From f4bef13c5c454dd0251f031766638e9e16ce8a3a Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 17 Sep 2026 14:12:30 +0800 Subject: [PATCH 01/21] fix: make archived log shard connections concurrency-safe and recoverable --- wafdb/db_monitor.go | 6 +- wafdb/localdb.go | 94 ++++++++++++---------------- wafdb/log_shard.go | 8 ++- wafdb/log_shard_cache.go | 131 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 58 deletions(-) create mode 100644 wafdb/log_shard_cache.go diff --git a/wafdb/db_monitor.go b/wafdb/db_monitor.go index c5fa2758..8751d431 100644 --- a/wafdb/db_monitor.go +++ b/wafdb/db_monitor.go @@ -108,9 +108,9 @@ func MonitorAllDatabases() ([]*DatabaseMetrics, error) { } } - // 监控自定义日志数据库 - if global.GDATA_CURRENT_LOG_DB_MAP != nil { - for fileName, db := range global.GDATA_CURRENT_LOG_DB_MAP { + // 监控自定义日志数据库（走快照，避免与按需打开/淘汰并发读写同一张 map） + { + for fileName, db := range snapshotShardDBs() { if db != nil { customDbPath := currentDir + "/data/" + fileName dbName := fmt.Sprintf("自定义日志数据库(%s)", fileName) diff --git a/wafdb/localdb.go b/wafdb/localdb.go index c2d24431..2110d3db 100644 --- a/wafdb/localdb.go +++ b/wafdb/localdb.go @@ -234,74 +234,62 @@ func InitLogDb(currentDir string) (bool, error) { } } -// 手工切换日志数据源 -func InitManaulLogDb(currentDir string, custFileName string) { +// InitManaulLogDb 按需打开一个归档日志分片，已打开的直接复用。 +// 连接统一由 log_shard_cache 托管（加锁、数量上限、空闲释放）。 +// 打开或迁移失败返回 error 交调用方降级：一个坏掉的归档文件不该让整个进程退出。 +func InitManaulLogDb(currentDir string, custFileName string) error { if dialect.Get().Name() != "sqlite" { // MySQL 模式下所有日志写入同一个库，无需手动切换分库 - return + return nil + } + if db := getShardDB(custFileName); db != nil { + zlog.Debug("自定义的库已存在", custFileName) + return nil } if currentDir == "" { currentDir = utils.GetCurrentDir() } - if global.GDATA_CURRENT_LOG_DB_MAP[custFileName] == nil { - zlog.Debug("初始化自定义的库", custFileName) - path := currentDir + "/data/" + custFileName - key := url.QueryEscape(global.GWAF_PWD_LOGDB) - dns := fmt.Sprintf("%s?_db_key=%s", path, key) - db, err := gorm.Open(sqlite.Open(dns), &gorm.Config{}) - if err != nil { - panic("failed to connect database") - } - // 日志/统计库使用 synchronous=NORMAL 提升高频写入吞吐，其余性能 pragma 统一设置 - applyPerfPragmas(db, true) - // 创建自定义日志记录器 - gormLogger := NewGormZLogger() - if global.GWAF_LOG_DEBUG_DB_ENABLE == true { - gormLogger = gormLogger.LogMode(logger.Info).(*GormZLogger) - // 启用调试模式 - db = db.Session(&gorm.Session{ - Logger: logger.Default.LogMode(logger.Info), // 设置为Info表示启用调试模式 - }) - } - global.GDATA_CURRENT_LOG_DB_MAP[custFileName] = db - //logDB.Use(crypto.NewCryptoPlugin()) - // 注册默认的AES加解密策略 - //crypto.RegisterCryptoStrategy(strategy.NewAesCryptoStrategy("3Y)(27EtO^tK8Bj~")) + zlog.Debug("初始化自定义的库", custFileName) + path := currentDir + "/data/" + custFileName + key := url.QueryEscape(global.GWAF_PWD_LOGDB) + dns := fmt.Sprintf("%s?_db_key=%s", path, key) + db, err := gorm.Open(sqlite.Open(dns), &gorm.Config{}) + if err != nil { + return fmt.Errorf("打开归档分片 %s 失败: %w", custFileName, err) + } + // 日志/统计库使用 synchronous=NORMAL 提升高频写入吞吐，其余性能 pragma 统一设置 + applyPerfPragmas(db, true) + // 创建自定义日志记录器 + gormLogger := NewGormZLogger() + if global.GWAF_LOG_DEBUG_DB_ENABLE == true { + gormLogger = gormLogger.LogMode(logger.Info).(*GormZLogger) + // 启用调试模式 + db = db.Session(&gorm.Session{ + Logger: logger.Default.LogMode(logger.Info), // 设置为Info表示启用调试模式 + }) + } - // ============ 使用 gormigrate 替代 AutoMigrate（完全向后兼容） ============ - zlog.Info("开始执行手动log数据库迁移...", "file", custFileName) - if err := RunLogDBMigrations(db); err != nil { - errStr := fmt.Sprintf("%v", err) - zlog.Error("手动log数据库迁移失败", "file", custFileName, "error_string", errStr, "error_type", fmt.Sprintf("%T", err)) - zlog.Error("手动log数据库迁移失败详细信息: " + errStr) - panic("manual log database migration failed: " + errStr) - } - // ============ 迁移代码结束 ============ + zlog.Info("开始执行手动log数据库迁移...", "file", custFileName) + if err := RunLogDBMigrations(db); err != nil { + zlog.Error("手动log数据库迁移失败", "file", custFileName, "error", fmt.Sprintf("%v", err)) + closeShardConn(custFileName, db) + return fmt.Errorf("归档分片 %s 迁移失败: %w", custFileName, err) + } - global.GDATA_CURRENT_LOG_DB_MAP[custFileName].Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query) - global.GDATA_CURRENT_LOG_DB_MAP[custFileName].Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update) + db.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query) + db.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update) - } else { - zlog.Debug("自定义的库已存在", custFileName) + // 并发下可能已有别的协程抢先放入，这时关掉自己开的这一份 + if _, duplicated := putShardDB(custFileName, db); duplicated { + closeShardConn(custFileName, db) } + return nil } // CloseManualLogDb 关闭并移除一个按需打开的归档日志分片连接（若存在）， // 供归档清理删除文件前调用，避免删正在被打开查询的 .db 文件。 func CloseManualLogDb(custFileName string) { - if global.GDATA_CURRENT_LOG_DB_MAP == nil { - return - } - db := global.GDATA_CURRENT_LOG_DB_MAP[custFileName] - if db == nil { - return - } - if sqlDB, err := db.DB(); err == nil { - if cerr := sqlDB.Close(); cerr != nil { - zlog.Warn("关闭归档分片连接失败", "file", custFileName, "error", cerr.Error()) - } - } - delete(global.GDATA_CURRENT_LOG_DB_MAP, custFileName) + closeShardDB(custFileName) } func InitStatsDb(currentDir string) (bool, error) { diff --git a/wafdb/log_shard.go b/wafdb/log_shard.go index 7199f048..4e39ec04 100644 --- a/wafdb/log_shard.go +++ b/wafdb/log_shard.go @@ -1,6 +1,7 @@ package wafdb import ( + "SamWaf/common/zlog" "SamWaf/enums" "SamWaf/global" "SamWaf/wafdb/dialect" @@ -43,8 +44,11 @@ func ResolveLogDB(currentDbName string) (*gorm.DB, string) { // Historical shard. if dialect.Get().IsFileBased() { // SQLite: open the archived .db file on demand and query its web_logs table. - InitManaulLogDb("", currentDbName) - if db := global.GDATA_CURRENT_LOG_DB_MAP[currentDbName]; db != nil { + if err := InitManaulLogDb("", currentDbName); err != nil { + zlog.Warn("归档分片不可用，降级查实时库", "file", currentDbName, "error", err.Error()) + return global.GWAF_LOCAL_LOG_DB, LogTableName + } + if db := getShardDB(currentDbName); db != nil { return db, LogTableName } // Shard file unavailable — degrade to live DB instead of panicking. diff --git a/wafdb/log_shard_cache.go b/wafdb/log_shard_cache.go new file mode 100644 index 00000000..4ea28b5f --- /dev/null +++ b/wafdb/log_shard_cache.go @@ -0,0 +1,131 @@ +package wafdb + +import ( + "SamWaf/common/zlog" + "SamWaf/global" + "sync" + "time" + + "gorm.io/gorm" +) + +// 归档分片连接缓存。 +// +// 这些连接原本直接读写 global.GDATA_CURRENT_LOG_DB_MAP 这张裸 map：并发查两个不同分片 +// 会命中 Go 的并发读写检测直接崩进程，且句柄一旦打开就永不释放。这里统一收口， +// 加锁之外再给两条上限：最多同时握 shardCacheMax 个连接，空闲超过 shardIdleTTL 的关掉。 +const ( + shardCacheMax = 8 + shardIdleTTL = 30 * time.Minute +) + +var ( + shardMu sync.Mutex + shardLastUse = map[string]time.Time{} +) + +// getShardDB 取一个已打开的分片连接并刷新它的使用时间；没有则返回 nil。 +func getShardDB(name string) *gorm.DB { + shardMu.Lock() + defer shardMu.Unlock() + if global.GDATA_CURRENT_LOG_DB_MAP == nil { + return nil + } + db := global.GDATA_CURRENT_LOG_DB_MAP[name] + if db != nil { + shardLastUse[name] = time.Now() + } + return db +} + +// putShardDB 登记一个新打开的连接。若同名连接已被别的协程抢先放入， +// 返回那一个并让调用方关掉自己开的，避免同一个文件握两份句柄。 +func putShardDB(name string, db *gorm.DB) (kept *gorm.DB, duplicated bool) { + shardMu.Lock() + defer shardMu.Unlock() + if global.GDATA_CURRENT_LOG_DB_MAP == nil { + global.GDATA_CURRENT_LOG_DB_MAP = map[string]*gorm.DB{} + } + if exist := global.GDATA_CURRENT_LOG_DB_MAP[name]; exist != nil { + shardLastUse[name] = time.Now() + return exist, true + } + global.GDATA_CURRENT_LOG_DB_MAP[name] = db + shardLastUse[name] = time.Now() + evictLocked() + return db, false +} + +// closeShardDB 关闭并移除一个分片连接。归档清理删文件前必须先调用， +// 否则 Windows 下文件被占用删不掉。 +func closeShardDB(name string) { + shardMu.Lock() + db := global.GDATA_CURRENT_LOG_DB_MAP[name] + delete(global.GDATA_CURRENT_LOG_DB_MAP, name) + delete(shardLastUse, name) + shardMu.Unlock() + + closeShardConn(name, db) +} + +// snapshotShardDBs 返回当前连接的快照，供监控之类只读遍历使用， +// 免得调用方直接遍历 map 时撞上并发写。 +func snapshotShardDBs() map[string]*gorm.DB { + shardMu.Lock() + defer shardMu.Unlock() + out := make(map[string]*gorm.DB, len(global.GDATA_CURRENT_LOG_DB_MAP)) + for k, v := range global.GDATA_CURRENT_LOG_DB_MAP { + out[k] = v + } + return out +} + +// evictLocked 在持锁状态下淘汰：先按空闲时间清，再按数量清最久未用的。 +// 实际关闭放到锁外做，关连接可能阻塞。 +func evictLocked() { + var victims []string + now := time.Now() + for name, last := range shardLastUse { + if now.Sub(last) > shardIdleTTL { + victims = append(victims, name) + } + } + for _, name := range victims { + db := global.GDATA_CURRENT_LOG_DB_MAP[name] + delete(global.GDATA_CURRENT_LOG_DB_MAP, name) + delete(shardLastUse, name) + go closeShardConn(name, db) + } + + for len(global.GDATA_CURRENT_LOG_DB_MAP) > shardCacheMax { + oldest := "" + var oldestAt time.Time + for name, last := range shardLastUse { + if oldest == "" || last.Before(oldestAt) { + oldest, oldestAt = name, last + } + } + if oldest == "" { + return + } + db := global.GDATA_CURRENT_LOG_DB_MAP[oldest] + delete(global.GDATA_CURRENT_LOG_DB_MAP, oldest) + delete(shardLastUse, oldest) + go closeShardConn(oldest, db) + } +} + +func closeShardConn(name string, db *gorm.DB) { + if db == nil { + return + } + sqlDB, err := db.DB() + if err != nil { + return + } + if cerr := sqlDB.Close(); cerr != nil { + zlog.Warn("关闭归档分片连接失败", "file", name, "error", cerr.Error()) + return + } + zlog.Debug("已释放归档分片连接", "file", name) +} From efb0727f0c74e24bce9c20f5cdec9360c37fb944 Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 17 Sep 2026 14:14:41 +0800 Subject: [PATCH 02/21] fix: build the WebSocket online table before its consumers start --- cmd/samwaf/main.go | 6 ++++-- wafqueue/message_queue.go | 6 ++++++ wafqueue/message_queue_ws_test.go | 22 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 wafqueue/message_queue_ws_test.go diff --git a/cmd/samwaf/main.go b/cmd/samwaf/main.go index 38c7935f..c1ca8c41 100644 --- a/cmd/samwaf/main.go +++ b/cmd/samwaf/main.go @@ -400,6 +400,10 @@ func (m *wafSystenService) run() { // 插件系统初始化失败不影响主程序启动 } + // WebSocket 在线表必须先于下面的消费协程和管理端建好：消息队列一开始消费就可能广播， + // 管理端一起来就可能有连接注册进来，而这里只是建两个空 map，没有任何外部依赖，放早没有副作用。 + global.GWebSocket = gwebsocket.InitWafWebSocket() + //初始化队列引擎 wafqueue.InitDequeEngine() //启动队列消费 @@ -498,8 +502,6 @@ func (m *wafSystenService) run() { webmanager.StartLocalServer() }() - //启动websocket - global.GWebSocket = gwebsocket.InitWafWebSocket() //定时取规则并更新（考虑后期定时拉取公共规则 待定，可能会影响实际生产） // 创建任务调度器 diff --git a/wafqueue/message_queue.go b/wafqueue/message_queue.go index 60bbdf93..d9064ab9 100644 --- a/wafqueue/message_queue.go +++ b/wafqueue/message_queue.go @@ -282,6 +282,12 @@ func handleIPBanMessage(msg innerbean.IPBanMessageInfo) { // 写入必须走 global.GWebSocket.Broadcast：它按连接加锁串行化并带写超时， // 直接对裸连接 WriteMessage 会与 ping 回显、定时任务撞成 concurrent write panic。 func sendToWebSocket(messageType, messageData string, messageAttach interface{}, cmdType string) { + // 在线表尚未建好时直接丢弃：本协程可能早于初始化跑起来，对 nil 调方法会 panic。 + // 正常启动顺序下不会走到这里，留着是防止将来调整启动顺序时再踩一次。 + if global.GWebSocket == nil { + zlog.Debug("WebSocket 在线表未就绪，跳过本条通知", messageType) + return + } dataPacket := model.MsgDataPacket{ MessageId: uuid.GenUUID(), MessageType: messageType, diff --git a/wafqueue/message_queue_ws_test.go b/wafqueue/message_queue_ws_test.go new file mode 100644 index 00000000..aed4a65a --- /dev/null +++ b/wafqueue/message_queue_ws_test.go @@ -0,0 +1,22 @@ +package wafqueue + +import ( + "SamWaf/global" + "testing" +) + +// 在线表未就绪时发通知不能 panic。 +// 曾经的现象：消息消费协程早于 global.GWebSocket 赋值启动，启动期广播一次就对 nil 调方法， +// 协程崩溃后被 NeverExit 拉起，日志里留一段 nil pointer dereference 堆栈。 +func TestSendToWebSocket_NilOnlineTable(t *testing.T) { + saved := global.GWebSocket + global.GWebSocket = nil + defer func() { + global.GWebSocket = saved + if r := recover(); r != nil { + t.Fatalf("在线表为 nil 时不应 panic: %v", r) + } + }() + + sendToWebSocket("测试通知", "内容", nil, "Test") +} From 1d0369af602f7e1b796f7f7403e1daebeb45d70c Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 17 Sep 2026 14:16:02 +0800 Subject: [PATCH 03/21] perf: cap stored payloads, index log lookups by IP and rule, batch IP tag writes --- innerbean/web_log.go | 3 + wafdb/dialect/dialect.go | 7 +++ wafdb/dialect/mysql_dialect.go | 7 +++ wafdb/dialect/postgres_dialect.go | 6 ++ wafdb/dialect/sqlite_dialect.go | 6 ++ wafdb/migrations_log.go | 51 +++++++++++++++++ wafqueue/log_queue.go | 4 +- wafqueue/log_truncate.go | 76 ++++++++++++++++++++++++ wafqueue/log_truncate_test.go | 89 +++++++++++++++++++++++++++++ waftask/stat_collector.go | 54 +++++++++-------- wafupgradenotice/upgrade_notes.yaml | 34 +++++++++++ 11 files changed, 311 insertions(+), 26 deletions(-) create mode 100644 wafqueue/log_truncate.go create mode 100644 wafqueue/log_truncate_test.go diff --git a/innerbean/web_log.go b/innerbean/web_log.go index cb718b0d..327716cf 100644 --- a/innerbean/web_log.go +++ b/innerbean/web_log.go @@ -61,6 +61,9 @@ type WebLog struct { IsBalance int `json:"is_balance"` //是否是负载均衡 1 是 0 不是 BalanceInfo string `gorm:"size:255" json:"balance_info"` //负载均衡IP端口信息 AI_SCORE float64 `json:"ai_score"` //AI检测得分[0,1]，0表示未经AI检测或未命中；命中(观察/拦截)时记录实际分数 + // Truncated 报文列因超长被截断的标记：0 未截断，1 已截断。 + // 截断只发生在落库这一步，内存对象与 Kafka 出口始终是原文。 + Truncated int `json:"truncated"` // GeoUnresolved 本次请求的地区无法判定（没有可用的地区库，或查询失败）， // 区别于"查出来是未知"。为 true 时规则引擎会跳过引用了 COUNTRY/PROVINCE/CITY 的规则， diff --git a/wafdb/dialect/dialect.go b/wafdb/dialect/dialect.go index 00cdadf3..6eb7a2cc 100644 --- a/wafdb/dialect/dialect.go +++ b/wafdb/dialect/dialect.go @@ -119,6 +119,13 @@ type DBDialect interface { // plain INSERT (web_logs has no primary key on any engine). InsertIgnoreSQL(table, quotedCols, rowPlaceholders string) string + // UpsertExcludedRef returns how the conflict branch of an upsert refers to the + // value the statement tried to insert, for accumulating updates such as + // "cnt = table.cnt + ". + // SQLite / PostgreSQL: excluded. + // MySQL: VALUES() + UpsertExcludedRef(col string) string + // FormatLocalTime returns a SQL expression that renders a DATETIME column as // 'YYYY-MM-DD HH:MM:SS' in local time, matching customtype.JsonTime.MarshalJSON. // diff --git a/wafdb/dialect/mysql_dialect.go b/wafdb/dialect/mysql_dialect.go index 793af1b8..7d2c92c4 100644 --- a/wafdb/dialect/mysql_dialect.go +++ b/wafdb/dialect/mysql_dialect.go @@ -38,6 +38,13 @@ func (d *MySQLDialect) InsertIgnoreSQL(table, quotedCols, rowPlaceholders string mysqlQuote(table), quotedCols, rowPlaceholders) } +// UpsertExcludedRef MySQL 的 ON DUPLICATE KEY UPDATE 用 VALUES() 引用待插入行。 +// 8.0.20 起官方推荐改用行别名，但 VALUES() 在 5.7/8.x 都仍然可用，这里取兼容面最广的写法。 +func (d *MySQLDialect) UpsertExcludedRef(col string) string { + return "VALUES(" + col + ")" +} + + // FormatLocalTime formats the column as-is: with loc=Local in the DSN the // DATETIME column already holds the local wall clock, so any timezone // conversion here would shift the value a second time. diff --git a/wafdb/dialect/postgres_dialect.go b/wafdb/dialect/postgres_dialect.go index e955a248..846d257b 100644 --- a/wafdb/dialect/postgres_dialect.go +++ b/wafdb/dialect/postgres_dialect.go @@ -64,6 +64,12 @@ func (d *PostgresDialect) InsertIgnoreSQL(table, quotedCols, rowPlaceholders str pgQuote(table), quotedCols, rowPlaceholders) } +// UpsertExcludedRef PostgreSQL 的 ON CONFLICT 用 excluded 伪表引用待插入行。 +func (d *PostgresDialect) UpsertExcludedRef(col string) string { + return "excluded." + col +} + + func (d *PostgresDialect) RenameTable(db *gorm.DB, src, dst string) error { return db.Exec(fmt.Sprintf("ALTER TABLE %s RENAME TO %s", pgQuote(src), pgQuote(dst))).Error } diff --git a/wafdb/dialect/sqlite_dialect.go b/wafdb/dialect/sqlite_dialect.go index 2393aad5..1be9e812 100644 --- a/wafdb/dialect/sqlite_dialect.go +++ b/wafdb/dialect/sqlite_dialect.go @@ -43,6 +43,12 @@ func (d *SQLiteDialect) InsertIgnoreSQL(table, quotedCols, rowPlaceholders strin sqliteQuote(table), quotedCols, rowPlaceholders) } +// UpsertExcludedRef SQLite 的 upsert 用 excluded 伪表引用待插入行。 +func (d *SQLiteDialect) UpsertExcludedRef(col string) string { + return "excluded." + col +} + + // FormatLocalTime adds the local UTC offset back: go-wxsqlite3 stores time.Time // as text carrying a zone suffix ('+08:00'), which SQLite normalizes to UTC // before applying the modifier. diff --git a/wafdb/migrations_log.go b/wafdb/migrations_log.go index 33439d45..9307638c 100644 --- a/wafdb/migrations_log.go +++ b/wafdb/migrations_log.go @@ -415,6 +415,57 @@ func RunLogDBMigrations(db *gorm.DB) error { return nil }, }, + // web_logs 增加 truncated 标记：报文列超过单列上限被截断时置 1， + // 让详情页知道看到的报文不完整，也让 AI 导出能跳过残缺样本。 + { + ID: "202609170001_add_web_logs_truncated", + Migrate: func(tx *gorm.DB) error { + if tx.Migrator().HasColumn(&innerbean.WebLog{}, "Truncated") { + return nil + } + zlog.Info("迁移 202609170001: web_logs 增加 truncated 列") + return tx.Migrator().AddColumn(&innerbean.WebLog{}, "Truncated") + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropColumn(&innerbean.WebLog{}, "Truncated") + }, + }, + // 按 IP / 规则 / 日期查询的索引。原有索引首列全是 unix_add_time， + // 「某 IP 全时段」「某规则全时段」只能扫整段时间索引再逐行过滤。 + // 大表上本次建索引耗时可能到分钟级，启动日志里有每条的耗时。 + { + ID: "202609170002_add_web_logs_query_indexes", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609170002: web_logs 增加 IP/规则/日期 查询索引") + // rule 是 text 列：MySQL 对 text 建索引必须给前缀长度，sqlite/pg 不需要 + ruleCol := "rule" + if tx.Dialector.Name() == "mysql" { + ruleCol = "rule(191)" + } + for _, ix := range []struct{ name, ddl string }{ + {"idx_web_ip_time", "CREATE INDEX IF NOT EXISTS idx_web_ip_time ON web_logs (tenant_id, user_code, src_ip, unix_add_time desc)"}, + {"idx_web_rule_time", "CREATE INDEX IF NOT EXISTS idx_web_rule_time ON web_logs (tenant_id, user_code, " + ruleCol + ", unix_add_time desc)"}, + {"idx_web_day_isbot", "CREATE INDEX IF NOT EXISTS idx_web_day_isbot ON web_logs (day, is_bot)"}, + } { + start := time.Now() + if err := safeCreateIndex(tx, "web_logs", ix.name, ix.ddl); err != nil { + // 建索引失败不阻断启动：查询只是慢，不是不能用 + zlog.Warn("创建索引失败", "index", ix.name, "error", err.Error()) + continue + } + zlog.Info("索引创建完成", "index", ix.name, "耗时", time.Since(start).String()) + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + for _, name := range []string{"idx_web_ip_time", "idx_web_rule_time", "idx_web_day_isbot"} { + if err := safeDropIndex(tx, "web_logs", name); err != nil { + zlog.Warn("删除索引失败", "index", name, "error", err.Error()) + } + } + return nil + }, + }, }) // 执行迁移 diff --git a/wafqueue/log_queue.go b/wafqueue/log_queue.go index aa0962c0..4fcc5f02 100644 --- a/wafqueue/log_queue.go +++ b/wafqueue/log_queue.go @@ -61,7 +61,9 @@ func ProcessLogDequeEngine() { } } if global.GCONFIG_LOG_PERSIST_ENABLED == 1 { - global.GWAF_LOCAL_LOG_DB.CreateInBatches(webLogArray, len(webLogArray)) + // 只有落库这一份做超长截断；下面的统计与出口仍用原始报文 + storeArray := truncateForStore(webLogArray) + global.GWAF_LOCAL_LOG_DB.CreateInBatches(storeArray, len(storeArray)) } // 日志流做统计 waftask.CollectStatsFromLogs(webLogArray) diff --git a/wafqueue/log_truncate.go b/wafqueue/log_truncate.go new file mode 100644 index 00000000..617c67d0 --- /dev/null +++ b/wafqueue/log_truncate.go @@ -0,0 +1,76 @@ +package wafqueue + +import ( + "SamWaf/innerbean" + "unicode/utf8" +) + +// payloadMaxBytes 单个报文列落库的字节上限，超出部分丢弃并置 Truncated=1。 +const payloadMaxBytes = 64 * 1024 + +// truncateForStore 返回用于落库的日志切片：超长报文列在副本上截断，未超长的元素原样复用。 +// +// 关键点是不改原对象——Kafka 出口与规则引擎共享同一批指针，且落库在它们之前执行， +// 就地截断会让下游也只能拿到半截报文。整批都没超长时直接返回原切片，不额外分配。 +func truncateForStore(logs []*innerbean.WebLog) []*innerbean.WebLog { + out := logs + copied := false + for i, lg := range logs { + if lg == nil || !needTruncate(lg) { + continue + } + if !copied { + out = make([]*innerbean.WebLog, len(logs)) + copy(out, logs) + copied = true + } + c := *lg + c.BODY = cutUTF8(c.BODY) + c.RES_BODY = cutUTF8(c.RES_BODY) + c.POST_FORM = cutUTF8(c.POST_FORM) + c.HEADER = cutUTF8(c.HEADER) + c.COOKIES = cutUTF8(c.COOKIES) + c.ResHeader = cutUTF8(c.ResHeader) + c.SrcByteBody = cutBytes(c.SrcByteBody) + c.SrcByteResBody = cutBytes(c.SrcByteResBody) + c.Truncated = 1 + out[i] = &c + } + return out +} + +func needTruncate(lg *innerbean.WebLog) bool { + return len(lg.BODY) > payloadMaxBytes || + len(lg.RES_BODY) > payloadMaxBytes || + len(lg.POST_FORM) > payloadMaxBytes || + len(lg.HEADER) > payloadMaxBytes || + len(lg.COOKIES) > payloadMaxBytes || + len(lg.ResHeader) > payloadMaxBytes || + len(lg.SrcByteBody) > payloadMaxBytes || + len(lg.SrcByteResBody) > payloadMaxBytes +} + +// cutUTF8 按字节上限截断，并回退到合法 UTF-8 边界（最多退 3 字节）， +// 免得把多字节字符切成半个，在 utf8mb4 列上落库报错。 +// 原文本身是二进制时回退会立即停下，最多多丢 3 字节。 +func cutUTF8(s string) string { + if len(s) <= payloadMaxBytes { + return s + } + cut := s[:payloadMaxBytes] + for i := 0; i < 3 && len(cut) > 0; i++ { + r, size := utf8.DecodeLastRuneInString(cut) + if r != utf8.RuneError || size > 1 { + break + } + cut = cut[:len(cut)-1] + } + return cut +} + +func cutBytes(b []byte) []byte { + if len(b) <= payloadMaxBytes { + return b + } + return b[:payloadMaxBytes] +} diff --git a/wafqueue/log_truncate_test.go b/wafqueue/log_truncate_test.go new file mode 100644 index 00000000..c7159819 --- /dev/null +++ b/wafqueue/log_truncate_test.go @@ -0,0 +1,89 @@ +package wafqueue + +import ( + "SamWaf/innerbean" + "strings" + "testing" + "unicode/utf8" +) + +// 未超长的批次原样返回，不做任何拷贝 +func TestTruncateForStore_NoOversize(t *testing.T) { + logs := []*innerbean.WebLog{ + {BODY: "hello", RES_BODY: "world"}, + {BODY: strings.Repeat("a", payloadMaxBytes)}, + } + out := truncateForStore(logs) + if &out[0] != &logs[0] { + t.Fatal("未超长时不应重新分配切片") + } + for i := range out { + if out[i] != logs[i] { + t.Fatalf("第 %d 条不应被替换", i) + } + if out[i].Truncated != 0 { + t.Fatalf("第 %d 条不应被标记截断", i) + } + } +} + +// 超长时截断落库副本，原对象必须保持原文（Kafka 出口与规则引擎共享它） +func TestTruncateForStore_KeepsOriginalIntact(t *testing.T) { + big := strings.Repeat("b", payloadMaxBytes+1000) + origin := &innerbean.WebLog{BODY: big, RES_BODY: "ok"} + logs := []*innerbean.WebLog{origin} + + out := truncateForStore(logs) + + if out[0] == origin { + t.Fatal("超长条目应换成副本，不能就地改原对象") + } + if len(origin.BODY) != len(big) || origin.Truncated != 0 { + t.Fatal("原对象被改动了，Kafka 出口会拿到半截报文") + } + if len(out[0].BODY) > payloadMaxBytes { + t.Fatalf("副本未被截断，长度 %d", len(out[0].BODY)) + } + if out[0].Truncated != 1 { + t.Fatal("副本应标记 Truncated=1") + } + if out[0].RES_BODY != "ok" { + t.Fatal("未超长的列不应被动") + } +} + +// 截断点落在多字节字符中间时要回退到合法边界 +func TestCutUTF8_RuneBoundary(t *testing.T) { + // 每个「中」占 3 字节，构造一个恰好在字符中间被切开的串 + s := strings.Repeat("中", payloadMaxBytes/3+10) + got := cutUTF8(s) + if len(got) > payloadMaxBytes { + t.Fatalf("超出上限: %d", len(got)) + } + if !utf8.ValidString(got) { + t.Fatal("截断后不是合法 UTF-8") + } + if payloadMaxBytes-len(got) > 3 { + t.Fatalf("回退过多，丢了 %d 字节", payloadMaxBytes-len(got)) + } +} + +// 二进制内容不会被一路回退到空 +func TestCutUTF8_BinaryPayload(t *testing.T) { + s := strings.Repeat("\xff", payloadMaxBytes+100) + got := cutUTF8(s) + if len(got) < payloadMaxBytes-3 { + t.Fatalf("二进制内容回退过多，只剩 %d 字节", len(got)) + } +} + +func TestCutBytes(t *testing.T) { + small := []byte("abc") + if got := cutBytes(small); len(got) != 3 { + t.Fatal("未超长的字节切片不应被截") + } + big := make([]byte, payloadMaxBytes+10) + if got := cutBytes(big); len(got) != payloadMaxBytes { + t.Fatalf("超长字节切片应截到上限，实际 %d", len(got)) + } +} diff --git a/waftask/stat_collector.go b/waftask/stat_collector.go index 23d91161..f51d3dc6 100644 --- a/waftask/stat_collector.go +++ b/waftask/stat_collector.go @@ -8,11 +8,18 @@ import ( "SamWaf/innerbean" "SamWaf/model" "SamWaf/model/baseorm" + "SamWaf/wafdb/dialect" "time" "gorm.io/gorm" + "gorm.io/gorm/clause" ) +// ipTagUpsertBatch 单条 upsert 语句里最多带多少行：太大容易撞上 +// SQLite 的 999 个绑定变量上限（IPTag 有 8 列，200 行约 1600 个参数， +// go-wxsqlite3 编译期上限是 32766，留足余量）。 +const ipTagUpsertBatch = 200 + // 从日志流批量聚合统计并写入统计库（不依赖日志入库）。 // 会按以下维度做增量统计： // 1) 主机日聚合 (host_code, action, day) @@ -324,23 +331,13 @@ func CollectStatsFromLogs(logs []*innerbean.WebLog) { zlog.Debug("城市聚合处理完成", "更新记录数", cityUpdateCount, "插入记录数", cityInsertCount) // 4) IPTag 增量（根据配置选择数据库） - ipTagUpdateCount := 0 - ipTagInsertCount := 0 + // 走唯一索引 uni_iptags_full 的 upsert：原来每个标签先 UPDATE 再按 RowsAffected 补 INSERT， + // 一批里有多少个 IP 就是多少次往返；改成整批一条语句后往返固定为一次。 ipTagDB := global.GetIPTagDB() // 使用封装方法获取数据库连接 - for k, delta := range ipTagAgg { - tx := ipTagDB.Model(&model.IPTag{}). - Where("tenant_id = ? and user_code = ? and ip = ? and ip_tag = ?", - global.GWAF_TENANT_ID, global.GWAF_USER_CODE, k.IP, k.Rule). - Updates(map[string]interface{}{ - "Cnt": gorm.Expr("Cnt + ?", delta), - "UPDATE_TIME": now, - }) - if tx.Error != nil { - zlog.Debug("IP标签更新失败", "错误", tx.Error.Error(), "IP", k.IP, "规则", k.Rule) - continue - } - if tx.RowsAffected == 0 { - err := ipTagDB.Create(&model.IPTag{ + if len(ipTagAgg) > 0 { + rows := make([]model.IPTag, 0, len(ipTagAgg)) + for k, delta := range ipTagAgg { + rows = append(rows, model.IPTag{ BaseOrm: baseorm.BaseOrm{ Id: uuid.GenUUID(), USER_CODE: global.GWAF_USER_CODE, @@ -352,17 +349,24 @@ func CollectStatsFromLogs(logs []*innerbean.WebLog) { IPTag: k.Rule, Cnt: delta, Remarks: "", - }).Error - if err != nil { - zlog.Debug("IP标签插入失败", "错误", err.Error(), "IP", k.IP, "规则", k.Rule) - } else { - ipTagInsertCount++ - } + }) + } + incoming := dialect.Get().UpsertExcludedRef("cnt") + err := ipTagDB.Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "user_code"}, {Name: "tenant_id"}, {Name: "ip"}, {Name: "ip_tag"}, + }, + DoUpdates: clause.Assignments(map[string]interface{}{ + "cnt": gorm.Expr("ip_tags.cnt + " + incoming), + "update_time": now, + }), + }).CreateInBatches(rows, ipTagUpsertBatch).Error + if err != nil { + zlog.Debug("IP标签写入失败", "错误", err.Error(), "条数", len(rows)) } else { - ipTagUpdateCount++ + zlog.Debug("IP标签处理完成", "条数", len(rows)) } } - zlog.Debug("IP标签处理完成", "更新记录数", ipTagUpdateCount, "插入记录数", ipTagInsertCount) // 5) 站点天级聚合 增量 siteDayUpdateCount := 0 @@ -476,7 +480,7 @@ func CollectStatsFromLogs(logs []*innerbean.WebLog) { "主机聚合", map[string]interface{}{"更新": hostUpdateCount, "插入": hostInsertCount}, "IP聚合", map[string]interface{}{"更新": ipUpdateCount, "插入": ipInsertCount}, "城市聚合", map[string]interface{}{"更新": cityUpdateCount, "插入": cityInsertCount}, - "IP标签", map[string]interface{}{"更新": ipTagUpdateCount, "插入": ipTagInsertCount}, + "IP标签", map[string]interface{}{"upsert": len(ipTagAgg)}, "站点天聚合", map[string]interface{}{"更新": siteDayUpdateCount, "插入": siteDayInsertCount}, "站点小时聚合", map[string]interface{}{"更新": siteHourUpdateCount, "插入": siteHourInsertCount}) } diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index 624ac5ce..504dec30 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -625,3 +625,37 @@ notes: backend is in use, and cache errors are recorded in the log and in the diagnostic package. Token lifetime is now refreshed on every request, matching the "expires after this long without activity" wording of that setting. + - id: v1_3_25_weblog_payload_truncate_and_index + version: v1.3.25 + kind: notice + level: normal + page: '' + doc: https://doc.samwaf.com/quickstart/Update.html + apply: + type: none + zh: + title: 超长报文落库时截断，并为访问日志按 IP / 规则查询补索引 + detail: >- + 单条请求的报文列（请求体、响应体、表单、请求头、Cookie、响应头）超过 64KB 时， + 只保留前 64KB 落库，该条日志会带上"已截断"标记，详情页据此提示报文不完整。 + 截断只发生在写入数据库这一步：检测引擎、自定义规则、Kafka 推送拿到的仍是完整报文， + 判定结果与推送内容都不受影响。日志库体积此前没有任何单条上限，一次大文件响应就能撑出一条巨大的记录。 + 本次同时为访问日志补了三条索引：按来源 IP 查、按触发规则查、按日期查机器人流量。 + 原有索引的首列都是时间，这三类查询只能扫完整段时间再逐行过滤，数据量大时明显变慢。 + 升级后首次启动会创建索引，日志量大的实例这一步可能需要数分钟，启动日志里会打印每条索引的耗时， + 期间请勿强制结束进程。另外 IP 标签的写入改为整批一条语句，高并发下对数据库的往返次数明显下降。 + en: + title: Oversized payloads are truncated on write, plus new access-log indexes for IP and rule lookups + detail: >- + When a request's payload columns (request body, response body, form data, headers, cookies, response + headers) exceed 64KB, only the first 64KB is stored and the entry is flagged as truncated, which the + detail page uses to warn that the payload is incomplete. Truncation happens only on the database write: + the detection engine, custom rules and the Kafka feed still receive the complete payload, so verdicts and + exported records are unaffected. Until now no per-record limit existed at all, so a single large response + could produce one enormous row. This release also adds three indexes to the access log: by source IP, by + triggered rule, and by date for bot traffic. Every existing index starts with the timestamp column, so + those three lookups had to scan the whole time range and filter row by row, which gets slow on large + datasets. The indexes are built on the first start after upgrading; on instances with a large log volume + this can take several minutes, the startup log prints the time each index took, and the process should + not be killed while it runs. IP tag counters are now written with a single batched statement, cutting + database round trips considerably under load. From d3d473be11a4cd1d5e9e3ee45b0920d9a65c1b7e Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 17 Sep 2026 16:17:00 +0800 Subject: [PATCH 04/21] fix: keep archived log shards readable when the live table gains a column --- .../cross_engine_shard_column_test.go | 57 ++++++++++++ service/waf_service/cross_engine_test.go | 9 ++ service/waf_service/waf_log.go | 91 +++++++++++++++---- 3 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 service/waf_service/cross_engine_shard_column_test.go diff --git a/service/waf_service/cross_engine_shard_column_test.go b/service/waf_service/cross_engine_shard_column_test.go new file mode 100644 index 00000000..c3bbfb86 --- /dev/null +++ b/service/waf_service/cross_engine_shard_column_test.go @@ -0,0 +1,57 @@ +//go:build crossdb + +// 归档分片缺列时的日志查询回归。 +// +// 归档分片是分库那一刻的结构快照，此后给 web_logs 加的列它没有。日志列表的 SELECT 是按当前 +// 结构体反射出来的显式列名，少一列整条查询就报错；而同一次请求里的 Count 不带列、照样数得出来， +// 页面于是变成「有分页、没数据」。这组用例钉住两件事：按交集列查得通，按全量列必然报错 +// （对照组一旦不报错，说明这个引擎不介意多选一列，用例本身就失去意义了）。 +package waf_service + +import ( + "SamWaf/innerbean" + "strings" + "testing" + + "gorm.io/gorm" +) + +func runShardColumnCases(t *testing.T, logdb *gorm.DB) { + want := getWebLogListColumns() + const dropped = "truncated" + + keep := make([]string, 0, len(want)) + for _, c := range want { + if c != dropped { + keep = append(keep, c) + } + } + if len(keep) == len(want) { + t.Fatalf("结构体里已经没有 %s 列了，用例需要换一列来模拟", dropped) + } + + name := "web_logs_xcol" + sfx() + must(t, logdb.Exec("CREATE TABLE "+name+" AS SELECT "+strings.Join(keep, ", ")+" FROM web_logs WHERE 1=0").Error) + defer logdb.Exec("DROP TABLE IF EXISTS " + name) + + sel := webLogSelect(logdb, name, name, want, "list") + if strings.Contains(sel, dropped) { + t.Fatalf("缺列的分片不该把 %s 放进 SELECT: %s", dropped, sel) + } + + var rows []innerbean.WebLog + if err := logdb.Table(name).Select(sel).Limit(1).Find(&rows).Error; err != nil { + t.Fatalf("按交集列查缺列分片仍然报错: %v", err) + } + + // 对照组：修复前就是这么查的 + if err := logdb.Table(name).Select(strings.Join(want, ", ")).Limit(1).Find(&rows).Error; err == nil { + t.Fatal("对照组没报错——这个引擎不介意多选一列，用例失去意义，需要换验证方式") + } + + // 列齐全的实时表不受影响，该选的一列都不能少 + live := webLogSelect(logdb, "", "web_logs", want, "list") + if !strings.Contains(live, dropped) { + t.Fatalf("实时表列是齐的，不该丢 %s: %s", dropped, live) + } +} diff --git a/service/waf_service/cross_engine_test.go b/service/waf_service/cross_engine_test.go index 97437966..09709a3a 100644 --- a/service/waf_service/cross_engine_test.go +++ b/service/waf_service/cross_engine_test.go @@ -285,6 +285,15 @@ func TestCrossEngine(t *testing.T) { // —— ip_tags 排除逻辑（见 cross_engine_iptag_test.go）—— t.Run("iptag", func(t *testing.T) { runIPTagCases(t, x.core) }) + // —— 归档分片缺列（见 cross_engine_shard_column_test.go）—— + t.Run("shardcolumn", func(t *testing.T) { runShardColumnCases(t, x.logdb) }) + + // —— 报文拆表读写（见 cross_engine_payload_test.go）—— + t.Run("payload", func(t *testing.T) { runPayloadCases(t, x.logdb) }) + + // —— ip_tags 跨库合并（见 cross_engine_payload_test.go）—— + t.Run("iptagmerge", func(t *testing.T) { runIPTagMergeCases(t, x) }) + // —— 有副作用 service 的 DB-only 路径（见 cross_engine_side_test.go）—— t.Run("side", func(t *testing.T) { runSideCases(t, x) }) diff --git a/service/waf_service/waf_log.go b/service/waf_service/waf_log.go index 40335ed1..4ab00943 100644 --- a/service/waf_service/waf_log.go +++ b/service/waf_service/waf_log.go @@ -2,8 +2,10 @@ package waf_service import ( "SamWaf/common/validfield" + "SamWaf/common/zlog" "SamWaf/global" "SamWaf/innerbean" + "SamWaf/model" "SamWaf/model/request" "SamWaf/wafdb" "SamWaf/wafdb/dialect" @@ -13,6 +15,7 @@ import ( "strings" "sync" + "gorm.io/gorm" "gorm.io/gorm/schema" ) @@ -34,32 +37,72 @@ var detailExcludeColumns = map[string]bool{ var ( webLogListSelectOnce sync.Once webLogDetailSelectOnce sync.Once - webLogListSelectCache string - webLogDetailSelectCache string + webLogListSelectCache []string + webLogDetailSelectCache []string + + // webLogShardSelect 每个分片一份可用列，key = 分片标识|表名|用途。 + webLogShardSelect sync.Map ) -// getWebLogListSelect 动态从 WebLog 结构体反射出列名并排除大字段，结果缓存复用。 -// 新增字段会自动纳入，旧版本数据库缺列也不影响（GORM 会忽略不存在的列）。 -func getWebLogListSelect() string { +// getWebLogListColumns 动态从 WebLog 结构体反射出列名并排除大字段，结果缓存复用。 +func getWebLogListColumns() []string { webLogListSelectOnce.Do(func() { - webLogListSelectCache = buildSelectExcluding(&innerbean.WebLog{}, listExcludeColumns) + webLogListSelectCache = columnsExcluding(&innerbean.WebLog{}, listExcludeColumns) }) return webLogListSelectCache } -// getWebLogDetailSelect 详情查询字段，包含文本 body 类字段，排除 blob。 -func getWebLogDetailSelect() string { +// getWebLogDetailColumns 详情查询字段，包含文本 body 类字段，排除 blob。 +func getWebLogDetailColumns() []string { webLogDetailSelectOnce.Do(func() { - webLogDetailSelectCache = buildSelectExcluding(&innerbean.WebLog{}, detailExcludeColumns) + webLogDetailSelectCache = columnsExcluding(&innerbean.WebLog{}, detailExcludeColumns) }) return webLogDetailSelectCache } -// buildSelectExcluding 通过 GORM schema 解析模型字段，返回排除指定列后的 SELECT 子句。 -func buildSelectExcluding(model interface{}, excludeDBNames map[string]bool) string { +// webLogSelect 给出这个分片上真正查得到的列。 +// +// 归档分片是分库那一刻的结构快照，之后给 web_logs 加的列它没有；而 SELECT 是按当前结构体 +// 反射出来的显式列名，少一列整条查询就报错。偏偏同一次请求里的 Count 不带列、照样数得出来， +// 页面于是变成「有分页、没数据」，错误还被 Find 吞掉，查都无从查起。 +// 所以按分片实际存在的列取一次交集，结果按分片缓存（归档分片的结构不会再变）。 +func webLogSelect(db *gorm.DB, shard, table string, want []string, usage string) string { + key := shard + "|" + table + "|" + usage + if v, ok := webLogShardSelect.Load(key); ok { + return v.(string) + } + sel := strings.Join(want, ", ") + if cols, err := dialect.Get().ColumnInfo(db, table); err == nil && len(cols) > 0 { + have := make(map[string]bool, len(cols)) + for _, c := range cols { + have[strings.ToLower(c.Name)] = true + } + kept := make([]string, 0, len(want)) + var missing []string + for _, w := range want { + if have[w] { + kept = append(kept, w) + continue + } + missing = append(missing, w) + } + // 一列都对不上多半是取列信息取错了表，这时宁可按结构体来，别把查询改成空 + if len(kept) > 0 { + sel = strings.Join(kept, ", ") + if len(missing) > 0 { + zlog.Info("归档分片缺列，本次查询跳过", "table", table, "columns", strings.Join(missing, ",")) + } + } + } + webLogShardSelect.Store(key, sel) + return sel +} + +// columnsExcluding 通过 GORM schema 解析模型字段，返回排除指定列后的列名（保持结构体顺序）。 +func columnsExcluding(model interface{}, excludeDBNames map[string]bool) []string { s, err := schema.Parse(model, &sync.Map{}, schema.NamingStrategy{}) if err != nil { - return "*" + return []string{"*"} } cols := make([]string, 0, len(s.Fields)) for _, field := range s.Fields { @@ -68,7 +111,7 @@ func buildSelectExcluding(model interface{}, excludeDBNames map[string]bool) str } cols = append(cols, field.DBName) } - return strings.Join(cols, ", ") + return cols } func (receiver *WafLogService) AddApi(log innerbean.WebLog) error { @@ -83,7 +126,12 @@ func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) ( var weblog innerbean.WebLog // 解析当前应查询的日志连接与表（live 或历史分片：SQLite 历史文件 / MySQL 历史表） logDB, logTable := wafdb.ResolveLogDB(req.CurrrentDbName) - logDB.Table(logTable).Select(getWebLogDetailSelect()).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog) + sel := webLogSelect(logDB, req.CurrrentDbName, logTable, getWebLogDetailColumns(), "detail") + if err := logDB.Table(logTable).Select(sel).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog).Error; err != nil { + return weblog, fmt.Errorf("查询日志详情失败: %w", err) + } + // 报文单独存在 event_payload 里，按主键点查补回来 + FillShardPayloads(req.CurrrentDbName, []*innerbean.WebLog{&weblog}) return weblog, nil } func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { @@ -238,8 +286,14 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn } else { return nil, 0, errors.New("输入排序字段不合法") } - logDB.Select(getWebLogListSelect()).Table(forceIndex).Limit(req.PageSize).Where(whereField, whereValues...).Offset(req.PageSize * (req.PageIndex - 1)).Order(orderInfo).Find(&weblogs) - logDB.Table(forceIndex).Where(whereField, whereValues...).Count(&total) + sel := webLogSelect(logDB, req.CurrrentDbName, logTable, getWebLogListColumns(), "list") + // 错误必须往上抛：吞掉它就只剩「有分页、没数据」，连从哪查起都不知道 + if err := logDB.Select(sel).Table(forceIndex).Limit(req.PageSize).Where(whereField, whereValues...).Offset(req.PageSize * (req.PageIndex - 1)).Order(orderInfo).Find(&weblogs).Error; err != nil { + return nil, 0, fmt.Errorf("查询日志失败: %w", err) + } + if err := logDB.Table(forceIndex).Where(whereField, whereValues...).Count(&total).Error; err != nil { + return nil, 0, fmt.Errorf("统计日志条数失败: %w", err) + } return weblogs, total, nil } func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { @@ -252,6 +306,11 @@ func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSear } func (receiver *WafLogService) DeleteHistory(day string) { global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", day).Delete(&innerbean.WebLog{}) + // 报文表自带同格式的 create_time，按同一条件删，不必回表对 req_uuid + if err := global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", day). + Delete(&model.EventPayload{}).Error; err != nil { + zlog.Warn("清理过期报文失败", "截止", day, "error", err.Error()) + } } // GetUnixTimeByCounter 依据开始时间和到期时间获取一个最新的时间戳 From f8548600e5cdc958fca07fb4a544907d7b3f7dff Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 17 Sep 2026 16:22:22 +0800 Subject: [PATCH 05/21] feat: merge IP tag history when their database location changes --- api/waf_log.go | 11 ++ global/global.go | 8 +- router/waf_log.go | 1 + .../cross_engine_iptag_merge_test.go | 81 ++++++++++++ .../waf_service/cross_engine_iptag_test.go | 9 +- service/waf_service/waf_ip_tag_merge.go | 122 ++++++++++++++++++ waftask/task_config.go | 15 +++ wafupgradenotice/upgrade_notes.yaml | 36 ++++++ 8 files changed, 281 insertions(+), 2 deletions(-) create mode 100644 service/waf_service/cross_engine_iptag_merge_test.go create mode 100644 service/waf_service/waf_ip_tag_merge.go diff --git a/api/waf_log.go b/api/waf_log.go index a7ccac4c..99b306c9 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -17,6 +17,7 @@ import ( "path/filepath" "regexp" "strings" + "sync/atomic" "time" "github.com/gin-gonic/gin" @@ -254,6 +255,16 @@ func (w *WafLogAPi) GetHttpCopyMaskApi(c *gin.Context) { } } +// GetIPTagDBStatusApi 报告 IP 标签的当前归属库与合并进度。 +// 切换归属会把另一个库里的历史标签并过来，量大时要跑一会儿， +// 页面据此显示「合并中」并在合并结束后重新拉取列表。 +func (w *WafLogAPi) GetIPTagDBStatusApi(c *gin.Context) { + response.OkWithData(gin.H{ + "db": global.GDATA_IP_TAG_DB, + "merging": atomic.LoadInt32(&global.GDATA_IP_TAG_MERGING) == 1, + }, c) +} + // GetAttackIPListApi 获取风险数据列表 func (w *WafLogAPi) GetAttackIPListApi(c *gin.Context) { var req request.WafAttackIpTagSearch diff --git a/global/global.go b/global/global.go index 950d6183..e95e4ca9 100644 --- a/global/global.go +++ b/global/global.go @@ -196,7 +196,13 @@ var ( GDATA_SHARE_DB_SIZE int64 = 100 * 10000 //100w 进行分库 100*10000 GDATA_SHARE_DB_FILE_SIZE int64 = 1024 //1024M 进行分库 GDATA_CURRENT_CHANGE bool = false //当前是否正在切换 - GDATA_IP_TAG_DB int64 = 0 //IP Tag 存放位置 0 是主库 1是读取 stat库 + // GDATA_IP_TAG_DB IP Tag 存放位置：0 核心库，1 统计库。 + // 新装默认统计库——ip_tags 是派生索引，写入量跟着攻击量走，和配置共用核心库会拖慢每一次配置读写。 + // 存量用户的取值来自 system_configs 里已有的那一行，不受这个默认值影响； + // 切换归属时 MergeIPTagsInto 会把另一个库里的历史标签并过来。 + GDATA_IP_TAG_DB int64 = 1 + // GDATA_IP_TAG_MERGING 标签合并任务是否在跑（atomic 读写）。界面据此显示"合并中"。 + GDATA_IP_TAG_MERGING int32 = 0 // GCONFIG_ATTACK_TAG_EXCLUDE 风险日志里不算风险的标签（逗号分隔）。"正常"永远排除，不用写这里。 // 这些标签既不出现在规则筛选列表里，也不计入「阻止数量」，而是算作放行。 GCONFIG_ATTACK_TAG_EXCLUDE string = "ACME证书校验,静态文件访问成功" diff --git a/router/waf_log.go b/router/waf_log.go index 508d6c32..243af71d 100644 --- a/router/waf_log.go +++ b/router/waf_log.go @@ -22,4 +22,5 @@ func (receiver *LogRouter) InitLogRouter(group *gin.RouterGroup) { wafLogRouter.POST("/api/v1/waflog/attack/attackiplist", logApi.GetAttackIPListApi) wafLogRouter.GET("/api/v1/waflog/attack/alliptag", logApi.GetAllIpTagApi) wafLogRouter.POST("/api/v1/waflog/attack/deletetagbyname", logApi.DeleteTagByNameApi) + wafLogRouter.GET("/api/v1/waflog/attack/iptagdbstatus", logApi.GetIPTagDBStatusApi) } diff --git a/service/waf_service/cross_engine_iptag_merge_test.go b/service/waf_service/cross_engine_iptag_merge_test.go new file mode 100644 index 00000000..ffecb664 --- /dev/null +++ b/service/waf_service/cross_engine_iptag_merge_test.go @@ -0,0 +1,81 @@ +//go:build crossdb + +// IP 标签跨库合并的三库回归：切换存放位置时把另一个库的历史并过来， +// 相同唯一键 cnt 相加、首次时间取早、最近时间取晚，且重复跑不改数。 +// 冲突分支要引用"本次待插入的值"，三个引擎写法不同，正是这里要钉住的地方。 +// 由 TestCrossEngine 每引擎调一次。 +package waf_service + +import ( + "SamWaf/common/uuid" + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "testing" + "time" +) + +func runIPTagMergeCases(t *testing.T, x *xdb) { + saved := global.GDATA_IP_TAG_DB + defer func() { global.GDATA_IP_TAG_DB = saved }() + + ip := "9.9." + sfx() + early := customtype.JsonTime(time.Now().Add(-48 * time.Hour)) + late := customtype.JsonTime(time.Now()) + + mk := func(id string, cnt int64, create, update customtype.JsonTime) *model.IPTag { + return &model.IPTag{ + BaseOrm: baseorm.BaseOrm{ + Id: id, USER_CODE: xtestUser, Tenant_ID: xtestTenant, + CREATE_TIME: create, UPDATE_TIME: update, + }, + IP: ip, IPTag: "SQL注入", Cnt: cnt, + } + } + + // 目标库(统计库)已有一行，来源库(核心库)有同键的另一行 + 一行独有的 + must(t, x.stats.Create(mk(uuid.GenUUID(), 3, late, late)).Error) + must(t, x.core.Create(mk(uuid.GenUUID(), 7, early, early)).Error) + onlyInCore := &model.IPTag{ + BaseOrm: baseorm.BaseOrm{ + Id: uuid.GenUUID(), USER_CODE: xtestUser, Tenant_ID: xtestTenant, + CREATE_TIME: early, UPDATE_TIME: early, + }, + IP: ip, IPTag: "XSS", Cnt: 5, + } + must(t, x.core.Create(onlyInCore).Error) + + global.GDATA_IP_TAG_DB = 1 + MergeIPTagsInto(1) + + var merged model.IPTag + firstBy(t, x.stats, &merged, "ip = ? and ip_tag = ?", ip, "SQL注入") + if merged.Cnt != 10 { + t.Fatalf("同键的 cnt 应相加为 10，实际 %d", merged.Cnt) + } + if time.Time(merged.CREATE_TIME).After(time.Time(early).Add(time.Second)) { + t.Fatalf("首次时间应取早的那个，实际 %v", time.Time(merged.CREATE_TIME)) + } + if time.Time(merged.UPDATE_TIME).Before(time.Time(late).Add(-time.Second)) { + t.Fatalf("最近时间应取晚的那个，实际 %v", time.Time(merged.UPDATE_TIME)) + } + + var moved model.IPTag + firstBy(t, x.stats, &moved, "ip = ? and ip_tag = ?", ip, "XSS") + if moved.Cnt != 5 { + t.Fatalf("来源库独有的标签应原样搬过来，实际 cnt=%d", moved.Cnt) + } + + if n := countBy(t, x.core, &model.IPTag{}, "ip = ?", ip); n != 0 { + t.Fatalf("合并完来源库应清空，还剩 %d 行", n) + } + + // 幂等：源库已空，再跑一次不能把计数翻倍 + MergeIPTagsInto(1) + var again model.IPTag + firstBy(t, x.stats, &again, "ip = ? and ip_tag = ?", ip, "SQL注入") + if again.Cnt != 10 { + t.Fatalf("重复合并把计数改了，实际 %d", again.Cnt) + } +} diff --git a/service/waf_service/cross_engine_iptag_test.go b/service/waf_service/cross_engine_iptag_test.go index e0404e37..ca6ed61c 100644 --- a/service/waf_service/cross_engine_iptag_test.go +++ b/service/waf_service/cross_engine_iptag_test.go @@ -50,7 +50,14 @@ func findAttackIP(list []model.AttackIPTag, ip string) (model.AttackIPTag, bool) func runIPTagCases(t *testing.T, coredb *gorm.DB) { svc := WafLogService{} oldExclude := global.GCONFIG_ATTACK_TAG_EXCLUDE - defer func() { global.GCONFIG_ATTACK_TAG_EXCLUDE = oldExclude }() + // 本用例直接往核心库塞数据，读取走 GetIPTagDB()，所以归属必须钉在核心库。 + // 新装默认已经是统计库，不钉住的话读的是另一个库，查出来全是空。 + oldTagDB := global.GDATA_IP_TAG_DB + global.GDATA_IP_TAG_DB = 0 + defer func() { + global.GCONFIG_ATTACK_TAG_EXCLUDE = oldExclude + global.GDATA_IP_TAG_DB = oldTagDB + }() // 干净起步：本用例独占 ip_tags must(t, coredb.Exec("DELETE FROM ip_tags").Error) diff --git a/service/waf_service/waf_ip_tag_merge.go b/service/waf_service/waf_ip_tag_merge.go new file mode 100644 index 00000000..61d6657d --- /dev/null +++ b/service/waf_service/waf_ip_tag_merge.go @@ -0,0 +1,122 @@ +package waf_service + +import ( + "SamWaf/common/zlog" + "SamWaf/global" + "SamWaf/model" + "SamWaf/wafdb/dialect" + "sync/atomic" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// 合并的节奏。ip_tags 是跟着攻击量长的派生表，实例跑久了几百万甚至上千万行都正常， +// 一次切换就要把它整张搬过去，所以这段循环必须让得出写锁： +// 批太小则往返次数爆炸（千万行按 500 一批就是两万轮、六万次往返）， +// 批太大则单个事务变长，SQLite 上会把正在落库的日志顶住。 +const ( + ipTagMergeBatch = 1000 + ipTagMergePause = 10 * time.Millisecond // 每批之间让一下写锁，日志落库不至于被饿死 + ipTagMergeLogEvery = 50000 // 每搬这么多行报一次进度，大表上别让人以为卡死了 + ipTagMergeSlowHint = 200000 // 超过这个量级先把预期讲清楚 +) + +// MergeIPTagsInto 把另一个库里的 ip_tags 并进 target 指定的库（0=核心库，1=统计库）。 +// +// ip_tags 的归属由 database.ip_tag_db 决定，读写都只认当前那一个库。 +// 切换归属之前积累的标签会留在原来的库里——数据没丢，但界面按新库查，看起来像"历史没了"。 +// 这个任务把它们搬过来：唯一键相同的 cnt 相加、首次时间取早、最近时间取晚。 +// +// 按内容驱动而不是按"切换事件"驱动：源库空就直接返回，所以启动时跑一次就能把 +// 更早版本切换时留下的历史一并收回来，重复跑也没有副作用。 +func MergeIPTagsInto(target int64) { + if !atomic.CompareAndSwapInt32(&global.GDATA_IP_TAG_MERGING, 0, 1) { + zlog.Debug("IP标签合并已在进行中，跳过本次") + return + } + defer atomic.StoreInt32(&global.GDATA_IP_TAG_MERGING, 0) + + dst, src := global.GWAF_LOCAL_STATS_DB, global.GWAF_LOCAL_DB + dstName, srcName := "统计库", "核心库" + if target != 1 { + dst, src = global.GWAF_LOCAL_DB, global.GWAF_LOCAL_STATS_DB + dstName, srcName = "核心库", "统计库" + } + if dst == nil || src == nil { + return + } + if !src.Migrator().HasTable(&model.IPTag{}) { + return + } + + var pending int64 + if err := src.Model(&model.IPTag{}).Count(&pending).Error; err != nil { + zlog.Warn("统计待合并IP标签失败", "来源", srcName, "error", err.Error()) + return + } + if pending == 0 { + return + } + + zlog.Info("开始合并IP标签", "来源", srcName, "目标", dstName, "待合并", pending) + if pending >= ipTagMergeSlowHint { + // 讲清楚而不是默默跑：合并期间风险日志页的计数会一直变，用户得知道那是正常的 + zlog.Warn("待合并的IP标签较多，合并会持续一段时间", + "条数", pending, "来源", srcName, "目标", dstName, + "说明", "合并期间风险日志页的标签计数会持续变动，完成后自动稳定；进程重启会从断点继续") + } + start := time.Now() + var merged, lastLogged int64 + for { + var rows []model.IPTag + if err := src.Model(&model.IPTag{}).Limit(ipTagMergeBatch).Find(&rows).Error; err != nil { + zlog.Warn("读取待合并IP标签失败", "来源", srcName, "已合并", merged, "error", err.Error()) + return + } + if len(rows) == 0 { + break + } + // 先写目标再删来源。两个库之间没有共同事务，只能选一头承担中断风险： + // 中断在这两步之间，这一批下次会再并一遍（计数偏大），比删掉之后没写进去要好收拾。 + if err := upsertIPTags(dst, rows); err != nil { + zlog.Warn("写入IP标签失败", "目标", dstName, "已合并", merged, "error", err.Error()) + return + } + ids := make([]string, 0, len(rows)) + for i := range rows { + ids = append(ids, rows[i].Id) + } + if err := src.Where("id in ?", ids).Delete(&model.IPTag{}).Error; err != nil { + zlog.Warn("清理来源IP标签失败", "来源", srcName, "已合并", merged, "error", err.Error()) + return + } + merged += int64(len(rows)) + + if merged-lastLogged >= ipTagMergeLogEvery { + lastLogged = merged + zlog.Info("IP标签合并进行中", "已合并", merged, "共", pending, "耗时", time.Since(start).String()) + } + time.Sleep(ipTagMergePause) + } + zlog.Info("IP标签合并完成", "来源", srcName, "目标", dstName, "条数", merged, "耗时", time.Since(start).String()) +} + +// upsertIPTags 按唯一索引 uni_iptags_full 合并写入：cnt 相加，首次时间取早、最近时间取晚。 +// 冲突分支要引用"本次待插入的值"，三个引擎写法不同，由方言给出。 +func upsertIPTags(dst *gorm.DB, rows []model.IPTag) error { + cnt := dialect.Get().UpsertExcludedRef("cnt") + upd := dialect.Get().UpsertExcludedRef("update_time") + cre := dialect.Get().UpsertExcludedRef("create_time") + return dst.Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "user_code"}, {Name: "tenant_id"}, {Name: "ip"}, {Name: "ip_tag"}, + }, + DoUpdates: clause.Assignments(map[string]interface{}{ + "cnt": gorm.Expr("ip_tags.cnt + " + cnt), + "update_time": gorm.Expr("CASE WHEN " + upd + " > ip_tags.update_time THEN " + upd + " ELSE ip_tags.update_time END"), + "create_time": gorm.Expr("CASE WHEN " + cre + " < ip_tags.create_time THEN " + cre + " ELSE ip_tags.create_time END"), + }), + }).CreateInBatches(rows, ipTagMergeBatch).Error +} diff --git a/waftask/task_config.go b/waftask/task_config.go index 03a00340..358b822e 100644 --- a/waftask/task_config.go +++ b/waftask/task_config.go @@ -214,7 +214,16 @@ func setConfigIntValue(name string, value int64, change int) { global.GCONFIG_ENABLE_SYSTEM_STATS_PUSH = value break case "ip_tag_db": + if value != 0 { + value = 1 + } + changed := global.GDATA_IP_TAG_DB != value global.GDATA_IP_TAG_DB = value + if changed { + // 归属换了：把另一个库里的历史标签并过来，否则界面按新库查，看起来像"切一下历史就没了"。 + // 合并可能要搬几十万行，放后台跑，界面用 GDATA_IP_TAG_MERGING 显示"合并中"。 + go waf_service.MergeIPTagsInto(value) + } break case "ip_failure_ban_enabled": global.GCONFIG_IP_FAILURE_BAN_ENABLED = value @@ -889,4 +898,10 @@ func TaskLoadSetting(initLoad bool) { zlog.Warn("管理端可信代理网段包含过宽条目(" + entry + ")：这种网段无法用来判定代理头里的哪个IP是客户端，代理头将不被采信、仍按网络层IP识别客户端(管理端IP白名单/登录失败锁定/令牌IP绑定也按它判定)。请在 conf/config.yml 把 security.manage_trusted_proxies 改成上游代理自身的地址，容器部署可填 private") } } + + // 把落在另一个库里的 IP 标签收回当前归属。按内容判断、源库空即返回， + // 所以更早版本切换归属时留下的历史也能在这里被收回来，重复跑无副作用。 + if initLoad { + go waf_service.MergeIPTagsInto(global.GDATA_IP_TAG_DB) + } } diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index 504dec30..089695ab 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -659,3 +659,39 @@ notes: this can take several minutes, the startup log prints the time each index took, and the process should not be killed while it runs. IP tag counters are now written with a single batched statement, cutting database round trips considerably under load. + - id: v1_3_25_weblog_payload_split_and_iptag_merge + version: v1.3.25 + kind: notice + level: normal + page: '' + doc: https://doc.samwaf.com/quickstart/Update.html + apply: + type: none + zh: + title: 报文改存独立表，IP 标签切换存放位置时自动合并历史 + detail: >- + 访问日志的报文（请求体、响应体、表单、Cookie、响应头、原始字节）不再和日志记录存在同一张表里， + 改为写入新的报文表，按请求编号一对一存放。日志列表、按 IP / 规则检索、统计扫描都不再顺带读这些大字段， + 看报文时才按主键取一次。升级后首次启动会自动建表，不搬历史数据：升级之前的日志报文仍在原处， + 详情页照常能看，随保留期自然到期。日志页面的展示与筛选没有变化。 + 另外，风险日志页的"标签数据存放位置"切换后，会自动把原库里已积累的标签并到新库： + 相同 IP + 标签的次数相加，首次访问时间取早的、最近访问时间取晚的，合并完清空原库的这部分数据。 + 以前切换会让历史标签看起来消失（数据其实还在原库），现在不会了；合并期间页面会显示"正在合并"， + 完成后自动刷新。新安装的实例默认把标签放统计库，已有实例保持你当前的设置不变。 + en: + title: Payloads move to their own table, and switching the IP tag location now merges the history + detail: >- + Access log payloads (request body, response body, form data, cookies, response headers, raw bytes) no + longer live in the same table as the log record. They are written to a new payload table, one row per + request id. Listing logs, searching by IP or rule, and statistics scans no longer drag those large + columns along; the payload is fetched by primary key only when someone opens it. The table is created on + the first start after upgrading and no history is moved: payloads recorded before the upgrade stay where + they are, remain visible in the detail view, and expire with the retention period as usual. Nothing + changes in what the log page shows or how it filters. + Separately, switching "tag data location" on the risk log page now merges the tags already accumulated in + the other database: counts for the same IP and tag are added up, the first-seen time keeps the earlier + value and the last-seen time the later one, and the merged rows are cleared from the source. Switching + used to make tag history look like it had vanished (the data was still in the old database); that no + longer happens. The page shows a merging indicator and refreshes itself when it finishes. New + installations store tags in the stats database by default; existing installations keep their current + setting. From 4de6438d1b3fc13a1dde741b2f525959085a6a50 Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 17 Sep 2026 16:24:16 +0800 Subject: [PATCH 06/21] refactor: move request payloads out of the access log into their own table --- api/waf_ai_api.go | 8 + model/event_payload.go | 52 ++++++ .../waf_service/cross_engine_payload_test.go | 109 ++++++++++++ service/waf_service/waf_ai_label_service.go | 10 ++ service/waf_service/waf_log_payload.go | 104 +++++++++++ wafdb/log_shard.go | 51 ++++++ wafdb/migrations_log.go | 18 ++ wafqueue/log_queue.go | 9 +- wafqueue/log_split.go | 124 +++++++++++++ wafqueue/log_split_db_test.go | 164 ++++++++++++++++++ wafqueue/log_split_test.go | 128 ++++++++++++++ wafqueue/log_truncate.go | 43 ----- wafqueue/log_truncate_test.go | 46 ----- waftask/task_db_sharding.go | 8 + 14 files changed, 782 insertions(+), 92 deletions(-) create mode 100644 model/event_payload.go create mode 100644 service/waf_service/cross_engine_payload_test.go create mode 100644 service/waf_service/waf_log_payload.go create mode 100644 wafqueue/log_split.go create mode 100644 wafqueue/log_split_db_test.go create mode 100644 wafqueue/log_split_test.go diff --git a/api/waf_ai_api.go b/api/waf_ai_api.go index 153aa415..b1349491 100644 --- a/api/waf_ai_api.go +++ b/api/waf_ai_api.go @@ -5,6 +5,7 @@ import ( "SamWaf/innerbean" "SamWaf/model/common/response" "SamWaf/model/request" + "SamWaf/service/waf_service" "SamWaf/utils" "SamWaf/wafai" "encoding/json" @@ -300,6 +301,13 @@ func runAIExport(req request.WafAIExportReq, maxCount int) (outPath string, nAtt } total = len(rows) + // BODY/POST_FORM 已搬到 event_payload，批量补回来再导出，否则样本只剩 URL + fillRows := make([]*innerbean.WebLog, 0, len(rows)) + for i := range rows { + fillRows = append(fillRows, &rows[i]) + } + waf_service.FillLivePayloads(fillRows) + dir := filepath.Join(utils.GetCurrentDir(), "data", aiExportDir) if err = os.MkdirAll(dir, 0750); err != nil { return "", 0, 0, 0, total, fmt.Errorf("创建导出目录失败: %w", err) diff --git a/model/event_payload.go b/model/event_payload.go new file mode 100644 index 00000000..46fc3eeb --- /dev/null +++ b/model/event_payload.go @@ -0,0 +1,52 @@ +package model + +// EventPayload 一条请求的报文，与 web_logs 按 req_uuid 一对一。 +// +// 拆出来是因为 web_logs 一张表同时承担了「列表翻页」「按 IP/规则检索」「统计扫描」「看报文」四件事， +// 而报文列（header/cookies/body/res_body/post_form/res_header + 原始字节）占了绝大部分体积。 +// 列表和统计从不读它们，却要把它们一起拖过磁盘。拆开后 web_logs 只剩窄列，报文按主键点查。 +// +// Kind 预留给后续的分层写入： +// - event 命中规则的请求（默认，也是当前唯一在写的值） +// - sample 放行请求里被蓄水池采到的负样本（供 AI 训练） +// - watch 观察名单内 IP 的全量留痕 +type EventPayload struct { + ReqUUID string `gorm:"column:req_uuid;size:64;primaryKey" json:"req_uuid"` + TenantId string `gorm:"size:64" json:"tenant_id"` + UserCode string `gorm:"size:64" json:"user_code"` + HostCode string `gorm:"size:64" json:"host_code"` + Kind string `gorm:"size:16" json:"kind"` + + HEADER string `gorm:"type:text" json:"header"` + COOKIES string `gorm:"type:text" json:"cookies"` + BODY string `gorm:"type:text" json:"body"` + RES_BODY string `gorm:"type:text" json:"res_body"` + POST_FORM string `gorm:"type:text" json:"post_form"` + ResHeader string `gorm:"type:text" json:"res_header"` + + SrcByteBody []byte `json:"src_byte_body"` + SrcByteResBody []byte `json:"src_byte_res_body"` + SrcURL []byte `json:"src_url"` + + // Truncated 与 web_logs 上的同名列一致：单列超过上限被截断时为 1。 + Truncated int `json:"truncated"` + + // CreateTime 与 web_logs.create_time 同格式同值，保留期清理按它删，不必回表。 + CreateTime string `gorm:"size:32;index:idx_ep_create_time" json:"create_time"` + UnixAddTime int64 `json:"unix_add_time"` + Day int `json:"day"` +} + +// EventPayloadTableName 实时报文表名。归档分片上是它加分片后缀（见 wafdb.ResolveLogTables）。 +const EventPayloadTableName = "event_payload" + +func (EventPayload) TableName() string { + return EventPayloadTableName +} + +// 报文列全空的请求不值得占一行（比如被提前拦下、连 header 都没读到的连接）。 +func (e *EventPayload) IsEmpty() bool { + return e.HEADER == "" && e.COOKIES == "" && e.BODY == "" && + e.RES_BODY == "" && e.POST_FORM == "" && e.ResHeader == "" && + len(e.SrcByteBody) == 0 && len(e.SrcByteResBody) == 0 && len(e.SrcURL) == 0 +} diff --git a/service/waf_service/cross_engine_payload_test.go b/service/waf_service/cross_engine_payload_test.go new file mode 100644 index 00000000..ebad30f7 --- /dev/null +++ b/service/waf_service/cross_engine_payload_test.go @@ -0,0 +1,109 @@ +//go:build crossdb + +// 报文垂直拆表的三库回归：报文按 req_uuid 点查、老数据回落读 web_logs 原列、 +// 保留期清理连报文一起删。由 TestCrossEngine 每引擎调一次。 +package waf_service + +import ( + "SamWaf/innerbean" + "SamWaf/model" + req "SamWaf/model/request" + "testing" + "time" + + "gorm.io/gorm" +) + +func runPayloadCases(t *testing.T, logdb *gorm.DB) { + now := time.Now() + + // 新数据：窄行不带报文，报文在 event_payload 里，详情按 req_uuid 点查补回来 + t.Run("拆表后详情能读到报文", func(t *testing.T) { + uid := "req_split_" + sfx() + must(t, logdb.Create(&innerbean.WebLog{ + REQ_UUID: uid, + HOST_CODE: "h1", + URL: "/login", + METHOD: "POST", + USER_CODE: xtestUser, + TenantId: xtestTenant, + UNIX_ADD_TIME: now.Unix(), + CREATE_TIME: now.Format("2006-01-02 15:04:05"), + HEADER: "User-Agent: curl", // HEADER 本次不搬，仍在窄行上 + }).Error) + must(t, logdb.Create(&model.EventPayload{ + ReqUUID: uid, + TenantId: xtestTenant, + UserCode: xtestUser, + HostCode: "h1", + Kind: "event", + BODY: "user=admin&pwd=1", + RES_BODY: "ok", + POST_FORM: "user=admin&pwd=1", + COOKIES: "sid=abc", + ResHeader: "Content-Type: text/html", + Truncated: 1, + CreateTime: now.Format("2006-01-02 15:04:05"), + UnixAddTime: now.Unix(), + Day: 20260918, + }).Error) + + got, err := WafLogServiceApp.GetDetailApi(req.WafAttackLogDetailReq{REQ_UUID: uid}) + fatalIf(t, err) + if got.BODY != "user=admin&pwd=1" { + t.Fatalf("BODY 没从报文表补回来: %q", got.BODY) + } + if got.RES_BODY != "ok" || got.COOKIES != "sid=abc" || got.ResHeader != "Content-Type: text/html" { + t.Fatalf("报文补得不全: %+v", got) + } + if got.HEADER != "User-Agent: curl" { + t.Fatalf("HEADER 留在窄行上，不该被报文行的空值抹掉: %q", got.HEADER) + } + if got.Truncated != 1 { + t.Fatal("截断标记应跟着报文一起回来") + } + }) + + // 存量数据：报文还在 web_logs 自己的列里，没有对应的报文行，详情必须照旧能看 + t.Run("改造前的存量行仍读原列", func(t *testing.T) { + uid := "req_legacy_" + sfx() + must(t, logdb.Create(&innerbean.WebLog{ + REQ_UUID: uid, + HOST_CODE: "h1", + URL: "/old", + USER_CODE: xtestUser, + TenantId: xtestTenant, + UNIX_ADD_TIME: now.Unix(), + CREATE_TIME: now.Format("2006-01-02 15:04:05"), + BODY: "legacy-body", + RES_BODY: "legacy-res", + COOKIES: "legacy-cookie", + }).Error) + + got, err := WafLogServiceApp.GetDetailApi(req.WafAttackLogDetailReq{REQ_UUID: uid}) + fatalIf(t, err) + if got.BODY != "legacy-body" || got.RES_BODY != "legacy-res" || got.COOKIES != "legacy-cookie" { + t.Fatalf("存量行的报文被抹掉了: %+v", got) + } + }) + + // 保留期清理要把报文一起删掉，否则报文表只增不减 + t.Run("保留期清理连报文一起删", func(t *testing.T) { + uid := "req_expire_" + sfx() + old := now.AddDate(0, 0, -30).Format("2006-01-02 15:04:05") + must(t, logdb.Create(&innerbean.WebLog{ + REQ_UUID: uid, HOST_CODE: "h1", USER_CODE: xtestUser, TenantId: xtestTenant, + UNIX_ADD_TIME: now.AddDate(0, 0, -30).Unix(), CREATE_TIME: old, + }).Error) + must(t, logdb.Create(&model.EventPayload{ + ReqUUID: uid, TenantId: xtestTenant, UserCode: xtestUser, + Kind: "event", BODY: "expired", CreateTime: old, + }).Error) + + WafLogServiceApp.DeleteHistory(now.AddDate(0, 0, -1).Format("2006-01-02 15:04")) + + if n := countBy(t, logdb, &model.EventPayload{}, "req_uuid = ?", uid); n != 0 { + t.Fatalf("过期报文没被清掉，还剩 %d 行", n) + } + }) +} diff --git a/service/waf_service/waf_ai_label_service.go b/service/waf_service/waf_ai_label_service.go index 99a3a7b8..d8e09059 100644 --- a/service/waf_service/waf_ai_label_service.go +++ b/service/waf_service/waf_ai_label_service.go @@ -43,6 +43,9 @@ func (receiver *WafAILabelService) MarkApi(req request.WafAILabelMarkReq) error global.GWAF_LOCAL_LOG_DB. Select("METHOD", "URL", "RawQuery", "BODY", "POST_FORM", "USER_AGENT", "ACTION", "RULE", "SRC_IP", "HOST_CODE", "LogOnlyMode"). Where("REQ_UUID = ?", req.ReqUuid).Limit(1).Find(&wl) + // BODY/POST_FORM 已搬到 event_payload，补回来才有东西做快照 + wl.REQ_UUID = req.ReqUuid + FillLivePayloads([]*innerbean.WebLog{&wl}) body := wl.BODY if body == "" { @@ -247,6 +250,13 @@ func (receiver *WafAILabelService) ListApi(req request.WafAILabelListReq) respon Order("ai_score desc").Order("unix_add_time desc"). Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&rows) + // BODY/POST_FORM 已搬到 event_payload，整页一次补回 + fillRows := make([]*innerbean.WebLog, 0, len(rows)) + for i := range rows { + fillRows = append(fillRows, &rows[i]) + } + FillLivePayloads(fillRows) + for i := range rows { r := &rows[i] body := r.BODY diff --git a/service/waf_service/waf_log_payload.go b/service/waf_service/waf_log_payload.go new file mode 100644 index 00000000..ad76325e --- /dev/null +++ b/service/waf_service/waf_log_payload.go @@ -0,0 +1,104 @@ +package waf_service + +import ( + "SamWaf/common/zlog" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/wafdb" +) + +// payloadFetchChunk 一次 IN 查询带的 req_uuid 个数。 +// SQLite 默认最多 999 个绑定变量，留足余量。 +const payloadFetchChunk = 500 + +// FillLivePayloads 给实时库查出来的日志补上报文列。 +func FillLivePayloads(rows []*innerbean.WebLog) { + FillShardPayloads("", rows) +} + +// FillShardPayloads 按 req_uuid 把报文补回日志对象。 +// +// 报文自本次改造起写在独立的 event_payload 表里，web_logs 那几列对新数据是空的。 +// 凡是要看报文的读取方，查完日志都得再走这一步，否则拿到的是空字符串。 +// +// 三种情况都能正确落地： +// - 新数据：找得到报文行，以它为准 +// - 改造前的存量行：找不到报文行，保留 web_logs 原列里的报文 +// - 改造前切出去的归档分片：整张报文表都不存在，直接返回，同样保留原列 +func FillShardPayloads(currentDbName string, rows []*innerbean.WebLog) { + if len(rows) == 0 { + return + } + db, _, payloadTable := wafdb.ResolveLogTables(currentDbName) + if db == nil || payloadTable == "" { + return + } + + index := make(map[string][]*innerbean.WebLog, len(rows)) + uuids := make([]string, 0, len(rows)) + for _, r := range rows { + if r == nil || r.REQ_UUID == "" { + continue + } + if _, ok := index[r.REQ_UUID]; !ok { + uuids = append(uuids, r.REQ_UUID) + } + index[r.REQ_UUID] = append(index[r.REQ_UUID], r) + } + if len(uuids) == 0 { + return + } + + for start := 0; start < len(uuids); start += payloadFetchChunk { + end := start + payloadFetchChunk + if end > len(uuids) { + end = len(uuids) + } + var found []model.EventPayload + err := db.Table(payloadTable). + Select("req_uuid", "header", "cookies", "body", "res_body", "post_form", "res_header", "truncated"). + Where("req_uuid in ?", uuids[start:end]).Find(&found).Error + if err != nil { + // 报文读不到不该让整个页面失败：日志本身已经查出来了，报文列留空即可 + zlog.Warn("读取报文失败", "table", payloadTable, "条数", end-start, "error", err.Error()) + return + } + for i := range found { + p := &found[i] + for _, r := range index[p.ReqUUID] { + applyPayload(r, p) + } + } + } +} + +// applyPayload 只用非空值覆盖，空值一律不动 web_logs 上的原值。 +// +// 这条规则同时管住三件事：报文表里某列为空时不会把窄行上仍然有效的值抹掉 +// （HEADER 这一列本次就还留在窄行里，见 wafqueue/log_split.go）； +// 存量行与新行混在一页时各取各的；将来把某一列挪进报文表也不用改这里。 +// +// 原始字节列(src_byte_*/src_url)不回填：详情与列表历来都不返回它们，回填只会把响应撑大。 +func applyPayload(r *innerbean.WebLog, p *model.EventPayload) { + if p.HEADER != "" { + r.HEADER = p.HEADER + } + if p.COOKIES != "" { + r.COOKIES = p.COOKIES + } + if p.BODY != "" { + r.BODY = p.BODY + } + if p.RES_BODY != "" { + r.RES_BODY = p.RES_BODY + } + if p.POST_FORM != "" { + r.POST_FORM = p.POST_FORM + } + if p.ResHeader != "" { + r.ResHeader = p.ResHeader + } + if p.Truncated == 1 { + r.Truncated = 1 + } +} diff --git a/wafdb/log_shard.go b/wafdb/log_shard.go index 4e39ec04..da962141 100644 --- a/wafdb/log_shard.go +++ b/wafdb/log_shard.go @@ -4,7 +4,10 @@ import ( "SamWaf/common/zlog" "SamWaf/enums" "SamWaf/global" + "SamWaf/model" "SamWaf/wafdb/dialect" + "strings" + "sync" "gorm.io/gorm" ) @@ -63,3 +66,51 @@ func ResolveLogDB(currentDbName string) (*gorm.DB, string) { } return global.GWAF_LOCAL_LOG_DB, LogTableName } + +// payloadTableSeen 记住哪些分片确认有报文表。只缓存"有"这一侧： +// 归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到，缓存下来会一直读不到报文。 +var payloadTableSeen sync.Map + +// ResolveLogTables 在 ResolveLogDB 的基础上再给出该分片的报文表名。 +// +// 报文表与日志表同进同出： +// - SQLite 按文件分片，归档文件里自带 event_payload +// - MySQL / PostgreSQL 按表分片，web_logs_ 对应 event_payload_ +// +// 返回空表名表示这个分片没有报文表——本次改造之前切出去的归档都是这样， +// 它们的报文还在 web_logs 自己的列里，读侧照旧读原列即可。 +func ResolveLogTables(currentDbName string) (*gorm.DB, string, string) { + db, logTable := ResolveLogDB(currentDbName) + if db == nil { + return db, logTable, "" + } + + // SQLite 的归档分片是整个文件，连上去之后表名仍是 web_logs / event_payload + if logTable == LogTableName { + return db, logTable, probePayloadTable(db, currentDbName, model.EventPayloadTableName) + } + + // MySQL / PostgreSQL：web_logs_ 对应 event_payload_ + shardPayload := model.EventPayloadTableName + strings.TrimPrefix(logTable, LogTableName) + if t := probePayloadTable(db, currentDbName, shardPayload); t != "" { + return db, logTable, t + } + // 分表时报文表没能一起换过去（换表失败或这批日志早于本次改造）： + // 报文按 req_uuid 寻址，留在实时表里照样找得到，退回去查它。 + return db, logTable, probePayloadTable(db, currentDbName, model.EventPayloadTableName) +} + +// probePayloadTable 探一次表存不存在，存在则返回表名。 +// 只缓存"存在"这一侧：归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到， +// 缓存下来会一直读不到报文。 +func probePayloadTable(db *gorm.DB, shard, table string) string { + key := shard + "|" + table + if _, ok := payloadTableSeen.Load(key); ok { + return table + } + if !dialect.Get().TableExists(db, table) { + return "" + } + payloadTableSeen.Store(key, struct{}{}) + return table +} diff --git a/wafdb/migrations_log.go b/wafdb/migrations_log.go index 9307638c..6f2e22f7 100644 --- a/wafdb/migrations_log.go +++ b/wafdb/migrations_log.go @@ -466,6 +466,24 @@ func RunLogDBMigrations(db *gorm.DB) error { return nil }, }, + // 报文垂直拆表：新写入的报文进 event_payload，web_logs 只留窄列。 + // 建表不搬历史——存量行的报文仍在 web_logs 自己的列里，读侧按 req_uuid 找不到报文行时回落读原列， + // 所以升级当下任何一条老日志的详情都照旧能看。 + { + ID: "202609170003_add_event_payload_table", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609170003: 创建报文表 event_payload") + if err := tx.AutoMigrate(&model.EventPayload{}); err != nil { + return fmt.Errorf("创建 event_payload 表失败: %w", err) + } + zlog.Info("event_payload 表创建成功") + return nil + }, + Rollback: func(tx *gorm.DB) error { + zlog.Info("回滚 202609170003: 删除 event_payload 表") + return tx.Migrator().DropTable(&model.EventPayload{}) + }, + }, }) // 执行迁移 diff --git a/wafqueue/log_queue.go b/wafqueue/log_queue.go index 4fcc5f02..bfd9989a 100644 --- a/wafqueue/log_queue.go +++ b/wafqueue/log_queue.go @@ -61,9 +61,12 @@ func ProcessLogDequeEngine() { } } if global.GCONFIG_LOG_PERSIST_ENABLED == 1 { - // 只有落库这一份做超长截断；下面的统计与出口仍用原始报文 - storeArray := truncateForStore(webLogArray) - global.GWAF_LOCAL_LOG_DB.CreateInBatches(storeArray, len(storeArray)) + // 只有落库这一份拆表并做超长截断；下面的统计与出口仍拿完整的原始对象 + storeArray, payloads := splitForStore(webLogArray) + if err := global.GWAF_LOCAL_LOG_DB.CreateInBatches(storeArray, len(storeArray)).Error; err != nil { + zlog.Warn("日志落库失败", "条数", len(storeArray), "error", err.Error()) + } + storePayloads(payloads) } // 日志流做统计 waftask.CollectStatsFromLogs(webLogArray) diff --git a/wafqueue/log_split.go b/wafqueue/log_split.go new file mode 100644 index 00000000..fdd5adaa --- /dev/null +++ b/wafqueue/log_split.go @@ -0,0 +1,124 @@ +package wafqueue + +import ( + "SamWaf/common/zlog" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + + "gorm.io/gorm/clause" +) + +// PayloadKindEvent 当前唯一在写的报文归属。采样负样本(sample)与观察名单(watch)在后续分层写入里接上。 +const PayloadKindEvent = "event" + +// splitForStore 把一批日志拆成两份落库数据：只剩窄列的 web_logs 行，和装报文的 event_payload 行。 +// +// 两条硬性约束： +// +// 1. 不改原对象。Kafka 出口、文件日志、统计与规则引擎共享同一批指针，且落库排在它们前面， +// 就地清列会让下游只拿到空报文。所以每条都是在副本上动手。 +// 2. req_uuid 为空的日志不拆。报文表以 req_uuid 为主键，没有它就没法再找回来， +// 这种日志维持老样子——报文留在 web_logs 自己的列里（仍然截断）。 +// +// 同一个 req_uuid 在一批里可能出现两次（一次请求分阶段入队，后一条信息更全）， +// web_logs 没有主键容得下两行，报文表容不下，所以按 req_uuid 去重取后到的那条。 +// +// HEADER 这一列本次不搬：访问日志页把它当独立列显示，还带一个 LIKE 全文筛选。 +// 搬走等于那一列立刻空掉、筛选也不再命中，而替代的筛选入口要等日志分层(access_log / +// security_event)拆出来之后才有地方放。它留在窄行里，跟着后续的视图拆分一起走。 +func splitForStore(logs []*innerbean.WebLog) ([]*innerbean.WebLog, []*model.EventPayload) { + narrow := make([]*innerbean.WebLog, 0, len(logs)) + payloads := make([]*model.EventPayload, 0, len(logs)) + seen := make(map[string]int, len(logs)) + + for _, lg := range logs { + if lg == nil { + continue + } + c := *lg + + header := cutUTF8(lg.HEADER) + cookies := cutUTF8(lg.COOKIES) + body := cutUTF8(lg.BODY) + resBody := cutUTF8(lg.RES_BODY) + postForm := cutUTF8(lg.POST_FORM) + resHeader := cutUTF8(lg.ResHeader) + srcBody := cutBytes(lg.SrcByteBody) + srcResBody := cutBytes(lg.SrcByteResBody) + srcURL := cutBytes(lg.SrcURL) + + if len(header) != len(lg.HEADER) || len(cookies) != len(lg.COOKIES) || + len(body) != len(lg.BODY) || len(resBody) != len(lg.RES_BODY) || + len(postForm) != len(lg.POST_FORM) || len(resHeader) != len(lg.ResHeader) || + len(srcBody) != len(lg.SrcByteBody) || len(srcResBody) != len(lg.SrcByteResBody) || + len(srcURL) != len(lg.SrcURL) { + c.Truncated = 1 + } + + c.HEADER = header + + if lg.REQ_UUID == "" { + c.COOKIES, c.BODY = cookies, body + c.RES_BODY, c.POST_FORM, c.ResHeader = resBody, postForm, resHeader + c.SrcByteBody, c.SrcByteResBody, c.SrcURL = srcBody, srcResBody, srcURL + narrow = append(narrow, &c) + continue + } + + p := &model.EventPayload{ + ReqUUID: lg.REQ_UUID, + TenantId: lg.TenantId, + UserCode: lg.USER_CODE, + HostCode: lg.HOST_CODE, + Kind: PayloadKindEvent, + COOKIES: cookies, + BODY: body, + RES_BODY: resBody, + POST_FORM: postForm, + ResHeader: resHeader, + SrcByteBody: srcBody, + SrcByteResBody: srcResBody, + SrcURL: srcURL, + Truncated: c.Truncated, + CreateTime: lg.CREATE_TIME, + UnixAddTime: lg.UNIX_ADD_TIME, + Day: lg.Day, + } + + c.COOKIES, c.BODY = "", "" + c.RES_BODY, c.POST_FORM, c.ResHeader = "", "", "" + c.SrcByteBody, c.SrcByteResBody, c.SrcURL = nil, nil, nil + narrow = append(narrow, &c) + + // 报文列全空就不占一行：读侧找不到报文行时会回落读 web_logs 的原列，结果一样是空。 + if p.IsEmpty() { + continue + } + if at, dup := seen[p.ReqUUID]; dup { + payloads[at] = p + continue + } + seen[p.ReqUUID] = len(payloads) + payloads = append(payloads, p) + } + + return narrow, payloads +} + +// storePayloads 写报文表。单独一条语句，且失败只告警： +// 日志行已经落库，报文丢了详情页显示「未留存报文」，比整批日志一起回滚强。 +// 跨批次可能撞上同一个 req_uuid（同一请求分两批入队），主键冲突按「保留先到的」跳过。 +func storePayloads(payloads []*model.EventPayload) { + if len(payloads) == 0 { + return + } + // 不指定冲突列：三个引擎对"无目标的 DO NOTHING"都认（MySQL 走 INSERT IGNORE）， + // 指定了反而要各写各的。这里只有主键一种冲突，无目标正合适。 + err := global.GWAF_LOCAL_LOG_DB. + Clauses(clause.OnConflict{DoNothing: true}). + CreateInBatches(payloads, len(payloads)).Error + if err != nil { + zlog.Warn("报文落库失败", "条数", len(payloads), "error", err.Error()) + } +} diff --git a/wafqueue/log_split_db_test.go b/wafqueue/log_split_db_test.go new file mode 100644 index 00000000..6ec9c53b --- /dev/null +++ b/wafqueue/log_split_db_test.go @@ -0,0 +1,164 @@ +//go:build crossdb + +// 报文落库这一步的三库回归。splitForStore 的纯逻辑由 log_split_test.go 覆盖， +// 这里盯的是真正压到数据库上的那条语句：报文表以 req_uuid 为主键， +// 同一请求分两批入队就会撞主键，三个引擎的"冲突即跳过"写法各不相同， +// 撞坏了整批报文都写不进去（日志行还在，用户看到的是报文凭空消失）。 +// +// go test -tags crossdb ./wafqueue/ -run TestStorePayloadsCrossEngine -v +package wafqueue + +import ( + "SamWaf/common/zlog" + "SamWaf/global" + "SamWaf/model" + "SamWaf/wafdb/dialect" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + sqlitedriver "github.com/samwafgo/sqlitedriver" + mysqldriver "gorm.io/driver/mysql" + pgdriver "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var silentCfg = &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)} + +type payloadEngine struct { + name string + setup func(t *testing.T) (*gorm.DB, func()) +} + +func payloadEngines() []payloadEngine { + return []payloadEngine{ + {"sqlite", func(t *testing.T) (*gorm.DB, func()) { + dialect.Register(&dialect.SQLiteDialect{}) + path := filepath.Join(t.TempDir(), "payload_test.db") + db, err := gorm.Open(sqlitedriver.Open(path), silentCfg) + if err != nil { + t.Logf("sqlite 打开失败（跳过）: %v", err) + return nil, nil + } + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + } + }}, + {"mysql", func(t *testing.T) (*gorm.DB, func()) { + dsn := os.Getenv("SAMWAF_TEST_MYSQL_DSN") + if dsn == "" { + dsn = "root:canteen1@tcp(127.0.0.1:3306)/" + } + name := fmt.Sprintf("samwaf_payload_%d", time.Now().UnixNano()%100000) + root, err := gorm.Open(mysqldriver.Open(dsn+"?charset=utf8mb4&parseTime=True&loc=Local"), silentCfg) + if err != nil { + t.Logf("mysql 连接失败（跳过）: %v", err) + return nil, nil + } + if err := root.Exec("CREATE DATABASE " + name + " DEFAULT CHARACTER SET utf8mb4").Error; err != nil { + t.Logf("mysql 建库失败（跳过）: %v", err) + return nil, nil + } + dialect.Register(&dialect.MySQLDialect{}) + db, err := gorm.Open(mysqldriver.Open(dsn+name+"?charset=utf8mb4&parseTime=True&loc=Local"), silentCfg) + if err != nil { + t.Fatalf("mysql 打开 %s: %v", name, err) + } + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + root.Exec("DROP DATABASE IF EXISTS " + name) + if s, e := root.DB(); e == nil { + s.Close() + } + } + }}, + {"postgres", func(t *testing.T) (*gorm.DB, func()) { + base := os.Getenv("SAMWAF_TEST_PG_DSN") + if base == "" { + base = "postgres://postgres:postgres@127.0.0.1:5432/" + } + name := fmt.Sprintf("samwaf_payload_%d", time.Now().UnixNano()%100000) + root, err := gorm.Open(pgdriver.Open(base+"postgres?sslmode=disable"), silentCfg) + if err != nil { + t.Logf("postgres 连接失败（跳过）: %v", err) + return nil, nil + } + if err := root.Exec("CREATE DATABASE " + name + " ENCODING 'UTF8'").Error; err != nil { + t.Logf("postgres 建库失败（跳过）: %v", err) + return nil, nil + } + dialect.Register(&dialect.PostgresDialect{}) + db, err := gorm.Open(pgdriver.Open(base+name+"?sslmode=disable&TimeZone=Asia/Shanghai"), silentCfg) + if err != nil { + t.Fatalf("postgres 打开 %s: %v", name, err) + } + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + root.Exec("DROP DATABASE IF EXISTS " + name) + if s, e := root.DB(); e == nil { + s.Close() + } + } + }}, + } +} + +func TestStorePayloadsCrossEngine(t *testing.T) { + zlog.InitZLog(false, "console") + saved := global.GWAF_LOCAL_LOG_DB + defer func() { global.GWAF_LOCAL_LOG_DB = saved }() + + for _, e := range payloadEngines() { + e := e + t.Run(e.name, func(t *testing.T) { + db, teardown := e.setup(t) + if db == nil { + t.Skipf("%s 未就绪，跳过", e.name) + return + } + defer teardown() + if err := db.AutoMigrate(&model.EventPayload{}); err != nil { + t.Fatalf("建 event_payload 失败: %v", err) + } + global.GWAF_LOCAL_LOG_DB = db + + rows := []*model.EventPayload{ + {ReqUUID: "u1", Kind: PayloadKindEvent, BODY: "a=1", CreateTime: "2026-09-17 10:00:00"}, + {ReqUUID: "u2", Kind: PayloadKindEvent, BODY: "b=2", CreateTime: "2026-09-17 10:00:00"}, + } + storePayloads(rows) + + var n int64 + db.Model(&model.EventPayload{}).Count(&n) + if n != 2 { + t.Fatalf("首次写入应有 2 行，实际 %d", n) + } + + // 同一请求分两批入队：主键撞上必须整批跳过而不是整批失败 + again := []*model.EventPayload{ + {ReqUUID: "u1", Kind: PayloadKindEvent, BODY: "a=1-late", CreateTime: "2026-09-17 10:00:01"}, + {ReqUUID: "u3", Kind: PayloadKindEvent, BODY: "c=3", CreateTime: "2026-09-17 10:00:01"}, + } + storePayloads(again) + + db.Model(&model.EventPayload{}).Count(&n) + if n != 3 { + t.Fatalf("撞主键应跳过冲突行、写入新行，期望 3 行实际 %d（整批失败了？）", n) + } + var got model.EventPayload + db.Where("req_uuid = ?", "u1").First(&got) + if got.BODY != "a=1" { + t.Fatalf("冲突行应保留先到的那条，实际 %q", got.BODY) + } + }) + } +} diff --git a/wafqueue/log_split_test.go b/wafqueue/log_split_test.go new file mode 100644 index 00000000..82ec09ca --- /dev/null +++ b/wafqueue/log_split_test.go @@ -0,0 +1,128 @@ +package wafqueue + +import ( + "SamWaf/innerbean" + "strings" + "testing" +) + +// 报文搬进 event_payload 行，web_logs 行只剩窄列 +func TestSplitForStore_MovesPayload(t *testing.T) { + origin := &innerbean.WebLog{ + REQ_UUID: "uuid-1", + HOST_CODE: "host-1", + HEADER: "User-Agent: curl", + BODY: "a=1", + RES_BODY: "ok", + POST_FORM: "a=1", + COOKIES: "sid=x", + ResHeader: "Content-Type: text/html", + URL: "/login", + } + + narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) + + if len(narrow) != 1 || len(payloads) != 1 { + t.Fatalf("应拆成 1 窄行 + 1 报文行，实际 %d/%d", len(narrow), len(payloads)) + } + n := narrow[0] + if n.BODY != "" || n.RES_BODY != "" || n.POST_FORM != "" || + n.COOKIES != "" || n.ResHeader != "" { + t.Fatal("窄行的报文列没清干净") + } + if n.URL != "/login" { + t.Fatal("非报文列不该被动") + } + // HEADER 本次不搬：访问日志页把它当独立列显示还带 LIKE 筛选 + if n.HEADER != "User-Agent: curl" { + t.Fatal("HEADER 必须留在窄行上") + } + p := payloads[0] + if p.ReqUUID != "uuid-1" || p.HostCode != "host-1" || p.Kind != PayloadKindEvent { + t.Fatalf("报文行归属字段不对: %+v", p) + } + if p.BODY != "a=1" || p.RES_BODY != "ok" || p.ResHeader != "Content-Type: text/html" { + t.Fatal("报文没搬全") + } + if p.HEADER != "" { + t.Fatal("HEADER 本次不进报文表，否则会存两份") + } +} + +// 原对象必须保持原文：Kafka 出口与规则引擎共享同一批指针，且落库排在它们前面 +func TestSplitForStore_KeepsOriginalIntact(t *testing.T) { + big := strings.Repeat("b", payloadMaxBytes+1000) + origin := &innerbean.WebLog{REQ_UUID: "uuid-2", BODY: big, RES_BODY: "ok"} + + narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) + + if narrow[0] == origin { + t.Fatal("窄行必须是副本，不能就地改原对象") + } + if len(origin.BODY) != len(big) || origin.Truncated != 0 { + t.Fatal("原对象被改动了，Kafka 出口会拿到半截报文") + } + if len(payloads[0].BODY) > payloadMaxBytes { + t.Fatalf("报文没截断，长度 %d", len(payloads[0].BODY)) + } + if payloads[0].Truncated != 1 || narrow[0].Truncated != 1 { + t.Fatal("截断标记两边都要有") + } +} + +// 没有 req_uuid 就没法再按主键找回报文，这种日志维持老样子留在 web_logs 列里 +func TestSplitForStore_NoUUIDKeepsInline(t *testing.T) { + origin := &innerbean.WebLog{BODY: "a=1", COOKIES: "sid=x"} + + narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) + + if len(payloads) != 0 { + t.Fatal("没有 req_uuid 不应产出报文行") + } + if narrow[0].BODY != "a=1" || narrow[0].COOKIES != "sid=x" { + t.Fatal("没有 req_uuid 时报文必须留在窄行里，否则就丢了") + } +} + +// 报文列全空不占一行 +func TestSplitForStore_SkipsEmptyPayload(t *testing.T) { + origin := &innerbean.WebLog{REQ_UUID: "uuid-3", URL: "/", SRC_IP: "1.2.3.4", HEADER: "H: 1"} + + narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) + + if len(narrow) != 1 { + t.Fatal("窄行还是要写的") + } + if len(payloads) != 0 { + t.Fatal("报文全空不该占一行") + } + if narrow[0].HEADER != "H: 1" { + t.Fatal("HEADER 留在窄行，不算报文") + } +} + +// 同一 req_uuid 在一批里出现两次（分阶段入队），报文表主键容不下两行，取后到的那条 +func TestSplitForStore_DedupByUUID(t *testing.T) { + first := &innerbean.WebLog{REQ_UUID: "uuid-4", BODY: "a=1"} + second := &innerbean.WebLog{REQ_UUID: "uuid-4", BODY: "a=1", RES_BODY: "done"} + + narrow, payloads := splitForStore([]*innerbean.WebLog{first, second}) + + if len(narrow) != 2 { + t.Fatal("web_logs 没有主键，两行都照写") + } + if len(payloads) != 1 { + t.Fatalf("报文行应去重成 1 条，实际 %d", len(payloads)) + } + if payloads[0].RES_BODY != "done" { + t.Fatal("应保留后到的那条（信息更全）") + } +} + +// nil 条目直接跳过，不能带崩整批 +func TestSplitForStore_SkipsNil(t *testing.T) { + narrow, payloads := splitForStore([]*innerbean.WebLog{nil, {REQ_UUID: "uuid-5", BODY: "x"}}) + if len(narrow) != 1 || len(payloads) != 1 { + t.Fatalf("nil 应被跳过，实际 %d/%d", len(narrow), len(payloads)) + } +} diff --git a/wafqueue/log_truncate.go b/wafqueue/log_truncate.go index 617c67d0..602ab8a7 100644 --- a/wafqueue/log_truncate.go +++ b/wafqueue/log_truncate.go @@ -1,55 +1,12 @@ package wafqueue import ( - "SamWaf/innerbean" "unicode/utf8" ) // payloadMaxBytes 单个报文列落库的字节上限，超出部分丢弃并置 Truncated=1。 const payloadMaxBytes = 64 * 1024 -// truncateForStore 返回用于落库的日志切片：超长报文列在副本上截断，未超长的元素原样复用。 -// -// 关键点是不改原对象——Kafka 出口与规则引擎共享同一批指针，且落库在它们之前执行， -// 就地截断会让下游也只能拿到半截报文。整批都没超长时直接返回原切片，不额外分配。 -func truncateForStore(logs []*innerbean.WebLog) []*innerbean.WebLog { - out := logs - copied := false - for i, lg := range logs { - if lg == nil || !needTruncate(lg) { - continue - } - if !copied { - out = make([]*innerbean.WebLog, len(logs)) - copy(out, logs) - copied = true - } - c := *lg - c.BODY = cutUTF8(c.BODY) - c.RES_BODY = cutUTF8(c.RES_BODY) - c.POST_FORM = cutUTF8(c.POST_FORM) - c.HEADER = cutUTF8(c.HEADER) - c.COOKIES = cutUTF8(c.COOKIES) - c.ResHeader = cutUTF8(c.ResHeader) - c.SrcByteBody = cutBytes(c.SrcByteBody) - c.SrcByteResBody = cutBytes(c.SrcByteResBody) - c.Truncated = 1 - out[i] = &c - } - return out -} - -func needTruncate(lg *innerbean.WebLog) bool { - return len(lg.BODY) > payloadMaxBytes || - len(lg.RES_BODY) > payloadMaxBytes || - len(lg.POST_FORM) > payloadMaxBytes || - len(lg.HEADER) > payloadMaxBytes || - len(lg.COOKIES) > payloadMaxBytes || - len(lg.ResHeader) > payloadMaxBytes || - len(lg.SrcByteBody) > payloadMaxBytes || - len(lg.SrcByteResBody) > payloadMaxBytes -} - // cutUTF8 按字节上限截断，并回退到合法 UTF-8 边界（最多退 3 字节）， // 免得把多字节字符切成半个，在 utf8mb4 列上落库报错。 // 原文本身是二进制时回退会立即停下，最多多丢 3 字节。 diff --git a/wafqueue/log_truncate_test.go b/wafqueue/log_truncate_test.go index c7159819..be2b24c4 100644 --- a/wafqueue/log_truncate_test.go +++ b/wafqueue/log_truncate_test.go @@ -1,57 +1,11 @@ package wafqueue import ( - "SamWaf/innerbean" "strings" "testing" "unicode/utf8" ) -// 未超长的批次原样返回，不做任何拷贝 -func TestTruncateForStore_NoOversize(t *testing.T) { - logs := []*innerbean.WebLog{ - {BODY: "hello", RES_BODY: "world"}, - {BODY: strings.Repeat("a", payloadMaxBytes)}, - } - out := truncateForStore(logs) - if &out[0] != &logs[0] { - t.Fatal("未超长时不应重新分配切片") - } - for i := range out { - if out[i] != logs[i] { - t.Fatalf("第 %d 条不应被替换", i) - } - if out[i].Truncated != 0 { - t.Fatalf("第 %d 条不应被标记截断", i) - } - } -} - -// 超长时截断落库副本，原对象必须保持原文（Kafka 出口与规则引擎共享它） -func TestTruncateForStore_KeepsOriginalIntact(t *testing.T) { - big := strings.Repeat("b", payloadMaxBytes+1000) - origin := &innerbean.WebLog{BODY: big, RES_BODY: "ok"} - logs := []*innerbean.WebLog{origin} - - out := truncateForStore(logs) - - if out[0] == origin { - t.Fatal("超长条目应换成副本，不能就地改原对象") - } - if len(origin.BODY) != len(big) || origin.Truncated != 0 { - t.Fatal("原对象被改动了，Kafka 出口会拿到半截报文") - } - if len(out[0].BODY) > payloadMaxBytes { - t.Fatalf("副本未被截断，长度 %d", len(out[0].BODY)) - } - if out[0].Truncated != 1 { - t.Fatal("副本应标记 Truncated=1") - } - if out[0].RES_BODY != "ok" { - t.Fatal("未超长的列不应被动") - } -} - // 截断点落在多字节字符中间时要回退到合法边界 func TestCutUTF8_RuneBoundary(t *testing.T) { // 每个「中」占 3 字节，构造一个恰好在字符中间被切开的串 diff --git a/waftask/task_db_sharding.go b/waftask/task_db_sharding.go index 1fcce26a..03d79abc 100644 --- a/waftask/task_db_sharding.go +++ b/waftask/task_db_sharding.go @@ -164,6 +164,14 @@ func TaskShareDbInfo() { if err := dialect.Get().ShardSwapTable(global.GWAF_LOCAL_LOG_DB, "web_logs", archiveName); err != nil { zlog.Error(innerLogName, "分表失败:", err) } else { + // 报文表跟着日志表一起归档，两张表的分片边界才对得上。 + // 换不过去也不回滚：报文按 req_uuid 寻址，留在实时表里读侧照样找得到（见 ResolveLogTables）。 + payloadArchive := model.EventPayloadTableName + fmt.Sprintf("_%v", ts) + if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.EventPayloadTableName) { + if err := dialect.Get().ShardSwapTable(global.GWAF_LOCAL_LOG_DB, model.EventPayloadTableName, payloadArchive); err != nil { + zlog.Warn(innerLogName, "报文表分表失败，报文留在实时表:", err) + } + } global.GWAF_LOCAL_DB.Create(sharDbBean) zlog.Info(innerLogName, "分表完成，归档表:", archiveName) } From a84b1c762d3718792f9fd2970021070434236a24 Mon Sep 17 00:00:00 2001 From: samwaf Date: Fri, 18 Sep 2026 15:10:37 +0800 Subject: [PATCH 07/21] feat: scope log database export to a time range and selected tiers --- api/waf_log.go | 51 ++++++++++++++++--- wafdb/log_export.go | 121 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 wafdb/log_export.go diff --git a/api/waf_log.go b/api/waf_log.go index 99b306c9..5d170956 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -90,8 +90,8 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) { } } func (w *WafLogAPi) ExportDBApi(c *gin.Context) { - // 该导出走 BackupDatabase 备份 .db 文件，仅文件型数据库(SQLite)支持； - // MySQL 等无日志文件，直接屏蔽，避免进入后台 goroutine 后才失败。 + // 导出物是「按时间段导出选定层」的新加密 SQLite 文件（见 wafdb.ExportLogRangeDb）， + // 仅文件型数据库(SQLite)支持；MySQL 等无日志文件，直接屏蔽，避免进入后台 goroutine 后才失败。 if !dialect.Get().SupportsBackup() { response.FailWithMessage("当前数据库不支持日志文件导出（仅 SQLite 支持）", c) return @@ -125,6 +125,38 @@ func (w *WafLogAPi) ExportDBApi(c *gin.Context) { return } + // 导出物重定义（C11）：不再是备份整个日志库文件，而是「按时间段导出选定层」。 + // start_time/end_time 形如 2006-01-02 15:04:05，留空不限；tiers 逗号分隔（access,event,payload,weblog），默认全选。 + startTime := strings.TrimSpace(c.Query("start_time")) + endTime := strings.TrimSpace(c.Query("end_time")) + for _, tm := range []string{startTime, endTime} { + if tm == "" { + continue + } + if _, err := time.ParseInLocation("2006-01-02 15:04:05", tm, time.Local); err != nil { + response.FailWithMessage("时间格式不正确（应为 2006-01-02 15:04:05）", c) + return + } + } + tiers := map[string]bool{} + tierParam := strings.TrimSpace(c.Query("tiers")) + if tierParam == "" { + tiers[wafdb.ExportTierAccess] = true + tiers[wafdb.ExportTierEvent] = true + tiers[wafdb.ExportTierPayload] = true + tiers[wafdb.ExportTierWeblog] = true + } else { + for _, t := range strings.Split(tierParam, ",") { + switch strings.TrimSpace(t) { + case wafdb.ExportTierAccess, wafdb.ExportTierEvent, wafdb.ExportTierPayload, wafdb.ExportTierWeblog: + tiers[strings.TrimSpace(t)] = true + default: + response.FailWithMessage("未知的导出层: "+t, c) + return + } + } + } + go func() { currentDir := utils.GetCurrentDir() downLoadDir := currentDir + "/download" @@ -140,22 +172,25 @@ func (w *WafLogAPi) ExportDBApi(c *gin.Context) { utils.DeleteOldFiles(downLoadDir, duration) // 创建下载文件 - downloadFileName := fmt.Sprintf("local_log_backup_%s.db", time.Now().Format("20060102150405")) + downloadFileName := fmt.Sprintf("local_log_export_%s.db", time.Now().Format("20060102150405")) downloadFilePath := filepath.Join(downLoadDir, downloadFileName) - err := wafdb.BackupDatabase(global.GWAF_LOCAL_LOG_DB, downloadFilePath) + counts, err := wafdb.ExportLogRangeDb(downloadFilePath, startTime, endTime, tiers) if err != nil { + _ = os.Remove(downloadFilePath) global.GQEQUE_MESSAGE_DB.Enqueue(innerbean.OpResultMessageInfo{ BaseMessageInfo: innerbean.BaseMessageInfo{OperaType: "DOWNLOAD_LOG", Server: global.GWAF_CUSTOM_SERVER_NAME}, - Msg: "导出失败", - Success: "true", + Msg: "导出失败: " + err.Error(), + Success: "false", }) } else { global.GWAF_RUNTIME_CURRENT_EXPORT_DB_LOG_FILE_PATH = downloadFilePath //发送websocket 推送消息 global.GQEQUE_MESSAGE_DB.Enqueue(innerbean.ExportResultMessageInfo{ BaseMessageInfo: innerbean.BaseMessageInfo{OperaType: "DOWNLOAD_LOG", Server: global.GWAF_CUSTOM_SERVER_NAME}, - Msg: "导出完毕", - Success: "true", + Msg: fmt.Sprintf("导出完毕（访问日志%d条/安全事件%d条/报文%d条/旧版日志%d条）", + counts[wafdb.ExportTierAccess], counts[wafdb.ExportTierEvent], + counts[wafdb.ExportTierPayload], counts[wafdb.ExportTierWeblog]), + Success: "true", }) } }() diff --git a/wafdb/log_export.go b/wafdb/log_export.go new file mode 100644 index 00000000..6cc896e5 --- /dev/null +++ b/wafdb/log_export.go @@ -0,0 +1,121 @@ +package wafdb + +import ( + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/wafdb/dialect" + "fmt" + "net/url" + "os" + + sqlite "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" +) + +// 日志导出的可选层 +const ( + ExportTierAccess = "access" // access_log 窄行 + ExportTierEvent = "event" // security_event 安全事件 + ExportTierPayload = "payload" // event_payload 报文（event/sample/watch） + ExportTierWeblog = "weblog" // 存量 web_logs（只读到期删的那部分） +) + +const exportBatchSize = 1000 + +// exportTierCopy 一个待导出的层：模型（导出文件建表用）+ 源表名 +type exportTierCopy struct { + name string + model interface{} + srcTable string +} + +// ExportLogRangeDb 把实时日志库按时间段导出成一个新的加密 SQLite 文件（C11）。 +// +// 分层后「导出一个 .db」不再是备份整个库文件：按选定层 + 时间段导出行， +// 导出件自带表结构（AutoMigrate），可直接用同一套密钥打开查阅。 +// startTime/endTime 为 "2006-01-02 15:04:05"，空 = 不限制。返回每层导出的行数。 +func ExportLogRangeDb(outPath, startTime, endTime string, tiers map[string]bool) (map[string]int64, error) { + counts := map[string]int64{} + if !dialect.Get().SupportsBackup() { + return counts, fmt.Errorf("按时间段导出仅在 SQLite 模式下可用，当前驱动: %s", dialect.Get().Name()) + } + if _, err := os.Stat(outPath); err == nil { + return counts, fmt.Errorf("导出文件已存在: %s", outPath) + } + + key := url.QueryEscape(global.GWAF_PWD_LOGDB) + dst, err := gorm.Open(sqlite.Open(fmt.Sprintf("%s?_db_key=%s", outPath, key)), + &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + return counts, fmt.Errorf("创建导出文件失败: %w", err) + } + defer func() { + if sqlDB, err := dst.DB(); err == nil { + sqlDB.Close() + } + }() + + src := global.GWAF_LOCAL_LOG_DB + if src == nil { + return counts, fmt.Errorf("日志库未初始化") + } + + // 各层：模型 + 源表存在性。web_logs 边界切割后可能已不在实时库 + copies := []exportTierCopy{} + if tiers[ExportTierAccess] && dialect.Get().TableExists(src, model.AccessLogTableName) { + copies = append(copies, exportTierCopy{ExportTierAccess, &model.AccessLog{}, model.AccessLogTableName}) + } + if tiers[ExportTierEvent] && dialect.Get().TableExists(src, model.SecurityEventTableName) { + copies = append(copies, exportTierCopy{ExportTierEvent, &model.SecurityEvent{}, model.SecurityEventTableName}) + } + if tiers[ExportTierPayload] && dialect.Get().TableExists(src, model.EventPayload{}.TableName()) { + copies = append(copies, exportTierCopy{ExportTierPayload, &model.EventPayload{}, model.EventPayloadTableName}) + } + if tiers[ExportTierWeblog] && dialect.Get().TableExists(src, LogTableName) { + copies = append(copies, exportTierCopy{ExportTierWeblog, &innerbean.WebLog{}, LogTableName}) + } + + for _, tc := range copies { + if err := dst.AutoMigrate(tc.model); err != nil { + return counts, fmt.Errorf("导出文件建表 %s 失败: %w", tc.srcTable, err) + } + n, err := exportCopyTier(src, dst, tc, startTime, endTime) + if err != nil { + return counts, err + } + counts[tc.name] = n + } + return counts, nil +} + +// exportCopyTier 按 create_time 区间分批读源表、批量写导出文件。主键冲突保留先到的（重复导出同一区间不会翻倍）。 +func exportCopyTier(src, dst *gorm.DB, tc exportTierCopy, startTime, endTime string) (int64, error) { + var total int64 + q := src.Table(tc.srcTable) + if startTime != "" { + q = q.Where("create_time >= ?", startTime) + } + if endTime != "" { + q = q.Where("create_time <= ?", endTime) + } + // 用目标模型接住行，保证列名与导出表一致（web_logs 列比新表多，互不影响） + rows := []map[string]interface{}{} + result := q.FindInBatches(&rows, exportBatchSize, func(tx *gorm.DB, batch int) error { + if len(rows) == 0 { + return nil + } + if err := dst.Table(tc.srcTable).Clauses(clause.OnConflict{DoNothing: true}). + Create(rows).Error; err != nil { + return fmt.Errorf("写入导出文件 %s 失败: %w", tc.srcTable, err) + } + total += int64(len(rows)) + return nil + }) + if result.Error != nil { + return total, fmt.Errorf("读取 %s 失败: %w", tc.srcTable, result.Error) + } + return total, nil +} From c340ee7e9b93536c043eb4c5bfd39d360fa02dc3 Mon Sep 17 00:00:00 2001 From: samwaf Date: Fri, 18 Sep 2026 15:16:59 +0800 Subject: [PATCH 08/21] feat: add an IP watchlist that full-captures watched IPs --- api/entrance.go | 1 + api/waf_ip_watchlist.go | 56 +++++++++++++ model/request/waf_ip_watchlist_req.go | 21 +++++ router/waf_log.go | 3 + service/waf_service/waf_ip_watchlist.go | 105 ++++++++++++++++++++++++ wafdb/migrations_core.go | 15 ++++ 6 files changed, 201 insertions(+) create mode 100644 api/waf_ip_watchlist.go create mode 100644 model/request/waf_ip_watchlist_req.go create mode 100644 service/waf_service/waf_ip_watchlist.go diff --git a/api/entrance.go b/api/entrance.go index 165b8f63..5ef2cf70 100644 --- a/api/entrance.go +++ b/api/entrance.go @@ -88,6 +88,7 @@ var APIGroupAPP = new(APIGroup) var ( wafHostService = waf_service.WafHostServiceApp wafLogService = waf_service.WafLogServiceApp + wafIPWatchlistService = waf_service.WafIPWatchlistServiceApp wafStatService = waf_service.WafStatServiceApp wafRuleService = waf_service.WafRuleServiceApp wafIpAllowService = waf_service.WafWhiteIpServiceApp diff --git a/api/waf_ip_watchlist.go b/api/waf_ip_watchlist.go new file mode 100644 index 00000000..55047b3a --- /dev/null +++ b/api/waf_ip_watchlist.go @@ -0,0 +1,56 @@ +package api + +import ( + "SamWaf/model/common/response" + "SamWaf/model/request" + + "github.com/gin-gonic/gin" +) + +// AddIPWatchlistApi 加入重点 IP 观察名单（或续期） +func (w *WafLogAPi) AddIPWatchlistApi(c *gin.Context) { + var req request.WafIPWatchlistAddReq + if err := c.ShouldBindJSON(&req); err != nil { + response.FailWithMessage("参数错误: "+err.Error(), c) + return + } + if err := wafIPWatchlistService.AddApi(req); err != nil { + response.FailWithMessage("加入观察名单失败: "+err.Error(), c) + return + } + response.OkWithMessage("已加入观察名单", c) +} + +// DelIPWatchlistApi 移出观察名单 +func (w *WafLogAPi) DelIPWatchlistApi(c *gin.Context) { + var req request.WafIPWatchlistDelReq + if err := c.ShouldBindJSON(&req); err != nil { + response.FailWithMessage("参数错误: "+err.Error(), c) + return + } + if err := wafIPWatchlistService.DelApi(req.IP); err != nil { + response.FailWithMessage("移除失败: "+err.Error(), c) + return + } + response.OkWithMessage("已移出观察名单", c) +} + +// GetIPWatchlistApi 观察名单分页 +func (w *WafLogAPi) GetIPWatchlistApi(c *gin.Context) { + var req request.WafIPWatchlistSearch + if err := c.ShouldBindJSON(&req); err != nil { + response.FailWithMessage("参数错误: "+err.Error(), c) + return + } + list, total, err := wafIPWatchlistService.ListApi(req) + if err != nil { + response.FailWithMessage("查询失败: "+err.Error(), c) + return + } + response.OkWithData(response.PageResult{ + List: list, + Total: total, + PageIndex: req.PageIndex, + PageSize: req.PageSize, + }, c) +} diff --git a/model/request/waf_ip_watchlist_req.go b/model/request/waf_ip_watchlist_req.go new file mode 100644 index 00000000..559754a6 --- /dev/null +++ b/model/request/waf_ip_watchlist_req.go @@ -0,0 +1,21 @@ +package request + +import "SamWaf/model/common/request" + +// WafIPWatchlistAddReq 加入/续期观察名单 +type WafIPWatchlistAddReq struct { + IP string `json:"ip" form:"ip" binding:"required"` + Days int `json:"days" form:"days"` // 观察天数，默认 7，上限 30 + Reason string `json:"reason" form:"reason"` // 加入原因（如触发规则名） +} + +// WafIPWatchlistDelReq 移出观察名单 +type WafIPWatchlistDelReq struct { + IP string `json:"ip" form:"ip" binding:"required"` +} + +// WafIPWatchlistSearch 名单分页查询 +type WafIPWatchlistSearch struct { + IP string `json:"ip" form:"ip"` + request.PageInfo +} diff --git a/router/waf_log.go b/router/waf_log.go index 243af71d..828923c8 100644 --- a/router/waf_log.go +++ b/router/waf_log.go @@ -23,4 +23,7 @@ func (receiver *LogRouter) InitLogRouter(group *gin.RouterGroup) { wafLogRouter.GET("/api/v1/waflog/attack/alliptag", logApi.GetAllIpTagApi) wafLogRouter.POST("/api/v1/waflog/attack/deletetagbyname", logApi.DeleteTagByNameApi) wafLogRouter.GET("/api/v1/waflog/attack/iptagdbstatus", logApi.GetIPTagDBStatusApi) + wafLogRouter.POST("/api/v1/waflog/attack/watchlist/add", logApi.AddIPWatchlistApi) + wafLogRouter.POST("/api/v1/waflog/attack/watchlist/del", logApi.DelIPWatchlistApi) + wafLogRouter.POST("/api/v1/waflog/attack/watchlist/list", logApi.GetIPWatchlistApi) } diff --git a/service/waf_service/waf_ip_watchlist.go b/service/waf_service/waf_ip_watchlist.go new file mode 100644 index 00000000..64105ed6 --- /dev/null +++ b/service/waf_service/waf_ip_watchlist.go @@ -0,0 +1,105 @@ +package waf_service + +import ( + "SamWaf/common/uuid" + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/model/request" + "errors" + "net" + "strings" + "time" + + "gorm.io/gorm" +) + +type WafIPWatchlistService struct{} + +var WafIPWatchlistServiceApp = new(WafIPWatchlistService) + +const ( + watchDefaultDays = 7 + watchMaxDays = 30 + watchReasonMaxLen = 255 +) + +// AddApi 加入观察名单；已存在则续期（到期时间取更晚者）并更新原因。 +func (receiver *WafIPWatchlistService) AddApi(req request.WafIPWatchlistAddReq) error { + ip := strings.TrimSpace(req.IP) + if net.ParseIP(ip) == nil { + return errors.New("IP 格式不正确") + } + days := req.Days + if days <= 0 { + days = watchDefaultDays + } + if days > watchMaxDays { + days = watchMaxDays + } + reason := req.Reason + if len(reason) > watchReasonMaxLen { + reason = reason[:watchReasonMaxLen] + } + expireAt := time.Now().AddDate(0, 0, days).Unix() + + var existing model.IPWatchlist + res := global.GWAF_LOCAL_DB.Where("ip = ? and tenant_id = ? and user_code = ?", + ip, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).First(&existing) + if res.Error == nil { + fields := map[string]interface{}{ + "reason": reason, + "update_time": customtype.JsonTime(time.Now()), + } + if existing.ExpireAt < expireAt { + fields["expire_at"] = expireAt + } + err := global.GWAF_LOCAL_DB.Model(&model.IPWatchlist{}).Where("id = ?", existing.Id).Updates(fields).Error + global.GWAF_IP_WATCH.Reload() + return err + } + bean := &model.IPWatchlist{ + BaseOrm: baseorm.BaseOrm{ + Id: uuid.GenUUID(), + USER_CODE: global.GWAF_USER_CODE, + Tenant_ID: global.GWAF_TENANT_ID, + CREATE_TIME: customtype.JsonTime(time.Now()), + UPDATE_TIME: customtype.JsonTime(time.Now()), + }, + IP: ip, + ExpireAt: expireAt, + Reason: reason, + } + err := global.GWAF_LOCAL_DB.Create(bean).Error + global.GWAF_IP_WATCH.Reload() + return err +} + +// DelApi 移出观察名单 +func (receiver *WafIPWatchlistService) DelApi(ip string) error { + err := global.GWAF_LOCAL_DB.Where("ip = ? and tenant_id = ? and user_code = ?", + strings.TrimSpace(ip), global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Delete(&model.IPWatchlist{}).Error + global.GWAF_IP_WATCH.Reload() + return err +} + +// ListApi 名单分页（顺带清掉已到期行）。 +func (receiver *WafIPWatchlistService) ListApi(req request.WafIPWatchlistSearch) ([]model.IPWatchlist, int64, error) { + var list []model.IPWatchlist + var total int64 + now := time.Now().Unix() + global.GWAF_LOCAL_DB.Where("expire_at <= ?", now).Delete(&model.IPWatchlist{}) + build := func() *gorm.DB { + q := global.GWAF_LOCAL_DB.Model(&model.IPWatchlist{}). + Where("tenant_id = ? and user_code = ? and expire_at > ?", global.GWAF_TENANT_ID, global.GWAF_USER_CODE, now) + if strings.TrimSpace(req.IP) != "" { + q = q.Where("ip like ?", "%"+strings.TrimSpace(req.IP)+"%") + } + return q + } + build().Count(&total) + err := build().Order("expire_at desc"). + Limit(req.PageSize).Offset(req.PageSize * (req.PageIndex - 1)).Find(&list).Error + return list, total, err +} diff --git a/wafdb/migrations_core.go b/wafdb/migrations_core.go index 2a26712a..cc4afda2 100644 --- a/wafdb/migrations_core.go +++ b/wafdb/migrations_core.go @@ -2261,6 +2261,21 @@ func RunCoreDBMigrations(db *gorm.DB) error { return nil }, }, + // 迁移: 创建重点 IP 观察名单表（命中后该 IP 的请求全量留痕） + { + ID: "202609180002_add_ip_watchlist", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609180002: 创建 ip_watchlist 表（重点IP观察名单）") + if err := tx.AutoMigrate(&model.IPWatchlist{}); err != nil { + return fmt.Errorf("创建 ip_watchlist 表失败: %w", err) + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + zlog.Info("回滚 202609180002: 删除 ip_watchlist 表") + return tx.Migrator().DropTable(&model.IPWatchlist{}) + }, + }, }) // 执行迁移 From d37b8e5fdeb698cedb05e36db4a97ad0b1402392 Mon Sep 17 00:00:00 2001 From: samwaf Date: Fri, 18 Sep 2026 15:17:41 +0800 Subject: [PATCH 09/21] feat: split log storage into security events, access rows, and payload tiers --- api/waf_ai_api.go | 56 +++- common/validfield/valid_filter_field.go | 4 +- global/global.go | 5 + global/ip_watch.go | 61 +++++ innerbean/web_log.go | 7 + model/access_log.go | 97 +++++++ model/ip_watchlist.go | 12 + model/request/waf_attack_log_req.go | 1 + .../waf_service/cross_engine_iptag_test.go | 27 +- service/waf_service/cross_engine_log_test.go | 71 +++++- .../waf_service/cross_engine_payload_test.go | 43 +++- service/waf_service/cross_engine_test.go | 2 +- service/waf_service/waf_ai_label_service.go | 20 +- service/waf_service/waf_ai_service.go | 6 +- service/waf_service/waf_analysis.go | 7 +- service/waf_service/waf_cc_threshold.go | 42 ++- service/waf_service/waf_ip_tag_log.go | 99 ++++++-- service/waf_service/waf_ip_tag_merge.go | 39 +++ service/waf_service/waf_log.go | 121 +++++++-- wafdb/log_shard.go | 118 +++++++-- wafdb/migrations_log.go | 47 ++++ wafenginecore/weblog_record.go | 2 +- wafqueue/log_keys.go | 133 ++++++++++ wafqueue/log_keys_test.go | 63 +++++ wafqueue/log_queue.go | 12 +- wafqueue/log_reservoir.go | 117 +++++++++ wafqueue/log_reservoir_test.go | 63 +++++ wafqueue/log_split.go | 98 +------ wafqueue/log_split_test.go | 128 ---------- wafqueue/log_tier.go | 240 ++++++++++++++++++ wafqueue/log_tier_db_test.go | 97 +++++++ wafqueue/log_tier_test.go | 225 ++++++++++++++++ waftask/stat_collector.go | 10 +- waftask/task_config.go | 20 +- waftask/task_db_sharding.go | 229 ++++++++++------- waftask/task_history.go | 4 +- wafupgradenotice/upgrade_notes.yaml | 39 +++ 37 files changed, 1937 insertions(+), 428 deletions(-) create mode 100644 global/ip_watch.go create mode 100644 model/access_log.go create mode 100644 model/ip_watchlist.go create mode 100644 wafqueue/log_keys.go create mode 100644 wafqueue/log_keys_test.go create mode 100644 wafqueue/log_reservoir.go create mode 100644 wafqueue/log_reservoir_test.go delete mode 100644 wafqueue/log_split_test.go create mode 100644 wafqueue/log_tier.go create mode 100644 wafqueue/log_tier_db_test.go create mode 100644 wafqueue/log_tier_test.go diff --git a/api/waf_ai_api.go b/api/waf_ai_api.go index b1349491..e3fb72e5 100644 --- a/api/waf_ai_api.go +++ b/api/waf_ai_api.go @@ -3,11 +3,13 @@ package api import ( "SamWaf/global" "SamWaf/innerbean" + "SamWaf/model" "SamWaf/model/common/response" "SamWaf/model/request" "SamWaf/service/waf_service" "SamWaf/utils" "SamWaf/wafai" + "SamWaf/wafdb/dialect" "encoding/json" "fmt" "io" @@ -19,6 +21,7 @@ import ( "time" "github.com/gin-gonic/gin" + "gorm.io/gorm" ) type WafAIApi struct { @@ -287,21 +290,60 @@ func (w *WafAIApi) ExportTrainDataApi(c *gin.Context) { } // runAIExport 实际执行训练数据导出，返回文件路径与各类计数。 +// +// 分层后的读源（C6）：正样本读 security_event（命中的请求全在这里）， +// 负样本读采样负样本池——event_payload(kind=sample) 里的报文配上 access_log 的窄行。 +// 正常请求不再全量落报文，训练要的真实负样本报文只能来自采样池（D2）。 +// 过渡期补充：采样池还没攒起来时，web_logs 里的存量行（升级前写入、尚在保留期内） +// 照旧能补负样本；它不再写入，随保留期自然退场。 func runAIExport(req request.WafAIExportReq, maxCount int) (outPath string, nAttack, nNormal, nDrop, total int, err error) { - query := global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}). - Select("REQ_UUID", "METHOD", "URL", "RawQuery", "BODY", "POST_FORM", "USER_AGENT", "ACTION", "RULE", "LogOnlyMode") + narrowCols := []string{"REQ_UUID", "METHOD", "URL", "RawQuery", "USER_AGENT", "ACTION", "RULE", "LogOnlyMode"} + cutoff := "" if req.Days > 0 { - cutoff := time.Now().AddDate(0, 0, -req.Days).Format("2006-01-02 15:04:05") - query = query.Where("create_time >= ?", cutoff) + cutoff = time.Now().AddDate(0, 0, -req.Days).Format("2006-01-02 15:04:05") + } + applyWindow := func(q *gorm.DB) *gorm.DB { + if cutoff != "" { + q = q.Where("create_time >= ?", cutoff) + } + return q } var rows []innerbean.WebLog - if err = query.Order("unix_add_time desc").Limit(maxCount).Find(&rows).Error; err != nil { - return "", 0, 0, 0, 0, fmt.Errorf("查询日志失败: %w", err) + + // 正样本：安全事件 + var attackRows []innerbean.WebLog + err = applyWindow(global.GWAF_LOCAL_LOG_DB.Model(&model.SecurityEvent{}).Select(narrowCols)). + Order("unix_add_time desc").Limit(maxCount).Find(&attackRows).Error + if err != nil { + return "", 0, 0, 0, 0, fmt.Errorf("查询安全事件失败: %w", err) + } + rows = append(rows, attackRows...) + + // 负样本：采样池（窄行在 access_log，报文在 event_payload kind=sample，按 req_uuid 对齐） + var normalRows []innerbean.WebLog + err = applyWindow(global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Select(narrowCols). + Where("req_uuid in (select req_uuid from event_payload where kind = 'sample')")). + Order("unix_add_time desc").Limit(maxCount).Find(&normalRows).Error + if err != nil { + return "", 0, 0, 0, 0, fmt.Errorf("查询采样负样本失败: %w", err) + } + rows = append(rows, normalRows...) + + // 过渡期：采样池未攒起来时，从存量 web_logs（不再写入，尚在保留期内的部分）补负样本 + if len(normalRows) < maxCount && dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, "web_logs") { + var legacy []innerbean.WebLog + err = applyWindow(global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}). + Select(append(append([]string{}, narrowCols...), "BODY", "POST_FORM")). + Where("ACTION = ? and RULE = ?", "放行", "")). + Order("unix_add_time desc").Limit(maxCount - len(normalRows)).Find(&legacy).Error + if err == nil { + rows = append(rows, legacy...) + } } total = len(rows) - // BODY/POST_FORM 已搬到 event_payload，批量补回来再导出，否则样本只剩 URL + // BODY/POST_FORM 在 event_payload 里，批量补回来再导出，否则样本只剩 URL fillRows := make([]*innerbean.WebLog, 0, len(rows)) for i := range rows { fillRows = append(fillRows, &rows[i]) diff --git a/common/validfield/valid_filter_field.go b/common/validfield/valid_filter_field.go index 53958fee..7ab65355 100644 --- a/common/validfield/valid_filter_field.go +++ b/common/validfield/valid_filter_field.go @@ -14,7 +14,9 @@ func IsValidHostFilterField(field string) bool { // IsValidWebLogFilterField 检测log字段是否合法 func IsValidWebLogFilterField(field string) bool { - var allowedFilterFields = []string{"header", "guest_identification", "req_uuid"} + // header=请求全文（仅安全事件视图，走报文子查询）；user_agent/referer 是窄行上的列， + // 供访问日志视图替代 header 做筛选 + var allowedFilterFields = []string{"header", "guest_identification", "req_uuid", "user_agent", "referer"} for _, allowedField := range allowedFilterFields { if field == allowedField { diff --git a/global/global.go b/global/global.go index e95e4ca9..2ead5032 100644 --- a/global/global.go +++ b/global/global.go @@ -196,6 +196,11 @@ var ( GDATA_SHARE_DB_SIZE int64 = 100 * 10000 //100w 进行分库 100*10000 GDATA_SHARE_DB_FILE_SIZE int64 = 1024 //1024M 进行分库 GDATA_CURRENT_CHANGE bool = false //当前是否正在切换 + // GDATA_ACCESS_LOG_MODE 访问日志窄行档位：off=只留安全事件，db=全量入库，sample=事件+采样。 + // off 会失去：CC 阈值推荐、AI 训练负样本、异常 IP 的正常行为回溯（全都要正常请求的全量/样本）。 + GDATA_ACCESS_LOG_MODE string = "db" + // GDATA_ACCESS_LOG_RETENTION_DAYS access_log 保留天数。它直接决定 CC 阈值推荐能回看多少天。 + GDATA_ACCESS_LOG_RETENTION_DAYS int64 = 30 // GDATA_IP_TAG_DB IP Tag 存放位置：0 核心库，1 统计库。 // 新装默认统计库——ip_tags 是派生索引，写入量跟着攻击量走，和配置共用核心库会拖慢每一次配置读写。 // 存量用户的取值来自 system_configs 里已有的那一行，不受这个默认值影响； diff --git a/global/ip_watch.go b/global/ip_watch.go new file mode 100644 index 00000000..aa29d790 --- /dev/null +++ b/global/ip_watch.go @@ -0,0 +1,61 @@ +package global + +import ( + "SamWaf/model" + "sync" + "time" +) + +// GWAF_IP_WATCH 重点 IP 观察名单的内存缓存。写入路径每条日志都要问一次，不能逐条查库； +// 名单是人工维护的、条数很小，全量放内存，定期重载兜底，增删后由服务层调 Reload 立即生效。 +var GWAF_IP_WATCH = &ipWatchCache{ips: map[string]int64{}} + +const ipWatchReloadInterval = 30 * time.Second + +type ipWatchCache struct { + mu sync.RWMutex + ips map[string]int64 // ip -> 到期 unix 秒 + loadedAt time.Time +} + +// Has 该 IP 当前是否在观察名单内（未到期）。DB 未初始化时一律视为不在。 +func (c *ipWatchCache) Has(ip string) bool { + if ip == "" { + return false + } + c.reloadIfDue(false) + c.mu.RLock() + exp, ok := c.ips[ip] + c.mu.RUnlock() + return ok && exp > time.Now().Unix() +} + +// Reload 名单增删后立即重载。 +func (c *ipWatchCache) Reload() { + c.reloadIfDue(true) +} + +func (c *ipWatchCache) reloadIfDue(force bool) { + if GWAF_LOCAL_DB == nil { + return + } + c.mu.RLock() + due := time.Since(c.loadedAt) > ipWatchReloadInterval + c.mu.RUnlock() + if !force && !due { + return + } + var rows []model.IPWatchlist + if err := GWAF_LOCAL_DB.Select("ip", "expire_at"). + Where("expire_at > ?", time.Now().Unix()).Find(&rows).Error; err != nil { + return + } + m := make(map[string]int64, len(rows)) + for _, r := range rows { + m[r.IP] = r.ExpireAt + } + c.mu.Lock() + c.ips = m + c.loadedAt = time.Now() + c.mu.Unlock() +} diff --git a/innerbean/web_log.go b/innerbean/web_log.go index 327716cf..a16858fb 100644 --- a/innerbean/web_log.go +++ b/innerbean/web_log.go @@ -123,6 +123,13 @@ func (w *WebLog) IsSafeBot() bool { return w.IsBot == 1 && w.RISK_LEVEL == 0 } +// IsSecurityEvent 判断这条请求算不算「安全事件」——命中了规则、被拦了、或处于仅记录模式。 +// 判定与引擎侧 weblog_record.go 的 abnormal 分支同一条规则，落库分层（security_event / +// access_log 双写窄行）与记录类型开关都靠它，改一处要同步另一处。 +func (w *WebLog) IsSecurityEvent() bool { + return w.ACTION != "放行" || w.RULE != "" || w.LogOnlyMode == 1 +} + // GetIPFailureCount 获取IP在指定时间窗口内的失败次数（用于规则引擎） // minutes: 时间窗口（分钟） // 返回: 失败次数 diff --git a/model/access_log.go b/model/access_log.go new file mode 100644 index 00000000..93c8d536 --- /dev/null +++ b/model/access_log.go @@ -0,0 +1,97 @@ +package model + +// LogNarrow 一条请求的窄行字段集，access_log 与 security_event 共用（gorm 嵌入展开）。 +// +// 窄行的取舍：列表翻页、按 IP/规则/时间检索、统计扫描、bot 分析、CC 阈值推荐要用的列都在； +// 报文列（header/cookies/body/res_body/post_form/res_header 与原始字节）一律不在—— +// 报文只跟「有报文的行」走，按 req_uuid 去 event_payload 点查。 +// url/raw_query 落库前截断到 2KB（wafqueue 侧动手），超出置 Truncated=1。 +// +// 三个分析键（actor_key/ua_hash/path_norm）必须在建表时就位：事后补列等于没有历史数据。 +type LogNarrow struct { + ReqUUID string `gorm:"column:req_uuid;size:64;primaryKey" json:"req_uuid"` + TenantId string `gorm:"size:64" json:"tenant_id"` + UserCode string `gorm:"size:64" json:"user_code"` + HostCode string `gorm:"size:64" json:"host_code"` + Host string `gorm:"size:255" json:"host"` + + URL string `gorm:"size:2048" json:"url"` + RawQuery string `gorm:"size:2048" json:"raw_query"` + Method string `gorm:"size:20" json:"method"` + Scheme string `gorm:"size:20" json:"scheme"` + REFERER string `gorm:"size:1024" json:"referer"` + USER_AGENT string `gorm:"size:500" json:"user_agent"` + + SRC_IP string `gorm:"size:64" json:"src_ip"` + SRC_PORT string `gorm:"size:10" json:"src_port"` + NetSrcIp string `gorm:"size:64" json:"net_src_ip"` // CC 阈值推荐在站点 IPMode=网卡时只认这一列 + COUNTRY string `gorm:"size:100" json:"country"` + PROVINCE string `gorm:"size:100" json:"province"` + CITY string `gorm:"size:100" json:"city"` + + ACTION string `gorm:"size:100" json:"action"` + RULE string `gorm:"type:text" json:"rule"` + STATUS string `gorm:"size:50" json:"status"` + STATUS_CODE int `json:"status_code"` + RISK_LEVEL int `json:"risk_level"` + AI_SCORE float64 `json:"ai_score"` + IsBot int `json:"is_bot"` + LogOnlyMode int `json:"log_only_mode"` + GUEST_IDENTIFICATION string `gorm:"column:guest_id_entification;size:191" json:"guest_identification"` + + TimeSpent int64 `json:"time_spent"` + CONTENT_LENGTH int64 `json:"content_length"` + ResContentLength int64 `json:"res_content_length"` + PreCheckCost int64 `json:"pre_check_cost"` + ForwardCost int64 `json:"forward_cost"` + BackendCheckCost int64 `json:"backend_check_cost"` + + IsBalance int `json:"is_balance"` + BalanceInfo string `gorm:"size:255" json:"balance_info"` + BodyHash string `gorm:"size:100" json:"body_hash"` + + // Truncated：url/raw_query 超 2KB 被截断时置 1（报文截断只看 event_payload 里的同名列）。 + Truncated int `json:"truncated"` + + // 分析键（M5 汇总与 facet 的数据地基，写入时算好）。 + // actor_key 的索引不在 tag 里建：LogNarrow 同时嵌进 access_log 与 security_event， + // 而 SQLite/PG 的索引名全库唯一，同名会撞——改在迁移里按表各建各的（idx_al_actor/idx_se_actor）。 + ActorKey string `gorm:"size:100" json:"actor_key"` // 访客身份优先，其次 IP + UaHash string `gorm:"size:64" json:"ua_hash"` // UA 指纹，供「同一人多少种 UA」 + PathNorm string `gorm:"size:512" json:"path_norm"` // 路径模板：数字段/UUID/hex 归一 + + CREATE_TIME string `gorm:"size:32" json:"create_time"` + UNIX_ADD_TIME int64 `json:"unix_add_time"` + Day int `json:"day"` +} + +// AccessLogTableName 实时访问日志窄行表名。归档分片上是它加分片后缀。 +const AccessLogTableName = "access_log" + +// AccessLog 所有被记录请求的窄行（命中的请求也双写一份，访问日志页因此不需要 UNION）。 +// 写多少由 access_log_mode 档位决定：db=全量，sample=事件+采样，off=仅事件。 +// 保留期短（默认 30 天，access_log_retention_days）。 +type AccessLog struct { + LogNarrow `gorm:"embedded"` +} + +func (AccessLog) TableName() string { + return AccessLogTableName +} + +// SecurityEventTableName 实时安全事件表名。归档分片上是它加分片后缀。 +const SecurityEventTableName = "security_event" + +// SecurityEvent 命中规则/被拦截/仅记录的请求（innerbean.WebLog.IsSecurityEvent 为真）。 +// 报文一对一放 event_payload(kind=event)，按 req_uuid 点查。保留期长（随日志保留天数，默认 180 天）。 +// +// PayloadHash 是本表独有的第四个键：规则 + 归一化命中内容指纹， +// 「同 hash 跨多个 IP」即分布式同一手法（M5 的聚类视图靠它）。 +type SecurityEvent struct { + LogNarrow `gorm:"embedded"` + PayloadHash string `gorm:"size:64" json:"payload_hash"` +} + +func (SecurityEvent) TableName() string { + return SecurityEventTableName +} diff --git a/model/ip_watchlist.go b/model/ip_watchlist.go new file mode 100644 index 00000000..65a0b083 --- /dev/null +++ b/model/ip_watchlist.go @@ -0,0 +1,12 @@ +package model + +import "SamWaf/model/baseorm" + +// IPWatchlist 重点 IP 观察名单：名单内 IP 的请求全量留痕（窄行进 access_log、 +// 报文进 event_payload kind=watch），到期自动失效。表在 core 库。 +type IPWatchlist struct { + baseorm.BaseOrm + IP string `gorm:"size:64;uniqueIndex:uni_watch_ip" json:"ip"` + ExpireAt int64 `gorm:"index" json:"expire_at"` // 到期时间（Unix 秒） + Reason string `gorm:"size:255" json:"reason"` // 加入原因（如触发规则名） +} diff --git a/model/request/waf_attack_log_req.go b/model/request/waf_attack_log_req.go index 2bfb30c0..718b8b40 100644 --- a/model/request/waf_attack_log_req.go +++ b/model/request/waf_attack_log_req.go @@ -16,6 +16,7 @@ type WafAttackLogDoExport struct { } type WafAttackLogSearch struct { CurrrentDbName string `json:"current_db_name"` + ViewType string `json:"view_type" form:"view_type"` //视图：access=访问日志(默认) event=安全事件 HostCode string `json:"host_code" form:"host_code"` //主机码 Rule string `json:"rule" form:"rule"` //规则名 ReqUuid string `json:"req_uuid" form:"req_uuid"` //请求UUID diff --git a/service/waf_service/cross_engine_iptag_test.go b/service/waf_service/cross_engine_iptag_test.go index ca6ed61c..cec7d89f 100644 --- a/service/waf_service/cross_engine_iptag_test.go +++ b/service/waf_service/cross_engine_iptag_test.go @@ -29,6 +29,18 @@ func seedIPTag(t *testing.T, db *gorm.DB, ip, tag string, cnt int64) { }).Error) } +// seedIPDay 种一条 stats_ip_days 放行/阻止计数（D7 后放行数量的真实来源） +func seedIPDay(t *testing.T, db *gorm.DB, ip, typ string, cnt int) { + t.Helper() + must(t, db.Create(&model.StatsIPDay{ + BaseOrm: newBase(uuid.GenUUID()), + IP: ip, + Type: typ, + Count: cnt, + Day: 20260918, + }).Error) +} + func findAllTag(list []model.AllIPTag, value string) (model.AllIPTag, bool) { for _, item := range list { if item.Value == value { @@ -47,7 +59,7 @@ func findAttackIP(list []model.AttackIPTag, ip string) (model.AttackIPTag, bool) return model.AttackIPTag{}, false } -func runIPTagCases(t *testing.T, coredb *gorm.DB) { +func runIPTagCases(t *testing.T, coredb *gorm.DB, statsdb *gorm.DB) { svc := WafLogService{} oldExclude := global.GCONFIG_ATTACK_TAG_EXCLUDE // 本用例直接往核心库塞数据，读取走 GetIPTagDB()，所以归属必须钉在核心库。 @@ -59,16 +71,21 @@ func runIPTagCases(t *testing.T, coredb *gorm.DB) { global.GDATA_IP_TAG_DB = oldTagDB }() - // 干净起步：本用例独占 ip_tags + // 干净起步：本用例独占 ip_tags 与 stats_ip_days must(t, coredb.Exec("DELETE FROM ip_tags").Error) + must(t, statsdb.Exec("DELETE FROM stats_ip_days").Error) - // 1.1.1.1 真攻击；2.2.2.2 只做过 ACME 校验；3.3.3.3 只有历史遗留的静态访问标签 - seedIPTag(t, coredb, "1.1.1.1", "正常", 10) + // 1.1.1.1 真攻击；2.2.2.2 只做过 ACME 校验；3.3.3.3 只有历史遗留的静态访问标签。 + // 「正常」标签不再产生（D7）：放行数量改由 stats_ip_days 的按天放行计数提供， + // 这里种两条验证跨库取数（ip_tags 在 core、stats_ip_days 在 stats，不能 join）。 seedIPTag(t, coredb, "1.1.1.1", "SQL注入", 3) - seedIPTag(t, coredb, "2.2.2.2", "正常", 5) seedIPTag(t, coredb, "2.2.2.2", "ACME证书校验", 7) seedIPTag(t, coredb, "3.3.3.3", "静态文件访问成功", 100) seedIPTag(t, coredb, "4.4.4.4", "XSS跨站注入", 2) + seedIPDay(t, statsdb, "1.1.1.1", "放行", 6) + seedIPDay(t, statsdb, "1.1.1.1", "放行", 4) // 同 IP 多天合计 = 10 + seedIPDay(t, statsdb, "1.1.1.1", "阻止", 3) + seedIPDay(t, statsdb, "2.2.2.2", "放行", 5) global.GCONFIG_ATTACK_TAG_EXCLUDE = "ACME证书校验,静态文件访问成功" diff --git a/service/waf_service/cross_engine_log_test.go b/service/waf_service/cross_engine_log_test.go index 10d93262..badfe152 100644 --- a/service/waf_service/cross_engine_log_test.go +++ b/service/waf_service/cross_engine_log_test.go @@ -47,18 +47,79 @@ func runLogCases(t *testing.T, logdb *gorm.DB) { t.Fatalf("WebLog TASK_FLAG 期望默认 -1，实际 %d", got.TASK_FLAG) } - // 列表查询：时间范围 + req_uuid 过滤（内部含 ForceIndexClause 方言分支） - list, total, err := WafLogServiceApp.GetListApi(req.WafAttackLogSearch{ + // 分层后的列表查询：访问日志视图读 access_log、安全事件视图读 security_event， + // 「请求」全文筛选只在事件视图可用（走报文子查询），访问视图应明确拒绝 + nowMs := time.Now().UnixNano() / 1e6 + nowStr := time.Now().Format("2006-01-02 15:04:05") + must(t, logdb.Create(&model.AccessLog{LogNarrow: model.LogNarrow{ + ReqUUID: uid, HostCode: "h1", URL: "/x", Method: "GET", SRC_IP: "1.2.3.4", + ACTION: "deny", RULE: "testrule", USER_AGENT: "crossdb-probe", UserCode: xtestUser, TenantId: xtestTenant, + UNIX_ADD_TIME: nowMs, CREATE_TIME: nowStr, Day: 20260918, + }}).Error) + must(t, logdb.Create(&model.SecurityEvent{LogNarrow: model.LogNarrow{ + ReqUUID: uid, HostCode: "h1", URL: "/x", Method: "GET", SRC_IP: "1.2.3.4", + ACTION: "deny", RULE: "testrule", USER_AGENT: "crossdb-probe", UserCode: xtestUser, TenantId: xtestTenant, + UNIX_ADD_TIME: nowMs, CREATE_TIME: nowStr, Day: 20260918, + }}).Error) + must(t, logdb.Create(&model.EventPayload{ + ReqUUID: uid, TenantId: xtestTenant, UserCode: xtestUser, HostCode: "h1", + Kind: "event", HEADER: "User-Agent: crossdb-probe", + CreateTime: nowStr, UnixAddTime: nowMs, Day: 20260918, + }).Error) + + base := req.WafAttackLogSearch{ ReqUuid: uid, UnixAddTimeBegin: "0", - UnixAddTimeEnd: fmt.Sprintf("%d", now+3600), + UnixAddTimeEnd: fmt.Sprintf("%d", nowMs+3600000), SortBy: "unix_add_time", SortDescending: "desc", PageInfo: request.PageInfo{PageIndex: 1, PageSize: 20}, - }) + } + + // 访问日志视图（默认）：按 req_uuid 查到窄行 + list, total, err := WafLogServiceApp.GetListApi(base) fatalIf(t, err) if total < 1 || len(list) < 1 { - t.Fatalf("WebLog 列表未查到刚写入的日志: total=%d len=%d", total, len(list)) + t.Fatalf("访问日志视图未查到刚写入的窄行: total=%d len=%d", total, len(list)) + } + + // 安全事件视图：同一 req_uuid 也查得到（事件双写），且报文被补回 + evReq := base + evReq.ViewType = "event" + evList, evTotal, err := WafLogServiceApp.GetListApi(evReq) + fatalIf(t, err) + if evTotal < 1 || len(evList) < 1 { + t.Fatalf("安全事件视图未查到刚写入的事件: total=%d len=%d", evTotal, len(evList)) + } + if evList[0].HEADER != "User-Agent: crossdb-probe" { + t.Fatalf("事件视图应按页补回报文，实际 header=%q", evList[0].HEADER) + } + + // 「请求」全文筛选：事件视图走报文子查询命中，访问视图明确拒绝 + hdrReq := base + hdrReq.ViewType = "event" + hdrReq.FilterBy = "header" + hdrReq.FilterValue = "crossdb-probe" + _, hdrTotal, err := WafLogServiceApp.GetListApi(hdrReq) + fatalIf(t, err) + if hdrTotal < 1 { + t.Fatal("事件视图的 header 全文筛选应命中报文子查询") + } + accHdr := base + accHdr.FilterBy = "header" + accHdr.FilterValue = "crossdb-probe" + if _, _, err = WafLogServiceApp.GetListApi(accHdr); err == nil { + t.Fatal("访问日志视图的 header 全文筛选应被拒绝（窄行没有这一列）") + } + + // UA 筛选两个视图都可用（窄行列），访问视图应命中刚写的窄行 + uaReq := base + uaReq.FilterBy = "user_agent" + uaReq.FilterValue = "crossdb-probe" + _, uaTotal, err := WafLogServiceApp.GetListApi(uaReq) + fatalIf(t, err) + if uaTotal < 1 { + t.Fatal("访问视图的 UA 筛选应命中窄行") } }) diff --git a/service/waf_service/cross_engine_payload_test.go b/service/waf_service/cross_engine_payload_test.go index ebad30f7..3b624b51 100644 --- a/service/waf_service/cross_engine_payload_test.go +++ b/service/waf_service/cross_engine_payload_test.go @@ -99,11 +99,52 @@ func runPayloadCases(t *testing.T, logdb *gorm.DB) { ReqUUID: uid, TenantId: xtestTenant, UserCode: xtestUser, Kind: "event", BODY: "expired", CreateTime: old, }).Error) + // 分层行：安全事件随长保留期删；访问窄行按 accessDay 删（更短）； + // 采样报文 30 天、观察名单报文 7 天 + must(t, logdb.Create(&model.SecurityEvent{LogNarrow: model.LogNarrow{ + ReqUUID: uid + "_se", UserCode: xtestUser, TenantId: xtestTenant, + UNIX_ADD_TIME: now.AddDate(0, 0, -30).Unix(), CREATE_TIME: old, + }}).Error) + must(t, logdb.Create(&model.AccessLog{LogNarrow: model.LogNarrow{ + ReqUUID: uid + "_al", UserCode: xtestUser, TenantId: xtestTenant, + UNIX_ADD_TIME: now.AddDate(0, 0, -8).Unix(), CREATE_TIME: now.AddDate(0, 0, -8).Format("2006-01-02 15:04:05"), + }}).Error) + must(t, logdb.Create(&model.EventPayload{ + ReqUUID: uid + "_watch", TenantId: xtestTenant, UserCode: xtestUser, + Kind: "watch", BODY: "expired-watch", CreateTime: now.AddDate(0, 0, -8).Format("2006-01-02 15:04:05"), + }).Error) + // 未到期对照组：2 天前的事件报文与访问行都应留下 + must(t, logdb.Create(&model.EventPayload{ + ReqUUID: uid + "_fresh", TenantId: xtestTenant, UserCode: xtestUser, + Kind: "event", BODY: "fresh", CreateTime: now.AddDate(0, 0, -2).Format("2006-01-02 15:04:05"), + }).Error) + must(t, logdb.Create(&model.AccessLog{LogNarrow: model.LogNarrow{ + ReqUUID: uid + "_al2", UserCode: xtestUser, TenantId: xtestTenant, + UNIX_ADD_TIME: now.AddDate(0, 0, -2).Unix(), CREATE_TIME: now.AddDate(0, 0, -2).Format("2006-01-02 15:04:05"), + }}).Error) - WafLogServiceApp.DeleteHistory(now.AddDate(0, 0, -1).Format("2006-01-02 15:04")) + // securityDay=4 天前（30 天前的删掉、2 天前的留下）、accessDay=7 天前 + WafLogServiceApp.DeleteHistory( + now.AddDate(0, 0, -4).Format("2006-01-02 15:04"), + now.AddDate(0, 0, -7).Format("2006-01-02 15:04")) if n := countBy(t, logdb, &model.EventPayload{}, "req_uuid = ?", uid); n != 0 { t.Fatalf("过期报文没被清掉，还剩 %d 行", n) } + if n := countBy(t, logdb, &model.SecurityEvent{}, "req_uuid = ?", uid+"_se"); n != 0 { + t.Fatalf("过期安全事件没被清掉，还剩 %d 行", n) + } + if n := countBy(t, logdb, &model.AccessLog{}, "req_uuid = ?", uid+"_al"); n != 0 { + t.Fatalf("超过访问日志保留期的窄行没被清掉，还剩 %d 行", n) + } + if n := countBy(t, logdb, &model.EventPayload{}, "req_uuid = ?", uid+"_watch"); n != 0 { + t.Fatalf("超过 7 天的观察名单报文没被清掉，还剩 %d 行", n) + } + if n := countBy(t, logdb, &model.EventPayload{}, "req_uuid = ?", uid+"_fresh"); n != 1 { + t.Fatalf("未到期报文被误删了") + } + if n := countBy(t, logdb, &model.AccessLog{}, "req_uuid = ?", uid+"_al2"); n != 1 { + t.Fatalf("未到期访问行被误删了") + } }) } diff --git a/service/waf_service/cross_engine_test.go b/service/waf_service/cross_engine_test.go index 09709a3a..e6336a66 100644 --- a/service/waf_service/cross_engine_test.go +++ b/service/waf_service/cross_engine_test.go @@ -283,7 +283,7 @@ func TestCrossEngine(t *testing.T) { t.Run("stats", func(t *testing.T) { runStatsCases(t, x.stats) }) // —— ip_tags 排除逻辑（见 cross_engine_iptag_test.go）—— - t.Run("iptag", func(t *testing.T) { runIPTagCases(t, x.core) }) + t.Run("iptag", func(t *testing.T) { runIPTagCases(t, x.core, x.stats) }) // —— 归档分片缺列（见 cross_engine_shard_column_test.go）—— t.Run("shardcolumn", func(t *testing.T) { runShardColumnCases(t, x.logdb) }) diff --git a/service/waf_service/waf_ai_label_service.go b/service/waf_service/waf_ai_label_service.go index d8e09059..df03f211 100644 --- a/service/waf_service/waf_ai_label_service.go +++ b/service/waf_service/waf_ai_label_service.go @@ -38,12 +38,19 @@ func (receiver *WafAILabelService) MarkApi(req request.WafAILabelMarkReq) error return errors.New("非法的标记类型") } - // 读取原始日志做快照（标记时日志通常仍存在） + // 读取原始日志做快照（标记时日志通常仍存在）。AI 命中的请求必是安全事件，先查 security_event； + // 升级前的存量行还在 web_logs 里，查不到再回落一次。 var wl innerbean.WebLog - global.GWAF_LOCAL_LOG_DB. - Select("METHOD", "URL", "RawQuery", "BODY", "POST_FORM", "USER_AGENT", "ACTION", "RULE", "SRC_IP", "HOST_CODE", "LogOnlyMode"). + narrowCols := []string{"METHOD", "URL", "RawQuery", "USER_AGENT", "ACTION", "RULE", "SRC_IP", "HOST_CODE", "LogOnlyMode"} + res := global.GWAF_LOCAL_LOG_DB.Model(&model.SecurityEvent{}). + Select(narrowCols). Where("REQ_UUID = ?", req.ReqUuid).Limit(1).Find(&wl) - // BODY/POST_FORM 已搬到 event_payload，补回来才有东西做快照 + if res.RowsAffected == 0 { + global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}). + Select(append(append([]string{}, narrowCols...), "BODY", "POST_FORM")). + Where("REQ_UUID = ?", req.ReqUuid).Limit(1).Find(&wl) + } + // BODY/POST_FORM 在 event_payload 里，补回来才有东西做快照 wl.REQ_UUID = req.ReqUuid FillLivePayloads([]*innerbean.WebLog{&wl}) @@ -222,7 +229,7 @@ func (receiver *WafAILabelService) ListApi(req request.WafAILabelListReq) respon } buildQ := func() *gorm.DB { - q := global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where("ai_score > 0") + q := global.GWAF_LOCAL_LOG_DB.Model(&model.SecurityEvent{}).Where("ai_score > 0") if req.StartDay > 0 && req.EndDay > 0 { q = q.Where("day between ? and ?", req.StartDay, req.EndDay) } @@ -245,8 +252,9 @@ func (receiver *WafAILabelService) ListApi(req request.WafAILabelListReq) respon buildQ().Count(&res.Total) var rows []innerbean.WebLog + // BODY/POST_FORM 不在窄行上（security_event 没有这两列），整页一次从 event_payload 补回 buildQ().Select("REQ_UUID", "CREATE_TIME", "HOST_CODE", "SRC_IP", "METHOD", "URL", - "RawQuery", "BODY", "POST_FORM", "USER_AGENT", "AI_SCORE", "RULE", "LogOnlyMode"). + "RawQuery", "USER_AGENT", "AI_SCORE", "RULE", "LogOnlyMode"). Order("ai_score desc").Order("unix_add_time desc"). Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&rows) diff --git a/service/waf_service/waf_ai_service.go b/service/waf_service/waf_ai_service.go index a80a5363..85e925cb 100644 --- a/service/waf_service/waf_ai_service.go +++ b/service/waf_service/waf_ai_service.go @@ -2,7 +2,7 @@ package waf_service import ( "SamWaf/global" - "SamWaf/innerbean" + "SamWaf/model" "SamWaf/model/request" response2 "SamWaf/model/response" "fmt" @@ -15,7 +15,7 @@ type WafAIService struct{} var WafAIServiceApp = new(WafAIService) // DashboardApi 聚合 AI 检测看板数据：按类别汇总、分数分布、observe/block 趋势。 -// 数据源为 web_logs 中 ai_score>0 的命中子集（相对较小），按 day 范围/站点过滤。 +// 数据源为 security_event 中 ai_score>0 的命中子集（AI 命中的请求必是安全事件），按 day 范围/站点过滤。 func (receiver *WafAIService) DashboardApi(req request.WafAIDashboardReq) response2.WafAIDashboard { var res response2.WafAIDashboard res.Categories = []response2.WafAINameValue{} @@ -28,7 +28,7 @@ func (receiver *WafAIService) DashboardApi(req request.WafAIDashboardReq) respon // 每次查询都用全新的 where 链，避免 GORM 条件被复用污染 base := func() *gorm.DB { - q := global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where("ai_score > 0") + q := global.GWAF_LOCAL_LOG_DB.Model(&model.SecurityEvent{}).Where("ai_score > 0") if req.StartDay > 0 && req.EndDay > 0 { q = q.Where("day between ? and ?", req.StartDay, req.EndDay) } diff --git a/service/waf_service/waf_analysis.go b/service/waf_service/waf_analysis.go index b5370de7..e56b7d42 100644 --- a/service/waf_service/waf_analysis.go +++ b/service/waf_service/waf_analysis.go @@ -2,7 +2,6 @@ package waf_service import ( "SamWaf/global" - "SamWaf/innerbean" "SamWaf/model" "SamWaf/model/request" response2 "SamWaf/model/response" @@ -28,15 +27,17 @@ func (receiver *WafAnalysisService) StatAnalysisDayCountryRangeApi(req request.W } // AnalysisSpiderApi 爬虫分析 +// 数据源是 access_log（分层后所有请求的窄行都在这里）；今天它也只看实时库， +// 与改造前只读 live web_logs 的口径一致。 func (receiver *WafAnalysisService) AnalysisSpiderApi(req request.WafAnalysisSpiderReq) []response2.WafAnalysisSpiderResp { var CountOfRange []response2.WafAnalysisSpiderResp if req.Host == "" { - global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where("day between ? and ? and is_bot=1 ", + global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Where("day between ? and ? and is_bot=1 ", req.StartDay, req.EndDay).Select(" guest_id_entification as Name ,count(1) as Value"). Group("guest_id_entification").Order("count(1) desc").Scan(&CountOfRange) } else { - global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where("day between ? and ? and is_bot=1 and host_code = ? ", + global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Where("day between ? and ? and is_bot=1 and host_code = ? ", req.StartDay, req.EndDay, req.Host). Select(" guest_id_entification as Name ,count(1) as Value"). Group("guest_id_entification").Order("count(1) desc").Scan(&CountOfRange) diff --git a/service/waf_service/waf_cc_threshold.go b/service/waf_service/waf_cc_threshold.go index 85ffa0ab..4c045afb 100644 --- a/service/waf_service/waf_cc_threshold.go +++ b/service/waf_service/waf_cc_threshold.go @@ -14,6 +14,7 @@ import ( "SamWaf/model/request" response2 "SamWaf/model/response" "SamWaf/wafdb" + "SamWaf/wafdb/dialect" "gorm.io/gorm" ) @@ -82,11 +83,25 @@ func (receiver *WafCCThresholdService) RecommendApi(req request.WafCCThresholdRe if days <= 0 { days = ccThDefaultDays } - if days > ccThMaxDays { - days = ccThMaxDays + // 可选天数上限跟随 access_log 的保留期：选到没有数据的区间只会算出偏小的推荐值 + // （阈值过低 = 误杀，方向危险）。保留期以外即便归档里还有改造前的数据也不去借， + // 那些数据随保留期退场，借了反而把「推荐能看多远」搞成随升级时间漂移。 + maxDays := ccThMaxDays + if global.GDATA_ACCESS_LOG_RETENTION_DAYS < int64(maxDays) { + maxDays = int(global.GDATA_ACCESS_LOG_RETENTION_DAYS) + } + if days > maxDays { + days = maxDays } rep.WindowSec = window + // off 档正常请求不入库，推荐功能失去数据基础——明说原因而不是算个偏小的值出来 + if global.GDATA_ACCESS_LOG_MODE == "off" { + rep.Reason = "访问日志档位为 off（仅安全事件入库），正常请求不留存，无法按历史流量推荐阈值。" + + "需要这个功能请在系统配置把 access_log_mode 调回 db 或 sample。" + return rep + } + // ── 先判定这套口径能不能从日志里可靠还原 ── switch req.StatDim { case "", model.CCStatDimIP, model.CCStatDimIPURI, model.CCStatDimHostTotal: @@ -286,10 +301,16 @@ func ccThURICond(c model.MatchCondition) (string, []interface{}) { // ccThResolveShards 列出需要查的日志分片：活跃表 + 时间范围有交集的归档分片。 // 只查活跃表的话，天数一长就只算到最近一片，推荐值会偏小。 +// 分层后读 access_log（全量窄行，含命中的双写）；改造前切出去的归档没有 access_log， +// 回落读它的 web_logs——两者都是「全量请求」，口径一致。 func ccThResolveShards(days int) []ccThShard { out := []ccThShard{} if global.GWAF_LOCAL_LOG_DB != nil { - out = append(out, ccThShard{db: global.GWAF_LOCAL_LOG_DB, table: wafdb.LogTableName}) + table := model.AccessLogTableName + if !dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, table) { + table = wafdb.LogTableName + } + out = append(out, ccThShard{db: global.GWAF_LOCAL_LOG_DB, table: table}) } from := time.Now().AddDate(0, 0, -days) all, err := WafShareDbServiceApp.GetAllShareDbApi() @@ -303,14 +324,21 @@ func ccThResolveShards(days int) []ccThShard { if time.Time(s.EndTime).Before(from) { continue } - db, table := wafdb.ResolveLogDB(s.FileName) - if db == nil || !ccThTableRe.MatchString(table) { + tier := wafdb.ResolveTierTables(s.FileName) + if tier.DB == nil { + continue + } + table := tier.Access + if table == "" { + table = tier.WebLog + } + if table == "" || !ccThTableRe.MatchString(table) { continue } - if db == global.GWAF_LOCAL_LOG_DB && table == wafdb.LogTableName { + if tier.DB == global.GWAF_LOCAL_LOG_DB && (table == out[0].table) { continue // 分片解析失败被降级回活跃表，别重复统计 } - out = append(out, ccThShard{db: db, table: table}) + out = append(out, ccThShard{db: tier.DB, table: table}) } return out } diff --git a/service/waf_service/waf_ip_tag_log.go b/service/waf_service/waf_ip_tag_log.go index 8d56bb75..03fd2680 100644 --- a/service/waf_service/waf_ip_tag_log.go +++ b/service/waf_service/waf_ip_tag_log.go @@ -70,9 +70,11 @@ func ValidateAttackTagExclude(value string) (string, bool) { return "", true } -// benignTags 返回「不算风险」的标签清单：固定的"正常" + 用户配置的 attack_tag_exclude。 -// 这些标签既不进规则筛选列表，也算放行而不是阻止——例如 ACME 证书校验是正常业务流量， +// benignTags 返回「不算风险」的标签清单：用户配置的 attack_tag_exclude，外加固定的"正常"。 +// 这些标签不进规则筛选列表，也算放行而不是阻止——例如 ACME 证书校验是正常业务流量， // 不排除的话「只做过证书校验」的 IP 会带着阻止数量>0 出现在风险日志里。 +// 分层后「正常」标签不再产生（D7），存量行由启动任务清掉；保留它在名单里是为了 +// 清理跑完之前的过渡期不把旧行当成风险。 func benignTags() []string { tags := []string{"正常"} seen := map[string]bool{"正常": true} @@ -135,13 +137,14 @@ func (receiver *WafLogService) GetAttackIpListApi(req request.WafAttackIpTagSear var results []model.AttackIPTag var total int64 - // 基础查询部分（update_time 落库即本地时间，方言层只负责渲染，不做时区换算） - firstTimeExpr := dialect.Get().FormatLocalTime("MIN(update_time)") - latestTimeExpr := dialect.Get().FormatLocalTime("MAX(update_time)") // 不算风险的标签（正常 + 用户配置的排除项）走参数绑定，SQL 里出现几次就补几份参数 tags := benignTags() notBenign := benignNotCond(len(tags)) - isBenign := benignIsCond(len(tags)) + // 首次/最近时间只看风险行：「正常」标签分层后不再更新，用户排除项（如 ACME 证书校验） + // 也不是风险活动——这个口径是「第一次/最近一次触发规则」，正是风险日志该有的语义。 + // （update_time 落库即本地时间，方言层只负责渲染，不做时区换算） + firstTimeExpr := dialect.Get().FormatLocalTime("MIN(CASE WHEN " + notBenign + " THEN update_time END)") + latestTimeExpr := dialect.Get().FormatLocalTime("MAX(CASE WHEN " + notBenign + " THEN update_time END)") // 聚合去重拼接：SQLite/MySQL 用 GROUP_CONCAT，PostgreSQL 用 string_agg ipTotalTagExpr := dialect.Get().GroupConcatDistinct("CASE WHEN " + notBenign + " THEN ip_tag END") query := ` @@ -149,7 +152,6 @@ func (receiver *WafLogService) GetAttackIpListApi(req request.WafAttackIpTagSear tenant_id, user_code, ip, - SUM(CASE WHEN ` + isBenign + ` THEN cnt ELSE 0 END) AS pass_num, SUM(CASE WHEN ` + notBenign + ` THEN cnt ELSE 0 END) AS deny_num, ` + firstTimeExpr + ` AS first_time, ` + latestTimeExpr + ` AS latest_time, @@ -168,21 +170,22 @@ func (receiver *WafLogService) GetAttackIpListApi(req request.WafAttackIpTagSear // 完成查询的其他部分 query += ` - GROUP BY - tenant_id, - user_code, + GROUP BY + tenant_id, + user_code, ip - HAVING - SUM(CASE WHEN ` + notBenign + ` THEN cnt ELSE 0 END) > 0 - ORDER BY + HAVING + SUM(CASE WHEN ` + notBenign + ` THEN cnt ELSE 0 END) > 0 + ORDER BY MAX(update_time) DESC LIMIT ? OFFSET ?` // 构建查询参数：顺序必须与占位符在 SQL 里出现的先后一致 - // pass_num -> deny_num -> ip_total_tag -> tenant/user -> [rule] -> [ip] -> having -> limit/offset + // deny_num -> first_time -> latest_time -> ip_total_tag -> tenant/user -> [rule] -> [ip] -> having -> limit/offset params := []interface{}{} - params = appendTags(params, tags) // pass_num params = appendTags(params, tags) // deny_num + params = appendTags(params, tags) // first_time + params = appendTags(params, tags) // latest_time params = appendTags(params, tags) // ip_total_tag params = append(params, global.GWAF_TENANT_ID, global.GWAF_USER_CODE) @@ -206,7 +209,7 @@ func (receiver *WafLogService) GetAttackIpListApi(req request.WafAttackIpTagSear } // 获取总记录数：等价于「至少有一条非排除标签且 cnt>0」的 IP 数。 - // 用 COUNT(DISTINCT ip) 而不是 GROUP BY+HAVING 子查询：97 万行实测 572ms -> 207ms， + // 用 COUNT(DISTINCT ip) 而不是 GROUP BY+HAVING 子查询：97 万行实测 NOT IN 572ms -> 207ms， // 因为过滤发生在分组之前，只有少量风险行需要去重（cnt 是计数器不会为负，两者结果一致）。 countQuery := ` SELECT @@ -238,9 +241,50 @@ func (receiver *WafLogService) GetAttackIpListApi(req request.WafAttackIpTagSear return nil, 0, err } + // 放行数量改取 stats_ip_days（D7）：ip_tags 不再记「正常」，而 stats_ip_days 本就按 + // 「每 IP 每天 放行/阻止」计数，是同一份数据不重复的源头。它在统计库，跟 ip_tags + // 可能不在一个库，不能 join——按本页 IP 单独查一回（每页一条查询）。 + fillPassNumFromStats(results) + return results, total, nil } +// fillPassNumFromStats 给本页结果补放行数量（stats_ip_days 的 放行 计数合计）。 +func fillPassNumFromStats(results []model.AttackIPTag) { + if len(results) == 0 || global.GWAF_LOCAL_STATS_DB == nil { + return + } + ips := make([]string, 0, len(results)) + for _, r := range results { + ips = append(ips, r.IP) + } + // SQLite 绑定变量上限 999，一页的 IP 分块查 + passes := map[string]int64{} + for i := 0; i < len(ips); i += 500 { + end := i + 500 + if end > len(ips) { + end = len(ips) + } + var rows []struct { + IP string + Cnt int64 + } + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). + Select("ip, SUM(count) as cnt"). + Where("tenant_id = ? and user_code = ? and type = ? and ip in ?", global.GWAF_TENANT_ID, global.GWAF_USER_CODE, "放行", ips[i:end]). + Group("ip").Scan(&rows).Error; err != nil { + zlog.Warn("放行数量查询失败", "error", err.Error()) + return + } + for _, row := range rows { + passes[row.IP] = row.Cnt + } + } + for i := range results { + results[i].PassNum = passes[results[i].IP] + } +} + // GetAllAttackIPTagListApi 获取所有攻击Tag // withBenign=true 时把被排除的标签（ACME证书校验、历史遗留的静态文件访问成功等）也带出来， // 供批量删除用——否则这些标签一旦被排除，界面上就再也没有入口清理它们残留的数据。 @@ -299,10 +343,29 @@ func (receiver *WafLogService) DeleteTagByNameApi(tagName string, deleteLogs boo // 2. 如果需要删除关联的日志数据 if deleteLogs { + batchSize := 1000 // 每批删除1000条 + // 分层后事件行在 security_event，报文按 req_uuid 跟着删； + // 存量 web_logs（不再写入、尚在保留期内的部分）照旧按老路径清。 + if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.SecurityEventTableName) { + for { + var ids []string + if err := global.GWAF_LOCAL_LOG_DB.Model(&model.SecurityEvent{}). + Where("tenant_id=? AND user_code=? AND rule=?", global.GWAF_TENANT_ID, global.GWAF_USER_CODE, tagName). + Limit(batchSize).Pluck("req_uuid", &ids).Error; err != nil { + return fmt.Errorf("查询关联安全事件失败: %v", err) + } + if len(ids) == 0 { + break + } + global.GWAF_LOCAL_LOG_DB.Where("req_uuid in ?", ids).Delete(&model.EventPayload{}) + if err := global.GWAF_LOCAL_LOG_DB.Where("req_uuid in ?", ids).Delete(&model.SecurityEvent{}).Error; err != nil { + return fmt.Errorf("删除关联安全事件失败: %v", err) + } + time.Sleep(10 * time.Millisecond) + } + } // 使用批量删除，避免内存溢出 // 每次删除一批数据，直到全部删除完成 - batchSize := 1000 // 每批删除1000条 - for { // 分批删除日志。web_logs 无主键，只能走各引擎的物理行标识（rowid/ctid）， // MySQL 两者都没有则退化成 DELETE ... LIMIT —— 统一交给方言层构造。 diff --git a/service/waf_service/waf_ip_tag_merge.go b/service/waf_service/waf_ip_tag_merge.go index 61d6657d..bef91716 100644 --- a/service/waf_service/waf_ip_tag_merge.go +++ b/service/waf_service/waf_ip_tag_merge.go @@ -103,6 +103,45 @@ func MergeIPTagsInto(target int64) { zlog.Info("IP标签合并完成", "来源", srcName, "目标", dstName, "条数", merged, "耗时", time.Since(start).String()) } +// CleanLegacyBenignIPTags 一次性清掉存量「正常」标签行（核心库与统计库都清）。 +// +// 分层后不再为未命中规则的请求生成「正常」标签（D7）——放行数量改由 stats_ip_days 承担， +// 那是同一份数据的重复记录，也是 ip_tags 行数的大头。存量行切换后不再更新但仍占表， +// 这里按批删掉；必须在归属合并（MergeIPTagsInto）之前跑，否则大表用户要先把一堆 +// 马上要删的行白搬一遍。 +func CleanLegacyBenignIPTags() { + for _, db := range []*gorm.DB{global.GWAF_LOCAL_DB, global.GWAF_LOCAL_STATS_DB} { + if db == nil || !db.Migrator().HasTable(&model.IPTag{}) { + continue + } + var total int64 + if err := db.Model(&model.IPTag{}).Where("ip_tag = ?", "正常").Count(&total).Error; err != nil || total == 0 { + continue + } + zlog.Info("清理存量「正常」IP标签", "条数", total) + start := time.Now() + var deleted int64 + for { + // 分批 + 让锁：这张表可能有几百万上千万行，一口气删会把日志落库顶死（SQLite 单写者）。 + // 各引擎的批量删法不同（PG 不认 DELETE...LIMIT），统一走方言层。 + res := db.Exec(dialect.Get().BatchDeleteSQL("ip_tags", "ip_tag=?", ipTagMergeBatch), "正常") + if res.Error != nil { + zlog.Warn("清理「正常」IP标签失败", "已删", deleted, "error", res.Error.Error()) + break + } + if res.RowsAffected == 0 { + break + } + deleted += res.RowsAffected + if deleted%ipTagMergeLogEvery < ipTagMergeBatch { + zlog.Info("清理「正常」IP标签进行中", "已删", deleted, "共", total, "耗时", time.Since(start).String()) + } + time.Sleep(ipTagMergePause) + } + zlog.Info("清理「正常」IP标签完成", "条数", deleted, "耗时", time.Since(start).String()) + } +} + // upsertIPTags 按唯一索引 uni_iptags_full 合并写入：cnt 相加，首次时间取早、最近时间取晚。 // 冲突分支要引用"本次待插入的值"，三个引擎写法不同，由方言给出。 func upsertIPTags(dst *gorm.DB, rows []model.IPTag) error { diff --git a/service/waf_service/waf_log.go b/service/waf_service/waf_log.go index 4ab00943..a914ed7d 100644 --- a/service/waf_service/waf_log.go +++ b/service/waf_service/waf_log.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "sync" + "time" "gorm.io/gorm" "gorm.io/gorm/schema" @@ -124,13 +125,28 @@ func (receiver *WafLogService) ModifyApi(log innerbean.WebLog) error { } func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) (innerbean.WebLog, error) { var weblog innerbean.WebLog - // 解析当前应查询的日志连接与表（live 或历史分片：SQLite 历史文件 / MySQL 历史表） - logDB, logTable := wafdb.ResolveLogDB(req.CurrrentDbName) - sel := webLogSelect(logDB, req.CurrrentDbName, logTable, getWebLogDetailColumns(), "detail") - if err := logDB.Table(logTable).Select(sel).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog).Error; err != nil { - return weblog, fmt.Errorf("查询日志详情失败: %w", err) + // 解析当前分片的三层表：安全事件 → 访问日志 → 存量 web_logs，按 req_uuid 逐层点查。 + // 事件双写了窄行，内容一致，但事件表保留期更长，优先从它读。 + tier := wafdb.ResolveTierTables(req.CurrrentDbName) + found := false + for _, table := range []string{tier.Event, tier.Access, tier.WebLog} { + if table == "" { + continue + } + sel := webLogSelect(tier.DB, req.CurrrentDbName, table, getWebLogDetailColumns(), "detail") + res := tier.DB.Table(table).Select(sel).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog) + if res.Error != nil { + return weblog, fmt.Errorf("查询日志详情失败: %w", res.Error) + } + if res.RowsAffected > 0 { + found = true + break + } + } + if !found { + return weblog, nil } - // 报文单独存在 event_payload 里，按主键点查补回来 + // 报文单独存在 event_payload 里，按主键点查补回来；没有报文行的（正常请求未采样）保持窄字段 FillShardPayloads(req.CurrrentDbName, []*innerbean.WebLog{&weblog}) return weblog, nil } @@ -140,10 +156,25 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn splitFilterBys := strings.Split(req.FilterBy, "|") splitFilterValues := strings.Split(req.FilterValue, "|") - // 解析当前应查询的日志连接与表（live 或历史分片：SQLite 历史文件 / MySQL 历史表） - logDB, logTable := wafdb.ResolveLogDB(req.CurrrentDbName) - /*强制索引*/ - var forceIndex = logTable + // 解析当前分片的三层表：访问日志视图读 access_log，安全事件视图读 security_event； + // 分层改造之前切出去的归档只有 web_logs，回落到它（列交集会自适应它的结构）。 + tier := wafdb.ResolveTierTables(req.CurrrentDbName) + logDB := tier.DB + isEventView := req.ViewType == "event" + logTable := tier.Access + if isEventView { + logTable = tier.Event + } + if logTable == "" { + logTable = tier.WebLog + } + if logTable == "" { + return nil, 0, errors.New("该分片没有可查询的日志表") + } + // 老分片上的安全事件视图：web_logs 里按事件条件过滤（与引擎 abnormal 判定同一条规则） + legacyEventView := isEventView && logTable == tier.WebLog + isLegacyTable := strings.HasPrefix(logTable, wafdb.LogTableName) + /*where条件*/ var whereField = "" var whereValues []interface{} @@ -151,6 +182,9 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn //where字段 { whereField = whereField + " (unix_add_time>=? and unix_add_time<=?)" + if legacyEventView { + whereField = whereField + " and (action<>? or rule<>? or log_only_mode=1)" + } if len(req.HostCode) > 0 { if len(whereField) > 0 { whereField = whereField + " and " @@ -211,16 +245,35 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn if by == "guest_identification" { by = "guest_id_entification" } - whereField = whereField + " " + by + " like ? " + if by == "header" && !isLegacyTable { + // header 随报文搬进 event_payload：访问日志视图没有这一列， + // 安全事件视图走报文子查询（该视图行数小、且必有报文）。 + if !isEventView { + return nil, 0, errors.New("「请求」全文筛选仅在安全事件视图可用，访问日志视图请改用 UA / Referer 筛选") + } + if tier.Payload == "" { + return nil, 0, errors.New("该分片没有报文表，无法按「请求」内容筛选") + } + whereField = whereField + " req_uuid in (select req_uuid from " + tier.Payload + " where header like ?) " + } else { + whereField = whereField + " " + by + " like ? " + } } } } //强制索引 + forceIndex := logTable { + idxTime, idxIP := "idx_web_time_desc_tenant_user_code", "idx_web_time_desc_tenant_user_code_ip" + if strings.HasPrefix(logTable, model.AccessLogTableName) { + idxTime, idxIP = "idx_al_time", "idx_al_ip_time" + } else if strings.HasPrefix(logTable, model.SecurityEventTableName) { + idxTime, idxIP = "idx_se_time", "idx_se_ip_time" + } if strings.Contains(whereField, "unix_add_time") && !strings.Contains(whereField, "src_ip") { - forceIndex = dialect.Get().ForceIndexClause(logTable, "idx_web_time_desc_tenant_user_code") + forceIndex = dialect.Get().ForceIndexClause(logTable, idxTime) } else if strings.Contains(whereField, "src_ip") { - forceIndex = dialect.Get().ForceIndexClause(logTable, "idx_web_time_desc_tenant_user_code_ip") + forceIndex = dialect.Get().ForceIndexClause(logTable, idxIP) } } @@ -241,6 +294,9 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn { whereValues = append(whereValues, unixBegin) whereValues = append(whereValues, unixEnd) + if legacyEventView { + whereValues = append(whereValues, "放行", "") + } if len(req.HostCode) > 0 { whereValues = append(whereValues, req.HostCode) } @@ -294,6 +350,14 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn if err := logDB.Table(forceIndex).Where(whereField, whereValues...).Count(&total).Error; err != nil { return nil, 0, fmt.Errorf("统计日志条数失败: %w", err) } + // 安全事件视图每页补一回报文（事件必有报文）：「请求」列与详情都靠它 + if isEventView && len(weblogs) > 0 { + rows := make([]*innerbean.WebLog, 0, len(weblogs)) + for i := range weblogs { + rows = append(rows, &weblogs[i]) + } + FillShardPayloads(req.CurrrentDbName, rows) + } return weblogs, total, nil } func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { @@ -304,15 +368,36 @@ func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSear global.GWAF_LOCAL_LOG_DB.Where("host_code = ?", log.HostCode).Model(&innerbean.WebLog{}).Count(&total) return weblogs, total, nil } -func (receiver *WafLogService) DeleteHistory(day string) { - global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", day).Delete(&innerbean.WebLog{}) - // 报文表自带同格式的 create_time，按同一条件删，不必回表对 req_uuid - if err := global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", day). +// DeleteHistory 分层保留期清理： +// - security_event 与 web_logs（存量，不再写入）按「日志保留天数」删 +// - access_log 按 access_log_retention_days 删（更短） +// - event_payload 跟属主走：kind=event 随安全事件，kind=sample 只留 30 天， +// kind=watch（观察名单全量留痕）只留 watchPayloadRetentionDays 天 +// +// 三张新表都带同格式的 create_time，按各自条件删，不必回表对 req_uuid。 +func (receiver *WafLogService) DeleteHistory(securityDay, accessDay string) { + global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", securityDay).Delete(&innerbean.WebLog{}) + global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", securityDay).Delete(&model.SecurityEvent{}) + if err := global.GWAF_LOCAL_LOG_DB.Where("create_time < ?", accessDay). + Delete(&model.AccessLog{}).Error; err != nil { + zlog.Warn("清理过期访问日志失败", "截止", accessDay, "error", err.Error()) + } + sampleDay := time.Now().AddDate(0, 0, -sampleRetentionDays).Format("2006-01-02 15:04") + watchDay := time.Now().AddDate(0, 0, -watchPayloadRetentionDays).Format("2006-01-02 15:04") + if err := global.GWAF_LOCAL_LOG_DB. + Where("(kind = ? and create_time < ?) or (kind = ? and create_time < ?) or (kind = ? and create_time < ?) or (kind = '' and create_time < ?)", + "event", securityDay, "sample", sampleDay, "watch", watchDay, securityDay). Delete(&model.EventPayload{}).Error; err != nil { - zlog.Warn("清理过期报文失败", "截止", day, "error", err.Error()) + zlog.Warn("清理过期报文失败", "截止", securityDay, "error", err.Error()) } } +// sampleRetentionDays 采样负样本池的保留天数（D2）。比 access_log 独立：它喂的是 AI 训练。 +const sampleRetentionDays = 30 + +// watchPayloadRetentionDays 观察名单全量留痕报文的保留天数（D9）。窄行仍随 access_log 保留期。 +const watchPayloadRetentionDays = 7 + // GetUnixTimeByCounter 依据开始时间和到期时间获取一个最新的时间戳 func (receiver *WafLogService) GetUnixTimeByCounter(lastStartCreateUnix int64, lastEndCreateUnix int64) innerbean.WebLog { var weblog innerbean.WebLog diff --git a/wafdb/log_shard.go b/wafdb/log_shard.go index da962141..4f28721e 100644 --- a/wafdb/log_shard.go +++ b/wafdb/log_shard.go @@ -67,17 +67,104 @@ func ResolveLogDB(currentDbName string) (*gorm.DB, string) { return global.GWAF_LOCAL_LOG_DB, LogTableName } -// payloadTableSeen 记住哪些分片确认有报文表。只缓存"有"这一侧： -// 归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到，缓存下来会一直读不到报文。 -var payloadTableSeen sync.Map +// shardTableSeen 记住哪些分片确认有某张表。只缓存"有"这一侧： +// 归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到，缓存下来会一直读不到。 +var shardTableSeen sync.Map + +// probeShardTable 探一次表存不存在，存在则返回表名。 +// 只缓存"存在"这一侧：归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到， +// 缓存下来会一直读不到报文。 +func probeShardTable(db *gorm.DB, shard, table string) string { + key := shard + "|" + table + if _, ok := shardTableSeen.Load(key); ok { + return table + } + if !dialect.Get().TableExists(db, table) { + return "" + } + shardTableSeen.Store(key, struct{}{}) + return table +} + +// TierTables 一个分片上三层存储各自的表名；空字符串 = 这个分片没有那一层 +// （分层改造之前切出去的归档只有 WebLog 与 Payload 两层）。 +type TierTables struct { + DB *gorm.DB + Access string // access_log / access_log_ + Event string // security_event / security_event_ + Payload string // event_payload / event_payload_ + WebLog string // web_logs / web_logs_ +} + +// ResolveTierTables 把分片标识（ShareDb.FileName，前端 current_db_name）解析成三层表名。 +// +// 分片标识的三种形态： +// - 空 / local_log.db / web_logs / access_log → 实时库 +// - SQLite 归档文件名（local_log_.db）→ 整个文件，表名不带后缀 +// - MySQL/PG 归档表名：改造前是 web_logs_，改造后是 access_log_； +// 两种都按后缀推同分片的其余表，探不到就当那一层不存在 +// +// 报文表探不到分片自带的时回落实时 event_payload：换表失败时报文就留在那里， +// 按 req_uuid 寻址照样找得到。 +func ResolveTierTables(currentDbName string) TierTables { + // 实时 + if len(currentDbName) == 0 || currentDbName == enums.DB_LOG || + currentDbName == LogTableName || currentDbName == model.AccessLogTableName { + return TierTables{ + DB: global.GWAF_LOCAL_LOG_DB, + Access: model.AccessLogTableName, + Event: model.SecurityEventTableName, + Payload: model.EventPayloadTableName, + WebLog: LogTableName, + } + } + + // SQLite 归档：整个文件，表名不带后缀；老归档文件里可能只有 web_logs（+event_payload） + if dialect.Get().IsFileBased() { + db, _ := ResolveLogDB(currentDbName) // 负责打开文件与降级 + return TierTables{ + DB: db, + Access: probeShardTable(db, currentDbName, model.AccessLogTableName), + Event: probeShardTable(db, currentDbName, model.SecurityEventTableName), + Payload: probeShardTable(db, currentDbName, model.EventPayloadTableName), + WebLog: probeShardTable(db, currentDbName, LogTableName), + } + } + + // MySQL/PG 归档：按表名后缀推同分片的其余表 + suffix := "" + switch { + case strings.HasPrefix(currentDbName, LogTableName+"_"): + suffix = strings.TrimPrefix(currentDbName, LogTableName) + case strings.HasPrefix(currentDbName, model.AccessLogTableName+"_"): + suffix = strings.TrimPrefix(currentDbName, model.AccessLogTableName) + default: + // 不认识的标识（脏数据）：降级实时库，沿用 ResolveLogDB 的老行为 + db, _ := ResolveLogDB(currentDbName) + return TierTables{DB: db, Access: model.AccessLogTableName, Event: model.SecurityEventTableName, Payload: model.EventPayloadTableName, WebLog: LogTableName} + } + db := global.GWAF_LOCAL_LOG_DB + t := TierTables{ + DB: db, + Access: probeShardTable(db, currentDbName, model.AccessLogTableName+suffix), + Event: probeShardTable(db, currentDbName, model.SecurityEventTableName+suffix), + Payload: probeShardTable(db, currentDbName, model.EventPayloadTableName+suffix), + WebLog: probeShardTable(db, currentDbName, LogTableName+suffix), + } + if t.Payload == "" { + t.Payload = probeShardTable(db, currentDbName, model.EventPayloadTableName) + } + return t +} // ResolveLogTables 在 ResolveLogDB 的基础上再给出该分片的报文表名。 +// 读侧正逐步迁到 ResolveTierTables；存量调用保持原语义。 // // 报文表与日志表同进同出： // - SQLite 按文件分片，归档文件里自带 event_payload // - MySQL / PostgreSQL 按表分片，web_logs_ 对应 event_payload_ // -// 返回空表名表示这个分片没有报文表——本次改造之前切出去的归档都是这样， +// 返回空表名表示这个分片没有报文表——垂直拆表之前切出去的归档都是这样， // 它们的报文还在 web_logs 自己的列里，读侧照旧读原列即可。 func ResolveLogTables(currentDbName string) (*gorm.DB, string, string) { db, logTable := ResolveLogDB(currentDbName) @@ -87,30 +174,15 @@ func ResolveLogTables(currentDbName string) (*gorm.DB, string, string) { // SQLite 的归档分片是整个文件，连上去之后表名仍是 web_logs / event_payload if logTable == LogTableName { - return db, logTable, probePayloadTable(db, currentDbName, model.EventPayloadTableName) + return db, logTable, probeShardTable(db, currentDbName, model.EventPayloadTableName) } // MySQL / PostgreSQL：web_logs_ 对应 event_payload_ shardPayload := model.EventPayloadTableName + strings.TrimPrefix(logTable, LogTableName) - if t := probePayloadTable(db, currentDbName, shardPayload); t != "" { + if t := probeShardTable(db, currentDbName, shardPayload); t != "" { return db, logTable, t } - // 分表时报文表没能一起换过去（换表失败或这批日志早于本次改造）： + // 分表时报文表没能一起换过去（换表失败或这批日志早于垂直拆表）： // 报文按 req_uuid 寻址，留在实时表里照样找得到，退回去查它。 - return db, logTable, probePayloadTable(db, currentDbName, model.EventPayloadTableName) -} - -// probePayloadTable 探一次表存不存在，存在则返回表名。 -// 只缓存"存在"这一侧：归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到， -// 缓存下来会一直读不到报文。 -func probePayloadTable(db *gorm.DB, shard, table string) string { - key := shard + "|" + table - if _, ok := payloadTableSeen.Load(key); ok { - return table - } - if !dialect.Get().TableExists(db, table) { - return "" - } - payloadTableSeen.Store(key, struct{}{}) - return table + return db, logTable, probeShardTable(db, currentDbName, model.EventPayloadTableName) } diff --git a/wafdb/migrations_log.go b/wafdb/migrations_log.go index 6f2e22f7..10697fc3 100644 --- a/wafdb/migrations_log.go +++ b/wafdb/migrations_log.go @@ -484,6 +484,53 @@ func RunLogDBMigrations(db *gorm.DB) error { return tx.Migrator().DropTable(&model.EventPayload{}) }, }, + // 分层写入的两张新表：access_log 装所有记录的窄行（安全事件双写一份进去）， + // security_event 只装事件、保留期更长。报文仍在 event_payload，按 req_uuid 一对一。 + // 三个分析键（actor_key/ua_hash/path_norm，事件表再加 payload_hash）随建表就位—— + // 事后补列等于没有历史数据。 + { + ID: "202609180001_add_tiered_log_tables", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609180001: 创建分层日志表 access_log / security_event") + if err := tx.AutoMigrate(&model.AccessLog{}, &model.SecurityEvent{}); err != nil { + return fmt.Errorf("创建分层日志表失败: %w", err) + } + // rule 是 text 列：MySQL 对 text 建索引必须给前缀长度，sqlite/pg 不需要 + ruleCol := "rule" + if tx.Dialector.Name() == "mysql" { + ruleCol = "rule(191)" + } + for _, ix := range []struct{ name, table, ddl string }{ + // 访问日志页默认排序（时间倒序 + 租户）与按 IP 查 + {"idx_al_time", "access_log", "CREATE INDEX IF NOT EXISTS idx_al_time ON access_log (tenant_id, user_code, unix_add_time desc)"}, + {"idx_al_ip_time", "access_log", "CREATE INDEX IF NOT EXISTS idx_al_ip_time ON access_log (tenant_id, user_code, src_ip, unix_add_time desc)"}, + // bot 分析按 day+is_bot 扫 + {"idx_al_day_isbot", "access_log", "CREATE INDEX IF NOT EXISTS idx_al_day_isbot ON access_log (day, is_bot)"}, + // 风险详情按 IP / 规则回查事件；事件视图默认按时间翻页 + {"idx_se_time", "security_event", "CREATE INDEX IF NOT EXISTS idx_se_time ON security_event (tenant_id, user_code, unix_add_time desc)"}, + {"idx_se_ip_time", "security_event", "CREATE INDEX IF NOT EXISTS idx_se_ip_time ON security_event (tenant_id, user_code, src_ip, unix_add_time desc)"}, + {"idx_se_rule_time", "security_event", "CREATE INDEX IF NOT EXISTS idx_se_rule_time ON security_event (tenant_id, user_code, " + ruleCol + ", unix_add_time desc)"}, + // 保留期清理按 create_time 删 + {"idx_al_create_time", "access_log", "CREATE INDEX IF NOT EXISTS idx_al_create_time ON access_log (create_time)"}, + {"idx_se_create_time", "security_event", "CREATE INDEX IF NOT EXISTS idx_se_create_time ON security_event (create_time)"}, + // 行为视角按访客键汇总（索引名全库唯一，两张表各建各的） + {"idx_al_actor", "access_log", "CREATE INDEX IF NOT EXISTS idx_al_actor ON access_log (tenant_id, user_code, actor_key, day)"}, + {"idx_se_actor", "security_event", "CREATE INDEX IF NOT EXISTS idx_se_actor ON security_event (tenant_id, user_code, actor_key, day)"}, + } { + start := time.Now() + if err := safeCreateIndex(tx, ix.table, ix.name, ix.ddl); err != nil { + zlog.Warn("创建索引失败", "index", ix.name, "error", err.Error()) + continue + } + zlog.Info("索引创建完成", "index", ix.name, "耗时", time.Since(start).String()) + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + zlog.Info("回滚 202609180001: 删除分层日志表") + return tx.Migrator().DropTable(&model.AccessLog{}, &model.SecurityEvent{}) + }, + }, }) // 执行迁移 diff --git a/wafenginecore/weblog_record.go b/wafenginecore/weblog_record.go index 8a01ae69..fbb416ae 100644 --- a/wafenginecore/weblog_record.go +++ b/wafenginecore/weblog_record.go @@ -40,7 +40,7 @@ func shouldRecordWebLog(weblog *innerbean.WebLog, excludeURLLog string) bool { case "all": return true case "abnormal": - return weblog.ACTION != "放行" || weblog.RULE != "" || weblog.LogOnlyMode == 1 + return weblog.IsSecurityEvent() } return false } diff --git a/wafqueue/log_keys.go b/wafqueue/log_keys.go new file mode 100644 index 00000000..45029832 --- /dev/null +++ b/wafqueue/log_keys.go @@ -0,0 +1,133 @@ +package wafqueue + +import ( + "crypto/sha1" + "encoding/hex" + "strings" +) + +// 窄行上的三个分析键（M5 汇总与 facet 的数据地基），落库这一步行级算好。 +// 归一化规则一改，历史数据与新数据就不可比（汇总失真），所以规则由 log_keys_test.go 钉死： +// 改这里的判定必须连测试一起改，并在提交记录里说明。 + +// ActorKey 访问者身份：有访客身份识别码就用它（换 IP 也认得出来），否则退回 IP。 +func ActorKey(guestID, ip string) string { + if guestID != "" { + return "g:" + guestID + } + if ip != "" { + return "ip:" + ip + } + return "" +} + +// UaHash UA 指纹：聚合「同一个访问者用过多少种 UA」用，不存原文。 +func UaHash(ua string) string { + if ua == "" { + return "" + } + sum := sha1.Sum([]byte(ua)) + return hex.EncodeToString(sum[:])[:16] +} + +// PayloadHash 手法指纹：规则 + 命中内容（body hash）的归一化哈希。 +// 「同 hash 出现在多个 IP 上」= 不同来源在用同一手法打。 +func PayloadHash(rule, bodyHash string) string { + if rule == "" && bodyHash == "" { + return "" + } + sum := sha1.Sum([]byte(rule + "|" + bodyHash)) + return hex.EncodeToString(sum[:])[:16] +} + +// pathNormMaxLen 与窄行表 path_norm 列宽一致。 +const pathNormMaxLen = 512 + +// NormalizePath 把 URL 路径模板化，否则 /item/123456 每个都是新路径，汇总会被爬虫扫目录打爆。 +// 规则（改动必须同步守护用例）： +// - 去掉 query 与 fragment，只留路径 +// - 纯数字段 → {n}；UUID（8-4-4-4-12）→ {id}；长度≥16 的纯 hex 段 → {h} +// - 末段保留扩展名：/img/12345.jpg → /img/{n}.jpg +// - 超过 32 字符的段（多半是 token/长随机串）→ {s} +func NormalizePath(rawURL string) string { + path := rawURL + if i := strings.IndexAny(path, "?#"); i >= 0 { + path = path[:i] + } + segs := strings.Split(path, "/") + for i, seg := range segs { + ext := "" + base := seg + if i == len(segs)-1 { // 末段保留扩展名 + if dot := strings.LastIndexByte(seg, '.'); dot > 0 && len(seg)-dot <= 10 { + ext = seg[dot:] + base = seg[:dot] + } + } + segs[i] = normSegment(base) + ext + } + out := strings.Join(segs, "/") + if len(out) > pathNormMaxLen { + out = out[:pathNormMaxLen] + } + return out +} + +func normSegment(seg string) string { + if seg == "" { + return seg + } + if isAllDigits(seg) { + return "{n}" + } + if isUUID(seg) { + return "{id}" + } + if len(seg) >= 16 && isHex(seg) { + return "{h}" + } + if len(seg) > 32 { + return "{s}" + } + return seg +} + +func isAllDigits(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return true +} + +func isHex(s string) bool { + for i := 0; i < len(s); i++ { + c := s[i] + if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F') { + return false + } + } + return true +} + +// isUUID 形如 8-4-4-4-12 的 hex 段 +func isUUID(s string) bool { + if len(s) != 36 { + return false + } + for i := 0; i < len(s); i++ { + switch i { + case 8, 13, 18, 23: + if s[i] != '-' { + return false + } + default: + c := s[i] + if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F') { + return false + } + } + } + return true +} diff --git a/wafqueue/log_keys_test.go b/wafqueue/log_keys_test.go new file mode 100644 index 00000000..8245625c --- /dev/null +++ b/wafqueue/log_keys_test.go @@ -0,0 +1,63 @@ +package wafqueue + +import "testing" + +// 归一化规则守护用例：这些规则是 M5 汇总表的地基，规则一改历史与新数据就不可比。 +// 要改规则就改这里，并在提交记录里写明——默默改 = 汇总悄悄失真。 +func TestNormalizePath(t *testing.T) { + cases := []struct{ in, want string }{ + {"/item/123456", "/item/{n}"}, + {"/item/123456?x=1&y=2", "/item/{n}"}, // query 去掉 + {"/p/page#frag", "/p/page"}, // fragment 去掉 + {"/u/550e8400-e29b-41d4-a716-446655440000", "/u/{id}"}, + {"/u/550e8400e29b41d4a716446655440000", "/u/{h}"}, // 32 位无横线 hex + {"/img/12345.jpg", "/img/{n}.jpg"}, // 末段保留扩展名 + {"/static/app.9f8c7d6b.js", "/static/app.9f8c7d6b.js"}, // 段内含点的短串不归一，扩展名保留 + {"/a/b/c", "/a/b/c"}, + {"/", "/"}, + {"", ""}, + {"/t/abasdhfkjahsdkjfhajksdhfkjashdfkjahsldkfhj", "/t/{s}"}, // 超长段当 token 处理 + {"/api/v2/2026/report", "/api/v2/{n}/report"}, + {"/d/123/456/789", "/d/{n}/{n}/{n}"}, + } + for _, c := range cases { + if got := NormalizePath(c.in); got != c.want { + t.Errorf("NormalizePath(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestActorKey(t *testing.T) { + if got := ActorKey("guest-1", "1.2.3.4"); got != "g:guest-1" { + t.Fatalf("有访客身份时优先用它，实际 %q", got) + } + if got := ActorKey("", "1.2.3.4"); got != "ip:1.2.3.4" { + t.Fatalf("没访客身份退回 IP，实际 %q", got) + } + if got := ActorKey("", ""); got != "" { + t.Fatalf("都没有应为空，实际 %q", got) + } + // 同一个访客换 IP 仍同键；同一 IP 不同访客不同键 + if ActorKey("guest-1", "1.2.3.4") != ActorKey("guest-1", "9.9.9.9") { + t.Fatal("换 IP 不该改变 actor_key") + } + if ActorKey("a", "1.2.3.4") == ActorKey("b", "1.2.3.4") { + t.Fatal("不同访客不该同键") + } +} + +func TestUaHashAndPayloadHash(t *testing.T) { + if UaHash("") != "" { + t.Fatal("空 UA 不留指纹") + } + if UaHash("Mozilla/5.0") != UaHash("Mozilla/5.0") || UaHash("Mozilla/5.0") == UaHash("curl/8.0") { + t.Fatal("UA 指纹应稳定且区分") + } + if PayloadHash("", "") != "" { + t.Fatal("无规则无内容不产手法指纹") + } + a := PayloadHash("SQLi:Union", "bodyhash1") + if a != PayloadHash("SQLi:Union", "bodyhash1") || a == PayloadHash("SQLi:Union", "bodyhash2") || a == PayloadHash("XSS", "bodyhash1") { + t.Fatal("手法指纹应稳定、随规则与内容变化") + } +} diff --git a/wafqueue/log_queue.go b/wafqueue/log_queue.go index bfd9989a..1a0b0e6b 100644 --- a/wafqueue/log_queue.go +++ b/wafqueue/log_queue.go @@ -61,12 +61,12 @@ func ProcessLogDequeEngine() { } } if global.GCONFIG_LOG_PERSIST_ENABLED == 1 { - // 只有落库这一份拆表并做超长截断；下面的统计与出口仍拿完整的原始对象 - storeArray, payloads := splitForStore(webLogArray) - if err := global.GWAF_LOCAL_LOG_DB.CreateInBatches(storeArray, len(storeArray)).Error; err != nil { - zlog.Warn("日志落库失败", "条数", len(storeArray), "error", err.Error()) - } - storePayloads(payloads) + // 分层落库：安全事件进 security_event（带报文），窄行进 access_log（按档位）， + // web_logs 不再写入（旧数据只读到期删）。 + // 全部在副本上动手，下面的统计与出口仍拿完整的原始对象。 + storeTiered(webLogArray) + // 蓄水池采到的正常请求报文攒在内存里，到点整池刷成 event_payload(kind=sample) + negSampler.FlushIfDue() } // 日志流做统计 waftask.CollectStatsFromLogs(webLogArray) diff --git a/wafqueue/log_reservoir.go b/wafqueue/log_reservoir.go new file mode 100644 index 00000000..292b5943 --- /dev/null +++ b/wafqueue/log_reservoir.go @@ -0,0 +1,117 @@ +package wafqueue + +import ( + "SamWaf/innerbean" + "SamWaf/model" + "math/rand" + "strconv" + "sync" + "time" +) + +// sampleNegPerSiteDay 每个站点每天最多留多少条正常请求的报文（AI 训练的负样本池，D2）。 +// 正常请求报文一律不留会把「空 body」学成正常，模型阈值随之失效；全留又失去了分层的意义。 +// 蓄水池采样保证当天流量无论多大，留下来的都是等概率的 500 条。 +const sampleNegPerSiteDay = 500 + +// sampleFlushInterval 蓄水池攒多久往 event_payload 落一次。 +// 采样行是「丢了不可惜」的数据，攒批换来的是写入次数与流量解耦。 +const sampleFlushInterval = 10 * time.Second + +// sampleReservoir 按「站点 × 天」做蓄水池采样（algorithm R：前 N 条全留， +// 之后第 i 条以 N/i 的概率替换池中随机一条，全程等概率、池子上限恒定）。 +// +// 为什么不能采中即落库：蓄水池的「替换」意味着后来被采中的会挤掉先来的， +// 即采即写会留下一堆早已被挤出池的行。所以池子留在内存（上限 500 条/站点/天， +// 截断后单条 ≤64KB），隔 sampleFlushInterval 把有变化的池子整池刷进 event_payload +// （主键 DoNothing，已在库里的行自动跳过）；进程退出丢失未刷的部分，样本数据可以接受。 +// +// 只有日志队列一个协程在写，锁为单测与将来可能的第二写者兜底。 +type sampleReservoir struct { + mu sync.Mutex + buckets map[string]*sampleBucket + lastFlush time.Time +} + +type sampleBucket struct { + day int + seen int + pick []*model.EventPayload + dirty bool +} + +var negSampler = &sampleReservoir{buckets: map[string]*sampleBucket{}} + +// Pick 把一条正常请求交给蓄水池；返回 true 表示它当前在池里（调用方在 sample 档下为它写窄行）。 +// 报文全空的请求不采（空报文对 AI 训练没有价值，留着只会挤占配额）。 +func (r *sampleReservoir) Pick(lg *innerbean.WebLog) bool { + p := extractPayload(lg, PayloadKindSample) + if p == nil { + return false + } + + today, _ := strconv.Atoi(time.Now().Format("20060102")) + day := lg.Day + if day <= 0 { + day = today + } + key := lg.HOST_CODE + "|" + strconv.Itoa(day) + + r.mu.Lock() + defer r.mu.Unlock() + b := r.buckets[key] + if b == nil || b.day != day { + b = &sampleBucket{day: day, pick: make([]*model.EventPayload, 0, sampleNegPerSiteDay)} + r.buckets[key] = b + // 顺路清掉过期 bucket，免得跨天运行的实例慢慢涨 + for k, old := range r.buckets { + if old.day != day && old.day != today { + delete(r.buckets, k) + } + } + } + b.seen++ + i := b.seen + if i <= sampleNegPerSiteDay { + b.pick = append(b.pick, p) + b.dirty = true + return true + } + if j := rand.Intn(i); j < sampleNegPerSiteDay { + b.pick[j] = p + b.dirty = true + return true + } + return false +} + +// FlushIfDue 到点就把有变化的池子刷进报文表。日志队列每轮都会调，节流在这层。 +func (r *sampleReservoir) FlushIfDue() { + r.mu.Lock() + if time.Since(r.lastFlush) < sampleFlushInterval { + r.mu.Unlock() + return + } + r.lastFlush = time.Now() + var out []*model.EventPayload + for _, b := range r.buckets { + if !b.dirty { + continue + } + b.dirty = false + out = append(out, b.pick...) + } + r.mu.Unlock() + storePayloads(out) +} + +// snapshot 当前池内容（单测用） +func (r *sampleReservoir) snapshot() []*model.EventPayload { + r.mu.Lock() + defer r.mu.Unlock() + var out []*model.EventPayload + for _, b := range r.buckets { + out = append(out, b.pick...) + } + return out +} diff --git a/wafqueue/log_reservoir_test.go b/wafqueue/log_reservoir_test.go new file mode 100644 index 00000000..61b4a5f7 --- /dev/null +++ b/wafqueue/log_reservoir_test.go @@ -0,0 +1,63 @@ +package wafqueue + +import ( + "SamWaf/innerbean" + "testing" +) + +func TestSampleReservoirCap(t *testing.T) { + r := &sampleReservoir{buckets: map[string]*sampleBucket{}} + for i := 0; i < 10000; i++ { + r.Pick(&innerbean.WebLog{HOST_CODE: "h1", Day: 20260901, REQ_UUID: itoa(i), BODY: "x"}) + } + picks := r.snapshot() + if len(picks) != sampleNegPerSiteDay { + t.Fatalf("池子上限应恒为 %d，实际 %d", sampleNegPerSiteDay, len(picks)) + } + // 池内不得有重复 req_uuid + seen := map[string]bool{} + for _, p := range picks { + if seen[p.ReqUUID] { + t.Fatalf("池内出现重复 %s", p.ReqUUID) + } + seen[p.ReqUUID] = true + if p.Kind != PayloadKindSample { + t.Fatalf("负样本 kind 应为 sample，实际 %q", p.Kind) + } + } + // 采中的条目把池子标脏，FlushIfDue 才会落库 + b := r.buckets["h1|20260901"] + if !b.dirty { + t.Fatal("有采中的条目池子应标脏") + } +} + +// 前 N 条必中；报文全空的不采（空报文对 AI 训练没有价值） +func TestSampleReservoirFirstNAndEmpty(t *testing.T) { + r := &sampleReservoir{buckets: map[string]*sampleBucket{}} + for i := 0; i < sampleNegPerSiteDay; i++ { + if !r.Pick(&innerbean.WebLog{HOST_CODE: "h1", Day: 20260901, REQ_UUID: itoa(i), BODY: "x"}) { + t.Fatalf("前 %d 条必须全中，第 %d 条漏了", sampleNegPerSiteDay, i+1) + } + } + if r.Pick(&innerbean.WebLog{HOST_CODE: "h1", Day: 20260901, REQ_UUID: "empty"}) { + t.Fatal("报文全空的请求不该占负样本配额") + } +} + +func TestSampleReservoirPerSitePerDay(t *testing.T) { + r := &sampleReservoir{buckets: map[string]*sampleBucket{}} + for i := 0; i < sampleNegPerSiteDay; i++ { + r.Pick(&innerbean.WebLog{HOST_CODE: "h1", Day: 20260901, BODY: "x"}) + } + if !r.Pick(&innerbean.WebLog{HOST_CODE: "h2", Day: 20260901, BODY: "x"}) { + t.Fatal("另一个站点的第一条必须采中（各自独立）") + } + // 新的一天到来后，更早的 bucket 被清（用两个都早于今天的日期，免得依赖真实日期） + r.Pick(&innerbean.WebLog{HOST_CODE: "h1", Day: 20260902, BODY: "x"}) + for k := range r.buckets { + if k == "h1|20260901" { + t.Fatal("新的一天到来后，更早的 bucket 应被清掉") + } + } +} diff --git a/wafqueue/log_split.go b/wafqueue/log_split.go index fdd5adaa..08dc50f3 100644 --- a/wafqueue/log_split.go +++ b/wafqueue/log_split.go @@ -3,109 +3,15 @@ package wafqueue import ( "SamWaf/common/zlog" "SamWaf/global" - "SamWaf/innerbean" "SamWaf/model" "gorm.io/gorm/clause" ) -// PayloadKindEvent 当前唯一在写的报文归属。采样负样本(sample)与观察名单(watch)在后续分层写入里接上。 +// PayloadKindEvent 安全事件的报文归属（事件全量留报文）。 +// 另外两种 sample（采样负样本）/ watch（观察名单）在 log_tier.go。 const PayloadKindEvent = "event" -// splitForStore 把一批日志拆成两份落库数据：只剩窄列的 web_logs 行，和装报文的 event_payload 行。 -// -// 两条硬性约束： -// -// 1. 不改原对象。Kafka 出口、文件日志、统计与规则引擎共享同一批指针，且落库排在它们前面， -// 就地清列会让下游只拿到空报文。所以每条都是在副本上动手。 -// 2. req_uuid 为空的日志不拆。报文表以 req_uuid 为主键，没有它就没法再找回来， -// 这种日志维持老样子——报文留在 web_logs 自己的列里（仍然截断）。 -// -// 同一个 req_uuid 在一批里可能出现两次（一次请求分阶段入队，后一条信息更全）， -// web_logs 没有主键容得下两行，报文表容不下，所以按 req_uuid 去重取后到的那条。 -// -// HEADER 这一列本次不搬：访问日志页把它当独立列显示，还带一个 LIKE 全文筛选。 -// 搬走等于那一列立刻空掉、筛选也不再命中，而替代的筛选入口要等日志分层(access_log / -// security_event)拆出来之后才有地方放。它留在窄行里，跟着后续的视图拆分一起走。 -func splitForStore(logs []*innerbean.WebLog) ([]*innerbean.WebLog, []*model.EventPayload) { - narrow := make([]*innerbean.WebLog, 0, len(logs)) - payloads := make([]*model.EventPayload, 0, len(logs)) - seen := make(map[string]int, len(logs)) - - for _, lg := range logs { - if lg == nil { - continue - } - c := *lg - - header := cutUTF8(lg.HEADER) - cookies := cutUTF8(lg.COOKIES) - body := cutUTF8(lg.BODY) - resBody := cutUTF8(lg.RES_BODY) - postForm := cutUTF8(lg.POST_FORM) - resHeader := cutUTF8(lg.ResHeader) - srcBody := cutBytes(lg.SrcByteBody) - srcResBody := cutBytes(lg.SrcByteResBody) - srcURL := cutBytes(lg.SrcURL) - - if len(header) != len(lg.HEADER) || len(cookies) != len(lg.COOKIES) || - len(body) != len(lg.BODY) || len(resBody) != len(lg.RES_BODY) || - len(postForm) != len(lg.POST_FORM) || len(resHeader) != len(lg.ResHeader) || - len(srcBody) != len(lg.SrcByteBody) || len(srcResBody) != len(lg.SrcByteResBody) || - len(srcURL) != len(lg.SrcURL) { - c.Truncated = 1 - } - - c.HEADER = header - - if lg.REQ_UUID == "" { - c.COOKIES, c.BODY = cookies, body - c.RES_BODY, c.POST_FORM, c.ResHeader = resBody, postForm, resHeader - c.SrcByteBody, c.SrcByteResBody, c.SrcURL = srcBody, srcResBody, srcURL - narrow = append(narrow, &c) - continue - } - - p := &model.EventPayload{ - ReqUUID: lg.REQ_UUID, - TenantId: lg.TenantId, - UserCode: lg.USER_CODE, - HostCode: lg.HOST_CODE, - Kind: PayloadKindEvent, - COOKIES: cookies, - BODY: body, - RES_BODY: resBody, - POST_FORM: postForm, - ResHeader: resHeader, - SrcByteBody: srcBody, - SrcByteResBody: srcResBody, - SrcURL: srcURL, - Truncated: c.Truncated, - CreateTime: lg.CREATE_TIME, - UnixAddTime: lg.UNIX_ADD_TIME, - Day: lg.Day, - } - - c.COOKIES, c.BODY = "", "" - c.RES_BODY, c.POST_FORM, c.ResHeader = "", "", "" - c.SrcByteBody, c.SrcByteResBody, c.SrcURL = nil, nil, nil - narrow = append(narrow, &c) - - // 报文列全空就不占一行：读侧找不到报文行时会回落读 web_logs 的原列，结果一样是空。 - if p.IsEmpty() { - continue - } - if at, dup := seen[p.ReqUUID]; dup { - payloads[at] = p - continue - } - seen[p.ReqUUID] = len(payloads) - payloads = append(payloads, p) - } - - return narrow, payloads -} - // storePayloads 写报文表。单独一条语句，且失败只告警： // 日志行已经落库，报文丢了详情页显示「未留存报文」，比整批日志一起回滚强。 // 跨批次可能撞上同一个 req_uuid（同一请求分两批入队），主键冲突按「保留先到的」跳过。 diff --git a/wafqueue/log_split_test.go b/wafqueue/log_split_test.go deleted file mode 100644 index 82ec09ca..00000000 --- a/wafqueue/log_split_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package wafqueue - -import ( - "SamWaf/innerbean" - "strings" - "testing" -) - -// 报文搬进 event_payload 行，web_logs 行只剩窄列 -func TestSplitForStore_MovesPayload(t *testing.T) { - origin := &innerbean.WebLog{ - REQ_UUID: "uuid-1", - HOST_CODE: "host-1", - HEADER: "User-Agent: curl", - BODY: "a=1", - RES_BODY: "ok", - POST_FORM: "a=1", - COOKIES: "sid=x", - ResHeader: "Content-Type: text/html", - URL: "/login", - } - - narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) - - if len(narrow) != 1 || len(payloads) != 1 { - t.Fatalf("应拆成 1 窄行 + 1 报文行，实际 %d/%d", len(narrow), len(payloads)) - } - n := narrow[0] - if n.BODY != "" || n.RES_BODY != "" || n.POST_FORM != "" || - n.COOKIES != "" || n.ResHeader != "" { - t.Fatal("窄行的报文列没清干净") - } - if n.URL != "/login" { - t.Fatal("非报文列不该被动") - } - // HEADER 本次不搬：访问日志页把它当独立列显示还带 LIKE 筛选 - if n.HEADER != "User-Agent: curl" { - t.Fatal("HEADER 必须留在窄行上") - } - p := payloads[0] - if p.ReqUUID != "uuid-1" || p.HostCode != "host-1" || p.Kind != PayloadKindEvent { - t.Fatalf("报文行归属字段不对: %+v", p) - } - if p.BODY != "a=1" || p.RES_BODY != "ok" || p.ResHeader != "Content-Type: text/html" { - t.Fatal("报文没搬全") - } - if p.HEADER != "" { - t.Fatal("HEADER 本次不进报文表，否则会存两份") - } -} - -// 原对象必须保持原文：Kafka 出口与规则引擎共享同一批指针，且落库排在它们前面 -func TestSplitForStore_KeepsOriginalIntact(t *testing.T) { - big := strings.Repeat("b", payloadMaxBytes+1000) - origin := &innerbean.WebLog{REQ_UUID: "uuid-2", BODY: big, RES_BODY: "ok"} - - narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) - - if narrow[0] == origin { - t.Fatal("窄行必须是副本，不能就地改原对象") - } - if len(origin.BODY) != len(big) || origin.Truncated != 0 { - t.Fatal("原对象被改动了，Kafka 出口会拿到半截报文") - } - if len(payloads[0].BODY) > payloadMaxBytes { - t.Fatalf("报文没截断，长度 %d", len(payloads[0].BODY)) - } - if payloads[0].Truncated != 1 || narrow[0].Truncated != 1 { - t.Fatal("截断标记两边都要有") - } -} - -// 没有 req_uuid 就没法再按主键找回报文，这种日志维持老样子留在 web_logs 列里 -func TestSplitForStore_NoUUIDKeepsInline(t *testing.T) { - origin := &innerbean.WebLog{BODY: "a=1", COOKIES: "sid=x"} - - narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) - - if len(payloads) != 0 { - t.Fatal("没有 req_uuid 不应产出报文行") - } - if narrow[0].BODY != "a=1" || narrow[0].COOKIES != "sid=x" { - t.Fatal("没有 req_uuid 时报文必须留在窄行里，否则就丢了") - } -} - -// 报文列全空不占一行 -func TestSplitForStore_SkipsEmptyPayload(t *testing.T) { - origin := &innerbean.WebLog{REQ_UUID: "uuid-3", URL: "/", SRC_IP: "1.2.3.4", HEADER: "H: 1"} - - narrow, payloads := splitForStore([]*innerbean.WebLog{origin}) - - if len(narrow) != 1 { - t.Fatal("窄行还是要写的") - } - if len(payloads) != 0 { - t.Fatal("报文全空不该占一行") - } - if narrow[0].HEADER != "H: 1" { - t.Fatal("HEADER 留在窄行，不算报文") - } -} - -// 同一 req_uuid 在一批里出现两次（分阶段入队），报文表主键容不下两行，取后到的那条 -func TestSplitForStore_DedupByUUID(t *testing.T) { - first := &innerbean.WebLog{REQ_UUID: "uuid-4", BODY: "a=1"} - second := &innerbean.WebLog{REQ_UUID: "uuid-4", BODY: "a=1", RES_BODY: "done"} - - narrow, payloads := splitForStore([]*innerbean.WebLog{first, second}) - - if len(narrow) != 2 { - t.Fatal("web_logs 没有主键，两行都照写") - } - if len(payloads) != 1 { - t.Fatalf("报文行应去重成 1 条，实际 %d", len(payloads)) - } - if payloads[0].RES_BODY != "done" { - t.Fatal("应保留后到的那条（信息更全）") - } -} - -// nil 条目直接跳过，不能带崩整批 -func TestSplitForStore_SkipsNil(t *testing.T) { - narrow, payloads := splitForStore([]*innerbean.WebLog{nil, {REQ_UUID: "uuid-5", BODY: "x"}}) - if len(narrow) != 1 || len(payloads) != 1 { - t.Fatalf("nil 应被跳过，实际 %d/%d", len(narrow), len(payloads)) - } -} diff --git a/wafqueue/log_tier.go b/wafqueue/log_tier.go new file mode 100644 index 00000000..359894cd --- /dev/null +++ b/wafqueue/log_tier.go @@ -0,0 +1,240 @@ +package wafqueue + +import ( + "SamWaf/common/uuid" + "SamWaf/common/zlog" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "unicode/utf8" + + "gorm.io/gorm/clause" +) + +// narrowURLMaxBytes url/raw_query 落库的字节上限（C8）。极长 URL 只在报文里看全。 +const narrowURLMaxBytes = 2048 + +// ipWatchHas 观察名单判定。包级变量，单测可替换。 +var ipWatchHas = func(ip string) bool { return global.GWAF_IP_WATCH.Has(ip) } + +// 报文行的三种归属：事件全量留、正常请求只留蓄水池采到的样本、观察名单内的 IP 全量留。 +const ( + PayloadKindSample = "sample" + PayloadKindWatch = "watch" +) + +// tierForStore 把一批日志分流到三层存储，全部在副本上动手（原对象还要给 Kafka 出口与统计）： +// +// - security_event：安全事件（IsSecurityEvent），必有；报文进 event_payload(kind=event) +// - access_log：窄行。db 档全量，sample 档事件+采到的样本，off 档仅事件（安全事件始终双写， +// 访问日志页因此不用做两表 UNION）；观察名单内的 IP 任意档位都写 +// +// 正常请求的报文不在这里写：蓄水池（log_reservoir.go）采中的攒在内存里定期刷成 +// event_payload(kind=sample)。本函数只问池子「这条有没有被采中」，决定 sample 档下写不写窄行。 +// 例外是观察名单：名单内 IP 的报文就地写 kind=watch，不等池子。 +// +// web_logs 从此不再写入：改造前的数据靠一次性切库变成归档分片，照旧可读（D6 只读到期删）。 +// 同一 req_uuid 一批里可能出现两次（分阶段入队），三张表都以它为主键，批内去重取后到的那条， +// 跨批次由主键 DoNothing 兜住。 +func tierForStore(logs []*innerbean.WebLog) (events []*model.SecurityEvent, accesses []*model.AccessLog, payloads []*model.EventPayload) { + mode := global.GDATA_ACCESS_LOG_MODE + seenEvent := make(map[string]int, len(logs)) + seenAccess := make(map[string]int, len(logs)) + seenPayload := make(map[string]int, len(logs)) + + for _, lg := range logs { + if lg == nil { + continue + } + // req_uuid 是三张表的寻址键。正常路径它一定在；真缺了就地补一个， + // 让这一行在库里也能被点开（Kafka 出口与这行用同一个对象，一并受益）。 + if lg.REQ_UUID == "" { + lg.REQ_UUID = uuid.GenUUID() + } + + isEvent := lg.IsSecurityEvent() + // 观察名单（C7）：名单内 IP 的正常请求也全量留痕——窄行必写、报文 kind=watch。 + // 已按安全事件处理的不再重复取报文（event_payload 以 req_uuid 为主键，一条只有一行）。 + watched := !isEvent && ipWatchHas(lg.SRC_IP) + sampled := false + if !isEvent && !watched && mode != "off" { + // 正常请求的报文只留蓄水池采中的那部分（D2 负样本池）。 + // 池子攒在内存里定期刷库（见 log_reservoir.go），这里只问「有没有被采中」。 + // off 档不采样：用户已明确放弃 AI 训练负样本（档位说明里列了这条）。 + // 观察名单内的已由 watch 全量留痕，不重复采样。 + sampled = negSampler.Pick(lg) + } + + ln := narrowFromLog(lg) + + if isEvent { + events = dedupByUUID(events, seenEvent, lg.REQ_UUID, + &model.SecurityEvent{LogNarrow: ln, PayloadHash: PayloadHash(lg.RULE, lg.BodyHash)}) + if p := extractPayload(lg, PayloadKindEvent); p != nil { + payloads = dedupByUUID(payloads, seenPayload, p.ReqUUID, p) + } + } + if watched { + if p := extractPayload(lg, PayloadKindWatch); p != nil { + payloads = dedupByUUID(payloads, seenPayload, p.ReqUUID, p) + } + } + if isEvent || watched || mode == "db" || (mode == "sample" && sampled) { + accesses = dedupByUUID(accesses, seenAccess, lg.REQ_UUID, &model.AccessLog{LogNarrow: ln}) + } + } + return events, accesses, payloads +} + +// dedupByUUID 批内按 req_uuid 去重，后到的那条覆盖先到的（分阶段入队，后一条信息更全）。 +func dedupByUUID[T any](dst []T, seen map[string]int, id string, item T) []T { + if at, dup := seen[id]; dup { + dst[at] = item + return dst + } + seen[id] = len(dst) + return append(dst, item) +} + +// narrowFromLog 从完整日志对象拷出窄行：url/raw_query 截断到 2KB，三个分析键算好。 +func narrowFromLog(lg *innerbean.WebLog) model.LogNarrow { + url := cutUTF8N(lg.URL, narrowURLMaxBytes) + rawQuery := cutUTF8N(lg.RawQuery, narrowURLMaxBytes) + truncated := 0 + if len(url) != len(lg.URL) || len(rawQuery) != len(lg.RawQuery) { + truncated = 1 + } + return model.LogNarrow{ + ReqUUID: lg.REQ_UUID, + TenantId: lg.TenantId, + UserCode: lg.USER_CODE, + HostCode: lg.HOST_CODE, + Host: lg.HOST, + URL: url, + RawQuery: rawQuery, + Method: lg.METHOD, + Scheme: lg.Scheme, + REFERER: cutUTF8N(lg.REFERER, 1024), + USER_AGENT: cutUTF8N(lg.USER_AGENT, 500), + SRC_IP: lg.SRC_IP, + SRC_PORT: lg.SRC_PORT, + NetSrcIp: lg.NetSrcIp, + COUNTRY: lg.COUNTRY, + PROVINCE: lg.PROVINCE, + CITY: lg.CITY, + ACTION: lg.ACTION, + RULE: lg.RULE, + STATUS: lg.STATUS, + STATUS_CODE: lg.STATUS_CODE, + RISK_LEVEL: lg.RISK_LEVEL, + AI_SCORE: lg.AI_SCORE, + IsBot: lg.IsBot, + LogOnlyMode: lg.LogOnlyMode, + GUEST_IDENTIFICATION: lg.GUEST_IDENTIFICATION, + TimeSpent: lg.TimeSpent, + CONTENT_LENGTH: lg.CONTENT_LENGTH, + ResContentLength: lg.RES_CONTENT_LENGTH, + PreCheckCost: lg.PreCheckCost, + ForwardCost: lg.ForwardCost, + BackendCheckCost: lg.BackendCheckCost, + IsBalance: lg.IsBalance, + BalanceInfo: lg.BalanceInfo, + BodyHash: lg.BodyHash, + Truncated: truncated, + ActorKey: ActorKey(lg.GUEST_IDENTIFICATION, lg.SRC_IP), + UaHash: UaHash(lg.USER_AGENT), + PathNorm: NormalizePath(lg.URL), + CREATE_TIME: lg.CREATE_TIME, + UNIX_ADD_TIME: lg.UNIX_ADD_TIME, + Day: lg.Day, + } +} + +// extractPayload 拷出报文行（含 HEADER——它随窄行拆出后只能跟报文走），单列 64KB 截断。 +// 报文列全空返回 nil：不占一行，读侧找不到报文行时本来就显示「未留存报文」。 +func extractPayload(lg *innerbean.WebLog, kind string) *model.EventPayload { + header := cutUTF8(lg.HEADER) + cookies := cutUTF8(lg.COOKIES) + body := cutUTF8(lg.BODY) + resBody := cutUTF8(lg.RES_BODY) + postForm := cutUTF8(lg.POST_FORM) + resHeader := cutUTF8(lg.ResHeader) + srcBody := cutBytes(lg.SrcByteBody) + srcResBody := cutBytes(lg.SrcByteResBody) + srcURL := cutBytes(lg.SrcURL) + + p := &model.EventPayload{ + ReqUUID: lg.REQ_UUID, + TenantId: lg.TenantId, + UserCode: lg.USER_CODE, + HostCode: lg.HOST_CODE, + Kind: kind, + HEADER: header, + COOKIES: cookies, + BODY: body, + RES_BODY: resBody, + POST_FORM: postForm, + ResHeader: resHeader, + SrcByteBody: srcBody, + SrcByteResBody: srcResBody, + SrcURL: srcURL, + CreateTime: lg.CREATE_TIME, + UnixAddTime: lg.UNIX_ADD_TIME, + Day: lg.Day, + } + if len(header) != len(lg.HEADER) || len(cookies) != len(lg.COOKIES) || + len(body) != len(lg.BODY) || len(resBody) != len(lg.RES_BODY) || + len(postForm) != len(lg.POST_FORM) || len(resHeader) != len(lg.ResHeader) || + len(srcBody) != len(lg.SrcByteBody) || len(srcResBody) != len(lg.SrcByteResBody) || + len(srcURL) != len(lg.SrcURL) { + p.Truncated = 1 + } + if p.IsEmpty() { + return nil + } + return p +} + +// storeTiered 三张表各一条批量语句，主键冲突（跨批次重复入队）保留先到的。 +// 任何一张失败只告警：行还在另外两张里，比整批回滚强。 +func storeTiered(logs []*innerbean.WebLog) { + events, accesses, payloads := tierForStore(logs) + if len(events) > 0 { + if err := conflictIgnoreInsert(events); err != nil { + zlog.Warn("安全事件落库失败", "条数", len(events), "error", err.Error()) + } + } + if len(accesses) > 0 { + if err := conflictIgnoreInsert(accesses); err != nil { + zlog.Warn("访问日志落库失败", "条数", len(accesses), "error", err.Error()) + } + } + storePayloads(payloads) +} + +// conflictIgnoreInsert 整批写入，撞主键的行跳过（保留先到的）。不指定冲突列： +// 三个引擎对"无目标的 DO NOTHING"都认（MySQL 走 INSERT IGNORE），与 storePayloads 同理。 +func conflictIgnoreInsert[T any](rows []T) error { + if len(rows) == 0 { + return nil + } + return global.GWAF_LOCAL_LOG_DB. + Clauses(clause.OnConflict{DoNothing: true}). + CreateInBatches(rows, len(rows)).Error +} + +// cutUTF8N 与 cutUTF8 同规则，上限可调（窄行 url 用 2KB，报文列用 64KB）。 +func cutUTF8N(s string, max int) string { + if len(s) <= max { + return s + } + cut := s[:max] + for i := 0; i < 3 && len(cut) > 0; i++ { + r, size := utf8.DecodeLastRuneInString(cut) + if r != utf8.RuneError || size > 1 { + break + } + cut = cut[:len(cut)-1] + } + return cut +} diff --git a/wafqueue/log_tier_db_test.go b/wafqueue/log_tier_db_test.go new file mode 100644 index 00000000..44ca0c80 --- /dev/null +++ b/wafqueue/log_tier_db_test.go @@ -0,0 +1,97 @@ +//go:build crossdb + +// 分层写入的三库回归。tierForStore 的纯逻辑由 log_tier_test.go 覆盖， +// 这里盯的是真正压到数据库上的写入：三张新表都以 req_uuid 为主键、 +// 整批 OnConflict DoNothing 的写法三个引擎各不相同（sqlite/pg=DO NOTHING，mysql=INSERT IGNORE）， +// 写错了一撞主键整批消失。 +// +// go test -tags crossdb ./wafqueue/ -run TestStoreTieredCrossEngine -v +package wafqueue + +import ( + "SamWaf/common/zlog" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "testing" +) + +func TestStoreTieredCrossEngine(t *testing.T) { + zlog.InitZLog(false, "console") + savedDB := global.GWAF_LOCAL_LOG_DB + savedMode := global.GDATA_ACCESS_LOG_MODE + savedSampler := negSampler + defer func() { + global.GWAF_LOCAL_LOG_DB = savedDB + global.GDATA_ACCESS_LOG_MODE = savedMode + negSampler = savedSampler + }() + + for _, e := range payloadEngines() { + e := e + t.Run(e.name, func(t *testing.T) { + db, teardown := e.setup(t) + if db == nil { + t.Skipf("%s 未就绪，跳过", e.name) + return + } + defer teardown() + if err := db.AutoMigrate(&model.AccessLog{}, &model.SecurityEvent{}, &model.EventPayload{}); err != nil { + t.Fatalf("建分层表失败: %v", err) + } + global.GWAF_LOCAL_LOG_DB = db + global.GDATA_ACCESS_LOG_MODE = "db" + negSampler = &sampleReservoir{buckets: map[string]*sampleBucket{}} + + mk := func(uid, ip, action, rule string) *innerbean.WebLog { + return &innerbean.WebLog{ + REQ_UUID: uid, HOST_CODE: "h1", SRC_IP: ip, URL: "/login", + METHOD: "POST", ACTION: action, RULE: rule, + USER_CODE: "u", TenantId: "t", + CREATE_TIME: "2026-09-18 10:00:00", UNIX_ADD_TIME: 1, Day: 20260918, + HEADER: "User-Agent: curl", BODY: "a=1", RES_BODY: "ok", + } + } + + // 事件 + 正常各一：事件三层全进（报文 kind=event、HEADER 随报文走），正常只有窄行 + storeTiered([]*innerbean.WebLog{mk("te1", "1.2.3.4", "阻止", "SQLi"), mk("tn1", "1.2.3.4", "放行", "")}) + + var nEvent, nAccess, nPayload int64 + db.Model(&model.SecurityEvent{}).Count(&nEvent) + db.Model(&model.AccessLog{}).Count(&nAccess) + db.Model(&model.EventPayload{}).Count(&nPayload) + if nEvent != 1 || nAccess != 2 || nPayload != 1 { + t.Fatalf("分层计数不对: event=%d access=%d payload=%d（期望 1/2/1）", nEvent, nAccess, nPayload) + } + var ev model.SecurityEvent + db.Where("req_uuid = ?", "te1").First(&ev) + if ev.ActorKey != "ip:1.2.3.4" || ev.PathNorm != "/login" || ev.PayloadHash == "" { + t.Fatalf("事件行的分析键没算好: %+v", ev.LogNarrow) + } + var p model.EventPayload + db.Where("req_uuid = ?", "te1").First(&p) + if p.Kind != PayloadKindEvent || p.HEADER != "User-Agent: curl" { + t.Fatalf("事件报文不对: kind=%q header=%q", p.Kind, p.HEADER) + } + + // 同一请求分两批入队：撞主键必须跳过冲突行而不是整批失败 + storeTiered([]*innerbean.WebLog{mk("te1", "1.2.3.4", "阻止", "SQLi"), mk("tn2", "5.6.7.8", "放行", "")}) + db.Model(&model.SecurityEvent{}).Count(&nEvent) + db.Model(&model.AccessLog{}).Count(&nAccess) + db.Model(&model.EventPayload{}).Count(&nPayload) + if nEvent != 1 || nAccess != 3 || nPayload != 1 { + t.Fatalf("重复入队后计数不对: event=%d access=%d payload=%d（期望 1/3/1）", nEvent, nAccess, nPayload) + } + + // off 档：正常请求三层都不留，事件照留 + global.GDATA_ACCESS_LOG_MODE = "off" + storeTiered([]*innerbean.WebLog{mk("to1", "9.9.9.9", "放行", "")}) + var offAccess int64 + db.Model(&model.AccessLog{}).Where("req_uuid = ?", "to1").Count(&offAccess) + if offAccess != 0 { + t.Fatalf("off 档正常请求不应有窄行，实际 %d", offAccess) + } + global.GDATA_ACCESS_LOG_MODE = "db" + }) + } +} diff --git a/wafqueue/log_tier_test.go b/wafqueue/log_tier_test.go new file mode 100644 index 00000000..60aaa0c4 --- /dev/null +++ b/wafqueue/log_tier_test.go @@ -0,0 +1,225 @@ +package wafqueue + +import ( + "SamWaf/global" + "SamWaf/innerbean" + "strings" + "testing" +) + +// 分层写入的纯逻辑用例。每个用例前重置档位与蓄水池，互不影响。 +func resetTier(mode string) func() { + savedMode := global.GDATA_ACCESS_LOG_MODE + savedSampler := negSampler + global.GDATA_ACCESS_LOG_MODE = mode + negSampler = &sampleReservoir{buckets: map[string]*sampleBucket{}} + return func() { + global.GDATA_ACCESS_LOG_MODE = savedMode + negSampler = savedSampler + } +} + +func mkLog(uuid, ip string) *innerbean.WebLog { + return &innerbean.WebLog{ + REQ_UUID: uuid, HOST_CODE: "h1", SRC_IP: ip, URL: "/login", + ACTION: "放行", CREATE_TIME: "2026-09-18 10:00:00", UNIX_ADD_TIME: 1, Day: 20260918, + HEADER: "User-Agent: curl", BODY: "a=1", RES_BODY: "ok", + } +} + +// 安全事件：窄行双写（security_event + access_log），报文进 event_payload(kind=event)，HEADER 随报文走 +func TestTier_EventWritesAllLayers(t *testing.T) { + defer resetTier("off")() // off 档也不影响事件 + lg := mkLog("e1", "1.2.3.4") + lg.ACTION = "阻止" + lg.RULE = "SQLi:Union" + + events, accesses, payloads := tierForStore([]*innerbean.WebLog{lg}) + + if len(events) != 1 || len(accesses) != 1 || len(payloads) != 1 { + t.Fatalf("事件应进三张表，实际 %d/%d/%d", len(events), len(accesses), len(payloads)) + } + if payloads[0].Kind != PayloadKindEvent { + t.Fatalf("事件报文 kind 应为 event，实际 %q", payloads[0].Kind) + } + if payloads[0].HEADER != "User-Agent: curl" { + t.Fatal("HEADER 随窄行拆分搬进报文表，不能丢") + } + if events[0].PayloadHash == "" { + t.Fatal("事件必须带手法指纹") + } + if events[0].ActorKey != "ip:1.2.3.4" || events[0].PathNorm != "/login" { + t.Fatalf("分析键没算好: %+v", events[0].LogNarrow) + } +} + +// db 档：正常请求窄行全量进 access_log；报文不进本批——被蓄水池采中的攒在池里定期刷库（D2） +func TestTier_DBModeNormalNoPayload(t *testing.T) { + defer resetTier("db")() + events, accesses, payloads := tierForStore([]*innerbean.WebLog{mkLog("n1", "1.2.3.4")}) + if len(events) != 0 || len(accesses) != 1 || len(payloads) != 0 { + t.Fatalf("db 档正常请求应只有窄行，实际 %d/%d/%d", len(events), len(accesses), len(payloads)) + } + // 第一条正常请求必被蓄水池采中 + if len(negSampler.snapshot()) != 1 { + t.Fatal("正常请求应进蓄水池") + } +} + +// sample 档：采中的正常请求有窄行；超过蓄水池上限后，被替换挤出的也写了窄行（报文以池为准，宁多勿缺） +func TestTier_SampleMode(t *testing.T) { + defer resetTier("sample")() + logs := make([]*innerbean.WebLog, 0, sampleNegPerSiteDay+100) + for i := 0; i < sampleNegPerSiteDay+100; i++ { + logs = append(logs, mkLog("nx"+itoa(i), "1.2.3.4")) + } + events, accesses, payloads := tierForStore(logs) + if len(events) != 0 || len(payloads) != 0 { + t.Fatal("全是正常请求不该有事件与事件报文") + } + if len(accesses) < sampleNegPerSiteDay || len(accesses) > len(logs) { + t.Fatalf("sample 档窄行数应在蓄水池上限与总量之间，实际 %d", len(accesses)) + } + if got := len(negSampler.snapshot()); got != sampleNegPerSiteDay { + t.Fatalf("蓄水池应恒为 %d 条，实际 %d", sampleNegPerSiteDay, got) + } +} + +// off 档：正常请求什么都不留，只留安全事件 +func TestTier_OffModeDropsNormal(t *testing.T) { + defer resetTier("off")() + events, accesses, payloads := tierForStore([]*innerbean.WebLog{mkLog("n9", "1.2.3.4")}) + if len(events) != 0 || len(accesses) != 0 || len(payloads) != 0 { + t.Fatalf("off 档正常请求应三层都不留，实际 %d/%d/%d", len(events), len(accesses), len(payloads)) + } +} + +// 命中了规则但没拦（仅记录/自定义规则放行）也算事件——判定与引擎 abnormal 分支同一条规则 +func TestTier_LogOnlyAndRulePassAreEvents(t *testing.T) { + defer resetTier("off")() + a := mkLog("lo1", "1.2.3.4") + a.LogOnlyMode = 1 + b := mkLog("lo2", "1.2.3.4") + b.RULE = "自定义规则放行" + events, accesses, _ := tierForStore([]*innerbean.WebLog{a, b}) + if len(events) != 2 || len(accesses) != 2 { + t.Fatalf("仅记录与带规则的放行都算事件，实际 %d/%d", len(events), len(accesses)) + } +} + +// url 超 2KB 截断并置 Truncated；报文列的 64KB 截断只标在报文行上 +func TestTier_URLTruncation(t *testing.T) { + defer resetTier("db")() + lg := mkLog("u1", "1.2.3.4") + lg.URL = "/p/" + strings.Repeat("x", narrowURLMaxBytes) + lg.ACTION = "阻止" + + events, accesses, _ := tierForStore([]*innerbean.WebLog{lg}) + if len(events[0].URL) > narrowURLMaxBytes || events[0].Truncated != 1 { + t.Fatal("事件窄行 url 应截断到 2KB 并置 Truncated") + } + if accesses[0].Truncated != 1 { + t.Fatal("access_log 窄行同样截断") + } + if len(lg.URL) <= narrowURLMaxBytes { + t.Fatal("原对象不能被改（Kafka 出口要原文）") + } +} + +// 没有 req_uuid 的日志补一个再写——三张表都以它寻址，空主键会让整批互相覆盖 +func TestTier_FillsMissingUUID(t *testing.T) { + defer resetTier("db")() + lg := mkLog("", "1.2.3.4") + events, accesses, _ := tierForStore([]*innerbean.WebLog{lg}) + if lg.REQ_UUID == "" { + t.Fatal("缺 req_uuid 应就地补上") + } + if len(accesses) != 1 || accesses[0].ReqUUID != lg.REQ_UUID { + t.Fatal("窄行必须带着补上的 req_uuid") + } + _ = events +} + +// 同一 req_uuid 一批来两条（分阶段入队），三张表都取后到的那条 +func TestTier_DedupByUUID(t *testing.T) { + defer resetTier("db")() + first := mkLog("dup1", "1.2.3.4") + first.ACTION = "阻止" + second := mkLog("dup1", "1.2.3.4") + second.ACTION = "阻止" + second.RES_BODY = "done" + + events, accesses, payloads := tierForStore([]*innerbean.WebLog{first, second}) + if len(events) != 1 || len(accesses) != 1 || len(payloads) != 1 { + t.Fatalf("三张表都应去重成 1 条，实际 %d/%d/%d", len(events), len(accesses), len(payloads)) + } + if payloads[0].RES_BODY != "done" { + t.Fatal("应保留后到的那条（信息更全）") + } +} + +// 原对象的报文必须原样留给 Kafka 出口与统计 +func TestTier_KeepsOriginalIntact(t *testing.T) { + defer resetTier("db")() + big := strings.Repeat("b", payloadMaxBytes+1000) + lg := mkLog("k1", "1.2.3.4") + lg.ACTION = "阻止" + lg.BODY = big + + tierForStore([]*innerbean.WebLog{lg}) + if len(lg.BODY) != len(big) || lg.Truncated != 0 { + t.Fatal("原对象被改动了，Kafka 出口会拿到半截报文") + } +} + +// nil 条目跳过，不能带崩整批 +func TestTier_SkipsNil(t *testing.T) { + defer resetTier("db")() + events, accesses, payloads := tierForStore([]*innerbean.WebLog{nil, mkLog("z1", "1.2.3.4")}) + if len(accesses) != 1 { + t.Fatalf("nil 应被跳过，实际 %d/%d/%d", len(events), len(accesses), len(payloads)) + } +} + +// 观察名单（C7）：名单内 IP 的正常请求任意档位都写窄行 + kind=watch 报文，且不再进采样池； +// 事件不受名单影响（报文已是 kind=event，一个 req_uuid 只有一条报文行） +func TestTier_WatchedIPFullCapture(t *testing.T) { + defer resetTier("off")() // off 档也不影响观察名单 + savedHas := ipWatchHas + ipWatchHas = func(ip string) bool { return ip == "9.9.9.9" } + defer func() { ipWatchHas = savedHas }() + + normal := mkLog("w1", "9.9.9.9") + event := mkLog("w2", "9.9.9.9") + event.RULE = "SQLi" + other := mkLog("w3", "1.2.3.4") + + events, accesses, payloads := tierForStore([]*innerbean.WebLog{normal, event, other}) + + if len(events) != 1 || len(accesses) != 2 || len(payloads) != 2 { + t.Fatalf("名单内正常+事件应各有窄行与报文，名单外 off 档不留，实际 %d/%d/%d", + len(events), len(accesses), len(payloads)) + } + kinds := map[string]string{} + for _, p := range payloads { + kinds[p.ReqUUID] = p.Kind + } + if kinds["w1"] != PayloadKindWatch || kinds["w2"] != PayloadKindEvent { + t.Fatalf("报文 kind 不对: %v", kinds) + } + if len(negSampler.snapshot()) != 0 { + t.Fatal("名单内的请求不重复进采样池") + } +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + s := "" + for i > 0 { + s = string(rune('0'+i%10)) + s + i /= 10 + } + return s +} diff --git a/waftask/stat_collector.go b/waftask/stat_collector.go index f51d3dc6..fdaa5518 100644 --- a/waftask/stat_collector.go +++ b/waftask/stat_collector.go @@ -144,12 +144,12 @@ func CollectStatsFromLogs(logs []*innerbean.WebLog) { } cityAgg[ck]++ - // IPTag 聚合 - rule := lg.RULE - if rule == "" { - rule = "正常" + // IPTag 聚合：只记风险标签（RULE 非空，含「自定义规则放行/验证通过/ACME证书校验」这类）。 + // 未命中规则的请求不再生成「正常」标签——放行数量由 stats_ip_days 承担（同一份数据的 + // 重复记录，也是 ip_tags 行数与写入量的大头，D7）；存量「正常」行由启动任务清掉。 + if lg.RULE != "" { + ipTagAgg[ipTagKey{IP: lg.SRC_IP, Rule: lg.RULE}]++ } - ipTagAgg[ipTagKey{IP: lg.SRC_IP, Rule: rule}]++ // 站点天级聚合 sdk := siteDayKey{ diff --git a/waftask/task_config.go b/waftask/task_config.go index 358b822e..50173589 100644 --- a/waftask/task_config.go +++ b/waftask/task_config.go @@ -86,6 +86,12 @@ func setConfigIntValue(name string, value int64, change int) { case "db_file_size": global.GDATA_SHARE_DB_FILE_SIZE = value break + case "access_log_retention_days": + if value < 1 { + value = 1 + } + global.GDATA_ACCESS_LOG_RETENTION_DAYS = value + break case "auto_load_ssl_file": global.GCONFIG_RECORD_AUTO_LOAD_SSL = value break @@ -357,6 +363,12 @@ func setConfigStringValue(name string, value string, change int) { case "record_log_type": global.GWAF_RUNTIME_RECORD_LOG_TYPE = value break + case "access_log_mode": + if value != "off" && value != "sample" { + value = "db" + } + global.GDATA_ACCESS_LOG_MODE = value + break case "attack_tag_exclude": global.GCONFIG_ATTACK_TAG_EXCLUDE = value break @@ -744,6 +756,8 @@ func TaskLoadSetting(initLoad bool) { updateConfigIntItem(initLoad, "system", "dns_timeout", global.GWAF_RUNTIME_DNS_TIMEOUT, "DNS 查询超时时间 单位毫秒", "int", "", configMap) updateConfigStringItem(initLoad, "system", "record_log_type", global.GWAF_RUNTIME_RECORD_LOG_TYPE, "日志记录类型", "options", "all|全部,abnormal|非正常", configMap) + updateConfigStringItem(initLoad, "system", "access_log_mode", global.GDATA_ACCESS_LOG_MODE, "访问日志窄行档位：db=全部请求入库(默认,保留期见下项)；sample=安全事件+采样入库；off=只留安全事件(高流量推荐，但会失去CC阈值推荐/AI训练负样本/异常IP的正常行为回溯)。db/sample 档均按站点每天采样 500 条正常请求报文供 AI 训练", "options", "db|全部入库,sample|采样入库,off|仅安全事件", configMap) + updateConfigIntItem(initLoad, "system", "access_log_retention_days", global.GDATA_ACCESS_LOG_RETENTION_DAYS, "访问日志窄行保留天数（默认30）。安全事件仍按「日志保留天数」走；本项直接决定CC阈值推荐能回看多少天", "int", "", configMap) updateConfigStringItem(initLoad, "system", "gwaf_proxy_header", global.GCONFIG_RECORD_PROXY_HEADER, "获取访客IP头信息（按照顺序）比如:X-Forwarded-For,X-Real-IP ,留空则提取的是直接访客IP", "string", "", configMap) updateConfigStringItem(initLoad, "system", "gwaf_manage_proxy_header", global.GCONFIG_MANAGE_PROXY_HEADER, "管理端获取客户端IP头信息（按优先级逗号分隔，如 X-Forwarded-For,X-Real-IP,CF-Connecting-IP），留空则直接取网络IP。安全起见需配合 conf/config.yml 的 security.manage_trusted_proxies：仅当直连来源属可信代理时才采信此头（容器/内网部署可直接填 private）", "string", "", configMap) @@ -901,7 +915,11 @@ func TaskLoadSetting(initLoad bool) { // 把落在另一个库里的 IP 标签收回当前归属。按内容判断、源库空即返回， // 所以更早版本切换归属时留下的历史也能在这里被收回来，重复跑无副作用。 + // 合并前先清存量「正常」行（分层后不再记它）：那是表里的大头，清完合并要搬的行数能少几个数量级。 if initLoad { - go waf_service.MergeIPTagsInto(global.GDATA_IP_TAG_DB) + go func() { + waf_service.CleanLegacyBenignIPTags() + waf_service.MergeIPTagsInto(global.GDATA_IP_TAG_DB) + }() } } diff --git a/waftask/task_db_sharding.go b/waftask/task_db_sharding.go index 03d79abc..d2240e2e 100644 --- a/waftask/task_db_sharding.go +++ b/waftask/task_db_sharding.go @@ -32,9 +32,27 @@ func TaskShareDbInfo() { return } - //获取当前日志数量 + // 分层改造的一次性边界：升级后 web_logs 里还躺着改造前的数据，而写入已切到 + // access_log / security_event / event_payload。先把它整体切成一个归档分片， + // 新表从空开始——此后「实时」与「归档」各归各位，旧数据照旧能在归档下拉里读（D6）。 + // 无标记可记：web_logs 不再写入，access_log 一旦有行就说明边界已经切过，天然幂等。 + if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.AccessLogTableName) { + var legacyCnt, accessCnt int64 + global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&legacyCnt) + global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Count(&accessCnt) + if legacyCnt > 0 && accessCnt == 0 { + doLogShardCut(innerLogName, fmt.Sprintf("分层改造边界切换（存量 %d 行转入归档）", legacyCnt), true) + return + } + } + + //获取当前日志数量（分层的写入主体是 access_log；老表不再写入，只作兜底） var total int64 = 0 - global.GWAF_LOCAL_LOG_DB.Table(dialect.Get().ForceIndexClause("web_logs", "idx_tenant_usercode_web_logs")).Count(&total) + if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.AccessLogTableName) { + global.GWAF_LOCAL_LOG_DB.Table(dialect.Get().ForceIndexClause(model.AccessLogTableName, "idx_al_time")).Count(&total) + } else { + global.GWAF_LOCAL_LOG_DB.Table(dialect.Get().ForceIndexClause("web_logs", "idx_tenant_usercode_web_logs")).Count(&total) + } //获取当前数据库文件大小 currentDir := utils.GetCurrentDir() @@ -50,7 +68,7 @@ func TaskShareDbInfo() { shardingReason = fmt.Sprintf("记录数量(%d)超过限制(%d)", total, global.GDATA_SHARE_DB_SIZE) } - // 检查大小是否超过限制：SQLite 用 .db 文件大小，MySQL 用 web_logs 表(数据+索引)大小 + // 检查大小是否超过限制：SQLite 用 .db 文件大小，MySQL/PG 用 access_log 表(数据+索引)大小 if dialect.Get().IsFileBased() { fileInfo, err := os.Stat(dbFilePath) if err == nil { @@ -68,116 +86,143 @@ func TaskShareDbInfo() { zlog.Error(innerLogName, "获取数据库文件大小失败:", err) } } else { - tableSizeMB, err := dialect.Get().TableSizeMB(global.GWAF_LOCAL_LOG_DB, "web_logs") + sizeTable := model.AccessLogTableName + if !dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, sizeTable) { + sizeTable = "web_logs" + } + tableSizeMB, err := dialect.Get().TableSizeMB(global.GWAF_LOCAL_LOG_DB, sizeTable) if err == nil { if tableSizeMB > global.GDATA_SHARE_DB_FILE_SIZE { needSharding = true shardingReason = fmt.Sprintf("表大小(%dMB)超过限制(%dMB)", tableSizeMB, global.GDATA_SHARE_DB_FILE_SIZE) } } else { - zlog.Error(innerLogName, "获取web_logs表大小失败:", err) + zlog.Error(innerLogName, "获取"+sizeTable+"表大小失败:", err) } } if needSharding { - global.GDATA_CURRENT_CHANGE = true - zlog.Info(innerLogName, "开始分库，原因:", shardingReason) - - ts := time.Now().Format("20060102150405") - newDBFilename := fmt.Sprintf("local_log_%v.db", ts) - // 归档标识：SQLite 为新文件名(.db)，MySQL 为归档表名(web_logs_) - archiveName := newDBFilename - if !dialect.Get().IsFileBased() { - archiveName = fmt.Sprintf("web_logs_%v", ts) - } + doLogShardCut(innerLogName, shardingReason, false) + } +} - var lastedDb model.ShareDb - err := global.GWAF_LOCAL_DB.Limit(1).Order("create_time desc").Find(&lastedDb).Error - startTime := customtype.JsonTime(time.Now()) - if err == nil { - startTime = lastedDb.EndTime - } - sharDbBean := model.ShareDb{ - BaseOrm: baseorm.BaseOrm{ - Id: uuid.GenUUID(), - USER_CODE: global.GWAF_USER_CODE, - Tenant_ID: global.GWAF_TENANT_ID, - CREATE_TIME: customtype.JsonTime(time.Now()), - UPDATE_TIME: customtype.JsonTime(time.Now()), - }, - DbLogicType: "log", - StartTime: startTime, - EndTime: customtype.JsonTime(time.Now()), - FileName: archiveName, - Cnt: total, +// doLogShardCut 执行一次日志库切分。swapWebLog=true 只用于分层改造的一次性边界切换 +// （把存量 web_logs + event_payload 切出去）；常规切分换的是三层新表， +// web_logs 不再写入也就无需再换。 +// +// 归档标识：SQLite 为新文件名(.db)，MySQL/PG 为归档表名（边界切换是 web_logs_， +// 常规切分是 access_log_；读侧 ResolveTierTables 按前缀两种都认）。 +// 注意：MySQL/PG 的归档表不再被 gormigrate 跟踪，今后给这些表加列时读旧分片可能缺列—— +// 读侧已按分片实际列取交集（webLogSelect），无需同步 ALTER。 +func doLogShardCut(innerLogName, reason string, swapWebLog bool) { + global.GDATA_CURRENT_CHANGE = true + defer func() { global.GDATA_CURRENT_CHANGE = false }() + zlog.Info(innerLogName, "开始分库，原因:", reason) + + ts := time.Now().Format("20060102150405") + + var total int64 + if swapWebLog { + global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&total) + } else { + global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Count(&total) + } + + newDBFilename := fmt.Sprintf("local_log_%v.db", ts) + archiveName := newDBFilename + if !dialect.Get().IsFileBased() { + if swapWebLog { + archiveName = fmt.Sprintf("web_logs_%v", ts) + } else { + archiveName = fmt.Sprintf("access_log_%v", ts) } + } - zlog.Info(innerLogName, "正在切库中...") - if dialect.Get().IsFileBased() { - // SQLite：关闭连接 → 重命名三个 WAL 文件 → 重建 LogDb - currentDir := utils.GetCurrentDir() - oldDBFilename = currentDir + "/data/" + oldDBFilename - newDBFilename = currentDir + "/data/" + newDBFilename + var lastedDb model.ShareDb + err := global.GWAF_LOCAL_DB.Limit(1).Order("create_time desc").Find(&lastedDb).Error + startTime := customtype.JsonTime(time.Now()) + if err == nil { + startTime = lastedDb.EndTime + } + sharDbBean := model.ShareDb{ + BaseOrm: baseorm.BaseOrm{ + Id: uuid.GenUUID(), + USER_CODE: global.GWAF_USER_CODE, + Tenant_ID: global.GWAF_TENANT_ID, + CREATE_TIME: customtype.JsonTime(time.Now()), + UPDATE_TIME: customtype.JsonTime(time.Now()), + }, + DbLogicType: "log", + StartTime: startTime, + EndTime: customtype.JsonTime(time.Now()), + FileName: archiveName, + Cnt: total, + } - sqlDB, err := global.GWAF_LOCAL_LOG_DB.DB() - if err != nil { + zlog.Info(innerLogName, "正在切库中...") + if dialect.Get().IsFileBased() { + // SQLite：关闭连接 → 重命名三个 WAL 文件 → 重建 LogDb + currentDir := utils.GetCurrentDir() + oldPath := currentDir + "/data/local_log.db" + newPath := currentDir + "/data/" + newDBFilename + + sqlDB, err := global.GWAF_LOCAL_LOG_DB.DB() + if err != nil { + zlog.Error(innerLogName, "切换关闭时候错误", err) + } else { + if err := sqlDB.Close(); err != nil { zlog.Error(innerLogName, "切换关闭时候错误", err) - } else { - if err := sqlDB.Close(); err != nil { - zlog.Error(innerLogName, "切换关闭时候错误", err) - } } - // 等待连接彻底关闭（Count 报错即连接已关闭，可安全重命名文件）。 - // 加最大重试上限，避免连接异常未报错时无限循环卡住切库（高频切库后该段执行更频繁）。 - var testTotal int64 - for attempt := 0; attempt < 10; attempt++ { - testError := global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&testTotal).Error - if testError != nil { - zlog.Debug(innerLogName, "连接已关闭，可切库", testError) - break - } - if attempt == 9 { - zlog.Warn(innerLogName, "等待日志库连接关闭超时，强制继续切库") - break - } - time.Sleep(1 * time.Second) + } + // 等待连接彻底关闭（Count 报错即连接已关闭，可安全重命名文件）。 + // 加最大重试上限，避免连接异常未报错时无限循环卡住切库（高频切库后该段执行更频繁）。 + var testTotal int64 + for attempt := 0; attempt < 10; attempt++ { + testError := global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&testTotal).Error + if testError != nil { + zlog.Debug(innerLogName, "连接已关闭，可切库", testError) + break } - - if err := os.Rename(oldDBFilename, newDBFilename); err != nil { - zlog.Error(innerLogName, "Error renaming database file:", err) + if attempt == 9 { + zlog.Warn(innerLogName, "等待日志库连接关闭超时，强制继续切库") + break } - if err := os.Rename(oldDBFilename+"-shm", newDBFilename+"-shm"); err != nil { - zlog.Error(innerLogName, "Error renaming .db-shm file:", err) + time.Sleep(1 * time.Second) + } + + if err := os.Rename(oldPath, newPath); err != nil { + zlog.Error(innerLogName, "Error renaming database file:", err) + } + if err := os.Rename(oldPath+"-shm", newPath+"-shm"); err != nil { + zlog.Error(innerLogName, "Error renaming .db-shm file:", err) + } + if err := os.Rename(oldPath+"-wal", newPath+"-wal"); err != nil { + zlog.Error(innerLogName, "Error renaming .db-wal file:", err) + } + global.GWAF_LOCAL_DB.Create(sharDbBean) + global.GWAF_LOCAL_LOG_DB = nil + wafdb.InitLogDb("") + } else { + // MySQL/PG：CREATE TABLE LIKE + 单语句原子 RENAME 换表，换表期间无写入空窗。 + // 任一张换失败不回滚：报文按 req_uuid 寻址、窄行按时间查，留在实时表里读侧照样找得到。 + swap := func(table string) { + if !dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, table) { + return } - if err := os.Rename(oldDBFilename+"-wal", newDBFilename+"-wal"); err != nil { - zlog.Error(innerLogName, "Error renaming .db-wal file:", err) + if err := dialect.Get().ShardSwapTable(global.GWAF_LOCAL_LOG_DB, table, table+"_"+ts); err != nil { + zlog.Warn(innerLogName, "分表失败，数据留在实时表:", "table", table, "error", err) } - global.GWAF_LOCAL_DB.Create(sharDbBean) - global.GWAF_LOCAL_LOG_DB = nil - wafdb.InitLogDb("") + } + if swapWebLog { + swap("web_logs") + swap(model.EventPayloadTableName) } else { - // MySQL：CREATE TABLE LIKE + 单语句原子 RENAME 换表。 - // 把 web_logs 归档为 archiveName 并重建同结构空表，换表期间无写入空窗， - // 同一连接、表已重建为空，无需像 SQLite 那样置 nil 重连。 - // 注意：归档表(web_logs_)不再被 gormigrate 跟踪，今后给 web_logs 加列时 - // 需同步 ALTER 历史分表，否则读旧分片可能缺列（见 ResolveLogDB 注释）。 - if err := dialect.Get().ShardSwapTable(global.GWAF_LOCAL_LOG_DB, "web_logs", archiveName); err != nil { - zlog.Error(innerLogName, "分表失败:", err) - } else { - // 报文表跟着日志表一起归档，两张表的分片边界才对得上。 - // 换不过去也不回滚：报文按 req_uuid 寻址，留在实时表里读侧照样找得到（见 ResolveLogTables）。 - payloadArchive := model.EventPayloadTableName + fmt.Sprintf("_%v", ts) - if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.EventPayloadTableName) { - if err := dialect.Get().ShardSwapTable(global.GWAF_LOCAL_LOG_DB, model.EventPayloadTableName, payloadArchive); err != nil { - zlog.Warn(innerLogName, "报文表分表失败，报文留在实时表:", err) - } - } - global.GWAF_LOCAL_DB.Create(sharDbBean) - zlog.Info(innerLogName, "分表完成，归档表:", archiveName) - } + swap(model.AccessLogTableName) + swap(model.SecurityEventTableName) + swap(model.EventPayloadTableName) } - global.GDATA_CURRENT_CHANGE = false - zlog.Info(innerLogName, "切库完成...") + global.GWAF_LOCAL_DB.Create(sharDbBean) + zlog.Info(innerLogName, "分表完成，归档表:", archiveName) } - + zlog.Info(innerLogName, "切库完成...") } diff --git a/waftask/task_history.go b/waftask/task_history.go index 447b5e9b..8f6af386 100644 --- a/waftask/task_history.go +++ b/waftask/task_history.go @@ -13,8 +13,10 @@ import ( // TaskDeleteHistoryInfo 定时删除指定历史信息 通过开关操作 func TaskDeleteHistoryInfo() { zlog.Debug("TaskDeleteHistoryInfo") + // 安全事件与存量 web_logs 走「日志保留天数」，access_log 走自己的（更短的）保留期 deleteBeforeDay := time.Now().AddDate(0, 0, -int(global.GDATA_DELETE_INTERVAL)).Format("2006-01-02 15:04") - waf_service.WafLogServiceApp.DeleteHistory(deleteBeforeDay) + accessBeforeDay := time.Now().AddDate(0, 0, -int(global.GDATA_ACCESS_LOG_RETENTION_DAYS)).Format("2006-01-02 15:04") + waf_service.WafLogServiceApp.DeleteHistory(deleteBeforeDay, accessBeforeDay) // 清理过期的归档分片文件（高频切库后 live 库只存最近数据，真正的保留期回收靠删归档文件） CleanExpiredArchiveShard() diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index 089695ab..696b8d58 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -695,3 +695,42 @@ notes: longer happens. The page shows a merging indicator and refreshes itself when it finishes. New installations store tags in the stats database by default; existing installations keep their current setting. + + - id: v1_3_25_log_tiered_storage + version: v1.3.25 + kind: notice + level: normal + page: '' + doc: https://doc.samwaf.com/quickstart/Update.html + apply: + type: none + zh: + title: 日志分层存储：访问窄行与安全事件分表，新增观察名单，日志导出改按时间段 + detail: >- + 日志现在按用途分层存放：命中规则或被拦截的请求进「安全事件」表并保留完整报文（随日志保留天数）； + 所有被记录的请求另写一行窄行进「访问日志」表（默认保留 30 天，可在日志配置里调）。 + 默认档位「全部入库」，升级后行为与之前一致；流量很大的站点可在日志配置里把「访问日志档位」调成 + 「仅安全事件」或「采样入库」，但这样会失去 CC 阈值推荐、AI 训练负样本与异常 IP 的正常行为回溯， + CC 阈值推荐界面在「仅安全事件」档会说明不可用原因。 + IP 标签不再为每个 IP 记「正常」标签，放行榜的次数改由统计数据给出，标签列表更干净。 + 风险日志页可以把可疑 IP「加入观察」：观察期内该 IP 的所有请求带完整报文留痕（默认 7 天）。 + 「导出日志库」不再是整库拷贝，改为按时间段导出选定层（访问日志/安全事件/报文/历史库），产物仍是一个加密 SQLite 文件。 + 升级后首次启动自动建表，历史数据不动、照旧可查。 + en: + title: Tiered log storage, an IP watchlist, and time-ranged log export + detail: >- + Logs are now stored in tiers by purpose: requests that hit a rule or get blocked go to the security-event + table with their full payload (kept for the log retention period); every recorded request also writes a + narrow row to the access-log table (kept 30 days by default, adjustable in log settings). + The default mode "store everything" behaves exactly as before. Very busy sites can switch the access-log + mode to "security events only" or "sampled" in log settings, at the cost of CC threshold + recommendations, AI training negative samples and the ability to look back at a suspicious IP's normal + traffic; the CC recommendation panel explains why it is unavailable in "security events only" mode. + IP tags no longer record a "normal" tag for every IP - pass counts on the pass ranking now come from + statistics data, keeping the tag list clean. + The risk log page can put a suspicious IP under observation: while observed, every request from that IP + is captured with its full payload (7 days by default). + "Export log database" no longer copies the whole database; it exports a chosen time range and tiers + (access log / security events / payloads / legacy) into a fresh encrypted SQLite file. + Tables are created on the first start after upgrading; existing data is untouched and stays readable. + From ad583fc300664843ad1101406f7472c4e465ae4e Mon Sep 17 00:00:00 2001 From: samwaf Date: Sun, 20 Sep 2026 16:24:08 +0800 Subject: [PATCH 10/21] feat: roll up requests per actor and path into daily analysis tables --- model/stats_analysis.go | 84 ++++++++ wafdb/migrations_core.go | 66 ++++++ wafdb/migrations_stats.go | 65 ++++++ wafqueue/log_keys.go | 37 +++- wafqueue/log_queue.go | 3 + waftask/retention_policy_seed_test.go | 71 +++++++ waftask/stat_analysis_collector.go | 265 ++++++++++++++++++++++++ waftask/stat_analysis_collector_test.go | 201 ++++++++++++++++++ waftask/stat_analysis_crossdb_test.go | 190 +++++++++++++++++ waftask/task_stats_cleanup.go | 6 +- 10 files changed, 982 insertions(+), 6 deletions(-) create mode 100644 model/stats_analysis.go create mode 100644 waftask/retention_policy_seed_test.go create mode 100644 waftask/stat_analysis_collector.go create mode 100644 waftask/stat_analysis_collector_test.go create mode 100644 waftask/stat_analysis_crossdb_test.go diff --git a/model/stats_analysis.go b/model/stats_analysis.go new file mode 100644 index 00000000..2dad2f27 --- /dev/null +++ b/model/stats_analysis.go @@ -0,0 +1,84 @@ +package model + +import "SamWaf/model/baseorm" + +// M5 分析层的天级汇总，喂三个视角：行为（谁）/ 目标（打哪）/ 手法（怎么打）。 +// +// 为什么去重数不做成列：去重计数没法跨批次累加——同一个 IP 在两个批次里各出现一次， +// 各 +1 就成了 2。所以这里只存**可累加的计数器**，去重数留给读侧现算： +// +// GROUP BY path_norm → 目标视角：SUM(req_cnt) / SUM(deny_cnt) / COUNT(DISTINCT actor_key)=去重 IP 数 +// GROUP BY actor_key → 行为视角：SUM(req_cnt) / SUM(deny_cnt) / COUNT(DISTINCT path_norm)=去重路径数 +// +// 一张细粒度表同时喂两个视角，数字精确，机制与 stats_ip_days 完全同款 +// （纯计数器：累加安全、重启安全、重复入队安全）。 + +// StatsPathNormLen 汇总表里路径模板的列宽，比窄行的 512 短。 +// 这几列都要进唯一索引，PostgreSQL 的 btree 单行上限约 2704 字节，得留余量； +// NormalizePath 已把超 32 字符的段归一成 {s}，正常模板远短于此。 +const StatsPathNormLen = 200 + +// StatsRuleLen 汇总表里规则名的列宽。日志里 RULE 是 text，做键要有界。 +const StatsRuleLen = 128 + +// StatsActorPathDay 「谁 打了 哪个路径」的天级计数，两个分析视角共同的地基。 +type StatsActorPathDay struct { + baseorm.BaseOrm + HostCode string `gorm:"size:64" json:"host_code"` //网站唯一码（主要键） + Day int `json:"day"` //年月日（主要键）如 20260919 + ActorKey string `gorm:"size:64" json:"actor_key"` //访问者身份，目前是 ip:+来源IP（主要键） + PathNorm string `gorm:"size:200" json:"path_norm"` //路径模板（主要键） + ReqCnt int64 `json:"req_cnt"` //请求数 + DenyCnt int64 `json:"deny_cnt"` //被拦截数 + // Err4xxCnt 只数后端给的 4xx，不含 WAF 拦截页（也是 403）： + // 混在一起就分不出「扫目录扫出一堆 404」和「被 WAF 挡了一堆」这两件事。 + Err4xxCnt int64 `gorm:"column:err4xx_cnt" json:"err4xx_cnt"` +} + +func (StatsActorPathDay) TableName() string { + return "stats_actor_path_days" +} + +// StatsActorUaDay 「谁 用了哪个 UA 指纹」的天级计数，ua_cnt = 按 actor 数行数。 +// 单独一张表而不并进上表：UA 与路径正交，合并会让行数变成两者的乘积。 +type StatsActorUaDay struct { + baseorm.BaseOrm + HostCode string `gorm:"size:64" json:"host_code"` //网站唯一码（主要键） + Day int `json:"day"` //年月日（主要键） + ActorKey string `gorm:"size:64" json:"actor_key"` //访问者身份（主要键） + UaHash string `gorm:"size:64" json:"ua_hash"` //UA 指纹（主要键） + Cnt int64 `json:"cnt"` //数量 +} + +func (StatsActorUaDay) TableName() string { + return "stats_actor_ua_days" +} + +// StatsPathRuleDay 「哪个路径 命中了哪条规则」的天级计数。 +// 存整个分布而不是只存一个 top_rule：写入成本一样，读侧既能取 top 也能看构成。 +type StatsPathRuleDay struct { + baseorm.BaseOrm + HostCode string `gorm:"size:64" json:"host_code"` //网站唯一码（主要键） + Day int `json:"day"` //年月日（主要键） + PathNorm string `gorm:"size:200" json:"path_norm"` //路径模板（主要键） + Rule string `gorm:"size:128" json:"rule"` //命中的规则（主要键） + Cnt int64 `json:"cnt"` //数量 +} + +func (StatsPathRuleDay) TableName() string { + return "stats_path_rule_days" +} + +// LogAnalysisRow 一条请求供分析层聚合用的最小投影。 +// 三个键由写入侧（wafqueue）用与窄行同一套函数算好传进来，汇总口径因此不会和明细漂移； +// 也避开了 waftask 反向 import wafqueue 的环（日志队列正在调 waftask）。 +type LogAnalysisRow struct { + HostCode string + Day int + ActorKey string + PathNorm string + UaHash string + Rule string + Action string + StatusCode int +} diff --git a/wafdb/migrations_core.go b/wafdb/migrations_core.go index cc4afda2..a60bd898 100644 --- a/wafdb/migrations_core.go +++ b/wafdb/migrations_core.go @@ -864,6 +864,72 @@ func RunCoreDBMigrations(db *gorm.DB) error { return tx.Migrator().DropTable(&model.DataRetentionPolicy{}) }, }, + // 迁移: 给分析层三张汇总表预置保留策略 + // 保留策略页只能改不能加（策略由系统预置），所以新表必须在这里补一行， + // 否则它既不会出现在页面上、也永远不会被清理任务碰到——只进 allowedCleanupTables 不够。 + { + ID: "202609200001_add_analysis_retention_policies", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609200001: 预置分析层汇总表的保留策略") + if err := tx.AutoMigrate(&model.DataRetentionPolicy{}); err != nil { + return fmt.Errorf("同步 data_retention_policies 表失败: %w", err) + } + + newPolicy := func(table string, rows int64, remarks string) model.DataRetentionPolicy { + return model.DataRetentionPolicy{ + BaseOrm: baseorm.BaseOrm{ + Id: uuid.GenUUID(), + USER_CODE: global.GWAF_USER_CODE, + Tenant_ID: global.GWAF_TENANT_ID, + CREATE_TIME: customtype.JsonTime(time.Now()), + UPDATE_TIME: customtype.JsonTime(time.Now()), + }, + TableName: table, + DbType: "stats", + RetainDays: 90, + RetainRows: rows, + DayField: "day", + DayFieldType: "int_day", + RowOrderField: "day", + RowOrderDir: "DESC", + // 与现有三条策略一致，默认禁用：清理是删数据，不替用户做这个决定。 + // 行数上限只是失控兜底，真正的日常过期要用户在页面上启用。 + CleanEnabled: 0, + Remarks: remarks, + } + } + policies := []model.DataRetentionPolicy{ + // 行数随「站点×天的 (IP,路径) 去重对数」增长，是分析层最大的一张 + newPolicy("stats_actor_path_days", 1000000, + "行为×目标日汇总-按day字段判断天数,保留day值最大(最新)的行"), + newPolicy("stats_actor_ua_days", 200000, + "行为×UA日汇总-按day字段判断天数,保留day值最大(最新)的行"), + newPolicy("stats_path_rule_days", 200000, + "目标×规则日汇总-按day字段判断天数,保留day值最大(最新)的行"), + } + + for _, policy := range policies { + var count int64 + tx.Model(&model.DataRetentionPolicy{}).Where("table_name = ?", policy.TableName).Count(&count) + if count > 0 { + zlog.Debug("保留策略已存在，跳过", "table", policy.TableName) + continue + } + if err := tx.Create(&policy).Error; err != nil { + return fmt.Errorf("初始化策略 %s 失败: %w", policy.TableName, err) + } + zlog.Info("分析层保留策略已创建", "table", policy.TableName) + } + zlog.Info("迁移 202609200001: 完成") + return nil + }, + Rollback: func(tx *gorm.DB) error { + zlog.Info("回滚 202609200001: 删除分析层保留策略") + return tx.Where("table_name in ?", []string{ + "stats_actor_path_days", "stats_actor_ua_days", "stats_path_rule_days", + }).Delete(&model.DataRetentionPolicy{}).Error + }, + }, // 迁移: 创建路径路由规则表 { ID: "202605130001_add_host_path_rules_table", diff --git a/wafdb/migrations_stats.go b/wafdb/migrations_stats.go index 44486557..523b5cbf 100644 --- a/wafdb/migrations_stats.go +++ b/wafdb/migrations_stats.go @@ -235,6 +235,71 @@ func RunStatsDBMigrations(db *gorm.DB) error { return nil }, }, + // 迁移6: 分析层天级汇总表（行为「谁」/ 目标「打哪」/ 手法「怎么打」三视角的地基） + { + ID: "202609190001_create_analysis_stats_tables", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609190001: 创建分析层汇总表") + if err := tx.AutoMigrate( + &model.StatsActorPathDay{}, + &model.StatsActorUaDay{}, + &model.StatsPathRuleDay{}, + ); err != nil { + return fmt.Errorf("创建分析层汇总表失败: %w", err) + } + + ddlDB := tx.Session(&gorm.Session{ + Logger: NewGormZLogger().LogMode(logger.Silent), + }) + indexes := []struct{ name, table, sql string }{ + // 唯一索引即 upsert 的冲突目标，列序与 analysisConflictCols 一致 + {"uni_stats_actor_path_days", "stats_actor_path_days", + "CREATE UNIQUE INDEX IF NOT EXISTS uni_stats_actor_path_days ON stats_actor_path_days (user_code, tenant_id, host_code, day, actor_key, path_norm)"}, + // 目标视角：按站点+天 GROUP BY path_norm + {"idx_stats_actor_path_days_path", "stats_actor_path_days", + "CREATE INDEX IF NOT EXISTS idx_stats_actor_path_days_path ON stats_actor_path_days (host_code, day, path_norm)"}, + // 行为视角：按站点+天 GROUP BY actor_key + {"idx_stats_actor_path_days_actor", "stats_actor_path_days", + "CREATE INDEX IF NOT EXISTS idx_stats_actor_path_days_actor ON stats_actor_path_days (host_code, day, actor_key)"}, + // 保留策略按天删 + {"idx_stats_actor_path_days_day", "stats_actor_path_days", + "CREATE INDEX IF NOT EXISTS idx_stats_actor_path_days_day ON stats_actor_path_days (day)"}, + + {"uni_stats_actor_ua_days", "stats_actor_ua_days", + "CREATE UNIQUE INDEX IF NOT EXISTS uni_stats_actor_ua_days ON stats_actor_ua_days (user_code, tenant_id, host_code, day, actor_key, ua_hash)"}, + {"idx_stats_actor_ua_days_day", "stats_actor_ua_days", + "CREATE INDEX IF NOT EXISTS idx_stats_actor_ua_days_day ON stats_actor_ua_days (day)"}, + + {"uni_stats_path_rule_days", "stats_path_rule_days", + "CREATE UNIQUE INDEX IF NOT EXISTS uni_stats_path_rule_days ON stats_path_rule_days (user_code, tenant_id, host_code, day, path_norm, rule)"}, + {"idx_stats_path_rule_days_day", "stats_path_rule_days", + "CREATE INDEX IF NOT EXISTS idx_stats_path_rule_days_day ON stats_path_rule_days (day)"}, + } + for _, idx := range indexes { + if err := safeCreateIndex(ddlDB, idx.table, idx.name, idx.sql); err != nil { + return fmt.Errorf("创建分析层索引 %s 失败: %w", idx.name, err) + } + } + zlog.Info("迁移 202609190001: 分析层汇总表与索引创建完成") + return nil + }, + Rollback: func(tx *gorm.DB) error { + zlog.Info("回滚 202609190001: 删除分析层汇总表") + for _, name := range []string{ + "uni_stats_actor_path_days", "idx_stats_actor_path_days_path", + "idx_stats_actor_path_days_actor", "idx_stats_actor_path_days_day", + "uni_stats_actor_ua_days", "idx_stats_actor_ua_days_day", + "uni_stats_path_rule_days", "idx_stats_path_rule_days_day", + } { + _ = tx.Exec("DROP INDEX IF EXISTS " + name).Error + } + return tx.Migrator().DropTable( + &model.StatsActorPathDay{}, + &model.StatsActorUaDay{}, + &model.StatsPathRuleDay{}, + ) + }, + }, }) // 执行迁移 diff --git a/wafqueue/log_keys.go b/wafqueue/log_keys.go index 45029832..f5e984a3 100644 --- a/wafqueue/log_keys.go +++ b/wafqueue/log_keys.go @@ -1,6 +1,8 @@ package wafqueue import ( + "SamWaf/innerbean" + "SamWaf/model" "crypto/sha1" "encoding/hex" "strings" @@ -10,11 +12,11 @@ import ( // 归一化规则一改，历史数据与新数据就不可比（汇总失真），所以规则由 log_keys_test.go 钉死： // 改这里的判定必须连测试一起改，并在提交记录里说明。 -// ActorKey 访问者身份：有访客身份识别码就用它（换 IP 也认得出来），否则退回 IP。 -func ActorKey(guestID, ip string) string { - if guestID != "" { - return "g:" + guestID - } +// ActorKey 访问者身份：只有 IP 可用。 +// 注意不能拿 GUEST_IDENTIFICATION 当身份——它是引擎写的分类标签 +// （正常访客/可疑用户/bot 名/触发敏感词…），全站只有几个取值， +// 当键会把所有正常访客聚成一个 actor。分类聚合直接查窄行的 guest_id_entification 列。 +func ActorKey(ip string) string { if ip != "" { return "ip:" + ip } @@ -131,3 +133,28 @@ func isUUID(s string) bool { } return true } + +// AnalysisRowsFromLogs 把一批日志投影成分析层要的最小行，三个键就地算好。 +// 键在这边算是刻意的：waftask 不能反向 import wafqueue（日志队列正在调它，会成环）， +// 而键又必须和落进 access_log / security_event 的列出自同一套函数，否则汇总口径会和明细漂移。 +// +// 不看档位：明细写多少由 tierForStore 决定，汇总永远吃全量请求流。 +func AnalysisRowsFromLogs(logs []*innerbean.WebLog) []model.LogAnalysisRow { + rows := make([]model.LogAnalysisRow, 0, len(logs)) + for _, lg := range logs { + if lg == nil { + continue + } + rows = append(rows, model.LogAnalysisRow{ + HostCode: lg.HOST_CODE, + Day: lg.Day, + ActorKey: ActorKey(lg.SRC_IP), + PathNorm: NormalizePath(lg.URL), + UaHash: UaHash(lg.USER_AGENT), + Rule: lg.RULE, + Action: lg.ACTION, + StatusCode: lg.STATUS_CODE, + }) + } + return rows +} diff --git a/wafqueue/log_queue.go b/wafqueue/log_queue.go index 1a0b0e6b..06426218 100644 --- a/wafqueue/log_queue.go +++ b/wafqueue/log_queue.go @@ -70,6 +70,9 @@ func ProcessLogDequeEngine() { } // 日志流做统计 waftask.CollectStatsFromLogs(webLogArray) + // 分析层天级汇总（行为 / 目标 / 手法三视角）。键在本包算好再传过去： + // waftask 不能反向 import 本包，而键必须与窄行同源。 + waftask.CollectAnalysisStats(AnalysisRowsFromLogs(webLogArray)) global.GNOTIFY_KAKFA_SERVICE.ProcessBatchLogs(webLogArray) // 文件日志写入 global.GNOTIFY_LOG_FILE_WRITER.ProcessBatchLogs(webLogArray) diff --git a/waftask/retention_policy_seed_test.go b/waftask/retention_policy_seed_test.go new file mode 100644 index 00000000..03c3c2d5 --- /dev/null +++ b/waftask/retention_policy_seed_test.go @@ -0,0 +1,71 @@ +package waftask + +import ( + "SamWaf/model" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "net/url" + "path/filepath" + "sort" + "testing" + + sqlitedriver "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// 保留策略页只能改不能加（策略由系统预置），所以「把表加进 allowedCleanupTables」 +// 只是拿到了清理许可，真正要它被清理还得在 migrations_core 里预置一行策略。 +// 两处漏掉任意一处，表就会安静地一直长——页面上根本看不到它。 +// 这条用例把两边钉在一起：白名单里的每张表都必须有预置策略。 +func TestEveryCleanupTableHasSeededPolicy(t *testing.T) { + dialect.Register(&dialect.SQLiteDialect{}) + dsn := filepath.Join(t.TempDir(), "core.db") + "?_db_key=" + url.QueryEscape("kcore") + db, err := gorm.Open(sqlitedriver.Open(dsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + t.Skipf("打不开 sqlite（缺 CGO？），跳过: %v", err) + } + t.Cleanup(func() { + if s, e := db.DB(); e == nil { + s.Close() + } + }) + if err := wafdb.RunCoreDBMigrations(db); err != nil { + t.Fatalf("core 迁移失败: %v", err) + } + + var policies []model.DataRetentionPolicy + if err := db.Find(&policies).Error; err != nil { + t.Fatalf("读保留策略失败: %v", err) + } + seeded := map[string]model.DataRetentionPolicy{} + for _, p := range policies { + seeded[p.TableName] = p + } + + var missing []string + for table := range allowedCleanupTables { + if _, ok := seeded[table]; !ok { + missing = append(missing, table) + } + } + if len(missing) > 0 { + sort.Strings(missing) + t.Fatalf("这些表在 allowedCleanupTables 里但没有预置策略，页面上加不了、也永远不会被清理：%v；"+ + "修法：在 wafdb/migrations_core.go 加一条迁移预置它们的 DataRetentionPolicy", missing) + } + + // 反向也查一遍：预置了策略却不在白名单，清理任务会因标识符校验失败跳过，同样是空转 + var orphan []string + for table := range seeded { + if _, ok := allowedCleanupTables[table]; !ok { + orphan = append(orphan, table) + } + } + if len(orphan) > 0 { + sort.Strings(orphan) + t.Fatalf("这些表预置了策略却不在 allowedCleanupTables，清理时会被标识符校验挡掉：%v", orphan) + } +} diff --git a/waftask/stat_analysis_collector.go b/waftask/stat_analysis_collector.go new file mode 100644 index 00000000..4070aaf9 --- /dev/null +++ b/waftask/stat_analysis_collector.go @@ -0,0 +1,265 @@ +package waftask + +import ( + "SamWaf/common/uuid" + "SamWaf/common/zlog" + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/wafdb/dialect" + "sync" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const ( + // pathTemplatePerSiteDay 每站每天最多认多少个路径模板（D10）。 + // 字典爆破会造出成千上万个互不相同的字面路径（/admin、/.env、/wp-login…）， + // NormalizePath 归一不掉它们；没有这道闸，汇总表会被扫目录打爆。 + pathTemplatePerSiteDay = 5000 + // pathOverflowTemplate 超出上限之后所有没见过的模板的去处。 + pathOverflowTemplate = "{overflow}" + // analysisUpsertBatch 单条 upsert 语句最多带多少行，同 ipTagUpsertBatch 的考量。 + analysisUpsertBatch = 200 +) + +// pathGate 记住每个「站点×天」已认下的路径模板，给 D10 的上限把门。 +// 只保留最近两天的桶（跨零点时新旧各一个），重启会清空—— +// 上限是防爆的闸不是精确不变式，重启后这一天最多再放行一批新模板，可以接受。 +type pathGate struct { + mu sync.Mutex + seen map[int]map[string]map[string]struct{} // day → host_code → 模板集合 +} + +var analysisPathGate = &pathGate{seen: map[int]map[string]map[string]struct{}{}} + +// admit 返回这个模板在汇总里该记成什么：认识的原样返回， +// 没见过且没到上限就收下，到了上限就归进 {overflow}。 +func (g *pathGate) admit(hostCode string, day int, path string) string { + if path == "" { + return path + } + g.mu.Lock() + defer g.mu.Unlock() + + dayBucket := g.seen[day] + if dayBucket == nil { + dayBucket = map[string]map[string]struct{}{} + g.seen[day] = dayBucket + g.pruneDaysLocked(day) + } + bucket := dayBucket[hostCode] + if bucket == nil { + bucket = make(map[string]struct{}, 256) + dayBucket[hostCode] = bucket + } + if _, ok := bucket[path]; ok { + return path + } + if len(bucket) >= pathTemplatePerSiteDay { + return pathOverflowTemplate + } + bucket[path] = struct{}{} + return path +} + +// pruneDaysLocked 只留 keep 和比它小的那一天，其余丢掉。调用方须持锁。 +func (g *pathGate) pruneDaysLocked(keep int) { + for len(g.seen) > 2 { + oldest := 0 + for d := range g.seen { + if oldest == 0 || d < oldest { + oldest = d + } + } + if oldest == keep || oldest == 0 { + return + } + delete(g.seen, oldest) + } +} + +// CollectAnalysisStats 把一批请求聚合进分析层的三张天级汇总表。 +// +// 与档位正交：吃的是全量请求流，access_log_mode 只决定明细写多少，不影响汇总口径—— +// 这是 stats_* 今天就有的性质（统计不依赖日志入库），不能丢。 +func CollectAnalysisStats(rows []model.LogAnalysisRow) { + if len(rows) == 0 || global.GWAF_LOCAL_STATS_DB == nil { + return + } + + type actorPathKey struct { + HostCode string + Day int + ActorKey string + PathNorm string + } + type actorUaKey struct { + HostCode string + Day int + ActorKey string + UaHash string + } + type pathRuleKey struct { + HostCode string + Day int + PathNorm string + Rule string + } + type actorPathVal struct { + Req int64 + Deny int64 + Err4xx int64 + } + + actorPathAgg := make(map[actorPathKey]*actorPathVal) + actorUaAgg := make(map[actorUaKey]int64) + pathRuleAgg := make(map[pathRuleKey]int64) + + for _, r := range rows { + if r.HostCode == "" || r.ActorKey == "" { + continue + } + path := analysisPathGate.admit(r.HostCode, r.Day, cutRunes(r.PathNorm, model.StatsPathNormLen)) + + apk := actorPathKey{HostCode: r.HostCode, Day: r.Day, ActorKey: r.ActorKey, PathNorm: path} + v := actorPathAgg[apk] + if v == nil { + v = &actorPathVal{} + actorPathAgg[apk] = v + } + v.Req++ + if r.Action == "阻止" { + v.Deny++ + } else if r.StatusCode >= 400 && r.StatusCode < 500 { + v.Err4xx++ + } + + if r.UaHash != "" { + actorUaAgg[actorUaKey{HostCode: r.HostCode, Day: r.Day, ActorKey: r.ActorKey, UaHash: r.UaHash}]++ + } + if r.Rule != "" { + pathRuleAgg[pathRuleKey{HostCode: r.HostCode, Day: r.Day, PathNorm: path, Rule: cutRunes(r.Rule, model.StatsRuleLen)}]++ + } + } + + now := customtype.JsonTime(time.Now()) + db := global.GWAF_LOCAL_STATS_DB + inc := dialect.Get().UpsertExcludedRef + + if len(actorPathAgg) > 0 { + batch := make([]model.StatsActorPathDay, 0, len(actorPathAgg)) + for k, v := range actorPathAgg { + batch = append(batch, model.StatsActorPathDay{ + BaseOrm: newStatsBaseOrm(now), + HostCode: k.HostCode, + Day: k.Day, + ActorKey: k.ActorKey, + PathNorm: k.PathNorm, + ReqCnt: v.Req, + DenyCnt: v.Deny, + Err4xxCnt: v.Err4xx, + }) + } + err := db.Clauses(clause.OnConflict{ + Columns: analysisConflictCols("host_code", "day", "actor_key", "path_norm"), + DoUpdates: clause.Assignments(map[string]interface{}{ + "req_cnt": gorm.Expr("stats_actor_path_days.req_cnt + " + inc("req_cnt")), + "deny_cnt": gorm.Expr("stats_actor_path_days.deny_cnt + " + inc("deny_cnt")), + "err4xx_cnt": gorm.Expr("stats_actor_path_days.err4xx_cnt + " + inc("err4xx_cnt")), + "update_time": now, + }), + }).CreateInBatches(batch, analysisUpsertBatch).Error + if err != nil { + zlog.Debug("分析层 行为×目标 聚合写入失败", "错误", err.Error(), "条数", len(batch)) + } + } + + if len(actorUaAgg) > 0 { + batch := make([]model.StatsActorUaDay, 0, len(actorUaAgg)) + for k, delta := range actorUaAgg { + batch = append(batch, model.StatsActorUaDay{ + BaseOrm: newStatsBaseOrm(now), + HostCode: k.HostCode, + Day: k.Day, + ActorKey: k.ActorKey, + UaHash: k.UaHash, + Cnt: delta, + }) + } + err := db.Clauses(clause.OnConflict{ + Columns: analysisConflictCols("host_code", "day", "actor_key", "ua_hash"), + DoUpdates: clause.Assignments(map[string]interface{}{ + "cnt": gorm.Expr("stats_actor_ua_days.cnt + " + inc("cnt")), + "update_time": now, + }), + }).CreateInBatches(batch, analysisUpsertBatch).Error + if err != nil { + zlog.Debug("分析层 行为×UA 聚合写入失败", "错误", err.Error(), "条数", len(batch)) + } + } + + if len(pathRuleAgg) > 0 { + batch := make([]model.StatsPathRuleDay, 0, len(pathRuleAgg)) + for k, delta := range pathRuleAgg { + batch = append(batch, model.StatsPathRuleDay{ + BaseOrm: newStatsBaseOrm(now), + HostCode: k.HostCode, + Day: k.Day, + PathNorm: k.PathNorm, + Rule: k.Rule, + Cnt: delta, + }) + } + err := db.Clauses(clause.OnConflict{ + Columns: analysisConflictCols("host_code", "day", "path_norm", "rule"), + DoUpdates: clause.Assignments(map[string]interface{}{ + "cnt": gorm.Expr("stats_path_rule_days.cnt + " + inc("cnt")), + "update_time": now, + }), + }).CreateInBatches(batch, analysisUpsertBatch).Error + if err != nil { + zlog.Debug("分析层 目标×规则 聚合写入失败", "错误", err.Error(), "条数", len(batch)) + } + } + + zlog.Debug("分析层聚合完成", + "行为×目标", len(actorPathAgg), "行为×UA", len(actorUaAgg), "目标×规则", len(pathRuleAgg)) +} + +// analysisConflictCols 唯一索引以 user_code/tenant_id 打头（与 ip_tags 同款），冲突目标要逐列对齐。 +func analysisConflictCols(cols ...string) []clause.Column { + out := []clause.Column{{Name: "user_code"}, {Name: "tenant_id"}} + for _, c := range cols { + out = append(out, clause.Column{Name: c}) + } + return out +} + +func newStatsBaseOrm(now customtype.JsonTime) baseorm.BaseOrm { + return baseorm.BaseOrm{ + Id: uuid.GenUUID(), + USER_CODE: global.GWAF_USER_CODE, + Tenant_ID: global.GWAF_TENANT_ID, + CREATE_TIME: now, + UPDATE_TIME: now, + } +} + +// cutRunes 按字符截断（列宽是字符数，不是字节数）。 +func cutRunes(s string, max int) string { + if max <= 0 || s == "" { + return s + } + n := 0 + for i := range s { + n++ + if n > max { + return s[:i] + } + } + return s +} diff --git a/waftask/stat_analysis_collector_test.go b/waftask/stat_analysis_collector_test.go new file mode 100644 index 00000000..de0fb549 --- /dev/null +++ b/waftask/stat_analysis_collector_test.go @@ -0,0 +1,201 @@ +package waftask + +import ( + "SamWaf/global" + "SamWaf/model" + "SamWaf/wafdb/dialect" + "net/url" + "path/filepath" + "testing" + + sqlitedriver "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func TestPathGateOverflow(t *testing.T) { + g := &pathGate{seen: map[int]map[string]map[string]struct{}{}} + const day = 20260919 + + // 上限内的模板原样收下，且认过的再来还是它自己（不会因为已满被改判） + first := g.admit("h1", day, "/login") + for i := 1; i < pathTemplatePerSiteDay; i++ { + g.admit("h1", day, "/p"+string(rune('a'+i%26))+string(rune(i))) + } + if first != "/login" { + t.Fatalf("上限内的模板应原样返回，实际 %q", first) + } + if got := g.admit("h1", day, "/login"); got != "/login" { + t.Fatalf("已认下的模板应一直原样返回，实际 %q", got) + } + + // 满了之后的新模板进 {overflow}——这是扫目录时汇总表不被打爆的唯一屏障 + if got := g.admit("h1", day, "/never-seen-before"); got != pathOverflowTemplate { + t.Fatalf("超上限的新模板应归进 %s，实际 %q", pathOverflowTemplate, got) + } + + // 另一个站点自己算自己的，不受隔壁影响 + if got := g.admit("h2", day, "/never-seen-before"); got != "/never-seen-before" { + t.Fatalf("上限是按站点算的，实际 %q", got) + } +} + +func TestPathGatePrunesOldDays(t *testing.T) { + g := &pathGate{seen: map[int]map[string]map[string]struct{}{}} + for _, day := range []int{20260917, 20260918, 20260919, 20260920} { + g.admit("h1", day, "/x") + } + if len(g.seen) > 2 { + t.Fatalf("跨天的桶应被清掉，只留最近两天，实际留了 %d 天", len(g.seen)) + } + if _, ok := g.seen[20260920]; !ok { + t.Fatal("当天的桶不该被清掉") + } +} + +func openAnalysisTestDB(t *testing.T) *gorm.DB { + t.Helper() + dialect.Register(&dialect.SQLiteDialect{}) + dsn := filepath.Join(t.TempDir(), "stats.db") + "?_db_key=" + url.QueryEscape("ktest") + db, err := gorm.Open(sqlitedriver.Open(dsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + t.Skipf("打不开 sqlite（缺 CGO？），跳过真实库用例: %v", err) + } + if err := db.AutoMigrate(&model.StatsActorPathDay{}, &model.StatsActorUaDay{}, &model.StatsPathRuleDay{}); err != nil { + t.Skipf("建表失败，跳过真实库用例: %v", err) + } + // upsert 要有冲突目标才会走累加分支，否则每批都新建行、计数翻倍 + for _, sql := range []string{ + "CREATE UNIQUE INDEX IF NOT EXISTS uni_stats_actor_path_days ON stats_actor_path_days (user_code, tenant_id, host_code, day, actor_key, path_norm)", + "CREATE UNIQUE INDEX IF NOT EXISTS uni_stats_actor_ua_days ON stats_actor_ua_days (user_code, tenant_id, host_code, day, actor_key, ua_hash)", + "CREATE UNIQUE INDEX IF NOT EXISTS uni_stats_path_rule_days ON stats_path_rule_days (user_code, tenant_id, host_code, day, path_norm, rule)", + } { + if err := db.Exec(sql).Error; err != nil { + t.Fatalf("建唯一索引失败: %v", err) + } + } + t.Cleanup(func() { + if sqlDB, e := db.DB(); e == nil { + sqlDB.Close() + } + }) + return db +} + +// 分析层的全部价值建立在「计数器可累加、去重数由 GROUP BY 现算」上。 +// 这个用例把两条都钉死：同一批键分两次进来必须累加成一行， +// 而去重 IP 数 / 去重路径数必须是真去重、不随批次翻倍。 +func TestCollectAnalysisStats_AccumulateAndDistinct(t *testing.T) { + db := openAnalysisTestDB(t) + prev := global.GWAF_LOCAL_STATS_DB + global.GWAF_LOCAL_STATS_DB = db + t.Cleanup(func() { global.GWAF_LOCAL_STATS_DB = prev }) + analysisPathGate = &pathGate{seen: map[int]map[string]map[string]struct{}{}} + + const day = 20260919 + row := func(actor, path, ua, rule, action string, code int) model.LogAnalysisRow { + return model.LogAnalysisRow{ + HostCode: "h1", Day: day, ActorKey: actor, PathNorm: path, + UaHash: ua, Rule: rule, Action: action, StatusCode: code, + } + } + + // 第一批：A 打 /login 和 /admin，B 打 /login + CollectAnalysisStats([]model.LogAnalysisRow{ + row("ip:1.1.1.1", "/login", "ua1", "", "放行", 200), + row("ip:1.1.1.1", "/admin", "ua1", "", "放行", 404), + row("ip:2.2.2.2", "/login", "ua2", "SQL注入", "阻止", 403), + }) + // 第二批：同样的键再来一遍，必须累加进同几行而不是新建 + CollectAnalysisStats([]model.LogAnalysisRow{ + row("ip:1.1.1.1", "/login", "ua1", "", "放行", 200), + row("ip:2.2.2.2", "/login", "ua2", "SQL注入", "阻止", 403), + }) + + var rows []model.StatsActorPathDay + if err := db.Order("actor_key, path_norm").Find(&rows).Error; err != nil { + t.Fatal(err) + } + if len(rows) != 3 { + t.Fatalf("三个 (actor,path) 对应三行，实际 %d 行（upsert 没走累加分支？）", len(rows)) + } + got := map[string]model.StatsActorPathDay{} + for _, r := range rows { + got[r.ActorKey+" "+r.PathNorm] = r + } + if v := got["ip:1.1.1.1 /login"]; v.ReqCnt != 2 { + t.Fatalf("同键两批应累加成 2，实际 %d", v.ReqCnt) + } + if v := got["ip:1.1.1.1 /admin"]; v.Err4xxCnt != 1 || v.DenyCnt != 0 { + t.Fatalf("后端 404 应只进 err4xx，实际 err4xx=%d deny=%d", v.Err4xxCnt, v.DenyCnt) + } + // WAF 拦截页也是 403，但它是「被挡」不是「扫出来的 4xx」，两者不能混 + if v := got["ip:2.2.2.2 /login"]; v.DenyCnt != 2 || v.Err4xxCnt != 0 { + t.Fatalf("WAF 拦截应只进 deny，实际 deny=%d err4xx=%d", v.DenyCnt, v.Err4xxCnt) + } + + // 目标视角：/login 被 2 个不同 IP 打过——这个数必须是 2，不能因为发了两批变 4 + var distinctIP int64 + if err := db.Model(&model.StatsActorPathDay{}). + Where("host_code = ? and day = ? and path_norm = ?", "h1", day, "/login"). + Distinct("actor_key").Count(&distinctIP).Error; err != nil { + t.Fatal(err) + } + if distinctIP != 2 { + t.Fatalf("/login 的去重 IP 数应为 2，实际 %d", distinctIP) + } + + // 行为视角：A 摸过 2 个不同路径（扫描信号就看这个数） + var distinctPath int64 + if err := db.Model(&model.StatsActorPathDay{}). + Where("host_code = ? and day = ? and actor_key = ?", "h1", day, "ip:1.1.1.1"). + Distinct("path_norm").Count(&distinctPath).Error; err != nil { + t.Fatal(err) + } + if distinctPath != 2 { + t.Fatalf("ip:1.1.1.1 的去重路径数应为 2，实际 %d", distinctPath) + } + + // UA 表：两个 actor 各一种 UA，且同样累加 + var uaRows []model.StatsActorUaDay + if err := db.Find(&uaRows).Error; err != nil { + t.Fatal(err) + } + if len(uaRows) != 2 { + t.Fatalf("两个 actor 各一种 UA 应是 2 行，实际 %d", len(uaRows)) + } + + // 规则表：只有命中规则的请求进，且累加 + var ruleRows []model.StatsPathRuleDay + if err := db.Find(&ruleRows).Error; err != nil { + t.Fatal(err) + } + if len(ruleRows) != 1 || ruleRows[0].Rule != "SQL注入" || ruleRows[0].Cnt != 2 { + t.Fatalf("规则分布应是 SQL注入×2 一行，实际 %+v", ruleRows) + } +} + +// 没有 host_code 或没有 actor_key 的行聚合不出任何有意义的键，必须丢掉， +// 否则会攒出一堆空键行把汇总表污染成垃圾。 +func TestCollectAnalysisStats_SkipsUnkeyedRows(t *testing.T) { + db := openAnalysisTestDB(t) + prev := global.GWAF_LOCAL_STATS_DB + global.GWAF_LOCAL_STATS_DB = db + t.Cleanup(func() { global.GWAF_LOCAL_STATS_DB = prev }) + analysisPathGate = &pathGate{seen: map[int]map[string]map[string]struct{}{}} + + CollectAnalysisStats([]model.LogAnalysisRow{ + {HostCode: "", Day: 20260919, ActorKey: "ip:1.1.1.1", PathNorm: "/x", Action: "放行"}, + {HostCode: "h1", Day: 20260919, ActorKey: "", PathNorm: "/x", Action: "放行"}, + }) + + var cnt int64 + if err := db.Model(&model.StatsActorPathDay{}).Count(&cnt).Error; err != nil { + t.Fatal(err) + } + if cnt != 0 { + t.Fatalf("无键的行不该落库，实际落了 %d 行", cnt) + } +} diff --git a/waftask/stat_analysis_crossdb_test.go b/waftask/stat_analysis_crossdb_test.go new file mode 100644 index 00000000..098468f6 --- /dev/null +++ b/waftask/stat_analysis_crossdb_test.go @@ -0,0 +1,190 @@ +//go:build crossdb + +// 分析层汇总的三库回归。 +// +// 这里的 upsert 走方言分支（UpsertExcludedRef 在 PG/SQLite 是 excluded.x、MySQL 是 VALUES(x)）， +// 冲突目标又落在一个六列的唯一索引上——任一引擎对不上，症状都是「每批新建一行、计数翻倍」 +// 这种不报错的静默错账，所以必须三边各跑一遍真实迁移 + 真实写入。 +// +// 跑法：go test -tags crossdb ./waftask/ -run TestCollectAnalysisStatsCrossEngine +// MySQL / PostgreSQL 连不上时跳过，不阻塞。 +package waftask + +import ( + "SamWaf/global" + "SamWaf/model" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "net/url" + "os" + "path/filepath" + "testing" + + mysqldriver "gorm.io/driver/mysql" + pgdriver "gorm.io/driver/postgres" + sqlitedriver "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +const xtestAnalysisDB = "samwaf_xtest_analysis" + +var analysisSilentCfg = &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)} + +func TestCollectAnalysisStatsCrossEngine(t *testing.T) { + engines := map[string]func(*testing.T) (*gorm.DB, func()){ + "sqlite": setupAnalysisSQLite, + "mysql": setupAnalysisMySQL, + "postgres": setupAnalysisPostgres, + } + for name, setup := range engines { + t.Run(name, func(t *testing.T) { + db, done := setup(t) + if db == nil { + t.Skip("引擎不可用，跳过") + } + defer done() + runAnalysisAssertions(t, db) + }) + } +} + +func runAnalysisAssertions(t *testing.T, db *gorm.DB) { + t.Helper() + prev := global.GWAF_LOCAL_STATS_DB + global.GWAF_LOCAL_STATS_DB = db + defer func() { global.GWAF_LOCAL_STATS_DB = prev }() + analysisPathGate = &pathGate{seen: map[int]map[string]map[string]struct{}{}} + + const day = 20260919 + row := func(actor, path, rule, action string, code int) model.LogAnalysisRow { + return model.LogAnalysisRow{ + HostCode: "h1", Day: day, ActorKey: actor, PathNorm: path, + UaHash: "ua1", Rule: rule, Action: action, StatusCode: code, + } + } + batch := []model.LogAnalysisRow{ + row("ip:1.1.1.1", "/login", "", "放行", 200), + row("ip:1.1.1.1", "/admin", "", "放行", 404), + row("ip:2.2.2.2", "/login", "SQL注入", "阻止", 403), + } + CollectAnalysisStats(batch) + CollectAnalysisStats(batch) + + var rows []model.StatsActorPathDay + if err := db.Order("actor_key, path_norm").Find(&rows).Error; err != nil { + t.Fatalf("读汇总失败: %v", err) + } + if len(rows) != 3 { + t.Fatalf("三个 (actor,path) 对应三行，实际 %d 行：upsert 没命中唯一索引", len(rows)) + } + for _, r := range rows { + if r.ReqCnt != 2 { + t.Fatalf("%s %s 两批应累加成 2，实际 %d", r.ActorKey, r.PathNorm, r.ReqCnt) + } + } + + var distinctIP int64 + if err := db.Model(&model.StatsActorPathDay{}). + Where("host_code = ? and day = ? and path_norm = ?", "h1", day, "/login"). + Distinct("actor_key").Count(&distinctIP).Error; err != nil { + t.Fatalf("去重 IP 统计失败: %v", err) + } + if distinctIP != 2 { + t.Fatalf("/login 去重 IP 数应为 2，实际 %d", distinctIP) + } + + var ruleRows []model.StatsPathRuleDay + if err := db.Find(&ruleRows).Error; err != nil { + t.Fatalf("读规则分布失败: %v", err) + } + if len(ruleRows) != 1 || ruleRows[0].Cnt != 2 { + t.Fatalf("规则分布应是一行 ×2，实际 %+v", ruleRows) + } +} + +func setupAnalysisSQLite(t *testing.T) (*gorm.DB, func()) { + dialect.Register(&dialect.SQLiteDialect{}) + dsn := filepath.Join(t.TempDir(), "stats.db") + "?_db_key=" + url.QueryEscape("kstats") + db, err := gorm.Open(sqlitedriver.Open(dsn), analysisSilentCfg) + if err != nil { + t.Logf("sqlite 打开失败（跳过）: %v", err) + return nil, nil + } + if err := wafdb.RunStatsDBMigrations(db); err != nil { + t.Fatalf("sqlite stats 迁移失败: %v", err) + } + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + } +} + +func setupAnalysisMySQL(t *testing.T) (*gorm.DB, func()) { + base := os.Getenv("SAMWAF_TEST_MYSQL_DSN") + if base == "" { + base = "root:canteen1@tcp(127.0.0.1:3306)/" + } + root, err := gorm.Open(mysqldriver.Open(base+"?parseTime=true"), analysisSilentCfg) + if err != nil { + t.Logf("mysql 连接失败（跳过）: %v", err) + return nil, nil + } + root.Exec("DROP DATABASE IF EXISTS " + xtestAnalysisDB) + if err := root.Exec("CREATE DATABASE " + xtestAnalysisDB + " CHARACTER SET utf8mb4").Error; err != nil { + t.Logf("mysql 建库失败（跳过）: %v", err) + return nil, nil + } + dialect.Register(&dialect.MySQLDialect{}) + db, err := gorm.Open(mysqldriver.Open(base+xtestAnalysisDB+"?charset=utf8mb4&parseTime=True&loc=Local"), analysisSilentCfg) + if err != nil { + t.Fatalf("mysql 打开失败: %v", err) + } + if err := wafdb.RunStatsDBMigrations(db); err != nil { + t.Fatalf("mysql stats 迁移失败: %v", err) + } + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + root.Exec("DROP DATABASE IF EXISTS " + xtestAnalysisDB) + if s, e := root.DB(); e == nil { + s.Close() + } + } +} + +func setupAnalysisPostgres(t *testing.T) (*gorm.DB, func()) { + base := os.Getenv("SAMWAF_TEST_PG_DSN") + if base == "" { + base = "postgres://postgres:postgres@127.0.0.1:5432/" + } + root, err := gorm.Open(pgdriver.Open(base+"postgres?sslmode=disable"), analysisSilentCfg) + if err != nil { + t.Logf("postgres 连接失败（跳过）: %v", err) + return nil, nil + } + root.Exec("DROP DATABASE IF EXISTS " + xtestAnalysisDB) + if err := root.Exec("CREATE DATABASE " + xtestAnalysisDB + " ENCODING 'UTF8'").Error; err != nil { + t.Logf("postgres 建库失败（跳过）: %v", err) + return nil, nil + } + dialect.Register(&dialect.PostgresDialect{}) + db, err := gorm.Open(pgdriver.Open(base+xtestAnalysisDB+"?sslmode=disable&TimeZone=Asia/Shanghai"), analysisSilentCfg) + if err != nil { + t.Fatalf("postgres 打开失败: %v", err) + } + if err := wafdb.RunStatsDBMigrations(db); err != nil { + t.Fatalf("postgres stats 迁移失败: %v", err) + } + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + root.Exec("DROP DATABASE IF EXISTS " + xtestAnalysisDB) + if s, e := root.DB(); e == nil { + s.Close() + } + } +} diff --git a/waftask/task_stats_cleanup.go b/waftask/task_stats_cleanup.go index 676272d1..8b624fc9 100644 --- a/waftask/task_stats_cleanup.go +++ b/waftask/task_stats_cleanup.go @@ -21,12 +21,16 @@ const cleanupBatchSleep = 50 * time.Millisecond var safeIdentPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) // allowedCleanupTables 数据保留清理任务允许操作的表白名单。 -// 目前仅这三张统计/标签表会被自动清理；任何不在此列的表一律拒绝（即便表名是合法标识符）， +// 只有登记在此的统计/标签表会被自动清理；任何不在此列的表一律拒绝（即便表名是合法标识符）， // 防止保留策略被篡改后误删/恶意删除业务表或核心表。新增可清理表时在此登记。 var allowedCleanupTables = map[string]struct{}{ "stats_ip_days": {}, "stats_ip_city_days": {}, "ip_tags": {}, + // 分析层天级汇总：行数随「站点×天」的去重对数增长，必须跟着保留策略走 + "stats_actor_path_days": {}, + "stats_actor_ua_days": {}, + "stats_path_rule_days": {}, } // validatePolicyIdentifiers 校验保留策略中所有会被拼接进 SQL 的标识符字段。 From 23c00d4c48248b62764cd29295c52202abbc02a5 Mon Sep 17 00:00:00 2001 From: samwaf Date: Sun, 20 Sep 2026 16:29:48 +0800 Subject: [PATCH 11/21] fix:actor key by ip --- model/access_log.go | 2 +- wafqueue/log_keys_test.go | 21 ++++++++------------- wafqueue/log_tier.go | 2 +- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/model/access_log.go b/model/access_log.go index 93c8d536..8e362929 100644 --- a/model/access_log.go +++ b/model/access_log.go @@ -56,7 +56,7 @@ type LogNarrow struct { // 分析键（M5 汇总与 facet 的数据地基，写入时算好）。 // actor_key 的索引不在 tag 里建：LogNarrow 同时嵌进 access_log 与 security_event， // 而 SQLite/PG 的索引名全库唯一，同名会撞——改在迁移里按表各建各的（idx_al_actor/idx_se_actor）。 - ActorKey string `gorm:"size:100" json:"actor_key"` // 访客身份优先，其次 IP + ActorKey string `gorm:"size:100" json:"actor_key"` // 访问者身份，目前只有 ip: 前缀的来源 IP UaHash string `gorm:"size:64" json:"ua_hash"` // UA 指纹，供「同一人多少种 UA」 PathNorm string `gorm:"size:512" json:"path_norm"` // 路径模板：数字段/UUID/hex 归一 diff --git a/wafqueue/log_keys_test.go b/wafqueue/log_keys_test.go index 8245625c..76bea23a 100644 --- a/wafqueue/log_keys_test.go +++ b/wafqueue/log_keys_test.go @@ -28,21 +28,16 @@ func TestNormalizePath(t *testing.T) { } func TestActorKey(t *testing.T) { - if got := ActorKey("guest-1", "1.2.3.4"); got != "g:guest-1" { - t.Fatalf("有访客身份时优先用它，实际 %q", got) + if got := ActorKey("1.2.3.4"); got != "ip:1.2.3.4" { + t.Fatalf("actor_key 取来源 IP，实际 %q", got) } - if got := ActorKey("", "1.2.3.4"); got != "ip:1.2.3.4" { - t.Fatalf("没访客身份退回 IP，实际 %q", got) + if got := ActorKey(""); got != "" { + t.Fatalf("没有 IP 应为空，实际 %q", got) } - if got := ActorKey("", ""); got != "" { - t.Fatalf("都没有应为空，实际 %q", got) - } - // 同一个访客换 IP 仍同键；同一 IP 不同访客不同键 - if ActorKey("guest-1", "1.2.3.4") != ActorKey("guest-1", "9.9.9.9") { - t.Fatal("换 IP 不该改变 actor_key") - } - if ActorKey("a", "1.2.3.4") == ActorKey("b", "1.2.3.4") { - t.Fatal("不同访客不该同键") + // 钉死：GUEST_IDENTIFICATION 是分类标签（正常访客/可疑用户/bot 名）， + // 不是身份，绝不能进 actor_key——否则全站正常访客聚成一个 actor。 + if ActorKey("1.2.3.4") == ActorKey("5.6.7.8") { + t.Fatal("不同 IP 不该同键") } } diff --git a/wafqueue/log_tier.go b/wafqueue/log_tier.go index 359894cd..53143d96 100644 --- a/wafqueue/log_tier.go +++ b/wafqueue/log_tier.go @@ -141,7 +141,7 @@ func narrowFromLog(lg *innerbean.WebLog) model.LogNarrow { BalanceInfo: lg.BalanceInfo, BodyHash: lg.BodyHash, Truncated: truncated, - ActorKey: ActorKey(lg.GUEST_IDENTIFICATION, lg.SRC_IP), + ActorKey: ActorKey(lg.SRC_IP), UaHash: UaHash(lg.USER_AGENT), PathNorm: NormalizePath(lg.URL), CREATE_TIME: lg.CREATE_TIME, From 5ad345202c326f3c0871e1c903ca829f32eb1d75 Mon Sep 17 00:00:00 2001 From: samwaf Date: Sun, 20 Sep 2026 16:30:39 +0800 Subject: [PATCH 12/21] feat: add a source and path analysis page over the daily rollups --- api/entrance.go | 1 + api/waf_analysis.go | 38 +++ global/global.go | 5 + model/request/waf_analysis_view_req.go | 29 ++ model/response/waf_analysis_view_resp.go | 86 ++++++ router/waf_analysis.go | 4 + service/waf_service/waf_analysis_view.go | 276 ++++++++++++++++++ service/waf_service/waf_analysis_view_test.go | 260 +++++++++++++++++ waftask/task_config.go | 14 + 9 files changed, 713 insertions(+) create mode 100644 model/request/waf_analysis_view_req.go create mode 100644 model/response/waf_analysis_view_resp.go create mode 100644 service/waf_service/waf_analysis_view.go create mode 100644 service/waf_service/waf_analysis_view_test.go diff --git a/api/entrance.go b/api/entrance.go index 5ef2cf70..9f8b0c05 100644 --- a/api/entrance.go +++ b/api/entrance.go @@ -145,6 +145,7 @@ var ( wafOtpService = waf_service.WafOtpServiceApp wafAnalysisService = waf_service.WafAnalysisServiceApp + wafAnalysisViewService = waf_service.WafAnalysisViewServiceApp wafAIService = waf_service.WafAIServiceApp wafAILabelService = waf_service.WafAILabelServiceApp diff --git a/api/waf_analysis.go b/api/waf_analysis.go index 6b253d97..9638ad2b 100644 --- a/api/waf_analysis.go +++ b/api/waf_analysis.go @@ -34,3 +34,41 @@ func (w *WafAnalysisApi) AnalysisSpiderRangeApi(c *gin.Context) { response.FailWithMessage("解析失败", c) } } + +// AnalysisActorListApi 来源与路径分析 - 行为视角（谁在打） +func (w *WafAnalysisApi) AnalysisActorListApi(c *gin.Context) { + var req request.WafAnalysisActorReq + if err := c.ShouldBind(&req); err != nil { + response.FailWithMessage("解析失败", c) + return + } + response.OkWithDetailed(wafAnalysisViewService.ActorListApi(req), "获取成功", c) +} + +// AnalysisPathListApi 来源与路径分析 - 目标视角（打哪里） +func (w *WafAnalysisApi) AnalysisPathListApi(c *gin.Context) { + var req request.WafAnalysisPathReq + if err := c.ShouldBind(&req); err != nil { + response.FailWithMessage("解析失败", c) + return + } + response.OkWithDetailed(wafAnalysisViewService.PathListApi(req), "获取成功", c) +} + +// AnalysisDetailApi 来源与路径分析 - 抽屉下钻 +func (w *WafAnalysisApi) AnalysisDetailApi(c *gin.Context) { + var req request.WafAnalysisDetailReq + if err := c.ShouldBind(&req); err != nil { + response.FailWithMessage("解析失败", c) + return + } + if req.Kind != "actor" && req.Kind != "path" { + response.FailWithMessage("kind 只能是 actor 或 path", c) + return + } + if req.Key == "" { + response.FailWithMessage("请传入要查看的 key", c) + return + } + response.OkWithDetailed(wafAnalysisViewService.DetailApi(req), "获取成功", c) +} diff --git a/global/global.go b/global/global.go index 2ead5032..8cfe8f38 100644 --- a/global/global.go +++ b/global/global.go @@ -201,6 +201,11 @@ var ( GDATA_ACCESS_LOG_MODE string = "db" // GDATA_ACCESS_LOG_RETENTION_DAYS access_log 保留天数。它直接决定 CC 阈值推荐能回看多少天。 GDATA_ACCESS_LOG_RETENTION_DAYS int64 = 30 + // GDATA_ANALYSIS_SCAN_PATH_THRESHOLD 来源分析里「疑似扫目录」的判定线：一天摸过多少个不同路径模板。 + // 只影响界面上的提示与筛选，不落库、不改任何拦截行为。 + GDATA_ANALYSIS_SCAN_PATH_THRESHOLD int64 = 20 + // GDATA_ANALYSIS_UA_THRESHOLD 来源分析里「疑似换 UA 试探」的判定线：一天用过多少种 UA 指纹。 + GDATA_ANALYSIS_UA_THRESHOLD int64 = 5 // GDATA_IP_TAG_DB IP Tag 存放位置：0 核心库，1 统计库。 // 新装默认统计库——ip_tags 是派生索引，写入量跟着攻击量走，和配置共用核心库会拖慢每一次配置读写。 // 存量用户的取值来自 system_configs 里已有的那一行，不受这个默认值影响； diff --git a/model/request/waf_analysis_view_req.go b/model/request/waf_analysis_view_req.go new file mode 100644 index 00000000..80970c06 --- /dev/null +++ b/model/request/waf_analysis_view_req.go @@ -0,0 +1,29 @@ +package request + +// 来源与路径分析（M5 / G3）的三个只读查询。 +// 都是 GET，gin 的 form 绑定只认 form tag，只写 json tag 会取不到值。 + +// WafAnalysisActorReq 行为视角：谁在打。 +type WafAnalysisActorReq struct { + Day int `json:"day" form:"day"` //年月日 如 20260920，留空取今天 + HostCode string `json:"host_code" form:"host_code"` //留空=全部站点 + SortBy string `json:"sort_by" form:"sort_by"` //排序字段，服务端白名单映射，非法值回落默认 + Limit int `json:"limit" form:"limit"` //Top N，默认 20 +} + +// WafAnalysisPathReq 目标视角：打哪里。 +type WafAnalysisPathReq struct { + Day int `json:"day" form:"day"` + HostCode string `json:"host_code" form:"host_code"` + SortBy string `json:"sort_by" form:"sort_by"` + Limit int `json:"limit" form:"limit"` +} + +// WafAnalysisDetailReq 抽屉下钻：Kind=actor 看某个来源摸过什么，Kind=path 看某个路径被谁打。 +type WafAnalysisDetailReq struct { + Day int `json:"day" form:"day"` + HostCode string `json:"host_code" form:"host_code"` + Kind string `json:"kind" form:"kind"` //actor | path + Key string `json:"key" form:"key"` //actor_key 或 path_norm + Limit int `json:"limit" form:"limit"` +} diff --git a/model/response/waf_analysis_view_resp.go b/model/response/waf_analysis_view_resp.go new file mode 100644 index 00000000..b0940ecf --- /dev/null +++ b/model/response/waf_analysis_view_resp.go @@ -0,0 +1,86 @@ +package response + +// 来源与路径分析（M5 / G3）的返回结构。 +// 去重数（distinct_path / distinct_actor）都是查询时 COUNT(DISTINCT) 现算的， +// 库里没有这两列——去重计数没法跨批次累加，存成列必错（见计划 §5.7）。 + +// WafAnalysisActorRow 行为视角一行：一个来源今天干了什么。 +type WafAnalysisActorRow struct { + ActorKey string `json:"actor_key"` //ip:1.2.3.4 + ReqCnt int64 `json:"req_cnt"` //请求数 + DenyCnt int64 `json:"deny_cnt"` //被拦截数 + Err4xxCnt int64 `json:"err4xx_cnt"` //后端返回的 4xx，不含 WAF 拦截页 + DistinctPath int64 `json:"distinct_path"` //摸过多少个不同路径模板——扫目录的信号 + UaCnt int64 `json:"ua_cnt"` //用过多少种 UA 指纹 +} + +// WafAnalysisActorResp 行为视角一页 + 该视角自己的几个统计数。 +type WafAnalysisActorResp struct { + Day int `json:"day"` + List []WafAnalysisActorRow `json:"list"` + TotalActor int64 `json:"total_actor"` //今天出现过的不同来源数 + ScanActor int64 `json:"scan_actor"` //达到扫目录阈值的来源数 + UaActor int64 `json:"ua_actor"` //达到换 UA 阈值的来源数 + ScanThreshold int64 `json:"scan_threshold"` //当前判定线，界面显示用 + UaThreshold int64 `json:"ua_threshold"` +} + +// WafAnalysisPathRow 目标视角一行：一个路径模板今天被怎么打。 +type WafAnalysisPathRow struct { + PathNorm string `json:"path_norm"` + ReqCnt int64 `json:"req_cnt"` + DenyCnt int64 `json:"deny_cnt"` + Err4xxCnt int64 `json:"err4xx_cnt"` + DistinctActor int64 `json:"distinct_actor"` //几个不同来源打过它 + TopRule string `json:"top_rule"` //命中最多的规则，来自 stats_path_rule_days +} + +type WafAnalysisPathResp struct { + Day int `json:"day"` + List []WafAnalysisPathRow `json:"list"` + TotalPath int64 `json:"total_path"` //今天出现过的路径模板数 +} + +// 抽屉下钻的几张小表 +type WafAnalysisDetailPath struct { + PathNorm string `json:"path_norm"` + ReqCnt int64 `json:"req_cnt"` + DenyCnt int64 `json:"deny_cnt"` + Err4xxCnt int64 `json:"err4xx_cnt"` +} +type WafAnalysisDetailActor struct { + ActorKey string `json:"actor_key"` + ReqCnt int64 `json:"req_cnt"` + DenyCnt int64 `json:"deny_cnt"` + Err4xxCnt int64 `json:"err4xx_cnt"` +} +type WafAnalysisDetailUa struct { + UaHash string `json:"ua_hash"` + Cnt int64 `json:"cnt"` +} +type WafAnalysisDetailRule struct { + Rule string `json:"rule"` + Cnt int64 `json:"cnt"` +} + +// WafAnalysisDetailHost 这个来源打过哪些站点。 +// 加黑/加白的接口 host_code 都是必填，而本页是跨站点聚合的， +// 一个来源可能横跨多站——界面靠这张表让用户选站点，不能瞎猜一个。 +type WafAnalysisDetailHost struct { + HostCode string `json:"host_code"` + ReqCnt int64 `json:"req_cnt"` + DenyCnt int64 `json:"deny_cnt"` +} + +type WafAnalysisDetailResp struct { + Kind string `json:"kind"` //actor | path + Key string `json:"key"` + ReqCnt int64 `json:"req_cnt"` + DenyCnt int64 `json:"deny_cnt"` + Err4xxCnt int64 `json:"err4xx_cnt"` + Paths []WafAnalysisDetailPath `json:"paths"` //kind=actor + Uas []WafAnalysisDetailUa `json:"uas"` //kind=actor + Hosts []WafAnalysisDetailHost `json:"hosts"` //kind=actor + Actors []WafAnalysisDetailActor `json:"actors"` //kind=path + Rules []WafAnalysisDetailRule `json:"rules"` //kind=path +} diff --git a/router/waf_analysis.go b/router/waf_analysis.go index b678713c..f13c839e 100644 --- a/router/waf_analysis.go +++ b/router/waf_analysis.go @@ -14,4 +14,8 @@ func (receiver *AnalysisRouter) InitAnalysisRouter(group *gin.RouterGroup) { //数据分析 router.GET("/api/v1/analysis/wafanalysisdaycountryrange", analysisApi.StatAnalysisDayCountryRangeApi) router.GET("/api/v1/analysis/spider", analysisApi.AnalysisSpiderRangeApi) + // 来源与路径分析（M5/G3）：两个视角 + 抽屉下钻，只读 + router.GET("/api/v1/analysis/actor/list", analysisApi.AnalysisActorListApi) + router.GET("/api/v1/analysis/path/list", analysisApi.AnalysisPathListApi) + router.GET("/api/v1/analysis/detail", analysisApi.AnalysisDetailApi) } diff --git a/service/waf_service/waf_analysis_view.go b/service/waf_service/waf_analysis_view.go new file mode 100644 index 00000000..48add248 --- /dev/null +++ b/service/waf_service/waf_analysis_view.go @@ -0,0 +1,276 @@ +package waf_service + +import ( + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/request" + response2 "SamWaf/model/response" + "strings" + "time" + + "gorm.io/gorm" +) + +// 来源与路径分析（M5 / G3）的读侧。 +// +// 数据来自 stats_actor_path_days 的两个 GROUP BY：按 actor_key 出「谁在打」， +// 按 path_norm 出「打哪里」。去重数是查询时 COUNT(DISTINCT) 现算的—— +// 它没法跨批次累加，所以库里不存这两列（见计划 §5.7 第 1 条）。 +// +// 三条硬约束： +// 1. 排序字段只走下面的白名单映射，前端传什么都不拼进 SQL； +// 2. 每条查询都必须带 day，索引是 (host_code, day, …)，不带 day 会全表扫； +// 3. 走本服务的专用只读接口，不要引导用户去数据查询页——那边的敏感列子串 +// "key" 会把 actor_key 静默隐掉，页面上主维度会是空的。 +type WafAnalysisViewService struct{} + +var WafAnalysisViewServiceApp = new(WafAnalysisViewService) + +const ( + analysisDefaultLimit = 20 + analysisMaxLimit = 200 + analysisDetailLimit = 50 +) + +// 排序白名单：前端传 key，服务端换成写死的 SQL 片段。 +var analysisActorSort = map[string]string{ + "distinct_path": "distinct_path desc", + "req_cnt": "req_cnt desc", + "deny_cnt": "deny_cnt desc", + "err4xx_cnt": "err4xx_cnt desc", +} +var analysisPathSort = map[string]string{ + "req_cnt": "req_cnt desc", + "deny_cnt": "deny_cnt desc", + "err4xx_cnt": "err4xx_cnt desc", + "distinct_actor": "distinct_actor desc", +} + +func analysisOrder(m map[string]string, key, fallback string) string { + if v, ok := m[key]; ok { + return v + } + return m[fallback] +} + +func analysisDay(day int) int { + if day > 0 { + return day + } + d := 0 + for _, c := range time.Now().Format("20060102") { + d = d*10 + int(c-'0') + } + return d +} + +func analysisLimit(n int, def int) int { + if n <= 0 { + return def + } + if n > analysisMaxLimit { + return analysisMaxLimit + } + return n +} + +// scoped 统一加上 day 与可选的站点过滤。day 是索引首要过滤列，任何查询都不能少。 +func (receiver *WafAnalysisViewService) scoped(tx *gorm.DB, day int, hostCode string) *gorm.DB { + tx = tx.Where("day = ?", day) + if strings.TrimSpace(hostCode) != "" { + tx = tx.Where("host_code = ?", hostCode) + } + return tx +} + +// ActorListApi 行为视角：谁在打。 +func (receiver *WafAnalysisViewService) ActorListApi(req request.WafAnalysisActorReq) response2.WafAnalysisActorResp { + day := analysisDay(req.Day) + limit := analysisLimit(req.Limit, analysisDefaultLimit) + resp := response2.WafAnalysisActorResp{ + Day: day, + List: []response2.WafAnalysisActorRow{}, + ScanThreshold: global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD, + UaThreshold: global.GDATA_ANALYSIS_UA_THRESHOLD, + } + statsDB := global.GWAF_LOCAL_STATS_DB + if statsDB == nil { + return resp + } + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Select("count(distinct actor_key)").Scan(&resp.TotalActor) + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Select("actor_key, sum(req_cnt) as req_cnt, sum(deny_cnt) as deny_cnt, " + + "sum(err4xx_cnt) as err4xx_cnt, count(distinct path_norm) as distinct_path"). + Group("actor_key"). + Order(analysisOrder(analysisActorSort, req.SortBy, "distinct_path")). + Limit(limit).Scan(&resp.List) + + receiver.fillUaCnt(statsDB, day, req.HostCode, resp.List) + + // 两个「疑似」计数：达到阈值的来源有几个。子查询必须带别名，PostgreSQL 不接受匿名派生表。 + scanSub := receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Select("actor_key").Group("actor_key"). + Having("count(distinct path_norm) >= ?", global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD) + statsDB.Table("(?) as t", scanSub).Select("count(*)").Scan(&resp.ScanActor) + + uaSub := receiver.scoped(statsDB.Model(&model.StatsActorUaDay{}), day, req.HostCode). + Select("actor_key").Group("actor_key"). + Having("count(*) >= ?", global.GDATA_ANALYSIS_UA_THRESHOLD) + statsDB.Table("(?) as t", uaSub).Select("count(*)").Scan(&resp.UaActor) + + return resp +} + +// fillUaCnt UA 种类在另一张表，单独查一次按 actor_key 贴回去。 +// 不做 join：两张表各有各的索引，join 的执行计划在三种引擎上不好保证， +// 而这里最多只有 limit 行，一次 IN 查询足够。 +func (receiver *WafAnalysisViewService) fillUaCnt(statsDB *gorm.DB, day int, hostCode string, rows []response2.WafAnalysisActorRow) { + if len(rows) == 0 { + return + } + keys := make([]string, 0, len(rows)) + for _, r := range rows { + keys = append(keys, r.ActorKey) + } + var uaRows []struct { + ActorKey string + Cnt int64 + } + receiver.scoped(statsDB.Model(&model.StatsActorUaDay{}), day, hostCode). + Where("actor_key in ?", keys). + Select("actor_key, count(*) as cnt").Group("actor_key").Scan(&uaRows) + + m := make(map[string]int64, len(uaRows)) + for _, u := range uaRows { + m[u.ActorKey] = u.Cnt + } + for i := range rows { + rows[i].UaCnt = m[rows[i].ActorKey] + } +} + +// PathListApi 目标视角：打哪里。 +func (receiver *WafAnalysisViewService) PathListApi(req request.WafAnalysisPathReq) response2.WafAnalysisPathResp { + day := analysisDay(req.Day) + limit := analysisLimit(req.Limit, analysisDefaultLimit) + resp := response2.WafAnalysisPathResp{Day: day, List: []response2.WafAnalysisPathRow{}} + statsDB := global.GWAF_LOCAL_STATS_DB + if statsDB == nil { + return resp + } + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Select("count(distinct path_norm)").Scan(&resp.TotalPath) + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Select("path_norm, sum(req_cnt) as req_cnt, sum(deny_cnt) as deny_cnt, " + + "sum(err4xx_cnt) as err4xx_cnt, count(distinct actor_key) as distinct_actor"). + Group("path_norm"). + Order(analysisOrder(analysisPathSort, req.SortBy, "req_cnt")). + Limit(limit).Scan(&resp.List) + + receiver.fillTopRule(statsDB, day, req.HostCode, resp.List) + return resp +} + +// fillTopRule 规则分布在另一张表，取回本页这些路径的所有规则行，在内存里挑各自最大的那条。 +// 存的是整个分布而不是一个 top_rule，读侧因此既能取 top 也能看构成。 +func (receiver *WafAnalysisViewService) fillTopRule(statsDB *gorm.DB, day int, hostCode string, rows []response2.WafAnalysisPathRow) { + if len(rows) == 0 { + return + } + paths := make([]string, 0, len(rows)) + for _, r := range rows { + paths = append(paths, r.PathNorm) + } + var ruleRows []struct { + PathNorm string + Rule string + Cnt int64 + } + receiver.scoped(statsDB.Model(&model.StatsPathRuleDay{}), day, hostCode). + Where("path_norm in ?", paths). + Select("path_norm, rule, sum(cnt) as cnt").Group("path_norm, rule").Scan(&ruleRows) + + type best struct { + rule string + cnt int64 + } + m := make(map[string]best, len(ruleRows)) + for _, r := range ruleRows { + if b, ok := m[r.PathNorm]; !ok || r.Cnt > b.cnt { + m[r.PathNorm] = best{rule: r.Rule, cnt: r.Cnt} + } + } + for i := range rows { + if b, ok := m[rows[i].PathNorm]; ok { + rows[i].TopRule = b.rule + } + } +} + +// DetailApi 抽屉下钻。Kind=actor 看这个来源摸过什么、用过什么 UA、打过哪些站点； +// Kind=path 看这个路径被谁打、命中过哪些规则。 +func (receiver *WafAnalysisViewService) DetailApi(req request.WafAnalysisDetailReq) response2.WafAnalysisDetailResp { + day := analysisDay(req.Day) + limit := analysisLimit(req.Limit, analysisDetailLimit) + resp := response2.WafAnalysisDetailResp{Kind: req.Kind, Key: req.Key} + statsDB := global.GWAF_LOCAL_STATS_DB + if statsDB == nil || strings.TrimSpace(req.Key) == "" { + return resp + } + + var totals struct { + ReqCnt int64 + DenyCnt int64 + Err4xxCnt int64 + } + if req.Kind == "path" { + resp.Actors = []response2.WafAnalysisDetailActor{} + resp.Rules = []response2.WafAnalysisDetailRule{} + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Where("path_norm = ?", req.Key). + Select("sum(req_cnt) as req_cnt, sum(deny_cnt) as deny_cnt, sum(err4xx_cnt) as err4xx_cnt"). + Scan(&totals) + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Where("path_norm = ?", req.Key). + Select("actor_key, sum(req_cnt) as req_cnt, sum(deny_cnt) as deny_cnt, sum(err4xx_cnt) as err4xx_cnt"). + Group("actor_key").Order("req_cnt desc").Limit(limit).Scan(&resp.Actors) + + receiver.scoped(statsDB.Model(&model.StatsPathRuleDay{}), day, req.HostCode). + Where("path_norm = ?", req.Key). + Select("rule, sum(cnt) as cnt").Group("rule").Order("cnt desc").Limit(limit).Scan(&resp.Rules) + } else { + resp.Kind = "actor" + resp.Paths = []response2.WafAnalysisDetailPath{} + resp.Uas = []response2.WafAnalysisDetailUa{} + resp.Hosts = []response2.WafAnalysisDetailHost{} + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Where("actor_key = ?", req.Key). + Select("sum(req_cnt) as req_cnt, sum(deny_cnt) as deny_cnt, sum(err4xx_cnt) as err4xx_cnt"). + Scan(&totals) + + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Where("actor_key = ?", req.Key). + Select("path_norm, sum(req_cnt) as req_cnt, sum(deny_cnt) as deny_cnt, sum(err4xx_cnt) as err4xx_cnt"). + Group("path_norm").Order("req_cnt desc").Limit(limit).Scan(&resp.Paths) + + receiver.scoped(statsDB.Model(&model.StatsActorUaDay{}), day, req.HostCode). + Where("actor_key = ?", req.Key). + Select("ua_hash, sum(cnt) as cnt").Group("ua_hash").Order("cnt desc").Limit(limit).Scan(&resp.Uas) + + // 站点维度：加黑/加白的 host_code 是必填，界面靠它给用户选，别在前端瞎猜 + receiver.scoped(statsDB.Model(&model.StatsActorPathDay{}), day, req.HostCode). + Where("actor_key = ?", req.Key). + Select("host_code, sum(req_cnt) as req_cnt, sum(deny_cnt) as deny_cnt"). + Group("host_code").Order("req_cnt desc").Limit(limit).Scan(&resp.Hosts) + } + resp.ReqCnt, resp.DenyCnt, resp.Err4xxCnt = totals.ReqCnt, totals.DenyCnt, totals.Err4xxCnt + return resp +} diff --git a/service/waf_service/waf_analysis_view_test.go b/service/waf_service/waf_analysis_view_test.go new file mode 100644 index 00000000..5b9ccc05 --- /dev/null +++ b/service/waf_service/waf_analysis_view_test.go @@ -0,0 +1,260 @@ +package waf_service + +import ( + "path/filepath" + "testing" + + "SamWaf/common/uuid" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/model/request" + "SamWaf/wafdb/dialect" + + sqlite "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +const avDay = 20260920 + +// setupAnalysisViewDB 建一个临时 stats 库并灌入一组刻意设计过的数据： +// 一个扫目录的、一个换 UA 的、两个正常访客。 +func setupAnalysisViewDB(t *testing.T) *gorm.DB { + t.Helper() + dialect.Register(&dialect.SQLiteDialect{}) + db, err := gorm.Open( + sqlite.Open(filepath.Join(t.TempDir(), "stats.db")), + &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}, + ) + if err != nil { + t.Skipf("打不开 sqlite（缺 CGO？），跳过: %v", err) + } + if err := db.AutoMigrate(&model.StatsActorPathDay{}, &model.StatsActorUaDay{}, &model.StatsPathRuleDay{}); err != nil { + t.Fatalf("建表失败: %v", err) + } + old := global.GWAF_LOCAL_STATS_DB + global.GWAF_LOCAL_STATS_DB = db + t.Cleanup(func() { + global.GWAF_LOCAL_STATS_DB = old + if s, e := db.DB(); e == nil { + s.Close() + } + }) + + ap := func(host, actor, path string, req, deny, err4xx int64) model.StatsActorPathDay { + return model.StatsActorPathDay{ + BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, + HostCode: host, Day: avDay, ActorKey: actor, PathNorm: path, + ReqCnt: req, DenyCnt: deny, Err4xxCnt: err4xx, + } + } + rows := []model.StatsActorPathDay{ + // 扫目录的：5 个不同路径，还跨了两个站点 + ap("h1", "ip:203.0.113.10", "/admin", 1, 0, 1), + ap("h1", "ip:203.0.113.10", "/.env", 1, 1, 0), + ap("h1", "ip:203.0.113.10", "/wp-login.php", 1, 0, 1), + ap("h1", "ip:203.0.113.10", "/item/{n}", 5, 0, 5), + ap("h2", "ip:203.0.113.10", "/backup.zip", 2, 0, 2), + // 换 UA 的：只摸一个路径，但请求最多 + ap("h1", "ip:203.0.113.20", "/api/products", 20, 0, 0), + // 两个正常访客 + ap("h1", "ip:198.51.100.10", "/", 6, 0, 0), + ap("h1", "ip:198.51.100.10", "/api/products", 3, 0, 0), + ap("h1", "ip:198.51.100.11", "/", 4, 0, 0), + } + if err := db.Create(&rows).Error; err != nil { + t.Fatalf("灌数据失败: %v", err) + } + uas := []model.StatsActorUaDay{ + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, ActorKey: "ip:203.0.113.20", UaHash: "ua1", Cnt: 3}, + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, ActorKey: "ip:203.0.113.20", UaHash: "ua2", Cnt: 3}, + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, ActorKey: "ip:203.0.113.20", UaHash: "ua3", Cnt: 3}, + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, ActorKey: "ip:203.0.113.10", UaHash: "ua9", Cnt: 10}, + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, ActorKey: "ip:198.51.100.10", UaHash: "ua1", Cnt: 9}, + } + if err := db.Create(&uas).Error; err != nil { + t.Fatalf("灌 UA 数据失败: %v", err) + } + rules := []model.StatsPathRuleDay{ + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, PathNorm: "/.env", Rule: "敏感文件", Cnt: 1}, + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, PathNorm: "/api/products", Rule: "SQL注入", Cnt: 5}, + {BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, HostCode: "h1", Day: avDay, PathNorm: "/api/products", Rule: "XSS跨站注入", Cnt: 2}, + } + if err := db.Create(&rules).Error; err != nil { + t.Fatalf("灌规则数据失败: %v", err) + } + return db +} + +// 行为视角的命门：distinct_path 必须是真去重，且默认按它倒序——扫目录的要排第一。 +func TestAnalysisActorList(t *testing.T) { + setupAnalysisViewDB(t) + resp := WafAnalysisViewServiceApp.ActorListApi(request.WafAnalysisActorReq{Day: avDay}) + + if len(resp.List) != 4 { + t.Fatalf("应有 4 个来源，实际 %d", len(resp.List)) + } + top := resp.List[0] + if top.ActorKey != "ip:203.0.113.10" { + t.Fatalf("默认按去重路径倒序，扫目录的该排第一，实际 %q", top.ActorKey) + } + if top.DistinctPath != 5 { + t.Fatalf("它摸了 5 个不同路径，实际 %d", top.DistinctPath) + } + if top.ReqCnt != 10 || top.DenyCnt != 1 || top.Err4xxCnt != 9 { + t.Fatalf("跨站点求和不对：req=%d deny=%d err4xx=%d", top.ReqCnt, top.DenyCnt, top.Err4xxCnt) + } + // UA 数来自另一张表，必须按 actor 正确贴回 + if top.UaCnt != 1 { + t.Fatalf("扫目录的只有 1 种 UA，实际 %d", top.UaCnt) + } + for _, r := range resp.List { + if r.ActorKey == "ip:203.0.113.20" && r.UaCnt != 3 { + t.Fatalf("换 UA 的应有 3 种 UA，实际 %d", r.UaCnt) + } + } + if resp.TotalActor != 4 { + t.Fatalf("总来源数应为 4，实际 %d", resp.TotalActor) + } +} + +// 排序字段是拼进 SQL 的东西，只能走白名单映射。 +// 传个注入串进来必须被丢弃回落默认序，而不是报错或者真去执行它。 +func TestAnalysisActorSortWhitelist(t *testing.T) { + setupAnalysisViewDB(t) + evil := request.WafAnalysisActorReq{Day: avDay, SortBy: "req_cnt desc; drop table stats_actor_path_days --"} + resp := WafAnalysisViewServiceApp.ActorListApi(evil) + if len(resp.List) != 4 { + t.Fatalf("非法排序值应被忽略并照常返回，实际 %d 行", len(resp.List)) + } + if resp.List[0].ActorKey != "ip:203.0.113.10" { + t.Fatalf("应回落到默认的去重路径倒序，实际首行 %q", resp.List[0].ActorKey) + } + // 表还在 + var n int64 + global.GWAF_LOCAL_STATS_DB.Model(&model.StatsActorPathDay{}).Count(&n) + if n == 0 { + t.Fatal("表没了——排序值被拼进 SQL 执行了") + } + + // 合法值要真的生效 + byReq := WafAnalysisViewServiceApp.ActorListApi(request.WafAnalysisActorReq{Day: avDay, SortBy: "req_cnt"}) + if byReq.List[0].ActorKey != "ip:203.0.113.20" { + t.Fatalf("按请求数倒序首位应是 20，实际 %q", byReq.List[0].ActorKey) + } +} + +// 目标视角：去重来源数 + top_rule 取最大的那条。 +func TestAnalysisPathList(t *testing.T) { + setupAnalysisViewDB(t) + resp := WafAnalysisViewServiceApp.PathListApi(request.WafAnalysisPathReq{Day: avDay}) + + var products *struct { + Req, Actor int64 + Rule string + } + for _, r := range resp.List { + if r.PathNorm == "/api/products" { + products = &struct { + Req, Actor int64 + Rule string + }{r.ReqCnt, r.DistinctActor, r.TopRule} + } + } + if products == nil { + t.Fatal("没查到 /api/products") + } + if products.Req != 23 { + t.Fatalf("/api/products 请求数应为 23，实际 %d", products.Req) + } + if products.Actor != 2 { + t.Fatalf("两个不同来源打过它，实际 %d", products.Actor) + } + if products.Rule != "SQL注入" { + t.Fatalf("top_rule 该取次数最多的 SQL注入，实际 %q", products.Rule) + } + // 7 个：/admin /.env /wp-login.php /item/{n} /backup.zip /api/products / + if resp.TotalPath != 7 { + t.Fatalf("路径模板数应为 7，实际 %d", resp.TotalPath) + } +} + +// 站点过滤要真的收窄：只看 h1 时，跨站的那个来源只剩 h1 那部分。 +func TestAnalysisHostFilter(t *testing.T) { + setupAnalysisViewDB(t) + resp := WafAnalysisViewServiceApp.ActorListApi(request.WafAnalysisActorReq{Day: avDay, HostCode: "h1"}) + for _, r := range resp.List { + if r.ActorKey == "ip:203.0.113.10" { + if r.DistinctPath != 4 || r.ReqCnt != 8 { + t.Fatalf("按 h1 过滤后该来源应是 4 个路径 / 8 次请求，实际 %d / %d", r.DistinctPath, r.ReqCnt) + } + return + } + } + t.Fatal("没查到该来源") +} + +// 抽屉：actor 模式要给出站点清单——加黑/加白的 host_code 必填，界面靠它选。 +func TestAnalysisDetailActor(t *testing.T) { + setupAnalysisViewDB(t) + resp := WafAnalysisViewServiceApp.DetailApi(request.WafAnalysisDetailReq{ + Day: avDay, Kind: "actor", Key: "ip:203.0.113.10", + }) + if len(resp.Paths) != 5 { + t.Fatalf("该来源摸过 5 个路径，实际 %d", len(resp.Paths)) + } + if len(resp.Hosts) != 2 { + t.Fatalf("它跨了 2 个站点，实际 %d——加黑时就没得选了", len(resp.Hosts)) + } + if resp.Hosts[0].HostCode != "h1" { + t.Fatalf("站点应按请求数倒序，首位该是 h1，实际 %q", resp.Hosts[0].HostCode) + } + if len(resp.Uas) != 1 { + t.Fatalf("它只有 1 种 UA，实际 %d", len(resp.Uas)) + } + if resp.ReqCnt != 10 { + t.Fatalf("合计请求数应为 10，实际 %d", resp.ReqCnt) + } +} + +func TestAnalysisDetailPath(t *testing.T) { + setupAnalysisViewDB(t) + resp := WafAnalysisViewServiceApp.DetailApi(request.WafAnalysisDetailReq{ + Day: avDay, Kind: "path", Key: "/api/products", + }) + if len(resp.Actors) != 2 { + t.Fatalf("两个来源打过它，实际 %d", len(resp.Actors)) + } + if resp.Actors[0].ActorKey != "ip:203.0.113.20" { + t.Fatalf("来源应按请求数倒序，实际首位 %q", resp.Actors[0].ActorKey) + } + if len(resp.Rules) != 2 || resp.Rules[0].Rule != "SQL注入" { + t.Fatalf("规则分布应是两条、SQL注入在前，实际 %+v", resp.Rules) + } +} + +// 阈值来自系统配置，改了要跟着走；「疑似」计数按阈值现算。 +func TestAnalysisThresholdCounts(t *testing.T) { + setupAnalysisViewDB(t) + oldScan, oldUa := global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD, global.GDATA_ANALYSIS_UA_THRESHOLD + t.Cleanup(func() { + global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD, global.GDATA_ANALYSIS_UA_THRESHOLD = oldScan, oldUa + }) + + global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD = 5 + global.GDATA_ANALYSIS_UA_THRESHOLD = 3 + resp := WafAnalysisViewServiceApp.ActorListApi(request.WafAnalysisActorReq{Day: avDay}) + if resp.ScanActor != 1 || resp.UaActor != 1 { + t.Fatalf("阈值 5/3 时各应命中 1 个，实际 scan=%d ua=%d", resp.ScanActor, resp.UaActor) + } + if resp.ScanThreshold != 5 || resp.UaThreshold != 3 { + t.Fatal("返回的阈值应是当前配置值，界面要拿它显示") + } + + global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD = 99 + resp = WafAnalysisViewServiceApp.ActorListApi(request.WafAnalysisActorReq{Day: avDay}) + if resp.ScanActor != 0 { + t.Fatalf("阈值调到 99 后不该有人命中，实际 %d", resp.ScanActor) + } +} diff --git a/waftask/task_config.go b/waftask/task_config.go index 50173589..3753ac0c 100644 --- a/waftask/task_config.go +++ b/waftask/task_config.go @@ -92,6 +92,18 @@ func setConfigIntValue(name string, value int64, change int) { } global.GDATA_ACCESS_LOG_RETENTION_DAYS = value break + case "analysis_scan_path_threshold": + if value < 1 { + value = 1 + } + global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD = value + break + case "analysis_ua_threshold": + if value < 1 { + value = 1 + } + global.GDATA_ANALYSIS_UA_THRESHOLD = value + break case "auto_load_ssl_file": global.GCONFIG_RECORD_AUTO_LOAD_SSL = value break @@ -758,6 +770,8 @@ func TaskLoadSetting(initLoad bool) { updateConfigStringItem(initLoad, "system", "record_log_type", global.GWAF_RUNTIME_RECORD_LOG_TYPE, "日志记录类型", "options", "all|全部,abnormal|非正常", configMap) updateConfigStringItem(initLoad, "system", "access_log_mode", global.GDATA_ACCESS_LOG_MODE, "访问日志窄行档位：db=全部请求入库(默认,保留期见下项)；sample=安全事件+采样入库；off=只留安全事件(高流量推荐，但会失去CC阈值推荐/AI训练负样本/异常IP的正常行为回溯)。db/sample 档均按站点每天采样 500 条正常请求报文供 AI 训练", "options", "db|全部入库,sample|采样入库,off|仅安全事件", configMap) updateConfigIntItem(initLoad, "system", "access_log_retention_days", global.GDATA_ACCESS_LOG_RETENTION_DAYS, "访问日志窄行保留天数（默认30）。安全事件仍按「日志保留天数」走；本项直接决定CC阈值推荐能回看多少天", "int", "", configMap) + updateConfigIntItem(initLoad, "system", "analysis_scan_path_threshold", global.GDATA_ANALYSIS_SCAN_PATH_THRESHOLD, "来源分析：一天摸过多少个不同路径模板判为「疑似扫目录」（默认20）。只影响界面提示与筛选，不改拦截行为", "int", "", configMap) + updateConfigIntItem(initLoad, "system", "analysis_ua_threshold", global.GDATA_ANALYSIS_UA_THRESHOLD, "来源分析：一天用过多少种UA指纹判为「疑似换UA试探」（默认5）。只影响界面提示与筛选，不改拦截行为", "int", "", configMap) updateConfigStringItem(initLoad, "system", "gwaf_proxy_header", global.GCONFIG_RECORD_PROXY_HEADER, "获取访客IP头信息（按照顺序）比如:X-Forwarded-For,X-Real-IP ,留空则提取的是直接访客IP", "string", "", configMap) updateConfigStringItem(initLoad, "system", "gwaf_manage_proxy_header", global.GCONFIG_MANAGE_PROXY_HEADER, "管理端获取客户端IP头信息（按优先级逗号分隔，如 X-Forwarded-For,X-Real-IP,CF-Connecting-IP），留空则直接取网络IP。安全起见需配合 conf/config.yml 的 security.manage_trusted_proxies：仅当直连来源属可信代理时才采信此头（容器/内网部署可直接填 private）", "string", "", configMap) From fe5dd8ce530f7f17eb927f4a1c7de587efe1d900 Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 21 Sep 2026 16:50:31 +0800 Subject: [PATCH 13/21] feat: exclude chosen source IPs from web log recording --- api/waf_host.go | 33 ++++++ api/waf_system_config.go | 5 + global/global.go | 1 + model/hosts.go | 1 + model/request/waf_host_req.go | 2 + model/wafenginmodel/hostsafe.go | 5 + service/waf_service/waf_host.go | 2 + wafdb/migrations_core.go | 22 ++++ wafenginecore/checkrule_scope_test.go | 3 +- wafenginecore/static_server.go | 2 +- wafenginecore/wafengine.go | 10 +- wafenginecore/wafworker.go | 2 + wafenginecore/weblog_exclude.go | 115 +++++++++++++++++++ wafenginecore/weblog_exclude_test.go | 155 ++++++++++++++++++++++++++ wafenginecore/weblog_record.go | 12 +- waftask/task_config.go | 6 + 16 files changed, 366 insertions(+), 10 deletions(-) create mode 100644 wafenginecore/weblog_exclude.go create mode 100644 wafenginecore/weblog_exclude_test.go diff --git a/api/waf_host.go b/api/waf_host.go index 08ff68cf..8bf44a5c 100644 --- a/api/waf_host.go +++ b/api/waf_host.go @@ -16,6 +16,7 @@ import ( "SamWaf/utils" "SamWaf/wafenginecore" "SamWaf/wafenginecore/clientip" + "SamWaf/wafenginecore/ipset" "errors" "fmt" "net" @@ -106,6 +107,30 @@ func checkIPSourceConfig(cfg *ipSourceConfig) error { return nil } +// checkExcludeIPLog 校验「排除记录日志的IP」清单：长度封顶 + 每行必须是可解析的 +// IP 模式（单IP/CIDR/通配符/区间）或 group:组短码。保存时拒绝，避免配错一行静默不生效。 +func checkExcludeIPLog(raw string) error { + if len(raw) > 10000 { + return errors.New("排除记录日志的IP清单过长（上限10000字符）") + } + for _, line := range strings.FieldsFunc(raw, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' }) { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if len(line) >= len("group:") && strings.EqualFold(line[:len("group:")], "group:") { + if strings.TrimSpace(line[len("group:"):]) == "" { + return errors.New("排除清单里的 group: 后面缺少组短码") + } + continue + } + if _, err := ipset.ParsePatternLenient(line); err != nil { + return fmt.Errorf("排除清单无法识别的IP模式: %s", line) + } + } + return nil +} + // checkCDNPresetTrustSource cdn_preset 模式必须至少有一个可信来源可用(中心库回源段 或 手填可信网段)。 // // 缺了它保存下去不是"少一层校验"，而是静默降级成更危险的状态：来源判定恒为 false， @@ -153,6 +178,10 @@ func (w *WafHostAPi) AddApi(c *gin.Context) { req.IPSourceMode, req.IPTrustDepth, req.IPRealHeader = ipCfg.Mode, ipCfg.Depth, ipCfg.Header req.IPTrustProxies, req.CDNProvider = ipCfg.TrustProxies, ipCfg.Provider + if verr := checkExcludeIPLog(req.EXCLUDE_IP_LOG); verr != nil { + response.FailWithMessage(verr.Error(), c) + return + } // 端口监听表校验（issue #955）：仅当本次显式携带时才阻断（脏数据/addr预留/HTTPS无证书一律拒绝） listens, verr := wafHostService.ValidatePortListensReq(req.PortListensJSON, req.Port, req.Ssl, req.AutoJumpHTTPS) if verr != nil { @@ -452,6 +481,10 @@ func (w *WafHostAPi) ModifyHostApi(c *gin.Context) { wafHostOld := wafHostService.GetDetailByCodeApi(req.CODE) + if verr := checkExcludeIPLog(req.EXCLUDE_IP_LOG); verr != nil { + response.FailWithMessage(verr.Error(), c) + return + } // 端口监听表校验（issue #955）：nil=本次未携带（旧前端），不校验不阻断（存量冲突不能卡死普通编辑） if req.PortListensJSON != nil && strings.TrimSpace(*req.PortListensJSON) != "" { listens, verr := wafHostService.ValidatePortListensReq(*req.PortListensJSON, req.Port, req.Ssl, req.AutoJumpHTTPS) diff --git a/api/waf_system_config.go b/api/waf_system_config.go index 71b094fc..93d48875 100644 --- a/api/waf_system_config.go +++ b/api/waf_system_config.go @@ -211,6 +211,11 @@ func checkSystemConfigValue(item string, value string) (string, bool) { if bad, ok := waf_service.ValidateAttackTagExclude(value); !ok { return fmt.Sprintf("排除标签不合法: %q（不能含引号/分号/反斜杠/注释符/控制字符，单项不超过 255 字符，最多 30 项）", bad), false } + case "exclude_ip_log": + // 与站点级清单同一套校验（checkExcludeIPLog 在 waf_host.go），存坏行当场拒绝 + if err := checkExcludeIPLog(value); err != nil { + return err.Error(), false + } } return "", true } diff --git a/global/global.go b/global/global.go index 8cfe8f38..961cb3f0 100644 --- a/global/global.go +++ b/global/global.go @@ -56,6 +56,7 @@ var ( GWAF_RUNTIME_DNS_TIMEOUT int64 = 500 // DNS 查询超时时间 单位毫秒 GWAF_RUNTIME_RECORD_LOG_TYPE string = "all" // 记录日志形式： 全部(all),非正常(abnormal) + GCONFIG_EXCLUDE_IP_LOG string = "" // 全局排除记录日志的IP清单(所有站点生效)，语法同站点级 exclude_ip_log GWAF_RUNTIME_IS_UPDATETING bool = false //是否正在升级中 GWAF_RUNTIME_CURRENT_EXEPATH string = "" //当前程序运行路径 diff --git a/model/hosts.go b/model/hosts.go index 0ad67fab..f36894d3 100644 --- a/model/hosts.go +++ b/model/hosts.go @@ -28,6 +28,7 @@ type Hosts struct { DEFENSE_JSON string `gorm:"type:text" json:"defense_json"` //自身防御 json START_STATUS int `json:"start_status"` //启动状态 如果是0 启动 ; 如果是1 不启动 EXCLUDE_URL_LOG string `gorm:"type:text" json:"exclude_url_log"` //排除的url开头的数据 换行隔开 + EXCLUDE_IP_LOG string `gorm:"type:text" json:"exclude_ip_log"` //排除记录日志的IP清单 换行或逗号隔开：单IP/CIDR/通配符/区间/group:组短码 IsEnableLoadBalance int `json:"is_enable_load_balance"` //是否激活负载 1 激活 非1 没有激活 LoadBalanceStage int `json:"load_balance_stage"` //负载策略 UnrestrictedPort int `json:"unrestricted_port"` //不限来源匹配端口 0 限制 1，不限制 diff --git a/model/request/waf_host_req.go b/model/request/waf_host_req.go index cf2dfa4e..6d78a131 100644 --- a/model/request/waf_host_req.go +++ b/model/request/waf_host_req.go @@ -19,6 +19,7 @@ type WafHostAddReq struct { DEFENSE_JSON string `json:"defense_json"` //自身防御 json START_STATUS int `json:"start_status"` //启动状态 EXCLUDE_URL_LOG string `json:"exclude_url_log"` //排除的url开头的数据 换行隔开 + EXCLUDE_IP_LOG string `json:"exclude_ip_log"` //排除记录日志的IP清单 换行或逗号隔开：单IP/CIDR/通配符/区间/group:组短码 IsEnableLoadBalance int `json:"is_enable_load_balance"` //是否激活负载 1 激活 非1 没有激活 LoadBalanceStage int `json:"load_balance_stage"` //负载策略 UnrestrictedPort int `json:"unrestricted_port"` //不限来源匹配端口 0 限制 1，不限制 @@ -86,6 +87,7 @@ type WafHostEditReq struct { DEFENSE_JSON string `json:"defense_json"` //自身防御 json START_STATUS int `json:"start_status"` //启动状态 EXCLUDE_URL_LOG string `json:"exclude_url_log"` //排除的url开头的数据 换行隔开 + EXCLUDE_IP_LOG string `json:"exclude_ip_log"` //排除记录日志的IP清单 换行或逗号隔开：单IP/CIDR/通配符/区间/group:组短码 IsEnableLoadBalance int `json:"is_enable_load_balance"` //是否激活负载 1 激活 非1 没有激活 LoadBalanceStage int `json:"load_balance_stage"` //负载策略 UnrestrictedPort int `json:"unrestricted_port"` //不限来源匹配端口 0 限制 1，不限制 diff --git a/model/wafenginmodel/hostsafe.go b/model/wafenginmodel/hostsafe.go index 82f840c0..6a077deb 100644 --- a/model/wafenginmodel/hostsafe.go +++ b/model/wafenginmodel/hostsafe.go @@ -36,6 +36,11 @@ type HostSafe struct { // 与其它字段一样受 RCU 约束：发布后不可就地 append 或改元素，热更新必须整体替换新切片。 IPBlockGroupCodes []string IPWhiteGroupCodes []string + // IPLogExcludeIndex / IPLogExcludeGroupCodes 是本站「排除记录日志的IP」清单的编译结果： + // 前者是单IP/CIDR/通配符/区间的快速索引，后者是清单里 group: 引用的 IP 组短码。 + // 组内容同样不在此缓存，运行时查 ipset 全局快照——改组内容无需重建任何站点。 + IPLogExcludeIndex *ipset.MatchSet + IPLogExcludeGroupCodes []string UrlBlockLists []model.URLBlockList //url 黑名单 LoadBalanceLists []model.LoadBalance //负载均衡 LoadBalanceRuntime *LoadBalanceRuntime //负载运行时 diff --git a/service/waf_service/waf_host.go b/service/waf_service/waf_host.go index 7261b3a0..e08b2815 100644 --- a/service/waf_service/waf_host.go +++ b/service/waf_service/waf_host.go @@ -87,6 +87,7 @@ func (receiver *WafHostService) AddApi(wafHostAddReq request.WafHostAddReq) (str DEFENSE_JSON: wafHostAddReq.DEFENSE_JSON, START_STATUS: wafHostAddReq.START_STATUS, EXCLUDE_URL_LOG: wafHostAddReq.EXCLUDE_URL_LOG, + EXCLUDE_IP_LOG: wafHostAddReq.EXCLUDE_IP_LOG, IsEnableLoadBalance: wafHostAddReq.IsEnableLoadBalance, LoadBalanceStage: wafHostAddReq.LoadBalanceStage, UnrestrictedPort: wafHostAddReq.UnrestrictedPort, @@ -178,6 +179,7 @@ func (receiver *WafHostService) ModifyApi(wafHostEditReq request.WafHostEditReq) "DEFENSE_JSON": wafHostEditReq.DEFENSE_JSON, "START_STATUS": wafHostEditReq.START_STATUS, "EXCLUDE_URL_LOG": wafHostEditReq.EXCLUDE_URL_LOG, + "EXCLUDE_IP_LOG": wafHostEditReq.EXCLUDE_IP_LOG, "IsEnableLoadBalance": wafHostEditReq.IsEnableLoadBalance, "LoadBalanceStage": wafHostEditReq.LoadBalanceStage, "UnrestrictedPort": wafHostEditReq.UnrestrictedPort, diff --git a/wafdb/migrations_core.go b/wafdb/migrations_core.go index a60bd898..3e739d0b 100644 --- a/wafdb/migrations_core.go +++ b/wafdb/migrations_core.go @@ -2342,6 +2342,28 @@ func RunCoreDBMigrations(db *gorm.DB) error { return tx.Migrator().DropTable(&model.IPWatchlist{}) }, }, + // 迁移: 网站表新增「排除记录日志的IP」清单列（M6 H1，全局级清单走系统配置 exclude_ip_log） + { + ID: "202609210001_add_hosts_exclude_ip_log", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609210001: 网站新增排除记录日志的IP清单列") + if tx.Migrator().HasColumn(&model.Hosts{}, "EXCLUDE_IP_LOG") { + return nil + } + if err := tx.Migrator().AddColumn(&model.Hosts{}, "EXCLUDE_IP_LOG"); err != nil { + return fmt.Errorf("新增网站排除IP列失败: %w", err) + } + zlog.Info("迁移 202609210001: 完成") + return nil + }, + Rollback: func(tx *gorm.DB) error { + zlog.Info("回滚 202609210001: 删除网站排除记录日志的IP清单列") + if tx.Migrator().HasColumn(&model.Hosts{}, "EXCLUDE_IP_LOG") { + return tx.Migrator().DropColumn(&model.Hosts{}, "EXCLUDE_IP_LOG") + } + return nil + }, + }, }) // 执行迁移 diff --git a/wafenginecore/checkrule_scope_test.go b/wafenginecore/checkrule_scope_test.go index 68ee0bc7..721b4717 100644 --- a/wafenginecore/checkrule_scope_test.go +++ b/wafenginecore/checkrule_scope_test.go @@ -230,7 +230,8 @@ func TestShouldRecordWebLog(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { global.GWAF_RUNTIME_RECORD_LOG_TYPE = tc.logType - if got := shouldRecordWebLog(tc.weblog, tc.excludeURL); got != tc.want { + hostSafe := &wafenginmodel.HostSafe{Host: model.Hosts{EXCLUDE_URL_LOG: tc.excludeURL}} + if got := shouldRecordWebLog(tc.weblog, hostSafe); got != tc.want { t.Errorf("期望 %v 实际 %v", tc.want, got) } }) diff --git a/wafenginecore/static_server.go b/wafenginecore/static_server.go index 6bf0526b..318c2ae7 100644 --- a/wafenginecore/static_server.go +++ b/wafenginecore/static_server.go @@ -614,7 +614,7 @@ func (waf *WafEngine) logStaticFileAccess(path, remoteAddr string, fileSize int6 // 按照全局日志记录策略决定是否记录 // abnormal 模式下静态文件成功访问(ACTION=放行)通常不记录，但若命中过自定义规则(RULE 非空)仍要留痕 - if shouldRecordWebLog(weblog, hostsafe.Host.EXCLUDE_URL_LOG) { + if shouldRecordWebLog(weblog, hostsafe) { global.GQEQUE_LOG_DB.Enqueue(weblog) } } diff --git a/wafenginecore/wafengine.go b/wafenginecore/wafengine.go index b65d3cff..493c7fb7 100644 --- a/wafenginecore/wafengine.go +++ b/wafenginecore/wafengine.go @@ -1184,7 +1184,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { weblogfrist.TASK_FLAG = 1 // 记录日志 - if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host].Host.EXCLUDE_URL_LOG) { + if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host]) { global.GQEQUE_LOG_DB.Enqueue(weblogfrist) } } @@ -1221,7 +1221,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { datetimeNow := time.Now() weblogfrist.TimeSpent = datetimeNow.UnixNano()/1e6 - weblogfrist.UNIX_ADD_TIME weblogfrist.BackendCheckCost = datetimeNow.UnixNano()/1e6 - backendCheckStart - if shouldRecordWebLog(weblogfrist, hostTarget.Host.EXCLUDE_URL_LOG) { + if shouldRecordWebLog(weblogfrist, hostTarget) { global.GQEQUE_LOG_DB.Enqueue(weblogfrist) } return nil @@ -1272,7 +1272,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { weblogfrist.BackendCheckCost = time.Now().UnixNano()/1e6 - backendCheckStart //响应数据处理时间 // 记录流式访问日志 - if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host].Host.EXCLUDE_URL_LOG) { + if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host]) { global.GQEQUE_LOG_DB.Enqueue(weblogfrist) } @@ -1439,7 +1439,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { weblogfrist.BackendCheckCost = time.Now().UnixNano()/1e6 - backendCheckStart // 记录日志 - 根据配置决定是否记录 - if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host].Host.EXCLUDE_URL_LOG) { + if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host]) { global.GQEQUE_LOG_DB.Enqueue(weblogfrist) } @@ -1457,7 +1457,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { weblogfrist.STATUS_CODE = resp.StatusCode weblogfrist.TASK_FLAG = 1 weblogfrist.BackendCheckCost = time.Now().UnixNano()/1e6 - backendCheckStart //响应数据处理时间 - if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host].Host.EXCLUDE_URL_LOG) { + if shouldRecordWebLog(weblogfrist, waf.rt().HostTarget[host]) { global.GQEQUE_LOG_DB.Enqueue(weblogfrist) } diff --git a/wafenginecore/wafworker.go b/wafenginecore/wafworker.go index a633fe37..be7294aa 100644 --- a/wafenginecore/wafworker.go +++ b/wafenginecore/wafworker.go @@ -204,6 +204,8 @@ func (waf *WafEngine) LoadHost(inHost model.Hosts) []innerbean.ServerRunTime { IPBlockLists: ipblocklist, IPBlockIndex: BuildIPBlockIndex(ipblocklist), IPBlockGroupCodes: ExtractBlockGroupCodes(ipblocklist), + IPLogExcludeIndex: BuildIPLogExcludeIndex(inHost.EXCLUDE_IP_LOG), + IPLogExcludeGroupCodes: ExtractIPLogExcludeGroupCodes(inHost.EXCLUDE_IP_LOG), UrlBlockLists: urlblocklist, AntiCCBean: anticcBean, HttpAuthBases: httpAuthList, diff --git a/wafenginecore/weblog_exclude.go b/wafenginecore/weblog_exclude.go new file mode 100644 index 00000000..8ca3a1be --- /dev/null +++ b/wafenginecore/weblog_exclude.go @@ -0,0 +1,115 @@ +package wafenginecore + +import ( + "SamWaf/model/wafenginmodel" + "SamWaf/wafenginecore/ipset" + "strings" + "sync/atomic" +) + +// 写入侧排除：按来源 IP 不记录访问日志（M6 H1）。 +// +// 清单是全局 + 站点两级（D15），行格式： +// - 单IP / CIDR / 通配符 / 区间（语法同 ipset.MatchSet） +// - group:组短码 引用一个 IP 组（组内容走 ipset 全局快照，改组即全部引用方生效） +// - # 开头是注释；空行跳过；逗号与换行都算分隔 +// +// 静音程度按 D13：只静音正常请求，安全事件照常留痕——办公出口多是 NAT 共享 IP， +// 一刀切静音意味着真被打了也看不见。判定挂在 shouldRecordWebLog（入队前）， +// 被排除的请求同样不进三层落库 / stats_* / 分析层汇总。 + +// parseIPLogExcludeLines 把清单文本拆成 IP 模式与组短码两组；组短码去重且保持出现顺序。 +func parseIPLogExcludeLines(raw string) (patterns []string, groupCodes []string) { + seen := map[string]struct{}{} + for _, line := range strings.FieldsFunc(raw, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' }) { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if len(line) >= len("group:") && strings.EqualFold(line[:len("group:")], "group:") { + code := strings.TrimSpace(line[len("group:"):]) + if code == "" { + continue + } + if _, dup := seen[code]; !dup { + seen[code] = struct{}{} + groupCodes = append(groupCodes, code) + } + continue + } + patterns = append(patterns, line) + } + return patterns, groupCodes +} + +// BuildIPLogExcludeIndex 编译站点级清单里的 IP 模式；空清单返回 nil。 +func BuildIPLogExcludeIndex(raw string) *ipset.MatchSet { + patterns, _ := parseIPLogExcludeLines(raw) + if len(patterns) == 0 { + return nil + } + return ipset.BuildMatchSet(patterns) +} + +// ExtractIPLogExcludeGroupCodes 抽出站点级清单里引用的组短码（去重保序）。 +func ExtractIPLogExcludeGroupCodes(raw string) []string { + _, codes := parseIPLogExcludeLines(raw) + return codes +} + +// ipLogExcludeCompiled 是全局清单的编译结果，配置热更新时整体替换。 +type ipLogExcludeCompiled struct { + index *ipset.MatchSet + groupCodes []string +} + +func (c *ipLogExcludeCompiled) match(clientIP string) bool { + if c == nil { + return false + } + if c.index.ContainsStr(clientIP) { + return true + } + for _, code := range c.groupCodes { + if ipset.GetGroupMatcher(code).ContainsStr(clientIP) { + return true + } + } + return false +} + +// globalIPLogExclude 全局排除清单的编译快照；nil 表示未配置。 +// MatchSet.ContainsStr / GetGroupMatcher 均为 nil 安全（返回 false）。 +var globalIPLogExclude atomic.Pointer[ipLogExcludeCompiled] + +// SetGlobalIPLogExclude 由系统配置热更新入口调用（waftask/task_config.go）。 +func SetGlobalIPLogExclude(raw string) { + patterns, codes := parseIPLogExcludeLines(raw) + var index *ipset.MatchSet + if len(patterns) > 0 { + index = ipset.BuildMatchSet(patterns) + } + globalIPLogExclude.Store(&ipLogExcludeCompiled{index: index, groupCodes: codes}) +} + +// isIPLogExcluded 判定来源 IP 是否命中全局或站点级排除清单。hostSafe 可为 nil（只判全局）。 +func isIPLogExcluded(clientIP string, hostSafe *wafenginmodel.HostSafe) bool { + if clientIP == "" { + return false + } + if globalIPLogExclude.Load().match(clientIP) { + return true + } + if hostSafe == nil { + return false + } + if hostSafe.IPLogExcludeIndex.ContainsStr(clientIP) { + return true + } + for _, code := range hostSafe.IPLogExcludeGroupCodes { + if ipset.GetGroupMatcher(code).ContainsStr(clientIP) { + return true + } + } + return false +} diff --git a/wafenginecore/weblog_exclude_test.go b/wafenginecore/weblog_exclude_test.go new file mode 100644 index 00000000..adc334d0 --- /dev/null +++ b/wafenginecore/weblog_exclude_test.go @@ -0,0 +1,155 @@ +package wafenginecore + +import ( + "SamWaf/common/queue" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/wafenginmodel" + "SamWaf/wafenginecore/accessgate" + "SamWaf/wafenginecore/ipset" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestParseIPLogExcludeLines(t *testing.T) { + patterns, codes := parseIPLogExcludeLines(` +# 办公出口 +203.0.113.7, 198.51.100.0/24 +10.0.0.1-10.0.0.9 +group:office-net +Group:Probe-Net +group:office-net +`) + if !reflect.DeepEqual(patterns, []string{"203.0.113.7", "198.51.100.0/24", "10.0.0.1-10.0.0.9"}) { + t.Errorf("patterns = %v", patterns) + } + // 组短码去重保序；group: 前缀大小写不敏感，组短码本身保持原样 + if !reflect.DeepEqual(codes, []string{"office-net", "Probe-Net"}) { + t.Errorf("codes = %v", codes) + } + + patterns, codes = parseIPLogExcludeLines(" \n# 只有注释\n") + if patterns != nil || codes != nil { + t.Errorf("空清单应得到 nil,nil，实际 %v %v", patterns, codes) + } +} + +func TestBuildIPLogExcludeIndexMatch(t *testing.T) { + idx := BuildIPLogExcludeIndex("203.0.113.7,198.51.100.0/24,10.*.*.*,坏行!,group:不会被收进索引") + if idx == nil { + t.Fatal("非空清单应返回索引") + } + for ip, want := range map[string]bool{ + "203.0.113.7": true, + "198.51.100.9": true, + "10.1.2.3": true, + "203.0.113.8": false, + "192.0.2.1": false, + } { + if got := idx.ContainsStr(ip); got != want { + t.Errorf("ContainsStr(%s) = %v, want %v", ip, got, want) + } + } + if BuildIPLogExcludeIndex("") != nil || BuildIPLogExcludeIndex("# 注释\n") != nil { + t.Error("空清单应返回 nil 索引") + } +} + +// TestShouldRecordWebLogIPExclude D13 语义：排除 IP 只静音正常请求，安全事件照记。 +func TestShouldRecordWebLogIPExclude(t *testing.T) { + origin := global.GWAF_RUNTIME_RECORD_LOG_TYPE + defer func() { global.GWAF_RUNTIME_RECORD_LOG_TYPE = origin }() + SetGlobalIPLogExclude("") + defer SetGlobalIPLogExclude("") + + mkHost := func(raw string) *wafenginmodel.HostSafe { + return &wafenginmodel.HostSafe{ + Host: model.Hosts{EXCLUDE_IP_LOG: raw}, + IPLogExcludeIndex: BuildIPLogExcludeIndex(raw), + IPLogExcludeGroupCodes: ExtractIPLogExcludeGroupCodes(raw), + } + } + cases := []struct { + name string + logType string + weblog *innerbean.WebLog + hostRaw string + want bool + }{ + {"all_排除IP的正常请求不记", "all", + &innerbean.WebLog{ACTION: "放行", URL: "/a", SRC_IP: "203.0.113.7"}, + "203.0.113.7", false}, + {"all_排除IP的安全事件照记", "all", + &innerbean.WebLog{ACTION: "阻止", RULE: "SQL注入", URL: "/a", SRC_IP: "203.0.113.7"}, + "203.0.113.7", true}, + {"abnormal_排除IP的安全事件照记", "abnormal", + &innerbean.WebLog{ACTION: "阻止", RULE: "SQL注入", URL: "/a", SRC_IP: "203.0.113.7"}, + "203.0.113.7", true}, + {"all_未排除IP不受影响", "all", + &innerbean.WebLog{ACTION: "放行", URL: "/a", SRC_IP: "198.51.100.1"}, + "203.0.113.7", true}, + {"all_排除网段命中", "all", + &innerbean.WebLog{ACTION: "放行", URL: "/a", SRC_IP: "10.0.0.9"}, + "10.0.0.0/8", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + global.GWAF_RUNTIME_RECORD_LOG_TYPE = tc.logType + if got := shouldRecordWebLog(tc.weblog, mkHost(tc.hostRaw)); got != tc.want { + t.Errorf("期望 %v 实际 %v", tc.want, got) + } + }) + } + + t.Run("all_排除清单里的IP组命中", func(t *testing.T) { + global.GWAF_RUNTIME_RECORD_LOG_TYPE = "all" + ipset.UpsertGroupMatcher("m6-test-grp", "测试组", ipset.BuildMatchSet([]string{"192.0.2.66"})) + defer ipset.RemoveGroupMatcher("m6-test-grp") + hostSafe := mkHost("group:m6-test-grp") + if got := shouldRecordWebLog(&innerbean.WebLog{ACTION: "放行", URL: "/a", SRC_IP: "192.0.2.66"}, hostSafe); got { + t.Error("组内 IP 的正常请求应被排除") + } + }) + + t.Run("all_全局排除清单命中", func(t *testing.T) { + global.GWAF_RUNTIME_RECORD_LOG_TYPE = "all" + SetGlobalIPLogExclude("203.0.113.99") + defer SetGlobalIPLogExclude("") + hostSafe := mkHost("") + if got := shouldRecordWebLog(&innerbean.WebLog{ACTION: "放行", URL: "/a", SRC_IP: "203.0.113.99"}, hostSafe); got { + t.Error("全局排除命中，正常请求不应记录") + } + if got := shouldRecordWebLog(&innerbean.WebLog{ACTION: "放行", URL: "/a", SRC_IP: "203.0.113.100"}, hostSafe); !got { + t.Error("未命中全局排除，照常记录") + } + }) +} + +// TestAccessPingNotEnqueued H2：自带 /<前缀>/ping 探测点由访问认证网关直接代答 204， +// 整条路径不入日志队列。该行为此前无测试钉住（实施计划 §5.8 曾误以为它会产生日志）。 +func TestAccessPingNotEnqueued(t *testing.T) { + accessgate.SetConfig(&accessgate.Config{PathPrefix: "/samwaf_access"}) + defer accessgate.SetConfig(nil) + + origin := global.GQEQUE_LOG_DB + global.GQEQUE_LOG_DB = queue.NewQueue() + defer func() { global.GQEQUE_LOG_DB = origin }() + + waf := &WafEngine{} + req := httptest.NewRequest(http.MethodGet, "http://www.example.com/samwaf_access/ping", nil) + rec := httptest.NewRecorder() + hostSafe := &wafenginmodel.HostSafe{Host: model.Hosts{Code: "m6-test"}} + weblog := &innerbean.WebLog{SRC_IP: "192.0.2.10", URL: "/samwaf_access/ping"} + + waf.handleAccessRequest(rec, req, hostSafe, weblog, accessgate.Get(), model.HostAccessConfig{}, "192.0.2.10") + + if rec.Code != http.StatusNoContent { + t.Errorf("ping 应回 204，实际 %d", rec.Code) + } + if global.GQEQUE_LOG_DB.Size() != 0 { + t.Error("ping 请求不应进日志队列") + } +} diff --git a/wafenginecore/weblog_record.go b/wafenginecore/weblog_record.go index fbb416ae..788ba1db 100644 --- a/wafenginecore/weblog_record.go +++ b/wafenginecore/weblog_record.go @@ -3,6 +3,7 @@ package wafenginecore import ( "SamWaf/global" "SamWaf/innerbean" + "SamWaf/model/wafenginmodel" "strings" ) @@ -29,11 +30,16 @@ func isURLLogExcluded(url string, excludeURLLog string) bool { // 自定义规则放行/仅记录、站点仅记录模式(LogOnlyMode)、AI 观察命中，这些请求最终 // ACTION 都会被 modifyResponse 覆写成"放行"，若只看 ACTION 就会被整条丢弃， // 导致白名单被谁用了、仅记录模式抓到了什么完全无法审计。 -func shouldRecordWebLog(weblog *innerbean.WebLog, excludeURLLog string) bool { - if weblog == nil { +// +// 排除清单两级：URL 前缀（彻底静音）与来源 IP（只静音正常请求，安全事件照记，见 weblog_exclude.go）。 +func shouldRecordWebLog(weblog *innerbean.WebLog, hostSafe *wafenginmodel.HostSafe) bool { + if weblog == nil || hostSafe == nil { + return false + } + if isURLLogExcluded(weblog.URL, hostSafe.Host.EXCLUDE_URL_LOG) { return false } - if isURLLogExcluded(weblog.URL, excludeURLLog) { + if !weblog.IsSecurityEvent() && isIPLogExcluded(weblog.SRC_IP, hostSafe) { return false } switch global.GWAF_RUNTIME_RECORD_LOG_TYPE { diff --git a/waftask/task_config.go b/waftask/task_config.go index 3753ac0c..85999037 100644 --- a/waftask/task_config.go +++ b/waftask/task_config.go @@ -384,6 +384,11 @@ func setConfigStringValue(name string, value string, change int) { case "attack_tag_exclude": global.GCONFIG_ATTACK_TAG_EXCLUDE = value break + case "exclude_ip_log": + global.GCONFIG_EXCLUDE_IP_LOG = value + // 同步编译全局排除快照，保存即生效，无需重启 + wafenginecore.SetGlobalIPLogExclude(value) + break case "gwaf_proxy_header": global.GCONFIG_RECORD_PROXY_HEADER = value break @@ -845,6 +850,7 @@ func TaskLoadSetting(initLoad bool) { updateConfigIntItem(initLoad, "database", "log_persist_enable", global.GCONFIG_LOG_PERSIST_ENABLED, "是否开启日志持久化（1开启 0关闭）", "options", "0|关闭,1|开启", configMap) updateConfigIntItem(initLoad, "database", "ip_tag_db", global.GDATA_IP_TAG_DB, "IP Tag 存放位置 0 是主库 1是读取 stat库", "int", "", configMap) updateConfigStringItem(initLoad, "system", "attack_tag_exclude", global.GCONFIG_ATTACK_TAG_EXCLUDE, "风险日志不算风险的标签(逗号分隔,如ACME证书校验)，这些标签不出现在规则筛选里也不计入阻止数量；正常 始终排除", "string", "", configMap) + updateConfigStringItem(initLoad, "system", "exclude_ip_log", global.GCONFIG_EXCLUDE_IP_LOG, "全局排除记录日志的IP（所有站点生效）：单IP/CIDR/通配符/区间，或 group:组短码 引用IP组；逗号或换行分隔，# 开头为注释。只静音正常请求，安全事件照常记录；站点级清单在网站编辑里", "string", "", configMap) // IP失败封禁相关配置 updateConfigStringItem(initLoad, "security", "ip_failure_status_codes", global.GCONFIG_IP_FAILURE_STATUS_CODES, "失败状态码配置，支持多个用|分隔，也支持正则表达式，例如：401|403|404|444|429|503 或 ^4[0-9]{2}$", "string", "", configMap) From 55e720256bd14156c1b7b916a1415421d903fe98 Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 24 Sep 2026 10:37:37 +0800 Subject: [PATCH 14/21] docs: add upgrade notes for the analysis page and log IP exclusion --- waftask/stat_iptag_crossdb_test.go | 124 ++++++++++++++++++++++++++++ wafupgradenotice/upgrade_notes.yaml | 91 ++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 waftask/stat_iptag_crossdb_test.go diff --git a/waftask/stat_iptag_crossdb_test.go b/waftask/stat_iptag_crossdb_test.go new file mode 100644 index 00000000..1e7e381c --- /dev/null +++ b/waftask/stat_iptag_crossdb_test.go @@ -0,0 +1,124 @@ +//go:build crossdb + +// ip_tags 增量写入的三库回归。 +// +// 这一步是整批一条 upsert（冲突目标 = uni_iptags_full 的四列，累加表达式走 +// dialect.UpsertExcludedRef，PG/SQLite 是 excluded.cnt、MySQL 是 VALUES(cnt)）。 +// 任一引擎对不上，症状都是不报错的静默错账：要么每批新建一行、同一个 IP+标签出现多行， +// 要么 cnt 被覆盖成最后一批的值而不是累加。所以三边各跑一遍真实迁移 + 真实写入。 +// +// 跑法：go test -tags crossdb ./waftask/ -run TestCollectStatsIPTagCrossEngine +// MySQL / PostgreSQL 连不上时跳过，不阻塞。 +package waftask + +import ( + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "testing" + + "gorm.io/gorm" +) + +func TestCollectStatsIPTagCrossEngine(t *testing.T) { + engines := map[string]func(*testing.T) (*gorm.DB, func()){ + "sqlite": setupAnalysisSQLite, + "mysql": setupAnalysisMySQL, + "postgres": setupAnalysisPostgres, + } + for name, setup := range engines { + t.Run(name, func(t *testing.T) { + db, done := setup(t) + if db == nil { + t.Skip("引擎不可用，跳过") + } + defer done() + runIPTagUpsertAssertions(t, db) + }) + } +} + +func runIPTagUpsertAssertions(t *testing.T, db *gorm.DB) { + t.Helper() + + prevStats, prevCore, prevTagDB := global.GWAF_LOCAL_STATS_DB, global.GWAF_LOCAL_DB, global.GDATA_IP_TAG_DB + prevUser, prevTenant := global.GWAF_USER_CODE, global.GWAF_TENANT_ID + global.GWAF_LOCAL_STATS_DB = db + global.GWAF_LOCAL_DB = db // 只用于 CollectStatsFromLogs 的非空检查 + global.GDATA_IP_TAG_DB = 1 + global.GWAF_USER_CODE = "xtest_user" + global.GWAF_TENANT_ID = "xtest_tenant" + defer func() { + global.GWAF_LOCAL_STATS_DB, global.GWAF_LOCAL_DB, global.GDATA_IP_TAG_DB = prevStats, prevCore, prevTagDB + global.GWAF_USER_CODE, global.GWAF_TENANT_ID = prevUser, prevTenant + }() + + const day = 20260924 + entry := func(ip, rule, action string) *innerbean.WebLog { + return &innerbean.WebLog{ + HOST_CODE: "h1", + HOST: "x.test.com", + SRC_IP: ip, + RULE: rule, + ACTION: action, + Day: day, + USER_CODE: "xtest_user", + TenantId: "xtest_tenant", + UNIX_ADD_TIME: 1758672000000, + } + } + batch := []*innerbean.WebLog{ + entry("1.1.1.1", "SQL注入", "阻止"), + entry("1.1.1.1", "SQL注入", "阻止"), + entry("1.1.1.1", "XSS", "阻止"), + entry("2.2.2.2", "SQL注入", "阻止"), + entry("3.3.3.3", "", "放行"), // 未命中规则：不该产生任何标签（D7） + } + + // 两批相同数据：第二批必须落在 upsert 的累加分支上，而不是新建一行 + CollectStatsFromLogs(batch) + CollectStatsFromLogs(batch) + + var rows []model.IPTag + if err := db.Order("ip, ip_tag").Find(&rows).Error; err != nil { + t.Fatalf("读 ip_tags 失败: %v", err) + } + if len(rows) != 3 { + t.Fatalf("三个 (ip,标签) 组合应只有 3 行，实际 %d 行：upsert 没命中 uni_iptags_full", len(rows)) + } + + want := map[string]int64{ + "1.1.1.1|SQL注入": 4, // 每批 2 次 × 2 批 + "1.1.1.1|XSS": 2, + "2.2.2.2|SQL注入": 2, + } + for _, r := range rows { + key := r.IP + "|" + r.IPTag + exp, ok := want[key] + if !ok { + t.Fatalf("出现意料外的标签行 %s（未命中规则的请求不该生成标签）", key) + } + if r.Cnt != exp { + t.Fatalf("%s 的 cnt 应累加成 %d，实际 %d（upsert 是覆盖而不是累加？）", key, exp, r.Cnt) + } + if r.USER_CODE != "xtest_user" || r.Tenant_ID != "xtest_tenant" { + t.Fatalf("%s 的归属列不对: user=%q tenant=%q", key, r.USER_CODE, r.Tenant_ID) + } + } + + // 放行的那条 IP 只该出现在 stats_ip_days 里，不该有标签 + var tagCnt int64 + if err := db.Model(&model.IPTag{}).Where("ip = ?", "3.3.3.3").Count(&tagCnt).Error; err != nil { + t.Fatalf("查放行 IP 标签失败: %v", err) + } + if tagCnt != 0 { + t.Fatalf("未命中规则的 IP 不该有标签，实际 %d 行", tagCnt) + } + var dayCnt int64 + if err := db.Model(&model.StatsIPDay{}).Where("ip = ? and day = ?", "3.3.3.3", day).Count(&dayCnt).Error; err != nil { + t.Fatalf("查 stats_ip_days 失败: %v", err) + } + if dayCnt == 0 { + t.Fatal("放行数量应由 stats_ip_days 承担，这里一行都没有") + } +} diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index 696b8d58..a4a2082c 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -734,3 +734,94 @@ notes: (access log / security events / payloads / legacy) into a fresh encrypted SQLite file. Tables are created on the first start after upgrading; existing data is untouched and stays readable. + - id: v1_3_25_log_source_path_analysis + version: v1.3.25 + kind: notice + level: normal + page: /wafanalysis/wafanalysissourcepath + doc: https://doc.samwaf.com/quickstart/Update.html + apply: + type: navigate + zh: + title: 新增「来源与路径分析」：从来源和路径两个视角看流量构成 + detail: >- + 「数据分析 - 来源与路径分析」是本版本新增的独立页面，把访问日志按天汇总后分两个视角呈现： + 按来源看哪些 IP 或访客打得最多（请求数、命中规则次数、去重路径数、UA 分布）， + 按路径看哪些接口被访问得最多（请求数、去重来源数、规则分布）。两个视角都可以按站点和天数筛选， + 表格里可以直接对某个来源 IP 加黑或加白（需要先选定站点，因为黑白名单是按站点生效的）。 + 数据来自本版本新增的按天汇总表，在日志写入时同步累加，不去扫原始日志表， + 所以即便把「访问日志档位」调成「采样入库」或「仅安全事件」，这个页面依然有数。 + 路径先模板化再汇总——数字段、UUID、hex 串归一成占位符，/user/1001 与 /user/1002 会聚成同一条； + 每个站点每天的模板数封顶 5000，超出的归入一条 {overflow}，避免随机路径把表刷爆。 + 需要知道的是：汇总只从升级之后的流量开始积累，历史日志不回填， + 所以刚升完级这个页面是空的，跑满一天之后才有完整的昨天数据。 + 新增的汇总表已经列入「数据保留策略」页，默认不自动清理，与现有各表的默认一致， + 觉得占空间可以自己在那里开启定期清理。 + en: + title: New Source and Path Analysis page, viewing traffic by origin and by path + detail: >- + Data Analysis - Source and Path Analysis is a new standalone page in this version. It rolls up access + logs per day and presents them from two angles: by source, showing which IPs or visitors send the most + requests (request count, rule hits, distinct paths, UA distribution), and by path, showing which + endpoints are hit the most (request count, distinct sources, rule distribution). Both angles filter by + website and number of days, and the table lets you block or allow a source IP directly - you pick the + website first, since block and allow lists apply per website. + The data comes from new daily rollup tables accumulated as logs are written rather than by scanning the + raw log tables, so the page still has data even when the access-log mode is set to sampled or + security-events-only. + Paths are templated before being rolled up - numeric segments, UUIDs and hex strings become + placeholders, so /user/1001 and /user/1002 collapse into one row. Each website is capped at 5000 + templates per day and anything beyond that lands in a single {overflow} row, so random paths cannot + flood the table. + Worth knowing: the rollups only start accumulating from traffic after the upgrade and historical logs + are not backfilled, so the page is empty right after upgrading and gives a complete picture of + yesterday only once a full day has passed. + The new rollup tables are listed on the Data Retention page with automatic cleanup off by default, + matching every other table there; turn it on yourself if the space matters to you. + + - id: v1_3_25_weblog_exclude_ip + version: v1.3.25 + kind: notice + level: normal + page: /waf/wafvisitlog + doc: https://doc.samwaf.com/quickstart/Update.html + apply: + type: navigate + zh: + title: 可按来源IP排除日志记录：办公出口与探测机的噪音不再进日志 + detail: >- + 以往只能按 URL 前缀排除记录（在网站编辑里），本版本新增按来源 IP 排除，两个作用域各一份： + 全局清单在「防护日志 - 访问日志」页的「日志配置」里，站点清单在网站编辑的「记录日志时排除IP」里。 + 清单语法与黑白名单完全一致——单个 IP、CIDR、通配符、区间，或者 group:组短码 引用一个 IP 组 + （改组内容对所有引用它的站点立即生效），逗号或换行分隔，# 开头为注释。 + 写错的模式在保存时就被拒绝并指出是哪一条，不会静默忽略；两份清单都不需要重启，保存即生效。 + 只静音正常请求：被排除 IP 的安全事件（命中规则、被拦截）照常记录。这是刻意这样定的—— + 办公出口往往是一个 NAT 共享地址，一刀切静音等于「真被人打了也看不见」。 + 被排除的请求不写日志、不计入请求数统计、也不进新的来源与路径汇总； + 但站点的进出流量字节仍然照计（字节计量在引擎侧，不经日志队列），所以流量图上还能看到它们， + 这是预期行为，不是漏排。 + 默认两份清单都是空的，升级后行为与之前完全一致，不需要任何操作。 + 另外顺带说明：SamWaf 自带的访问认证探测点（`/<访问认证前缀>/ping`，恒回 204）从来不产生访问日志， + 不需要也无法为它配置排除。 + en: + title: Exclude log recording by source IP, keeping office exits and probes out of the logs + detail: >- + Until now records could only be excluded by URL prefix in the website editor. This version adds + exclusion by source IP, with one list per scope: the global list lives in Log Settings on the + Protection Logs - Access Log page, and the per-site list in the website editor as "Exclude IP When + Logging". + The syntax matches the block and allow lists exactly - a single IP, CIDR, wildcard, range, or + group:CODE referencing an IP group (changing the group's contents takes effect immediately for every + website naming it). Entries are comma- or newline-separated and # starts a comment. + A malformed pattern is rejected when you save, naming the offending line rather than being silently + ignored, and neither list needs a restart - saving is enough. + Only plain requests are silenced: security events from an excluded IP (rule hits, blocked requests) are + still recorded. That is deliberate - an office exit is often one NAT-shared address, and silencing it + outright would mean a real attack from behind it becomes invisible. + Excluded requests write no log row, are not counted in request statistics, and do not reach the new + source and path rollups. Their inbound and outbound bytes are still metered, because byte accounting + happens in the engine and never passes through the log queue, so they remain visible on the traffic + charts - that is expected, not a missed exclusion. + Both lists are empty by default, so behavior after upgrading is exactly as before and nothing needs to + be done. As a side note, SamWaf's own access-authentication probe endpoint (`//ping`, + always 204) never produces an access log, so there is nothing to exclude for it. From 13220fe5f69c3bc40a412ab2a58528268a3406e9 Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 28 Sep 2026 09:47:18 +0800 Subject: [PATCH 15/21] feat: derive the log partition from the time range and look up identifiers across all of them --- api/waf_log.go | 50 +- .../cross_engine_logfanout_test.go | 344 +++++++++ service/waf_service/cross_engine_test.go | 6 + service/waf_service/waf_log.go | 232 +----- service/waf_service/waf_log_query.go | 689 ++++++++++++++++++ wafupgradenotice/upgrade_notes.yaml | 105 +++ 6 files changed, 1205 insertions(+), 221 deletions(-) create mode 100644 service/waf_service/cross_engine_logfanout_test.go create mode 100644 service/waf_service/waf_log_query.go diff --git a/api/waf_log.go b/api/waf_log.go index 5d170956..f6144fa4 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -73,15 +73,24 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) { response.FailWithMessage("正在切换数据库请等待", c) return } - wafLogs, total, err2 := wafLogService.GetListApi(req) + wafLogs, total, meta, err2 := wafLogService.GetListApiWithMeta(req) if err2 != nil { response.FailWithMessage("访问列表失败:"+err2.Error(), c) } else { - response.OkWithDetailed(response.PageResult{ - List: wafLogs, - Total: total, - PageIndex: req.PageIndex, - PageSize: req.PageSize, + // 除了列表本身，还要告诉前端这次到底查了哪些分区： + // 自动模式下用户没选分区，界面得能说明数据是从哪来的、识别码是在哪找到的 + response.OkWithDetailed(gin.H{ + "list": wafLogs, + "total": total, + "pageIndex": req.PageIndex, + "pageSize": req.PageSize, + "shards": meta.Shards, + "found_in": meta.FoundIn, + "scanned": meta.Scanned, + "uuid_lookup": meta.UuidLookup, + "sort_forced_time": meta.SortForcedTime, + "partial": meta.Partial, + "took_ms": meta.TookMs, }, "获取成功", c) } @@ -250,7 +259,7 @@ func (w *WafLogAPi) GetListByHostCodeApi(c *gin.Context) { } func (w *WafLogAPi) GetAllShareDbApi(c *gin.Context) { - wafShareList, _ := wafShareDbService.GetAllShareDbApi() + wafShareList, _ := wafShareDbService.GetAllShareDbWithTiers() liveName := wafdb.LiveLogName() // 当前驱动的实时分片标识，用于标记默认选中项 allShareDbRep := make([]response2.AllShareDbRep, len(wafShareList)) // 创建数组 for i, _ := range wafShareList { @@ -261,12 +270,39 @@ func (w *WafLogAPi) GetAllShareDbApi(c *gin.Context) { FileName: wafShareList[i].FileName, Cnt: wafShareList[i].Cnt, IsCurrent: wafShareList[i].FileName == liveName, + PeriodKey: wafShareList[i].PeriodKey, + Tiers: wafShareList[i].Tiers, } } response.OkWithDetailed(allShareDbRep, "获取成功", c) } +// DelShardApi 主动删除一个归档分区（不看保留期）。 +// +// 以前想立刻腾空间只能去「文件管理」删 .db 文件——那条路只对 SQLite 有效， +// 而且删完 share_dbs 记录还在，归档下拉里会留着一个已经不存在的分区。 +// 这里按分区删：文件型删文件、服务型丢表，两边都把记录与计数缓存一并清掉。 +func (w *WafLogAPi) DelShardApi(c *gin.Context) { + var req struct { + FileName string `json:"file_name"` + } + if err := c.ShouldBindJSON(&req); err != nil { + response.FailWithMessage("解析失败", c) + return + } + if global.GDATA_CURRENT_CHANGE { + response.FailWithMessage("正在切换数据库请等待", c) + return + } + detail, err := wafShareDbService.ForceDeleteShard(req.FileName) + if err != nil { + response.FailWithMessage("删除分区失败："+err.Error(), c) + return + } + response.OkWithMessage(detail, c) +} + // http 原始请求并进行脱敏处理 func (w *WafLogAPi) GetHttpCopyMaskApi(c *gin.Context) { var req request.WafAttackLogDetailReq diff --git a/service/waf_service/cross_engine_logfanout_test.go b/service/waf_service/cross_engine_logfanout_test.go new file mode 100644 index 00000000..7b986298 --- /dev/null +++ b/service/waf_service/cross_engine_logfanout_test.go @@ -0,0 +1,344 @@ +//go:build crossdb + +// 日志查询的分区扇出与识别码直查（E5）的三库回归。 +// +// 这两条路都会「悄悄地错」：扇出翻页算错偏移就是漏行或重复行，用户看不出来； +// 识别码直查要是没忽略时间条件，就会出现「码是对的、却说查不到」。所以必须在真库上按行核对。 +// MySQL/PG 的归档分区是同库里的表，建起来就能测；SQLite 的分区是独立文件， +// 这里只验它的实时库那一路（多文件分片由 m3_verify.py 在真实实例上验）。 +package waf_service + +import ( + "SamWaf/customtype" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/model/request" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "SamWaf/wafdb/partition" + "fmt" + "testing" + "time" + + "gorm.io/gorm" +) + +func runLogFanoutCases(t *testing.T, core, logdb *gorm.DB) { + d := dialect.Get() + base := model.AccessLogTableName + now := time.Now() + + // 三段时间，各 3 行；两段进归档分区，一段留在实时表 + keyOld := partition.KeyOf(now.AddDate(0, -2, 0)) + keyMid := partition.KeyOf(now.AddDate(0, -1, 0)) + if keyOld == keyMid { + t.Skip("跨月边界导致周期键相同，跳过") + } + partOld := partition.TableName(base, keyOld) + partMid := partition.TableName(base, keyMid) + tag := sfx() + // 用独一份的站点码把本用例的行圈出来：同一次 crossdb 跑里别的用例也往实时表写 h1 的行 + hostCode := "fanh_" + tag + + mkRow := func(table string, at time.Time, i int) string { + uid := fmt.Sprintf("fan_%s_%s_%d", tag, at.Format("200601"), i) + row := model.AccessLog{LogNarrow: model.LogNarrow{ + ReqUUID: uid, TenantId: xtestTenant, UserCode: xtestUser, + HostCode: hostCode, Host: "fan.test.com", URL: "/p" + fmt.Sprint(i), + Method: "GET", SRC_IP: "203.0.113.5", ACTION: "放行", + Day: at.Year()*10000 + int(at.Month())*100 + at.Day(), UNIX_ADD_TIME: at.UnixMilli(), + CREATE_TIME: at.Format("2006-01-02 15:04:05"), + }} + must(t, logdb.Table(table).Create(&row).Error) + return uid + } + + cleanup := func() { + logdb.Table(base).Where("req_uuid like ?", "fan_"+tag+"%").Delete(&model.AccessLog{}) + core.Where("file_name in ?", []string{partOld, partMid}).Delete(&model.ShareDb{}) + if !d.IsFileBased() { + _ = d.DropPartition(logdb, base, partOld) + _ = d.DropPartition(logdb, base, partMid) + } + } + cleanup() + defer cleanup() + + // 实时表里放 3 行（本月） + var liveUUIDs []string + for i := 0; i < 3; i++ { + liveUUIDs = append(liveUUIDs, mkRow(base, now.Add(-time.Duration(i+1)*time.Minute), i)) + } + + multi := !d.IsFileBased() + var oldestUUID string + if multi { + for _, p := range []struct { + table string + at time.Time + }{{partMid, now.AddDate(0, -1, 0)}, {partOld, now.AddDate(0, -2, 0)}} { + if err := d.CreatePartition(logdb, base, p.table); err != nil { + t.Fatalf("建分区 %s 失败: %v", p.table, err) + } + for i := 0; i < 3; i++ { + uid := mkRow(p.table, p.at.Add(-time.Duration(i+1)*time.Minute), i) + if p.table == partOld { + oldestUUID = uid + } + } + end := p.at.AddDate(0, 0, 1) + must(t, core.Create(&model.ShareDb{ + BaseOrm: baseorm.BaseOrm{ + Id: "fanshard_" + tag + "_" + p.table, USER_CODE: xtestUser, Tenant_ID: xtestTenant, + CREATE_TIME: customtype.JsonTime(time.Now()), UPDATE_TIME: customtype.JsonTime(time.Now()), + }, + DbLogicType: "log", + StartTime: customtype.JsonTime(p.at.AddDate(0, 0, -1)), + EndTime: customtype.JsonTime(end), + FileName: p.table, + PeriodKey: partition.KeyOf(p.at), + Cnt: 3, + }).Error) + } + } + + baseReq := func() request.WafAttackLogSearch { + r := request.WafAttackLogSearch{} + r.PageIndex = 1 + r.PageSize = 10 + r.SortBy = "unix_add_time" + r.SortDescending = "desc" + r.HostCode = hostCode + r.CurrrentDbName = AutoShard + r.UnixAddTimeBegin = fmt.Sprint(now.AddDate(0, -3, 0).UnixMilli()) + r.UnixAddTimeEnd = fmt.Sprint(now.Add(time.Hour).UnixMilli()) + return r + } + + t.Run("自动模式按时间范围扇出", func(t *testing.T) { + rows, total, meta, err := WafLogServiceApp.GetListApiWithMeta(baseReq()) + fatalIf(t, err) + want := int64(3) + if multi { + want = 9 + } + if total != want { + t.Fatalf("总数应为 %d（三段各 3 行），实际 %d；覆盖分区 %+v", want, total, meta.Shards) + } + if int64(len(rows)) != want { + t.Fatalf("一页取 10 条应全部返回 %d 行，实际 %d 行", want, len(rows)) + } + // 时间必须整体倒序：跨分区拼接如果顺序错了，用户看到的就是乱序 + for i := 1; i < len(rows); i++ { + if rows[i-1].UNIX_ADD_TIME < rows[i].UNIX_ADD_TIME { + t.Fatalf("第 %d 行比前一行更新，跨分区顺序错了", i) + } + } + if multi && len(meta.Shards) != 3 { + t.Fatalf("应覆盖 3 个分区，实际 %+v", meta.Shards) + } + // 每行都要带上来源分区：界面靠它标注，详情链接靠它直达 + for i, r := range rows { + if r.ShardName == "" { + t.Fatalf("第 %d 行没带来源分区", i) + } + } + }) + + if multi { + t.Run("跨分区翻页不漏不重", func(t *testing.T) { + seen := map[string]int{} + var order []string + for page := 1; page <= 5; page++ { + req := baseReq() + req.PageSize = 2 + req.PageIndex = page + rows, total, _, err := WafLogServiceApp.GetListApiWithMeta(req) + fatalIf(t, err) + if total != 9 { + t.Fatalf("每页返回的总数都应是 9，第 %d 页拿到 %d", page, total) + } + for _, r := range rows { + seen[r.REQ_UUID]++ + order = append(order, r.REQ_UUID) + } + } + if len(order) != 9 { + t.Fatalf("2 条一页翻 5 页应正好取完 9 行，实际 %d 行", len(order)) + } + for uid, n := range seen { + if n != 1 { + t.Fatalf("%s 出现了 %d 次：跨分区偏移算错会重复或漏行", uid, n) + } + } + }) + + t.Run("时间范围只落在旧分区时不查实时", func(t *testing.T) { + req := baseReq() + at := now.AddDate(0, -2, 0) + req.UnixAddTimeBegin = fmt.Sprint(at.AddDate(0, 0, -2).UnixMilli()) + req.UnixAddTimeEnd = fmt.Sprint(at.AddDate(0, 0, 1).UnixMilli()) + _, total, meta, err := WafLogServiceApp.GetListApiWithMeta(req) + fatalIf(t, err) + if total != 3 { + t.Fatalf("只该命中最老那段的 3 行，实际 %d（覆盖 %+v）", total, meta.Shards) + } + }) + } + + t.Run("重复查询走计数缓存且实时分区不被缓存", func(t *testing.T) { + _, first, _, err := WafLogServiceApp.GetListApiWithMeta(baseReq()) + fatalIf(t, err) + _, second, meta, err := WafLogServiceApp.GetListApiWithMeta(baseReq()) + fatalIf(t, err) + if first != second { + t.Fatalf("同样的查询两次总数不一致：%d vs %d", first, second) + } + if meta.Partial { + t.Fatal("这点数据量不该触发预算超时") + } + // 实时分区的计数绝不能被缓存：新写进来的行必须立刻算得进去 + extra := mkRow(base, now, 99) + defer logdb.Table(base).Where("req_uuid = ?", extra).Delete(&model.AccessLog{}) + _, third, _, err := WafLogServiceApp.GetListApiWithMeta(baseReq()) + fatalIf(t, err) + if third != second+1 { + t.Fatalf("实时分区新增一行后总数应 +1（%d → %d），实时分区被缓存了？", second, third) + } + }) + + t.Run("识别码直查忽略时间与分区", func(t *testing.T) { + target := liveUUIDs[0] + if multi { + target = oldestUUID // 最老的那个分区，离默认时间范围最远 + } + req := baseReq() + req.ReqUuid = target + // 故意把时间范围设成「今天以后」，证明识别码直查压根不看时间 + req.UnixAddTimeBegin = fmt.Sprint(now.AddDate(0, 0, 1).UnixMilli()) + req.UnixAddTimeEnd = fmt.Sprint(now.AddDate(0, 0, 2).UnixMilli()) + rows, total, meta, err := WafLogServiceApp.GetListApiWithMeta(req) + fatalIf(t, err) + if !meta.UuidLookup { + t.Fatal("填了识别码就该走直查") + } + if total != 1 || len(rows) != 1 || rows[0].REQ_UUID != target { + t.Fatalf("应正好查到 %s，实际 total=%d rows=%d", target, total, len(rows)) + } + if meta.FoundIn == "" { + t.Fatal("命中之后要告诉前端是在哪个分区找到的") + } + if meta.Scanned < 1 { + t.Fatal("翻过的分区数要记下来，查不到时界面要用它说明") + } + }) + + t.Run("识别码查不到不是错误", func(t *testing.T) { + req := baseReq() + req.ReqUuid = "fan_" + tag + "_not_exist" + rows, total, meta, err := WafLogServiceApp.GetListApiWithMeta(req) + fatalIf(t, err) + if total != 0 || len(rows) != 0 { + t.Fatalf("不存在的识别码应返回空，实际 total=%d rows=%d", total, len(rows)) + } + if !meta.UuidLookup || meta.FoundIn != "" { + t.Fatalf("没找到时 found_in 必须为空，实际 %+v", meta) + } + }) + + t.Run("指定分区时只查它", func(t *testing.T) { + req := baseReq() + req.CurrrentDbName = "" // 空 = 实时库 + _, total, meta, err := WafLogServiceApp.GetListApiWithMeta(req) + fatalIf(t, err) + if total != 3 { + t.Fatalf("实时库里只有 3 行，实际 %d", total) + } + if len(meta.Shards) != 1 { + t.Fatalf("锁定单分区时覆盖信息应只有一条，实际 %+v", meta.Shards) + } + }) + + if multi { + t.Run("详情按识别码自动定位分区", func(t *testing.T) { + got, err := WafLogServiceApp.GetDetailApi(request.WafAttackLogDetailReq{REQ_UUID: oldestUUID}) + fatalIf(t, err) + if got.REQ_UUID != oldestUUID { + t.Fatalf("没给分区标识时详情也该找得到归档里的记录，实际 %q", got.REQ_UUID) + } + if got.ShardName == "" { + t.Fatal("详情要告诉界面这条记录在哪个分区") + } + }) + } + + if multi { + t.Run("主动删除分区", func(t *testing.T) { + // 用一个专属周期，别动上面几个用例赖以断言的分区 + key := partition.KeyOf(now.AddDate(0, -4, 0)) + name := partition.TableName(base, key) + tiers := []string{model.AccessLogTableName, model.SecurityEventTableName, model.EventPayloadTableName} + for _, b := range tiers { + fatalIf(t, d.CreatePartition(logdb, b, partition.TableName(b, key))) + } + must(t, core.Create(&model.ShareDb{ + BaseOrm: baseorm.BaseOrm{ + Id: "delshard_" + tag + "_" + key, USER_CODE: xtestUser, Tenant_ID: xtestTenant, + CREATE_TIME: customtype.JsonTime(time.Now()), UPDATE_TIME: customtype.JsonTime(time.Now()), + }, + DbLogicType: "log", FileName: name, PeriodKey: key, Cnt: 0, + StartTime: customtype.JsonTime(now.AddDate(0, -4, -1)), + EndTime: customtype.JsonTime(now.AddDate(0, -4, 1)), + }).Error) + defer core.Where("file_name = ?", name).Delete(&model.ShareDb{}) + + // 列表要标出这个分区现存哪些层 + infos, err := WafShareDbServiceApp.GetAllShareDbWithTiers() + fatalIf(t, err) + found := false + for _, it := range infos { + if it.FileName == name { + found = true + if len(it.Tiers) != len(tiers) { + t.Fatalf("应标出 %d 层，实际 %+v", len(tiers), it.Tiers) + } + } + } + if !found { + t.Fatal("分片列表里找不到刚建的分区") + } + + // 实时库与不存在的名字都必须被拒绝——这是不可逆动作，入口要收窄 + if _, err := WafShareDbServiceApp.ForceDeleteShard(wafdb.LiveLogName()); err == nil { + t.Error("实时库不该被删掉") + } + if _, err := WafShareDbServiceApp.ForceDeleteShard(model.AccessLogTableName + "_209912"); err == nil { + t.Error("share_dbs 里没有登记的名字不该能删") + } + + // 正常删：三层表与分片记录都要没 + if _, err := WafShareDbServiceApp.ForceDeleteShard(name); err != nil { + t.Fatalf("删除分区失败: %v", err) + } + for _, b := range tiers { + if d.TableExists(logdb, partition.TableName(b, key)) { + t.Errorf("%s 应已被丢掉", partition.TableName(b, key)) + } + } + var left int64 + must(t, core.Model(&model.ShareDb{}).Where("file_name = ?", name).Count(&left).Error) + if left != 0 { + t.Errorf("分片记录应一并删除，实际还有 %d 条（下拉里会留下一个不存在的分区）", left) + } + // 实时表一张都不能少 + for _, b := range tiers { + if !d.TableExists(logdb, b) { + t.Fatalf("实时表 %s 被误删了", b) + } + } + }) + } + + _ = innerbean.WebLog{} +} diff --git a/service/waf_service/cross_engine_test.go b/service/waf_service/cross_engine_test.go index e6336a66..31d3038b 100644 --- a/service/waf_service/cross_engine_test.go +++ b/service/waf_service/cross_engine_test.go @@ -291,6 +291,12 @@ func TestCrossEngine(t *testing.T) { // —— 报文拆表读写（见 cross_engine_payload_test.go）—— t.Run("payload", func(t *testing.T) { runPayloadCases(t, x.logdb) }) + // —— 时间分区三动词（见 cross_engine_partition_test.go）—— + t.Run("partition", func(t *testing.T) { runPartitionCases(t, x.logdb) }) + + // —— 日志分区扇出与识别码直查（见 cross_engine_logfanout_test.go）—— + t.Run("logfanout", func(t *testing.T) { runLogFanoutCases(t, x.core, x.logdb) }) + // —— ip_tags 跨库合并（见 cross_engine_payload_test.go）—— t.Run("iptagmerge", func(t *testing.T) { runIPTagMergeCases(t, x) }) diff --git a/service/waf_service/waf_log.go b/service/waf_service/waf_log.go index a914ed7d..a41393b3 100644 --- a/service/waf_service/waf_log.go +++ b/service/waf_service/waf_log.go @@ -1,7 +1,6 @@ package waf_service import ( - "SamWaf/common/validfield" "SamWaf/common/zlog" "SamWaf/global" "SamWaf/innerbean" @@ -9,9 +8,7 @@ import ( "SamWaf/model/request" "SamWaf/wafdb" "SamWaf/wafdb/dialect" - "errors" "fmt" - "strconv" "strings" "sync" "time" @@ -127,13 +124,16 @@ func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) ( var weblog innerbean.WebLog // 解析当前分片的三层表：安全事件 → 访问日志 → 存量 web_logs，按 req_uuid 逐层点查。 // 事件双写了窄行，内容一致，但事件表保留期更长，优先从它读。 - tier := wafdb.ResolveTierTables(req.CurrrentDbName) + // 分区标识为空或 auto 时先按识别码定位分区：详情最常见的来路就是 + // 用户手里只有一串识别码，不知道那次访问落在哪个分区 + shardName := ResolveDetailShard(req.CurrrentDbName, req.REQ_UUID) + tier := wafdb.ResolveTierTables(shardName) found := false for _, table := range []string{tier.Event, tier.Access, tier.WebLog} { if table == "" { continue } - sel := webLogSelect(tier.DB, req.CurrrentDbName, table, getWebLogDetailColumns(), "detail") + sel := webLogSelect(tier.DB, shardName, table, getWebLogDetailColumns(), "detail") res := tier.DB.Table(table).Select(sel).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog) if res.Error != nil { return weblog, fmt.Errorf("查询日志详情失败: %w", res.Error) @@ -146,220 +146,23 @@ func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) ( if !found { return weblog, nil } + // 告诉界面这条是在哪个分区找到的：auto 时用户并没有选分区，得有个地方说清楚 + weblog.ShardName = shardName + if weblog.ShardName == "" { + weblog.ShardName = wafdb.LiveLogName() + } // 报文单独存在 event_payload 里，按主键点查补回来；没有报文行的（正常请求未采样）保持窄字段 - FillShardPayloads(req.CurrrentDbName, []*innerbean.WebLog{&weblog}) + FillShardPayloads(shardName, []*innerbean.WebLog{&weblog}) return weblog, nil } -func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { - var total int64 = 0 - var weblogs []innerbean.WebLog - - splitFilterBys := strings.Split(req.FilterBy, "|") - splitFilterValues := strings.Split(req.FilterValue, "|") - // 解析当前分片的三层表：访问日志视图读 access_log，安全事件视图读 security_event； - // 分层改造之前切出去的归档只有 web_logs，回落到它（列交集会自适应它的结构）。 - tier := wafdb.ResolveTierTables(req.CurrrentDbName) - logDB := tier.DB - isEventView := req.ViewType == "event" - logTable := tier.Access - if isEventView { - logTable = tier.Event - } - if logTable == "" { - logTable = tier.WebLog - } - if logTable == "" { - return nil, 0, errors.New("该分片没有可查询的日志表") - } - // 老分片上的安全事件视图：web_logs 里按事件条件过滤（与引擎 abnormal 判定同一条规则） - legacyEventView := isEventView && logTable == tier.WebLog - isLegacyTable := strings.HasPrefix(logTable, wafdb.LogTableName) - - /*where条件*/ - var whereField = "" - var whereValues []interface{} - - //where字段 - { - whereField = whereField + " (unix_add_time>=? and unix_add_time<=?)" - if legacyEventView { - whereField = whereField + " and (action<>? or rule<>? or log_only_mode=1)" - } - if len(req.HostCode) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " host_code=? " - } - if len(req.Rule) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " rule=? " - } - if len(req.ReqUuid) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " req_uuid=? " - } - if len(req.Action) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " action=? " - } - if len(req.SrcIp) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " src_ip=? " - } - if len(req.StatusCode) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " status_code=? " - } - if len(req.Method) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " method=? " - } - if len(req.LogOnlyMode) > 0 { - if len(whereField) > 0 { - whereField = whereField + " and " - } - whereField = whereField + " log_only_mode=? " - } - for _, by := range splitFilterBys { - - if len(by) > 0 { - if !validfield.IsValidWebLogFilterField(by) { - return nil, 0, errors.New("输入过滤字段不合法") - } - if len(whereField) > 0 { - whereField = whereField + " and " - } - if by == "guest_identification" { - by = "guest_id_entification" - } - if by == "header" && !isLegacyTable { - // header 随报文搬进 event_payload：访问日志视图没有这一列， - // 安全事件视图走报文子查询（该视图行数小、且必有报文）。 - if !isEventView { - return nil, 0, errors.New("「请求」全文筛选仅在安全事件视图可用，访问日志视图请改用 UA / Referer 筛选") - } - if tier.Payload == "" { - return nil, 0, errors.New("该分片没有报文表，无法按「请求」内容筛选") - } - whereField = whereField + " req_uuid in (select req_uuid from " + tier.Payload + " where header like ?) " - } else { - whereField = whereField + " " + by + " like ? " - } - } - } - } - //强制索引 - forceIndex := logTable - { - idxTime, idxIP := "idx_web_time_desc_tenant_user_code", "idx_web_time_desc_tenant_user_code_ip" - if strings.HasPrefix(logTable, model.AccessLogTableName) { - idxTime, idxIP = "idx_al_time", "idx_al_ip_time" - } else if strings.HasPrefix(logTable, model.SecurityEventTableName) { - idxTime, idxIP = "idx_se_time", "idx_se_ip_time" - } - if strings.Contains(whereField, "unix_add_time") && !strings.Contains(whereField, "src_ip") { - forceIndex = dialect.Get().ForceIndexClause(logTable, idxTime) - } else if strings.Contains(whereField, "src_ip") { - forceIndex = dialect.Get().ForceIndexClause(logTable, idxIP) - } - } - - // 将字符串转换为 int64 类型 - unixBegin, err := strconv.ParseInt(req.UnixAddTimeBegin, 10, 64) - if err != nil { - fmt.Println("Error converting UnixAddTimeBegin to int64:", err) - - } - - unixEnd, err := strconv.ParseInt(req.UnixAddTimeEnd, 10, 64) - if err != nil { - fmt.Println("Error converting UnixAddTimeEnd to int64:", err) - - } - //where字段赋值 - { - whereValues = append(whereValues, unixBegin) - whereValues = append(whereValues, unixEnd) - if legacyEventView { - whereValues = append(whereValues, "放行", "") - } - if len(req.HostCode) > 0 { - whereValues = append(whereValues, req.HostCode) - } - if len(req.Rule) > 0 { - whereValues = append(whereValues, req.Rule) - } - if len(req.ReqUuid) > 0 { - whereValues = append(whereValues, req.ReqUuid) - } - if len(req.Action) > 0 { - whereValues = append(whereValues, req.Action) - } - if len(req.SrcIp) > 0 { - whereValues = append(whereValues, req.SrcIp) - } - if len(req.StatusCode) > 0 { - whereValues = append(whereValues, req.StatusCode) - } - if len(req.Method) > 0 { - whereValues = append(whereValues, req.Method) - } - if len(req.LogOnlyMode) > 0 { - whereValues = append(whereValues, req.LogOnlyMode) - } - for _, val := range splitFilterValues { - if len(val) > 0 { - whereValues = append(whereValues, "%"+val+"%") - } - } - } - - orderInfo := "" - - /** - 排序 - */ - if receiver.isValidSortField(req.SortBy) { - if req.SortDescending == "desc" { - orderInfo = req.SortBy + " desc" - } else { - orderInfo = req.SortBy + " asc" - } - } else { - return nil, 0, errors.New("输入排序字段不合法") - } - sel := webLogSelect(logDB, req.CurrrentDbName, logTable, getWebLogListColumns(), "list") - // 错误必须往上抛：吞掉它就只剩「有分页、没数据」，连从哪查起都不知道 - if err := logDB.Select(sel).Table(forceIndex).Limit(req.PageSize).Where(whereField, whereValues...).Offset(req.PageSize * (req.PageIndex - 1)).Order(orderInfo).Find(&weblogs).Error; err != nil { - return nil, 0, fmt.Errorf("查询日志失败: %w", err) - } - if err := logDB.Table(forceIndex).Where(whereField, whereValues...).Count(&total).Error; err != nil { - return nil, 0, fmt.Errorf("统计日志条数失败: %w", err) - } - // 安全事件视图每页补一回报文（事件必有报文）：「请求」列与详情都靠它 - if isEventView && len(weblogs) > 0 { - rows := make([]*innerbean.WebLog, 0, len(weblogs)) - for i := range weblogs { - rows = append(rows, &weblogs[i]) - } - FillShardPayloads(req.CurrrentDbName, rows) - } - return weblogs, total, nil +// GetListApi 日志列表查询。条件组装、分区解析与跨分区扇出都在 waf_log_query.go， +// 这里保留旧签名给不关心「这次查了哪些分区」的调用方。 +func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { + rows, total, _, err := receiver.GetListApiWithMeta(req) + return rows, total, err } + func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { var total int64 = 0 var weblogs []innerbean.WebLog @@ -368,6 +171,7 @@ func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSear global.GWAF_LOCAL_LOG_DB.Where("host_code = ?", log.HostCode).Model(&innerbean.WebLog{}).Count(&total) return weblogs, total, nil } + // DeleteHistory 分层保留期清理： // - security_event 与 web_logs（存量，不再写入）按「日志保留天数」删 // - access_log 按 access_log_retention_days 删（更短） diff --git a/service/waf_service/waf_log_query.go b/service/waf_service/waf_log_query.go new file mode 100644 index 00000000..d464aaa5 --- /dev/null +++ b/service/waf_service/waf_log_query.go @@ -0,0 +1,689 @@ +package waf_service + +// 日志列表查询的分区扇出（M3 E5）。 +// +// E4 之前「访问日期」与「日志归档库」各管一半时间语义，互不联动：日期选今天、分区停在实时， +// 结果就是空的，而界面不会告诉你数据在另一个分区里。M3 之后分区本身就是一段时间， +// 于是这里让分区**由时间范围算出来**： +// +// - shardName == "auto"（前端默认）：按时间范围挑出候选分区，从新到旧顺序取数、跨分区翻页； +// - 填了访问识别码：忽略时间与分区，按分区从新到旧逐个主键点查、命中即停； +// - 明确指定了某个分区：照旧只查它（排障与导出要的就是这个）。 +// +// 顺序取数之所以成立，是因为分区按时间互不重叠：整体按时间排序 == 各分区内排序后按分区时间拼接。 +// 其它排序字段没有这个性质，所以跨分区时强制按时间排序（SortForcedTime 会告诉前端说明一句）。 + +import ( + "SamWaf/common/validfield" + "SamWaf/common/zlog" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/request" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "context" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "gorm.io/gorm" +) + +// AutoShard 前端默认传的分区标识：让后端按时间范围自己算 +const AutoShard = "auto" + +// fanoutBudget 一次扇出查询的时间预算。比前端默认超时（20 秒）短一截： +// 宁可返回「部分结果 + 说明」，也不要让浏览器那边白屏超时，那种失败什么线索都留不下。 +const fanoutBudget = 8 * time.Second + +// fanoutParallel 同时统计几个分区。分区之间互不相干，串行 count 是把每个分区的 +// 索引扫描时间直接相加——分区一多就必然超时。并发度不开太大：SQLite 下每个分区是独立文件， +// 开太多只会互相抢磁盘。 +const fanoutParallel = 4 + +// maxUuidLookupShards 识别码直查最多翻几个分区。分区量级是「一个月一个」， +// 正常远达不到；设上限只是防止分片表被手工塞出成百上千条时把一次查询拖垮。 +const maxUuidLookupShards = 60 + +// LogShardHit 一次查询里某个分区贡献了多少行 +type LogShardHit struct { + Name string `json:"name"` // 分片标识（share_dbs.file_name / 实时标识） + Count int64 `json:"count"` // 该分区内命中条数 +} + +// LogQueryMeta 告诉前端这次查询到底查了哪里 +type LogQueryMeta struct { + Shards []LogShardHit `json:"shards"` // 本次覆盖的分区（按时间从新到旧） + FoundIn string `json:"found_in"` // 识别码直查命中的分区 + Scanned int `json:"scanned"` // 识别码直查翻了几个分区 + UuidLookup bool `json:"uuid_lookup"` // 是否走了识别码直查 + SortForcedTime bool `json:"sort_forced_time"` // 跨分区时排序被强制成时间 + Partial bool `json:"partial"` // 查询超出时间预算，结果与总数都只是一部分 + TookMs int64 `json:"took_ms"` // 本次耗时，界面用来解释「为什么只给了一部分」 +} + +// shardCountCache 归档分区的计数缓存。 +// +// 归档分区是**不可变**的——写入只发生在实时库，归档只会被整体丢掉。所以同一组过滤条件 +// 在同一个归档分区上的计数永远是同一个值，算一次就能一直用。 +// 只有「时间范围完整覆盖该分区」时才缓存：这时时间条件是个空条件，计数与用户选的具体区间无关， +// 缓存键才稳定得下来（否则每次点「最近30天」毫秒数都不同，缓存永远不命中）。 +// 实时分区一律不缓存。 +var shardCountCache sync.Map + +// shardCountCacheMax 缓存条目上限，超了整个清空。条目本身很小，这里只是防止 +// 「每天不同过滤条件」日积月累把它撑大。 +const shardCountCacheMax = 2000 + +var shardCountCacheN atomic.Int64 + +// InvalidateShardCounts 分区发生变化（切库、丢过期分区）时清空计数缓存。 +func InvalidateShardCounts() { + shardCountCache.Range(func(k, _ any) bool { + shardCountCache.Delete(k) + return true + }) + shardCountCacheN.Store(0) +} + +// countCacheKey 非时间条件的签名 + 分区名。时间条件不进键——只有完整覆盖时才会用到本缓存。 +func countCacheKey(req request.WafAttackLogSearch, shard string) string { + return strings.Join([]string{ + shard, req.ViewType, req.HostCode, req.Rule, req.ReqUuid, req.Action, + req.SrcIp, req.StatusCode, req.Method, req.LogOnlyMode, req.FilterBy, req.FilterValue, + }, "") +} + +// logQuery 一个分区上组装好的查询：条件、参数、选列与强制索引都定死了， +// 之后只差 count 与 find 两个动作。 +type logQuery struct { + db *gorm.DB + tier wafdb.TierTables + shardName string + logTable string + forceIndex string + whereField string + whereValues []interface{} + orderInfo string + isEventView bool +} + +// buildLogQuery 在指定分区上组装查询条件。ignoreTime=true 时不带时间范围（识别码直查用）。 +func buildLogQuery(req request.WafAttackLogSearch, shardName string, ignoreTime bool) (*logQuery, error) { + splitFilterBys := strings.Split(req.FilterBy, "|") + splitFilterValues := strings.Split(req.FilterValue, "|") + // 解析当前分片的三层表：访问日志视图读 access_log，安全事件视图读 security_event； + // 分层改造之前切出去的归档只有 web_logs，回落到它（列交集会自适应它的结构）。 + tier := wafdb.ResolveTierTables(shardName) + logDB := tier.DB + isEventView := req.ViewType == "event" + logTable := tier.Access + if isEventView { + logTable = tier.Event + } + if logTable == "" { + logTable = tier.WebLog + } + if logTable == "" { + return nil, errors.New("该分片没有可查询的日志表") + } + // 老分片上的安全事件视图：web_logs 里按事件条件过滤（与引擎 abnormal 判定同一条规则） + legacyEventView := isEventView && logTable == tier.WebLog + isLegacyTable := strings.HasPrefix(logTable, wafdb.LogTableName) + + /*where条件*/ + var whereField = "" + var whereValues []interface{} + + //where字段 + { + // 按识别码直查时不带时间条件：识别码是主键，用户手里往往只有这串码、并不知道是哪天 + if !ignoreTime { + whereField = whereField + " (unix_add_time>=? and unix_add_time<=?)" + } + if legacyEventView { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " (action<>? or rule<>? or log_only_mode=1) " + } + if len(req.HostCode) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " host_code=? " + } + if len(req.Rule) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " rule=? " + } + if len(req.ReqUuid) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " req_uuid=? " + } + if len(req.Action) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " action=? " + } + if len(req.SrcIp) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " src_ip=? " + } + if len(req.StatusCode) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " status_code=? " + } + if len(req.Method) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " method=? " + } + if len(req.LogOnlyMode) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " log_only_mode=? " + } + for _, by := range splitFilterBys { + + if len(by) > 0 { + if !validfield.IsValidWebLogFilterField(by) { + return nil, errors.New("输入过滤字段不合法") + } + if len(whereField) > 0 { + whereField = whereField + " and " + } + if by == "guest_identification" { + by = "guest_id_entification" + } + if by == "header" && !isLegacyTable { + // header 随报文搬进 event_payload：访问日志视图没有这一列， + // 安全事件视图走报文子查询（该视图行数小、且必有报文）。 + if !isEventView { + return nil, errors.New("「请求」全文筛选仅在安全事件视图可用，访问日志视图请改用 UA / Referer 筛选") + } + if tier.Payload == "" { + return nil, errors.New("该分片没有报文表，无法按「请求」内容筛选") + } + whereField = whereField + " req_uuid in (select req_uuid from " + tier.Payload + " where header like ?) " + } else { + whereField = whereField + " " + by + " like ? " + } + } + } + } + //强制索引 + forceIndex := logTable + { + idxTime, idxIP := "idx_web_time_desc_tenant_user_code", "idx_web_time_desc_tenant_user_code_ip" + if strings.HasPrefix(logTable, model.AccessLogTableName) { + idxTime, idxIP = "idx_al_time", "idx_al_ip_time" + } else if strings.HasPrefix(logTable, model.SecurityEventTableName) { + idxTime, idxIP = "idx_se_time", "idx_se_ip_time" + } + if strings.Contains(whereField, "unix_add_time") && !strings.Contains(whereField, "src_ip") { + forceIndex = dialect.Get().ForceIndexClause(logTable, idxTime) + } else if strings.Contains(whereField, "src_ip") { + forceIndex = dialect.Get().ForceIndexClause(logTable, idxIP) + } + } + + // 将字符串转换为 int64 类型 + unixBegin, err := strconv.ParseInt(req.UnixAddTimeBegin, 10, 64) + if err != nil { + fmt.Println("Error converting UnixAddTimeBegin to int64:", err) + + } + + unixEnd, err := strconv.ParseInt(req.UnixAddTimeEnd, 10, 64) + if err != nil { + fmt.Println("Error converting UnixAddTimeEnd to int64:", err) + + } + + //where字段赋值 + { + if !ignoreTime { + whereValues = append(whereValues, unixBegin) + whereValues = append(whereValues, unixEnd) + } + if legacyEventView { + whereValues = append(whereValues, "放行", "") + } + if len(req.HostCode) > 0 { + whereValues = append(whereValues, req.HostCode) + } + if len(req.Rule) > 0 { + whereValues = append(whereValues, req.Rule) + } + if len(req.ReqUuid) > 0 { + whereValues = append(whereValues, req.ReqUuid) + } + if len(req.Action) > 0 { + whereValues = append(whereValues, req.Action) + } + if len(req.SrcIp) > 0 { + whereValues = append(whereValues, req.SrcIp) + } + if len(req.StatusCode) > 0 { + whereValues = append(whereValues, req.StatusCode) + } + if len(req.Method) > 0 { + whereValues = append(whereValues, req.Method) + } + if len(req.LogOnlyMode) > 0 { + whereValues = append(whereValues, req.LogOnlyMode) + } + for _, val := range splitFilterValues { + if len(val) > 0 { + whereValues = append(whereValues, "%"+val+"%") + } + } + } + + orderInfo := "" + + /** + 排序 + */ + if WafLogServiceApp.isValidSortField(req.SortBy) { + if req.SortDescending == "desc" { + orderInfo = req.SortBy + " desc" + } else { + orderInfo = req.SortBy + " asc" + } + } else { + return nil, errors.New("输入排序字段不合法") + } + if whereField == "" { + // 条件全空（识别码直查又没给识别码）不该发出去：那是全表扫 + return nil, errors.New("查询条件为空") + } + return &logQuery{ + db: logDB, + tier: tier, + shardName: shardName, + logTable: logTable, + forceIndex: forceIndex, + whereField: whereField, + whereValues: whereValues, + orderInfo: orderInfo, + isEventView: isEventView, + }, nil +} + +// count 该分区内命中多少行。ctx 到期时底层驱动会取消查询，不会把整次请求拖死。 +func (q *logQuery) count(ctx context.Context) (int64, error) { + var total int64 + if err := q.db.WithContext(ctx).Table(q.forceIndex).Where(q.whereField, q.whereValues...).Count(&total).Error; err != nil { + return 0, fmt.Errorf("统计日志条数失败: %w", err) + } + return total, nil +} + +// find 取一页。orderOverride 非空时覆盖排序（跨分区时统一按时间）。 +func (q *logQuery) find(ctx context.Context, offset, limit int, orderOverride string) ([]innerbean.WebLog, error) { + order := q.orderInfo + if orderOverride != "" { + order = orderOverride + } + var rows []innerbean.WebLog + sel := webLogSelect(q.db, q.shardName, q.logTable, getWebLogListColumns(), "list") + // 错误必须往上抛：吞掉它就只剩「有分页、没数据」，连从哪查起都不知道 + if err := q.db.WithContext(ctx).Select(sel).Table(q.forceIndex).Limit(limit).Offset(offset). + Where(q.whereField, q.whereValues...).Order(order).Find(&rows).Error; err != nil { + return nil, fmt.Errorf("查询日志失败: %w", err) + } + // 标上来源分区：跨分区时界面要能说明这行从哪来，详情链接也能据此直达 + for i := range rows { + rows[i].ShardName = q.shardName + } + // 安全事件视图每页补一回报文（事件必有报文）：「请求」列与详情都靠它 + if q.isEventView && len(rows) > 0 { + ptrs := make([]*innerbean.WebLog, 0, len(rows)) + for i := range rows { + ptrs = append(ptrs, &rows[i]) + } + FillShardPayloads(q.shardName, ptrs) + } + return rows, nil +} + +// logShard 一个候选分区：标识 + 它装着的时间范围 +type logShard struct { + Name string + Start time.Time + End time.Time + Live bool +} + +// candidateShards 挑出与 [fromMs, toMs] 有交集的分区，按时间**从新到旧**排列。 +// ignoreTime=true 时不做时间过滤，返回全部分区（识别码直查用）。 +// +// 实时分区的起点取「最新归档分片的结束时间」：归档之后写进来的都还在实时库里。 +// 一条归档记录的起止时间缺失时按「覆盖到边界」处理——宁可多查一个分区，也不要漏掉数据。 +func candidateShards(fromMs, toMs int64, ignoreTime bool) []logShard { + live := logShard{Name: wafdb.LiveLogName(), Live: true, End: time.Now()} + + all, err := WafShareDbServiceApp.GetAllShareDbApi() + if err != nil { + zlog.Warn("日志分区扇出", "读取分片列表失败，只查实时库", "error", err.Error()) + return []logShard{live} + } + + shards := make([]logShard, 0, len(all)+1) + for _, s := range all { + if s.DbLogicType != "" && s.DbLogicType != "log" { + continue + } + if s.FileName == "" || s.FileName == live.Name { + continue + } + st, en := time.Time(s.StartTime), time.Time(s.EndTime) + if en.After(live.Start) { + live.Start = en + } + shards = append(shards, logShard{Name: s.FileName, Start: st, End: en}) + } + shards = append(shards, live) + sort.Slice(shards, func(i, j int) bool { return shards[i].End.After(shards[j].End) }) + + if ignoreTime { + return shards + } + from, to := time.UnixMilli(fromMs), time.UnixMilli(toMs) + picked := make([]logShard, 0, len(shards)) + for _, sh := range shards { + if !sh.Start.IsZero() && sh.Start.After(to) { + continue + } + if !sh.End.IsZero() && sh.End.Before(from) { + continue + } + picked = append(picked, sh) + } + if len(picked) == 0 { + // 时间范围落在所有分区之外：仍然查一次实时库，让「查不到」是查出来的结论而不是算出来的 + return []logShard{live} + } + return picked +} + +// GetListApiWithMeta 日志列表查询，并告诉调用方这次到底查了哪些分区。 +// +// 三条路：填了识别码走直查（忽略时间与分区）／auto 走按时间范围扇出／指定了分区就只查它。 +func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, LogQueryMeta, error) { + var meta LogQueryMeta + shardName := strings.TrimSpace(req.CurrrentDbName) + // 只有明确传 auto 才扇出。空值沿用旧语义（实时库）——存量调用方与 Vue3 还在传空， + // 悄悄改成扇出会让它们的每次查询都多打几个分区 + auto := shardName == AutoShard + + if auto && strings.TrimSpace(req.ReqUuid) != "" { + return receiver.lookupByUuid(req) + } + + ctx, cancel := context.WithTimeout(context.Background(), fanoutBudget) + defer cancel() + started := time.Now() + defer func() { meta.TookMs = time.Since(started).Milliseconds() }() + + if !auto { + q, err := buildLogQuery(req, shardName, false) + if err != nil { + return nil, 0, meta, err + } + total, err := q.count(ctx) + if err != nil { + return nil, 0, meta, err + } + rows, err := q.find(ctx, req.PageSize*(req.PageIndex-1), req.PageSize, "") + if err != nil { + return nil, 0, meta, err + } + meta.Shards = []LogShardHit{{Name: shardName, Count: total}} + return rows, total, meta, nil + } + + // —— 自动：按时间范围扇出 —— + fromMs, _ := strconv.ParseInt(req.UnixAddTimeBegin, 10, 64) + toMs, _ := strconv.ParseInt(req.UnixAddTimeEnd, 10, 64) + shards := candidateShards(fromMs, toMs, false) + + // 跨分区时统一按时间排序：分区之间只有时间是可比的，按别的列排出来的「全局顺序」是假的 + order := "" + if len(shards) > 1 { + dir := "desc" + if req.SortDescending == "asc" { + dir = "asc" + } + order = "unix_add_time " + dir + meta.SortForcedTime = req.SortBy != "unix_add_time" + if dir == "asc" { + for i, j := 0, len(shards)-1; i < j; i, j = i+1, j-1 { + shards[i], shards[j] = shards[j], shards[i] + } + } + } + + // 先把各分区的查询组装出来（只是探表拼条件，很便宜），统计放到后面并发做。 + // 组装不并发：SQLite 下解析分区会按需打开归档文件，串行更稳。 + type plan struct { + q *logQuery + sh logShard + cnt int64 + name string + cached bool + } + plans := make([]plan, 0, len(shards)) + var firstErr error + for _, sh := range shards { + q, err := buildLogQuery(req, sh.Name, false) + if err != nil { + // 某个分区没有可查的表（改造前的老分片）不该让整次查询失败，跳过即可 + if firstErr == nil { + firstErr = err + } + zlog.Debug("日志分区扇出", "跳过分区", sh.Name, "原因", err.Error()) + continue + } + plans = append(plans, plan{q: q, sh: sh, name: sh.Name}) + } + if len(plans) == 0 { + if firstErr != nil { + return nil, 0, meta, firstErr + } + return []innerbean.WebLog{}, 0, meta, nil + } + + // 时间范围完整覆盖的归档分区：计数与具体区间无关，可以复用上次算过的值。 + // 归档不可变，缓存一直有效，直到切库或丢分区把它清掉。 + from, to := time.UnixMilli(fromMs), time.UnixMilli(toMs) + cacheable := func(sh logShard) bool { + if sh.Live || sh.Start.IsZero() || sh.End.IsZero() { + return false + } + return !sh.Start.Before(from) && !sh.End.After(to) + } + + var wg sync.WaitGroup + sem := make(chan struct{}, fanoutParallel) + for i := range plans { + i := i + key := "" + if cacheable(plans[i].sh) { + key = countCacheKey(req, plans[i].name) + if v, ok := shardCountCache.Load(key); ok { + plans[i].cnt, plans[i].cached = v.(int64), true + continue + } + } + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + cnt, err := plans[i].q.count(ctx) + if err != nil { + zlog.Warn("日志分区扇出", "统计分区失败", plans[i].name, "error", err.Error()) + plans[i].cnt = -1 // 统计失败：这个分区的条数算不进总数，但它的行照取 + return + } + plans[i].cnt = cnt + if key != "" { + if shardCountCacheN.Add(1) > shardCountCacheMax { + InvalidateShardCounts() + } + shardCountCache.Store(key, cnt) + } + }() + } + wg.Wait() + + var total int64 + for i := range plans { + if plans[i].cnt < 0 { + meta.Partial = true // 有分区没统计上，总数只是一部分 + plans[i].cnt = 0 + continue + } + total += plans[i].cnt + } + if ctx.Err() != nil { + meta.Partial = true + } + + offset := int64(req.PageSize * (req.PageIndex - 1)) + remaining := req.PageSize + rows := make([]innerbean.WebLog, 0, req.PageSize) + for _, p := range plans { + if p.cnt > 0 { + meta.Shards = append(meta.Shards, LogShardHit{Name: p.name, Count: p.cnt}) + } + if remaining <= 0 || p.cnt == 0 { + continue + } + if offset >= p.cnt { + offset -= p.cnt + continue + } + if ctx.Err() != nil { + // 预算用完了：把已经拿到的行还回去并标 partial， + // 让界面提示「缩小时间范围或锁定分区」，而不是让浏览器那边干等到超时 + meta.Partial = true + break + } + part, err := p.q.find(ctx, int(offset), remaining, order) + if err != nil { + if ctx.Err() != nil { + meta.Partial = true + break + } + return nil, 0, meta, err + } + rows = append(rows, part...) + remaining -= len(part) + offset = 0 + } + return rows, total, meta, nil +} + +// lookupByUuid 按访问识别码直查：忽略时间与分区，从新到旧逐个分区做主键点查，命中即停。 +// +// 识别码是三张日志表的主键，一次点查就是一次索引命中；最常见的来路是访客把拦截页上的 +// 识别码报给管理员——他手里只有这串码，不知道是哪天，更不知道在哪个分区。 +// 查不到不是错误：调用方按空结果处理，界面负责解释可能的原因（已过保留期 / 抄错 / 本就没记录）。 +func (receiver *WafLogService) lookupByUuid(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, LogQueryMeta, error) { + meta := LogQueryMeta{UuidLookup: true} + ctx, cancel := context.WithTimeout(context.Background(), fanoutBudget) + defer cancel() + started := time.Now() + defer func() { meta.TookMs = time.Since(started).Milliseconds() }() + + shards := candidateShards(0, 0, true) + var firstErr error + for i, sh := range shards { + if i >= maxUuidLookupShards { + zlog.Warn("识别码直查", "分区过多，已停在上限", maxUuidLookupShards) + break + } + if ctx.Err() != nil { + meta.Partial = true + break + } + meta.Scanned = i + 1 + q, err := buildLogQuery(req, sh.Name, true) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + found, err := q.find(ctx, 0, req.PageSize, "") + if err != nil { + if firstErr == nil { + firstErr = err + } + zlog.Warn("识别码直查", "分区查询失败", sh.Name, "error", err.Error()) + continue + } + if len(found) > 0 { + meta.FoundIn = sh.Name + meta.Shards = []LogShardHit{{Name: sh.Name, Count: int64(len(found))}} + return found, int64(len(found)), meta, nil + } + } + return []innerbean.WebLog{}, 0, meta, nil +} + +// ResolveDetailShard 详情按识别码定位分区：明确给了就用它， +// auto / 空 时从新到旧找出第一个有这条记录的分区。找不到返回空字符串（调用方回落实时库）。 +func ResolveDetailShard(shardName, reqUuid string) string { + name := strings.TrimSpace(shardName) + if name != "" && name != AutoShard { + return name + } + if strings.TrimSpace(reqUuid) == "" { + return name + } + ctx, cancel := context.WithTimeout(context.Background(), fanoutBudget) + defer cancel() + for i, sh := range candidateShards(0, 0, true) { + if i >= maxUuidLookupShards || ctx.Err() != nil { + break + } + tier := wafdb.ResolveTierTables(sh.Name) + if tier.DB == nil { + continue + } + for _, table := range []string{tier.Event, tier.Access, tier.WebLog} { + if table == "" { + continue + } + var cnt int64 + if err := tier.DB.WithContext(ctx).Table(table).Where("req_uuid = ?", reqUuid).Count(&cnt).Error; err != nil { + continue + } + if cnt > 0 { + return sh.Name + } + } + } + return "" +} diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index a4a2082c..36cb506f 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -825,3 +825,108 @@ notes: Both lists are empty by default, so behavior after upgrading is exactly as before and nothing needs to be done. As a side note, SamWaf's own access-authentication probe endpoint (`//ping`, always 204) never produces an access log, so there is nothing to exclude for it. + + - id: v1_3_25_log_time_partition + version: v1.3.25 + kind: notice + level: high + page: /waf/wafvisitlog + doc: https://doc.samwaf.com/quickstart/Update.html + apply: + type: navigate + zh: + title: 日志归档改为按月切分区，过期回收改为丢整个分区（MySQL/PostgreSQL 的归档表首次开始回收） + detail: >- + 日志归档以前由行数与文件大小决定什么时候切，现在改成**按月**切：一个月一个分区， + 同一个月里真被写爆了才会提前切一次（分区名带序号）作为兜底。好处是「某段时间的日志在哪个分区」 + 变成算得出来的，不再需要挨个猜；「访问日志」页的归档下拉因此显示**时间段**而不是文件名或表名， + 按月切的显示成 2026-09，升级前按体积切的显示它记录的起止日期，鼠标悬停仍能看到原始名字。 + 过期回收同时改成丢整个分区：SQLite 删掉整个归档文件，MySQL/PostgreSQL 丢掉整张归档表， + 不再逐行删除加 VACUUM。判断过期用的是分片记录里那批数据真实的最晚时间， + 所以停机几天或同月切了两次都不会把还在保留期内的数据算成过期。 + **MySQL/PostgreSQL 用户请先看这一段**：此前归档表没有任何清理逻辑，很可能已经堆了多张 + access_log_ 或 web_logs_ 开头的历史表；升级后第一次执行「删除历史日志」任务（每天 05:00）时， + 其中**已超过保留期**的会被丢掉，系统日志里逐张记录表名、所用保留天数与数据截止时间。 + 这是保留期设置本来就承诺的行为，也让两种部署终于一致。如果有哪一段历史想留： + 先把日志保留天数调长，或用数据库自己的工具把那张表复制到别处，再让清理继续。 + 另外 MySQL/PostgreSQL 上每一层是独立的表，因此可以按层各走各的保留期—— + 访问日志窄行走「访问日志保留天数」（默认 30 天），安全事件与报文走「日志保留天数」； + SQLite 的一个分区是一个文件、三层都在里面，只能按较长的「日志保留天数」整体回收。 + 升级前切出来的旧分片（包括最早那种只有 web_logs 的）**照旧可以查**，不需要任何迁移动作， + 读取时会自动适配它实际有哪些表和哪些列，到期后同样整体回收。 + en: + title: Log archives are now cut per month and expired by dropping whole partitions (MySQL/PostgreSQL archive tables start being recycled) + detail: >- + Log archives used to be cut when a row count or file size threshold was crossed. They are now cut + **per month**: one partition per month, with an extra numbered partition only as a fallback when a + single month really overflows. The point is that which partition holds a given time range becomes + something that can be computed instead of guessed, so the archive dropdown on the access log page now + shows a **time range** rather than a file or table name - 2026-09 for monthly partitions, and the + recorded start and end dates for shards cut by size before the upgrade. Hovering still shows the + original name. + Expiry now drops a whole partition: SQLite deletes the entire archive file and MySQL/PostgreSQL drops + the archive table, instead of deleting rows and running VACUUM. Expiry is judged by the real latest + timestamp of the data recorded for that shard, so being offline for days, or cutting twice within one + month, never marks data still inside the retention window as expired. + **MySQL/PostgreSQL users should read this part first**: archive tables previously had no cleanup at + all, so you may already have many history tables starting with access_log_ or web_logs_. The first run + of the daily 05:00 history cleanup after upgrading drops those **past their retention period**, writing + one log line per table with its name, the retention days applied and the data cutoff. This is what the + retention setting has always promised, and it finally makes both deployment types behave alike. To keep + a particular stretch of history, raise the log retention days first, or copy that table elsewhere with + your database tools, then let cleanup proceed. + On MySQL/PostgreSQL each tier is its own table, so tiers expire on their own schedules - access-log + narrow rows follow "access log retention days" (30 by default) while security events and payloads + follow "delete history log (days)". On SQLite one partition is one file holding all tiers, so it can + only be recycled as a whole on the longer "delete history log (days)". + Shards cut before the upgrade, including the earliest ones that only hold web_logs, **remain + queryable** with no migration on your part: reads adapt to whichever tables and columns a shard + actually has, and they are recycled whole once they expire. + + - id: v1_3_25_log_auto_partition_and_uuid + version: v1.3.25 + kind: notice + level: normal + page: /waf/wafvisitlog + doc: https://doc.samwaf.com/quickstart/Update.html + apply: + type: navigate + zh: + title: 查日志不用再挑分区：时间范围自动决定查哪里，按访问识别码可直接全库查找 + detail: >- + 「日志归档库」下拉的默认值改成了**自动（按时间范围）**：分区由你选的时间范围算出来， + 跨多个分区时会自动合并翻页，列表上方会说明这次覆盖了哪几个分区。 + 此前日期与归档是两个互不联动的条件，选了上个月的日期却停在实时分区，结果永远是空的。 + 需要盯住某一个分区时（排障、导出），下拉里仍然可以手动锁定，并且按「实时 / 按月分区 / + 升级前按体积切的旧分片」分了组。跨分区时列表统一按时间排序——分区之间只有时间是可比的。 + **填了「访问识别码」时会忽略时间与分区，直接跨全部分区查找**，命中即停并告诉你是在哪个分区找到的。 + 访问识别码是日志表的主键，一次查找就是一次索引命中；最常见的场景正是访客把拦截页上的识别码 + 报给管理员——管理员手里只有这串码，并不知道那次访问是哪天、更不知道在哪个分区。 + 查不到时会明确列出三种可能：记录已过保留期被回收、识别码抄错或被截断、 + 那次请求本来就没有被记录（来源IP在排除清单里，或访问日志档位是「仅安全事件」而它是一次正常请求）。 + 另外两处顺带改了：归档下拉里同一天切出来的多个旧分片现在带上时分（2026-08-19 13:51~14:40） + 并把条数写成「13.5 万条」，不再是七个一模一样的日期；风险日志的「访问 IP 明细」弹窗 + 补上了时间范围（默认最近 30 天），不用再一个归档一个归档地翻。 + en: + title: "No more picking a partition - the time range decides where to look, and a visit identifier is looked up across all of them" + detail: >- + The archive dropdown now defaults to **Auto (by time range)**: partitions are derived from the time + range you pick, paging spans them automatically, and a line above the list names the partitions this + query covered. Previously the date and the archive were two conditions that knew nothing about each + other, so picking last month's dates while the archive stayed on the live partition always returned + nothing. A specific partition can still be locked from the dropdown when troubleshooting or exporting, + and the entries are grouped into Live, monthly partitions, and shards cut by size before the upgrade. + When a query spans partitions the list is sorted by time, since time is the only thing comparable + across them. + **Filling in the visit identifier ignores the time range and the partition and searches all of them**, + stopping at the first hit and telling you which partition it came from. The identifier is the primary + key of the log tables, so each check is a single index lookup. This is exactly the common case where a + visitor reports the identifier from a block page: the administrator has only that string and knows + neither the date nor the partition. When nothing is found, three possible reasons are stated: the + record passed its retention period and was recycled, the identifier was mistyped or truncated, or the + request was never recorded (its source IP is on an exclusion list, or the access log mode is + "security events only" and it was a plain request). + Two smaller fixes come along: several old shards cut on the same day now show their times + (2026-08-19 13:51~14:40) with counts written as readable numbers instead of seven identical dates, and + the Access IP Details dialog on the risk log page gained a time range (last 30 days by default) so it + no longer takes going through archives one by one. From 785b607b10368e35e933f9e5a2125e6fd16a9cb7 Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 28 Sep 2026 09:47:53 +0800 Subject: [PATCH 16/21] feat: add time partition verbs and the period naming they share --- .../cross_engine_partition_test.go | 145 +++++++++++++++++ wafdb/dialect/dialect.go | 20 +++ wafdb/dialect/mysql_dialect.go | 28 ++++ wafdb/dialect/partition.go | 86 ++++++++++ wafdb/dialect/partition_guard_test.go | 54 +++++++ wafdb/dialect/postgres_dialect.go | 30 ++++ wafdb/dialect/sqlite_dialect.go | 17 ++ wafdb/partition/partition.go | 134 ++++++++++++++++ wafdb/partition/partition_test.go | 148 ++++++++++++++++++ 9 files changed, 662 insertions(+) create mode 100644 service/waf_service/cross_engine_partition_test.go create mode 100644 wafdb/dialect/partition.go create mode 100644 wafdb/dialect/partition_guard_test.go create mode 100644 wafdb/partition/partition.go create mode 100644 wafdb/partition/partition_test.go diff --git a/service/waf_service/cross_engine_partition_test.go b/service/waf_service/cross_engine_partition_test.go new file mode 100644 index 00000000..5d65146e --- /dev/null +++ b/service/waf_service/cross_engine_partition_test.go @@ -0,0 +1,145 @@ +//go:build crossdb + +// 时间分区三动词（E1）的三库回归：建 / 列 / 丢。 +// +// 三个引擎的语义故意不一样，所以必须各跑一遍： +// MySQL 是 CREATE TABLE LIKE、PG 是 LIKE ... INCLUDING ALL（索引名由 PG 自己生成， +// 写成复制原名会撞），SQLite 按文件分区、建与丢一律返回「不适用」让调用方分流。 +// 丢分区是不可逆动作，这里同时钉住「基表 + 分区表」的配对校验：名字不同族就必须拒绝。 +// 由 TestCrossEngine 每引擎调一次。 +package waf_service + +import ( + "SamWaf/model" + "SamWaf/wafdb/dialect" + "SamWaf/wafdb/partition" + "testing" + + "gorm.io/gorm" +) + +func runPartitionCases(t *testing.T, logdb *gorm.DB) { + d := dialect.Get() + base := model.AccessLogTableName + part := partition.TableName(base, "209901") // 远期周期键，不会和真实分片撞 + + // 收尾：无论断言走到哪一步都别留下测试表 + defer func() { + if !d.IsFileBased() { + _ = d.DropPartition(logdb, base, part) + } + }() + + if d.IsFileBased() { + // SQLite：建与丢发生在文件层，dialect 必须明确说不适用， + // 而不是静默成功——静默成功会让清理任务以为分区已经删掉了 + if err := d.CreatePartition(logdb, base, part); err == nil { + t.Error("SQLite 的 CreatePartition 应返回不适用错误") + } + if err := d.DropPartition(logdb, base, part); err == nil { + t.Error("SQLite 的 DropPartition 应返回不适用错误") + } + // 列分区在 SQLite 上返回空是正常的（一个文件里不会有周期表），不该报错 + parts, err := d.ListPartitions(logdb, base) + if err != nil { + t.Errorf("SQLite 列分区不该报错: %v", err) + } + for _, p := range parts { + if p == part { + t.Errorf("SQLite 上不该出现分区表 %s", p) + } + } + return + } + + // 1) 建分区：与基表同构 + if err := d.CreatePartition(logdb, base, part); err != nil { + t.Fatalf("建分区 %s 失败: %v", part, err) + } + // 幂等：同一个周期重复建不能报错（切库重试、任务重跑都会碰到） + if err := d.CreatePartition(logdb, base, part); err != nil { + t.Fatalf("重复建同一个分区应幂等，实际报错: %v", err) + } + if !d.TableExists(logdb, part) { + t.Fatalf("分区表 %s 建完却不存在", part) + } + + // 结构可用：基表的列都在，且能写能读 + baseCols, err := d.ColumnInfo(logdb, base) + fatalIf(t, err) + partCols, err := d.ColumnInfo(logdb, part) + fatalIf(t, err) + if len(partCols) != len(baseCols) { + t.Fatalf("分区表列数 %d 与基表 %d 不一致（结构没复制全）", len(partCols), len(baseCols)) + } + uid := "part_" + sfx() + must(t, logdb.Table(part).Create(&model.AccessLog{ + LogNarrow: model.LogNarrow{ + ReqUUID: uid, + HostCode: "h1", + UserCode: xtestUser, + TenantId: xtestTenant, + Day: 20990101, + }, + }).Error) + var cnt int64 + must(t, logdb.Table(part).Where("req_uuid = ?", uid).Count(&cnt).Error) + if cnt != 1 { + t.Fatalf("分区表里应能读到刚写的行，实际 %d 行", cnt) + } + // 写进分区的行不该出现在基表里（两张独立的表，不是同一份数据） + var liveCnt int64 + must(t, logdb.Table(base).Where("req_uuid = ?", uid).Count(&liveCnt).Error) + if liveCnt != 0 { + t.Fatalf("分区表的行漏进基表了，基表命中 %d 行", liveCnt) + } + + // 2) 列分区：认自己的分区，不认别的基表的 + parts, err := d.ListPartitions(logdb, base) + fatalIf(t, err) + found := false + for _, p := range parts { + if p == part { + found = true + } + if p == base { + t.Errorf("基表 %s 自己不该出现在分区列表里", base) + } + } + if !found { + t.Fatalf("分区列表里找不到 %s，实际 %v", part, parts) + } + otherParts, err := d.ListPartitions(logdb, model.SecurityEventTableName) + fatalIf(t, err) + for _, p := range otherParts { + if p == part { + t.Errorf("%s 是 %s 的分区，不该出现在 %s 的分区列表里", p, base, model.SecurityEventTableName) + } + } + + // 3) 丢分区：名字不同族一律拒绝，别把无关的表丢掉 + if err := d.DropPartition(logdb, base, model.SecurityEventTableName); err == nil { + t.Errorf("拿 %s 当基表去丢 %s 应被拒绝", base, model.SecurityEventTableName) + } + if !d.TableExists(logdb, model.SecurityEventTableName) { + t.Fatalf("%s 表被误删了", model.SecurityEventTableName) + } + if err := d.DropPartition(logdb, base, base); err == nil { + t.Error("把基表自己当分区丢应被拒绝") + } + if !d.TableExists(logdb, base) { + t.Fatal("基表被误删了") + } + + // 正常丢 + if err := d.DropPartition(logdb, base, part); err != nil { + t.Fatalf("丢分区 %s 失败: %v", part, err) + } + if d.TableExists(logdb, part) { + t.Fatalf("分区表 %s 丢完却还在", part) + } + // 幂等：清理任务重跑不能因为已经没了而报错 + if err := d.DropPartition(logdb, base, part); err != nil { + t.Fatalf("重复丢同一个分区应幂等，实际报错: %v", err) + } +} diff --git a/wafdb/dialect/dialect.go b/wafdb/dialect/dialect.go index 6eb7a2cc..2112ac07 100644 --- a/wafdb/dialect/dialect.go +++ b/wafdb/dialect/dialect.go @@ -150,6 +150,26 @@ type DBDialect interface { // return an error here ("not supported"). ShardSwapTable(db *gorm.DB, liveTable, archiveTable string) error + // CreatePartition creates an empty time partition of baseTable, cloning its + // structure and indexes. partTable must be named "_". + // MySQL: CREATE TABLE LIKE + // Postgres: CREATE TABLE (LIKE INCLUDING ALL) + // A partition here is a period-named table, not a native RANGE partition - + // see wafdb/dialect/partition.go for why. File-based databases (SQLite) + // partition by file and return an error; check IsFileBased() first. + CreatePartition(db *gorm.DB, baseTable, partTable string) error + + // ListPartitions returns the partition tables of baseTable ("_"), + // sorted by name. File-based databases (SQLite) partition by file, so this + // legitimately returns nothing there. + ListPartitions(db *gorm.DB, baseTable string) ([]string, error) + + // DropPartition drops one whole partition table, replacing per-row DELETE + + // VACUUM for expiry. partTable must be named "_" - the pair + // is validated so an unrelated table cannot be dropped by mistake. + // File-based databases (SQLite) delete the partition file instead and return an error. + DropPartition(db *gorm.DB, baseTable, partTable string) error + // TableSizeMB returns the on-disk size (data + index) of a table in MB, // used by the sharding task to detect the size threshold on server databases. // File-based databases (SQLite) return 0 (the caller uses the file size instead). diff --git a/wafdb/dialect/mysql_dialect.go b/wafdb/dialect/mysql_dialect.go index 7d2c92c4..74c4c4aa 100644 --- a/wafdb/dialect/mysql_dialect.go +++ b/wafdb/dialect/mysql_dialect.go @@ -226,3 +226,31 @@ func BuildMySQLRootDSN(host string, port int, user, password, charset string) st func mysqlQuote(name string) string { return "`" + strings.ReplaceAll(name, "`", "``") + "`" } + +// CreatePartition 建一张与基表同构（含索引）的空周期表 +func (d *MySQLDialect) CreatePartition(db *gorm.DB, baseTable, partTable string) error { + if err := checkPartitionPair(baseTable, partTable); err != nil { + return err + } + sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s LIKE %s", mysqlQuote(partTable), mysqlQuote(baseTable)) + if err := db.Exec(sql).Error; err != nil { + return fmt.Errorf("mysql: 建分区表 %s 失败: %w", partTable, err) + } + return nil +} + +// ListPartitions 列出 <基表>_<后缀> 形态的表 +func (d *MySQLDialect) ListPartitions(db *gorm.DB, baseTable string) ([]string, error) { + return listPartitionsByPrefix(d, db, baseTable) +} + +// DropPartition 丢掉整张周期表，替代逐行 DELETE + VACUUM +func (d *MySQLDialect) DropPartition(db *gorm.DB, baseTable, partTable string) error { + if err := checkPartitionPair(baseTable, partTable); err != nil { + return err + } + if err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", mysqlQuote(partTable))).Error; err != nil { + return fmt.Errorf("mysql: 丢分区表 %s 失败: %w", partTable, err) + } + return nil +} diff --git a/wafdb/dialect/partition.go b/wafdb/dialect/partition.go new file mode 100644 index 00000000..04b2842b --- /dev/null +++ b/wafdb/dialect/partition.go @@ -0,0 +1,86 @@ +package dialect + +import ( + "fmt" + "regexp" + "sort" + "strings" + + "gorm.io/gorm" +) + +// 时间分区三动词的共用部分（M3/E1）。 +// +// 分区在这里的形态是「一段时间一个存储单元」：MySQL/PG 是一张以周期命名的表 +// （access_log_202609），SQLite 是一个 .db 文件。**没有走 PG/MySQL 的原生 +// PARTITION BY RANGE**：两个引擎都要求分区列出现在每个唯一键里，而 access_log 与 +// event_payload 都以 req_uuid 为主键，改成原生分区就得改主键并重建整表搬数据—— +// 存量部署上这是一次大爆炸，计划 §5.4 明确要避免。周期表沿用已有的换表与读侧扇出， +// 一行读写代码都不用改，丢分区同样是一条 DROP TABLE。 +// +// 文件型引擎（SQLite）的建与丢发生在文件层，dialect 拿不到数据目录， +// 所以这两个动词在 SQLite 上返回明确错误，调用方必须先看 IsFileBased()—— +// 与 ShardSwapTable 一致的分工。 + +// partIdentPattern 分区表名允许的字符。名字是程序按周期拼出来的， +// 但它要拼进 DDL（DDL 不接受占位符），所以进 SQL 前必须先卡一遍。 +var partIdentPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// maxIdentLen 三个引擎里最短的标识符上限（MySQL 64） +const maxIdentLen = 64 + +// checkPartIdent 校验要拼进 DDL 的表名 +func checkPartIdent(name string) error { + if name == "" { + return fmt.Errorf("分区表名为空") + } + if len(name) > maxIdentLen { + return fmt.Errorf("分区表名 %q 超过 %d 字符", name, maxIdentLen) + } + if !partIdentPattern.MatchString(name) { + return fmt.Errorf("分区表名 %q 含非法字符（只允许字母、数字、下划线，且不以数字开头）", name) + } + return nil +} + +// checkPartitionPair 校验「基表 + 分区表」这一对：分区表必须真的是该基表的分区， +// 免得把无关的表当分区丢掉。 +func checkPartitionPair(baseTable, partTable string) error { + if err := checkPartIdent(baseTable); err != nil { + return err + } + if err := checkPartIdent(partTable); err != nil { + return err + } + if !strings.HasPrefix(partTable, baseTable+"_") { + return fmt.Errorf("分区表 %q 不是 %q 的分区（名字必须是 <基表>_<周期>）", partTable, baseTable) + } + return nil +} + +// shardTmpSuffix ShardSwapTable 换表期间的临时表后缀，不是分区 +const shardTmpSuffix = "_shardtmp" + +// listPartitionsByPrefix 列分区的共用实现：拿全部表名，筛出 <基表>_<后缀> 形态的。 +// +// 三个引擎共用一份：分区就是「名字带周期后缀的表」，各引擎只是 ListTables 的查法不同。 +// SQLite 上正常返回空——它按文件分区，一个文件里不会有周期表，这不是错误。 +func listPartitionsByPrefix(d DBDialect, db *gorm.DB, baseTable string) ([]string, error) { + if err := checkPartIdent(baseTable); err != nil { + return nil, err + } + tables, err := d.ListTables(db) + if err != nil { + return nil, err + } + prefix := baseTable + "_" + var parts []string + for _, t := range tables { + if !strings.HasPrefix(t, prefix) || strings.HasSuffix(t, shardTmpSuffix) { + continue + } + parts = append(parts, t) + } + sort.Strings(parts) + return parts, nil +} diff --git a/wafdb/dialect/partition_guard_test.go b/wafdb/dialect/partition_guard_test.go new file mode 100644 index 00000000..4ec4f33d --- /dev/null +++ b/wafdb/dialect/partition_guard_test.go @@ -0,0 +1,54 @@ +package dialect + +import ( + "strings" + "testing" +) + +// 分区表名要拼进 DDL（DDL 不接受占位符），所以校验是这里唯一的防线。 +func TestCheckPartIdentRejectsUnsafeNames(t *testing.T) { + bad := []string{ + "", + "access log", + "access_log; DROP TABLE hosts", + "access_log\"", + "access_log`", + "access_log'", + "access_log--", + "202609_access", // 以数字开头 + strings.Repeat("a", 65), + } + for _, name := range bad { + if err := checkPartIdent(name); err == nil { + t.Errorf("%q 应被拒绝", name) + } + } + for _, name := range []string{"access_log", "access_log_202609", "event_payload_202512"} { + if err := checkPartIdent(name); err != nil { + t.Errorf("%q 应通过: %v", name, err) + } + } +} + +// 丢分区是不可逆动作，必须确认这张表真是该基表的分区 +func TestCheckPartitionPairRequiresPrefix(t *testing.T) { + if err := checkPartitionPair("access_log", "access_log_202609"); err != nil { + t.Fatalf("同族分区应通过: %v", err) + } + for _, part := range []string{"hosts", "security_event_202609", "access_log", "xaccess_log_202609"} { + if err := checkPartitionPair("access_log", part); err == nil { + t.Errorf("%q 不是 access_log 的分区，应被拒绝", part) + } + } +} + +// SQLite 按文件分区：建与丢必须明确报错，免得调用方以为已经建好/删掉了 +func TestSQLitePartitionVerbsAreNotApplicable(t *testing.T) { + d := &SQLiteDialect{} + if err := d.CreatePartition(nil, "access_log", "access_log_202609"); err == nil { + t.Error("SQLite 的 CreatePartition 应返回不适用错误") + } + if err := d.DropPartition(nil, "access_log", "access_log_202609"); err == nil { + t.Error("SQLite 的 DropPartition 应返回不适用错误") + } +} diff --git a/wafdb/dialect/postgres_dialect.go b/wafdb/dialect/postgres_dialect.go index 846d257b..938cdcca 100644 --- a/wafdb/dialect/postgres_dialect.go +++ b/wafdb/dialect/postgres_dialect.go @@ -383,3 +383,33 @@ func BuildPostgresMaintenanceDSN(host string, port int, user, password, maintena func pgQuote(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +// CreatePartition 建一张与基表同构的空周期表。INCLUDING ALL 连索引、默认值、 +// 非空约束一起复制，索引名由 PG 自己生成，不会和基表的撞。 +func (d *PostgresDialect) CreatePartition(db *gorm.DB, baseTable, partTable string) error { + if err := checkPartitionPair(baseTable, partTable); err != nil { + return err + } + sql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (LIKE %s INCLUDING ALL)", + pgQuote(partTable), pgQuote(baseTable)) + if err := db.Exec(sql).Error; err != nil { + return fmt.Errorf("postgres: 建分区表 %s 失败: %w", partTable, err) + } + return nil +} + +// ListPartitions 列出 <基表>_<后缀> 形态的表 +func (d *PostgresDialect) ListPartitions(db *gorm.DB, baseTable string) ([]string, error) { + return listPartitionsByPrefix(d, db, baseTable) +} + +// DropPartition 丢掉整张周期表，替代逐行 DELETE + VACUUM +func (d *PostgresDialect) DropPartition(db *gorm.DB, baseTable, partTable string) error { + if err := checkPartitionPair(baseTable, partTable); err != nil { + return err + } + if err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", pgQuote(partTable))).Error; err != nil { + return fmt.Errorf("postgres: 丢分区表 %s 失败: %w", partTable, err) + } + return nil +} diff --git a/wafdb/dialect/sqlite_dialect.go b/wafdb/dialect/sqlite_dialect.go index 1be9e812..dfe9c61a 100644 --- a/wafdb/dialect/sqlite_dialect.go +++ b/wafdb/dialect/sqlite_dialect.go @@ -217,3 +217,20 @@ func (d *SQLiteDialect) CollectMetrics(db *gorm.DB, name, path string) (*DBMetri func sqliteQuote(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +// CreatePartition 不适用于 SQLite：分区就是一个 .db 文件，建分区等于建文件 + 跑迁移， +// 得由掌握数据目录的任务层做（与 ShardSwapTable 同样的分工）。 +func (d *SQLiteDialect) CreatePartition(db *gorm.DB, baseTable, partTable string) error { + return fmt.Errorf("CreatePartition 不适用于 SQLite（按文件分区，建分区=建库文件），当前驱动: sqlite") +} + +// ListPartitions 列本文件内 <基表>_<后缀> 形态的表。SQLite 按文件分区， +// 正常情况下返回空——这不是错误，调用方靠 IsFileBased() 分流去查分区文件。 +func (d *SQLiteDialect) ListPartitions(db *gorm.DB, baseTable string) ([]string, error) { + return listPartitionsByPrefix(d, db, baseTable) +} + +// DropPartition 不适用于 SQLite：丢分区等于删文件，同样在任务层。 +func (d *SQLiteDialect) DropPartition(db *gorm.DB, baseTable, partTable string) error { + return fmt.Errorf("DropPartition 不适用于 SQLite（按文件分区，丢分区=删库文件），当前驱动: sqlite") +} diff --git a/wafdb/partition/partition.go b/wafdb/partition/partition.go new file mode 100644 index 00000000..48656229 --- /dev/null +++ b/wafdb/partition/partition.go @@ -0,0 +1,134 @@ +// Package partition 时间分区的口径：分区键怎么算、分区叫什么名、一段时间范围落在哪些分区上。 +// +// 纯函数、不碰数据库；真正的建/列/丢在 `wafdb/dialect` 的三个动词里。 +// +// 粒度 = 月（D3 定案）。一个分区就是「一段时间的独立存储单元」： +// SQLite 下是一个 `.db` 文件，MySQL/PG 下是一张以周期命名的表。两边都沿用已有的 +// 换文件 / 换表 + 读侧扇出机制，所以这里只需要让「周期键 ↔ 时间范围 ↔ 存储单元名字」能互相换算。 +// +// 与按体积切分的旧分片共存：旧分片名字带 14 位时间戳（`access_log_20260921153045`）， +// 周期分区是 6 位（`access_log_202609`，同周期内被迫再切时是 `access_log_202609_02`）。 +// `KeyFromName` 只认 6 位，旧分片认不出来是**故意的**—— +// 它们的时间范围只能查 `share_dbs` 的起止时间，不能从名字算。 +package partition + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// KeyLayout 周期键的格式：年月，如 202609 +const KeyLayout = "200601" + +// keyLen 周期键长度，用来和旧的 14 位时间戳分片区分开 +const keyLen = 6 + +// KeyOf 返回某个时刻所属的周期键。按本地时区算——保留期、清理任务、 +// 界面上的「时间段」都是本地时间口径，这里跟着走才不会在月初差一天。 +func KeyOf(t time.Time) string { + return t.Format(KeyLayout) +} + +// Range 返回周期键覆盖的时间范围，左闭右开 [start, end)。 +func Range(key string) (time.Time, time.Time, error) { + if !IsKey(key) { + return time.Time{}, time.Time{}, fmt.Errorf("非法周期键: %q（应形如 202609）", key) + } + start, err := time.ParseInLocation(KeyLayout, key, time.Local) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("解析周期键 %q 失败: %w", key, err) + } + return start, start.AddDate(0, 1, 0), nil +} + +// IsKey 判断一个字符串是不是合法周期键（6 位数字且月份在 1~12） +func IsKey(key string) bool { + if len(key) != keyLen { + return false + } + n, err := strconv.Atoi(key) + if err != nil || n <= 0 { + return false + } + month := n % 100 + return month >= 1 && month <= 12 +} + +// TableName 返回周期分区表名，如 TableName("access_log", "202609") = access_log_202609 +func TableName(base, key string) string { + return base + "_" + key +} + +// FileName 返回 SQLite 周期分区的文件名，如 FileName("202609") = local_log_202609.db +func FileName(key string) string { + return "local_log_" + key + ".db" +} + +// TableNameSeq 返回同一周期内第 seq 个分区表名（seq >= 2 才带序号）。 +// 正常一个周期一个分区，只有体积/行数在周期内就超限、被迫提前切的时候才会用到—— +// 那是异常兜底，不是常态。 +func TableNameSeq(base, key string, seq int) string { + if seq <= 1 { + return TableName(base, key) + } + return fmt.Sprintf("%s_%s_%02d", base, key, seq) +} + +// FileNameSeq 同 TableNameSeq，SQLite 文件版 +func FileNameSeq(key string, seq int) string { + if seq <= 1 { + return FileName(key) + } + return fmt.Sprintf("local_log_%s_%02d.db", key, seq) +} + +// KeyFromName 从存储单元名字里取出周期键：表名（access_log_202609）、 +// 文件名（local_log_202609.db）、带兜底序号的（access_log_202609_02）都认。 +// 按体积切出来的旧分片（14 位时间戳）返回 false。 +// +// 只看最后两段：再往前找就会把业务表名里的数字段误当周期键。 +func KeyFromName(name string) (string, bool) { + name = strings.TrimSuffix(name, ".db") + parts := strings.Split(name, "_") + for i := len(parts) - 1; i >= 0 && i >= len(parts)-2; i-- { + if IsKey(parts[i]) { + return parts[i], true + } + } + return "", false +} + +// KeysBetween 返回覆盖 [from, to] 的全部周期键，升序、去重。 +// from 晚于 to 时返回 nil。读侧的「候选分区裁剪」就靠它—— +// 从「猜哪些分片可能有数据」变成纯算术。 +func KeysBetween(from, to time.Time) []string { + if from.After(to) { + return nil + } + var keys []string + cur := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.Local) + last := time.Date(to.Year(), to.Month(), 1, 0, 0, 0, 0, time.Local) + for !cur.After(last) { + keys = append(keys, KeyOf(cur)) + cur = cur.AddDate(0, 1, 0) + } + return keys +} + +// Expired 判断一个周期分区是否整段都超出了保留期。 +// +// 判据是分区的**结束时刻**而不是开始时刻：9 月的分区装着 9/1~9/30 的数据， +// 保留 30 天时 10/5 还有 9/5 之后的数据在保留期内，这时候丢整个分区就是丢用户还该看到的数据。 +// retentionDays <= 0 表示不按天清理，一律不过期。 +func Expired(key string, retentionDays int, now time.Time) bool { + if retentionDays <= 0 { + return false + } + _, end, err := Range(key) + if err != nil { + return false + } + return !end.After(now.AddDate(0, 0, -retentionDays)) +} diff --git a/wafdb/partition/partition_test.go b/wafdb/partition/partition_test.go new file mode 100644 index 00000000..42873001 --- /dev/null +++ b/wafdb/partition/partition_test.go @@ -0,0 +1,148 @@ +package partition + +import ( + "testing" + "time" +) + +func TestKeyOfAndRange(t *testing.T) { + key := KeyOf(time.Date(2026, 9, 21, 15, 30, 0, 0, time.Local)) + if key != "202609" { + t.Fatalf("周期键应为 202609，实际 %q", key) + } + start, end, err := Range(key) + if err != nil { + t.Fatalf("Range 失败: %v", err) + } + if start.Year() != 2026 || start.Month() != time.September || start.Day() != 1 { + t.Fatalf("起点应是 2026-09-01，实际 %v", start) + } + // 右开：9 月分区的终点是 10/1 零点，不是 9/30 23:59:59。 + // 写成右闭会在月末最后一秒漏掉数据。 + if end.Year() != 2026 || end.Month() != time.October || end.Day() != 1 { + t.Fatalf("终点应是 2026-10-01，实际 %v", end) + } +} + +func TestRangeCrossYear(t *testing.T) { + _, end, err := Range("202612") + if err != nil { + t.Fatalf("Range 失败: %v", err) + } + if end.Year() != 2027 || end.Month() != time.January { + t.Fatalf("12 月的终点应跨年到 2027-01，实际 %v", end) + } +} + +func TestIsKeyRejectsJunk(t *testing.T) { + for _, bad := range []string{"", "2026", "20260921150405", "202613", "202600", "20260a", "-20260"} { + if IsKey(bad) { + t.Errorf("%q 不该被当成周期键", bad) + } + } + for _, ok := range []string{"202601", "202612", "202509"} { + if !IsKey(ok) { + t.Errorf("%q 应是合法周期键", ok) + } + } +} + +func TestKeyFromName(t *testing.T) { + cases := []struct { + name string + want string + ok bool + }{ + {"access_log_202609", "202609", true}, + {"event_payload_202512", "202512", true}, + {"local_log_202609.db", "202609", true}, + // 按体积切出来的旧分片：14 位时间戳，认不出来是故意的， + // 它们的时间范围只能查 share_dbs，不能从名字算 + {"access_log_20260921153045", "", false}, + {"local_log_20260921153045.db", "", false}, + {"access_log", "", false}, + {"access_log_", "", false}, + {"access_log_shardtmp", "", false}, + // 同周期内体积超限被迫再切的兜底命名，仍要认得出周期 + {"access_log_202609_02", "202609", true}, + {"local_log_202609_02.db", "202609", true}, + } + for _, c := range cases { + got, ok := KeyFromName(c.name) + if ok != c.ok || got != c.want { + t.Errorf("KeyFromName(%q) = (%q,%v)，期望 (%q,%v)", c.name, got, ok, c.want, c.ok) + } + } +} + +func TestTableAndFileNameSeq(t *testing.T) { + // seq<=1 不带序号：一个周期一个分区是常态，名字里不该多出噪音 + if got := TableNameSeq("access_log", "202609", 1); got != "access_log_202609" { + t.Errorf("seq=1 应不带序号，实际 %q", got) + } + if got := TableNameSeq("access_log", "202609", 2); got != "access_log_202609_02" { + t.Errorf("seq=2 表名 = %q", got) + } + if got := FileNameSeq("202609", 3); got != "local_log_202609_03.db" { + t.Errorf("seq=3 文件名 = %q", got) + } + if got := FileNameSeq("202609", 1); got != "local_log_202609.db" { + t.Errorf("seq=1 应不带序号，实际 %q", got) + } +} + +func TestTableAndFileName(t *testing.T) { + if got := TableName("access_log", "202609"); got != "access_log_202609" { + t.Errorf("表名 = %q", got) + } + if got := FileName("202609"); got != "local_log_202609.db" { + t.Errorf("文件名 = %q", got) + } +} + +func TestKeysBetween(t *testing.T) { + from := time.Date(2026, 11, 20, 0, 0, 0, 0, time.Local) + to := time.Date(2027, 2, 3, 0, 0, 0, 0, time.Local) + got := KeysBetween(from, to) + want := []string{"202611", "202612", "202701", "202702"} + if len(got) != len(want) { + t.Fatalf("跨年区间应有 %d 个周期，实际 %v", len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("第 %d 个周期 = %q，期望 %q（全部：%v）", i, got[i], want[i], got) + } + } + + // 同月内的区间只有一个周期 + same := KeysBetween(from, from.AddDate(0, 0, 3)) + if len(same) != 1 || same[0] != "202611" { + t.Fatalf("同月区间应只有 202611，实际 %v", same) + } + + // 起点晚于终点：返回空而不是倒序或一整年 + if got := KeysBetween(to, from); got != nil { + t.Fatalf("起点晚于终点应返回 nil，实际 %v", got) + } +} + +func TestExpiredJudgesByEndNotStart(t *testing.T) { + const key = "202609" // 装 9/1~9/30 的数据，终点 10/1 + + // 保留 30 天、当前 10/5：截止点 9/5，9 月分区里 9/5 之后的数据还在保留期内 → 不能丢 + if Expired(key, 30, time.Date(2026, 10, 5, 12, 0, 0, 0, time.Local)) { + t.Error("分区里还有在保留期内的数据，不该判为过期（按开始时刻判就会错在这里）") + } + // 当前 11/5：截止点 10/6 已越过分区终点 → 整段过期 + if !Expired(key, 30, time.Date(2026, 11, 5, 12, 0, 0, 0, time.Local)) { + t.Error("整段都超出保留期了，应判为过期") + } + // 不按天清理 + if Expired(key, 0, time.Date(2030, 1, 1, 0, 0, 0, 0, time.Local)) { + t.Error("retentionDays<=0 表示不按天清理，不该过期") + } + // 非法键不判过期（宁可留着也不误删） + if Expired("20260921153045", 30, time.Date(2030, 1, 1, 0, 0, 0, 0, time.Local)) { + t.Error("认不出的名字不该被判过期") + } +} From ab2ce5e0a27fcf88a8e15969dfdff127b6deb0d3 Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 28 Sep 2026 09:48:24 +0800 Subject: [PATCH 17/21] feat: cut log archives per month and expire them by dropping partitions --- enums/task_enum.go | 4 +- model/response/all_host.go | 2 + model/sharedb.go | 8 + router/waf_log.go | 1 + wafdb/migrations_core.go | 21 +++ waftask/clean_archive_crossdb_test.go | 239 ++++++++++++++++++++++++++ waftask/task_clean_archive.go | 236 +++++++++++++++++++++---- waftask/task_clean_archive_test.go | 80 +++++++++ waftask/task_db_sharding.go | 138 +++++++++++++-- waftask/task_history.go | 3 +- 10 files changed, 681 insertions(+), 51 deletions(-) create mode 100644 waftask/clean_archive_crossdb_test.go create mode 100644 waftask/task_clean_archive_test.go diff --git a/enums/task_enum.go b/enums/task_enum.go index 5a6f3ece..cdb30c05 100644 --- a/enums/task_enum.go +++ b/enums/task_enum.go @@ -3,8 +3,8 @@ package enums const ( TASK_RUNTIME_QPS_CLEAN = "task_runtime_qps_clean" //清空运行QPS TASK_HOST_QPS_CLEAN = "task_host_qps_clean" //清空主机QPS - TASK_SHARE_DB = "task_share_db" //分库检测(每天兜底) - TASK_SHARE_DB_CHECK = "task_share_db_check" //分库高频检测(每5分钟按大小切库) + TASK_SHARE_DB = "task_share_db" //分区检测(每天兜底) + TASK_SHARE_DB_CHECK = "task_share_db_check" //分区高频检测(每5分钟；按周期切，体积超限兜底) TASK_COUNTER = "task_counter" //统计 TASK_DELAY_INFO = "task_delay_info" //延迟统计信息 TASK_LOAD_CONFIG = "task_load_config" //获取配置信息 diff --git a/model/response/all_host.go b/model/response/all_host.go index d1659d22..50f796cc 100644 --- a/model/response/all_host.go +++ b/model/response/all_host.go @@ -20,6 +20,8 @@ type AllShareDbRep struct { FileName string `json:"file_name"` //文件名 Cnt int64 `json:"cnt"` //当前数量 IsCurrent bool `json:"is_current"` //是否为当前(实时)分片：前端据此设默认选中项 + PeriodKey string `json:"period_key"` //周期键（月，如 202609）；按体积切出来的旧分片为空，前端回落显示起止日期 + Tiers []string `json:"tiers"` //该分区现存哪些层（仅服务型数据库给；按层过期后可能只剩安全事件与报文） } // AllDomainRep 域名信息 diff --git a/model/sharedb.go b/model/sharedb.go index 8a2079fa..50841eaf 100644 --- a/model/sharedb.go +++ b/model/sharedb.go @@ -17,6 +17,14 @@ type ShareDb struct { EndTime customtype.JsonTime `json:"end_time"` //结束时间 FileName string `gorm:"size:255" json:"file_name"` //文件名：SQLite 下为 .db 文件名；MySQL 下为表名（无后缀） Cnt int64 `json:"cnt"` //当前数量 + PeriodKey string `gorm:"size:16" json:"period_key"` //周期键（月，如 202609）。按体积切出来的旧分片为空 +} + +// IsPeriodShard 返回 true 表示这是按时间周期切出来的分区（M3）， +// false 表示按行数/体积切出来的旧分片——两者的时间范围都记在 StartTime/EndTime 里， +// 区别只在「名字能不能算出周期」以及界面上怎么标。 +func (s ShareDb) IsPeriodShard() bool { + return s.PeriodKey != "" } // IsTableShard 在 MySQL/SQL Server 模式下返回 true（FileName 存的是表名而非文件名） diff --git a/router/waf_log.go b/router/waf_log.go index 828923c8..12f6af09 100644 --- a/router/waf_log.go +++ b/router/waf_log.go @@ -18,6 +18,7 @@ func (receiver *LogRouter) InitLogRouter(group *gin.RouterGroup) { wafLogRouter.GET("/api/v1/waflog/attack/download", logApi.DownloadApi) wafLogRouter.GET("/api/v1/waflog/attack/detail", logApi.GetDetailApi) wafLogRouter.GET("/api/v1/waflog/attack/allsharedb", logApi.GetAllShareDbApi) + wafLogRouter.POST("/api/v1/waflog/attack/shard/del", logApi.DelShardApi) wafLogRouter.GET("/api/v1/waflog/attack/httpcopymask", logApi.GetHttpCopyMaskApi) wafLogRouter.POST("/api/v1/waflog/attack/attackiplist", logApi.GetAttackIPListApi) wafLogRouter.GET("/api/v1/waflog/attack/alliptag", logApi.GetAllIpTagApi) diff --git a/wafdb/migrations_core.go b/wafdb/migrations_core.go index 3e739d0b..35596c11 100644 --- a/wafdb/migrations_core.go +++ b/wafdb/migrations_core.go @@ -2364,6 +2364,27 @@ func RunCoreDBMigrations(db *gorm.DB) error { return nil }, }, + // 迁移: 日志分片表新增周期键列（M3 E1/E2，按时间周期切分区；旧的按体积分片该列为空） + { + ID: "202609240001_add_share_dbs_period_key", + Migrate: func(tx *gorm.DB) error { + zlog.Info("迁移 202609240001: 日志分片新增周期键列") + if tx.Migrator().HasColumn(&model.ShareDb{}, "PeriodKey") { + return nil + } + if err := tx.Migrator().AddColumn(&model.ShareDb{}, "PeriodKey"); err != nil { + return fmt.Errorf("新增分片周期键列失败: %w", err) + } + zlog.Info("迁移 202609240001: 完成") + return nil + }, + Rollback: func(tx *gorm.DB) error { + if !tx.Migrator().HasColumn(&model.ShareDb{}, "PeriodKey") { + return nil + } + return tx.Migrator().DropColumn(&model.ShareDb{}, "PeriodKey") + }, + }, }) // 执行迁移 diff --git a/waftask/clean_archive_crossdb_test.go b/waftask/clean_archive_crossdb_test.go new file mode 100644 index 00000000..f72f716b --- /dev/null +++ b/waftask/clean_archive_crossdb_test.go @@ -0,0 +1,239 @@ +//go:build crossdb + +// 过期回收（E3）的三库回归。 +// +// 这条路径会**真的删数据**，而且 MySQL/PG 上原来根本没人清归档表，所以必须在真库上跑： +// 按层各走各的保留期（窄行短、事件与报文长）、只过期一层时分片记录要留着、 +// 没有分片记录的孤儿分区按周期终点判、实时表一张都不能碰。 +// +// 跑法：go test -tags crossdb ./waftask/ -run TestCleanExpiredArchiveShardCrossEngine +// MySQL / PostgreSQL 连不上时跳过，不阻塞；SQLite 走删文件那条路，本用例只管服务型数据库。 +package waftask + +import ( + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "SamWaf/wafdb/partition" + "os" + "testing" + "time" + + mysqldriver "gorm.io/driver/mysql" + pgdriver "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +const xtestArchiveDB = "samwaf_xtest_archive" + +func TestCleanExpiredArchiveShardCrossEngine(t *testing.T) { + engines := map[string]func(*testing.T) (*gorm.DB, func()){ + "mysql": setupArchiveMySQL, + "postgres": setupArchivePostgres, + } + for name, setup := range engines { + t.Run(name, func(t *testing.T) { + db, done := setup(t) + if db == nil { + t.Skip("引擎不可用，跳过") + } + defer done() + runArchiveCleanupAssertions(t, db) + }) + } +} + +func runArchiveCleanupAssertions(t *testing.T, db *gorm.DB) { + t.Helper() + d := dialect.Get() + + prevCore, prevLog := global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB + prevDel, prevAccess := global.GDATA_DELETE_INTERVAL, global.GDATA_ACCESS_LOG_RETENTION_DAYS + // 分片元数据与日志表放同一个库只是测试便利；生产里前者在核心库。 + global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB = db, db + global.GDATA_DELETE_INTERVAL = 180 + global.GDATA_ACCESS_LOG_RETENTION_DAYS = 30 + defer func() { + global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB = prevCore, prevLog + global.GDATA_DELETE_INTERVAL, global.GDATA_ACCESS_LOG_RETENTION_DAYS = prevDel, prevAccess + }() + + now := time.Now() + tiers := []string{model.AccessLogTableName, model.SecurityEventTableName, model.EventPayloadTableName} + + mkParts := func(key string) { + for _, base := range tiers { + if err := d.CreatePartition(db, base, partition.TableName(base, key)); err != nil { + t.Fatalf("建分区 %s 失败: %v", partition.TableName(base, key), err) + } + } + } + mkShard := func(key string, end time.Time) { + must(t, db.Create(&model.ShareDb{ + BaseOrm: baseOrmFor("xshard_" + key), + DbLogicType: "log", + StartTime: customtype.JsonTime(end.AddDate(0, 0, -30)), + EndTime: customtype.JsonTime(end), + FileName: partition.TableName(model.AccessLogTableName, key), + PeriodKey: key, + Cnt: 1, + }).Error) + } + + // 场景 A：整段远超两个保留期 —— 三层全丢、分片记录一并删掉 + keyOld := partition.KeyOf(now.AddDate(0, 0, -400)) + mkParts(keyOld) + mkShard(keyOld, now.AddDate(0, 0, -390)) + + // 场景 B：过了窄行保留期(30)但没过日志保留期(180) —— 只丢窄行，其余留着，记录也要留着 + keyMid := partition.KeyOf(now.AddDate(0, 0, -60)) + if keyMid == keyOld { + t.Fatalf("测试周期键撞车了: %s", keyMid) + } + mkParts(keyMid) + mkShard(keyMid, now.AddDate(0, 0, -60)) + + // 场景 C：孤儿分区（有表、没分片记录），周期终点远在保留期之外 —— 该被丢掉 + keyOrphan := partition.KeyOf(now.AddDate(0, 0, -800)) + mkParts(keyOrphan) + + CleanExpiredArchiveShard() + + // A：三层都该没了 + for _, base := range tiers { + if name := partition.TableName(base, keyOld); d.TableExists(db, name) { + t.Errorf("整段过期的分区 %s 应被丢掉", name) + } + } + var cntOld int64 + must(t, db.Model(&model.ShareDb{}).Where("period_key = ?", keyOld).Count(&cntOld).Error) + if cntOld != 0 { + t.Errorf("整个分区都没了，分片记录应一并删除，实际还有 %d 条", cntOld) + } + + // B：窄行丢掉、事件与报文留下、记录留下 + if name := partition.TableName(model.AccessLogTableName, keyMid); d.TableExists(db, name) { + t.Errorf("窄行 %s 已过访问日志保留期，应被丢掉", name) + } + for _, base := range []string{model.SecurityEventTableName, model.EventPayloadTableName} { + if name := partition.TableName(base, keyMid); !d.TableExists(db, name) { + t.Errorf("%s 还在日志保留期内，不该被丢", name) + } + } + var cntMid int64 + must(t, db.Model(&model.ShareDb{}).Where("period_key = ?", keyMid).Count(&cntMid).Error) + if cntMid != 1 { + t.Errorf("还有层没过期时分片记录必须留着，实际 %d 条", cntMid) + } + + // C：孤儿分区被丢 + for _, base := range tiers { + if name := partition.TableName(base, keyOrphan); d.TableExists(db, name) { + t.Errorf("没有分片记录且早已过期的孤儿分区 %s 应被丢掉", name) + } + } + + // 实时表一张都不能少 + for _, base := range append(tiers, wafdb.LogTableName) { + if !d.TableExists(db, base) { + t.Fatalf("实时表 %s 被误删了", base) + } + } +} + +// baseOrmFor 造一个够用的 BaseOrm（回收逻辑只认 Id 与归属列） +func baseOrmFor(id string) baseorm.BaseOrm { + return baseorm.BaseOrm{ + Id: id, + USER_CODE: "xtest_user", + Tenant_ID: "xtest_tenant", + CREATE_TIME: customtype.JsonTime(time.Now()), + UPDATE_TIME: customtype.JsonTime(time.Now()), + } +} + +func setupArchiveMySQL(t *testing.T) (*gorm.DB, func()) { + base := os.Getenv("SAMWAF_TEST_MYSQL_DSN") + if base == "" { + base = "root:canteen1@tcp(127.0.0.1:3306)/" + } + root, err := gorm.Open(mysqldriver.Open(base+"?parseTime=true"), analysisSilentCfg) + if err != nil { + t.Logf("mysql 连接失败（跳过）: %v", err) + return nil, nil + } + root.Exec("DROP DATABASE IF EXISTS " + xtestArchiveDB) + if err := root.Exec("CREATE DATABASE " + xtestArchiveDB + " CHARACTER SET utf8mb4").Error; err != nil { + t.Logf("mysql 建库失败（跳过）: %v", err) + return nil, nil + } + dialect.Register(&dialect.MySQLDialect{}) + db, err := gorm.Open(mysqldriver.Open(base+xtestArchiveDB+"?charset=utf8mb4&parseTime=True&loc=Local"), analysisSilentCfg) + if err != nil { + t.Fatalf("mysql 打开失败: %v", err) + } + migrateArchiveSchema(t, db) + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + root.Exec("DROP DATABASE IF EXISTS " + xtestArchiveDB) + if s, e := root.DB(); e == nil { + s.Close() + } + } +} + +func setupArchivePostgres(t *testing.T) (*gorm.DB, func()) { + base := os.Getenv("SAMWAF_TEST_PG_DSN") + if base == "" { + base = "postgres://postgres:postgres@127.0.0.1:5432/" + } + root, err := gorm.Open(pgdriver.Open(base+"postgres?sslmode=disable"), analysisSilentCfg) + if err != nil { + t.Logf("postgres 连接失败（跳过）: %v", err) + return nil, nil + } + root.Exec("DROP DATABASE IF EXISTS " + xtestArchiveDB) + if err := root.Exec("CREATE DATABASE " + xtestArchiveDB + " ENCODING 'UTF8'").Error; err != nil { + t.Logf("postgres 建库失败（跳过）: %v", err) + return nil, nil + } + dialect.Register(&dialect.PostgresDialect{}) + db, err := gorm.Open(pgdriver.Open(base+xtestArchiveDB+"?sslmode=disable&TimeZone=Asia/Shanghai"), analysisSilentCfg) + if err != nil { + t.Fatalf("postgres 打开失败: %v", err) + } + migrateArchiveSchema(t, db) + return db, func() { + if s, e := db.DB(); e == nil { + s.Close() + } + root.Exec("DROP DATABASE IF EXISTS " + xtestArchiveDB) + if s, e := root.DB(); e == nil { + s.Close() + } + } +} + +// migrateArchiveSchema 建出用例要的表：日志三层走真实迁移（嵌入结构的索引名在多张表上会撞， +// 直接 AutoMigrate 会失败，这是 M2 踩过的坑），分片元数据表单独 AutoMigrate。 +func migrateArchiveSchema(t *testing.T, db *gorm.DB) { + t.Helper() + if err := wafdb.RunLogDBMigrations(db); err != nil { + t.Fatalf("日志库迁移失败: %v", err) + } + if err := db.AutoMigrate(&model.ShareDb{}); err != nil { + t.Fatalf("分片表迁移失败: %v", err) + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatalf("执行失败: %v", err) + } +} diff --git a/waftask/task_clean_archive.go b/waftask/task_clean_archive.go index cd8ec8a7..93ca432d 100644 --- a/waftask/task_clean_archive.go +++ b/waftask/task_clean_archive.go @@ -2,31 +2,88 @@ package waftask import ( "SamWaf/common/zlog" - "SamWaf/enums" "SamWaf/global" + "SamWaf/model" "SamWaf/service/waf_service" "SamWaf/utils" "SamWaf/wafdb" "SamWaf/wafdb/dialect" + "SamWaf/wafdb/partition" "os" "time" ) -// CleanExpiredArchiveShard 清理超过保留期(GDATA_DELETE_INTERVAL 天)的归档日志分片文件。 +// 过期回收 = 丢整个分区（M3 E3）。 // -// 背景：高频切库(见 TaskShareDbInfo)后会产生大量 local_log_.db 归档文件， -// 而原有的删历史逻辑只对 live 库做 DELETE，从不清理归档文件，会很快堆满磁盘。 -// 真正的 N 天保留改由本函数按分片 EndTime 删除整个过期归档文件实现。 +// 原来只有 SQLite 的归档文件会被删，MySQL/PG 的归档表**没人清**（注释里写的 +// 「由其各自策略处理」并不存在），归档表会一直堆着。现在两边统一： // -// 仅 SQLite(文件型驱动)生效；MySQL 的归档是表，由其各自策略处理。 -func CleanExpiredArchiveShard() { - innerLogName := "CleanExpiredArchiveShard" - if !dialect.Get().IsFileBased() { - return +// - SQLite：一个分区就是一个 .db 文件，整体删除。文件里装着三层，只能按**较长**的 +// 那个保留期（日志保留天数）回收——要更快回收窄行，就把日志保留天数调短， +// 或者换到服务型数据库。 +// - MySQL/PG：一个分区是每层各一张表，可以**按层各走各的保留期**： +// 窄行 access_log_* 走访问日志保留天数（默认 30 天，通常更短）， +// 安全事件 / 报文 / 存量 web_logs 走日志保留天数。某一层先过期就先丢那一层， +// 整个分区都没了才删掉分片记录。 +// +// 实时库里的过期数据仍然按行删（DeleteHistory）：保留期可能短于一个周期， +// 这时候当前周期的分区里就有该删的行，丢分区帮不上。丢分区省掉的是**历史**那一大堆 +// 逐行 DELETE + VACUUM。 + +// tierRetention 一层存储 + 它该用哪个保留期 +type tierRetention struct { + base string + days int +} + +// archiveTiers 归档分区里可能有的四层，以及各自的保留期来源 +func archiveTiers() []tierRetention { + logDays := int(global.GDATA_DELETE_INTERVAL) + accessDays := int(global.GDATA_ACCESS_LOG_RETENTION_DAYS) + if accessDays <= 0 { + accessDays = logDays + } + return []tierRetention{ + {model.AccessLogTableName, accessDays}, + {model.SecurityEventTableName, logDays}, + {model.EventPayloadTableName, logDays}, + {wafdb.LogTableName, logDays}, } +} + +// shardEnd 给出一个分片里数据的结束时刻：优先用 share_dbs 记的 EndTime（真实数据边界）， +// 缺失时回落到周期键算出来的周期终点。两个都没有就返回 false —— 认不出时间范围的分片宁可留着。 +func shardEnd(shard model.ShareDb) (time.Time, bool) { + if end := time.Time(shard.EndTime); !end.IsZero() && end.Unix() > 0 { + return end, true + } + if key, ok := partition.KeyFromName(shard.FileName); ok { + if _, end, err := partition.Range(key); err == nil { + return end, true + } + } + return time.Time{}, false +} - cutoff := time.Now().AddDate(0, 0, -int(global.GDATA_DELETE_INTERVAL)) - currentDir := utils.GetCurrentDir() +// expiredBy 数据结束时刻早于「现在 - 保留天数」即过期。days<=0 表示不按天清理。 +func expiredBy(end time.Time, days int, now time.Time) bool { + if days <= 0 { + return false + } + return end.Before(now.AddDate(0, 0, -days)) +} + +// isLiveShardName / archiveSuffix 与「分区管理」用的是同一套判定（waf_service 里）， +// 这里只是转调：回收与主动删除对「什么是实时库」「后缀怎么取」必须完全一致， +// 各写一份早晚会漂。 +func isLiveShardName(name string) bool { return waf_service.IsLiveShardName(name) } + +func archiveSuffix(name string) string { return waf_service.ArchiveSuffix(name) } + +// CleanExpiredArchiveShard 回收过期的归档分区。 +func CleanExpiredArchiveShard() { + innerLogName := "CleanExpiredArchiveShard" + now := time.Now() shards, err := waf_service.WafShareDbServiceApp.GetAllShareDbApi() if err != nil { @@ -34,41 +91,150 @@ func CleanExpiredArchiveShard() { return } - removedFiles := 0 + removed := 0 for _, shard := range shards { - // 只处理日志分片；跳过 live 库本身 - if shard.DbLogicType != "log" { + if shard.DbLogicType != "" && shard.DbLogicType != "log" { continue } - if shard.FileName == "" || shard.FileName == enums.DB_LOG || shard.FileName == "local_log.db" { + if isLiveShardName(shard.FileName) { continue } - // 整个分片(截止 EndTime)早于保留期才删除 - if !time.Time(shard.EndTime).Before(cutoff) { + end, ok := shardEnd(shard) + if !ok { + zlog.Debug(innerLogName, "分片没有可判断的时间范围，跳过", "name", shard.FileName) continue } - // 删除前先关闭可能已按需打开的连接，避免删正在查询的文件 - wafdb.CloseManualLogDb(shard.FileName) - - dbPath := currentDir + "/data/" + shard.FileName - for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} { - if err := os.Remove(p); err != nil && !os.IsNotExist(err) { - zlog.Warn(innerLogName, "删除归档文件失败", "file", p, "error", err.Error()) + if dialect.Get().IsFileBased() { + if !expiredBy(end, int(global.GDATA_DELETE_INTERVAL), now) { + continue } + if dropShardFile(innerLogName, shard) { + removed++ + } + continue + } + + if dropShardTables(innerLogName, shard, end, now) { + removed++ + } + } + + // 孤儿分区：库里有分区表、share_dbs 里却没有对应记录（记录被手工删了、或切表时记录没写成）。 + // 只处理名字能算出周期键的，按周期终点判过期——认不出周期的一律不动。 + orphan := dropOrphanPartitions(innerLogName, shards, now) + + if removed > 0 || orphan > 0 { + waf_service.InvalidateShardCounts() + zlog.Info(innerLogName, "归档回收完成", "分片", removed, "孤儿分区表", orphan) + } +} + +// dropShardFile 删除 SQLite 分区文件（含 -wal / -shm）与分片记录 +func dropShardFile(innerLogName string, shard model.ShareDb) bool { + // 删除前先关掉可能已按需打开的连接，避免删正在被查询的文件 + wafdb.CloseManualLogDb(shard.FileName) + + dbPath := utils.GetCurrentDir() + "/data/" + shard.FileName + for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} { + if err := os.Remove(p); err != nil && !os.IsNotExist(err) { + zlog.Warn(innerLogName, "删除归档文件失败", "file", p, "error", err.Error()) + } + } + if err := waf_service.WafShareDbServiceApp.DeleteById(shard.Id); err != nil { + zlog.Warn(innerLogName, "删除归档记录失败", "file", shard.FileName, "error", err.Error()) + return false + } + zlog.Info(innerLogName, "已回收过期归档分区", "file", shard.FileName, + "end_time", time.Time(shard.EndTime).Format("2006-01-02 15:04:05")) + return true +} + +// dropShardTables 丢掉 MySQL/PG 分区里已过期的那些层。 +// 返回 true 表示这个分片已经整体没了、分片记录也删掉了。 +func dropShardTables(innerLogName string, shard model.ShareDb, end, now time.Time) bool { + suffix := archiveSuffix(shard.FileName) + if suffix == "" { + zlog.Debug(innerLogName, "认不出归档标识，跳过", "name", shard.FileName) + return false + } + db := global.GWAF_LOCAL_LOG_DB + if db == nil { + return false + } + + left := 0 + dropped := 0 + for _, tier := range archiveTiers() { + part := tier.base + "_" + suffix + if !dialect.Get().TableExists(db, part) { + continue } - removedFiles++ - - // 删除归档元数据记录 - if err := waf_service.WafShareDbServiceApp.DeleteById(shard.Id); err != nil { - zlog.Warn(innerLogName, "删除归档记录失败", "file", shard.FileName, "error", err.Error()) - } else { - zlog.Info(innerLogName, "已清理过期归档分片", "file", shard.FileName, - "end_time", time.Time(shard.EndTime).Format("2006-01-02 15:04:05")) + if !expiredBy(end, tier.days, now) { + left++ + continue + } + // DropPartition 自己会校验「分区必须属于该基表」，实时表传不进来 + if err := dialect.Get().DropPartition(db, tier.base, part); err != nil { + zlog.Warn(innerLogName, "丢分区表失败", "table", part, "error", err.Error()) + left++ + continue } + dropped++ + zlog.Info(innerLogName, "已丢过期分区表", "table", part, "保留天数", tier.days, + "数据截止", end.Format("2006-01-02 15:04:05")) } - if removedFiles > 0 { - zlog.Info(innerLogName, "归档清理完成", "removed", removedFiles, "cutoff", cutoff.Format("2006-01-02")) + if left > 0 || dropped == 0 { + return false + } + if err := waf_service.WafShareDbServiceApp.DeleteById(shard.Id); err != nil { + zlog.Warn(innerLogName, "删除归档记录失败", "name", shard.FileName, "error", err.Error()) + return false + } + return true +} + +// dropOrphanPartitions 丢掉没有分片记录、且周期已过期的分区表。 +// 保守起见只动「名字能算出周期键」的表：认不出周期的可能是用户自己建的表。 +func dropOrphanPartitions(innerLogName string, shards []model.ShareDb, now time.Time) int { + if dialect.Get().IsFileBased() || global.GWAF_LOCAL_LOG_DB == nil { + return 0 + } + known := make(map[string]struct{}, len(shards)*4) + for _, s := range shards { + if suffix := archiveSuffix(s.FileName); suffix != "" { + for _, tier := range archiveTiers() { + known[tier.base+"_"+suffix] = struct{}{} + } + } + } + + dropped := 0 + for _, tier := range archiveTiers() { + parts, err := dialect.Get().ListPartitions(global.GWAF_LOCAL_LOG_DB, tier.base) + if err != nil { + zlog.Warn(innerLogName, "列分区失败", "base", tier.base, "error", err.Error()) + continue + } + for _, part := range parts { + if _, ok := known[part]; ok { + continue + } + key, ok := partition.KeyFromName(part) + if !ok { + continue // 认不出周期：不是我们切的，别动 + } + if !partition.Expired(key, tier.days, now) { + continue + } + if err := dialect.Get().DropPartition(global.GWAF_LOCAL_LOG_DB, tier.base, part); err != nil { + zlog.Warn(innerLogName, "丢孤儿分区表失败", "table", part, "error", err.Error()) + continue + } + dropped++ + zlog.Info(innerLogName, "已丢无记录的过期分区表", "table", part, "周期", key, "保留天数", tier.days) + } } + return dropped } diff --git a/waftask/task_clean_archive_test.go b/waftask/task_clean_archive_test.go new file mode 100644 index 00000000..3d9fca85 --- /dev/null +++ b/waftask/task_clean_archive_test.go @@ -0,0 +1,80 @@ +package waftask + +import ( + "SamWaf/customtype" + "SamWaf/model" + "testing" + "time" +) + +// 回收会真的删数据，所以「什么算过期」「认不出的一律不动」这两条判断必须单独钉住。 + +func TestExpiredByUsesRetentionDays(t *testing.T) { + now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.Local) + end := time.Date(2026, 9, 30, 23, 59, 0, 0, time.Local) + + if expiredBy(end, 30, now) { + t.Error("数据截止 9/30、保留 30 天、现在 10/10：还在保留期内，不该过期") + } + if !expiredBy(end, 5, now) { + t.Error("保留 5 天时 9/30 的数据已过期") + } + // 0 / 负数 = 不按天清理，任何数据都不能被删 + if expiredBy(time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), 0, now) { + t.Error("保留天数 <=0 表示不按天清理，不该判过期") + } + if expiredBy(time.Date(2000, 1, 1, 0, 0, 0, 0, time.Local), -1, now) { + t.Error("保留天数为负同样不该判过期") + } +} + +func TestShardEndPrefersRecordedTime(t *testing.T) { + want := time.Date(2026, 9, 21, 15, 4, 5, 0, time.Local) + got, ok := shardEnd(model.ShareDb{FileName: "access_log_202609", EndTime: customtype.JsonTime(want)}) + if !ok || !got.Equal(want) { + t.Fatalf("有 EndTime 时应直接用它，实际 (%v,%v)", got, ok) + } + + // EndTime 缺失：回落周期键算出来的周期终点（9 月 → 10/1 零点） + got, ok = shardEnd(model.ShareDb{FileName: "access_log_202609"}) + if !ok { + t.Fatal("EndTime 缺失时应能靠周期键算出终点") + } + if got.Year() != 2026 || got.Month() != time.October || got.Day() != 1 { + t.Fatalf("周期终点应是 2026-10-01，实际 %v", got) + } + + // 既没有 EndTime、名字也算不出周期（按体积切的旧分片）：认不出就别动它 + if _, ok := shardEnd(model.ShareDb{FileName: "access_log_20260921153045"}); ok { + t.Error("时间范围认不出来的分片必须返回 false，宁可留着也不能误删") + } +} + +func TestIsLiveShardNameCoversEveryDriver(t *testing.T) { + for _, name := range []string{"", "local_log.db", "web_logs", "access_log"} { + if !isLiveShardName(name) { + t.Errorf("%q 是实时库标识，必须被挡在回收之外", name) + } + } + for _, name := range []string{"local_log_202609.db", "access_log_202609", "web_logs_20260921153045"} { + if isLiveShardName(name) { + t.Errorf("%q 是归档分片，不该被当成实时库", name) + } + } +} + +func TestArchiveSuffix(t *testing.T) { + cases := map[string]string{ + "access_log_202609": "202609", + "access_log_202609_02": "202609_02", + "web_logs_20260921153045": "20260921153045", + "local_log_202609.db": "", // SQLite 走删文件那条路，不需要后缀 + "access_log": "", // 实时表：取不出后缀，回收逻辑因此不会碰它 + "security_event_202609": "", // 只认 web_logs_/access_log_ 两种标识，与读侧口径一致 + } + for name, want := range cases { + if got := archiveSuffix(name); got != want { + t.Errorf("archiveSuffix(%q) = %q，期望 %q", name, got, want) + } + } +} diff --git a/waftask/task_db_sharding.go b/waftask/task_db_sharding.go index d2240e2e..7c9a10d3 100644 --- a/waftask/task_db_sharding.go +++ b/waftask/task_db_sharding.go @@ -8,15 +8,67 @@ import ( "SamWaf/innerbean" "SamWaf/model" "SamWaf/model/baseorm" + "SamWaf/service/waf_service" "SamWaf/utils" "SamWaf/wafdb" "SamWaf/wafdb/dialect" + "SamWaf/wafdb/partition" "fmt" "os" "time" ) -// 检测库是否切换 +// livePeriodKey 返回实时库里最早一条日志所属的周期键。 +// +// 「实时库现在装的是哪个周期」这件事不另存状态，直接问数据本身： +// 最早一条落在上一个周期，就说明周期边界已经过了、该切了。这样重启、停机几天、 +// 手工删表都不会让状态和事实脱节（存一个全局变量反而要处理这些不一致）。 +// 表不存在或没有数据时返回 false —— 没数据就没什么可切的。 +func livePeriodKey() (string, bool) { + db := global.GWAF_LOCAL_LOG_DB + if db == nil { + return "", false + } + table := model.AccessLogTableName + if !dialect.Get().TableExists(db, table) { + table = "web_logs" + if !dialect.Get().TableExists(db, table) { + return "", false + } + } + var oldest *int64 + if err := db.Table(table).Select("MIN(unix_add_time)").Scan(&oldest).Error; err != nil { + zlog.Debug("TaskDBSharding", "取最早日志时间失败", err.Error()) + return "", false + } + if oldest == nil || *oldest <= 0 { + return "", false + } + // UNIX_ADD_TIME 是毫秒 + return partition.KeyOf(time.UnixMilli(*oldest)), true +} + +// nextPeriodSeq 返回某个周期下一个可用的序号：正常是 1（一个周期一个分区）， +// 已经有同周期分区时才往上加——那是「同周期内体积超限被迫再切」的异常兜底。 +func nextPeriodSeq(periodKey string) int { + if global.GWAF_LOCAL_DB == nil { + return 1 + } + var cnt int64 + if err := global.GWAF_LOCAL_DB.Model(&model.ShareDb{}). + Where("db_logic_type = ? and period_key = ?", "log", periodKey).Count(&cnt).Error; err != nil { + zlog.Debug("TaskDBSharding", "统计同周期分区数失败", err.Error()) + return 1 + } + return int(cnt) + 1 +} + +// 检测库是否切换。 +// +// 切分口径（M3 E2）：**按时间周期切**，一个月一个分区。 +// 行数与体积超限降级为异常兜底——同一个周期内真被写爆了才提前切，分区名带序号 +// （access_log_202609_02）。这样「一段时间落在哪些分区上」是算出来的而不是猜的， +// 过期也变成丢整个分区（见 CleanExpiredArchiveShard）。 func TaskShareDbInfo() { innerLogName := "TaskDBSharding" zlog.Debug(innerLogName, "检测是否需要进行分库") @@ -41,11 +93,20 @@ func TaskShareDbInfo() { global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&legacyCnt) global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Count(&accessCnt) if legacyCnt > 0 && accessCnt == 0 { - doLogShardCut(innerLogName, fmt.Sprintf("分层改造边界切换（存量 %d 行转入归档）", legacyCnt), true) + // 存量 web_logs 里的数据跨很多个月，给它安一个周期键是假的，沿用时间戳命名 + doLogShardCut(innerLogName, fmt.Sprintf("分层改造边界切换（存量 %d 行转入归档）", legacyCnt), true, "") return } } + // 周期边界优先：实时库里最早一条日志落在上一个周期，就把这一段整体切成那个周期的分区。 + // 放在体积判断之前——常态就该按周期切，体积只是兜底。 + curKey := partition.KeyOf(time.Now()) + if oldKey, ok := livePeriodKey(); ok && oldKey != curKey { + doLogShardCut(innerLogName, fmt.Sprintf("跨周期切分（%s → %s）", oldKey, curKey), false, oldKey) + return + } + //获取当前日志数量（分层的写入主体是 access_log；老表不再写入，只作兜底） var total int64 = 0 if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.AccessLogTableName) { @@ -102,7 +163,8 @@ func TaskShareDbInfo() { } if needSharding { - doLogShardCut(innerLogName, shardingReason, false) + // 同周期内被写爆了：仍然按当前周期命名，序号从 02 起，读侧与过期判断照样认得出周期 + doLogShardCut(innerLogName, shardingReason+"（同周期内兜底切分）", false, curKey) } } @@ -110,17 +172,25 @@ func TaskShareDbInfo() { // （把存量 web_logs + event_payload 切出去）；常规切分换的是三层新表， // web_logs 不再写入也就无需再换。 // +// periodKey 非空则按周期命名（local_log_202609.db / access_log_202609，同周期第二个起带 _02 序号）， +// 为空则沿用 14 位时间戳命名——只有分层改造的边界切换会走后者，那批存量数据跨很多个月， +// 安一个周期键是假的。 +// // 归档标识：SQLite 为新文件名(.db)，MySQL/PG 为归档表名（边界切换是 web_logs_， -// 常规切分是 access_log_；读侧 ResolveTierTables 按前缀两种都认）。 +// 常规切分是 access_log_<周期或时间戳>；读侧 ResolveTierTables 按前缀两种都认）。 // 注意：MySQL/PG 的归档表不再被 gormigrate 跟踪，今后给这些表加列时读旧分片可能缺列—— // 读侧已按分片实际列取交集（webLogSelect），无需同步 ALTER。 -func doLogShardCut(innerLogName, reason string, swapWebLog bool) { +func doLogShardCut(innerLogName, reason string, swapWebLog bool, periodKey string) { global.GDATA_CURRENT_CHANGE = true defer func() { global.GDATA_CURRENT_CHANGE = false }() zlog.Info(innerLogName, "开始分库，原因:", reason) ts := time.Now().Format("20060102150405") + cutTable := model.AccessLogTableName + if swapWebLog { + cutTable = "web_logs" + } var total int64 if swapWebLog { global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&total) @@ -128,21 +198,38 @@ func doLogShardCut(innerLogName, reason string, swapWebLog bool) { global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Count(&total) } - newDBFilename := fmt.Sprintf("local_log_%v.db", ts) + // 分区命名：有周期键按周期来，同周期第二个起带序号；没有则沿用时间戳 + suffix := ts + seq := 1 + if periodKey != "" { + seq = nextPeriodSeq(periodKey) + suffix = periodKey + if seq > 1 { + suffix = fmt.Sprintf("%s_%02d", periodKey, seq) + } + } + newDBFilename := fmt.Sprintf("local_log_%v.db", suffix) archiveName := newDBFilename if !dialect.Get().IsFileBased() { if swapWebLog { - archiveName = fmt.Sprintf("web_logs_%v", ts) + archiveName = "web_logs_" + suffix } else { - archiveName = fmt.Sprintf("access_log_%v", ts) + archiveName = model.AccessLogTableName + "_" + suffix } } - var lastedDb model.ShareDb - err := global.GWAF_LOCAL_DB.Limit(1).Order("create_time desc").Find(&lastedDb).Error + // 起止时间用**这批数据自己的**最早/最晚时间，而不是「上一个分片的结束时间 → 现在」。 + // 过期回收按 EndTime 判（见 CleanExpiredArchiveShard），拿真实边界才不会把还在保留期内的 + // 数据算成过期；停机几天再启动、或同周期内兜底切分时，这个差别很要紧。 startTime := customtype.JsonTime(time.Now()) - if err == nil { - startTime = lastedDb.EndTime + endTime := customtype.JsonTime(time.Now()) + if lo, hi, ok := cutDataRange(cutTable); ok { + startTime, endTime = customtype.JsonTime(lo), customtype.JsonTime(hi) + } else { + var lastedDb model.ShareDb + if err := global.GWAF_LOCAL_DB.Limit(1).Order("create_time desc").Find(&lastedDb).Error; err == nil { + startTime = lastedDb.EndTime + } } sharDbBean := model.ShareDb{ BaseOrm: baseorm.BaseOrm{ @@ -154,9 +241,10 @@ func doLogShardCut(innerLogName, reason string, swapWebLog bool) { }, DbLogicType: "log", StartTime: startTime, - EndTime: customtype.JsonTime(time.Now()), + EndTime: endTime, FileName: archiveName, Cnt: total, + PeriodKey: periodKey, } zlog.Info(innerLogName, "正在切库中...") @@ -224,5 +312,29 @@ func doLogShardCut(innerLogName, reason string, swapWebLog bool) { global.GWAF_LOCAL_DB.Create(sharDbBean) zlog.Info(innerLogName, "分表完成，归档表:", archiveName) } + // 分区构成变了，归档计数缓存跟着作废 + waf_service.InvalidateShardCounts() zlog.Info(innerLogName, "切库完成...") } + +// cutDataRange 返回待切表里数据的真实时间边界 [最早, 最晚]。 +// 表空或查不到返回 false，调用方回落到旧口径。 +func cutDataRange(table string) (time.Time, time.Time, bool) { + db := global.GWAF_LOCAL_LOG_DB + if db == nil || !dialect.Get().TableExists(db, table) { + return time.Time{}, time.Time{}, false + } + var row struct { + Lo *int64 + Hi *int64 + } + if err := db.Table(table).Select("MIN(unix_add_time) as lo, MAX(unix_add_time) as hi").Scan(&row).Error; err != nil { + zlog.Debug("TaskDBSharding", "取数据时间边界失败", err.Error()) + return time.Time{}, time.Time{}, false + } + if row.Lo == nil || row.Hi == nil || *row.Lo <= 0 || *row.Hi <= 0 { + return time.Time{}, time.Time{}, false + } + // UNIX_ADD_TIME 是毫秒 + return time.UnixMilli(*row.Lo), time.UnixMilli(*row.Hi), true +} diff --git a/waftask/task_history.go b/waftask/task_history.go index 8f6af386..a47d8fdb 100644 --- a/waftask/task_history.go +++ b/waftask/task_history.go @@ -18,7 +18,8 @@ func TaskDeleteHistoryInfo() { accessBeforeDay := time.Now().AddDate(0, 0, -int(global.GDATA_ACCESS_LOG_RETENTION_DAYS)).Format("2006-01-02 15:04") waf_service.WafLogServiceApp.DeleteHistory(deleteBeforeDay, accessBeforeDay) - // 清理过期的归档分片文件（高频切库后 live 库只存最近数据，真正的保留期回收靠删归档文件） + // 回收过期的归档分区：实时库只装当前周期，历史数据的保留期靠丢整个分区实现 + // （SQLite 删文件，MySQL/PG 丢表，按层各走各的保留期） CleanExpiredArchiveShard() // 仅 SQLite：DELETE 不会收缩文件，主动 checkpoint 截断 WAL 并 VACUUM 回收空间。 From 25ba6f2373127860d39dbceaf6ca10711d57eade Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 28 Sep 2026 09:54:02 +0800 Subject: [PATCH 18/21] feat: manage log partitions, and answer why an IP has no logs --- innerbean/web_log.go | 5 + service/waf_service/waf_ip_lookup_service.go | 104 +++++++++++- service/waf_service/waf_sharedb_delete.go | 167 +++++++++++++++++++ wafenginecore/ipset/listtext.go | 55 ++++++ wafenginecore/weblog_exclude.go | 36 +--- 5 files changed, 337 insertions(+), 30 deletions(-) create mode 100644 service/waf_service/waf_sharedb_delete.go create mode 100644 wafenginecore/ipset/listtext.go diff --git a/innerbean/web_log.go b/innerbean/web_log.go index a16858fb..87a2fadc 100644 --- a/innerbean/web_log.go +++ b/innerbean/web_log.go @@ -70,6 +70,11 @@ type WebLog struct { // 避免 `MF.COUNTRY != "中国"` 这类规则在 IPv6 地区库缺失时把访客整片误杀。 // 仅运行期使用，不落库、不出接口。 GeoUnresolved bool `gorm:"-" json:"-"` + + // ShardName 这条记录是从哪个分区读出来的。**只在读侧回填**（列表扇出与详情定位时）， + // 不落库；引擎侧的对象恒为空，omitempty 保证它不会混进 Kafka 出口的报文里。 + // 界面靠它标注「这条在哪个分区」，详情链接也靠它直达而不必再逐个分区找。 + ShardName string `gorm:"-" json:"shard_name,omitempty"` } // GetHeaderValue 从HEADER字段中提取指定header的值 diff --git a/service/waf_service/waf_ip_lookup_service.go b/service/waf_service/waf_ip_lookup_service.go index b8eb3bcc..a2cd6c5b 100644 --- a/service/waf_service/waf_ip_lookup_service.go +++ b/service/waf_service/waf_ip_lookup_service.go @@ -148,7 +148,7 @@ func (r *WafIPLookupService) Lookup(ipStr string, sources []string) (*response2. Degraded: make([]string, 0), } for _, src := range []string{ - srcIPWhite, srcIPBlack, srcIPGroup, srcThreatIP, + srcIPWhite, srcIPBlack, srcIPGroup, srcLogExclude, srcThreatIP, srcIPFailure, srcCCBan, srcFirewall, srcCDN, } { if pick(src) { @@ -178,6 +178,9 @@ func (r *WafIPLookupService) Lookup(ipStr string, sources []string) (*response2. if pick(srcIPGroup) { r.matchIPGroup(ipStr, resp) } + if pick(srcLogExclude) { + r.matchLogExclude(ipStr, resp) + } if pick(srcThreatIP) { r.matchThreatIP(ipStr, parsed, resp) } @@ -317,6 +320,7 @@ func (r *WafIPLookupService) matchIPGroup(ip string, resp *response2.IPLookupRes } refs := r.groupRefs() + logRefs := r.logExcludeGroupRefs() for _, g := range groups { for _, it := range byGroup[g.GroupCode] { if it.Ip == "" || !utils.MatchIPPattern(ip, it.Ip) { @@ -327,6 +331,13 @@ func (r *WafIPLookupService) matchIPGroup(ip string, resp *response2.IPLookupRes if ref, ok := refs[g.GroupCode]; ok { effect = ref.effect detail = ref.text + } else if text, ok := logRefs[g.GroupCode]; ok { + // 组没被黑白名单引用、却被日志排除清单引用时，原来的文案会让人以为这组没任何作用 + effect = "log_skip" + detail = text + } + if text, ok := logRefs[g.GroupCode]; ok && effect != "log_skip" { + detail = detail + "；同时" + text } resp.Hits = append(resp.Hits, response2.IPLookupHit{ Source: srcIPGroup, @@ -682,3 +693,94 @@ func (r *WafIPLookupService) matchCDN(ip string, resp *response2.IPLookupResp) { }) } } + +// srcLogExclude 日志排除清单（M6）：命中它的 IP，正常请求不记访问日志。 +// +// 为什么要进「IP归属查询」：这条链路常常隔着一层——把 IP 放进一个 IP 组、 +// 组被某个站点的「记录日志时排除IP」引用，事后自己都想不起来为什么这个 IP 没有日志。 +// 归属查询本来就是回答「这个 IP 现在被什么规则罩着」，日志排除属于同一个问题。 +const srcLogExclude = "log_exclude" + +// logExcludeList 一份生效中的排除清单：来自全局配置或某个站点 +type logExcludeList struct { + scope string // 「全局」或站点名 + raw string +} + +// collectLogExcludeLists 收齐全局与各站点的排除清单（空的跳过） +func (r *WafIPLookupService) collectLogExcludeLists() []logExcludeList { + lists := make([]logExcludeList, 0, 4) + if strings.TrimSpace(global.GCONFIG_EXCLUDE_IP_LOG) != "" { + lists = append(lists, logExcludeList{scope: "全局", raw: global.GCONFIG_EXCLUDE_IP_LOG}) + } + var hosts []model.Hosts + if err := global.GWAF_LOCAL_DB.Where("exclude_ip_log <> ''").Find(&hosts).Error; err == nil { + for _, h := range hosts { + if strings.TrimSpace(h.EXCLUDE_IP_LOG) == "" { + continue + } + name := h.Host + if name == "" { + name = h.Code + } + lists = append(lists, logExcludeList{scope: name, raw: h.EXCLUDE_IP_LOG}) + } + } + return lists +} + +// matchLogExclude 查这个 IP 是否被某份日志排除清单命中，直接命中与经由 IP 组命中都要报出来 +func (r *WafIPLookupService) matchLogExclude(ip string, resp *response2.IPLookupResp) { + for _, l := range r.collectLogExcludeLists() { + patterns, codes := ipset.ParseListText(l.raw) + + // 直接写在清单里的模式 + if len(patterns) > 0 { + if set := ipset.BuildMatchSet(patterns); set != nil && set.ContainsStr(ip) { + matched := "" + for _, p := range patterns { + if one := ipset.BuildMatchSet([]string{p}); one != nil && one.ContainsStr(ip) { + matched = p + break + } + } + resp.Hits = append(resp.Hits, response2.IPLookupHit{ + Source: srcLogExclude, SourceName: "日志排除", Scope: l.scope, + Matched: matched, Effect: "log_skip", + Detail: "命中「记录日志时排除IP」，该IP的正常请求不记访问日志（安全事件照常记录）", + }) + } + } + + // 经由 group: 引用命中——用户最容易忘掉的就是这一层 + for _, code := range codes { + m := ipset.GetGroupMatcher(code) + if m == nil || !m.ContainsStr(ip) { + continue + } + resp.Hits = append(resp.Hits, response2.IPLookupHit{ + Source: srcLogExclude, SourceName: "日志排除", Scope: l.scope, + Matched: "group:" + code, Effect: "log_skip", + Detail: "通过IP组 " + code + " 被「记录日志时排除IP」命中，该IP的正常请求不记访问日志（安全事件照常记录）", + }) + } + } +} + +// logExcludeGroupRefs 返回被日志排除清单引用的组短码 → 说明文案 +func (r *WafIPLookupService) logExcludeGroupRefs() map[string]string { + refs := map[string]string{} + for _, l := range r.collectLogExcludeLists() { + for _, code := range ipset.GroupCodesOf(l.raw) { + if old, ok := refs[code]; ok { + refs[code] = old + "、" + l.scope + continue + } + refs[code] = "被「记录日志时排除IP」引用（" + l.scope + } + } + for code, text := range refs { + refs[code] = text + "）：组内IP的正常请求不记访问日志" + } + return refs +} diff --git a/service/waf_service/waf_sharedb_delete.go b/service/waf_service/waf_sharedb_delete.go new file mode 100644 index 00000000..6cd2d3d6 --- /dev/null +++ b/service/waf_service/waf_sharedb_delete.go @@ -0,0 +1,167 @@ +package waf_service + +import ( + "SamWaf/common/zlog" + "SamWaf/enums" + "SamWaf/global" + "SamWaf/model" + "SamWaf/utils" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "fmt" + "os" + "strings" +) + +// 分区的「还剩哪些层」与「主动删除」（E6）。 +// +// 两件事以前都没有： +// - 按层过期之后，一个分区可能只剩安全事件与报文（窄行先到期被丢了）， +// 但归档下拉里看不出这一点，选中它再切到访问日志视图就只会得到一句生硬的报错。 +// - 想立刻腾空间时只能去「文件管理」删 .db 文件——那条路**只对 SQLite 有效**， +// 而且删完 share_dbs 记录还在，下拉里仍然列着一个已经不存在的分区。 +// 这里按「分区」来删：文件型删文件、服务型丢表，两种部署都把元数据记录一并清掉。 + +// ShardTierBases 一个归档分区里可能存在的四层基表 +var ShardTierBases = []string{ + model.AccessLogTableName, + model.SecurityEventTableName, + model.EventPayloadTableName, + wafdb.LogTableName, +} + +// IsLiveShardName 实时库自己的标识，永远不能被当成归档删除 +func IsLiveShardName(name string) bool { + name = strings.TrimSpace(name) + return name == "" || name == AutoShard || name == enums.DB_LOG || name == "local_log.db" || + name == wafdb.LogTableName || name == model.AccessLogTableName +} + +// ArchiveSuffix 从归档标识里取出分片后缀（周期键或时间戳），认不出返回空。 +// 只认 web_logs_ / access_log_ 两种前缀——与读侧 ResolveTierTables 的口径一致。 +func ArchiveSuffix(name string) string { + for _, base := range []string{wafdb.LogTableName, model.AccessLogTableName} { + if strings.HasPrefix(name, base+"_") { + return strings.TrimPrefix(name, base+"_") + } + } + return "" +} + +// ShardTierInfo 一个分片 + 它现在还剩哪些层 +type ShardTierInfo struct { + model.ShareDb + Tiers []string `json:"tiers"` +} + +// GetAllShareDbWithTiers 列出分片，并（仅服务型数据库）标出每个分片还剩哪些层。 +// +// SQLite 不给层信息：它的分区是独立文件，要判断层就得把每个归档文件都打开一遍， +// 列表接口不该付这个代价。前端拿不到 tiers 时就不显示，不去猜。 +func (receiver *WafShareDbService) GetAllShareDbWithTiers() ([]ShardTierInfo, error) { + shards, err := receiver.GetAllShareDbApi() + if err != nil { + return nil, err + } + out := make([]ShardTierInfo, 0, len(shards)) + + var existing map[string]struct{} + if !dialect.Get().IsFileBased() && global.GWAF_LOCAL_LOG_DB != nil { + // 一次列表 + 内存里判在不在，比每张表各查一次 information_schema 便宜得多 + if tables, terr := dialect.Get().ListTables(global.GWAF_LOCAL_LOG_DB); terr == nil { + existing = make(map[string]struct{}, len(tables)) + for _, t := range tables { + existing[t] = struct{}{} + } + } + } + + for _, s := range shards { + info := ShardTierInfo{ShareDb: s} + if existing != nil && !IsLiveShardName(s.FileName) { + if suffix := ArchiveSuffix(s.FileName); suffix != "" { + for _, base := range ShardTierBases { + if _, ok := existing[base+"_"+suffix]; ok { + info.Tiers = append(info.Tiers, base) + } + } + } + } + out = append(out, info) + } + return out, nil +} + +// ForceDeleteShard 主动删除一个归档分区，**不看保留期**。 +// +// 只删 share_dbs 里登记过的归档分片：名字得对得上记录，才不会被当成「随便丢一张表」的入口。 +// 实时库一律拒绝。删完把分片记录与计数缓存一并清掉，否则下拉里会留下一个已经不存在的分区。 +func (receiver *WafShareDbService) ForceDeleteShard(name string) (string, error) { + name = strings.TrimSpace(name) + if IsLiveShardName(name) { + return "", fmt.Errorf("实时库不能删除") + } + + shards, err := receiver.GetAllShareDbApi() + if err != nil { + return "", err + } + var target *model.ShareDb + for i := range shards { + if shards[i].FileName == name { + target = &shards[i] + break + } + } + if target == nil { + return "", fmt.Errorf("没有找到分区 %s，请刷新后重试", name) + } + + detail := "" + if dialect.Get().IsFileBased() { + // 先关掉可能已按需打开的连接，避免删正在被查询的文件 + wafdb.CloseManualLogDb(name) + base := utils.GetCurrentDir() + "/data/" + name + removed := 0 + for _, p := range []string{base, base + "-wal", base + "-shm"} { + if rerr := os.Remove(p); rerr != nil && !os.IsNotExist(rerr) { + return "", fmt.Errorf("删除归档文件失败: %w", rerr) + } else if rerr == nil { + removed++ + } + } + detail = fmt.Sprintf("已删除归档文件 %s（含 %d 个关联文件）", name, removed) + } else { + suffix := ArchiveSuffix(name) + if suffix == "" { + return "", fmt.Errorf("认不出的归档标识：%s", name) + } + db := global.GWAF_LOCAL_LOG_DB + if db == nil { + return "", fmt.Errorf("日志库未就绪") + } + var dropped []string + for _, base := range ShardTierBases { + part := base + "_" + suffix + if !dialect.Get().TableExists(db, part) { + continue + } + // DropPartition 会校验「分区必须属于该基表」，实时表传不进来 + if derr := dialect.Get().DropPartition(db, base, part); derr != nil { + return "", fmt.Errorf("丢分区表 %s 失败: %w", part, derr) + } + dropped = append(dropped, part) + } + detail = fmt.Sprintf("已丢分区表 %s", strings.Join(dropped, ", ")) + if len(dropped) == 0 { + detail = "分区表已不存在，仅清理记录" + } + } + + if derr := receiver.DeleteById(target.Id); derr != nil { + return detail, fmt.Errorf("存储已删除，但清理分片记录失败: %w", derr) + } + InvalidateShardCounts() + zlog.Info("分区管理", "主动删除归档分区", name, "详情", detail) + return detail, nil +} diff --git a/wafenginecore/ipset/listtext.go b/wafenginecore/ipset/listtext.go new file mode 100644 index 00000000..33ec8671 --- /dev/null +++ b/wafenginecore/ipset/listtext.go @@ -0,0 +1,55 @@ +package ipset + +import "strings" + +// 「IP 清单文本」的统一解析。 +// +// 放在 ipset 这个叶子包的理由和 groups.go 一样：**判定侧与查询侧必须用同一套解析**。 +// 引擎按它决定「这个请求记不记日志」，IP归属查询按它回答「这个 IP 为什么没有日志」—— +// 两边各写一份，早晚会在某个边角（大小写、空格、逗号换行混用）分叉， +// 而那种分叉的表现是「界面说没排除，实际排除了」，最难查。 + +// listGroupPrefix 引用 IP 组的前缀 +const listGroupPrefix = "group:" + +// ParseListText 把一段清单文本拆成 IP 模式与组短码两组。 +// +// 行格式：单IP / CIDR / 通配符 / 区间（语法同 MatchSet），或 group:组短码； +// `#` 开头是注释，空行跳过，逗号与换行都算分隔。组短码去重且保持出现顺序。 +func ParseListText(raw string) (patterns []string, groupCodes []string) { + seen := map[string]struct{}{} + for _, line := range strings.FieldsFunc(raw, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' }) { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if len(line) >= len(listGroupPrefix) && strings.EqualFold(line[:len(listGroupPrefix)], listGroupPrefix) { + code := strings.TrimSpace(line[len(listGroupPrefix):]) + if code == "" { + continue + } + if _, dup := seen[code]; !dup { + seen[code] = struct{}{} + groupCodes = append(groupCodes, code) + } + continue + } + patterns = append(patterns, line) + } + return patterns, groupCodes +} + +// BuildFromListText 编译清单文本里的 IP 模式（不含组引用）；没有模式时返回 nil。 +func BuildFromListText(raw string) *MatchSet { + patterns, _ := ParseListText(raw) + if len(patterns) == 0 { + return nil + } + return BuildMatchSet(patterns) +} + +// GroupCodesOf 取出清单文本里引用的组短码（去重保序）。 +func GroupCodesOf(raw string) []string { + _, codes := ParseListText(raw) + return codes +} diff --git a/wafenginecore/weblog_exclude.go b/wafenginecore/weblog_exclude.go index 8ca3a1be..bd71d3d7 100644 --- a/wafenginecore/weblog_exclude.go +++ b/wafenginecore/weblog_exclude.go @@ -3,7 +3,6 @@ package wafenginecore import ( "SamWaf/model/wafenginmodel" "SamWaf/wafenginecore/ipset" - "strings" "sync/atomic" ) @@ -18,43 +17,22 @@ import ( // 一刀切静音意味着真被打了也看不见。判定挂在 shouldRecordWebLog（入队前）， // 被排除的请求同样不进三层落库 / stats_* / 分析层汇总。 -// parseIPLogExcludeLines 把清单文本拆成 IP 模式与组短码两组；组短码去重且保持出现顺序。 +// 清单文本的解析与编译统一放在 ipset（叶子包）里：引擎按它决定记不记日志， +// IP归属查询按它回答「这个 IP 为什么没有日志」，两边必须是同一套解析。 + +// parseIPLogExcludeLines 把清单文本拆成 IP 模式与组短码两组 func parseIPLogExcludeLines(raw string) (patterns []string, groupCodes []string) { - seen := map[string]struct{}{} - for _, line := range strings.FieldsFunc(raw, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' }) { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - if len(line) >= len("group:") && strings.EqualFold(line[:len("group:")], "group:") { - code := strings.TrimSpace(line[len("group:"):]) - if code == "" { - continue - } - if _, dup := seen[code]; !dup { - seen[code] = struct{}{} - groupCodes = append(groupCodes, code) - } - continue - } - patterns = append(patterns, line) - } - return patterns, groupCodes + return ipset.ParseListText(raw) } // BuildIPLogExcludeIndex 编译站点级清单里的 IP 模式；空清单返回 nil。 func BuildIPLogExcludeIndex(raw string) *ipset.MatchSet { - patterns, _ := parseIPLogExcludeLines(raw) - if len(patterns) == 0 { - return nil - } - return ipset.BuildMatchSet(patterns) + return ipset.BuildFromListText(raw) } // ExtractIPLogExcludeGroupCodes 抽出站点级清单里引用的组短码（去重保序）。 func ExtractIPLogExcludeGroupCodes(raw string) []string { - _, codes := parseIPLogExcludeLines(raw) - return codes + return ipset.GroupCodesOf(raw) } // ipLogExcludeCompiled 是全局清单的编译结果，配置热更新时整体替换。 From 8f7b951f2fdad590a66ca71035997cb84198db0f Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 28 Sep 2026 14:52:00 +0800 Subject: [PATCH 19/21] fix: read archived SQLite log shards read-only and choose tiers by data --- api/waf_log.go | 2 + cmd/samwaf/main.go | 2 + model/response/all_host.go | 1 + .../cross_engine_sqlite_archive_test.go | 176 ++++++++++ service/waf_service/cross_engine_test.go | 3 + service/waf_service/waf_log.go | 7 + service/waf_service/waf_log_query.go | 198 ++++++++--- service/waf_service/waf_sharedb_delete.go | 19 +- wafdb/localdb.go | 81 ++--- wafdb/log_shard.go | 330 +++++++++++------- wafdb/log_shard_cache.go | 62 +++- wafdb/log_shard_test.go | 267 ++++++++++++++ wafdb/mysql_localdb.go | 32 +- wafdb/postgres_localdb.go | 32 +- waftask/task_db_sharding.go | 42 ++- wafupgradenotice/upgrade_notes.yaml | 8 + 16 files changed, 963 insertions(+), 299 deletions(-) create mode 100644 service/waf_service/cross_engine_sqlite_archive_test.go create mode 100644 wafdb/log_shard_test.go diff --git a/api/waf_log.go b/api/waf_log.go index f6144fa4..ed4cf0fb 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -91,6 +91,7 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) { "sort_forced_time": meta.SortForcedTime, "partial": meta.Partial, "took_ms": meta.TookMs, + "issues": meta.Issues, }, "获取成功", c) } @@ -272,6 +273,7 @@ func (w *WafLogAPi) GetAllShareDbApi(c *gin.Context) { IsCurrent: wafShareList[i].FileName == liveName, PeriodKey: wafShareList[i].PeriodKey, Tiers: wafShareList[i].Tiers, + Missing: wafShareList[i].Missing, } } diff --git a/cmd/samwaf/main.go b/cmd/samwaf/main.go index c1ca8c41..b12f56e0 100644 --- a/cmd/samwaf/main.go +++ b/cmd/samwaf/main.go @@ -382,6 +382,8 @@ func (m *wafSystenService) run() { zlog.Error("初始化统计数据库失败，程序退出，请检查conf/config.yml数据库配置是否正确", "error", err) os.Exit(1) } + // 分层改造的边界切换要赶在接流量之前：条件是新表为空，一旦有请求写进来就不再成立 + waftask.CutTierBoundaryIfNeeded() // 全新安装引导：账户表为空(新用户判定)时创建默认管理员并生成随机初始口令。 // 放在启动初始化(核心库就绪后)执行，不再等首次登录才触发，便于新装即时拿到 data/initial_password.txt。 diff --git a/model/response/all_host.go b/model/response/all_host.go index 50f796cc..ffc7c269 100644 --- a/model/response/all_host.go +++ b/model/response/all_host.go @@ -22,6 +22,7 @@ type AllShareDbRep struct { IsCurrent bool `json:"is_current"` //是否为当前(实时)分片：前端据此设默认选中项 PeriodKey string `json:"period_key"` //周期键（月，如 202609）；按体积切出来的旧分片为空，前端回落显示起止日期 Tiers []string `json:"tiers"` //该分区现存哪些层（仅服务型数据库给；按层过期后可能只剩安全事件与报文） + Missing bool `json:"missing"` //登记还在但存储已不在（SQLite 文件被删 / 分区表都不存在） } // AllDomainRep 域名信息 diff --git a/service/waf_service/cross_engine_sqlite_archive_test.go b/service/waf_service/cross_engine_sqlite_archive_test.go new file mode 100644 index 00000000..6ee0a460 --- /dev/null +++ b/service/waf_service/cross_engine_sqlite_archive_test.go @@ -0,0 +1,176 @@ +//go:build crossdb + +// SQLite 历史分区：只读打开、按数据选层、缺失标注。 +// +// 分层改造边界切出来的分片（以及被旧版打开过、跑过迁移的归档）同时有 web_logs 数据与空的新表， +// 按「表在不在」选层会读到空表——下拉里标着有数据、选中却一条都查不到。 +// 分片登记还在而文件已不在时，要标出来，不能在原位置建出空库，也不能回落到实时库。 +package waf_service + +import ( + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/model/request" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "SamWaf/wafdb/partition" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + sqlitedriver "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" +) + +func runSQLiteArchiveCases(t *testing.T, core, logdb *gorm.DB) { + if !dialect.Get().IsFileBased() { + t.Skip("只针对文件型分区") + } + t.Setenv("SamWafIDE", "1") // 让 utils.GetCurrentDir() 返回 "."，归档文件落到临时目录的 data/ 下 + t.Chdir(t.TempDir()) + must(t, os.MkdirAll("data", 0o755)) + + now := time.Now() + midAt := now.AddDate(0, -1, 0) + oldAt := now.AddDate(0, -2, 0) + keyMid, keyOld := partition.KeyOf(midAt), partition.KeyOf(oldAt) + if keyMid == keyOld { + t.Skip("跨月边界导致周期键相同，跳过") + } + tag := sfx() + hostCode := "arch_" + tag + polluted := "local_log_" + keyMid + ".db" + missing := "local_log_" + keyOld + ".db" + + // 被污染的归档：web_logs 3 行（其中 1 行是拦截），新三表存在但为空 + path := filepath.Join("data", polluted) + adb, err := gorm.Open(sqlitedriver.Open(path+"?_db_key="+url.QueryEscape(global.GWAF_PWD_LOGDB)), silentCfg) + fatalIf(t, err) + fatalIf(t, adb.AutoMigrate(&innerbean.WebLog{}, &model.AccessLog{}, &model.SecurityEvent{}, &model.EventPayload{})) + for i := 0; i < 3; i++ { + at := midAt.Add(-time.Duration(i+1) * time.Minute) + action, rule := "放行", "" + if i == 0 { + action, rule = "阻止", "sqli" + } + fatalIf(t, adb.Create(&innerbean.WebLog{ + REQ_UUID: fmt.Sprintf("arch_%s_%d", tag, i), TenantId: xtestTenant, USER_CODE: xtestUser, + HOST_CODE: hostCode, ACTION: action, RULE: rule, SRC_IP: "203.0.113.9", + UNIX_ADD_TIME: at.UnixMilli(), CREATE_TIME: at.Format("2006-01-02 15:04:05"), + Day: at.Year()*10000 + int(at.Month())*100 + at.Day(), + }).Error) + } + if s, e := adb.DB(); e == nil { + _ = s.Close() + } + before, err := os.Stat(path) + fatalIf(t, err) + + register := func(name string, at time.Time, cnt int64) { + must(t, core.Create(&model.ShareDb{ + BaseOrm: baseorm.BaseOrm{ + Id: "arch_" + tag + "_" + name, USER_CODE: xtestUser, Tenant_ID: xtestTenant, + CREATE_TIME: customtype.JsonTime(now), UPDATE_TIME: customtype.JsonTime(now), + }, + DbLogicType: "log", + StartTime: customtype.JsonTime(at.AddDate(0, 0, -1)), + EndTime: customtype.JsonTime(at.AddDate(0, 0, 1)), + FileName: name, + PeriodKey: partition.KeyOf(at), + Cnt: cnt, + }).Error) + } + register(polluted, midAt, 3) + register(missing, oldAt, 5) + InvalidateShardCounts() + defer func() { + wafdb.CloseManualLogDb(polluted) + core.Where("id like ?", "arch_"+tag+"%").Delete(&model.ShareDb{}) + InvalidateShardCounts() + }() + + req := func(shard, view string) request.WafAttackLogSearch { + r := request.WafAttackLogSearch{} + r.PageIndex, r.PageSize = 1, 10 + r.SortBy, r.SortDescending = "unix_add_time", "desc" + r.HostCode = hostCode + r.CurrrentDbName = shard + r.ViewType = view + r.UnixAddTimeBegin = fmt.Sprint(now.AddDate(0, -3, 0).UnixMilli()) + r.UnixAddTimeEnd = fmt.Sprint(now.Add(time.Hour).UnixMilli()) + return r + } + + t.Run("被污染的归档按 web_logs 读", func(t *testing.T) { + rows, total, _, err := WafLogServiceApp.GetListApiWithMeta(req(polluted, "access")) + fatalIf(t, err) + if total != 3 || len(rows) != 3 { + t.Fatalf("访问日志视图应读到 3 行，实际 total=%d rows=%d", total, len(rows)) + } + _, total, _, err = WafLogServiceApp.GetListApiWithMeta(req(polluted, "event")) + fatalIf(t, err) + if total != 1 { + t.Fatalf("安全事件视图应读到 1 行拦截，实际 %d", total) + } + }) + + t.Run("自动扇出跳过缺失分区并标出", func(t *testing.T) { + rows, total, meta, err := WafLogServiceApp.GetListApiWithMeta(req(AutoShard, "access")) + fatalIf(t, err) + if total != 3 || len(rows) != 3 { + t.Fatalf("应只读到被污染归档里的 3 行，实际 total=%d rows=%d", total, len(rows)) + } + var got *LogShardIssue + for i := range meta.Issues { + if meta.Issues[i].Name == missing { + got = &meta.Issues[i] + } + } + if got == nil || got.Kind != ShardIssueMissing || got.Registered != 5 { + t.Fatalf("缺失分区应以 missing 标出且带登记条数 5，实际 %+v", meta.Issues) + } + if _, err := os.Stat(filepath.Join("data", missing)); !os.IsNotExist(err) { + t.Fatal("查询缺失分区时在原位置建出了文件") + } + }) + + t.Run("明确选中缺失分区给出说明", func(t *testing.T) { + _, _, meta, err := WafLogServiceApp.GetListApiWithMeta(req(missing, "access")) + if err == nil || !strings.Contains(err.Error(), "已不存在") { + t.Fatalf("应报存储已不存在，实际 %v", err) + } + if len(meta.Issues) != 1 || meta.Issues[0].Kind != ShardIssueMissing { + t.Fatalf("应带一条 missing 问题，实际 %+v", meta.Issues) + } + }) + + t.Run("列表标出缺失", func(t *testing.T) { + list, err := WafShareDbServiceApp.GetAllShareDbWithTiers() + fatalIf(t, err) + seen := map[string]bool{} + for _, s := range list { + if s.FileName == polluted || s.FileName == missing { + seen[s.FileName] = s.Missing + } + } + if len(seen) != 2 || seen[polluted] || !seen[missing] { + t.Fatalf("应只有缺失的那一个标 missing，实际 %+v", seen) + } + }) + + t.Run("归档文件未被改动", func(t *testing.T) { + wafdb.CloseManualLogDb(polluted) + after, err := os.Stat(path) + fatalIf(t, err) + if after.Size() != before.Size() || !after.ModTime().Equal(before.ModTime()) { + t.Fatalf("归档文件被改动：%d/%v → %d/%v", before.Size(), before.ModTime(), after.Size(), after.ModTime()) + } + }) +} diff --git a/service/waf_service/cross_engine_test.go b/service/waf_service/cross_engine_test.go index 31d3038b..3a4e476c 100644 --- a/service/waf_service/cross_engine_test.go +++ b/service/waf_service/cross_engine_test.go @@ -297,6 +297,9 @@ func TestCrossEngine(t *testing.T) { // —— 日志分区扇出与识别码直查（见 cross_engine_logfanout_test.go）—— t.Run("logfanout", func(t *testing.T) { runLogFanoutCases(t, x.core, x.logdb) }) + // —— SQLite 历史分区只读与缺失标注（见 cross_engine_sqlite_archive_test.go）—— + t.Run("sqlitearchive", func(t *testing.T) { runSQLiteArchiveCases(t, x.core, x.logdb) }) + // —— ip_tags 跨库合并（见 cross_engine_payload_test.go）—— t.Run("iptagmerge", func(t *testing.T) { runIPTagMergeCases(t, x) }) diff --git a/service/waf_service/waf_log.go b/service/waf_service/waf_log.go index a41393b3..84011f71 100644 --- a/service/waf_service/waf_log.go +++ b/service/waf_service/waf_log.go @@ -8,6 +8,7 @@ import ( "SamWaf/model/request" "SamWaf/wafdb" "SamWaf/wafdb/dialect" + "errors" "fmt" "strings" "sync" @@ -128,6 +129,12 @@ func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) ( // 用户手里只有一串识别码，不知道那次访问落在哪个分区 shardName := ResolveDetailShard(req.CurrrentDbName, req.REQ_UUID) tier := wafdb.ResolveTierTables(shardName) + if tier.Err != nil { + if errors.Is(tier.Err, wafdb.ErrShardMissing) { + return weblog, fmt.Errorf("该分区的存储已不存在，可在「分区管理」里删除这条登记") + } + return weblog, fmt.Errorf("打开日志分区失败: %w", tier.Err) + } found := false for _, table := range []string{tier.Event, tier.Access, tier.WebLog} { if table == "" { diff --git a/service/waf_service/waf_log_query.go b/service/waf_service/waf_log_query.go index d464aaa5..1f006be8 100644 --- a/service/waf_service/waf_log_query.go +++ b/service/waf_service/waf_log_query.go @@ -58,13 +58,55 @@ type LogShardHit struct { // LogQueryMeta 告诉前端这次查询到底查了哪里 type LogQueryMeta struct { - Shards []LogShardHit `json:"shards"` // 本次覆盖的分区（按时间从新到旧） - FoundIn string `json:"found_in"` // 识别码直查命中的分区 - Scanned int `json:"scanned"` // 识别码直查翻了几个分区 - UuidLookup bool `json:"uuid_lookup"` // 是否走了识别码直查 - SortForcedTime bool `json:"sort_forced_time"` // 跨分区时排序被强制成时间 - Partial bool `json:"partial"` // 查询超出时间预算，结果与总数都只是一部分 - TookMs int64 `json:"took_ms"` // 本次耗时，界面用来解释「为什么只给了一部分」 + Shards []LogShardHit `json:"shards"` // 本次覆盖的分区（按时间从新到旧） + FoundIn string `json:"found_in"` // 识别码直查命中的分区 + Scanned int `json:"scanned"` // 识别码直查翻了几个分区 + UuidLookup bool `json:"uuid_lookup"` // 是否走了识别码直查 + SortForcedTime bool `json:"sort_forced_time"` // 跨分区时排序被强制成时间 + Partial bool `json:"partial"` // 查询超出时间预算，结果与总数都只是一部分 + TookMs int64 `json:"took_ms"` // 本次耗时，界面用来解释「为什么只给了一部分」 + Issues []LogShardIssue `json:"issues"` // 本次碰到的存储缺失 / 空壳分区 +} + +// 分区问题的两种类型 +const ( + ShardIssueMissing = "missing" // 登记还在，文件（或分区表）已不在 + ShardIssueEmpty = "empty" // 文件在，但一行数据都没有，而登记有条数 +) + +// LogShardIssue 一个有问题的分区，界面据此提示用户去「分区管理」处理 +type LogShardIssue struct { + Name string `json:"name"` + Kind string `json:"kind"` + Registered int64 `json:"registered"` // share_dbs 登记的条数 +} + +// errShardNoTable 分区里没有任何可查询的日志表 +var errShardNoTable = errors.New("该分片没有可查询的日志表") + +// shardIssueOf 查询某个分区出错或读到空壳时，归成一条问题；不属于这两类返回 nil。 +func shardIssueOf(name string, registered int64, err error, tier wafdb.TierTables) *LogShardIssue { + if err != nil && errors.Is(err, wafdb.ErrShardMissing) { + return &LogShardIssue{Name: name, Kind: ShardIssueMissing, Registered: registered} + } + if registered > 0 && (errors.Is(err, errShardNoTable) || (err == nil && tier.Empty)) { + return &LogShardIssue{Name: name, Kind: ShardIssueEmpty, Registered: registered} + } + return nil +} + +// registeredShardCount share_dbs 里某个分区登记的条数，找不到返回 0 +func registeredShardCount(name string) int64 { + all, err := WafShareDbServiceApp.GetAllShareDbApi() + if err != nil { + return 0 + } + for _, s := range all { + if s.FileName == name { + return s.Cnt + } + } + return 0 } // shardCountCache 归档分区的计数缓存。 @@ -89,6 +131,11 @@ func InvalidateShardCounts() { return true }) shardCountCacheN.Store(0) + webLogShardSelect.Range(func(k, _ any) bool { + webLogShardSelect.Delete(k) + return true + }) + wafdb.InvalidateShardTierCache() } // countCacheKey 非时间条件的签名 + 分区名。时间条件不进键——只有完整覆盖时才会用到本缓存。 @@ -120,6 +167,9 @@ func buildLogQuery(req request.WafAttackLogSearch, shardName string, ignoreTime // 解析当前分片的三层表：访问日志视图读 access_log，安全事件视图读 security_event； // 分层改造之前切出去的归档只有 web_logs，回落到它（列交集会自适应它的结构）。 tier := wafdb.ResolveTierTables(shardName) + if tier.Err != nil { + return nil, tier.Err + } logDB := tier.DB isEventView := req.ViewType == "event" logTable := tier.Access @@ -130,7 +180,7 @@ func buildLogQuery(req request.WafAttackLogSearch, shardName string, ignoreTime logTable = tier.WebLog } if logTable == "" { - return nil, errors.New("该分片没有可查询的日志表") + return nil, errShardNoTable } // 老分片上的安全事件视图：web_logs 里按事件条件过滤（与引擎 abnormal 判定同一条规则） legacyEventView := isEventView && logTable == tier.WebLog @@ -237,10 +287,15 @@ func buildLogQuery(req request.WafAttackLogSearch, shardName string, ignoreTime } else if strings.HasPrefix(logTable, model.SecurityEventTableName) { idxTime, idxIP = "idx_se_time", "idx_se_ip_time" } + // 历史分区不跑迁移，早年切出来的可能没有这些索引：有才强制，没有交给优化器 if strings.Contains(whereField, "unix_add_time") && !strings.Contains(whereField, "src_ip") { - forceIndex = dialect.Get().ForceIndexClause(logTable, idxTime) + if wafdb.ShardHasIndex(logDB, shardName, logTable, idxTime) { + forceIndex = dialect.Get().ForceIndexClause(logTable, idxTime) + } } else if strings.Contains(whereField, "src_ip") { - forceIndex = dialect.Get().ForceIndexClause(logTable, idxIP) + if wafdb.ShardHasIndex(logDB, shardName, logTable, idxIP) { + forceIndex = dialect.Get().ForceIndexClause(logTable, idxIP) + } } } @@ -367,10 +422,11 @@ func (q *logQuery) find(ctx context.Context, offset, limit int, orderOverride st // logShard 一个候选分区：标识 + 它装着的时间范围 type logShard struct { - Name string - Start time.Time - End time.Time - Live bool + Name string + Start time.Time + End time.Time + Live bool + Registered int64 // share_dbs 登记的条数 } // candidateShards 挑出与 [fromMs, toMs] 有交集的分区，按时间**从新到旧**排列。 @@ -399,7 +455,7 @@ func candidateShards(fromMs, toMs int64, ignoreTime bool) []logShard { if en.After(live.Start) { live.Start = en } - shards = append(shards, logShard{Name: s.FileName, Start: st, End: en}) + shards = append(shards, logShard{Name: s.FileName, Start: st, End: en, Registered: s.Cnt}) } shards = append(shards, live) sort.Slice(shards, func(i, j int) bool { return shards[i].End.After(shards[j].End) }) @@ -446,16 +502,37 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch if !auto { q, err := buildLogQuery(req, shardName, false) + var tier wafdb.TierTables + if q != nil { + tier = q.tier + } + if issue := shardIssueOf(shardName, registeredShardCount(shardName), err, tier); issue != nil { + meta.Issues = append(meta.Issues, *issue) + if issue.Kind == ShardIssueMissing { + return nil, 0, meta, fmt.Errorf("该分区的存储已不存在（登记 %d 条），可在「分区管理」里删除这条登记", issue.Registered) + } + if err != nil { + return []innerbean.WebLog{}, 0, meta, nil + } + } if err != nil { return nil, 0, meta, err } - total, err := q.count(ctx) + // 先取数再统计：取一页走时间索引、凑够一页就停；统计要数完整个分区。 + // 分层改造之前的大分片上，安全事件视图得逐行看 action/rule，统计可能超出时间预算—— + // 这时仍把这一页给出去并标 partial，而不是整页报错。 + offset := req.PageSize * (req.PageIndex - 1) + rows, err := q.find(ctx, offset, req.PageSize, "") if err != nil { return nil, 0, meta, err } - rows, err := q.find(ctx, req.PageSize*(req.PageIndex-1), req.PageSize, "") + total, err := q.count(ctx) if err != nil { - return nil, 0, meta, err + if ctx.Err() == nil { + return nil, 0, meta, err + } + meta.Partial = true + total = int64(offset + len(rows)) } meta.Shards = []LogShardHit{{Name: shardName, Count: total}} return rows, total, meta, nil @@ -482,34 +559,27 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch } } - // 先把各分区的查询组装出来（只是探表拼条件，很便宜），统计放到后面并发做。 - // 组装不并发：SQLite 下解析分区会按需打开归档文件，串行更稳。 + // 分区用到才打开：统计命中缓存的不打开，统计与取数各自在时间预算内解析分区。 + // 先把全部候选一次性打开，候选一多就会超出分片连接缓存的上限，也不受时间预算约束。 type plan struct { q *logQuery + err error + built bool sh logShard cnt int64 name string cached bool } - plans := make([]plan, 0, len(shards)) - var firstErr error - for _, sh := range shards { - q, err := buildLogQuery(req, sh.Name, false) - if err != nil { - // 某个分区没有可查的表（改造前的老分片）不该让整次查询失败，跳过即可 - if firstErr == nil { - firstErr = err - } - zlog.Debug("日志分区扇出", "跳过分区", sh.Name, "原因", err.Error()) - continue - } - plans = append(plans, plan{q: q, sh: sh, name: sh.Name}) + plans := make([]plan, len(shards)) + for i, sh := range shards { + plans[i] = plan{sh: sh, name: sh.Name} } - if len(plans) == 0 { - if firstErr != nil { - return nil, 0, meta, firstErr + build := func(p *plan) error { + if !p.built { + p.q, p.err = buildLogQuery(req, p.name, false) + p.built = true } - return []innerbean.WebLog{}, 0, meta, nil + return p.err } // 时间范围完整覆盖的归档分区：计数与具体区间无关，可以复用上次算过的值。 @@ -539,6 +609,14 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch go func() { defer wg.Done() defer func() { <-sem }() + if ctx.Err() != nil { + plans[i].cnt = -1 + return + } + if err := build(&plans[i]); err != nil { + plans[i].cnt = 0 + return + } cnt, err := plans[i].q.count(ctx) if err != nil { zlog.Warn("日志分区扇出", "统计分区失败", plans[i].name, "error", err.Error()) @@ -557,13 +635,36 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch wg.Wait() var total int64 + var firstErr error + usable := 0 for i := range plans { - if plans[i].cnt < 0 { + p := &plans[i] + var tier wafdb.TierTables + if p.q != nil { + tier = p.q.tier + } + if issue := shardIssueOf(p.name, p.sh.Registered, p.err, tier); issue != nil { + meta.Issues = append(meta.Issues, *issue) + } + if p.err != nil { + if !errors.Is(p.err, wafdb.ErrShardMissing) && !errors.Is(p.err, errShardNoTable) { + zlog.Warn("日志分区扇出", "跳过分区", p.name, "原因", p.err.Error()) + if firstErr == nil { + firstErr = p.err + } + } + continue + } + usable++ + if p.cnt < 0 { meta.Partial = true // 有分区没统计上，总数只是一部分 - plans[i].cnt = 0 + p.cnt = 0 continue } - total += plans[i].cnt + total += p.cnt + } + if usable == 0 && firstErr != nil { + return nil, 0, meta, firstErr } if ctx.Err() != nil { meta.Partial = true @@ -572,11 +673,12 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch offset := int64(req.PageSize * (req.PageIndex - 1)) remaining := req.PageSize rows := make([]innerbean.WebLog, 0, req.PageSize) - for _, p := range plans { + for i := range plans { + p := &plans[i] if p.cnt > 0 { meta.Shards = append(meta.Shards, LogShardHit{Name: p.name, Count: p.cnt}) } - if remaining <= 0 || p.cnt == 0 { + if remaining <= 0 || p.cnt <= 0 { continue } if offset >= p.cnt { @@ -589,6 +691,14 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch meta.Partial = true break } + // 计数来自缓存的分区到这里才打开 + if err := build(p); err != nil { + if issue := shardIssueOf(p.name, p.sh.Registered, err, wafdb.TierTables{}); issue != nil { + meta.Issues = append(meta.Issues, *issue) + } + meta.Partial = true + continue + } part, err := p.q.find(ctx, int(offset), remaining, order) if err != nil { if ctx.Err() != nil { @@ -630,6 +740,10 @@ func (receiver *WafLogService) lookupByUuid(req request.WafAttackLogSearch) ([]i meta.Scanned = i + 1 q, err := buildLogQuery(req, sh.Name, true) if err != nil { + if issue := shardIssueOf(sh.Name, sh.Registered, err, wafdb.TierTables{}); issue != nil { + meta.Issues = append(meta.Issues, *issue) + continue + } if firstErr == nil { firstErr = err } diff --git a/service/waf_service/waf_sharedb_delete.go b/service/waf_service/waf_sharedb_delete.go index 6cd2d3d6..887d39d2 100644 --- a/service/waf_service/waf_sharedb_delete.go +++ b/service/waf_service/waf_sharedb_delete.go @@ -51,7 +51,8 @@ func ArchiveSuffix(name string) string { // ShardTierInfo 一个分片 + 它现在还剩哪些层 type ShardTierInfo struct { model.ShareDb - Tiers []string `json:"tiers"` + Tiers []string `json:"tiers"` + Missing bool `json:"missing"` // 登记还在，存储已不在 } // GetAllShareDbWithTiers 列出分片，并（仅服务型数据库）标出每个分片还剩哪些层。 @@ -78,13 +79,19 @@ func (receiver *WafShareDbService) GetAllShareDbWithTiers() ([]ShardTierInfo, er for _, s := range shards { info := ShardTierInfo{ShareDb: s} - if existing != nil && !IsLiveShardName(s.FileName) { - if suffix := ArchiveSuffix(s.FileName); suffix != "" { - for _, base := range ShardTierBases { - if _, ok := existing[base+"_"+suffix]; ok { - info.Tiers = append(info.Tiers, base) + if !IsLiveShardName(s.FileName) { + if dialect.Get().IsFileBased() { + // 只看文件在不在，不打开：分片文件多时列表照样秒回 + info.Missing = wafdb.ShardFileMissing(s.FileName) + } else if existing != nil { + if suffix := ArchiveSuffix(s.FileName); suffix != "" { + for _, base := range ShardTierBases { + if _, ok := existing[base+"_"+suffix]; ok { + info.Tiers = append(info.Tiers, base) + } } } + info.Missing = len(info.Tiers) == 0 } } out = append(out, info) diff --git a/wafdb/localdb.go b/wafdb/localdb.go index 2110d3db..7805a43d 100644 --- a/wafdb/localdb.go +++ b/wafdb/localdb.go @@ -1,13 +1,9 @@ package wafdb import ( - "SamWaf/common/uuid" "SamWaf/common/zlog" - "SamWaf/customtype" "SamWaf/global" - "SamWaf/innerbean" "SamWaf/model" - "SamWaf/model/baseorm" "SamWaf/utils" "bufio" "context" @@ -148,6 +144,13 @@ func InitCoreDb(currentDir string) (bool, error) { } } +// 归档分片只读连接的页缓存（负数为 KB）与连接数上限。实时库用 64MB 页缓存， +// 归档若也按 64MB，同时打开 8 个分片扫一遍就要五六百 MB；只读查询 8MB 足够。 +const ( + archiveCacheSizeKB = -8192 + archiveMaxOpenConns = 2 +) + func InitLogDb(currentDir string) (bool, error) { switch dialect.Get().Name() { case "mysql": @@ -204,29 +207,7 @@ func InitLogDb(currentDir string) (bool, error) { global.GWAF_LOCAL_LOG_DB.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update) pathLogSql(db) - var total int64 = 0 - global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Count(&total) - if total == 0 { - - var logtotal int64 = 0 - global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&logtotal) - - sharDbBean := model.ShareDb{ - BaseOrm: baseorm.BaseOrm{ - Id: uuid.GenUUID(), - USER_CODE: global.GWAF_USER_CODE, - Tenant_ID: global.GWAF_TENANT_ID, - CREATE_TIME: customtype.JsonTime(time.Now()), - UPDATE_TIME: customtype.JsonTime(time.Now()), - }, - DbLogicType: "log", - StartTime: customtype.JsonTime(time.Now()), - EndTime: customtype.JsonTime(time.Now()), - FileName: "local_log.db", - Cnt: logtotal, - } - global.GWAF_LOCAL_DB.Create(sharDbBean) - } + ensureLiveShardRecord(global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB, LiveLogName()) return isNewDb, nil } else { @@ -234,48 +215,48 @@ func InitLogDb(currentDir string) (bool, error) { } } -// InitManaulLogDb 按需打开一个归档日志分片，已打开的直接复用。 -// 连接统一由 log_shard_cache 托管（加锁、数量上限、空闲释放）。 -// 打开或迁移失败返回 error 交调用方降级：一个坏掉的归档文件不该让整个进程退出。 +// InitManaulLogDb 按需只读打开一个 SQLite 归档日志分片，已打开的直接复用。 +// +// 归档文件只读：不跑迁移、连接层 query_only，打开前先确认文件在——不在就返回 ErrShardMissing， +// 不会在原位置建出一个空库。读侧按分片实际有的表与列取数（ResolveTierTables / webLogSelect）， +// 不需要归档跟着实时库补表补列。页缓存与连接数按只读场景收小，多个分片同时打开时内存可控。 func InitManaulLogDb(currentDir string, custFileName string) error { if dialect.Get().Name() != "sqlite" { // MySQL 模式下所有日志写入同一个库，无需手动切换分库 return nil } if db := getShardDB(custFileName); db != nil { - zlog.Debug("自定义的库已存在", custFileName) return nil } - if currentDir == "" { - currentDir = utils.GetCurrentDir() + path, err := ShardFilePath(custFileName) + if err != nil { + return err + } + if currentDir != "" { + path = currentDir + "/data/" + custFileName + } + if _, serr := os.Stat(path); serr != nil { + if os.IsNotExist(serr) { + return fmt.Errorf("%w: %s", ErrShardMissing, custFileName) + } + return fmt.Errorf("读取归档分片 %s 失败: %w", custFileName, serr) } - zlog.Debug("初始化自定义的库", custFileName) - path := currentDir + "/data/" + custFileName key := url.QueryEscape(global.GWAF_PWD_LOGDB) - dns := fmt.Sprintf("%s?_db_key=%s", path, key) + dns := fmt.Sprintf("%s?_db_key=%s&_query_only=1&_cache_size=%d&_busy_timeout=5000", path, key, archiveCacheSizeKB) db, err := gorm.Open(sqlite.Open(dns), &gorm.Config{}) if err != nil { return fmt.Errorf("打开归档分片 %s 失败: %w", custFileName, err) } - // 日志/统计库使用 synchronous=NORMAL 提升高频写入吞吐，其余性能 pragma 统一设置 - applyPerfPragmas(db, true) - // 创建自定义日志记录器 - gormLogger := NewGormZLogger() + if sqlDB, derr := db.DB(); derr == nil { + sqlDB.SetMaxOpenConns(archiveMaxOpenConns) + sqlDB.SetMaxIdleConns(archiveMaxOpenConns) + } if global.GWAF_LOG_DEBUG_DB_ENABLE == true { - gormLogger = gormLogger.LogMode(logger.Info).(*GormZLogger) - // 启用调试模式 db = db.Session(&gorm.Session{ - Logger: logger.Default.LogMode(logger.Info), // 设置为Info表示启用调试模式 + Logger: logger.Default.LogMode(logger.Info), }) } - zlog.Info("开始执行手动log数据库迁移...", "file", custFileName) - if err := RunLogDBMigrations(db); err != nil { - zlog.Error("手动log数据库迁移失败", "file", custFileName, "error", fmt.Sprintf("%v", err)) - closeShardConn(custFileName, db) - return fmt.Errorf("归档分片 %s 迁移失败: %w", custFileName, err) - } - db.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query) db.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update) diff --git a/wafdb/log_shard.go b/wafdb/log_shard.go index 4f28721e..dc1140d1 100644 --- a/wafdb/log_shard.go +++ b/wafdb/log_shard.go @@ -1,13 +1,23 @@ package wafdb import ( + "SamWaf/common/uuid" "SamWaf/common/zlog" + "SamWaf/customtype" "SamWaf/enums" "SamWaf/global" + "SamWaf/innerbean" "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/utils" "SamWaf/wafdb/dialect" + "errors" + "fmt" + "os" + "regexp" "strings" "sync" + "time" "gorm.io/gorm" ) @@ -25,164 +35,248 @@ func LiveLogName() string { return LogTableName // "web_logs" } -// ResolveLogDB returns the *gorm.DB connection and table name to query for the -// given log shard identifier (ShareDb.FileName, passed from the front-end as -// current_db_name). +// ensureLiveShardRecord 确保 share_dbs 里有一条实时分片记录（file_name = liveName），没有才补。 +// 按这一行在不在判断，不看表里总数：已有历史分片时总数不为 0，实时记录缺了也永远补不回来。 +func ensureLiveShardRecord(coreDB, logDB *gorm.DB, liveName string) { + var liveCount int64 + if err := coreDB.Model(&model.ShareDb{}).Where("file_name = ?", liveName).Count(&liveCount).Error; err != nil { + zlog.Error("查询实时分片记录失败", "file_name", liveName, "error", err.Error()) + return + } + if liveCount > 0 { + return + } + var logTotal int64 + logDB.Model(&innerbean.WebLog{}).Count(&logTotal) + now := customtype.JsonTime(time.Now()) + coreDB.Create(&model.ShareDb{ + BaseOrm: baseorm.BaseOrm{ + Id: uuid.GenUUID(), + USER_CODE: global.GWAF_USER_CODE, + Tenant_ID: global.GWAF_TENANT_ID, + CREATE_TIME: now, + UPDATE_TIME: now, + }, + DbLogicType: "log", + StartTime: now, + EndTime: now, + FileName: liveName, + Cnt: logTotal, + }) +} + +// ErrShardMissing 归档分区的存储已不在：SQLite 文件不存在，或 MySQL/PG 下该分区的表一张都没有。 +var ErrShardMissing = errors.New("归档分区的存储已不存在") + +// archiveFileRe SQLite 归档文件名：local_log_<周期或时间戳>[_序号].db，只允许纯文件名 +var archiveFileRe = regexp.MustCompile(`^local_log_[0-9]+(_[0-9]+)?\.db$`) + +// isLiveIdent 实时库的几种标识：空值、SQLite 库名、两张实时表名 +func isLiveIdent(name string) bool { + return name == "" || name == enums.DB_LOG || name == LogTableName || name == model.AccessLogTableName +} + +// ShardFilePath SQLite 归档文件的完整路径；名字不是合法的归档文件名时返回错误。 +func ShardFilePath(name string) (string, error) { + if !archiveFileRe.MatchString(name) { + return "", fmt.Errorf("不是合法的归档文件名: %q", name) + } + return utils.GetCurrentDir() + "/data/" + name, nil +} + +// ShardFileMissing SQLite 归档文件是否已不在（只看文件，不打开）。名字不合法也按缺失算。 +func ShardFileMissing(name string) bool { + path, err := ShardFilePath(name) + if err != nil { + return true + } + _, err = os.Stat(path) + return os.IsNotExist(err) +} + +// ResolveLogDB 返回某个分片标识对应的连接与表名。 // -// - empty / live identifier → live log DB + "web_logs" -// - SQLite historical shard → on-demand opened shard .db file + "web_logs" -// - MySQL historical shard → same log DB connection + shard table name +// - 空值 / 实时标识 → 实时库 + web_logs +// - SQLite 归档文件 → 按需只读打开该文件 + web_logs +// - MySQL/PG 归档表 → 实时库连接 + 该表名 // -// It never returns a nil *gorm.DB: if a SQLite shard cannot be opened it falls -// back to the live connection, guarding against the nil-map dereference panic -// that the previous inline read paths were exposed to under MySQL. -func ResolveLogDB(currentDbName string) (*gorm.DB, string) { - // Treat as "live" (current log store): the empty value, the legacy default - // "local_log.db" (still sent by the front-end as its default selection under - // any driver), and the MySQL live table name "web_logs". - if len(currentDbName) == 0 || currentDbName == enums.DB_LOG || currentDbName == LogTableName { - return global.GWAF_LOCAL_LOG_DB, LogTableName - } - - // Historical shard. +// 打不开（文件不在、表不在、名字不合法）时返回错误，不回落实时库： +// 回落会把实时数据当成这个分区的内容显示出来。 +func ResolveLogDB(currentDbName string) (*gorm.DB, string, error) { + if isLiveIdent(currentDbName) { + return global.GWAF_LOCAL_LOG_DB, LogTableName, nil + } if dialect.Get().IsFileBased() { - // SQLite: open the archived .db file on demand and query its web_logs table. if err := InitManaulLogDb("", currentDbName); err != nil { - zlog.Warn("归档分片不可用，降级查实时库", "file", currentDbName, "error", err.Error()) - return global.GWAF_LOCAL_LOG_DB, LogTableName + return nil, "", err } if db := getShardDB(currentDbName); db != nil { - return db, LogTableName + return db, LogTableName, nil } - // Shard file unavailable — degrade to live DB instead of panicking. - return global.GWAF_LOCAL_LOG_DB, LogTableName + return nil, "", fmt.Errorf("归档分片 %s 连接不可用", currentDbName) } - - // MySQL: the archived shard is a table (web_logs_) in the same database. - // Guard against a non-existent table name (e.g. stale share_dbs rows that - // stored the database name instead of a table name) by falling back to live. if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, currentDbName) { - return global.GWAF_LOCAL_LOG_DB, currentDbName - } - return global.GWAF_LOCAL_LOG_DB, LogTableName -} - -// shardTableSeen 记住哪些分片确认有某张表。只缓存"有"这一侧： -// 归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到，缓存下来会一直读不到。 -var shardTableSeen sync.Map - -// probeShardTable 探一次表存不存在，存在则返回表名。 -// 只缓存"存在"这一侧：归档分片一旦有就永远有，而"没有"可能只是升级迁移还没跑到， -// 缓存下来会一直读不到报文。 -func probeShardTable(db *gorm.DB, shard, table string) string { - key := shard + "|" + table - if _, ok := shardTableSeen.Load(key); ok { - return table - } - if !dialect.Get().TableExists(db, table) { - return "" + return global.GWAF_LOCAL_LOG_DB, currentDbName, nil } - shardTableSeen.Store(key, struct{}{}) - return table + return nil, "", fmt.Errorf("%w: %s", ErrShardMissing, currentDbName) } -// TierTables 一个分片上三层存储各自的表名；空字符串 = 这个分片没有那一层 -// （分层改造之前切出去的归档只有 WebLog 与 Payload 两层）。 +// TierTables 一个分片上各层实际要读的表名；空字符串 = 这个分片没有那一层。 type TierTables struct { DB *gorm.DB Access string // access_log / access_log_ Event string // security_event / security_event_ Payload string // event_payload / event_payload_ - WebLog string // web_logs / web_logs_ + WebLog string // web_logs / web_logs_（分层改造之前的分片才有值） + Empty bool // 归档里一行数据都没有 + Err error // 分片打不开；errors.Is(Err, ErrShardMissing) 表示存储已不在 +} + +// shardTierNames 选层结果，按分片缓存。归档只读，内容不再变； +// 分区被删或按层过期时由 InvalidateShardTierCache / forgetShardTiers 作废。 +type shardTierNames struct { + Access, Event, Payload, WebLog string + Empty bool +} + +var shardTierCache sync.Map + +func forgetShardTiers(name string) { shardTierCache.Delete(name) } + +// InvalidateShardTierCache 分区构成变化后清空选层与索引缓存。 +func InvalidateShardTierCache() { + shardTierCache.Range(func(k, _ any) bool { shardTierCache.Delete(k); return true }) + shardIndexCache.Range(func(k, _ any) bool { shardIndexCache.Delete(k); return true }) } -// ResolveTierTables 把分片标识（ShareDb.FileName，前端 current_db_name）解析成三层表名。 +// tableHasRows 表存在且至少有一行（当前租户可见的）。 +func tableHasRows(db *gorm.DB, table string) bool { + if !dialect.Get().TableExists(db, table) { + return false + } + var one []int + res := db.Table(table).Select("1 AS x").Limit(1).Find(&one) + return res.Error == nil && res.RowsAffected > 0 +} + +// classifyShardTiers 按数据判断分片属于哪个年代，而不是按表在不在： +// 分层改造边界切出来的分片（以及被旧版打开过、跑过迁移的归档）同时有 web_logs 与空的新表， +// 按存在性选会读到空的新表。新表里有行才按新表读，否则读 web_logs。 +func classifyShardTiers(db *gorm.DB, shard, access, event, payload, weblog string) shardTierNames { + exists := func(t string) string { + if dialect.Get().TableExists(db, t) { + return t + } + return "" + } + var r shardTierNames + if tableHasRows(db, access) || tableHasRows(db, event) { + r.Access, r.Event, r.Payload = exists(access), exists(event), exists(payload) + if tableHasRows(db, weblog) { + zlog.Warn("归档分片同时有分层数据与 web_logs 存量行，按分层读取", "shard", shard) + } + return r + } + r.WebLog = exists(weblog) + if tableHasRows(db, payload) { + r.Payload = payload + } + r.Empty = r.WebLog == "" || !tableHasRows(db, weblog) + return r +} + +// ResolveTierTables 把分片标识（ShareDb.FileName，前端 current_db_name）解析成各层表名。 // -// 分片标识的三种形态： // - 空 / local_log.db / web_logs / access_log → 实时库 -// - SQLite 归档文件名（local_log_.db）→ 整个文件，表名不带后缀 -// - MySQL/PG 归档表名：改造前是 web_logs_，改造后是 access_log_； -// 两种都按后缀推同分片的其余表，探不到就当那一层不存在 +// - SQLite 归档文件名（local_log_.db）→ 只读打开整个文件，表名不带后缀 +// - MySQL/PG 归档表名：改造前是 web_logs_，改造后是 access_log_，按后缀推同分片其余表 // -// 报文表探不到分片自带的时回落实时 event_payload：换表失败时报文就留在那里， -// 按 req_uuid 寻址照样找得到。 +// MySQL/PG 下分片自带的报文表没有时回落实时 event_payload：换表失败时报文留在那里，按 req_uuid 仍找得到。 func ResolveTierTables(currentDbName string) TierTables { - // 实时 - if len(currentDbName) == 0 || currentDbName == enums.DB_LOG || - currentDbName == LogTableName || currentDbName == model.AccessLogTableName { + if isLiveIdent(currentDbName) { return TierTables{ DB: global.GWAF_LOCAL_LOG_DB, Access: model.AccessLogTableName, Event: model.SecurityEventTableName, Payload: model.EventPayloadTableName, - WebLog: LogTableName, + WebLog: LogTableName, // 详情按识别码点查时兜底读改造前的存量行 } } - // SQLite 归档：整个文件，表名不带后缀；老归档文件里可能只有 web_logs（+event_payload） + var db *gorm.DB + suffix := "" if dialect.Get().IsFileBased() { - db, _ := ResolveLogDB(currentDbName) // 负责打开文件与降级 - return TierTables{ - DB: db, - Access: probeShardTable(db, currentDbName, model.AccessLogTableName), - Event: probeShardTable(db, currentDbName, model.SecurityEventTableName), - Payload: probeShardTable(db, currentDbName, model.EventPayloadTableName), - WebLog: probeShardTable(db, currentDbName, LogTableName), + d, _, err := ResolveLogDB(currentDbName) + if err != nil { + return TierTables{Err: err} + } + db = d + } else { + switch { + case strings.HasPrefix(currentDbName, LogTableName+"_"): + suffix = strings.TrimPrefix(currentDbName, LogTableName) + case strings.HasPrefix(currentDbName, model.AccessLogTableName+"_"): + suffix = strings.TrimPrefix(currentDbName, model.AccessLogTableName) + default: + return TierTables{Err: fmt.Errorf("认不出的归档分区标识: %q", currentDbName)} + } + db = global.GWAF_LOCAL_LOG_DB + if db == nil { + return TierTables{Err: errors.New("日志库未就绪")} } } - // MySQL/PG 归档：按表名后缀推同分片的其余表 - suffix := "" - switch { - case strings.HasPrefix(currentDbName, LogTableName+"_"): - suffix = strings.TrimPrefix(currentDbName, LogTableName) - case strings.HasPrefix(currentDbName, model.AccessLogTableName+"_"): - suffix = strings.TrimPrefix(currentDbName, model.AccessLogTableName) - default: - // 不认识的标识（脏数据）：降级实时库，沿用 ResolveLogDB 的老行为 - db, _ := ResolveLogDB(currentDbName) - return TierTables{DB: db, Access: model.AccessLogTableName, Event: model.SecurityEventTableName, Payload: model.EventPayloadTableName, WebLog: LogTableName} - } - db := global.GWAF_LOCAL_LOG_DB - t := TierTables{ - DB: db, - Access: probeShardTable(db, currentDbName, model.AccessLogTableName+suffix), - Event: probeShardTable(db, currentDbName, model.SecurityEventTableName+suffix), - Payload: probeShardTable(db, currentDbName, model.EventPayloadTableName+suffix), - WebLog: probeShardTable(db, currentDbName, LogTableName+suffix), - } - if t.Payload == "" { - t.Payload = probeShardTable(db, currentDbName, model.EventPayloadTableName) - } - return t + var names shardTierNames + if v, ok := shardTierCache.Load(currentDbName); ok { + names = v.(shardTierNames) + } else { + names = classifyShardTiers(db, currentDbName, + model.AccessLogTableName+suffix, model.SecurityEventTableName+suffix, + model.EventPayloadTableName+suffix, LogTableName+suffix) + if suffix != "" { + if names.Access == "" && names.Event == "" && names.WebLog == "" && names.Payload == "" { + return TierTables{Err: fmt.Errorf("%w: %s", ErrShardMissing, currentDbName)} + } + if names.Payload == "" && dialect.Get().TableExists(db, model.EventPayloadTableName) { + names.Payload = model.EventPayloadTableName + } + } + shardTierCache.Store(currentDbName, names) + } + return TierTables{DB: db, Access: names.Access, Event: names.Event, Payload: names.Payload, + WebLog: names.WebLog, Empty: names.Empty} } -// ResolveLogTables 在 ResolveLogDB 的基础上再给出该分片的报文表名。 -// 读侧正逐步迁到 ResolveTierTables；存量调用保持原语义。 -// -// 报文表与日志表同进同出： -// - SQLite 按文件分片，归档文件里自带 event_payload -// - MySQL / PostgreSQL 按表分片，web_logs_ 对应 event_payload_ -// -// 返回空表名表示这个分片没有报文表——垂直拆表之前切出去的归档都是这样， -// 它们的报文还在 web_logs 自己的列里，读侧照旧读原列即可。 +// ResolveLogTables 给报文补全用：返回分片连接、主日志表与报文表。打不开时连接为 nil。 func ResolveLogTables(currentDbName string) (*gorm.DB, string, string) { - db, logTable := ResolveLogDB(currentDbName) - if db == nil { - return db, logTable, "" + t := ResolveTierTables(currentDbName) + if t.Err != nil || t.DB == nil { + return nil, "", "" } - - // SQLite 的归档分片是整个文件，连上去之后表名仍是 web_logs / event_payload - if logTable == LogTableName { - return db, logTable, probeShardTable(db, currentDbName, model.EventPayloadTableName) + logTable := t.WebLog + if logTable == "" { + logTable = t.Access } + return t.DB, logTable, t.Payload +} + +// shardIndexCache 分片上某个索引在不在，按 分片|表|索引 缓存 +var shardIndexCache sync.Map - // MySQL / PostgreSQL：web_logs_ 对应 event_payload_ - shardPayload := model.EventPayloadTableName + strings.TrimPrefix(logTable, LogTableName) - if t := probeShardTable(db, currentDbName, shardPayload); t != "" { - return db, logTable, t +// ShardHasIndex 分片上有没有这个索引。强制索引前必须先问：历史文件不再跑迁移， +// 更早切出来的分片可能没有后来才加的索引，强制一个不存在的索引整条 SQL 会报错。 +func ShardHasIndex(db *gorm.DB, shard, table, index string) bool { + if db == nil { + return false + } + if isLiveIdent(shard) { + shard = "live" + } + key := shard + "|" + table + "|" + index + if v, ok := shardIndexCache.Load(key); ok { + return v.(bool) } - // 分表时报文表没能一起换过去（换表失败或这批日志早于垂直拆表）： - // 报文按 req_uuid 寻址，留在实时表里照样找得到，退回去查它。 - return db, logTable, probeShardTable(db, currentDbName, model.EventPayloadTableName) + has := db.Migrator().HasIndex(table, index) + shardIndexCache.Store(key, has) + return has } diff --git a/wafdb/log_shard_cache.go b/wafdb/log_shard_cache.go index 4ea28b5f..e648d283 100644 --- a/wafdb/log_shard_cache.go +++ b/wafdb/log_shard_cache.go @@ -14,14 +14,19 @@ import ( // 这些连接原本直接读写 global.GDATA_CURRENT_LOG_DB_MAP 这张裸 map：并发查两个不同分片 // 会命中 Go 的并发读写检测直接崩进程，且句柄一旦打开就永不释放。这里统一收口， // 加锁之外再给两条上限：最多同时握 shardCacheMax 个连接，空闲超过 shardIdleTTL 的关掉。 +// +// 淘汰出缓存的连接不立刻关：同一次请求里可能还握着它（扇出先统计、后取数）， +// 立刻关会让后半段拿到 database is closed。放进 retired，过 shardRetireGrace 再关。 const ( - shardCacheMax = 8 - shardIdleTTL = 30 * time.Minute + shardCacheMax = 8 + shardIdleTTL = 30 * time.Minute + shardRetireGrace = 2 * time.Minute ) var ( shardMu sync.Mutex shardLastUse = map[string]time.Time{} + shardRetired = map[string][]*gorm.DB{} ) // getShardDB 取一个已打开的分片连接并刷新它的使用时间；没有则返回 nil。 @@ -52,20 +57,63 @@ func putShardDB(name string, db *gorm.DB) (kept *gorm.DB, duplicated bool) { } global.GDATA_CURRENT_LOG_DB_MAP[name] = db shardLastUse[name] = time.Now() + // 重新打开的分片（例如文件被拷回来）重新选层 + forgetShardTiers(name) evictLocked() return db, false } // closeShardDB 关闭并移除一个分片连接。归档清理删文件前必须先调用， // 否则 Windows 下文件被占用删不掉。 +// 宽限期里还没关的同名连接一并关掉。 func closeShardDB(name string) { shardMu.Lock() db := global.GDATA_CURRENT_LOG_DB_MAP[name] delete(global.GDATA_CURRENT_LOG_DB_MAP, name) delete(shardLastUse, name) + retired := shardRetired[name] + delete(shardRetired, name) shardMu.Unlock() closeShardConn(name, db) + for _, r := range retired { + closeShardConn(name, r) + } + forgetShardTiers(name) +} + +// retireLocked 把一个连接移出缓存，宽限期后再关。调用方持锁。 +func retireLocked(name string) { + db := global.GDATA_CURRENT_LOG_DB_MAP[name] + delete(global.GDATA_CURRENT_LOG_DB_MAP, name) + delete(shardLastUse, name) + if db == nil { + return + } + shardRetired[name] = append(shardRetired[name], db) + time.AfterFunc(shardRetireGrace, func() { + shardMu.Lock() + list := shardRetired[name] + kept := list[:0] + found := false + for _, r := range list { + if r == db { + found = true + continue + } + kept = append(kept, r) + } + if len(kept) == 0 { + delete(shardRetired, name) + } else { + shardRetired[name] = kept + } + shardMu.Unlock() + // 已被 closeShardDB 提前关掉的不再关第二次 + if found { + closeShardConn(name, db) + } + }) } // snapshotShardDBs 返回当前连接的快照，供监控之类只读遍历使用， @@ -91,10 +139,7 @@ func evictLocked() { } } for _, name := range victims { - db := global.GDATA_CURRENT_LOG_DB_MAP[name] - delete(global.GDATA_CURRENT_LOG_DB_MAP, name) - delete(shardLastUse, name) - go closeShardConn(name, db) + retireLocked(name) } for len(global.GDATA_CURRENT_LOG_DB_MAP) > shardCacheMax { @@ -108,10 +153,7 @@ func evictLocked() { if oldest == "" { return } - db := global.GDATA_CURRENT_LOG_DB_MAP[oldest] - delete(global.GDATA_CURRENT_LOG_DB_MAP, oldest) - delete(shardLastUse, oldest) - go closeShardConn(oldest, db) + retireLocked(oldest) } } diff --git a/wafdb/log_shard_test.go b/wafdb/log_shard_test.go new file mode 100644 index 00000000..cc6423ca --- /dev/null +++ b/wafdb/log_shard_test.go @@ -0,0 +1,267 @@ +package wafdb + +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + + "SamWaf/common/uuid" + "SamWaf/enums" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/baseorm" + + sqlite "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func openShardTestDBs(t *testing.T) (core, logDB *gorm.DB) { + t.Helper() + core = openTenantTestDB(t, "sqlite") + if err := core.AutoMigrate(&model.ShareDb{}); err != nil { + t.Fatalf("建 share_dbs 失败: %v", err) + } + logDB = openTenantTestDB(t, "sqlite") + if err := logDB.AutoMigrate(&innerbean.WebLog{}); err != nil { + t.Fatalf("建 web_logs 失败: %v", err) + } + return core, logDB +} + +func countShard(t *testing.T, core *gorm.DB, name string) int64 { + t.Helper() + var n int64 + if err := core.Model(&model.ShareDb{}).Where("file_name = ?", name).Count(&n).Error; err != nil { + t.Fatalf("统计 %s 失败: %v", name, err) + } + return n +} + +// 已有归档分片、却没有实时记录时也要补上：只在整表为空时才补的话，这种库会一直缺这一行， +// 归档列表里就没有任何一项被标为 is_current。 +func TestEnsureLiveShardRecordWithArchivesOnly(t *testing.T) { + core, logDB := openShardTestDBs(t) + for _, name := range []string{"local_log_20260615173225.db", "local_log_20260616092350.db"} { + if err := core.Create(&model.ShareDb{BaseOrm: baseorm.BaseOrm{Id: uuid.GenUUID()}, DbLogicType: "log", FileName: name}).Error; err != nil { + t.Fatalf("写归档记录失败: %v", err) + } + } + + ensureLiveShardRecord(core, logDB, enums.DB_LOG) + + if got := countShard(t, core, enums.DB_LOG); got != 1 { + t.Fatalf("实时记录条数 = %d，应为 1", got) + } +} + +func TestEnsureLiveShardRecordIdempotent(t *testing.T) { + core, logDB := openShardTestDBs(t) + + ensureLiveShardRecord(core, logDB, enums.DB_LOG) + ensureLiveShardRecord(core, logDB, enums.DB_LOG) + + if got := countShard(t, core, enums.DB_LOG); got != 1 { + t.Fatalf("重复调用后实时记录条数 = %d，应为 1", got) + } +} + +// 补登用的名字必须和归档列表判 is_current 用的是同一个。 +func TestLiveLogNameSQLite(t *testing.T) { + if got := LiveLogName(); got != enums.DB_LOG { + t.Fatalf("SQLite 下 LiveLogName() = %q，应为 %q", got, enums.DB_LOG) + } +} + +// —— 归档分片只读打开与按数据选层 —— + +// makeArchive 在 ./data 下造一个归档文件：建出全部四张日志表（模拟被旧版打开过、跑过迁移的归档）， +// legacyRows 行写进 web_logs，accessRows 行写进 access_log。 +func makeArchive(t *testing.T, name string, legacyRows, accessRows int) string { + t.Helper() + if err := os.MkdirAll("data", 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join("data", name) + dsn := path + "?_db_key=" + url.QueryEscape(global.GWAF_PWD_LOGDB) + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatalf("建归档文件失败: %v", err) + } + if err := db.AutoMigrate(&innerbean.WebLog{}, &model.AccessLog{}, &model.SecurityEvent{}, &model.EventPayload{}); err != nil { + t.Fatalf("建表失败: %v", err) + } + for i := 0; i < legacyRows; i++ { + row := &innerbean.WebLog{REQ_UUID: fmt.Sprintf("%s_w%d", name, i), TenantId: global.GWAF_TENANT_ID, + USER_CODE: global.GWAF_USER_CODE, UNIX_ADD_TIME: int64(i + 1)} + if err := db.Create(row).Error; err != nil { + t.Fatalf("写 web_logs 失败: %v", err) + } + } + for i := 0; i < accessRows; i++ { + row := &model.AccessLog{LogNarrow: model.LogNarrow{ReqUUID: fmt.Sprintf("%s_a%d", name, i), + TenantId: global.GWAF_TENANT_ID, UserCode: global.GWAF_USER_CODE, UNIX_ADD_TIME: int64(i + 1)}} + if err := db.Create(row).Error; err != nil { + t.Fatalf("写 access_log 失败: %v", err) + } + } + if s, e := db.DB(); e == nil { + _ = s.Close() + } + return path +} + +// 改造边界切出来的分片（以及被旧版打开过的归档）同时有 web_logs 数据与空的新表： +// 必须读 web_logs，不能因为新表「存在」就去读空表。 +func TestResolveTierTablesPollutedArchiveReadsWebLogs(t *testing.T) { + const name = "local_log_20260101000001.db" + path := makeArchive(t, name, 3, 0) + t.Cleanup(func() { closeShardDB(name) }) + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + + tier := ResolveTierTables(name) + if tier.Err != nil { + t.Fatalf("打开归档失败: %v", tier.Err) + } + // 对照：新表确实存在，按存在性选层就会读到空表 + if !tier.DB.Migrator().HasTable(model.AccessLogTableName) { + t.Fatal("对照组失效：归档里应当有空的 access_log") + } + if tier.WebLog != LogTableName || tier.Access != "" || tier.Event != "" { + t.Fatalf("应读 web_logs、新表置空，实际 %+v", tier) + } + var n int64 + if err := tier.DB.Table(tier.WebLog).Count(&n).Error; err != nil || n != 3 { + t.Fatalf("应读到 3 行，实际 %d (%v)", n, err) + } + if tier.Empty { + t.Fatal("有数据的归档不该标成空") + } + + // 只读：连接层拒写，文件大小与修改时间不变 + if err := tier.DB.Exec("DELETE FROM web_logs").Error; err == nil { + t.Fatal("归档连接不该能写") + } + closeShardDB(name) + after, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if after.Size() != before.Size() || !after.ModTime().Equal(before.ModTime()) { + t.Fatalf("归档文件被改动：%d/%v → %d/%v", before.Size(), before.ModTime(), after.Size(), after.ModTime()) + } + if _, err := os.Stat(path + "-wal"); err == nil { + if fi, _ := os.Stat(path + "-wal"); fi.Size() > 0 { + t.Fatal("只读打开不该留下有内容的 WAL") + } + } +} + +func TestResolveTierTablesTieredArchiveReadsNewTables(t *testing.T) { + const name = "local_log_20260101000002.db" + makeArchive(t, name, 0, 2) + t.Cleanup(func() { closeShardDB(name) }) + + tier := ResolveTierTables(name) + if tier.Err != nil { + t.Fatalf("打开归档失败: %v", tier.Err) + } + if tier.Access != model.AccessLogTableName || tier.Event != model.SecurityEventTableName || tier.WebLog != "" { + t.Fatalf("分层之后的归档应读新表，实际 %+v", tier) + } +} + +func TestResolveTierTablesEmptyArchive(t *testing.T) { + const name = "local_log_20260101000003.db" + makeArchive(t, name, 0, 0) + t.Cleanup(func() { closeShardDB(name) }) + + tier := ResolveTierTables(name) + if tier.Err != nil || !tier.Empty { + t.Fatalf("空归档应可打开且标为空，实际 %+v", tier) + } +} + +// 文件不在时报缺失，且不能在原位置建出一个空库，也不能回落实时库 +func TestResolveTierTablesMissingArchive(t *testing.T) { + const name = "local_log_20260101000004.db" + tier := ResolveTierTables(name) + if !errors.Is(tier.Err, ErrShardMissing) { + t.Fatalf("应报 ErrShardMissing，实际 %v", tier.Err) + } + if tier.DB != nil { + t.Fatal("缺失的分片不该返回任何连接") + } + if _, err := os.Stat(filepath.Join("data", name)); !os.IsNotExist(err) { + t.Fatal("打开缺失分片时在原位置建出了文件") + } + if !ShardFileMissing(name) { + t.Fatal("ShardFileMissing 应为 true") + } +} + +func TestResolveTierTablesRejectsBadNames(t *testing.T) { + for _, name := range []string{"../local.db", "local_log_1.db/../../x.db", "local.db", "local_log_abc.db"} { + if tier := ResolveTierTables(name); tier.Err == nil { + t.Fatalf("%q 不该被当成归档打开", name) + } + } +} + +// 淘汰出缓存的连接在宽限期内仍可用；显式关闭时连同宽限期里的一起关掉 +func TestShardRetireKeepsConnectionUsable(t *testing.T) { + names := make([]string, 0, shardCacheMax+1) + for i := 0; i <= shardCacheMax; i++ { + name := fmt.Sprintf("local_log_2026020100%04d.db", i) + makeArchive(t, name, 1, 0) + names = append(names, name) + } + t.Cleanup(func() { + for _, n := range names { + closeShardDB(n) + } + }) + first := ResolveTierTables(names[0]) + if first.Err != nil { + t.Fatal(first.Err) + } + for _, n := range names[1:] { + if tier := ResolveTierTables(n); tier.Err != nil { + t.Fatal(tier.Err) + } + } + if getShardDB(names[0]) != nil { + t.Fatal("超过上限后最早的连接应被移出缓存") + } + var n int64 + if err := first.DB.Table(LogTableName).Count(&n).Error; err != nil { + t.Fatalf("宽限期内被淘汰的连接应仍可用: %v", err) + } + closeShardDB(names[0]) + if err := first.DB.Table(LogTableName).Count(&n).Error; err == nil { + t.Fatal("显式关闭后宽限期里的连接也应已关") + } +} + +func TestShardHasIndex(t *testing.T) { + const name = "local_log_20260101000005.db" + makeArchive(t, name, 1, 0) + t.Cleanup(func() { closeShardDB(name) }) + tier := ResolveTierTables(name) + if tier.Err != nil { + t.Fatal(tier.Err) + } + if ShardHasIndex(tier.DB, name, LogTableName, "idx_no_such_index") { + t.Fatal("不存在的索引应返回 false") + } + if !ShardHasIndex(tier.DB, name, LogTableName, "idx_weblog_time") { + t.Fatal("结构体上声明的索引应存在") + } +} diff --git a/wafdb/mysql_localdb.go b/wafdb/mysql_localdb.go index 16ff71c4..301025b9 100644 --- a/wafdb/mysql_localdb.go +++ b/wafdb/mysql_localdb.go @@ -5,13 +5,9 @@ package wafdb // activation is handled via the go.mod dependency on gorm.io/driver/mysql. import ( - "SamWaf/common/uuid" "SamWaf/common/zlog" - "SamWaf/customtype" "SamWaf/global" - "SamWaf/innerbean" "SamWaf/model" - "SamWaf/model/baseorm" "SamWaf/wafdb/dialect" "fmt" "strconv" @@ -276,32 +272,8 @@ func InitLogDbMySQL() (bool, error) { pathLogSql(db) - // 确保存在一条 live 分片记录(web_logs)。幂等：仅当该记录不存在时创建。 - // 不能用 share_dbs 总数判断——从 SQLite 迁移过来时表里已有 .db 历史分片，总数!=0 会导致 live 记录缺失。 - var liveCount int64 - global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Where("file_name = ?", "web_logs").Count(&liveCount) - if liveCount == 0 { - var logTotal int64 - global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&logTotal) - - sharDbBean := model.ShareDb{ - BaseOrm: baseorm.BaseOrm{ - Id: uuid.GenUUID(), - USER_CODE: global.GWAF_USER_CODE, - Tenant_ID: global.GWAF_TENANT_ID, - CREATE_TIME: customtype.JsonTime(time.Now()), - UPDATE_TIME: customtype.JsonTime(time.Now()), - }, - DbLogicType: "log", - StartTime: customtype.JsonTime(time.Now()), - EndTime: customtype.JsonTime(time.Now()), - // live 分片标识用 web_logs 表名（与 ResolveLogDB 的 live 判定一致）； - // 历史分片由分表任务写入 web_logs_ 表名。不能用库名，否则读取会误入历史分支。 - FileName: "web_logs", - Cnt: logTotal, - } - global.GWAF_LOCAL_DB.Create(sharDbBean) - } + // live 分片标识用 web_logs 表名（与 ResolveLogDB 的 live 判定一致），不能用库名，否则读取会误入历史分支 + ensureLiveShardRecord(global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB, LiveLogName()) return false, nil } diff --git a/wafdb/postgres_localdb.go b/wafdb/postgres_localdb.go index 627e47af..018287ae 100644 --- a/wafdb/postgres_localdb.go +++ b/wafdb/postgres_localdb.go @@ -4,13 +4,9 @@ package wafdb // Mirrors mysql_localdb.go; see that file for the shape this follows. import ( - "SamWaf/common/uuid" "SamWaf/common/zlog" - "SamWaf/customtype" "SamWaf/global" - "SamWaf/innerbean" "SamWaf/model" - "SamWaf/model/baseorm" "SamWaf/wafdb/dialect" "fmt" "time" @@ -199,32 +195,8 @@ func InitLogDbPostgres() (bool, error) { pathLogSql(db) - // 确保存在一条 live 分片记录(web_logs)。幂等：仅当该记录不存在时创建。 - // 不能用 share_dbs 总数判断——从 SQLite 迁移过来时表里已有 .db 历史分片，总数!=0 会导致 live 记录缺失。 - var liveCount int64 - global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Where("file_name = ?", "web_logs").Count(&liveCount) - if liveCount == 0 { - var logTotal int64 - global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&logTotal) - - sharDbBean := model.ShareDb{ - BaseOrm: baseorm.BaseOrm{ - Id: uuid.GenUUID(), - USER_CODE: global.GWAF_USER_CODE, - Tenant_ID: global.GWAF_TENANT_ID, - CREATE_TIME: customtype.JsonTime(time.Now()), - UPDATE_TIME: customtype.JsonTime(time.Now()), - }, - DbLogicType: "log", - StartTime: customtype.JsonTime(time.Now()), - EndTime: customtype.JsonTime(time.Now()), - // live 分片标识用 web_logs 表名（与 ResolveLogDB 的 live 判定一致）； - // 历史分片由分表任务写入 web_logs_ 表名。不能用库名，否则读取会误入历史分支。 - FileName: "web_logs", - Cnt: logTotal, - } - global.GWAF_LOCAL_DB.Create(sharDbBean) - } + // live 分片标识用 web_logs 表名（与 ResolveLogDB 的 live 判定一致），不能用库名，否则读取会误入历史分支 + ensureLiveShardRecord(global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB, LiveLogName()) return false, nil } diff --git a/waftask/task_db_sharding.go b/waftask/task_db_sharding.go index 7c9a10d3..7b625e5a 100644 --- a/waftask/task_db_sharding.go +++ b/waftask/task_db_sharding.go @@ -84,19 +84,9 @@ func TaskShareDbInfo() { return } - // 分层改造的一次性边界：升级后 web_logs 里还躺着改造前的数据，而写入已切到 - // access_log / security_event / event_payload。先把它整体切成一个归档分片， - // 新表从空开始——此后「实时」与「归档」各归各位，旧数据照旧能在归档下拉里读（D6）。 - // 无标记可记：web_logs 不再写入，access_log 一旦有行就说明边界已经切过，天然幂等。 - if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.AccessLogTableName) { - var legacyCnt, accessCnt int64 - global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&legacyCnt) - global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Count(&accessCnt) - if legacyCnt > 0 && accessCnt == 0 { - // 存量 web_logs 里的数据跨很多个月，给它安一个周期键是假的，沿用时间戳命名 - doLogShardCut(innerLogName, fmt.Sprintf("分层改造边界切换（存量 %d 行转入归档）", legacyCnt), true, "") - return - } + // 启动时已同步做过一次（CutTierBoundaryIfNeeded），这里是兜底 + if CutTierBoundaryIfNeeded() { + return } // 周期边界优先：实时库里最早一条日志落在上一个周期，就把这一段整体切成那个周期的分区。 @@ -168,6 +158,32 @@ func TaskShareDbInfo() { } } +// CutTierBoundaryIfNeeded 分层改造的一次性边界：升级后 web_logs 里还躺着改造前的数据，而写入已切到 +// access_log / security_event / event_payload。先把它整体切成一个归档分片，新表从空开始—— +// 此后「实时」与「归档」各归各位，旧数据照旧能在归档下拉里读（D6）。 +// +// 必须在开始接流量之前调用：判定条件是 access_log 为空，一旦有新请求写进来就不再成立， +// 存量数据会一直留在实时库的 web_logs 里，而实时视图只读新表。 +// 无标记可记：web_logs 不再写入，access_log 一旦有行就说明边界已经切过，天然幂等。 +// 返回 true 表示本次做了切分。 +func CutTierBoundaryIfNeeded() bool { + if global.GWAF_LOCAL_DB == nil || global.GWAF_LOCAL_LOG_DB == nil { + return false + } + if !dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, model.AccessLogTableName) { + return false + } + var legacyCnt, accessCnt int64 + global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&legacyCnt) + global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Count(&accessCnt) + if legacyCnt == 0 || accessCnt > 0 { + return false + } + // 存量 web_logs 里的数据跨很多个月，给它安一个周期键是假的，沿用时间戳命名 + doLogShardCut("TaskDBSharding", fmt.Sprintf("分层改造边界切换（存量 %d 行转入归档）", legacyCnt), true, "") + return true +} + // doLogShardCut 执行一次日志库切分。swapWebLog=true 只用于分层改造的一次性边界切换 // （把存量 web_logs + event_payload 切出去）；常规切分换的是三层新表， // web_logs 不再写入也就无需再换。 diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index 36cb506f..1107046f 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -854,6 +854,9 @@ notes: SQLite 的一个分区是一个文件、三层都在里面，只能按较长的「日志保留天数」整体回收。 升级前切出来的旧分片（包括最早那种只有 web_logs 的）**照旧可以查**，不需要任何迁移动作， 读取时会自动适配它实际有哪些表和哪些列，到期后同样整体回收。 + 历史归档文件一律**只读**打开，查询不会再改动它们（不补表、不补列、不建索引）， + 同时打开多个历史分区时内存占用也收小了。分片记录还在、文件却已不在（例如被手工删掉）的分区， + 归档下拉里会标「文件缺失」且不可选，查询时会说明跳过了哪些分区，可在「分区管理」里删除这条记录。 en: title: Log archives are now cut per month and expired by dropping whole partitions (MySQL/PostgreSQL archive tables start being recycled) detail: >- @@ -882,6 +885,11 @@ notes: Shards cut before the upgrade, including the earliest ones that only hold web_logs, **remain queryable** with no migration on your part: reads adapt to whichever tables and columns a shard actually has, and they are recycled whole once they expire. + Archived history files are always opened **read-only**: queries never modify them (no tables, columns + or indexes are added), and opening several history partitions at once uses much less memory. A + partition whose record still exists but whose file is gone (for example deleted by hand) is marked + "file missing" and cannot be selected in the archive dropdown; queries say which partitions were + skipped, and the record can be removed under Partitions. - id: v1_3_25_log_auto_partition_and_uuid version: v1.3.25 From 8e952733a484685761eb82f089600e3717f61cde Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 28 Sep 2026 17:16:54 +0800 Subject: [PATCH 20/21] fix: count the live log shard from access_log instead of a stale snapshot --- service/waf_service/waf_sharedb_delete.go | 39 ++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/service/waf_service/waf_sharedb_delete.go b/service/waf_service/waf_sharedb_delete.go index 887d39d2..d9538247 100644 --- a/service/waf_service/waf_sharedb_delete.go +++ b/service/waf_service/waf_sharedb_delete.go @@ -11,6 +11,8 @@ import ( "fmt" "os" "strings" + "sync" + "time" ) // 分区的「还剩哪些层」与「主动删除」（E6）。 @@ -79,7 +81,11 @@ func (receiver *WafShareDbService) GetAllShareDbWithTiers() ([]ShardTierInfo, er for _, s := range shards { info := ShardTierInfo{ShareDb: s} - if !IsLiveShardName(s.FileName) { + if IsLiveShardName(s.FileName) { + // 实时行的 Cnt 只是补登那一刻对 web_logs 的快照，且该表分层后已停写，永远定格。 + // 归档分区的 Cnt 是切库快照（归档不可变，快照即准确），实时的只能现场数。 + info.Cnt = liveShardCnt() + } else { if dialect.Get().IsFileBased() { // 只看文件在不在，不打开：分片文件多时列表照样秒回 info.Missing = wafdb.ShardFileMissing(s.FileName) @@ -99,6 +105,37 @@ func (receiver *WafShareDbService) GetAllShareDbWithTiers() ([]ShardTierInfo, er return out, nil } +// 实时分片条数的 30 秒缓存：下拉每次打开都全表 COUNT 不值当，30 秒的滞后对「一直在涨」 +// 的实时库没有体感差别。 +var liveCntCache struct { + mu sync.Mutex + at time.Time + value int64 +} + +// liveShardCnt 数实时库当前有多少条日志。正常档位（db/sample）下事件双写窄行， +// access_log 一条不缺；off 档不记窄行，退而数 security_event。 +func liveShardCnt() int64 { + liveCntCache.mu.Lock() + defer liveCntCache.mu.Unlock() + if time.Since(liveCntCache.at) < 30*time.Second { + return liveCntCache.value + } + if global.GWAF_LOCAL_LOG_DB == nil { + return liveCntCache.value + } + var n int64 + if err := global.GWAF_LOCAL_LOG_DB.Model(&model.AccessLog{}).Count(&n).Error; err == nil && n > 0 { + liveCntCache.value, liveCntCache.at = n, time.Now() + return n + } + var ev int64 + if err := global.GWAF_LOCAL_LOG_DB.Model(&model.SecurityEvent{}).Count(&ev).Error; err == nil { + liveCntCache.value, liveCntCache.at = ev, time.Now() + } + return liveCntCache.value +} + // ForceDeleteShard 主动删除一个归档分区，**不看保留期**。 // // 只删 share_dbs 里登记过的归档分片：名字得对得上记录，才不会被当成「随便丢一张表」的入口。 From baccdb19322933e0b71ac90528053b2857655235 Mon Sep 17 00:00:00 2001 From: samwaf Date: Tue, 29 Sep 2026 13:59:01 +0800 Subject: [PATCH 21/21] fix:export db bug --- .gitignore | 3 +- api/waf_log.go | 26 ++-- middleware/replay_protect_middleware.go | 15 ++- model/common/response/response.go | 7 +- service/waf_service/waf_log_export.go | 41 ++++++ wafdb/log_export.go | 144 ++++++++++++++------ wafdb/log_export_test.go | 166 ++++++++++++++++++++++++ 7 files changed, 349 insertions(+), 53 deletions(-) create mode 100644 service/waf_service/waf_log_export.go create mode 100644 wafdb/log_export_test.go diff --git a/.gitignore b/.gitignore index a9a00366..6bb7a8ef 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ dist/ # 只挡 wafsec/ 下这两个固定文件名，不影响内嵌资源 cmd/samwaf/exedata/public_key.pem。 /wafsec/private_key.pem /wafsec/public_key.pem -/wafinit/testdata \ No newline at end of file +/wafinit/testdata +/download/ \ No newline at end of file diff --git a/api/waf_log.go b/api/waf_log.go index ed4cf0fb..6e8591a3 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -100,8 +100,8 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) { } } func (w *WafLogAPi) ExportDBApi(c *gin.Context) { - // 导出物是「按时间段导出选定层」的新加密 SQLite 文件（见 wafdb.ExportLogRangeDb）， - // 仅文件型数据库(SQLite)支持；MySQL 等无日志文件，直接屏蔽，避免进入后台 goroutine 后才失败。 + // 导出物是「按时间段导出选定层」的新加密 SQLite 文件（service 按时间段扇出实时库与归档分区， + // 逐分片拷贝见 wafdb.ExportLogRangeDb），仅文件型数据库(SQLite)支持；MySQL 等无日志文件，直接屏蔽，避免进入后台 goroutine 后才失败。 if !dialect.Get().SupportsBackup() { response.FailWithMessage("当前数据库不支持日志文件导出（仅 SQLite 支持）", c) return @@ -120,7 +120,9 @@ func (w *WafLogAPi) ExportDBApi(c *gin.Context) { Msg: "当前不允许导出", Success: "false", }) - response.FailWithMessage("当前不允许导出", c) + // 专属码 EXPORT_DISABLED：前端据此给出「如何开启」的指引，而不是一句报错 + response.Result(response.EXPORT_DISABLED, map[string]interface{}{}, + "日志导出功能未开启：请在 conf/config.yml 中设置 export_download: true 并重启后再试", c) return } if global.GDATA_CURRENT_CHANGE { @@ -184,7 +186,7 @@ func (w *WafLogAPi) ExportDBApi(c *gin.Context) { // 创建下载文件 downloadFileName := fmt.Sprintf("local_log_export_%s.db", time.Now().Format("20060102150405")) downloadFilePath := filepath.Join(downLoadDir, downloadFileName) - counts, err := wafdb.ExportLogRangeDb(downloadFilePath, startTime, endTime, tiers) + counts, err := wafLogService.ExportLogRange(downloadFilePath, startTime, endTime, tiers) if err != nil { _ = os.Remove(downloadFilePath) global.GQEQUE_MESSAGE_DB.Enqueue(innerbean.OpResultMessageInfo{ @@ -204,6 +206,9 @@ func (w *WafLogAPi) ExportDBApi(c *gin.Context) { }) } }() + // 导出走后台 goroutine，受理本身要立刻落响应：此前处理函数什么都不写， + // 前端拿到 200 空体，只能把「已经开始」误报成失败 + response.OkWithMessage("导出已开始，完成后会推送下载通知", c) } func (w *WafLogAPi) DownloadApi(c *gin.Context) { if global.GWAF_CAN_EXPORT_DOWNLOAD_LOG == false { @@ -220,7 +225,7 @@ func (w *WafLogAPi) DownloadApi(c *gin.Context) { Msg: "当前不允许下载", Success: "false", }) - c.JSON(http.StatusInternalServerError, gin.H{"message": "当前不允许下载"}) + c.JSON(http.StatusInternalServerError, gin.H{"message": "当前不允许下载：请在 conf/config.yml 中设置 export_download: true 并重启后再试"}) return } if len(global.GWAF_RUNTIME_CURRENT_EXPORT_DB_LOG_FILE_PATH) == 0 { @@ -228,15 +233,12 @@ func (w *WafLogAPi) DownloadApi(c *gin.Context) { return } // 提供文件下载 - c.FileAttachment(global.GWAF_RUNTIME_CURRENT_EXPORT_DB_LOG_FILE_PATH, "log.db") + filePath := global.GWAF_RUNTIME_CURRENT_EXPORT_DB_LOG_FILE_PATH + c.FileAttachment(filePath, "log.db") global.GWAF_RUNTIME_CURRENT_EXPORT_DB_LOG_FILE_PATH = "" - // 下载完成后删除文件 - err := os.Remove(global.GWAF_RUNTIME_CURRENT_EXPORT_DB_LOG_FILE_PATH) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"message": "Failed to delete file"}) - return - } + // 下载完成后删除文件；删不掉有 download/ 的定时清理兜底 + _ = os.Remove(filePath) } func (w *WafLogAPi) GetListByHostCodeApi(c *gin.Context) { var req request.WafAttackLogSearch diff --git a/middleware/replay_protect_middleware.go b/middleware/replay_protect_middleware.go index 16bd750c..9c0740d3 100644 --- a/middleware/replay_protect_middleware.go +++ b/middleware/replay_protect_middleware.go @@ -5,6 +5,7 @@ import ( "SamWaf/global" "SamWaf/model/common/response" "strconv" + "strings" "time" "github.com/gin-gonic/gin" @@ -36,6 +37,19 @@ func ReplayProtect() gin.HandlerFunc { // 校验 X-Request-Time tsStr := c.GetHeader(replayTimeHeader) + nonce := c.GetHeader(replayNonceHeader) + if tsStr == "" || nonce == "" { + // 日志下载走 window.open，浏览器带不了自定义头，时间戳与 nonce 只能放查询串 + // （与 extractTokenStr 对同一路径的特判保持一致；仅限这条路径，其余接口仍只认头） + if strings.HasPrefix(c.Request.URL.Path, "/api/v1/waflog/attack/download") { + if tsStr == "" { + tsStr = c.Query(replayTimeHeader) + } + if nonce == "" { + nonce = c.Query(replayNonceHeader) + } + } + } if tsStr == "" { response.FailWithMessage("请求缺少时间标头", c) c.Abort() @@ -55,7 +69,6 @@ func ReplayProtect() gin.HandlerFunc { } // 校验 X-Request-Id（Nonce） - nonce := c.GetHeader(replayNonceHeader) if nonce == "" || len(nonce) < 16 || len(nonce) > 128 { response.FailWithMessage("请求标识无效", c) c.Abort() diff --git a/model/common/response/response.go b/model/common/response/response.go index 3651cb76..09169f5b 100644 --- a/model/common/response/response.go +++ b/model/common/response/response.go @@ -26,8 +26,11 @@ const ( // BACKEND_UNAVAILABLE 依赖的存储/缓存后端本次不可用，请求未能完成。 // 与 AUTHFAIL 的区别：登录状态没有问题，客户端应保留登录态并稍后重试。 BACKEND_UNAVAILABLE = -6 - FORBIDDEN = -403 - AUTHFAIL = -999 + // EXPORT_DISABLED 日志导出被配置关闭（conf/config.yml 的 export_download 未开启）。 + // 前端据此给出开启方法的指引，而不是一句「当前不允许导出」。 + EXPORT_DISABLED = -7 + FORBIDDEN = -403 + AUTHFAIL = -999 ) // HeaderKeyID 是客户端声明本次会话密钥的请求头，与 X-Sec-Ver: 2 配套。 diff --git a/service/waf_service/waf_log_export.go b/service/waf_service/waf_log_export.go new file mode 100644 index 00000000..cc792f24 --- /dev/null +++ b/service/waf_service/waf_log_export.go @@ -0,0 +1,41 @@ +package waf_service + +import ( + "SamWaf/common/zlog" + "SamWaf/wafdb" + "time" +) + +// ExportLogRange 按时间段导出日志：与列表查询同一套按时间扇出选片， +// 实时库与命中的归档分区都作为数据源。只读实时库会在「时间段落在已归档分区」时导出空表。 +// startTime/endTime 形如 2006-01-02 15:04:05，空 = 不限制。 +func (receiver *WafLogService) ExportLogRange(outPath, startTime, endTime string, tiers map[string]bool) (map[string]int64, error) { + fromMs := int64(0) + toMs := time.Now().UnixMilli() + if startTime != "" { + if t, err := time.ParseInLocation("2006-01-02 15:04:05", startTime, time.Local); err == nil { + fromMs = t.UnixMilli() + } + } + if endTime != "" { + if t, err := time.ParseInLocation("2006-01-02 15:04:05", endTime, time.Local); err == nil { + toMs = t.UnixMilli() + } + } + + sources := make([]wafdb.ExportSource, 0, 4) + for _, sh := range candidateShards(fromMs, toMs, false) { + tables := wafdb.ResolveExportTables(sh.Name) + switch { + case tables.Err != nil: + // 打不开的分区跳过而不是整单失败：剩下的分区照常导， + // 缺的那个在日志里留名，界面按层计数也能看出少没少 + zlog.Warn("日志导出跳过分片", "shard", sh.Name, "error", tables.Err.Error()) + continue + case tables.Empty: + continue + } + sources = append(sources, wafdb.ExportSource{Shard: sh.Name, Tables: tables}) + } + return wafdb.ExportLogRangeDb(outPath, startTime, endTime, tiers, sources) +} diff --git a/wafdb/log_export.go b/wafdb/log_export.go index 6cc896e5..8f7faa28 100644 --- a/wafdb/log_export.go +++ b/wafdb/log_export.go @@ -25,19 +25,39 @@ const ( const exportBatchSize = 1000 -// exportTierCopy 一个待导出的层：模型（导出文件建表用）+ 源表名 -type exportTierCopy struct { +// exportInsertBatch 单条 INSERT 的最大行数：驱动有效变量上限是 999， +// 列最多的表（web_logs，60+ 列）按 10 行一批也只占 600+ 个变量，留有余量。 +const exportInsertBatch = 10 + +// ExportSource 一个导出数据源：分片连接 + 该分片各层实际的表名（空 = 该分片没有这一层）。 +// 来源列表由 service 层按时间段扇出得出（与列表查询同一套选片逻辑）： +// 只导实时库会在「时间段落在已归档分区」时导出空表。 +type ExportSource struct { + Shard string + Tables TierTables +} + +// exportTierDef 一个可导出的层：模型（导出文件建表用）+ 导出件里的标准表名 + 从分片取源表名 +type exportTierDef struct { name string model interface{} - srcTable string + dstTable string + pick func(TierTables) string +} + +var exportTierDefs = []exportTierDef{ + {ExportTierAccess, &model.AccessLog{}, model.AccessLogTableName, func(t TierTables) string { return t.Access }}, + {ExportTierEvent, &model.SecurityEvent{}, model.SecurityEventTableName, func(t TierTables) string { return t.Event }}, + {ExportTierPayload, &model.EventPayload{}, model.EventPayloadTableName, func(t TierTables) string { return t.Payload }}, + {ExportTierWeblog, &innerbean.WebLog{}, LogTableName, func(t TierTables) string { return t.WebLog }}, } -// ExportLogRangeDb 把实时日志库按时间段导出成一个新的加密 SQLite 文件（C11）。 +// ExportLogRangeDb 把各数据源（实时库 + 命中的归档分区）按时间段导出成一个新的加密 SQLite 文件（C11）。 // // 分层后「导出一个 .db」不再是备份整个库文件：按选定层 + 时间段导出行， // 导出件自带表结构（AutoMigrate），可直接用同一套密钥打开查阅。 // startTime/endTime 为 "2006-01-02 15:04:05"，空 = 不限制。返回每层导出的行数。 -func ExportLogRangeDb(outPath, startTime, endTime string, tiers map[string]bool) (map[string]int64, error) { +func ExportLogRangeDb(outPath, startTime, endTime string, tiers map[string]bool, sources []ExportSource) (map[string]int64, error) { counts := map[string]int64{} if !dialect.Get().SupportsBackup() { return counts, fmt.Errorf("按时间段导出仅在 SQLite 模式下可用，当前驱动: %s", dialect.Get().Name()) @@ -58,64 +78,114 @@ func ExportLogRangeDb(outPath, startTime, endTime string, tiers map[string]bool) } }() - src := global.GWAF_LOCAL_LOG_DB - if src == nil { - return counts, fmt.Errorf("日志库未初始化") + for _, td := range exportTierDefs { + if !tiers[td.name] { + continue + } + // 有任何一个源带着这一层才建表：一张都没有时导出件里不留空表， + // 打开导出件一眼能看出这次确实没选/没有这一层 + migrated := false + for _, s := range sources { + if s.Tables.DB == nil { + continue + } + srcTable := td.pick(s.Tables) + if srcTable == "" || !dialect.Get().TableExists(s.Tables.DB, srcTable) { + continue + } + if !migrated { + if err := dst.AutoMigrate(td.model); err != nil { + return counts, fmt.Errorf("导出文件建表 %s 失败: %w", td.dstTable, err) + } + migrated = true + } + var n int64 + var err error + switch td.name { + case ExportTierAccess: + n, err = exportCopyTier[model.AccessLog](s.Tables.DB, dst, srcTable, td.dstTable, startTime, endTime) + case ExportTierEvent: + n, err = exportCopyTier[model.SecurityEvent](s.Tables.DB, dst, srcTable, td.dstTable, startTime, endTime) + case ExportTierPayload: + n, err = exportCopyTier[model.EventPayload](s.Tables.DB, dst, srcTable, td.dstTable, startTime, endTime) + case ExportTierWeblog: + n, err = exportCopyTier[innerbean.WebLog](s.Tables.DB, dst, srcTable, td.dstTable, startTime, endTime) + } + if err != nil { + return counts, fmt.Errorf("分片 %s: %w", s.Shard, err) + } + counts[td.name] += n + } } + return counts, nil +} - // 各层：模型 + 源表存在性。web_logs 边界切割后可能已不在实时库 - copies := []exportTierCopy{} - if tiers[ExportTierAccess] && dialect.Get().TableExists(src, model.AccessLogTableName) { - copies = append(copies, exportTierCopy{ExportTierAccess, &model.AccessLog{}, model.AccessLogTableName}) - } - if tiers[ExportTierEvent] && dialect.Get().TableExists(src, model.SecurityEventTableName) { - copies = append(copies, exportTierCopy{ExportTierEvent, &model.SecurityEvent{}, model.SecurityEventTableName}) +// ResolveExportTables 导出专用的层解析：每层独立按「表存在且有行」判断。 +// 读侧的 ResolveTierTables 是「选一个年代」——分层数据有行时 web_logs 存量行被忽略； +// 导出要把数据都带上，边界分片里新旧两层并存时两层都导。 +// 仅支持文件型（SQLite）：导出本身就只在 SQLite 下开放。 +func ResolveExportTables(currentDbName string) TierTables { + if isLiveIdent(currentDbName) { + return TierTables{ + DB: global.GWAF_LOCAL_LOG_DB, + Access: model.AccessLogTableName, + Event: model.SecurityEventTableName, + Payload: model.EventPayloadTableName, + WebLog: LogTableName, + } } - if tiers[ExportTierPayload] && dialect.Get().TableExists(src, model.EventPayload{}.TableName()) { - copies = append(copies, exportTierCopy{ExportTierPayload, &model.EventPayload{}, model.EventPayloadTableName}) + if !dialect.Get().IsFileBased() { + return TierTables{Err: fmt.Errorf("导出仅支持 SQLite，认不出的分区标识: %q", currentDbName)} } - if tiers[ExportTierWeblog] && dialect.Get().TableExists(src, LogTableName) { - copies = append(copies, exportTierCopy{ExportTierWeblog, &innerbean.WebLog{}, LogTableName}) + db, _, err := ResolveLogDB(currentDbName) + if err != nil { + return TierTables{Err: err} } - - for _, tc := range copies { - if err := dst.AutoMigrate(tc.model); err != nil { - return counts, fmt.Errorf("导出文件建表 %s 失败: %w", tc.srcTable, err) - } - n, err := exportCopyTier(src, dst, tc, startTime, endTime) - if err != nil { - return counts, err + has := func(name string) string { + if tableHasRows(db, name) { + return name } - counts[tc.name] = n + return "" } - return counts, nil + t := TierTables{DB: db} + t.Access = has(model.AccessLogTableName) + t.Event = has(model.SecurityEventTableName) + t.Payload = has(model.EventPayloadTableName) + t.WebLog = has(LogTableName) + t.Empty = t.Access == "" && t.Event == "" && t.Payload == "" && t.WebLog == "" + return t } // exportCopyTier 按 create_time 区间分批读源表、批量写导出文件。主键冲突保留先到的（重复导出同一区间不会翻倍）。 -func exportCopyTier(src, dst *gorm.DB, tc exportTierCopy, startTime, endTime string) (int64, error) { +// +// 必须用带主键的模型切片接行：FindInBatches 靠目标类型的主键续批（WHERE pk > 上一批最大主键）， +// 导出期间实时表仍在写入也不会错位；用 map 接行没有主键信息，GORM 会报 model value required。 +func exportCopyTier[T any](src, dst *gorm.DB, srcTable, dstTable, startTime, endTime string) (int64, error) { var total int64 - q := src.Table(tc.srcTable) + q := src.Table(srcTable) if startTime != "" { q = q.Where("create_time >= ?", startTime) } if endTime != "" { q = q.Where("create_time <= ?", endTime) } - // 用目标模型接住行，保证列名与导出表一致（web_logs 列比新表多，互不影响） - rows := []map[string]interface{}{} + rows := []T{} result := q.FindInBatches(&rows, exportBatchSize, func(tx *gorm.DB, batch int) error { if len(rows) == 0 { return nil } - if err := dst.Table(tc.srcTable).Clauses(clause.OnConflict{DoNothing: true}). - Create(rows).Error; err != nil { - return fmt.Errorf("写入导出文件 %s 失败: %w", tc.srcTable, err) + // 一批 1000 行 × 几十列远超变量上限，按 exportInsertBatch 拆成小 INSERT； + // 切片 Create 默认包在一个事务里，拆开不会引入逐行提交的开销。 + if err := dst.Session(&gorm.Session{CreateBatchSize: exportInsertBatch}). + Table(dstTable).Clauses(clause.OnConflict{DoNothing: true}). + Create(&rows).Error; err != nil { + return fmt.Errorf("写入导出文件 %s 失败: %w", dstTable, err) } total += int64(len(rows)) return nil }) if result.Error != nil { - return total, fmt.Errorf("读取 %s 失败: %w", tc.srcTable, result.Error) + return total, fmt.Errorf("读取 %s 失败: %w", srcTable, result.Error) } return total, nil } diff --git a/wafdb/log_export_test.go b/wafdb/log_export_test.go new file mode 100644 index 00000000..1baec6fe --- /dev/null +++ b/wafdb/log_export_test.go @@ -0,0 +1,166 @@ +package wafdb + +import ( + "fmt" + "net/url" + "path/filepath" + "testing" + "time" + + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + + sqlite "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// 守护用例：分层导出的分批拷贝曾用 map 接行，FindInBatches 拿不到主键信息， +// 报 "model value required"、拼出 `ORDER BY access_log.` 的坏 SQL，导出整批失败。 +// 这里走真实两库过一遍 exportCopyTier：时间段过滤要对、跨多个批次续批不能漏不能重。 +func TestExportCopyTierBatchesByPrimaryKey(t *testing.T) { + srcDB := openTenantTestDB(t, "sqlite") + dstDB := openTenantTestDB(t, "sqlite") + if err := srcDB.AutoMigrate(&model.AccessLog{}); err != nil { + t.Fatalf("源库建表失败: %v", err) + } + if err := dstDB.AutoMigrate(&model.AccessLog{}); err != nil { + t.Fatalf("导出库建表失败: %v", err) + } + + // 2500 行（> 2 个批次）：create_time 每分钟一行，时间段只覆盖前 1500 行 + base := time.Date(2026, 9, 29, 0, 0, 0, 0, time.Local) + const total = 2500 + batch := make([]model.AccessLog, 0, 500) + flush := func() { + if len(batch) == 0 { + return + } + if err := srcDB.Create(&batch).Error; err != nil { + t.Fatalf("造数失败: %v", err) + } + batch = batch[:0] + } + for i := 0; i < total; i++ { + batch = append(batch, model.AccessLog{LogNarrow: model.LogNarrow{ + ReqUUID: fmt.Sprintf("uuid-%05d", i), + CREATE_TIME: base.Add(time.Duration(i) * time.Minute).Format("2006-01-02 15:04:05"), + }}) + if len(batch) == 500 { + flush() + } + } + flush() + + start := base.Format("2006-01-02 15:04:05") + end := base.Add(1499 * time.Minute).Format("2006-01-02 15:04:05") + n, err := exportCopyTier[model.AccessLog](srcDB, dstDB, model.AccessLogTableName, model.AccessLogTableName, start, end) + if err != nil { + t.Fatalf("导出失败: %v", err) + } + if n != 1500 { + t.Fatalf("应导出 1500 行，实际 %d", n) + } + var got int64 + dstDB.Model(&model.AccessLog{}).Count(&got) + if got != 1500 { + t.Fatalf("导出文件应有 1500 行，实际 %d（分批续批漏行或重复）", got) + } + + // 同区间再导一次：主键冲突保留先到，不翻倍 + if _, err := exportCopyTier[model.AccessLog](srcDB, dstDB, model.AccessLogTableName, model.AccessLogTableName, start, end); err != nil { + t.Fatalf("重复导出失败: %v", err) + } + dstDB.Model(&model.AccessLog{}).Count(&got) + if got != 1500 { + t.Fatalf("重复导出后应仍 1500 行，实际 %d", got) + } +} + +// 守护用例：导出必须跨数据源扇出——只导实时库时，时间段落在归档分区里的数据会整张空表。 +// 造「实时 + 边界归档（access_log 与 web_logs 并存）」两个源，行数要合并且两层都要带出来。 +func TestExportLogRangeDbFansOutAcrossSources(t *testing.T) { + liveDB := openTenantTestDB(t, "sqlite") + archDB := openTenantTestDB(t, "sqlite") + for _, db := range []*gorm.DB{liveDB, archDB} { + if err := db.AutoMigrate(&model.AccessLog{}, &innerbean.WebLog{}); err != nil { + t.Fatalf("建表失败: %v", err) + } + } + base := time.Date(2026, 9, 28, 12, 0, 0, 0, time.Local) + ts := func(i int) string { return base.Add(time.Duration(i) * time.Minute).Format("2006-01-02 15:04:05") } + + put := func(db *gorm.DB, prefix string, n int) { + for i := 0; i < n; i++ { + row := &model.AccessLog{LogNarrow: model.LogNarrow{ + ReqUUID: fmt.Sprintf("%s-a%d", prefix, i), CREATE_TIME: ts(i)}} + if err := db.Create(row).Error; err != nil { + t.Fatalf("写 access_log 失败: %v", err) + } + } + } + put(liveDB, "live", 3) + put(archDB, "arch", 2) + for i := 0; i < 2; i++ { + row := &innerbean.WebLog{REQ_UUID: fmt.Sprintf("arch-w%d", i), CREATE_TIME: ts(i)} + if err := archDB.Create(row).Error; err != nil { + t.Fatalf("写 web_logs 失败: %v", err) + } + } + + out := filepath.Join(t.TempDir(), "export.db") + sources := []ExportSource{ + {Shard: "local_log.db", Tables: TierTables{DB: liveDB, Access: model.AccessLogTableName, WebLog: LogTableName}}, + {Shard: "local_log_202609.db", Tables: TierTables{DB: archDB, Access: model.AccessLogTableName, WebLog: LogTableName}}, + } + tiers := map[string]bool{ExportTierAccess: true, ExportTierWeblog: true} + counts, err := ExportLogRangeDb(out, ts(0), ts(10), tiers, sources) + if err != nil { + t.Fatalf("导出失败: %v", err) + } + if counts[ExportTierAccess] != 5 || counts[ExportTierWeblog] != 2 { + t.Fatalf("行数应为 access=5 weblog=2，实际 %+v", counts) + } + + // 用同一密钥打开导出件复核 + dsn := out + "?_db_key=" + url.QueryEscape(global.GWAF_PWD_LOGDB) + check, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatalf("打开导出件失败: %v", err) + } + defer func() { + if s, e := check.DB(); e == nil { + _ = s.Close() + } + }() + var n int64 + if err := check.Table(model.AccessLogTableName).Count(&n).Error; err != nil || n != 5 { + t.Fatalf("导出件 access_log 应 5 行，实际 %d (%v)", n, err) + } + if err := check.Table(LogTableName).Count(&n).Error; err != nil || n != 2 { + t.Fatalf("导出件 web_logs 应 2 行，实际 %d (%v)", n, err) + } + // 未选中的层不该在导出件里留下空表 + if check.Migrator().HasTable(model.SecurityEventTableName) { + t.Fatal("未选中 event 层，导出件里不该有 security_event 表") + } +} + +// 导出专用的层解析：边界分片新旧两层并存时两层都要（读侧只选新层，导出要把数据都带上）。 +func TestResolveExportTablesBoundaryArchiveKeepsBothTiers(t *testing.T) { + const name = "local_log_20260101000009.db" + makeArchive(t, name, 3, 2) + t.Cleanup(func() { closeShardDB(name) }) + + tier := ResolveExportTables(name) + if tier.Err != nil { + t.Fatalf("打开归档失败: %v", tier.Err) + } + if tier.Access == "" || tier.WebLog == "" { + t.Fatalf("边界分片应同时导出两层，实际 %+v", tier) + } + if tier.Empty { + t.Fatal("有数据的归档不该标成空") + } +}