Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 35 additions & 7 deletions build_test_update.bat
Original file line number Diff line number Diff line change
Expand Up @@ -3,34 +3,62 @@ setlocal

set "CURDIR=%~dp0"
set "CURDIR=%CURDIR:~0,-1%"
set "TESTDIR=%CURDIR%\release\testupdate"

SET CGO_ENABLED=1
SET GOOS=windows
SET GOARCH=amd64
SET GIN_MODE=release

:: ---- Step 1: Build v1.1.0 ----
echo [1/3] Building v1.1.0...
echo [1/6] Building v1.1.0...
if not exist "%CURDIR%\release\githubci\v1.1.0" mkdir "%CURDIR%\release\githubci\v1.1.0"
go build -ldflags="-X SamWaf/global.GWAF_RELEASE=true -X SamWaf/global.GWAF_RELEASE_VERSION_NAME=20260224 -X SamWaf/global.GWAF_RELEASE_VERSION=v1.1.0 -X SamWaf/global.GUPDATE_VERSION_URL=http://127.0.0.1:8111/ -s -w" -o "%CURDIR%\release\githubci\v1.1.0\SamWaf64.exe" ./cmd/samwaf/main.go
if %ERRORLEVEL% neq 0 ( echo FAILED: v1.1.0 build error & pause & exit /b 1 )
echo OK: release\githubci\v1.1.0\SamWaf64.exe

:: ---- Step 2: Build v1.1.1 ----
echo [2/3] Building v1.1.1...
:: ---- Step 2: Stop test backend and deploy v1.1.0 ----
echo [2/6] Stopping test backend and deploying v1.1.0...
if not exist "%TESTDIR%" mkdir "%TESTDIR%"
powershell -NoProfile -ExecutionPolicy Bypass -Command "$d='%TESTDIR%'; $p=Get-Process -Name SamWaf64 -ErrorAction SilentlyContinue | Where-Object { $_.Path -and $_.Path.StartsWith($d, 'OrdinalIgnoreCase') }; if ($p) { Write-Host ('stopping ' + $p.Count + ' process(es)'); $p | Stop-Process -Force; Start-Sleep -Milliseconds 1500 } else { Write-Host 'no running process' }"
copy /y "%CURDIR%\release\githubci\v1.1.0\SamWaf64.exe" "%TESTDIR%\SamWaf64.exe" >nul
if %ERRORLEVEL% neq 0 ( echo FAILED: copy to test backend error ^(file locked?^) & pause & exit /b 1 )
echo OK: %TESTDIR%\SamWaf64.exe

:: ---- Step 3: Build v1.1.1 ----
echo [3/6] Building v1.1.1...
if not exist "%CURDIR%\release\githubci\v1.1.1" mkdir "%CURDIR%\release\githubci\v1.1.1"
go build -ldflags="-X SamWaf/global.GWAF_RELEASE=true -X SamWaf/global.GWAF_RELEASE_VERSION_NAME=20260224 -X SamWaf/global.GWAF_RELEASE_VERSION=v1.1.1 -X SamWaf/global.GUPDATE_VERSION_URL=http://127.0.0.1:8111/ -s -w" -o "%CURDIR%\release\githubci\v1.1.1\SamWaf64.exe" ./cmd/samwaf/main.go
if %ERRORLEVEL% neq 0 ( echo FAILED: v1.1.1 build error & pause & exit /b 1 )
echo OK: release\githubci\v1.1.1\SamWaf64.exe

:: ---- Step 3: Package v1.1.1 update ----
echo [3/3] Packaging v1.1.1...
:: ---- Step 4: Package v1.1.1 update ----
echo [4/6] Packaging v1.1.1...
"%CURDIR%\setup\go_gen_updatefile\go_gen_updatefile.exe" -desc "local-test-1.1.1" -o "%CURDIR%\release\web\samwaf_update" -platform windows-amd64 "%CURDIR%\release\githubci\v1.1.1\SamWaf64.exe" v1.1.1
if %ERRORLEVEL% neq 0 ( echo FAILED: package error & pause & exit /b 1 )
echo OK: release\web\samwaf_update\v1.1.1\windows-amd64.gz

:: ---- Step 5: Start test backend service ----
echo [5/6] Starting test backend v1.1.0...
pushd "%TESTDIR%"
.\SamWaf64.exe start
if %ERRORLEVEL% neq 0 ( echo WARN: SamWaf64.exe start failed ^(service not installed or no admin rights?^) & pause )
popd

:: ---- Step 6: Start local update HTTP server ----
echo [6/6] Starting HTTP server...
if not exist "%CURDIR%\release\web" mkdir "%CURDIR%\release\web"
where python >nul 2>nul
if %ERRORLEVEL% neq 0 ( echo FAILED: python not found in PATH & pause & exit /b 1 )

echo.
echo All done. Test backend v1.1.0 is running in %TESTDIR%.
echo.
echo All done. Start v1.1.0 to test upgrade.
echo HTTP server started: http://127.0.0.1:8111/
echo Root: %CURDIR%\release\web
echo Press Ctrl+C to stop.
echo.
pause
pushd "%CURDIR%\release\web"
python -m http.server 8111
popd
endlocal
124 changes: 124 additions & 0 deletions build_test_upgrade_twice.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
@echo off
setlocal

:: =====================================================================
:: Reproduce the Windows self-update "Access is denied" bug.
::
:: Why build_test_update.bat can NOT reproduce it:
:: that script kills every SamWaf64 process and overwrites the exe on
:: EVERY run, so the test dir is always clean (no leftover .old, no
:: long-lived old Supervisor). It only ever exercises the FIRST upgrade.
::
:: The bug needs TWO upgrades in a row with NO process kill in between:
:: upgrade #1 renames SamWaf64.exe -> .SamWaf64.exe.old and succeeds,
:: but the Supervisor never exits, so its running image IS that .old
:: file and it can never be deleted (Windows). Upgrade #2 then tries to
:: rename onto that still-in-use .old and gets ERROR_ACCESS_DENIED.
::
:: This script prepares the ground and then gets out of the way: it
:: publishes ONLY v1.1.1 so upgrade #1 is a real single step. Publish
:: v1.1.2 later with publish_test_version.bat to trigger upgrade #2.
:: =====================================================================

set "CURDIR=%~dp0"
set "CURDIR=%CURDIR:~0,-1%"
set "TESTDIR=%CURDIR%\release\testupdate"
set "WEBDIR=%CURDIR%\release\web"
set "GENTOOL=%CURDIR%\setup\go_gen_updatefile\go_gen_updatefile.exe"
set "VERNAME=20260224"
set "UPURL=http://127.0.0.1:8111/"

SET CGO_ENABLED=1
SET GOOS=windows
SET GOARCH=amd64
SET GIN_MODE=release

:: ---- Step 1: Build v1.1.0 / v1.1.1 / v1.1.2 ----
echo [1/6] Building v1.1.0, v1.1.1, v1.1.2...
call :build v1.1.0 || exit /b 1
call :build v1.1.1 || exit /b 1
call :build v1.1.2 || exit /b 1

:: ---- Step 2: Stop test backend, wipe leftovers, deploy v1.1.0 ----
:: This is the ONLY point where processes are killed. After this the test
:: must run untouched, otherwise the bug cannot surface.
echo [2/6] Stopping test backend, cleaning leftovers, deploying v1.1.0...
if not exist "%TESTDIR%" mkdir "%TESTDIR%"
powershell -NoProfile -ExecutionPolicy Bypass -Command "$d='%TESTDIR%'; $p=Get-Process -Name SamWaf64 -ErrorAction SilentlyContinue | Where-Object { $_.Path -and $_.Path.StartsWith($d, 'OrdinalIgnoreCase') }; if ($p) { Write-Host ('stopping ' + $p.Count + ' process(es)'); $p | Stop-Process -Force; Start-Sleep -Milliseconds 1500 } else { Write-Host 'no running process' }"

:: Wipe .SamWaf64.exe.old / .new left by earlier runs so the test starts
:: from a genuinely clean state (they are HIDDEN, so -Force is required).
powershell -NoProfile -ExecutionPolicy Bypass -Command "$d='%TESTDIR%'; $f=Get-ChildItem -Force -LiteralPath $d -Filter '.SamWaf64.exe.*' -ErrorAction SilentlyContinue; if ($f) { foreach ($i in $f) { try { $i.Attributes='Normal'; Remove-Item -Force -LiteralPath $i.FullName -ErrorAction Stop; Write-Host ('wiped ' + $i.Name) } catch { Write-Host ('CANNOT WIPE ' + $i.Name + ' -> ' + $_.Exception.Message); Write-Host 'A process is still holding it. Stop it and re-run this script.'; exit 1 } } } else { Write-Host 'no leftover .old/.new' }"
if %ERRORLEVEL% neq 0 ( echo FAILED: leftover cleanup & pause & exit /b 1 )

copy /y "%CURDIR%\release\githubci\v1.1.0\SamWaf64.exe" "%TESTDIR%\SamWaf64.exe" >nul
if %ERRORLEVEL% neq 0 ( echo FAILED: copy to test backend error ^(file locked?^) & pause & exit /b 1 )
echo OK: %TESTDIR%\SamWaf64.exe is v1.1.0

:: ---- Step 3: Publish ONLY v1.1.1 ----
:: windows-amd64.json holds a single target version, so publishing v1.1.2
:: now would make v1.1.0 jump straight to v1.1.2 and skip the two-step.
echo [3/6] Publishing v1.1.1 to the local update server...
call :publish v1.1.1 || exit /b 1

:: ---- Step 4: Start test backend ----
echo [4/6] Starting test backend v1.1.0...
pushd "%TESTDIR%"
.\SamWaf64.exe start
if %ERRORLEVEL% neq 0 ( echo WARN: SamWaf64.exe start failed ^(service not installed or no admin rights?^) & pause )
popd

:: ---- Step 5: Baseline snapshot ----
echo [5/6] Baseline state:
powershell -NoProfile -ExecutionPolicy Bypass -File "%CURDIR%\check_upgrade_state.ps1" -Dir "%TESTDIR%"

:: ---- Step 6: Instructions + HTTP server ----
echo [6/6] Starting HTTP server...
where python >nul 2>nul
if %ERRORLEVEL% neq 0 ( echo FAILED: python not found in PATH & pause & exit /b 1 )

echo.
echo =====================================================================
echo DO NOT kill any SamWaf64 process from here on, and DO NOT re-run
echo this script until the whole sequence below is finished.
echo.
echo 1. Open the admin UI and upgrade v1.1.0 -^> v1.1.1. Expect SUCCESS.
echo.
echo 2. In another window, inspect the state:
echo powershell -File check_upgrade_state.ps1
echo Expected (this is what proves the root cause):
echo - .SamWaf64.exe.old exists and is HIDDEN
echo - the Supervisor process Path now points AT that .old file
echo - deleting it by hand fails with "Access is denied"
echo.
echo 3. Publish the next version:
echo publish_test_version.bat v1.1.2
echo.
echo 4. Upgrade v1.1.1 -^> v1.1.2 in the UI. Expect FAILURE:
echo upgrade error:rename ...\SamWaf64.exe ...\.SamWaf64.exe.old:
echo Access is denied.
echo =====================================================================
echo.
echo HTTP server started: %UPURL%
echo Root: %WEBDIR%
echo Press Ctrl+C to stop.
echo.
pushd "%WEBDIR%"
python -m http.server 8111
popd
endlocal
exit /b 0

:: ---------------------------------------------------------------------
:build
if not exist "%CURDIR%\release\githubci\%~1" mkdir "%CURDIR%\release\githubci\%~1"
go build -ldflags="-X SamWaf/global.GWAF_RELEASE=true -X SamWaf/global.GWAF_RELEASE_VERSION_NAME=%VERNAME% -X SamWaf/global.GWAF_RELEASE_VERSION=%~1 -X SamWaf/global.GUPDATE_VERSION_URL=%UPURL% -s -w" -o "%CURDIR%\release\githubci\%~1\SamWaf64.exe" ./cmd/samwaf/main.go
if %ERRORLEVEL% neq 0 ( echo FAILED: %~1 build error & pause & exit /b 1 )
echo OK: release\githubci\%~1\SamWaf64.exe
exit /b 0

:publish
"%GENTOOL%" -desc "local-test-%~1" -o "%WEBDIR%\samwaf_update" -platform windows-amd64 "%CURDIR%\release\githubci\%~1\SamWaf64.exe" %~1
if %ERRORLEVEL% neq 0 ( echo FAILED: publish %~1 error & pause & exit /b 1 )
echo OK: release\web\samwaf_update\%~1\windows-amd64.gz
exit /b 0
89 changes: 89 additions & 0 deletions check_upgrade_state.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# 升级现场观测脚本 —— 用来确认 Windows 自升级 "Access is denied" 的根因。
#
# powershell -File check_upgrade_state.ps1
# powershell -File check_upgrade_state.ps1 -TryDelete
#
# 三项观测:
# 1) SamWaf64 进程及其映像路径 —— 升级一次后,Supervisor 的 Path 会变成
# .SamWaf64.exe.old(rename 只改目录项,进程仍映射同一个文件对象)
# 2) .SamWaf64.exe.old* 残留清单与 HIDDEN 属性
# 3) -TryDelete 时手工删一次 .old,预期"拒绝访问"
#
# 本脚本只读(除非显式加 -TryDelete),不会动任何进程。

param(
[string] $Dir = (Join-Path $PSScriptRoot 'release\testupdate'),
[switch] $TryDelete
)

$ErrorActionPreference = 'Continue'
$resolved = Resolve-Path -LiteralPath $Dir -ErrorAction SilentlyContinue
if (-not $resolved) {
Write-Host "目录不存在: $Dir 请先跑 build_test_upgrade_twice.bat" -ForegroundColor Red
exit 1
}
$Dir = $resolved.Path

Write-Host ''
Write-Host "=== 观测目录: $Dir ===" -ForegroundColor Cyan

# --- 1) 进程与映像路径 -------------------------------------------------
Write-Host ''
Write-Host '[1] SamWaf64 进程(关注 Path 是不是 .old)' -ForegroundColor Yellow
$procs = Get-Process -Name SamWaf64 -ErrorAction SilentlyContinue |
Where-Object { $_.Path -and $_.Path.StartsWith($Dir, 'OrdinalIgnoreCase') }
if (-not $procs) {
Write-Host ' (无) 该目录下没有运行中的 SamWaf64'
} else {
foreach ($p in $procs) {
$name = Split-Path $p.Path -Leaf
$pinned = $name -like '*.old*'
$tag = if ($pinned) { ' <== 钉住了 .old,第二次升级会失败' } else { '' }
$line = ' pid={0,-6} started={1} {2}{3}' -f $p.Id, $p.StartTime.ToString('HH:mm:ss'), $p.Path, $tag
if ($pinned) { Write-Host $line -ForegroundColor Red } else { Write-Host $line }
}
}

# --- 2) 残留文件 -------------------------------------------------------
Write-Host ''
Write-Host '[2] .SamWaf64.exe.* 残留(-Force 才看得到 HIDDEN)' -ForegroundColor Yellow
$leftovers = Get-ChildItem -Force -LiteralPath $Dir -Filter '.SamWaf64.exe.*' -ErrorAction SilentlyContinue
if (-not $leftovers) {
Write-Host ' (无)'
} else {
foreach ($f in $leftovers) {
$line = ' {0,-40} {1,12:N0} bytes attrs={2} {3}' -f `
$f.Name, $f.Length, $f.Attributes, $f.LastWriteTime.ToString('MM-dd HH:mm:ss')
Write-Host $line -ForegroundColor Red
}
}

# --- 3) 当前更新服务器指向的版本 ---------------------------------------
$manifest = Join-Path (Split-Path $Dir -Parent) 'web\samwaf_update\windows-amd64.json'
Write-Host ''
Write-Host '[3] 本地更新服务器当前发布的版本' -ForegroundColor Yellow
if (Test-Path -LiteralPath $manifest) {
$j = Get-Content -Raw -LiteralPath $manifest | ConvertFrom-Json
Write-Host (' Version={0} Desc={1} UpdateTime={2}' -f $j.Version, $j.Desc, $j.UpdateTime)
} else {
Write-Host ' (未发布)'
}

# --- 4) 可选:手工删 .old,验证"拒绝访问" -----------------------------
if ($TryDelete -and $leftovers) {
Write-Host ''
Write-Host '[4] 手工删除 .old(预期:拒绝访问)' -ForegroundColor Yellow
foreach ($f in $leftovers) {
try {
Remove-Item -Force -LiteralPath $f.FullName -ErrorAction Stop
Write-Host (' {0} -> 删除成功(说明没被进程钉住)' -f $f.Name) -ForegroundColor Green
} catch {
Write-Host (' {0} -> {1}' -f $f.Name, $_.Exception.Message) -ForegroundColor Red
}
}
}

Write-Host ''
Write-Host '判据:升级一次后若 [1] 出现红色的 .old 路径、[2] 有 HIDDEN 残留、' -ForegroundColor Cyan
Write-Host ' 且 -TryDelete 报拒绝访问,则第二次升级必然 Access is denied。' -ForegroundColor Cyan
Write-Host ''
65 changes: 65 additions & 0 deletions cmd/samwaf/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,11 @@ func (m *wafSystenService) run() {
// cron 任务调度同样是单例(避免新旧 Worker 重复执行定时任务);takeover 模式延迟到 ACTIVATE 再启动。
if !global.GWAF_RUNTIME_IS_TAKEOVER {
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler.Start()
// 独占单例已在本函数内联启动完毕,这里把 activateOnce 消费掉,
// 使后续任何 ACTIVATE 都成为空操作。Supervisor 重启后"原地认领"存活 Worker 时
// 会补发一次 ACTIVATE(它无从判断该 Worker 当初是否已接管过单例),
// 没有这道守卫就会把隧道/应用/cron 重复启动一遍。
markSingletonsActivated()
}

//脱敏处理初始化
Expand Down Expand Up @@ -1064,6 +1069,11 @@ func main() {
if isWorker, _, _, _ := parseWorkerRole(); !isWorker {
zlog.FileName = "supervisor.log"
}
// 自升级交接助手是与 Supervisor 并存的第三个短命进程,同理单独写一个日志文件,
// 免得三个进程抢同一个 supervisor.log 的滚动 rename。
if len(os.Args) > 1 && os.Args[1] == supervisor.HandoffRestartArg {
zlog.FileName = "handoff.log"
}
//初始化日志
zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, global.GWAF_LOG_OUTPUT_FORMAT)
if v := recover(); v != nil {
Expand Down Expand Up @@ -1138,6 +1148,9 @@ func main() {
ExePath: exePath,
DataDir: dataDir,
DrainTimeout: int(global.GCONFIG_RECORD_DRAIN_TIMEOUT),
// 自升级交接(Windows)靠"退出 + 助手把服务重新 start",前台运行时退出等于停服,
// 故把"是否由服务管理器托管"传下去,由 Supervisor 决定是否交接。
ServiceManaged: !service.Interactive(),
})
prg := &supervisorService{sup: sup}
s, err := service.New(prg, svcConfig)
Expand Down Expand Up @@ -1166,6 +1179,11 @@ func main() {
}
fmt.Printf("Samwaf has successfully executed the '%s' command.\n", command)
break
case supervisor.HandoffRestartArg:
// 自升级交接助手(Windows,隐藏命令,由 Supervisor 自己拉起)。
// 老 Supervisor 交接前起本进程后就退出了;本进程轮询把服务重新 start 起来,
// 让 Supervisor 换到新二进制。期间 Worker 作为孤儿继续转发业务,不中断。
runHandoffRestart(s)
case "rolling-restart": // 零停机滚动重启:通知运行中的 Supervisor 换 Worker(业务不中断),亦用于测试升级编排
if err := supervisor.TriggerUpgrade(dataDir); err != nil {
fmt.Println("滚动重启触发失败:", err)
Expand Down Expand Up @@ -1348,6 +1366,46 @@ type workerCtrl struct {
// gWorkerCtrl 当前进程的 Worker 控制客户端(仅 worker 角色下非 nil)。
var gWorkerCtrl *workerCtrl

// runHandoffRestart 是 Supervisor 自升级交接的"重启助手"(Windows 隐藏命令)。
//
// Windows 没有 execve,Supervisor 换二进制只能退出再被拉起来。这里不依赖 SCM 的
// 失败恢复动作(那是服务安装时写入的,老版本装的服务可能压根没配),而是由老
// Supervisor 先起本进程、随后自己干净退出,本进程轮询把服务 start 回来。
//
// 期间 Worker 作为孤儿继续转发业务,新 Supervisor 起来后靠 supervisor.state 里的
// ctrl 端口 + token 把它收编回去(同版本原地认领,不重起 Worker)。
func runHandoffRestart(s service.Service) {
const (
interval = 1 * time.Second
timeout = 60 * time.Second
)
zlog.Info("[交接助手] 等待旧 Supervisor 退出后把服务重新拉起...")
deadline := time.Now().Add(timeout)
sawStopped := false
for time.Now().Before(deadline) {
time.Sleep(interval)
if st, err := s.Status(); err == nil && st == service.StatusRunning {
if sawStopped {
// 服务停过又跑起来了——可能是 SCM 的失败恢复动作先我们一步。目的已达成。
zlog.Info("[交接助手] 服务已重新运行(由服务管理器先行拉起),交接完成")
return
}
// 旧 Supervisor 还没退干净,继续等。
continue
}
sawStopped = true
if err := service.Control(s, "start"); err != nil {
zlog.Debug("[交接助手] start 暂未成功,稍后重试: " + err.Error())
continue
}
zlog.Info("[交接助手] 服务已以新二进制重新启动,交接完成")
return
}
// 拉不起来不代表业务断了:Worker 仍在转发流量,只是暂时没有监护进程。
zlog.Error("[交接助手] " + timeout.String() + " 内未能把服务拉起来。Worker 仍在服务," +
"但监护进程缺席,请人工执行 SamWaf64.exe start")
}

// activateOnce 保证独占单例(应用/隧道/cron)只被接管启动一次。
var activateOnce sync.Once

Expand All @@ -1369,6 +1427,13 @@ func activateSingletons() {
})
}

// markSingletonsActivated 把 activateOnce 消费掉而不做任何事,
// 供"独占单例已在 run() 内联启动"的非 takeover 路径调用,让之后收到的
// ACTIVATE 变成空操作(幂等),避免隧道/应用/cron 被重复启动。
func markSingletonsActivated() {
activateOnce.Do(func() {})
}

// startWorkerControl 启动 Worker 控制通道客户端(后台重连循环)。
func startWorkerControl(addr, token string, prg *wafSystenService) {
gWorkerCtrl = &workerCtrl{addr: addr, token: token, prg: prg}
Expand Down
Loading
Loading