diff --git a/plugin/action/hash/normalize/token_normalizer.go b/plugin/action/hash/normalize/token_normalizer.go index 366d6f773..aadb95d5b 100644 --- a/plugin/action/hash/normalize/token_normalizer.go +++ b/plugin/action/hash/normalize/token_normalizer.go @@ -207,7 +207,12 @@ func initTokens(lexer *lexmachine.Lexer, addTokens := func(patterns []TokenPattern) { for _, p := range patterns { if p.mask == 0 || builtinPatterns&p.mask != 0 { - lexer.Add([]byte(p.RE), newToken(p.Placeholder)) + switch p.mask { + case pFilepath: + lexer.Add([]byte(p.RE), newFilepathToken(p.Placeholder)) + default: + lexer.Add([]byte(p.RE), newToken(p.Placeholder)) + } } } } @@ -263,6 +268,23 @@ func newToken(placeholder string) lexmachine.Action { } } +func newFilepathToken(placeholder string) lexmachine.Action { + return func(s *lexmachine.Scanner, m *machines.Match) (any, error) { + // skip `\w\w` + if m.TC > 0 && isWord(s.Text[m.TC-1]) || + m.TC+len(m.Bytes) < len(s.Text) && isWord(s.Text[m.TC+len(m.Bytes)]) { + s.TC = m.TC + 1 + return nil, nil + } + + return token{ + placeholder: placeholder, + begin: m.TC, + end: m.TC + len(m.Bytes), + }, nil + } +} + func (n *tokenNormalizer) normalizeByScanner(out []byte, scanner *lexmachine.Scanner) []byte { prevEnd := 0 for tokRaw, err, eos := scanner.Next(); !eos; tokRaw, err, eos = scanner.Next() { diff --git a/plugin/action/hash/normalize/token_normalizer_test.go b/plugin/action/hash/normalize/token_normalizer_test.go index 6bd51f1d9..fb932523d 100644 --- a/plugin/action/hash/normalize/token_normalizer_test.go +++ b/plugin/action/hash/normalize/token_normalizer_test.go @@ -468,9 +468,9 @@ func TestTokenNormalizerCustom(t *testing.T) { }, }, inputs: []string{ - `2006/01/02 15:04:05 error occurred, client: 10.125.172.251, upstream: "http://10.117.246.15:84/download", host: "mpm-youtube-downloader-38.name.com:84"`, + `2006/01/02 15:04:05 error occurred, client: 10.125.172.251, upstream: "http://10.117.246.15:84/download", host: "mpm-youtube-downloader-38.name.com:84", part/offset: 10117/2461584`, }, - want: " error occurred, client: , upstream: , host: ", + want: " error occurred, client: , upstream: , host: , part/offset: /", }, { name: "empty_patterns", @@ -527,6 +527,7 @@ func genBenchInput(count int) []byte { "0x13eb85e69dfbc0758b12acdaae36287d", // hex "-4.56", // float "123", // int + "truE faLse", } var sb strings.Builder