Improving normalization patterns

plugin/action/hash/normalize/token_normalizer.go

@@ -207,7 +207,12 @@ func initTokens(lexer *lexmachine.Lexer,
 	addTokens := func(patterns []TokenPattern) {
 		for _, p := range patterns {
 			if p.mask == 0 || builtinPatterns&p.mask != 0 {
-				lexer.Add([]byte(p.RE), newToken(p.Placeholder))
+				switch p.mask {
+				case pFilepath:
+					lexer.Add([]byte(p.RE), newFilepathToken(p.Placeholder))
+				default:
+					lexer.Add([]byte(p.RE), newToken(p.Placeholder))
+				}
 			}
 		}
 	}
@@ -263,6 +268,23 @@ func newToken(placeholder string) lexmachine.Action {
 	}
 }
 
+func newFilepathToken(placeholder string) lexmachine.Action {
+	return func(s *lexmachine.Scanner, m *machines.Match) (any, error) {
+		// skip `\w<match>\w`
+		if m.TC > 0 && isWord(s.Text[m.TC-1]) ||
+			m.TC+len(m.Bytes) < len(s.Text) && isWord(s.Text[m.TC+len(m.Bytes)]) {
+			s.TC = m.TC + 1
+			return nil, nil
+		}
+
+		return token{
+			placeholder: placeholder,
+			begin:       m.TC,
+			end:         m.TC + len(m.Bytes),
+		}, nil
+	}
+}
+
 func (n *tokenNormalizer) normalizeByScanner(out []byte, scanner *lexmachine.Scanner) []byte {
 	prevEnd := 0
 	for tokRaw, err, eos := scanner.Next(); !eos; tokRaw, err, eos = scanner.Next() {

plugin/action/hash/normalize/token_normalizer_test.go

@@ -468,9 +468,9 @@ func TestTokenNormalizerCustom(t *testing.T) {
 				},
 			},
 			inputs: []string{
-				`2006/01/02 15:04:05 error occurred, client: 10.125.172.251, upstream: "http://10.117.246.15:84/download", host: "mpm-youtube-downloader-38.name.com:84"`,
+				`2006/01/02 15:04:05 error occurred, client: 10.125.172.251, upstream: "http://10.117.246.15:84/download", host: "mpm-youtube-downloader-38.name.com:84", part/offset: 10117/2461584`,
 			},
-			want: "<nginx_datetime> error occurred, client: <ip>, upstream: <double_quoted>, host: <double_quoted>",
+			want: "<nginx_datetime> error occurred, client: <ip>, upstream: <double_quoted>, host: <double_quoted>, part/offset: <int>/<int>",
 		},
 		{
 			name:    "empty_patterns",
@@ -527,6 +527,7 @@ func genBenchInput(count int) []byte {
 		"0x13eb85e69dfbc0758b12acdaae36287d",                               // hex
 		"-4.56",                                                            // float
 		"123",                                                              // int
+		"truE faLse",
 	}
 
 	var sb strings.Builder