From 7a6226ba4c4228d54e3b375789d5061c09795ed0 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 27 May 2026 11:30:58 +0300 Subject: [PATCH 1/8] pupu --- config/config.example.yaml | 2 + config/config.multicluster.yaml | 2 + internal/app/config/config.go | 418 +------------------------- internal/app/config/config_v1.go | 476 ++++++++++++++++++++++++++++++ internal/app/config/config_v2.go | 491 +++++++++++++++++++++++++++++++ internal/app/config/migrate.go | 1 + 6 files changed, 975 insertions(+), 415 deletions(-) create mode 100644 internal/app/config/config_v1.go create mode 100644 internal/app/config/config_v2.go create mode 100644 internal/app/config/migrate.go diff --git a/config/config.example.yaml b/config/config.example.yaml index 604c16c..8e0a5fa 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -1,3 +1,5 @@ +version: 2 + server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/config/config.multicluster.yaml b/config/config.multicluster.yaml index 94fb312..3d1eb81 100644 --- a/config/config.multicluster.yaml +++ b/config/config.multicluster.yaml @@ -1,3 +1,5 @@ +version: 2 + server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/internal/app/config/config.go b/internal/app/config/config.go index c744b57..6bea697 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -1,12 +1,9 @@ package config import ( - "bytes" "fmt" "os" "time" - - "gopkg.in/yaml.v3" ) const ( @@ -52,424 +49,15 @@ const ( defaultClickHouseReadTimeout = 30 * time.Second ) -type Config struct { - Server *Server `yaml:"server"` - Clients *Clients `yaml:"clients"` - Handlers *Handlers `yaml:"handlers"` -} - -type CORS struct { - AllowedOrigins []string `yaml:"allowed_origins"` - AllowedMethods []string `yaml:"allowed_methods"` - AllowedHeaders []string `yaml:"allowed_headers"` - ExposedHeaders []string `yaml:"exposed_headers"` - AllowCredentials bool `yaml:"allow_credentials"` - MaxAge int `yaml:"max_age"` - OptionsPassthrough bool `yaml:"options_passthrough"` -} - -type OIDC struct { - SkipVerify bool `yaml:"skip_verify"` - AuthURLs []string `yaml:"auth_urls"` - RootCA string `yaml:"root_ca"` - CACert string `yaml:"ca_cert"` - PrivateKey string `yaml:"private_key"` - SSLSkipVerify bool `yaml:"ssl_skip_verify"` - AllowedClients []string `yaml:"allowed_clients"` - CacheSecretKey string `yaml:"cache_secret_key"` -} - -type DB struct { - Name string `yaml:"name"` - Host string `yaml:"host"` - Port int64 `yaml:"port"` - Pass string `yaml:"pass"` - User string `yaml:"user"` - RequestTimeout time.Duration `yaml:"request_timeout"` - ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` - UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` -} - -func (db *DB) ConnString() string { - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) -} - -type CH struct { - Addrs []string `yaml:"addrs"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Sharded bool `yaml:"sharded"` - DialTimeout time.Duration `yaml:"dial_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` -} - -type ( - RateLimiter struct { - RatePerSec int `yaml:"rate_per_sec"` - MaxBurst int `yaml:"max_burst"` - StoreMaxKeys int `yaml:"store_max_keys"` - PerHandler bool `yaml:"per_handler"` - } - - UserToRateLimiter map[string]RateLimiter - - ApiRateLimiters struct { - Default RateLimiter `yaml:"default"` - SpecialUsers UserToRateLimiter `yaml:"spec_users"` - } - - ApiToRateLimiters map[string]ApiRateLimiters -) - -type InmemoryCache struct { - NumCounters int64 `yaml:"num_counters"` - MaxCost int64 `yaml:"max_cost"` - BufferItems int64 `yaml:"buffer_items"` -} - -type Redis struct { - Addr string `yaml:"addr"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Timeout time.Duration `yaml:"timeout"` - MaxRetries int `yaml:"max_retries"` - MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type Cache struct { - Inmemory InmemoryCache `yaml:"inmemory"` - Redis *Redis `yaml:"redis"` -} - -type S3 struct { - Endpoint string `yaml:"endpoint"` - AccessKeyID string `yaml:"access_key_id"` - SecretAccessKey string `yaml:"secret_access_key"` - BucketName string `yaml:"bucket_name"` - EnableSSl bool `yaml:"enable_ssl"` -} - -type SeqProxyDownloader struct { - Delay time.Duration `yaml:"delay"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type SessionStore struct { - Redis Redis `yaml:"redis"` - ExportLifetime time.Duration `yaml:"export_lifetime"` -} - -type FileStore struct { - S3 *S3 `yaml:"s3"` -} - -type MassExport struct { - BatchSize uint64 `yaml:"batch_size"` - WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` - URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` - FileStore *FileStore `yaml:"file_store"` - SessionStore *SessionStore `yaml:"session_store"` - SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` -} - -type Server struct { - DebugAddr string `yaml:"debug_addr"` - HTTPAddr string `yaml:"http_addr"` - GRPCAddr string `yaml:"grpc_addr"` - CORS *CORS `yaml:"cors"` - OIDC *OIDC `yaml:"oidc"` - GRPCConnectionTimeout time.Duration `yaml:"grpc_connection_timeout"` - HTTPReadHeaderTimeout time.Duration `yaml:"http_read_header_timeout"` - HTTPReadTimeout time.Duration `yaml:"http_read_timeout"` - HTTPWriteTimeout time.Duration `yaml:"http_write_timeout"` - DB *DB `yaml:"db"` - CH *CH `yaml:"clickhouse"` - RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` - Cache Cache `yaml:"cache"` - JWTSecretKey string `yaml:"jwt_secret_key"` -} - -type GRPCKeepaliveParams struct { - // After a duration of this time if the client doesn't see any activity it - // pings the server to see if the transport is still alive. - // If set below 10s, a minimum value of 10s will be used instead. - Time time.Duration `yaml:"time"` - // After having pinged for keepalive check, the client waits for a duration - // of Timeout and if no activity is seen even after that the connection is - // closed. If set below 1s, a minimum value of 1s will be used instead. - Timeout time.Duration `yaml:"timeout"` - // If true, client sends keepalive pings even with no active RPCs. If false, - // when there are no active RPCs, Time and Timeout will be ignored and no - // keepalive pings will be sent. False by default. - PermitWithoutStream bool `yaml:"permit_without_stream"` -} - -type SeqDBClient struct { - ID string `yaml:"id"` - Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` - Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ClientMode string `yaml:"client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` -} - -type Clients struct { - SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` - SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` - SeqDBAddrs []string `yaml:"seq_db_addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ProxyClientMode string `yaml:"proxy_client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` - SeqDB []SeqDBClient `yaml:"seq_db"` -} - -type Handlers struct { - SeqAPI SeqAPI `yaml:"seq_api"` - ErrorGroups ErrorGroups `yaml:"error_groups"` - MassExport *MassExport `yaml:"mass_export"` - AsyncSearch AsyncSearch `yaml:"async_search"` -} - -type Field struct { - Name string `yaml:"name"` - Type string `yaml:"type"` -} - -type SeqAPI struct { - *SeqAPIOptions `yaml:",inline"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` -} - -type SeqAPIEnv struct { - SeqDB string `yaml:"seq_db_id"` - Options *SeqAPIOptions `yaml:"options"` -} - -type SeqAPIOptions struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` - EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` - LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` - LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` - FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` - Masking *Masking `yaml:"masking"` -} - -type Masking struct { - Masks []Mask `yaml:"masks"` - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` -} - -type Mask struct { - Re string `yaml:"re"` - Groups []int `yaml:"groups"` - Mode string `yaml:"mode"` // "mask" or "replace" or "cut" - ReplaceWord string `yaml:"replace_word"` // for mode:replace - - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` - - FieldFilters *FieldFilterSet `yaml:"field_filters"` -} - -type FieldFilter struct { - Field string `yaml:"field"` - Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" - Values []string `yaml:"values"` -} - -type FieldFilterSet struct { - Condition string `yaml:"condition"` // "and" or "or" or "not" - Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not -} - -type LogTagsMapping struct { - Release []string `yaml:"release"` - Env []string `yaml:"env"` -} - -type ErrorGroups struct { - LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` - QueryFilter map[string]string `yaml:"query_filter"` -} - -type AsyncSearch struct { - AdminUsers []string `yaml:"admin_users"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` -} - -// FromFile parse config from config path. func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + cfgBytes, err := os.ReadFile(cfgPath) if err != nil { return Config{}, fmt.Errorf("error reading file: %s", err) } - cfg, err := parse(cfgBytes) - if err != nil { - return Config{}, fmt.Errorf("error parsing file: %s", err) - } - - proxyClientMode := cfg.Clients.ProxyClientMode - if proxyClientMode == "" { - cfg.Clients.ProxyClientMode = ProxyClientModeGRPC - } else if proxyClientMode != ProxyClientModeGRPC { - return Config{}, fmt.Errorf( - "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", - proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, - ) - } - - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) - - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - } - - if cfg.Server.DB != nil && cfg.Server.DB.UsePreparedStatements == nil { - cfg.Server.DB.UsePreparedStatements = new(bool) - *cfg.Server.DB.UsePreparedStatements = true - } - - if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { - cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout - } - - if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { - cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout - } - - if cfg.Clients.GRPCKeepaliveParams != nil { - if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime - } - if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout - } - } - - if cfg.Server.Cache.Inmemory.NumCounters <= 0 { - cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters - } - if cfg.Server.Cache.Inmemory.MaxCost <= 0 { - cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost - } - if cfg.Server.Cache.Inmemory.BufferItems <= 0 { - cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems - } - - if len(cfg.Clients.SeqDB) == 0 { - defaultClient := SeqDBClient{ - ID: DefaultSeqDBClientID, - Timeout: cfg.Clients.SeqDBTimeout, - AvgDocSize: cfg.Clients.SeqDBAvgDocSize, - Addrs: cfg.Clients.SeqDBAddrs, - RequestRetries: cfg.Clients.RequestRetries, - InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, - MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, - ClientMode: cfg.Clients.ProxyClientMode, - GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, - } - cfg.Clients.SeqDB = []SeqDBClient{defaultClient} - } - - clientIDs := make(map[string]struct{}) - for _, client := range cfg.Clients.SeqDB { - if client.ID == "" { - return Config{}, fmt.Errorf("seq_db client ID cannot be empty") - } - if _, ok := clientIDs[client.ID]; ok { - return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) - } - clientIDs[client.ID] = struct{}{} - } - - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return Config{}, fmt.Errorf("default_env must be specified when using envs") - } - - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := clientIDs[envConfig.SeqDB]; !ok { - return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) - } - - if envConfig.Options == nil { - envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions - } else { - setSeqAPIOptionsDefaults(envConfig.Options) - } - - cfg.Handlers.SeqAPI.Envs[envName] = envConfig - } - } - - return cfg, nil -} - -func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { - if options.MaxAggregationsPerRequest <= 0 { - options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest - } - if options.MaxBucketsPerAggregationTs <= 0 { - options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs - } - if options.MaxParallelExportRequests <= 0 { - options.MaxParallelExportRequests = defaultMaxParallelExportRequests - } - if options.MaxSearchTotalLimit <= 0 { - options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit - } - if options.MaxSearchOffsetLimit <= 0 { - options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit - } - if options.MaxExportLimit <= 0 { - options.MaxExportLimit = defaultMaxExportLimit - } - if options.EventsCacheTTL <= 0 { - options.EventsCacheTTL = defaultEventsCacheTTL - } - if options.LogsLifespanCacheKey == "" { - options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey - } - if options.LogsLifespanCacheTTL <= 0 { - options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL - } + return fromBytes(cfgBytes) } -func parse(cfg []byte) (Config, error) { - result := Config{} - - decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) - } +func fromBytes(cfgBytes []byte) (Config, error) { - return result, nil } diff --git a/internal/app/config/config_v1.go b/internal/app/config/config_v1.go new file mode 100644 index 0000000..589e661 --- /dev/null +++ b/internal/app/config/config_v1.go @@ -0,0 +1,476 @@ +package config + +import ( + "bytes" + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + DefaultSeqDBClientID = "default" + + ProxyClientModeGRPC = "grpc" + + MaskModeMask = "mask" + MaskModeReplace = "replace" + MaskModeCut = "cut" + + FieldFilterConditionAnd = "and" + FieldFilterConditionOr = "or" + FieldFilterConditionNot = "not" + + FieldFilterModeEqual = "equal" + FieldFilterModeContains = "contains" + FieldFilterModePrefix = "prefix" + FieldFilterModeSuffix = "suffix" + + minGRPCKeepaliveTime = 10 * time.Second + minGRPCKeepaliveTimeout = 1 * time.Second + + defaultAsyncSearchListQueryLengthLimit = 1000 + + defaultMaxSearchTotalLimit = 1000000 + defaultMaxSearchOffsetLimit = 1000000 + defaultMaxExportLimit = 100000 + defaultMaxAggregationsPerRequest = 1 + defaultMaxBucketsPerAggregationTs = 200 + defaultMaxParallelExportRequests = 1 + + defaultInmemCacheNumCounters = 10000000 + defaultInmemCacheMaxCost = 1000000 + defaultInmemCacheBufferItems = 64 + + defaultEventsCacheTTL = 24 * time.Hour + + defaultLogsLifespanCacheKey = "logs_lifespan" + defaultLogsLifespanCacheTTL = 10 * time.Minute + + defaultClickHouseDialTimeout = 5 * time.Second + defaultClickHouseReadTimeout = 30 * time.Second +) + +type Config struct { + Version *int `yaml:"version,omitempty"` + Server *Server `yaml:"server"` + Clients *Clients `yaml:"clients"` + Handlers *Handlers `yaml:"handlers"` +} + +type CORS struct { + AllowedOrigins []string `yaml:"allowed_origins"` + AllowedMethods []string `yaml:"allowed_methods"` + AllowedHeaders []string `yaml:"allowed_headers"` + ExposedHeaders []string `yaml:"exposed_headers"` + AllowCredentials bool `yaml:"allow_credentials"` + MaxAge int `yaml:"max_age"` + OptionsPassthrough bool `yaml:"options_passthrough"` +} + +type OIDC struct { + SkipVerify bool `yaml:"skip_verify"` + AuthURLs []string `yaml:"auth_urls"` + RootCA string `yaml:"root_ca"` + CACert string `yaml:"ca_cert"` + PrivateKey string `yaml:"private_key"` + SSLSkipVerify bool `yaml:"ssl_skip_verify"` + AllowedClients []string `yaml:"allowed_clients"` + CacheSecretKey string `yaml:"cache_secret_key"` +} + +type DB struct { + Name string `yaml:"name"` + Host string `yaml:"host"` + Port int64 `yaml:"port"` + Pass string `yaml:"pass"` + User string `yaml:"user"` + RequestTimeout time.Duration `yaml:"request_timeout"` + ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` + UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` +} + +func (db *DB) ConnString() string { + return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) +} + +type CH struct { + Addrs []string `yaml:"addrs"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Sharded bool `yaml:"sharded"` + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` +} + +type ( + RateLimiter struct { + RatePerSec int `yaml:"rate_per_sec"` + MaxBurst int `yaml:"max_burst"` + StoreMaxKeys int `yaml:"store_max_keys"` + PerHandler bool `yaml:"per_handler"` + } + + UserToRateLimiter map[string]RateLimiter + + ApiRateLimiters struct { + Default RateLimiter `yaml:"default"` + SpecialUsers UserToRateLimiter `yaml:"spec_users"` + } + + ApiToRateLimiters map[string]ApiRateLimiters +) + +type InmemoryCache struct { + NumCounters int64 `yaml:"num_counters"` + MaxCost int64 `yaml:"max_cost"` + BufferItems int64 `yaml:"buffer_items"` +} + +type Redis struct { + Addr string `yaml:"addr"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Timeout time.Duration `yaml:"timeout"` + MaxRetries int `yaml:"max_retries"` + MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type Cache struct { + Inmemory InmemoryCache `yaml:"inmemory"` + Redis *Redis `yaml:"redis"` +} + +type S3 struct { + Endpoint string `yaml:"endpoint"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + BucketName string `yaml:"bucket_name"` + EnableSSl bool `yaml:"enable_ssl"` +} + +type SeqProxyDownloader struct { + Delay time.Duration `yaml:"delay"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type SessionStore struct { + Redis Redis `yaml:"redis"` + ExportLifetime time.Duration `yaml:"export_lifetime"` +} + +type FileStore struct { + S3 *S3 `yaml:"s3"` +} + +type MassExport struct { + BatchSize uint64 `yaml:"batch_size"` + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore *FileStore `yaml:"file_store"` + SessionStore *SessionStore `yaml:"session_store"` + SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` +} + +type Server struct { + DebugAddr string `yaml:"debug_addr"` + HTTPAddr string `yaml:"http_addr"` + GRPCAddr string `yaml:"grpc_addr"` + CORS *CORS `yaml:"cors"` + OIDC *OIDC `yaml:"oidc"` + GRPCConnectionTimeout time.Duration `yaml:"grpc_connection_timeout"` + HTTPReadHeaderTimeout time.Duration `yaml:"http_read_header_timeout"` + HTTPReadTimeout time.Duration `yaml:"http_read_timeout"` + HTTPWriteTimeout time.Duration `yaml:"http_write_timeout"` + DB *DB `yaml:"db"` + CH *CH `yaml:"clickhouse"` + RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` + Cache Cache `yaml:"cache"` + JWTSecretKey string `yaml:"jwt_secret_key"` +} + +type GRPCKeepaliveParams struct { + // After a duration of this time if the client doesn't see any activity it + // pings the server to see if the transport is still alive. + // If set below 10s, a minimum value of 10s will be used instead. + Time time.Duration `yaml:"time"` + // After having pinged for keepalive check, the client waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. If set below 1s, a minimum value of 1s will be used instead. + Timeout time.Duration `yaml:"timeout"` + // If true, client sends keepalive pings even with no active RPCs. If false, + // when there are no active RPCs, Time and Timeout will be ignored and no + // keepalive pings will be sent. False by default. + PermitWithoutStream bool `yaml:"permit_without_stream"` +} + +type SeqDBClient struct { + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ClientMode string `yaml:"client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` +} + +type Clients struct { + SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` + SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` + SeqDBAddrs []string `yaml:"seq_db_addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ProxyClientMode string `yaml:"proxy_client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` + SeqDB []SeqDBClient `yaml:"seq_db"` +} + +type Handlers struct { + SeqAPI SeqAPI `yaml:"seq_api"` + ErrorGroups ErrorGroups `yaml:"error_groups"` + MassExport *MassExport `yaml:"mass_export"` + AsyncSearch AsyncSearch `yaml:"async_search"` +} + +type Field struct { + Name string `yaml:"name"` + Type string `yaml:"type"` +} + +type SeqAPI struct { + *SeqAPIOptions `yaml:",inline"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type SeqAPIEnv struct { + SeqDB string `yaml:"seq_db_id"` + Options *SeqAPIOptions `yaml:"options"` +} + +type SeqAPIOptions struct { + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` + MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` + EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` + LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` + LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` + FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` + Masking *Masking `yaml:"masking"` +} + +type Masking struct { + Masks []Mask `yaml:"masks"` + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` +} + +type Mask struct { + Re string `yaml:"re"` + Groups []int `yaml:"groups"` + Mode string `yaml:"mode"` // "mask" or "replace" or "cut" + ReplaceWord string `yaml:"replace_word"` // for mode:replace + + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` + + FieldFilters *FieldFilterSet `yaml:"field_filters"` +} + +type FieldFilter struct { + Field string `yaml:"field"` + Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" + Values []string `yaml:"values"` +} + +type FieldFilterSet struct { + Condition string `yaml:"condition"` // "and" or "or" or "not" + Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not +} + +type LogTagsMapping struct { + Release []string `yaml:"release"` + Env []string `yaml:"env"` +} + +type ErrorGroups struct { + LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` + QueryFilter map[string]string `yaml:"query_filter"` +} + +type AsyncSearch struct { + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` +} + +// FromFile parse config from config path. +func FromFile(cfgPath string) (Config, error) { + cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + if err != nil { + return Config{}, fmt.Errorf("error reading file: %s", err) + } + + cfg, err := parse(cfgBytes) + if err != nil { + return Config{}, fmt.Errorf("error parsing file: %s", err) + } + + proxyClientMode := cfg.Clients.ProxyClientMode + if proxyClientMode == "" { + cfg.Clients.ProxyClientMode = ProxyClientModeGRPC + } else if proxyClientMode != ProxyClientModeGRPC { + return Config{}, fmt.Errorf( + "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", + proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, + ) + } + + setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + + if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + } + + if cfg.Server.DB != nil && cfg.Server.DB.UsePreparedStatements == nil { + cfg.Server.DB.UsePreparedStatements = new(bool) + *cfg.Server.DB.UsePreparedStatements = true + } + + if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { + cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout + } + + if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { + cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout + } + + if cfg.Clients.GRPCKeepaliveParams != nil { + if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { + cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + } + if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { + cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + } + } + + if cfg.Server.Cache.Inmemory.NumCounters <= 0 { + cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters + } + if cfg.Server.Cache.Inmemory.MaxCost <= 0 { + cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost + } + if cfg.Server.Cache.Inmemory.BufferItems <= 0 { + cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems + } + + if len(cfg.Clients.SeqDB) == 0 { + defaultClient := SeqDBClient{ + ID: DefaultSeqDBClientID, + Timeout: cfg.Clients.SeqDBTimeout, + AvgDocSize: cfg.Clients.SeqDBAvgDocSize, + Addrs: cfg.Clients.SeqDBAddrs, + RequestRetries: cfg.Clients.RequestRetries, + InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, + MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, + ClientMode: cfg.Clients.ProxyClientMode, + GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, + } + cfg.Clients.SeqDB = []SeqDBClient{defaultClient} + } + + clientIDs := make(map[string]struct{}) + for _, client := range cfg.Clients.SeqDB { + if client.ID == "" { + return Config{}, fmt.Errorf("seq_db client ID cannot be empty") + } + if _, ok := clientIDs[client.ID]; ok { + return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) + } + clientIDs[client.ID] = struct{}{} + } + + if len(cfg.Handlers.SeqAPI.Envs) > 0 { + if cfg.Handlers.SeqAPI.DefaultEnv == "" { + return Config{}, fmt.Errorf("default_env must be specified when using envs") + } + + if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { + return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) + } + + for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { + if _, ok := clientIDs[envConfig.SeqDB]; !ok { + return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) + } + + if envConfig.Options == nil { + envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions + } else { + setSeqAPIOptionsDefaults(envConfig.Options) + } + + cfg.Handlers.SeqAPI.Envs[envName] = envConfig + } + } + + return cfg, nil +} + +func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { + if options.MaxAggregationsPerRequest <= 0 { + options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest + } + if options.MaxBucketsPerAggregationTs <= 0 { + options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs + } + if options.MaxParallelExportRequests <= 0 { + options.MaxParallelExportRequests = defaultMaxParallelExportRequests + } + if options.MaxSearchTotalLimit <= 0 { + options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit + } + if options.MaxSearchOffsetLimit <= 0 { + options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit + } + if options.MaxExportLimit <= 0 { + options.MaxExportLimit = defaultMaxExportLimit + } + if options.EventsCacheTTL <= 0 { + options.EventsCacheTTL = defaultEventsCacheTTL + } + if options.LogsLifespanCacheKey == "" { + options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey + } + if options.LogsLifespanCacheTTL <= 0 { + options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL + } +} + +func parse(cfg []byte) (Config, error) { + result := Config{} + + decoder := yaml.NewDecoder(bytes.NewReader(cfg)) + decoder.KnownFields(true) + if err := decoder.Decode(&result); err != nil { + return result, fmt.Errorf("error parsing config: %w", err) + } + + return result, nil +} diff --git a/internal/app/config/config_v2.go b/internal/app/config/config_v2.go new file mode 100644 index 0000000..f350a4e --- /dev/null +++ b/internal/app/config/config_v2.go @@ -0,0 +1,491 @@ +package config + +import ( + "bytes" + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + DefaultSeqDBClientID = "default" + + ProxyClientModeGRPC = "grpc" + + MaskModeMask = "mask" + MaskModeReplace = "replace" + MaskModeCut = "cut" + + FieldFilterConditionAnd = "and" + FieldFilterConditionOr = "or" + FieldFilterConditionNot = "not" + + FieldFilterModeEqual = "equal" + FieldFilterModeContains = "contains" + FieldFilterModePrefix = "prefix" + FieldFilterModeSuffix = "suffix" + + minGRPCKeepaliveTime = 10 * time.Second + minGRPCKeepaliveTimeout = 1 * time.Second + + defaultAsyncSearchListQueryLengthLimit = 1000 + + defaultMaxSearchTotalLimit = 1000000 + defaultMaxSearchOffsetLimit = 1000000 + defaultMaxExportLimit = 100000 + defaultMaxAggregationsPerRequest = 1 + defaultMaxBucketsPerAggregationTs = 200 + defaultMaxParallelExportRequests = 1 + + defaultInmemCacheNumCounters = 10000000 + defaultInmemCacheMaxCost = 1000000 + defaultInmemCacheBufferItems = 64 + + defaultEventsCacheTTL = 24 * time.Hour + + defaultLogsLifespanCacheKey = "logs_lifespan" + defaultLogsLifespanCacheTTL = 10 * time.Minute + + defaultClickHouseDialTimeout = 5 * time.Second + defaultClickHouseReadTimeout = 30 * time.Second +) + +type Config struct { + Server *Server `yaml:"server"` + Clients *Clients `yaml:"clients"` + Handlers *Handlers `yaml:"handlers"` + DB *DB `yaml:"db"` +} + +type CORS struct { + AllowedOrigins []string `yaml:"allowed_origins"` + AllowedMethods []string `yaml:"allowed_methods"` + AllowedHeaders []string `yaml:"allowed_headers"` + ExposedHeaders []string `yaml:"exposed_headers"` + AllowCredentials bool `yaml:"allow_credentials"` + MaxAge int `yaml:"max_age"` + OptionsPassthrough bool `yaml:"options_passthrough"` +} + +type OIDC struct { + SkipVerify bool `yaml:"skip_verify"` + AuthURLs []string `yaml:"auth_urls"` + RootCA string `yaml:"root_ca"` + CACert string `yaml:"ca_cert"` + PrivateKey string `yaml:"private_key"` + SSLSkipVerify bool `yaml:"ssl_skip_verify"` + AllowedClients []string `yaml:"allowed_clients"` + CacheSecretKey string `yaml:"cache_secret_key"` +} + +type DB struct { + Name string `yaml:"name"` + Host string `yaml:"host"` + Port int64 `yaml:"port"` + Pass string `yaml:"pass"` + User string `yaml:"user"` + RequestTimeout time.Duration `yaml:"request_timeout"` + ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` + UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` +} + +func (db *DB) ConnString() string { + return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) +} + +type CH struct { + Addrs []string `yaml:"addrs"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Sharded bool `yaml:"sharded"` + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` +} + +type ( + RateLimiter struct { + RatePerSec int `yaml:"rate_per_sec"` + MaxBurst int `yaml:"max_burst"` + StoreMaxKeys int `yaml:"store_max_keys"` + PerHandler bool `yaml:"per_handler"` + } + + UserToRateLimiter map[string]RateLimiter + + ApiRateLimiters struct { + Default RateLimiter `yaml:"default"` + SpecialUsers UserToRateLimiter `yaml:"spec_users"` + } + + ApiToRateLimiters map[string]ApiRateLimiters +) + +type InmemoryCache struct { + NumCounters int64 `yaml:"num_counters"` + MaxCost int64 `yaml:"max_cost"` + BufferItems int64 `yaml:"buffer_items"` +} + +type Redis struct { + Addr string `yaml:"addr"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Timeout time.Duration `yaml:"timeout"` + MaxRetries int `yaml:"max_retries"` + MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type Cache struct { + Inmemory InmemoryCache `yaml:"inmemory"` + Redis *Redis `yaml:"redis"` +} + +type S3 struct { + Endpoint string `yaml:"endpoint"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + BucketName string `yaml:"bucket_name"` + EnableSSl bool `yaml:"enable_ssl"` +} + +type SeqProxyDownloader struct { + Delay time.Duration `yaml:"delay"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` +} + +type SessionStore struct { + Redis Redis `yaml:"redis"` + ExportLifetime time.Duration `yaml:"export_lifetime"` +} + +type FileStore struct { + S3 *S3 `yaml:"s3"` +} + +type MassExport struct { + BatchSize uint64 `yaml:"batch_size"` + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore *FileStore `yaml:"file_store"` + SessionStore *SessionStore `yaml:"session_store"` + SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` +} + +type HTTP struct { + Addr string `yaml:"addr"` + ReadHeaderTimeout time.Duration `yaml:"read_header_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` + WriteTimeout time.Duration `yaml:"write_timeout"` + CORS *CORS `yaml:"cors"` +} + +type GRPC struct { + Addr string `yaml:"addr"` + ConnectionTimeout time.Duration `yaml:"connection_timeout"` +} + +type Debug struct { + Addr string `yaml:"addr"` +} + +type Auth struct { + OIDC *OIDC `yaml:"oidc"` + JWTSecretKey string `yaml:"jwt_secret_key"` +} + +type Server struct { + HTTP HTTP `yaml:"http"` + GRPC GRPC `yaml:"grpc"` + Debug Debug `yaml:"debug"` + Auth Auth `yaml:"auth"` + RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` + // CH *CH `yaml:"clickhouse"` + // Cache Cache `yaml:"cache"` +} + +type GRPCKeepaliveParams struct { + // After a duration of this time if the client doesn't see any activity it + // pings the server to see if the transport is still alive. + // If set below 10s, a minimum value of 10s will be used instead. + Time time.Duration `yaml:"time"` + // After having pinged for keepalive check, the client waits for a duration + // of Timeout and if no activity is seen even after that the connection is + // closed. If set below 1s, a minimum value of 1s will be used instead. + Timeout time.Duration `yaml:"timeout"` + // If true, client sends keepalive pings even with no active RPCs. If false, + // when there are no active RPCs, Time and Timeout will be ignored and no + // keepalive pings will be sent. False by default. + PermitWithoutStream bool `yaml:"permit_without_stream"` +} + +type SeqDBClient struct { + ID string `yaml:"id"` + Timeout time.Duration `yaml:"timeout"` + AvgDocSize int `yaml:"avg_doc_size"` + Addrs []string `yaml:"addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ClientMode string `yaml:"client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` +} + +type Clients struct { + SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` + SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` + SeqDBAddrs []string `yaml:"seq_db_addrs"` + RequestRetries int `yaml:"request_retries"` + InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` + MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` + ProxyClientMode string `yaml:"proxy_client_mode"` + GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` + SeqDB []SeqDBClient `yaml:"seq_db"` +} + +type Handlers struct { + SeqAPI SeqAPI `yaml:"seq_api"` + ErrorGroups ErrorGroups `yaml:"error_groups"` + MassExport *MassExport `yaml:"mass_export"` + AsyncSearch AsyncSearch `yaml:"async_search"` +} + +type Field struct { + Name string `yaml:"name"` + Type string `yaml:"type"` +} + +type SeqAPI struct { + *SeqAPIOptions `yaml:",inline"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type SeqAPIEnv struct { + SeqDB string `yaml:"seq_db_id"` + Options *SeqAPIOptions `yaml:"options"` +} + +type SeqAPIOptions struct { + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` + MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` + EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` + LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` + LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` + FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` + Masking *Masking `yaml:"masking"` +} + +type Masking struct { + Masks []Mask `yaml:"masks"` + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` +} + +type Mask struct { + Re string `yaml:"re"` + Groups []int `yaml:"groups"` + Mode string `yaml:"mode"` // "mask" or "replace" or "cut" + ReplaceWord string `yaml:"replace_word"` // for mode:replace + + ProcessFields []string `yaml:"process_fields"` + IgnoreFields []string `yaml:"ignore_fields"` + + FieldFilters *FieldFilterSet `yaml:"field_filters"` +} + +type FieldFilter struct { + Field string `yaml:"field"` + Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" + Values []string `yaml:"values"` +} + +type FieldFilterSet struct { + Condition string `yaml:"condition"` // "and" or "or" or "not" + Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not +} + +type LogTagsMapping struct { + Release []string `yaml:"release"` + Env []string `yaml:"env"` +} + +type ErrorGroups struct { + LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` + QueryFilter map[string]string `yaml:"query_filter"` +} + +type AsyncSearch struct { + AdminUsers []string `yaml:"admin_users"` + ListQueryLengthLimit int `yaml:"list_query_length_limit"` +} + +// FromFile parse config from config path. +func FromFile(cfgPath string) (Config, error) { + cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + if err != nil { + return Config{}, fmt.Errorf("error reading file: %s", err) + } + + cfg, err := parse(cfgBytes) + if err != nil { + return Config{}, fmt.Errorf("error parsing file: %s", err) + } + + proxyClientMode := cfg.Clients.ProxyClientMode + if proxyClientMode == "" { + cfg.Clients.ProxyClientMode = ProxyClientModeGRPC + } else if proxyClientMode != ProxyClientModeGRPC { + return Config{}, fmt.Errorf( + "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", + proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, + ) + } + + setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + + if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + } + + if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { + cfg.DB.UsePreparedStatements = new(bool) + *cfg.DB.UsePreparedStatements = true + } + + if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { + cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout + } + + if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { + cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout + } + + if cfg.Clients.GRPCKeepaliveParams != nil { + if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { + cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + } + if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { + cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + } + } + + if cfg.Server.Cache.Inmemory.NumCounters <= 0 { + cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters + } + if cfg.Server.Cache.Inmemory.MaxCost <= 0 { + cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost + } + if cfg.Server.Cache.Inmemory.BufferItems <= 0 { + cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems + } + + if len(cfg.Clients.SeqDB) == 0 { + defaultClient := SeqDBClient{ + ID: DefaultSeqDBClientID, + Timeout: cfg.Clients.SeqDBTimeout, + AvgDocSize: cfg.Clients.SeqDBAvgDocSize, + Addrs: cfg.Clients.SeqDBAddrs, + RequestRetries: cfg.Clients.RequestRetries, + InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, + MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, + ClientMode: cfg.Clients.ProxyClientMode, + GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, + } + cfg.Clients.SeqDB = []SeqDBClient{defaultClient} + } + + clientIDs := make(map[string]struct{}) + for _, client := range cfg.Clients.SeqDB { + if client.ID == "" { + return Config{}, fmt.Errorf("seq_db client ID cannot be empty") + } + if _, ok := clientIDs[client.ID]; ok { + return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) + } + clientIDs[client.ID] = struct{}{} + } + + if len(cfg.Handlers.SeqAPI.Envs) > 0 { + if cfg.Handlers.SeqAPI.DefaultEnv == "" { + return Config{}, fmt.Errorf("default_env must be specified when using envs") + } + + if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { + return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) + } + + for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { + if _, ok := clientIDs[envConfig.SeqDB]; !ok { + return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) + } + + if envConfig.Options == nil { + envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions + } else { + setSeqAPIOptionsDefaults(envConfig.Options) + } + + cfg.Handlers.SeqAPI.Envs[envName] = envConfig + } + } + + return cfg, nil +} + +func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { + if options.MaxAggregationsPerRequest <= 0 { + options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest + } + if options.MaxBucketsPerAggregationTs <= 0 { + options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs + } + if options.MaxParallelExportRequests <= 0 { + options.MaxParallelExportRequests = defaultMaxParallelExportRequests + } + if options.MaxSearchTotalLimit <= 0 { + options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit + } + if options.MaxSearchOffsetLimit <= 0 { + options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit + } + if options.MaxExportLimit <= 0 { + options.MaxExportLimit = defaultMaxExportLimit + } + if options.EventsCacheTTL <= 0 { + options.EventsCacheTTL = defaultEventsCacheTTL + } + if options.LogsLifespanCacheKey == "" { + options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey + } + if options.LogsLifespanCacheTTL <= 0 { + options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL + } +} + +func parse(cfg []byte) (Config, error) { + result := Config{} + + decoder := yaml.NewDecoder(bytes.NewReader(cfg)) + decoder.KnownFields(true) + if err := decoder.Decode(&result); err != nil { + return result, fmt.Errorf("error parsing config: %w", err) + } + + return result, nil +} diff --git a/internal/app/config/migrate.go b/internal/app/config/migrate.go new file mode 100644 index 0000000..d912156 --- /dev/null +++ b/internal/app/config/migrate.go @@ -0,0 +1 @@ +package config From 61143435b375e8b043f72936040b2d3539ae83da Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Mon, 13 Jul 2026 14:22:44 +0300 Subject: [PATCH 2/8] draft config v2 --- cmd/config-migrate/main.go | 48 ++ cmd/seq-ui/main.go | 8 +- docker-compose.yml | 2 +- internal/app/config/config.go | 359 ++------------ internal/app/config/migrate.go | 1 - internal/app/config/migrate_v1_v2.go | 443 ++++++++++++++++++ .../app/config/{config_v1.go => v1/config.go} | 214 +-------- .../app/config/{config_v2.go => v2/config.go} | 273 ++++++----- 8 files changed, 694 insertions(+), 654 deletions(-) create mode 100644 cmd/config-migrate/main.go delete mode 100644 internal/app/config/migrate.go create mode 100644 internal/app/config/migrate_v1_v2.go rename internal/app/config/{config_v1.go => v1/config.go} (60%) rename internal/app/config/{config_v2.go => v2/config.go} (70%) diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go new file mode 100644 index 0000000..08fd1b8 --- /dev/null +++ b/cmd/config-migrate/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "errors" + "flag" + "os" + + "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/logger" + "go.uber.org/zap" +) + +var ( + source = flag.String("source", "", "path to the config file to migrate in place") +) + +func main() { + flag.Parse() + + run(*source) +} + +func run(source string) { + if source == "" { + logger.Fatal("missing required parameter", zap.String("param", "-source")) + } + + legacyCfg, err := os.ReadFile(source) + if err != nil { + logger.Fatal("error reading legacy config", zap.String("source", source), zap.Error(err)) + } + + backupPath := source + ".bak" + + if _, err := os.Stat(backupPath); err == nil { + logger.Fatal("backup already exists, remove it before re-running", zap.String("backup", backupPath)) + } else if !errors.Is(err, os.ErrNotExist) { + logger.Fatal("stat backup", zap.String("backup", backupPath), zap.Error(err)) + } + if err := os.WriteFile(backupPath, legacyCfg, 0644); err != nil { + logger.Fatal("error writing backup legacy config", zap.String("backup", backupPath), zap.Error(err)) + } + + backupCfg, err := config.FromFile(backupPath) + if err != nil { + logger.Fatal("read backup config file error", zap.Error(err)) + } +} diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 9450352..f587b15 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -24,7 +24,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/profiles" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" - "github.com/ozontech/seq-ui/internal/app/config" + "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/server" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" @@ -103,7 +103,7 @@ func run(ctx context.Context) { } } -func initApp(ctx context.Context, cfg config.Config) *api.Registrar { +func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { @@ -145,7 +145,7 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { } logger.Info("initializing db") - db, err := initDb(ctx, cfg.Server.DB) + db, err := initDb(ctx, cfg.DB) if err != nil { logger.Fatal("failed to init db", zap.Error(err)) } @@ -157,7 +157,7 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { dashboardsV1 *dashboards_v1.Dashboards ) if db != nil { - repo := repository.New(db, cfg.Server.DB.RequestTimeout) + repo := repository.New(db, cfg.DB.RequestTimeout) svc := service.New(repo) p = profiles.New(svc) diff --git a/docker-compose.yml b/docker-compose.yml index 37cf965..ee1ca48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.8' +version: "3.8" services: postgres: diff --git a/internal/app/config/config.go b/internal/app/config/config.go index a3b1a5b..b861a80 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -1,335 +1,72 @@ package config import ( + "bytes" "fmt" "os" - "time" + + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" + v2 "github.com/ozontech/seq-ui/internal/app/config/v2" + "gopkg.in/yaml.v3" ) const ( - DefaultSeqDBClientID = "default" - - ProxyClientModeGRPC = "grpc" - - MaskModeMask = "mask" - MaskModeReplace = "replace" - MaskModeCut = "cut" - - FieldFilterConditionAnd = "and" - FieldFilterConditionOr = "or" - FieldFilterConditionNot = "not" - - FieldFilterModeEqual = "equal" - FieldFilterModeContains = "contains" - FieldFilterModePrefix = "prefix" - FieldFilterModeSuffix = "suffix" - - minGRPCKeepaliveTime = 10 * time.Second - minGRPCKeepaliveTimeout = 1 * time.Second - - defaultAsyncSearchListQueryLengthLimit = 1000 - - defaultMaxSearchTotalLimit = 1000000 - defaultMaxSearchOffsetLimit = 1000000 - defaultMaxExportLimit = 100000 - defaultMaxAggregationsPerRequest = 1 - defaultMaxBucketsPerAggregationTs = 200 - defaultMaxParallelExportRequests = 1 - - defaultInmemCacheNumCounters = 10000000 - defaultInmemCacheMaxCost = 1000000 - defaultInmemCacheBufferItems = 64 - - defaultEventsCacheTTL = 24 * time.Hour - - defaultLogsLifespanCacheKey = "logs_lifespan" - defaultLogsLifespanCacheTTL = 10 * time.Minute - - defaultClickHouseDialTimeout = 5 * time.Second - defaultClickHouseReadTimeout = 30 * time.Second + configV1Name = "config_v1" + configV2Name = "config_v2" + previousVersion = 1 + currentVersion = 2 ) -<<<<<<< HEAD -======= -type Config struct { - Server *Server `yaml:"server"` - Clients *Clients `yaml:"clients"` - Handlers *Handlers `yaml:"handlers"` -} - -type CORS struct { - AllowedOrigins []string `yaml:"allowed_origins"` - AllowedMethods []string `yaml:"allowed_methods"` - AllowedHeaders []string `yaml:"allowed_headers"` - ExposedHeaders []string `yaml:"exposed_headers"` - AllowCredentials bool `yaml:"allow_credentials"` - MaxAge int `yaml:"max_age"` - OptionsPassthrough bool `yaml:"options_passthrough"` -} - -type OIDC struct { - SkipVerify bool `yaml:"skip_verify"` - AuthURLs []string `yaml:"auth_urls"` - RootCA string `yaml:"root_ca"` - CACert string `yaml:"ca_cert"` - PrivateKey string `yaml:"private_key"` - SSLSkipVerify bool `yaml:"ssl_skip_verify"` - AllowedClients []string `yaml:"allowed_clients"` - CacheSecretKey string `yaml:"cache_secret_key"` +type configMeta struct { + Version *int `yaml:"version"` } -type DB struct { - Name string `yaml:"name"` - Host string `yaml:"host"` - Port int64 `yaml:"port"` - Pass string `yaml:"pass"` - User string `yaml:"user"` - RequestTimeout time.Duration `yaml:"request_timeout"` - ConnectionPoolCapacity int64 `yaml:"connection_pool_capacity"` - UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` -} - -func (db *DB) ConnString() string { - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) -} - -type CH struct { - Addrs []string `yaml:"addrs"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Sharded bool `yaml:"sharded"` - DialTimeout time.Duration `yaml:"dial_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` -} - -type ( - RateLimiter struct { - RatePerSec int `yaml:"rate_per_sec"` - MaxBurst int `yaml:"max_burst"` - StoreMaxKeys int `yaml:"store_max_keys"` - PerHandler bool `yaml:"per_handler"` +// FromFile parse config from config path. +func FromFile(cfgPath string) (v2.Config, error) { + cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec + if err != nil { + return v2.Config{}, fmt.Errorf("error reading file: %s", err) } - UserToRateLimiter map[string]RateLimiter - - ApiRateLimiters struct { - Default RateLimiter `yaml:"default"` - SpecialUsers UserToRateLimiter `yaml:"spec_users"` + meta, err := parse[configMeta](cfgBytes, false) + if err != nil { + return v2.Config{}, err } - ApiToRateLimiters map[string]ApiRateLimiters -) - -type InmemoryCache struct { - NumCounters int64 `yaml:"num_counters"` - MaxCost int64 `yaml:"max_cost"` - BufferItems int64 `yaml:"buffer_items"` -} - -type Redis struct { - Addr string `yaml:"addr"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Timeout time.Duration `yaml:"timeout"` - MaxRetries int `yaml:"max_retries"` - MinRetryBackoff time.Duration `yaml:"min_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type Cache struct { - Inmemory InmemoryCache `yaml:"inmemory"` - Redis *Redis `yaml:"redis"` -} - -type S3 struct { - Endpoint string `yaml:"endpoint"` - AccessKeyID string `yaml:"access_key_id"` - SecretAccessKey string `yaml:"secret_access_key"` - BucketName string `yaml:"bucket_name"` - EnableSSl bool `yaml:"enable_ssl"` -} - -type SeqProxyDownloader struct { - Delay time.Duration `yaml:"delay"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` -} - -type SessionStore struct { - Redis Redis `yaml:"redis"` - ExportLifetime time.Duration `yaml:"export_lifetime"` -} - -type FileStore struct { - S3 *S3 `yaml:"s3"` -} - -type MassExport struct { - BatchSize uint64 `yaml:"batch_size"` - WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` - URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` - FileStore *FileStore `yaml:"file_store"` - SessionStore *SessionStore `yaml:"session_store"` - SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` -} - -type Server struct { - DebugAddr string `yaml:"debug_addr"` - HTTPAddr string `yaml:"http_addr"` - GRPCAddr string `yaml:"grpc_addr"` - CORS *CORS `yaml:"cors"` - OIDC *OIDC `yaml:"oidc"` - GRPCConnectionTimeout time.Duration `yaml:"grpc_connection_timeout"` - HTTPReadHeaderTimeout time.Duration `yaml:"http_read_header_timeout"` - HTTPReadTimeout time.Duration `yaml:"http_read_timeout"` - HTTPWriteTimeout time.Duration `yaml:"http_write_timeout"` - DB *DB `yaml:"db"` - CH *CH `yaml:"clickhouse"` - RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` - Cache Cache `yaml:"cache"` - JWTSecretKey string `yaml:"jwt_secret_key"` -} - -type GRPCKeepaliveParams struct { - // After a duration of this time if the client doesn't see any activity it - // pings the server to see if the transport is still alive. - // If set below 10s, a minimum value of 10s will be used instead. - Time time.Duration `yaml:"time"` - // After having pinged for keepalive check, the client waits for a duration - // of Timeout and if no activity is seen even after that the connection is - // closed. If set below 1s, a minimum value of 1s will be used instead. - Timeout time.Duration `yaml:"timeout"` - // If true, client sends keepalive pings even with no active RPCs. If false, - // when there are no active RPCs, Time and Timeout will be ignored and no - // keepalive pings will be sent. False by default. - PermitWithoutStream bool `yaml:"permit_without_stream"` -} - -type SeqDBClient struct { - ID string `yaml:"id"` - Timeout time.Duration `yaml:"timeout"` - AvgDocSize int `yaml:"avg_doc_size"` - Addrs []string `yaml:"addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ClientMode string `yaml:"client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` -} - -type Clients struct { - SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` - SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` - SeqDBAddrs []string `yaml:"seq_db_addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ProxyClientMode string `yaml:"proxy_client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` - SeqDB []SeqDBClient `yaml:"seq_db"` -} - -type Handlers struct { - SeqAPI SeqAPI `yaml:"seq_api"` - ErrorGroups ErrorGroups `yaml:"error_groups"` - MassExport *MassExport `yaml:"mass_export"` - AsyncSearch AsyncSearch `yaml:"async_search"` -} - -type Field struct { - Name string `yaml:"name"` - Type string `yaml:"type"` -} - -type SeqAPI struct { - *SeqAPIOptions `yaml:",inline"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` -} - -type SeqAPIEnv struct { - SeqDB string `yaml:"seq_db_id"` - Options *SeqAPIOptions `yaml:"options"` -} - -type SeqAPIOptions struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` - EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` - LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` - LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` - FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` - Masking *Masking `yaml:"masking"` -} - -type Masking struct { - Masks []Mask `yaml:"masks"` - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` -} - -type Mask struct { - Re string `yaml:"re"` - Groups []int `yaml:"groups"` - Mode string `yaml:"mode"` // "mask" or "replace" or "cut" - ReplaceWord string `yaml:"replace_word"` // for mode:replace - - ProcessFields []string `yaml:"process_fields"` - IgnoreFields []string `yaml:"ignore_fields"` - - FieldFilters *FieldFilterSet `yaml:"field_filters"` -} - -type FieldFilter struct { - Field string `yaml:"field"` - Mode string `yaml:"mode"` // "equal" or "contains" or "prefix" or "suffix" - Values []string `yaml:"values"` -} - -type FieldFilterSet struct { - Condition string `yaml:"condition"` // "and" or "or" or "not" - Filters []FieldFilter `yaml:"filters"` // max 1 if condition:not -} + cfg := v2.Config{} + switch { + case meta.Version == nil || *meta.Version == previousVersion: + cfgV1, err := parse[v1.Config](cfgBytes, true) + if err != nil { + return v2.Config{}, err + } + cfg = migrateV1ToV2(cfgV1) + case *meta.Version == currentVersion: + cfg, err = parse[v2.Config](cfgBytes, true) + if err != nil { + return v2.Config{}, err + } + default: + return v2.Config{}, fmt.Errorf("unsupported config version: %d", *meta.Version) + } -type LogTagsMapping struct { - Env []string `yaml:"env"` - Service []string `yaml:"service"` - Release []string `yaml:"release"` -} + if err := v2.Normalize(&cfg); err != nil { + return v2.Config{}, fmt.Errorf("normalize config: %w") + } -type ErrorGroups struct { - LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` - QueryFilter map[string]string `yaml:"query_filter"` + return cfg, nil } -type AsyncSearch struct { - AdminUsers []string `yaml:"admin_users"` - ListQueryLengthLimit int `yaml:"list_query_length_limit"` -} +func parse[T any](cfg []byte, strict bool) (T, error) { + var result T -// FromFile parse config from config path. ->>>>>>> main -func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) - if err != nil { - return Config{}, fmt.Errorf("error reading file: %s", err) + decoder := yaml.NewDecoder(bytes.NewReader(cfg)) + if strict { + decoder.KnownFields(true) + } + if err := decoder.Decode(&result); err != nil { + return result, fmt.Errorf("error parsing config: %w", err) } - return fromBytes(cfgBytes) -} - -func fromBytes(cfgBytes []byte) (Config, error) { - + return result, nil } diff --git a/internal/app/config/migrate.go b/internal/app/config/migrate.go deleted file mode 100644 index d912156..0000000 --- a/internal/app/config/migrate.go +++ /dev/null @@ -1 +0,0 @@ -package config diff --git a/internal/app/config/migrate_v1_v2.go b/internal/app/config/migrate_v1_v2.go new file mode 100644 index 0000000..8dc9df8 --- /dev/null +++ b/internal/app/config/migrate_v1_v2.go @@ -0,0 +1,443 @@ +package config + +import ( + v1 "github.com/ozontech/seq-ui/internal/app/config/v1" + v2 "github.com/ozontech/seq-ui/internal/app/config/v2" +) + +func migrateV1ToV2(src v1.Config) v2.Config { + dst := v2.Config{Version: currentVersion} + + dst.Server = migrateServer(src.Server) + dst.Cache = migrateCache(src.Server) + dst.Clients = migrateClients(src) + dst.DB = migrateDB(src.Server) + dst.Handlers = migrateHandlers(src.Handlers, &dst) + + return dst +} + +func migrateServer(src *v1.Server) *v2.Server { + dst := &v2.Server{ + HTTP: v2.HTTP{ + Addr: src.HTTPAddr, + ReadTimeout: src.HTTPReadTimeout, + ReadHeaderTimeout: src.HTTPReadHeaderTimeout, + WriteTimeout: src.HTTPWriteTimeout, + CORS: migrateCORS(src.CORS), + }, + GRPC: v2.GRPC{ + Addr: src.GRPCAddr, + ConnectionTimeout: src.GRPCConnectionTimeout, + }, + Debug: v2.Debug{ + Addr: src.DebugAddr, + }, + RateLimiters: migrateApiRateLimiters(src.RateLimiters), + } + + if src.OIDC != nil { + dst.Auth = &v2.Auth{ + OIDC: &v2.OIDC{ + SkipVerify: src.OIDC.SkipVerify, + AuthURLs: src.OIDC.AuthURLs, + RootCA: src.OIDC.RootCA, + CACert: src.OIDC.CACert, + PrivateKey: src.OIDC.PrivateKey, + SSLSkipVerify: src.OIDC.SSLSkipVerify, + AllowedClients: src.OIDC.AllowedClients, + CacheSecretKey: src.OIDC.CacheSecretKey, + }, + } + } + + if src.JWTSecretKey != "" { + dst.Auth.JWT = &v2.JWT{SecretKey: src.JWTSecretKey} + } + + return dst +} + +func migrateCORS(src *v1.CORS) *v2.CORS { + if src == nil { + return nil + } + + return &v2.CORS{ + AllowedOrigins: src.AllowedOrigins, + AllowedMethods: src.AllowedMethods, + AllowedHeaders: src.AllowedHeaders, + ExposedHeaders: src.ExposedHeaders, + AllowCredentials: src.AllowCredentials, + MaxAge: src.MaxAge, + OptionsPassthrough: src.OptionsPassthrough, + } +} + +func migrateApiRateLimiters(src v1.ApiToRateLimiters) v2.ApiToRateLimiters { + if src == nil { + return nil + } + + dst := make(v2.ApiToRateLimiters, len(src)) + for api, rl := range src { + dst[api] = v2.ApiRateLimiters{ + Default: v2.RateLimiter{ + RatePerSec: rl.Default.RatePerSec, + MaxBurst: rl.Default.MaxBurst, + StoreMaxKeys: rl.Default.StoreMaxKeys, + PerHandler: rl.Default.PerHandler, + }, + SpecialUsers: migrateUserToRateLimiter(rl.SpecialUsers), + } + } + + return dst +} + +func migrateUserToRateLimiter(src v1.UserToRateLimiter) v2.UserToRateLimiter { + if src == nil { + return nil + } + + dst := make(v2.UserToRateLimiter, len(src)) + for k, rl := range src { + dst[k] = v2.RateLimiter{ + RatePerSec: rl.RatePerSec, + MaxBurst: rl.MaxBurst, + StoreMaxKeys: rl.StoreMaxKeys, + PerHandler: rl.PerHandler, + } + } + + return dst +} + +func migrateGRPCKeepaliveParams(src *v1.GRPCKeepaliveParams) *v2.GRPCKeepaliveParams { + if src == nil { + return nil + } + + return &v2.GRPCKeepaliveParams{ + Time: src.Time, + Timeout: src.Timeout, + PermitWithoutStream: src.PermitWithoutStream, + } +} + +func migrateClients(src v1.Config) *v2.Clients { + dst := &v2.Clients{} + + if src.Clients != nil { + if len(src.Clients.SeqDB) > 0 { + dst.SeqDB = make([]v2.SeqDBClient, 0, len(src.Clients.SeqDB)) + for _, s := range src.Clients.SeqDB { + dst.SeqDB = append(dst.SeqDB, migrateSeqDBClient(s)) + } + } else { + dst.SeqDB = []v2.SeqDBClient{{ + ID: v2.DefaultSeqDBClientID, + Timeout: src.Clients.SeqDBTimeout, + AvgDocSize: src.Clients.SeqDBAvgDocSize, + Addrs: src.Clients.SeqDBAddrs, + RequestRetries: src.Clients.RequestRetries, + InitialRetryBackoff: src.Clients.InitialRetryBackoff, + MaxRetryBackoff: src.Clients.MaxRetryBackoff, + ClientMode: src.Clients.ProxyClientMode, + GRPCKeepaliveParams: migrateGRPCKeepaliveParams(src.Clients.GRPCKeepaliveParams), + }} + } + } + + if src.Server != nil && src.Server.CH != nil { + ch := src.Server.CH + dst.ClickHouse = []v2.CHClient{{ + ID: v2.DefaultSeqDBClientID, + Addrs: ch.Addrs, + Database: ch.Database, + Username: ch.Username, + Password: ch.Password, + Sharded: ch.Sharded, + DialTimeout: ch.DialTimeout, + ReadTimeout: ch.ReadTimeout, + }} + } + + return dst +} + +func migrateSeqDBClient(src v1.SeqDBClient) v2.SeqDBClient { + return v2.SeqDBClient{ + ID: src.ID, + Timeout: src.Timeout, + AvgDocSize: src.AvgDocSize, + Addrs: src.Addrs, + RequestRetries: src.RequestRetries, + InitialRetryBackoff: src.InitialRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + ClientMode: src.ClientMode, + GRPCKeepaliveParams: migrateGRPCKeepaliveParams(src.GRPCKeepaliveParams), + } +} + +func migrateDB(src *v1.Server) *v2.DB { + if src == nil || src.DB == nil { + return nil + } + + return &v2.DB{ + Name: src.DB.Name, + Host: src.DB.Host, + Port: src.DB.Port, + Pass: src.DB.Pass, + User: src.DB.User, + RequestTimeout: src.DB.RequestTimeout, + ConnectionPoolCapacity: src.DB.ConnectionPoolCapacity, + UsePreparedStatements: src.DB.UsePreparedStatements, + } +} + +func migrateCache(src *v1.Server) *v2.Cache { + if src == nil { + return &v2.Cache{} + } + + cache := &v2.Cache{} + cache.Inmemory = append(cache.Inmemory, v2.InmemoryCache{ + ID: v2.DefaultInmemCacheID, + NumCounters: src.Cache.Inmemory.NumCounters, + MaxCost: src.Cache.Inmemory.MaxCost, + BufferItems: src.Cache.Inmemory.BufferItems, + }) + + if src.Cache.Redis != nil { + cache.Redis = append(cache.Redis, migrateRedis(src.Cache.Redis, v2.DefaultRedisID, v2.DefaultInmemCacheID)) + } + + return cache +} + +func migrateRedis(src *v1.Redis, id, withInmemID string) v2.Redis { + return v2.Redis{ + ID: id, + WithInmemID: withInmemID, + Addr: src.Addr, + Username: src.Username, + Password: src.Password, + Timeout: src.Timeout, + MaxRetries: src.MaxRetries, + MinRetryBackoff: src.MinRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + } +} + +func migrateHandlers(src *v1.Handlers, cfg *v2.Config) *v2.Handlers { + if src == nil { + return &v2.Handlers{} + } + + dst := &v2.Handlers{ + SeqAPI: migrateSeqAPI(src.SeqAPI), + ErrorGroups: migrateErrorGroups(src.ErrorGroups), + AsyncSearch: migrateAsyncSearch(src.AsyncSearch), + } + + if src.MassExport != nil { + dst.MassExport = migrateMassExport(src.MassExport, cfg) + } + + return dst +} + +func migrateSeqAPI(src v1.SeqAPI) v2.SeqAPI { + return v2.SeqAPI{ + SeqAPIOptions: migrateSeqAPIOptions(src.SeqAPIOptions), + Envs: migrateSeqAPIEnvs(src.Envs), + DefaultEnv: src.DefaultEnv, + } +} + +func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { + if len(envs) == 0 { + return nil + } + + dst := make(map[string]v2.SeqAPIEnv, len(envs)) + for name, cfg := range envs { + dst[name] = v2.SeqAPIEnv{ + SeqDB: cfg.SeqDB, + Options: migrateSeqAPIOptions(cfg.Options), + } + } + + return dst +} + +func migrateSeqAPIOptions(options *v1.SeqAPIOptions) *v2.SeqAPIOptions { + if options == nil { + return nil + } + + return &v2.SeqAPIOptions{ + MaxSearchLimit: options.MaxSearchLimit, + MaxSearchTotalLimit: options.MaxSearchTotalLimit, + MaxSearchOffsetLimit: options.MaxSearchOffsetLimit, + MaxExportLimit: options.MaxExportLimit, + SeqCLIMaxSearchLimit: options.SeqCLIMaxSearchLimit, + MaxParallelExportRequests: options.MaxParallelExportRequests, + MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, + MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, + EventsCacheTTL: options.EventsCacheTTL, + PinnedFields: migrateFields(options.PinnedFields), + SystemFields: migrateFields(options.SystemFields), + LogsLifespanCacheKey: options.LogsLifespanCacheKey, + LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, + FieldsCacheTTL: options.FieldsCacheTTL, + Masking: migrateMasking(options.Masking), + } +} + +func migrateFields(fs []v1.Field) []v2.Field { + if fs == nil { + return nil + } + + dst := make([]v2.Field, len(fs)) + for i, f := range fs { + dst[i] = v2.Field{ + Name: f.Name, + Type: f.Type, + } + } + + return dst +} + +func migrateMasking(src *v1.Masking) *v2.Masking { + if src == nil { + return nil + } + + return &v2.Masking{ + Masks: migrateMasks(src.Masks), + ProcessFields: src.ProcessFields, + IgnoreFields: src.IgnoreFields, + } +} + +func migrateMasks(ms []v1.Mask) []v2.Mask { + if ms == nil { + return nil + } + + dst := make([]v2.Mask, len(ms)) + for i, m := range ms { + dst[i] = v2.Mask{ + Re: m.Re, + Groups: m.Groups, + Mode: m.Mode, + ReplaceWord: m.ReplaceWord, + ProcessFields: m.ProcessFields, + IgnoreFields: m.IgnoreFields, + FieldFilters: migrateFieldFilters(m.FieldFilters), + } + } + + return dst +} + +func migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { + if src == nil { + return nil + } + + dst := &v2.FieldFilterSet{ + Condition: src.Condition, + } + + if src.Filters != nil { + dst.Filters = make([]v2.FieldFilter, len(src.Filters)) + for i, f := range src.Filters { + dst.Filters[i] = v2.FieldFilter{ + Field: f.Field, + Mode: f.Mode, + Values: f.Values, + } + } + } + + return dst +} + +func migrateErrorGroups(eg v1.ErrorGroups) v2.ErrorGroups { + return v2.ErrorGroups{ + LogTagsMapping: v2.LogTagsMapping{ + Env: eg.LogTagsMapping.Env, + Service: eg.LogTagsMapping.Service, + Release: eg.LogTagsMapping.Release, + }, + QueryFilter: eg.QueryFilter, + } +} + +func migrateAsyncSearch(a v1.AsyncSearch) v2.AsyncSearch { + return v2.AsyncSearch{ + AdminUsers: a.AdminUsers, + ListQueryLengthLimit: a.ListQueryLengthLimit, + } +} + +func migrateMassExport(me *v1.MassExport, cfg *v2.Config) *v2.MassExport { + dst := &v2.MassExport{ + BatchSize: me.BatchSize, + WorkersCount: me.WorkersCount, + TasksChannelSize: me.TasksChannelSize, + PartLength: me.PartLength, + URLPrefix: me.URLPrefix, + AllowedUsers: me.AllowedUsers, + FileStore: migrateFileStore(me.FileStore), + DownloadParams: migrateDownloadParams(me.SeqProxyDownloader), + } + + if me.SessionStore != nil { + cfg.Cache.Redis = append(cfg.Cache.Redis, migrateRedis(&me.SessionStore.Redis, v2.DefaultMassExportRedisID, "")) + + dst.SessionStore = &v2.SessionStore{ + RedisID: v2.DefaultMassExportRedisID, + ExportLifetime: me.SessionStore.ExportLifetime, + } + } + + return dst +} + +func migrateFileStore(fs *v1.FileStore) *v2.FileStore { + if fs == nil { + return nil + } + + dst := &v2.FileStore{} + if fs.S3 != nil { + dst.S3 = &v2.S3{ + Endpoint: fs.S3.Endpoint, + AccessKeyID: fs.S3.AccessKeyID, + SecretAccessKey: fs.S3.SecretAccessKey, + BucketName: fs.S3.BucketName, + EnableSSl: fs.S3.EnableSSl, + } + } + + return dst +} + +func migrateDownloadParams(src *v1.SeqProxyDownloader) *v2.DownloadParams { + if src == nil { + return nil + } + + return &v2.DownloadParams{ + Delay: src.Delay, + InitialRetryBackoff: src.InitialRetryBackoff, + MaxRetryBackoff: src.MaxRetryBackoff, + } +} diff --git a/internal/app/config/config_v1.go b/internal/app/config/v1/config.go similarity index 60% rename from internal/app/config/config_v1.go rename to internal/app/config/v1/config.go index 589e661..9851298 100644 --- a/internal/app/config/config_v1.go +++ b/internal/app/config/v1/config.go @@ -1,59 +1,11 @@ -package config +package v1 import ( - "bytes" - "fmt" - "os" "time" - - "gopkg.in/yaml.v3" -) - -const ( - DefaultSeqDBClientID = "default" - - ProxyClientModeGRPC = "grpc" - - MaskModeMask = "mask" - MaskModeReplace = "replace" - MaskModeCut = "cut" - - FieldFilterConditionAnd = "and" - FieldFilterConditionOr = "or" - FieldFilterConditionNot = "not" - - FieldFilterModeEqual = "equal" - FieldFilterModeContains = "contains" - FieldFilterModePrefix = "prefix" - FieldFilterModeSuffix = "suffix" - - minGRPCKeepaliveTime = 10 * time.Second - minGRPCKeepaliveTimeout = 1 * time.Second - - defaultAsyncSearchListQueryLengthLimit = 1000 - - defaultMaxSearchTotalLimit = 1000000 - defaultMaxSearchOffsetLimit = 1000000 - defaultMaxExportLimit = 100000 - defaultMaxAggregationsPerRequest = 1 - defaultMaxBucketsPerAggregationTs = 200 - defaultMaxParallelExportRequests = 1 - - defaultInmemCacheNumCounters = 10000000 - defaultInmemCacheMaxCost = 1000000 - defaultInmemCacheBufferItems = 64 - - defaultEventsCacheTTL = 24 * time.Hour - - defaultLogsLifespanCacheKey = "logs_lifespan" - defaultLogsLifespanCacheTTL = 10 * time.Minute - - defaultClickHouseDialTimeout = 5 * time.Second - defaultClickHouseReadTimeout = 30 * time.Second ) type Config struct { - Version *int `yaml:"version,omitempty"` + Version *int `yaml:"version"` Server *Server `yaml:"server"` Clients *Clients `yaml:"clients"` Handlers *Handlers `yaml:"handlers"` @@ -91,10 +43,6 @@ type DB struct { UsePreparedStatements *bool `yaml:"use_prepared_statements,omitempty"` } -func (db *DB) ConnString() string { - return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) -} - type CH struct { Addrs []string `yaml:"addrs"` Database string `yaml:"database"` @@ -306,8 +254,9 @@ type FieldFilterSet struct { } type LogTagsMapping struct { - Release []string `yaml:"release"` Env []string `yaml:"env"` + Service []string `yaml:"service"` + Release []string `yaml:"release"` } type ErrorGroups struct { @@ -319,158 +268,3 @@ type AsyncSearch struct { AdminUsers []string `yaml:"admin_users"` ListQueryLengthLimit int `yaml:"list_query_length_limit"` } - -// FromFile parse config from config path. -func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec - if err != nil { - return Config{}, fmt.Errorf("error reading file: %s", err) - } - - cfg, err := parse(cfgBytes) - if err != nil { - return Config{}, fmt.Errorf("error parsing file: %s", err) - } - - proxyClientMode := cfg.Clients.ProxyClientMode - if proxyClientMode == "" { - cfg.Clients.ProxyClientMode = ProxyClientModeGRPC - } else if proxyClientMode != ProxyClientModeGRPC { - return Config{}, fmt.Errorf( - "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", - proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, - ) - } - - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) - - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit - } - - if cfg.Server.DB != nil && cfg.Server.DB.UsePreparedStatements == nil { - cfg.Server.DB.UsePreparedStatements = new(bool) - *cfg.Server.DB.UsePreparedStatements = true - } - - if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { - cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout - } - - if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { - cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout - } - - if cfg.Clients.GRPCKeepaliveParams != nil { - if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime - } - if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout - } - } - - if cfg.Server.Cache.Inmemory.NumCounters <= 0 { - cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters - } - if cfg.Server.Cache.Inmemory.MaxCost <= 0 { - cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost - } - if cfg.Server.Cache.Inmemory.BufferItems <= 0 { - cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems - } - - if len(cfg.Clients.SeqDB) == 0 { - defaultClient := SeqDBClient{ - ID: DefaultSeqDBClientID, - Timeout: cfg.Clients.SeqDBTimeout, - AvgDocSize: cfg.Clients.SeqDBAvgDocSize, - Addrs: cfg.Clients.SeqDBAddrs, - RequestRetries: cfg.Clients.RequestRetries, - InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, - MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, - ClientMode: cfg.Clients.ProxyClientMode, - GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, - } - cfg.Clients.SeqDB = []SeqDBClient{defaultClient} - } - - clientIDs := make(map[string]struct{}) - for _, client := range cfg.Clients.SeqDB { - if client.ID == "" { - return Config{}, fmt.Errorf("seq_db client ID cannot be empty") - } - if _, ok := clientIDs[client.ID]; ok { - return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) - } - clientIDs[client.ID] = struct{}{} - } - - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return Config{}, fmt.Errorf("default_env must be specified when using envs") - } - - if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) - } - - for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := clientIDs[envConfig.SeqDB]; !ok { - return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) - } - - if envConfig.Options == nil { - envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions - } else { - setSeqAPIOptionsDefaults(envConfig.Options) - } - - cfg.Handlers.SeqAPI.Envs[envName] = envConfig - } - } - - return cfg, nil -} - -func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { - if options.MaxAggregationsPerRequest <= 0 { - options.MaxAggregationsPerRequest = defaultMaxAggregationsPerRequest - } - if options.MaxBucketsPerAggregationTs <= 0 { - options.MaxBucketsPerAggregationTs = defaultMaxBucketsPerAggregationTs - } - if options.MaxParallelExportRequests <= 0 { - options.MaxParallelExportRequests = defaultMaxParallelExportRequests - } - if options.MaxSearchTotalLimit <= 0 { - options.MaxSearchTotalLimit = defaultMaxSearchTotalLimit - } - if options.MaxSearchOffsetLimit <= 0 { - options.MaxSearchOffsetLimit = defaultMaxSearchOffsetLimit - } - if options.MaxExportLimit <= 0 { - options.MaxExportLimit = defaultMaxExportLimit - } - if options.EventsCacheTTL <= 0 { - options.EventsCacheTTL = defaultEventsCacheTTL - } - if options.LogsLifespanCacheKey == "" { - options.LogsLifespanCacheKey = defaultLogsLifespanCacheKey - } - if options.LogsLifespanCacheTTL <= 0 { - options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL - } -} - -func parse(cfg []byte) (Config, error) { - result := Config{} - - decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) - } - - return result, nil -} diff --git a/internal/app/config/config_v2.go b/internal/app/config/v2/config.go similarity index 70% rename from internal/app/config/config_v2.go rename to internal/app/config/v2/config.go index f350a4e..882c00a 100644 --- a/internal/app/config/config_v2.go +++ b/internal/app/config/v2/config.go @@ -1,16 +1,15 @@ -package config +package v2 import ( - "bytes" "fmt" - "os" "time" - - "gopkg.in/yaml.v3" ) const ( - DefaultSeqDBClientID = "default" + DefaultSeqDBClientID = "default" + DefaultInmemCacheID = "seqapi" + DefaultRedisID = "default" + DefaultMassExportRedisID = "mass_export" ProxyClientModeGRPC = "grpc" @@ -53,10 +52,12 @@ const ( ) type Config struct { + Version int `yaml:"version"` Server *Server `yaml:"server"` Clients *Clients `yaml:"clients"` Handlers *Handlers `yaml:"handlers"` DB *DB `yaml:"db"` + Cache *Cache `yaml:"cache"` } type CORS struct { @@ -95,16 +96,6 @@ func (db *DB) ConnString() string { return fmt.Sprintf("host=%s port=%d dbname=%s user=%s password=%s pool_max_conns=%d", db.Host, db.Port, db.Name, db.User, db.Pass, db.ConnectionPoolCapacity) } -type CH struct { - Addrs []string `yaml:"addrs"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - Sharded bool `yaml:"sharded"` - DialTimeout time.Duration `yaml:"dial_timeout"` - ReadTimeout time.Duration `yaml:"read_timeout"` -} - type ( RateLimiter struct { RatePerSec int `yaml:"rate_per_sec"` @@ -124,12 +115,15 @@ type ( ) type InmemoryCache struct { - NumCounters int64 `yaml:"num_counters"` - MaxCost int64 `yaml:"max_cost"` - BufferItems int64 `yaml:"buffer_items"` + ID string `yaml:"id"` + NumCounters int64 `yaml:"num_counters"` + MaxCost int64 `yaml:"max_cost"` + BufferItems int64 `yaml:"buffer_items"` } type Redis struct { + ID string `yaml:"id"` + WithInmemID string `yaml:"with_inmem_id"` Addr string `yaml:"addr"` Username string `yaml:"username"` Password string `yaml:"password"` @@ -140,8 +134,8 @@ type Redis struct { } type Cache struct { - Inmemory InmemoryCache `yaml:"inmemory"` - Redis *Redis `yaml:"redis"` + Inmemory []InmemoryCache `yaml:"inmemory"` + Redis []Redis `yaml:"redis"` } type S3 struct { @@ -152,14 +146,14 @@ type S3 struct { EnableSSl bool `yaml:"enable_ssl"` } -type SeqProxyDownloader struct { +type DownloadParams struct { Delay time.Duration `yaml:"delay"` InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` } type SessionStore struct { - Redis Redis `yaml:"redis"` + RedisID string `yaml:"redis_id"` ExportLifetime time.Duration `yaml:"export_lifetime"` } @@ -168,15 +162,15 @@ type FileStore struct { } type MassExport struct { - BatchSize uint64 `yaml:"batch_size"` - WorkersCount int `yaml:"workers_count"` - TasksChannelSize int `yaml:"tasks_channel_size"` - PartLength time.Duration `yaml:"part_length"` - URLPrefix string `yaml:"url_prefix"` - AllowedUsers []string `yaml:"allowed_users"` - FileStore *FileStore `yaml:"file_store"` - SessionStore *SessionStore `yaml:"session_store"` - SeqProxyDownloader *SeqProxyDownloader `yaml:"seq_proxy_downloader"` + BatchSize uint64 `yaml:"batch_size"` + WorkersCount int `yaml:"workers_count"` + TasksChannelSize int `yaml:"tasks_channel_size"` + PartLength time.Duration `yaml:"part_length"` + URLPrefix string `yaml:"url_prefix"` + AllowedUsers []string `yaml:"allowed_users"` + FileStore *FileStore `yaml:"file_store"` + SessionStore *SessionStore `yaml:"session_store"` + DownloadParams *DownloadParams `yaml:"download_params"` } type HTTP struct { @@ -196,19 +190,21 @@ type Debug struct { Addr string `yaml:"addr"` } +type JWT struct { + SecretKey string `yaml:"secret_key"` +} + type Auth struct { - OIDC *OIDC `yaml:"oidc"` - JWTSecretKey string `yaml:"jwt_secret_key"` + OIDC *OIDC `yaml:"oidc"` + JWT *JWT `yaml:"jwt"` } type Server struct { HTTP HTTP `yaml:"http"` GRPC GRPC `yaml:"grpc"` Debug Debug `yaml:"debug"` - Auth Auth `yaml:"auth"` + Auth *Auth `yaml:"auth"` RateLimiters ApiToRateLimiters `yaml:"rate_limiters"` - // CH *CH `yaml:"clickhouse"` - // Cache Cache `yaml:"cache"` } type GRPCKeepaliveParams struct { @@ -236,18 +232,23 @@ type SeqDBClient struct { MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` ClientMode string `yaml:"client_mode"` GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` + DownloadParams *DownloadParams `yaml:"download_params"` +} + +type CHClient struct { + ID string `yaml:"id"` + Addrs []string `yaml:"addrs"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Sharded bool `yaml:"sharded"` + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` } type Clients struct { - SeqDBTimeout time.Duration `yaml:"seq_db_timeout"` - SeqDBAvgDocSize int `yaml:"seq_db_avg_doc_size"` - SeqDBAddrs []string `yaml:"seq_db_addrs"` - RequestRetries int `yaml:"request_retries"` - InitialRetryBackoff time.Duration `yaml:"initial_retry_backoff"` - MaxRetryBackoff time.Duration `yaml:"max_retry_backoff"` - ProxyClientMode string `yaml:"proxy_client_mode"` - GRPCKeepaliveParams *GRPCKeepaliveParams `yaml:"grpc_keepalive_params"` - SeqDB []SeqDBClient `yaml:"seq_db"` + SeqDB []SeqDBClient `yaml:"seq_db"` + ClickHouse []CHClient `yaml:"clickhouse"` } type Handlers struct { @@ -322,6 +323,7 @@ type FieldFilterSet struct { type LogTagsMapping struct { Release []string `yaml:"release"` + Service []string `yaml:"service"` Env []string `yaml:"env"` } @@ -335,104 +337,124 @@ type AsyncSearch struct { ListQueryLengthLimit int `yaml:"list_query_length_limit"` } -// FromFile parse config from config path. -func FromFile(cfgPath string) (Config, error) { - cfgBytes, err := os.ReadFile(cfgPath) //nolint:gosec - if err != nil { - return Config{}, fmt.Errorf("error reading file: %s", err) +func Normalize(cfg *Config) error { + if len(cfg.Clients.SeqDB) == 0 { + return fmt.Errorf("clients.seq_db must contain at least one client") } - cfg, err := parse(cfgBytes) - if err != nil { - return Config{}, fmt.Errorf("error parsing file: %s", err) - } + seqDBIDs := make(map[string]struct{}, len(cfg.Clients.SeqDB)) + for i := range cfg.Clients.SeqDB { + c := &cfg.Clients.SeqDB[i] + if c.ID == "" { + return fmt.Errorf("seq_db client ID cannot be empty") + } + if _, ok := seqDBIDs[c.ID]; ok { + return fmt.Errorf("duplicate seq_db client ID: %s", c.ID) + } - proxyClientMode := cfg.Clients.ProxyClientMode - if proxyClientMode == "" { - cfg.Clients.ProxyClientMode = ProxyClientModeGRPC - } else if proxyClientMode != ProxyClientModeGRPC { - return Config{}, fmt.Errorf( - "invalid value for clients.proxy_client_mode: %q. Allowed values are empty string (defaults to %q) or %q", - proxyClientMode, ProxyClientModeGRPC, ProxyClientModeGRPC, - ) - } + seqDBIDs[c.ID] = struct{}{} - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + if c.ClientMode == "" { + c.ClientMode = ProxyClientModeGRPC + } else if c.ClientMode != ProxyClientModeGRPC { + return fmt.Errorf("invalid clients.seq_db[%s].client_mode: %q (allowed: %q)", c.ID, c.ClientMode, ProxyClientModeGRPC) + } - if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { - cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit + if c.GRPCKeepaliveParams != nil { + if c.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { + c.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + } + if c.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { + c.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + } + } } - if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { - cfg.DB.UsePreparedStatements = new(bool) - *cfg.DB.UsePreparedStatements = true - } + chIDs := make(map[string]struct{}, len(cfg.Clients.ClickHouse)) + for i := range cfg.Clients.ClickHouse { + ch := &cfg.Clients.ClickHouse[i] + if ch.ID == "" { + return fmt.Errorf("clickhouse client ID cannot be empty") + } + if _, ok := chIDs[ch.ID]; ok { + return fmt.Errorf("duplicate clickhouse client ID: %s", ch.ID) + } - if cfg.Server.CH != nil && cfg.Server.CH.DialTimeout <= 0 { - cfg.Server.CH.DialTimeout = defaultClickHouseDialTimeout - } + chIDs[ch.ID] = struct{}{} - if cfg.Server.CH != nil && cfg.Server.CH.ReadTimeout <= 0 { - cfg.Server.CH.ReadTimeout = defaultClickHouseReadTimeout + if ch.DialTimeout <= 0 { + ch.DialTimeout = defaultClickHouseDialTimeout + } + if ch.ReadTimeout <= 0 { + ch.ReadTimeout = defaultClickHouseReadTimeout + } } - if cfg.Clients.GRPCKeepaliveParams != nil { - if cfg.Clients.GRPCKeepaliveParams.Time < minGRPCKeepaliveTime { - cfg.Clients.GRPCKeepaliveParams.Time = minGRPCKeepaliveTime + inmemIDs := make(map[string]struct{}, len(cfg.Cache.Inmemory)) + for i := range cfg.Cache.Inmemory { + inm := &cfg.Cache.Inmemory[i] + if inm.ID == "" { + return fmt.Errorf("inmemory cache ID cannot be empty") } - if cfg.Clients.GRPCKeepaliveParams.Timeout < minGRPCKeepaliveTimeout { - cfg.Clients.GRPCKeepaliveParams.Timeout = minGRPCKeepaliveTimeout + if _, ok := inmemIDs[inm.ID]; ok { + return fmt.Errorf("duplicate inmemory cache ID: %s", inm.ID) } - } - if cfg.Server.Cache.Inmemory.NumCounters <= 0 { - cfg.Server.Cache.Inmemory.NumCounters = defaultInmemCacheNumCounters - } - if cfg.Server.Cache.Inmemory.MaxCost <= 0 { - cfg.Server.Cache.Inmemory.MaxCost = defaultInmemCacheMaxCost - } - if cfg.Server.Cache.Inmemory.BufferItems <= 0 { - cfg.Server.Cache.Inmemory.BufferItems = defaultInmemCacheBufferItems - } + inmemIDs[inm.ID] = struct{}{} - if len(cfg.Clients.SeqDB) == 0 { - defaultClient := SeqDBClient{ - ID: DefaultSeqDBClientID, - Timeout: cfg.Clients.SeqDBTimeout, - AvgDocSize: cfg.Clients.SeqDBAvgDocSize, - Addrs: cfg.Clients.SeqDBAddrs, - RequestRetries: cfg.Clients.RequestRetries, - InitialRetryBackoff: cfg.Clients.InitialRetryBackoff, - MaxRetryBackoff: cfg.Clients.MaxRetryBackoff, - ClientMode: cfg.Clients.ProxyClientMode, - GRPCKeepaliveParams: cfg.Clients.GRPCKeepaliveParams, + if inm.NumCounters <= 0 { + inm.NumCounters = defaultInmemCacheNumCounters + } + if inm.MaxCost <= 0 { + inm.MaxCost = defaultInmemCacheMaxCost + } + if inm.BufferItems <= 0 { + inm.BufferItems = defaultInmemCacheBufferItems } - cfg.Clients.SeqDB = []SeqDBClient{defaultClient} } - clientIDs := make(map[string]struct{}) - for _, client := range cfg.Clients.SeqDB { - if client.ID == "" { - return Config{}, fmt.Errorf("seq_db client ID cannot be empty") + redisIDs := make(map[string]struct{}, len(cfg.Cache.Redis)) + for i := range cfg.Cache.Redis { + r := &cfg.Cache.Redis[i] + if r.ID == "" { + return fmt.Errorf("redis cache ID cannot be empty") } - if _, ok := clientIDs[client.ID]; ok { - return Config{}, fmt.Errorf("duplicate seq_db client ID: %s", client.ID) + if _, ok := inmemIDs[r.ID]; ok { + return fmt.Errorf("duplicate redis cache ID: %s", r.ID) } - clientIDs[client.ID] = struct{}{} + + redisIDs[r.ID] = struct{}{} + + if r.WithInmemID != "" { + if _, ok := redisIDs[r.WithInmemID]; !ok { + return fmt.Errorf("redis cache %q references unknown inmem cache id %q", r.ID, r.WithInmemID) + } + } + } + + if cfg.DB != nil && cfg.DB.UsePreparedStatements == nil { + cfg.DB.UsePreparedStatements = new(bool) + *cfg.DB.UsePreparedStatements = true + } + + if cfg.Handlers.AsyncSearch.ListQueryLengthLimit <= 0 { + cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } + setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + if len(cfg.Handlers.SeqAPI.Envs) > 0 { if cfg.Handlers.SeqAPI.DefaultEnv == "" { - return Config{}, fmt.Errorf("default_env must be specified when using envs") + return fmt.Errorf("default_env must be specified when using envs") } if _, exists := cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv]; !exists { - return Config{}, fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) + return fmt.Errorf("default_env '%s' not found in seq_api.envs", cfg.Handlers.SeqAPI.DefaultEnv) } for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := clientIDs[envConfig.SeqDB]; !ok { - return Config{}, fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) + if _, ok := seqDBIDs[envConfig.SeqDB]; !ok { + return fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) } if envConfig.Options == nil { @@ -445,7 +467,16 @@ func FromFile(cfgPath string) (Config, error) { } } - return cfg, nil + if cfg.Handlers.MassExport != nil { + if cfg.Handlers.MassExport.SessionStore.RedisID == "" { + return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") + } + + if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) + } + } + return nil } func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { @@ -477,15 +508,3 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { options.LogsLifespanCacheTTL = defaultLogsLifespanCacheTTL } } - -func parse(cfg []byte) (Config, error) { - result := Config{} - - decoder := yaml.NewDecoder(bytes.NewReader(cfg)) - decoder.KnownFields(true) - if err := decoder.Decode(&result); err != nil { - return result, fmt.Errorf("error parsing config: %w", err) - } - - return result, nil -} From 4dc75725b8693e504548d00fe2cfefca6fe23886 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 22 Jul 2026 12:44:28 +0300 Subject: [PATCH 3/8] fix --- cmd/seq-ui/m.md | 0 config/config.example.yaml | 2 - config/config.multicluster.yaml | 2 - internal/app/config/config.go | 3 +- .../{migrate_v1_v2.go => migrate/v1_v2.go} | 6 +- internal/app/config/v1/config.go | 210 +++++++++++++++++ internal/app/config/v2/config.go | 218 +++++++++++++++++- 7 files changed, 427 insertions(+), 14 deletions(-) create mode 100644 cmd/seq-ui/m.md rename internal/app/config/{migrate_v1_v2.go => migrate/v1_v2.go} (99%) diff --git a/cmd/seq-ui/m.md b/cmd/seq-ui/m.md new file mode 100644 index 0000000..e69de29 diff --git a/config/config.example.yaml b/config/config.example.yaml index 8e0a5fa..604c16c 100644 --- a/config/config.example.yaml +++ b/config/config.example.yaml @@ -1,5 +1,3 @@ -version: 2 - server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/config/config.multicluster.yaml b/config/config.multicluster.yaml index 3d1eb81..94fb312 100644 --- a/config/config.multicluster.yaml +++ b/config/config.multicluster.yaml @@ -1,5 +1,3 @@ -version: 2 - server: http_addr: ":5555" grpc_addr: ":5556" diff --git a/internal/app/config/config.go b/internal/app/config/config.go index b861a80..9f46c43 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/ozontech/seq-ui/internal/app/config/migrate" v1 "github.com/ozontech/seq-ui/internal/app/config/v1" v2 "github.com/ozontech/seq-ui/internal/app/config/v2" "gopkg.in/yaml.v3" @@ -40,7 +41,7 @@ func FromFile(cfgPath string) (v2.Config, error) { if err != nil { return v2.Config{}, err } - cfg = migrateV1ToV2(cfgV1) + cfg = migrate.V1ToV2(cfgV1) case *meta.Version == currentVersion: cfg, err = parse[v2.Config](cfgBytes, true) if err != nil { diff --git a/internal/app/config/migrate_v1_v2.go b/internal/app/config/migrate/v1_v2.go similarity index 99% rename from internal/app/config/migrate_v1_v2.go rename to internal/app/config/migrate/v1_v2.go index 8dc9df8..522fc50 100644 --- a/internal/app/config/migrate_v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -1,12 +1,12 @@ -package config +package migrate import ( v1 "github.com/ozontech/seq-ui/internal/app/config/v1" v2 "github.com/ozontech/seq-ui/internal/app/config/v2" ) -func migrateV1ToV2(src v1.Config) v2.Config { - dst := v2.Config{Version: currentVersion} +func V1ToV2(src v1.Config) v2.Config { + dst := v2.Config{Version: 2} dst.Server = migrateServer(src.Server) dst.Cache = migrateCache(src.Server) diff --git a/internal/app/config/v1/config.go b/internal/app/config/v1/config.go index 9851298..e3deb74 100644 --- a/internal/app/config/v1/config.go +++ b/internal/app/config/v1/config.go @@ -4,6 +4,216 @@ import ( "time" ) +// Deprecated configuration scheme +// version: +// server: +// http_addr: +// grpc_addr: +// debug_addr: +// grpc_connection_timeout: +// http_read_timeout: +// http_read_header_timeout: +// http_write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: +// jwt_secret_key: +// oidc: +// skip_verify: +// auth_urls: +// root_ca: +// ca_cert: +// private_key: +// ssl_skip_verify: +// allowed_clients: +// cache_secret_key: +// rate_limiters: +// : +// default: +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// spec_users: +// : +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// cache: +// inmemory: +// num_counters: +// max_cost: +// buffer_items: +// redis: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: +// db: +// name: +// host: +// port: +// pass: +// user: +// request_timeout: +// connection_pool_capacity: +// use_prepared_statements: +// clickhouse: +// addrs: +// database: +// username: +// password: +// sharded: +// dial_timeout: +// read_timeout: +// clients: +// seq_db_timeout: +// seq_db_avg_doc_size: +// seq_db_addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// proxy_client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// seq_db: +// id: +// timeout: +// avg_doc_size: +// addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// handlers: +// seq_api: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// envs: +// : +// seq_db_id: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// default_env: +// error_groups: +// log_tags_mapping: +// env: +// service: +// release: +// query_filter: +// : +// mass_export: +// batch_size: +// workers_count: +// tasks_channel_size: +// part_length: +// url_prefix: +// allowed_users: +// file_store: +// s3: +// endpoint: +// access_key_id: +// secret_access_key: +// bucket_name: +// enable_ssl: +// session_store: +// redis: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: +// export_lifetime: +// seq_proxy_downloader: +// delay: +// initial_retry_backoff: +// max_retry_backoff: +// async_search: +// admin_users: +// list_query_length_limit: + type Config struct { Version *int `yaml:"version"` Server *Server `yaml:"server"` diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 882c00a..481625a 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -5,6 +5,212 @@ import ( "time" ) +// Actual configuration scheme +// version: +// server: +// http: +// addr: +// read_timeout: +// read_header_timeout: +// write_timeout: +// cors: +// allowed_origins: +// allowed_methods: +// allowed_headers: +// exposed_headers: +// allow_credentials: +// max_age: +// options_passthrough: +// grpc: +// addr: +// connection_timeout: +// debug: +// addr: +// auth: +// oidc: +// skip_verify: +// auth_urls: +// root_ca: +// ca_cert: +// private_key: +// ssl_skip_verify: +// allowed_clients: +// cache_secret_key: +// jwt: +// secret_key: +// rate_limiters: +// : +// default: +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// spec_users: +// : +// rate_per_sec: +// max_burst: +// store_max_keys: +// per_handler: +// clients: +// seq_db: +// id: +// timeout: +// avg_doc_size: +// addrs: +// request_retries: +// initial_retry_backoff: +// max_retry_backoff: +// client_mode: +// grpc_keepalive_params: +// time: +// timeout: +// permit_without_stream: +// download_params: +// delay: +// initial_retry_backoff: +// max_retry_backoff: +// clickhouse: +// id: +// addrs: +// database: +// username: +// password: +// sharded: +// dial_timeout: +// read_timeout: +// handlers: +// seq_api: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// envs: +// : +// seq_db_id: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: +// events_cache_ttl: +// pinned_fields: +// name: +// type: +// system_fields: +// name: +// type: +// logs_lifespan_cache_key: +// logs_lifespan_cache_ttl: +// fields_cache_ttl: +// masking: +// masks: +// re: +// groups: +// mode: +// replace_word: +// process_fields: +// ignore_fields: +// field_filters: +// condition: +// filters: +// field: +// mode: +// values: +// process_fields: +// ignore_fields: +// default_env: +// error_groups: +// log_tags_mapping: +// env: +// service: +// release: +// query_filter: +// : +// mass_export: +// batch_size: +// workers_count: +// tasks_channel_size: +// part_length: +// url_prefix: +// allowed_users: +// file_store: +// s3: +// endpoint: +// access_key_id: +// secret_access_key: +// bucket_name: +// enable_ssl: +// session_store: +// redis_id: +// export_lifetime: +// download_params: +// delay: +// initial_retry_backoff: +// max_retry_backoff: +// async_search: +// admin_users: +// list_query_length_limit: +// db: +// name: +// host: +// port: +// pass: +// user: +// request_timeout: +// connection_pool_capacity: +// use_prepared_statements: +// cache: +// inmemory: +// id: +// num_counters: +// max_cost: +// buffer_items: +// redis: +// id: +// with_inmem_id: +// addr: +// username: +// password: +// timeout: +// max_retries: +// min_retry_backoff: +// max_retry_backoff: + const ( DefaultSeqDBClientID = "default" DefaultInmemCacheID = "seqapi" @@ -264,9 +470,9 @@ type Field struct { } type SeqAPI struct { - *SeqAPIOptions `yaml:",inline"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + Options SeqAPIOptions `yaml:"options"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` } type SeqAPIEnv struct { @@ -419,14 +625,14 @@ func Normalize(cfg *Config) error { if r.ID == "" { return fmt.Errorf("redis cache ID cannot be empty") } - if _, ok := inmemIDs[r.ID]; ok { + if _, ok := redisIDs[r.ID]; ok { return fmt.Errorf("duplicate redis cache ID: %s", r.ID) } redisIDs[r.ID] = struct{}{} if r.WithInmemID != "" { - if _, ok := redisIDs[r.WithInmemID]; !ok { + if _, ok := inmemIDs[r.WithInmemID]; !ok { return fmt.Errorf("redis cache %q references unknown inmem cache id %q", r.ID, r.WithInmemID) } } @@ -441,7 +647,7 @@ func Normalize(cfg *Config) error { cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } - setSeqAPIOptionsDefaults(cfg.Handlers.SeqAPI.SeqAPIOptions) + setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) if len(cfg.Handlers.SeqAPI.Envs) > 0 { if cfg.Handlers.SeqAPI.DefaultEnv == "" { From b01ac86bb20dc521dda7401474f88ebfe185df67 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Thu, 23 Jul 2026 23:19:59 +0300 Subject: [PATCH 4/8] some refactor --- cmd/seq-ui/m.md | 0 cmd/seq-ui/main.go | 44 +++--- internal/api/seqapi/v1/http/aggregation.go | 4 +- .../api/seqapi/v1/http/aggregation_test.go | 14 +- .../api/seqapi/v1/http/aggregation_ts_test.go | 16 +-- internal/api/seqapi/v1/http/api.go | 126 +++++++----------- internal/api/seqapi/v1/http/events.go | 8 +- internal/api/seqapi/v1/seqapi.go | 2 +- internal/api/seqapi/v1/test/data.go | 20 +-- internal/app/config/config.go | 4 +- internal/app/config/migrate/v1_v2.go | 48 ++++--- internal/app/config/v2/config.go | 96 +++++++++---- internal/app/mw/auth.go | 2 +- internal/app/mw/ratelimiter.go | 2 +- internal/app/ratelimiter/ratelimiter.go | 2 +- internal/app/server/init.go | 30 ++--- internal/app/server/run.go | 14 +- internal/app/server/server.go | 2 +- internal/pkg/cache/ctor.go | 2 +- internal/pkg/cache/inmemory.go | 2 +- internal/pkg/cache/redis.go | 2 +- internal/pkg/mask/field_filter.go | 2 +- internal/pkg/mask/field_filter_test.go | 2 +- internal/pkg/mask/mask.go | 2 +- internal/pkg/mask/mask_test.go | 2 +- internal/pkg/mask/masker.go | 2 +- internal/pkg/mask/masker_test.go | 2 +- internal/pkg/redisclient/client.go | 2 +- .../pkg/service/async_searches/service.go | 2 +- internal/pkg/service/errorgroups/service.go | 2 +- .../pkg/service/errorgroups/service_test.go | 2 +- .../pkg/service/massexport/filestore/s3.go | 2 +- .../massexport/seq_proxy_downloader.go | 2 +- internal/pkg/service/massexport/service.go | 2 +- .../service/massexport/sessionstore/redis.go | 2 +- 35 files changed, 251 insertions(+), 217 deletions(-) delete mode 100644 cmd/seq-ui/m.md diff --git a/cmd/seq-ui/m.md b/cmd/seq-ui/m.md deleted file mode 100644 index e69de29..0000000 diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 6fa2344..8980870 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -23,6 +23,7 @@ import ( massexport_v1 "github.com/ozontech/seq-ui/internal/api/massexport/v1" seqapi_v1 "github.com/ozontech/seq-ui/internal/api/seqapi/v1" userprofile_v1 "github.com/ozontech/seq-ui/internal/api/userprofile/v1" + configloader "github.com/ozontech/seq-ui/internal/app/config" config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/server" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -74,7 +75,7 @@ func run(ctx context.Context) { logger.Warn("app uses the default config file, to provide your own config use -config flag") } - cfg, err := config.FromFile(*configPath) + cfg, err := configloader.FromFile(*configPath) if err != nil { logger.Fatal("read config file error", zap.Error(err)) } @@ -104,28 +105,21 @@ func run(ctx context.Context) { } } -func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { +func initApp(ctx context.Context, cfg config.Config) *api.Registrar { logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { logger.Fatal("failed to init seq-db client", zap.Error(err)) } - defaultClientID := config.DefaultSeqDBClientID - if len(cfg.Handlers.SeqAPI.Envs) > 0 { - defaultClientID = cfg.Handlers.SeqAPI.Envs[cfg.Handlers.SeqAPI.DefaultEnv].SeqDB - } - - defaultClient, exists := seqDBClients[defaultClientID] - if !exists { - logger.Fatal("default seq-db client not found", - zap.String("clientID", defaultClientID), - ) - } - var massExportV1 *massexport_v1.MassExport if cfg.Handlers.MassExport != nil { - exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, defaultClient) + massExportClient, ok := seqDBClients[cfg.Handlers.MassExport.SeqDBID] + if !ok { + logger.Fatal("mass_export seq_db client not found", zap.String("seq_db_id", cfg.Handlers.MassExport.SeqDBID)) + } + + exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, massExportClient) if err != nil { logger.Fatal("can't init export server", zap.Error(err)) } @@ -157,7 +151,7 @@ func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { dashboardsV1 *dashboards_v1.Dashboards ) if db != nil { - repo := repository.New(db, cfg.Server.DB.RequestTimeout) + repo := repository.New(db, cfg.DB.RequestTimeout) userProfilesSvc := userprofile.New(repo.UserProfiles, repo.FavoriteQueries) dashboardsSvc := dashboards.New(repo.Dashboards) profiles.InitProfiles(repo.UserProfiles.GetOrCreate) @@ -165,7 +159,11 @@ func initApp(ctx context.Context, cfg config_v2.Config) *api.Registrar { userProfileV1 = userprofile_v1.New(userProfilesSvc) dashboardsV1 = dashboards_v1.New(dashboardsSvc) - asyncSearchesService = asyncsearches.New(ctx, repo, defaultClient, cfg.Handlers.AsyncSearch) + asyncSearchClient, ok := seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID] + if !ok { + logger.Fatal("async_search seq_db client not found", zap.String("seq_db_id", cfg.Handlers.AsyncSearch.SeqDBID)) + } + asyncSearchesService = asyncsearches.New(ctx, repo, asyncSearchClient, cfg.Handlers.AsyncSearch) } seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService) @@ -201,7 +199,17 @@ func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb. } func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI config.SeqAPI) (seqdb.Client, error) { - clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * int(seqAPI.MaxSearchLimit) + maxSearchLimit := int(seqAPI.Options.MaxSearchLimit) + for _, env := range seqAPI.Envs { + if env.SeqDBID != cfg.ID { + continue + } + if l := int(env.Options.MaxSearchLimit); l > maxSearchLimit { + maxSearchLimit = l + } + } + + clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * maxSearchLimit if clientMaxRecvMsgSize < defaultClientMaxRecvMsgSize { clientMaxRecvMsgSize = defaultClientMaxRecvMsgSize } diff --git a/internal/api/seqapi/v1/http/aggregation.go b/internal/api/seqapi/v1/http/aggregation.go index b4ffebb..e423159 100644 --- a/internal/api/seqapi/v1/http/aggregation.go +++ b/internal/api/seqapi/v1/http/aggregation.go @@ -94,7 +94,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { getAggResp := getAggregationResponseFromProto(resp) - if params.masker != nil { + if a.globalParams.masker != nil { buf := make([]string, 0) for i, agg := range getAggResp.Aggregations { buf = buf[:0] @@ -108,7 +108,7 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) { field = aggReq.GroupBy } - buf = params.masker.MaskAgg(field, buf) + buf = a.globalParams.masker.MaskAgg(field, buf) for j, key := range buf { getAggResp.Aggregations[i].Buckets[j].Key = key diff --git a/internal/api/seqapi/v1/http/aggregation_test.go b/internal/api/seqapi/v1/http/aggregation_test.go index 6a675f4..6b75c75 100644 --- a/internal/api/seqapi/v1/http/aggregation_test.go +++ b/internal/api/seqapi/v1/http/aggregation_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -63,7 +63,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -106,7 +106,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -168,7 +168,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: false, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -203,7 +203,7 @@ func TestServeGetAggregation(t *testing.T) { PartialResponse: true, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -218,7 +218,7 @@ func TestServeGetAggregation(t *testing.T) { }, wantErr: true, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 2, }, }, @@ -242,7 +242,7 @@ func TestServeGetAggregation(t *testing.T) { }, wantErr: true, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, diff --git a/internal/api/seqapi/v1/http/aggregation_ts_test.go b/internal/api/seqapi/v1/http/aggregation_ts_test.go index 9c4f86c..cd06b34 100644 --- a/internal/api/seqapi/v1/http/aggregation_ts_test.go +++ b/internal/api/seqapi/v1/http/aggregation_ts_test.go @@ -17,7 +17,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -106,7 +106,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":1}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":2}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":3}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -162,7 +162,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"test_count1":"test1"},"values":[{"timestamp":1695637231,"value":3}]},{"metric":{"test_count1":"test2"},"values":[{"timestamp":1695637232,"value":6}]},{"metric":{"test_count1":"test3"},"values":[{"timestamp":1695637233,"value":9}]}]}},{"data":{"result":[{"metric":{"test_count2":"test1"},"values":[{"timestamp":1695637231,"value":2}]},{"metric":{"test_count2":"test2"},"values":[{"timestamp":1695637232,"value":4}]},{"metric":{"test_count2":"test3"},"values":[{"timestamp":1695637233,"value":6}]}],"target_bucket_rate":"2s"}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -221,7 +221,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":[{"data":{"result":[{"metric":{"quantile":"p95","service":"test1"},"values":[{"timestamp":1695637231,"value":100}]},{"metric":{"quantile":"p99","service":"test1"},"values":[{"timestamp":1695637231,"value":150}]},{"metric":{"quantile":"p95","service":"test2"},"values":[{"timestamp":1695637232,"value":100}]},{"metric":{"quantile":"p99","service":"test2"},"values":[{"timestamp":1695637232,"value":150}]},{"metric":{"quantile":"p95","service":"test3"},"values":[{"timestamp":1695637233,"value":100}]},{"metric":{"quantile":"p99","service":"test3"},"values":[{"timestamp":1695637233,"value":150}]}]}}],"error":{"code":"ERROR_CODE_NO"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -247,7 +247,7 @@ func TestServeGetAggregationTs(t *testing.T) { wantRespBody: `{"aggregations":null,"error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`, wantStatus: http.StatusOK, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 100, }, @@ -263,7 +263,7 @@ func TestServeGetAggregationTs(t *testing.T) { reqBody: formatReqBody(aggregationTsQueries{{}, {}, {}}), wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 2, MaxBucketsPerAggregationTs: 100, }, @@ -278,7 +278,7 @@ func TestServeGetAggregationTs(t *testing.T) { }), wantStatus: http.StatusBadRequest, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, MaxBucketsPerAggregationTs: 1, }, @@ -297,7 +297,7 @@ func TestServeGetAggregationTs(t *testing.T) { }, wantStatus: http.StatusInternalServerError, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index c762883..bb567fc 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -10,7 +10,7 @@ import ( "github.com/gofrs/uuid" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tokenlimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" @@ -24,115 +24,89 @@ import ( type apiParams struct { client seqdb.Client options *config.SeqAPIOptions - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields fields - systemFields fields exportLimiter *tokenlimiter.Limiter } +type apiGlobalParams struct { + fieldsCache *fieldsCache + masker *mask.Masker + pinnedFields fields + systemFields fields +} + type API struct { - config config.SeqAPI - params apiParams - paramsByEnv map[string]apiParams - inmemWithRedisCache cache.Cache - redisCache cache.Cache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse getEnvsResponse + config config.SeqAPI + globalParams apiGlobalParams + paramsByEnv map[string]apiParams + cache cache.Cache + nowFn func() time.Time + asyncSearches asyncsearches.Service + envsResponse getEnvsResponse } func New( cfg config.SeqAPI, seqDBСlients map[string]seqdb.Client, - inmemWithRedisCache cache.Cache, - redisCache cache.Cache, + cache cache.Cache, asyncSearches asyncsearches.Service, ) *API { - var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) + globalParams := apiGlobalParams{ + pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), + systemFields: parseFields(cfg.GlobalOptions.SystemFields), + } + if cfg.GlobalOptions.FieldsCacheTTL > 0 { + globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.Masking) + globalMasker, err := mask.New(cfg.GlobalOptions.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } - globalPinnedFields := parseFields(cfg.PinnedFields) - globalSystemFields := parseFields(cfg.SystemFields) - globalExportLimiter := tokenlimiter.New(cfg.MaxParallelExportRequests) - - var params apiParams - var paramsByEnv map[string]apiParams + globalParams.masker = globalMasker + globalExportLimiter := tokenlimiter.New(cfg.Options.MaxParallelExportRequests) + paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { - paramsByEnv = make(map[string]apiParams) for envName, envConfig := range cfg.Envs { - client := seqDBСlients[envConfig.SeqDB] - options := envConfig.Options - - var envfCache *fieldsCache - if options.FieldsCacheTTL > 0 { - envfCache = newFieldsCache(options.FieldsCacheTTL) - } - - var envMasker, err = mask.New(options.Masking) - if err != nil { - logger.Fatal("failed to init env masking", zap.Error(err)) - } - - envPinnedFields := parseFields(options.PinnedFields) - envSystemFields := parseFields(options.SystemFields) - envExportLimiter := tokenlimiter.New(options.MaxParallelExportRequests) - paramsByEnv[envName] = apiParams{ - client: client, - options: options, - fieldsCache: envfCache, - masker: envMasker, - pinnedFields: envPinnedFields, - systemFields: envSystemFields, - exportLimiter: envExportLimiter, + client: seqDBСlients[envConfig.SeqDBID], + options: envConfig.Options, + exportLimiter: tokenlimiter.New(envConfig.Options.MaxParallelExportRequests), } } } else { - client, exists := seqDBСlients[config.DefaultSeqDBClientID] - if !exists { - logger.Fatal("default client not found", - zap.String("clientID", config.DefaultSeqDBClientID)) + if len(seqDBСlients) != 1 { + logger.Fatal("seq_api.envs is empty, configure envs when clients.seq_db has more than one instance") + } + + var client seqdb.Client + for _, c := range seqDBСlients { + client = c } - params = apiParams{ + paramsByEnv[""] = apiParams{ client: client, - options: cfg.SeqAPIOptions, - fieldsCache: globalfCache, - masker: globalMasker, - pinnedFields: globalPinnedFields, - systemFields: globalSystemFields, + options: &cfg.Options, exportLimiter: globalExportLimiter, } } + // for export - if len(cfg.Envs) > 0 { + if globalParams.masker != nil { for _, param := range paramsByEnv { - if param.masker != nil { - param.client.WithMasking(param.masker) - } + param.client.WithMasking(globalParams.masker) } - } else if params.masker != nil { - params.client.WithMasking(params.masker) } return &API{ - config: cfg, - params: params, - paramsByEnv: paramsByEnv, - inmemWithRedisCache: inmemWithRedisCache, - redisCache: redisCache, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + config: cfg, + globalParams: globalParams, + paramsByEnv: paramsByEnv, + cache: cache, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), } } @@ -219,7 +193,7 @@ func parseEnvs(cfg config.SeqAPI) getEnvsResponse { envs = append(envs, createEnvInfo(name, envConfig.Options)) } } else { - envs = []envInfo{createEnvInfo("", cfg.SeqAPIOptions)} + envs = []envInfo{createEnvInfo("", &cfg.Options)} } return getEnvsResponse{Envs: envs} } @@ -236,7 +210,7 @@ const ( func (a *API) GetEnvParams(env string) (apiParams, error) { if len(a.config.Envs) == 0 { - return a.params, nil + return a.paramsByEnv[""], nil } if env == "" { diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index 006293a..6f8154a 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -57,8 +57,8 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { if cached, err := a.inmemWithRedisCache.Get(ctx, id); err == nil { e := &seqapi.Event{} if err = proto.Unmarshal([]byte(cached), e); err == nil { - if params.masker != nil { - params.masker.Mask(e.Data) + if a.globalParams.masker != nil { + a.globalParams.masker.Mask(e.Data) } wr.WriteJson(getEventResponseFromProto(&seqapi.GetEventResponse{Event: e})) return @@ -74,8 +74,8 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { return } - if params.masker != nil && resp.Event != nil { - params.masker.Mask(resp.Event.Data) + if a.globalParams.masker != nil && resp.Event != nil { + a.globalParams.masker.Mask(resp.Event.Data) } if data, err := proto.Marshal(resp.Event); err == nil { diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index f1a08c4..f32d657 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -5,7 +5,7 @@ import ( grpc_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/grpc" http_api "github.com/ozontech/seq-ui/internal/api/seqapi/v1/http" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" diff --git a/internal/api/seqapi/v1/test/data.go b/internal/api/seqapi/v1/test/data.go index 0ab1d44..b32962f 100644 --- a/internal/api/seqapi/v1/test/data.go +++ b/internal/api/seqapi/v1/test/data.go @@ -6,7 +6,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" mock_asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches/mock" @@ -111,22 +111,22 @@ func MakeAggregations(aggCount, bucketCount int, opts *MakeAggOpts) []*seqapi.Ag } func SetCfgDefaults(cfg config.SeqAPI) config.SeqAPI { - if cfg.MaxAggregationsPerRequest <= 0 { - cfg.MaxAggregationsPerRequest = 1 + if cfg.Options.MaxAggregationsPerRequest <= 0 { + cfg.Options.MaxAggregationsPerRequest = 1 } - if cfg.MaxParallelExportRequests <= 0 { - cfg.MaxParallelExportRequests = 1 + if cfg.Options.MaxParallelExportRequests <= 0 { + cfg.Options.MaxParallelExportRequests = 1 } - if cfg.MaxSearchTotalLimit <= 0 { - cfg.MaxSearchTotalLimit = 1000000 + if cfg.Options.MaxSearchTotalLimit <= 0 { + cfg.Options.MaxSearchTotalLimit = 1000000 } - if cfg.MaxSearchOffsetLimit <= 0 { - cfg.MaxSearchOffsetLimit = 1000000 + if cfg.Options.MaxSearchOffsetLimit <= 0 { + cfg.Options.MaxSearchOffsetLimit = 1000000 } for envName, envConfig := range cfg.Envs { if envConfig.Options == nil { - envConfig.Options = cfg.SeqAPIOptions + envConfig.Options = &cfg.Options cfg.Envs[envName] = envConfig } } diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 9f46c43..000e69d 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -12,8 +12,6 @@ import ( ) const ( - configV1Name = "config_v1" - configV2Name = "config_v2" previousVersion = 1 currentVersion = 2 ) @@ -52,7 +50,7 @@ func FromFile(cfgPath string) (v2.Config, error) { } if err := v2.Normalize(&cfg); err != nil { - return v2.Config{}, fmt.Errorf("normalize config: %w") + return v2.Config{}, fmt.Errorf("normalize config: %w", err) } return cfg, nil diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index 522fc50..f0b2bc4 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -250,11 +250,17 @@ func migrateHandlers(src *v1.Handlers, cfg *v2.Config) *v2.Handlers { } func migrateSeqAPI(src v1.SeqAPI) v2.SeqAPI { - return v2.SeqAPI{ - SeqAPIOptions: migrateSeqAPIOptions(src.SeqAPIOptions), - Envs: migrateSeqAPIEnvs(src.Envs), - DefaultEnv: src.DefaultEnv, + dst := v2.SeqAPI{ + Envs: migrateSeqAPIEnvs(src.Envs), + DefaultEnv: src.DefaultEnv, } + + if src.SeqAPIOptions != nil { + dst.GlobalOptions = migrateSeqAPIOptionsToGlobal(src.SeqAPIOptions) + dst.Options = migrateSeqAPIOptions(src.SeqAPIOptions) + } + + return dst } func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { @@ -264,21 +270,34 @@ func migrateSeqAPIEnvs(envs map[string]v1.SeqAPIEnv) map[string]v2.SeqAPIEnv { dst := make(map[string]v2.SeqAPIEnv, len(envs)) for name, cfg := range envs { - dst[name] = v2.SeqAPIEnv{ - SeqDB: cfg.SeqDB, - Options: migrateSeqAPIOptions(cfg.Options), + env := v2.SeqAPIEnv{ + SeqDBID: cfg.SeqDB, } + if cfg.Options != nil { + envOptions := migrateSeqAPIOptions(cfg.Options) + env.Options = &envOptions + } + + dst[name] = env } return dst } -func migrateSeqAPIOptions(options *v1.SeqAPIOptions) *v2.SeqAPIOptions { - if options == nil { - return nil +func migrateSeqAPIOptionsToGlobal(options *v1.SeqAPIOptions) v2.SeqAPIGlobalOptions { + return v2.SeqAPIGlobalOptions{ + EventsCacheTTL: options.EventsCacheTTL, + PinnedFields: migrateFields(options.PinnedFields), + SystemFields: migrateFields(options.SystemFields), + LogsLifespanCacheKey: options.LogsLifespanCacheKey, + LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, + FieldsCacheTTL: options.FieldsCacheTTL, + Masking: migrateMasking(options.Masking), } +} - return &v2.SeqAPIOptions{ +func migrateSeqAPIOptions(options *v1.SeqAPIOptions) v2.SeqAPIOptions { + return v2.SeqAPIOptions{ MaxSearchLimit: options.MaxSearchLimit, MaxSearchTotalLimit: options.MaxSearchTotalLimit, MaxSearchOffsetLimit: options.MaxSearchOffsetLimit, @@ -287,13 +306,6 @@ func migrateSeqAPIOptions(options *v1.SeqAPIOptions) *v2.SeqAPIOptions { MaxParallelExportRequests: options.MaxParallelExportRequests, MaxAggregationsPerRequest: options.MaxAggregationsPerRequest, MaxBucketsPerAggregationTs: options.MaxBucketsPerAggregationTs, - EventsCacheTTL: options.EventsCacheTTL, - PinnedFields: migrateFields(options.PinnedFields), - SystemFields: migrateFields(options.SystemFields), - LogsLifespanCacheKey: options.LogsLifespanCacheKey, - LogsLifespanCacheTTL: options.LogsLifespanCacheTTL, - FieldsCacheTTL: options.FieldsCacheTTL, - Masking: migrateMasking(options.Masking), } } diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 481625a..9350474 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -368,6 +368,7 @@ type FileStore struct { } type MassExport struct { + SeqDBID string `yaml:"seq_db_id"` BatchSize uint64 `yaml:"batch_size"` WorkersCount int `yaml:"workers_count"` TasksChannelSize int `yaml:"tasks_channel_size"` @@ -470,32 +471,37 @@ type Field struct { } type SeqAPI struct { - Options SeqAPIOptions `yaml:"options"` - Envs map[string]SeqAPIEnv `yaml:"envs"` - DefaultEnv string `yaml:"default_env"` + CacheID string `yaml:"cache_id"` + GlobalOptions SeqAPIGlobalOptions `yaml:"global_options"` + Options SeqAPIOptions `yaml:"options"` + Envs map[string]SeqAPIEnv `yaml:"envs"` + DefaultEnv string `yaml:"default_env"` +} + +type SeqAPIGlobalOptions struct { + PinnedFields []Field `yaml:"pinned_fields"` + SystemFields []Field `yaml:"system_fields"` + Masking *Masking `yaml:"masking"` + LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` + EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` + LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` + FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` } type SeqAPIEnv struct { - SeqDB string `yaml:"seq_db_id"` + SeqDBID string `yaml:"seq_db_id"` Options *SeqAPIOptions `yaml:"options"` } type SeqAPIOptions struct { - MaxSearchLimit int32 `yaml:"max_search_limit"` - MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` - MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` - MaxExportLimit int32 `yaml:"max_export_limit"` - SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` - MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` - MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` - MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` - EventsCacheTTL time.Duration `yaml:"events_cache_ttl"` - PinnedFields []Field `yaml:"pinned_fields"` - SystemFields []Field `yaml:"system_fields"` - LogsLifespanCacheKey string `yaml:"logs_lifespan_cache_key"` - LogsLifespanCacheTTL time.Duration `yaml:"logs_lifespan_cache_ttl"` - FieldsCacheTTL time.Duration `yaml:"fields_cache_ttl"` - Masking *Masking `yaml:"masking"` + MaxSearchLimit int32 `yaml:"max_search_limit"` + MaxSearchTotalLimit int64 `yaml:"max_search_total_limit"` + MaxSearchOffsetLimit int32 `yaml:"max_search_offset_limit"` + MaxExportLimit int32 `yaml:"max_export_limit"` + SeqCLIMaxSearchLimit int `yaml:"seq_cli_max_search_limit"` + MaxParallelExportRequests int `yaml:"max_parallel_export_requests"` + MaxAggregationsPerRequest int `yaml:"max_aggregations_per_request"` + MaxBucketsPerAggregationTs int `yaml:"max_buckets_per_aggregation_ts"` } type Masking struct { @@ -534,11 +540,13 @@ type LogTagsMapping struct { } type ErrorGroups struct { + CHID string `yaml:"ch_id"` LogTagsMapping LogTagsMapping `yaml:"log_tags_mapping"` QueryFilter map[string]string `yaml:"query_filter"` } type AsyncSearch struct { + SeqDBID string `yaml:"seq_db_id"` AdminUsers []string `yaml:"admin_users"` ListQueryLengthLimit int `yaml:"list_query_length_limit"` } @@ -647,6 +655,7 @@ func Normalize(cfg *Config) error { cfg.Handlers.AsyncSearch.ListQueryLengthLimit = defaultAsyncSearchListQueryLengthLimit } + setSeqAPIGlobalOptionsDefaults(&cfg.Handlers.SeqAPI.GlobalOptions) setSeqAPIOptionsDefaults(&cfg.Handlers.SeqAPI.Options) if len(cfg.Handlers.SeqAPI.Envs) > 0 { @@ -659,16 +668,12 @@ func Normalize(cfg *Config) error { } for envName, envConfig := range cfg.Handlers.SeqAPI.Envs { - if _, ok := seqDBIDs[envConfig.SeqDB]; !ok { - return fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDB, envName) - } - - if envConfig.Options == nil { - envConfig.Options = cfg.Handlers.SeqAPI.SeqAPIOptions - } else { - setSeqAPIOptionsDefaults(envConfig.Options) + if _, ok := seqDBIDs[envConfig.SeqDBID]; !ok { + return fmt.Errorf("client '%s' for env '%s' not found", envConfig.SeqDBID, envName) } + merged := mergeSeqAPIOptions(cfg.Handlers.SeqAPI.Options, envConfig.Options) + envConfig.Options = &merged cfg.Handlers.SeqAPI.Envs[envName] = envConfig } } @@ -704,6 +709,43 @@ func setSeqAPIOptionsDefaults(options *SeqAPIOptions) { if options.MaxExportLimit <= 0 { options.MaxExportLimit = defaultMaxExportLimit } +} + +func mergeSeqAPIOptions(global SeqAPIOptions, envOptions *SeqAPIOptions) SeqAPIOptions { + merged := global + if envOptions == nil { + return merged + } + + if envOptions.MaxAggregationsPerRequest > 0 { + merged.MaxAggregationsPerRequest = envOptions.MaxAggregationsPerRequest + } + if envOptions.MaxBucketsPerAggregationTs > 0 { + merged.MaxBucketsPerAggregationTs = envOptions.MaxBucketsPerAggregationTs + } + if envOptions.MaxParallelExportRequests > 0 { + merged.MaxParallelExportRequests = envOptions.MaxParallelExportRequests + } + if envOptions.MaxSearchLimit > 0 { + merged.MaxSearchLimit = envOptions.MaxSearchLimit + } + if envOptions.MaxSearchTotalLimit > 0 { + merged.MaxSearchTotalLimit = envOptions.MaxSearchTotalLimit + } + if envOptions.MaxSearchOffsetLimit > 0 { + merged.MaxSearchOffsetLimit = envOptions.MaxSearchOffsetLimit + } + if envOptions.MaxExportLimit > 0 { + merged.MaxExportLimit = envOptions.MaxExportLimit + } + if envOptions.SeqCLIMaxSearchLimit > 0 { + merged.SeqCLIMaxSearchLimit = envOptions.SeqCLIMaxSearchLimit + } + + return merged +} + +func setSeqAPIGlobalOptionsDefaults(options *SeqAPIGlobalOptions) { if options.EventsCacheTTL <= 0 { options.EventsCacheTTL = defaultEventsCacheTTL } diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index a927e2d..ef8ee74 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/ozontech/seq-ui/internal/app/auth" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" ) diff --git a/internal/app/mw/ratelimiter.go b/internal/app/mw/ratelimiter.go index f181616..ccee177 100644 --- a/internal/app/mw/ratelimiter.go +++ b/internal/app/mw/ratelimiter.go @@ -7,7 +7,7 @@ import ( "net/http" "strconv" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/ratelimiter" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/metric" diff --git a/internal/app/ratelimiter/ratelimiter.go b/internal/app/ratelimiter/ratelimiter.go index dadee28..096ac91 100644 --- a/internal/app/ratelimiter/ratelimiter.go +++ b/internal/app/ratelimiter/ratelimiter.go @@ -8,7 +8,7 @@ import ( "github.com/throttled/throttled/v2" "github.com/throttled/throttled/v2/store/memstore" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) const rateLimiterQuantityPerCall = 1 diff --git a/internal/app/server/init.go b/internal/app/server/init.go index 26d31a7..affc9c8 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -30,7 +30,7 @@ var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { var err error - s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.JWTSecretKey, s.config.OIDC, s.config.Cache) + s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.Auth.JWT.SecretKey, s.config.Auth.OIDC, s.Cache) if err != nil { return err } @@ -54,22 +54,22 @@ func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { // setupCORS applies CORS policies set in config to the provided mux. func (s *Server) setupCORS(mux *chi.Mux) { - allowedOrigins := s.config.CORS.AllowedOrigins + allowedOrigins := s.config.HTTP.CORS.AllowedOrigins if len(allowedOrigins) == 0 { allowedOrigins = append(allowedOrigins, defaultCORSAllowedOrigins) } - allowedMethods := s.config.CORS.AllowedMethods + allowedMethods := s.config.HTTP.CORS.AllowedMethods if len(allowedMethods) == 0 { allowedMethods = append(allowedMethods, defaultCORSAllowedMethods...) } mux.Use(cors.Handler(cors.Options{ AllowedOrigins: allowedOrigins, AllowedMethods: allowedMethods, - AllowedHeaders: s.config.CORS.AllowedHeaders, - ExposedHeaders: s.config.CORS.ExposedHeaders, - AllowCredentials: s.config.CORS.AllowCredentials, - MaxAge: s.config.CORS.MaxAge, - OptionsPassthrough: s.config.CORS.OptionsPassthrough, + AllowedHeaders: s.config.HTTP.CORS.AllowedHeaders, + ExposedHeaders: s.config.HTTP.CORS.ExposedHeaders, + AllowCredentials: s.config.HTTP.CORS.AllowCredentials, + MaxAge: s.config.HTTP.CORS.MaxAge, + OptionsPassthrough: s.config.HTTP.CORS.OptionsPassthrough, })) } @@ -130,7 +130,7 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { opts := []grpc.ServerOption{ grpc.UnaryInterceptor(grpc_mw.ChainUnaryServer(interceptors...)), - grpc.ConnectionTimeout(s.config.GRPCConnectionTimeout), + grpc.ConnectionTimeout(s.config.GRPC.ConnectionTimeout), } s.grpcServer = grpc.NewServer(opts...) @@ -143,7 +143,7 @@ func (s *Server) prepareGRPCServer(registrar *api.Registrar) { // prepareHTTPServer prepares HTTP server with applied CORS policies and added interceptors. func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar) { mux := chi.NewMux() - if s.config.CORS != nil { + if s.config.HTTP.CORS != nil { s.setupCORS(mux) } interceptors := chi.Middlewares{ @@ -172,7 +172,7 @@ func (s *Server) prepareHTTPServer(ctx context.Context, registrar *api.Registrar // prepareHTTPMux prepares debug HTTP server mux with applied CORS policies and added interceptors. func (s *Server) prepareDebugServer(ctx context.Context) error { mux := chi.NewMux() - if s.config.CORS != nil { + if s.config.HTTP.CORS != nil { s.setupCORS(mux) } interceptors := chi.Middlewares{ @@ -181,7 +181,7 @@ func (s *Server) prepareDebugServer(ctx context.Context) error { mux.Use(interceptors...) mux.Handle("/metrics", promhttp.Handler()) serveHealth(mux) - _, port, err := net.SplitHostPort(s.config.HTTPAddr) + _, port, err := net.SplitHostPort(s.config.HTTP.Addr) if err != nil { return err } @@ -195,9 +195,9 @@ func (s *Server) prepareDebugServer(ctx context.Context) error { func (s *Server) makeHTTPServer(ctx context.Context, mux *chi.Mux) *http.Server { return &http.Server{ Handler: mux, - ReadHeaderTimeout: s.config.HTTPReadHeaderTimeout, - ReadTimeout: s.config.HTTPReadTimeout, - WriteTimeout: s.config.HTTPWriteTimeout, + ReadHeaderTimeout: s.config.HTTP.ReadHeaderTimeout, + ReadTimeout: s.config.HTTP.ReadTimeout, + WriteTimeout: s.config.HTTP.WriteTimeout, MaxHeaderBytes: maxHTTPHeaderBytes, // 4 KiB BaseContext: func(_ net.Listener) context.Context { return ctx diff --git a/internal/app/server/run.go b/internal/app/server/run.go index ce7dc2c..a7346ba 100644 --- a/internal/app/server/run.go +++ b/internal/app/server/run.go @@ -18,7 +18,7 @@ func (s *Server) Run(ctx context.Context) error { // run gRPC server errWg.Go(func() error { - lis, err := net.Listen("tcp", s.config.GRPCAddr) + lis, err := net.Listen("tcp", s.config.GRPC.Addr) if err != nil { return err } @@ -28,7 +28,7 @@ func (s *Server) Run(ctx context.Context) error { // run HTTP server errWg.Go(func() error { - l, err := net.Listen("tcp", s.config.HTTPAddr) + l, err := net.Listen("tcp", s.config.HTTP.Addr) if err != nil { return err } @@ -38,7 +38,7 @@ func (s *Server) Run(ctx context.Context) error { // run debug server errWg.Go(func() error { - l, err := net.Listen("tcp", s.config.DebugAddr) + l, err := net.Listen("tcp", s.config.Debug.Addr) if err != nil { return err } @@ -47,12 +47,12 @@ func (s *Server) Run(ctx context.Context) error { }) errWg.Go(func() error { - swaggerAddr := s.config.DebugAddr + swaggerUIPrefix + swaggerAddr := s.config.Debug.Addr + swaggerUIPrefix logger.Info("app started", - zap.String("http", s.config.HTTPAddr), - zap.String("debug", s.config.DebugAddr), + zap.String("http", s.config.HTTP.Addr), + zap.String("debug", s.config.Debug.Addr), zap.String("swagger", swaggerAddr), - zap.String("grpc", s.config.GRPCAddr), + zap.String("grpc", s.config.GRPC.Addr), ) return nil }) diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 6872959..3d9e256 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -8,7 +8,7 @@ import ( "google.golang.org/grpc" "github.com/ozontech/seq-ui/internal/api" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" ) diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index b14bc3a..b9030c0 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -6,7 +6,7 @@ import ( "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" ) diff --git a/internal/pkg/cache/inmemory.go b/internal/pkg/cache/inmemory.go index acf69cf..1236b87 100644 --- a/internal/pkg/cache/inmemory.go +++ b/internal/pkg/cache/inmemory.go @@ -7,7 +7,7 @@ import ( "github.com/dgraph-io/ristretto" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/metric" ) diff --git a/internal/pkg/cache/redis.go b/internal/pkg/cache/redis.go index 4e60d44..a36782c 100644 --- a/internal/pkg/cache/redis.go +++ b/internal/pkg/cache/redis.go @@ -8,7 +8,7 @@ import ( "github.com/redis/go-redis/v9" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/metric" ) diff --git a/internal/pkg/mask/field_filter.go b/internal/pkg/mask/field_filter.go index 2eba3dc..3f108ba 100644 --- a/internal/pkg/mask/field_filter.go +++ b/internal/pkg/mask/field_filter.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) type filterCondition int8 diff --git a/internal/pkg/mask/field_filter_test.go b/internal/pkg/mask/field_filter_test.go index 4c368bc..fe69ba7 100644 --- a/internal/pkg/mask/field_filter_test.go +++ b/internal/pkg/mask/field_filter_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestFieldFilterSetCreate(t *testing.T) { diff --git a/internal/pkg/mask/mask.go b/internal/pkg/mask/mask.go index f3d4000..c16035f 100644 --- a/internal/pkg/mask/mask.go +++ b/internal/pkg/mask/mask.go @@ -7,7 +7,7 @@ import ( "slices" "strings" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) const maskSymbol = byte('*') diff --git a/internal/pkg/mask/mask_test.go b/internal/pkg/mask/mask_test.go index afe054e..612d237 100644 --- a/internal/pkg/mask/mask_test.go +++ b/internal/pkg/mask/mask_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskCompile(t *testing.T) { diff --git a/internal/pkg/mask/masker.go b/internal/pkg/mask/masker.go index 90adb62..7d75b26 100644 --- a/internal/pkg/mask/masker.go +++ b/internal/pkg/mask/masker.go @@ -3,7 +3,7 @@ package mask import ( "fmt" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) type Masker struct { diff --git a/internal/pkg/mask/masker_test.go b/internal/pkg/mask/masker_test.go index b8da9d2..3150786 100644 --- a/internal/pkg/mask/masker_test.go +++ b/internal/pkg/mask/masker_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestMaskerMask(t *testing.T) { diff --git a/internal/pkg/redisclient/client.go b/internal/pkg/redisclient/client.go index e9a879b..00f5d7a 100644 --- a/internal/pkg/redisclient/client.go +++ b/internal/pkg/redisclient/client.go @@ -6,7 +6,7 @@ import ( "github.com/redis/go-redis/v9" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func New(ctx context.Context, cfg *config.Redis) (*redis.Client, error) { diff --git a/internal/pkg/service/async_searches/service.go b/internal/pkg/service/async_searches/service.go index d6f4825..1c55396 100644 --- a/internal/pkg/service/async_searches/service.go +++ b/internal/pkg/service/async_searches/service.go @@ -10,7 +10,7 @@ import ( "github.com/cenkalti/backoff/v4" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/repository" diff --git a/internal/pkg/service/errorgroups/service.go b/internal/pkg/service/errorgroups/service.go index f6af295..b537bde 100644 --- a/internal/pkg/service/errorgroups/service.go +++ b/internal/pkg/service/errorgroups/service.go @@ -8,7 +8,7 @@ import ( "golang.org/x/sync/errgroup" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" repositorych "github.com/ozontech/seq-ui/internal/pkg/repository_ch" ) diff --git a/internal/pkg/service/errorgroups/service_test.go b/internal/pkg/service/errorgroups/service_test.go index be857e9..49cc37d 100644 --- a/internal/pkg/service/errorgroups/service_test.go +++ b/internal/pkg/service/errorgroups/service_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" mock "github.com/ozontech/seq-ui/internal/pkg/repository_ch/mock" ) diff --git a/internal/pkg/service/massexport/filestore/s3.go b/internal/pkg/service/massexport/filestore/s3.go index 469e465..2903e6e 100644 --- a/internal/pkg/service/massexport/filestore/s3.go +++ b/internal/pkg/service/massexport/filestore/s3.go @@ -11,7 +11,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) type s3FileStore struct { diff --git a/internal/pkg/service/massexport/seq_proxy_downloader.go b/internal/pkg/service/massexport/seq_proxy_downloader.go index 3ff9163..4a92f3e 100644 --- a/internal/pkg/service/massexport/seq_proxy_downloader.go +++ b/internal/pkg/service/massexport/seq_proxy_downloader.go @@ -9,7 +9,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/logger" "github.com/ozontech/seq-ui/metric" diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index 3c21309..e65ba6b 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -7,7 +7,7 @@ import ( "slices" "time" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" "github.com/ozontech/seq-ui/internal/pkg/service/massexport/filestore" diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index f24d9b1..e5a7988 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -11,7 +11,7 @@ import ( "github.com/redis/go-redis/v9" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/redisclient" "github.com/ozontech/seq-ui/logger" From 1eb14b824ef4f25a57cc8a888a47f423608df473 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Tue, 28 Jul 2026 17:17:44 +0300 Subject: [PATCH 5/8] update --- cmd/config-migrate/main.go | 3 ++- cmd/seq-ui/main.go | 27 ++++++++------------------- internal/app/config/config.go | 3 ++- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/cmd/config-migrate/main.go b/cmd/config-migrate/main.go index 08fd1b8..3dd1f2c 100644 --- a/cmd/config-migrate/main.go +++ b/cmd/config-migrate/main.go @@ -5,9 +5,10 @@ import ( "flag" "os" + "go.uber.org/zap" + "github.com/ozontech/seq-ui/internal/app/config" "github.com/ozontech/seq-ui/logger" - "go.uber.org/zap" ) var ( diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 8980870..c35fc71 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -111,14 +111,12 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { if err != nil { logger.Fatal("failed to init seq-db client", zap.Error(err)) } - var massExportV1 *massexport_v1.MassExport if cfg.Handlers.MassExport != nil { massExportClient, ok := seqDBClients[cfg.Handlers.MassExport.SeqDBID] if !ok { logger.Fatal("mass_export seq_db client not found", zap.String("seq_db_id", cfg.Handlers.MassExport.SeqDBID)) } - exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, massExportClient) if err != nil { logger.Fatal("can't init export server", zap.Error(err)) @@ -126,17 +124,10 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { massExportV1 = massexport_v1.New(exportServer) } - - logger.Info("initializing inmemory with redis seqapi cache") - inmemWithRedisCache, err := cache.NewInmemoryWithRedisOrInmemory(ctx, cfg.Server.Cache) - if err != nil { - logger.Fatal("failed to init inmemory with redis seqapi cache", zap.Error(err)) - } - - logger.Info("initializing redis seqapi cache") - redisCache, err := cache.NewRedisOrInmemory(ctx, cfg.Server.Cache) + logger.Info("initializing caches") + caches, err := cache.FromConfig(ctx, cfg.Cache) if err != nil { - logger.Fatal("failed to init redis seqapi cache", zap.Error(err)) + logger.Fatal("failed to init caches", zap.Error(err)) } logger.Info("initializing db") @@ -166,7 +157,10 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { asyncSearchesService = asyncsearches.New(ctx, repo, asyncSearchClient, cfg.Handlers.AsyncSearch) } - seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, inmemWithRedisCache, redisCache, asyncSearchesService) + seqApiCache := caches[cfg.Handlers.SeqAPI.CacheID] + seqApiRedisCache := caches[cfg.Handlers.SeqAPI.RedisID] + + seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) logger.Info("initializing clickhouse") ch, err := initClickHouse(ctx, cfg.Server.CH) @@ -176,7 +170,7 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { var errorGroupsV1 *errorgroups_v1.ErrorGroups if ch != nil { - repo := repositorych.New(ch, cfg.Server.CH.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) + repo := repositorych.New(ch, cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) errorGroupsV1 = errorgroups_v1.New(svc) @@ -208,12 +202,10 @@ func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI confi maxSearchLimit = l } } - clientMaxRecvMsgSize := cfg.AvgDocSize * 1024 * maxSearchLimit if clientMaxRecvMsgSize < defaultClientMaxRecvMsgSize { clientMaxRecvMsgSize = defaultClientMaxRecvMsgSize } - clientParams := seqdb.ClientParams{ Addrs: cfg.Addrs, Timeout: cfg.Timeout, @@ -232,18 +224,15 @@ func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI confi return seqdb.NewGRPCClient(ctx, clientParams) } - func initDb(ctx context.Context, cfg *config_v2.DB) (*pgxpool.Pool, error) { if cfg == nil { logger.Warn("db config is nil, running without db") return nil, nil } - pgxCfg, err := pgxpool.ParseConfig(cfg.ConnString()) if err != nil { return nil, fmt.Errorf("can't parse connection string: %w", err) } - if !*cfg.UsePreparedStatements { // By default, pgx uses the QueryExecModeCacheStatement and automatically prepares and caches prepared statements. // However, this may be incompatible with proxies such as PGBouncer. diff --git a/internal/app/config/config.go b/internal/app/config/config.go index 000e69d..78e7638 100644 --- a/internal/app/config/config.go +++ b/internal/app/config/config.go @@ -5,10 +5,11 @@ import ( "fmt" "os" + "gopkg.in/yaml.v3" + "github.com/ozontech/seq-ui/internal/app/config/migrate" v1 "github.com/ozontech/seq-ui/internal/app/config/v1" v2 "github.com/ozontech/seq-ui/internal/app/config/v2" - "gopkg.in/yaml.v3" ) const ( From ebcf7a8c5489dcef27774809953d42e53467391d Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 29 Jul 2026 13:00:14 +0300 Subject: [PATCH 6/8] update --- cmd/seq-ui/main.go | 22 +++-- .../api/seqapi/v1/grpc/aggregation_test.go | 12 +-- internal/api/seqapi/v1/http/api.go | 54 ++++++++----- internal/api/seqapi/v1/http/events.go | 2 +- internal/api/seqapi/v1/http/fields.go | 26 ++---- internal/api/seqapi/v1/http/fields_test.go | 2 +- internal/api/seqapi/v1/http/logs_lifespan.go | 6 +- internal/api/seqapi/v1/http/search.go | 4 +- .../api/seqapi/v1/http/start_async_search.go | 9 ++- internal/app/auth/oidc.go | 17 +--- internal/app/config/v2/config.go | 48 +++++++++++ internal/app/mw/auth.go | 5 +- internal/app/server/init.go | 2 +- internal/app/server/server.go | 6 +- internal/pkg/cache/ctor.go | 80 +++++++++++++------ 15 files changed, 192 insertions(+), 103 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index c35fc71..f75116d 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -91,9 +91,20 @@ func run(ctx context.Context) { zap.Float64("sampler_param", tracingCfg.SamplerParam)) } - registrar := initApp(ctx, cfg) + logger.Info("initializing caches") + caches, err := cache.FromConfig(ctx, cfg.Cache) + if err != nil { + logger.Fatal("failed to init caches", zap.Error(err)) + } + + var oidcCache cache.Cache + if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { + oidcCache = caches[cfg.Server.Auth.OIDC.CacheID] + } - serv, err := server.New(ctx, cfg.Server, registrar) + registrar := initApp(ctx, cfg, caches) + + serv, err := server.New(ctx, cfg.Server, registrar, oidcCache) if err != nil { logger.Fatal("app init error", zap.Error(err)) } @@ -105,7 +116,7 @@ func run(ctx context.Context) { } } -func initApp(ctx context.Context, cfg config.Config) *api.Registrar { +func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cache) *api.Registrar { logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { @@ -124,11 +135,6 @@ func initApp(ctx context.Context, cfg config.Config) *api.Registrar { massExportV1 = massexport_v1.New(exportServer) } - logger.Info("initializing caches") - caches, err := cache.FromConfig(ctx, cfg.Cache) - if err != nil { - logger.Fatal("failed to init caches", zap.Error(err)) - } logger.Info("initializing db") db, err := initDb(ctx, cfg.DB) diff --git a/internal/api/seqapi/v1/grpc/aggregation_test.go b/internal/api/seqapi/v1/grpc/aggregation_test.go index 785a824..7ee158b 100644 --- a/internal/api/seqapi/v1/grpc/aggregation_test.go +++ b/internal/api/seqapi/v1/grpc/aggregation_test.go @@ -12,7 +12,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -50,7 +50,7 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -65,7 +65,7 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 2, }, }, @@ -82,7 +82,7 @@ func TestGetAggregation(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 1, }, }, @@ -178,7 +178,7 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, @@ -221,7 +221,7 @@ func TestGetAggregationWithNormalization(t *testing.T) { }, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxAggregationsPerRequest: 3, }, }, diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index bb567fc..b8c5f43 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -28,32 +28,43 @@ type apiParams struct { } type apiGlobalParams struct { - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields fields - systemFields fields + fieldsCache *fieldsCache + masker *mask.Masker + pinnedFields fields + systemFields fields + eventsCacheTTL time.Duration + logsLifespanCacheKey string + logsLifespanCacheTTL time.Duration + fieldsCacheTTL time.Duration } type API struct { - config config.SeqAPI - globalParams apiGlobalParams - paramsByEnv map[string]apiParams - cache cache.Cache - nowFn func() time.Time - asyncSearches asyncsearches.Service - envsResponse getEnvsResponse + config config.SeqAPI + globalParams apiGlobalParams + paramsByEnv map[string]apiParams + inmemWithRedisCache cache.Cache + redisCache cache.Cache + nowFn func() time.Time + asyncSearches asyncsearches.Service + envsResponse getEnvsResponse } func New( cfg config.SeqAPI, seqDBСlients map[string]seqdb.Client, - cache cache.Cache, + inmemWithRedisCache cache.Cache, + redisCache cache.Cache, asyncSearches asyncsearches.Service, ) *API { globalParams := apiGlobalParams{ - pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), - systemFields: parseFields(cfg.GlobalOptions.SystemFields), + pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), + systemFields: parseFields(cfg.GlobalOptions.SystemFields), + eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, + logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, + fieldsCacheTTL: cfg.GlobalOptions.FieldsCacheTTL, + logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, } + if cfg.GlobalOptions.FieldsCacheTTL > 0 { globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) } @@ -100,13 +111,14 @@ func New( } return &API{ - config: cfg, - globalParams: globalParams, - paramsByEnv: paramsByEnv, - cache: cache, - nowFn: time.Now, - asyncSearches: asyncSearches, - envsResponse: parseEnvs(cfg), + config: cfg, + globalParams: globalParams, + paramsByEnv: paramsByEnv, + inmemWithRedisCache: inmemWithRedisCache, + redisCache: redisCache, + nowFn: time.Now, + asyncSearches: asyncSearches, + envsResponse: parseEnvs(cfg), } } diff --git a/internal/api/seqapi/v1/http/events.go b/internal/api/seqapi/v1/http/events.go index 6f8154a..f56cd42 100644 --- a/internal/api/seqapi/v1/http/events.go +++ b/internal/api/seqapi/v1/http/events.go @@ -79,7 +79,7 @@ func (a *API) serveGetEvent(w http.ResponseWriter, r *http.Request) { } if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, id, string(data), params.options.EventsCacheTTL) + _ = a.inmemWithRedisCache.SetWithTTL(ctx, id, string(data), a.globalParams.eventsCacheTTL) } else { logger.Error("failed to marshal event proto for caching", zap.String("id", id), zap.Error(err)) } diff --git a/internal/api/seqapi/v1/http/fields.go b/internal/api/seqapi/v1/http/fields.go index 556e8f4..8e99fcc 100644 --- a/internal/api/seqapi/v1/http/fields.go +++ b/internal/api/seqapi/v1/http/fields.go @@ -34,7 +34,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - if params.fieldsCache == nil { + if a.globalParams.fieldsCache == nil { resp, err := params.client.GetFields(ctx, &seqapi.GetFieldsRequest{}) if err != nil { wr.Error(err, http.StatusInternalServerError) @@ -43,14 +43,14 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, } wr.WriteJson(res) return } - rawFields, cached, isActual := params.fieldsCache.getFields() + rawFields, cached, isActual := a.globalParams.fieldsCache.getFields() if cached && isActual { _, _ = wr.Write(rawFields) return @@ -70,8 +70,8 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { res := getFieldsResponse{ Fields: fieldsFromProto(resp.GetFields()), - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, } resData, err := json.Marshal(res) if err != nil { @@ -85,7 +85,7 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { return } - params.fieldsCache.setFields(resData) + a.globalParams.fieldsCache.setFields(resData) _, _ = wr.Write(resData) } @@ -98,18 +98,8 @@ func (a *API) serveGetFields(w http.ResponseWriter, r *http.Request) { // @Success 200 {object} getFieldsResponse "A successful response" // @Failure default {object} httputil.Error "An unexpected error response" func (a *API) serveGetPinnedFields(w http.ResponseWriter, r *http.Request) { - ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_get_pinned_fields") - defer span.End() - - env := getEnvFromContext(ctx) - params, err := a.GetEnvParams(env) - if err != nil { - httputil.NewWriter(w).Error(err, http.StatusBadRequest) - return - } - httputil.NewWriter(w).WriteJson(getFieldsResponse{ - Fields: params.pinnedFields, + Fields: a.globalParams.pinnedFields, }) } diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 81bfb7b..3911f54 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -83,7 +83,7 @@ func TestServeGetFields(t *testing.T) { }, }, }, - cfg: config.SeqAPIOptions{ + cfg: config.Options{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, diff --git a/internal/api/seqapi/v1/http/logs_lifespan.go b/internal/api/seqapi/v1/http/logs_lifespan.go index 50f9a98..9e52e55 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan.go +++ b/internal/api/seqapi/v1/http/logs_lifespan.go @@ -37,9 +37,7 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { return } - cacheKey := params.options.LogsLifespanCacheKey - - if resStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { + if resStr, err := a.redisCache.Get(ctx, a.globalParams.logsLifespanCacheKey); err == nil { res := 0 res, err = strconv.Atoi(resStr) if err == nil { @@ -66,7 +64,7 @@ func (a *API) serveGetLogsLifespan(w http.ResponseWriter, r *http.Request) { res := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(res), params.options.LogsLifespanCacheTTL) + err = a.redisCache.SetWithTTL(ctx, a.globalParams.logsLifespanCacheKey, strconv.Itoa(res), a.globalParams.logsLifespanCacheTTL) if err != nil { logger.Error("can't set logs lifespan to cache", zap.Error(err)) } diff --git a/internal/api/seqapi/v1/http/search.go b/internal/api/seqapi/v1/http/search.go index fd19854..6916242 100644 --- a/internal/api/seqapi/v1/http/search.go +++ b/internal/api/seqapi/v1/http/search.go @@ -140,9 +140,9 @@ func (a *API) serveSearch(w http.ResponseWriter, r *http.Request) { } searchResp := searchResponseFromProto(resp, httpReq.WithTotal) - if params.masker != nil { + if a.globalParams.masker != nil { for i := range searchResp.Events { - params.masker.Mask(searchResp.Events[i].Data) + a.globalParams.masker.Mask(searchResp.Events[i].Data) } } diff --git a/internal/api/seqapi/v1/http/start_async_search.go b/internal/api/seqapi/v1/http/start_async_search.go index 1096ec8..f2b548c 100644 --- a/internal/api/seqapi/v1/http/start_async_search.go +++ b/internal/api/seqapi/v1/http/start_async_search.go @@ -37,6 +37,13 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { ctx, span := tracing.StartSpan(r.Context(), "seqapi_v1_start_async_search") defer span.End() + env := getEnvFromContext(ctx) + params, err := a.GetEnvParams(env) + if err != nil { + httputil.NewWriter(w).Error(err, http.StatusBadRequest) + return + } + var httpReq startAsyncSearchRequest if err := json.NewDecoder(r.Body).Decode(&httpReq); err != nil { wr.Error(fmt.Errorf("failed to parse search request: %w", err), http.StatusBadRequest) @@ -54,7 +61,7 @@ func (a *API) serveStartAsyncSearch(w http.ResponseWriter, r *http.Request) { continue } if err := api_error.CheckAggregationTsInterval(agg.Interval, httpReq.From, httpReq.To, - a.config.MaxBucketsPerAggregationTs, + params.options.MaxBucketsPerAggregationTs, ); err != nil { wr.Error(err, http.StatusBadRequest) return diff --git a/internal/app/auth/oidc.go b/internal/app/auth/oidc.go index 15caf35..4a53756 100644 --- a/internal/app/auth/oidc.go +++ b/internal/app/auth/oidc.go @@ -13,7 +13,7 @@ import ( "github.com/coreos/go-oidc/v3/oidc" "go.uber.org/zap" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/tls" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" @@ -42,21 +42,12 @@ type oidcProvider struct { allowedClients []string } -func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, cacheCfg config.Cache) (OIDCProvider, error) { +func NewOIDCProvider(ctx context.Context, cfg *config.OIDC, oidcCache cache.Cache) (OIDCProvider, error) { var ( - err error - oidcCache cache.Cache - oidcCtx context.Context + err error + oidcCtx context.Context ) - if cfg.CacheSecretKey != "" { - logger.Info("initializing oidc cache") - oidcCache, err = cache.NewInmemoryWithRedisOrInmemory(ctx, cacheCfg) - if err != nil { - return nil, fmt.Errorf("init oidc cache: %w", err) - } - } - if !cfg.SkipVerify { oidcCtx, err = newHTTPContext( context.Background(), diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 9350474..d9c7110 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -277,6 +277,7 @@ type CORS struct { } type OIDC struct { + CacheID string `yaml:"cache_id"` SkipVerify bool `yaml:"skip_verify"` AuthURLs []string `yaml:"auth_urls"` RootCA string `yaml:"root_ca"` @@ -344,6 +345,34 @@ type Cache struct { Redis []Redis `yaml:"redis"` } +func (c *Cache) InmemByID(id string) *InmemoryCache { + if c == nil { + return nil + } + + for i := range c.Inmemory { + if c.Inmemory[i].ID == id { + return &c.Inmemory[i] + } + } + + return nil +} + +func (c *Cache) RedisByID(id string) *Redis { + if c == nil { + return nil + } + + for i := range c.Redis { + if c.Redis[i].ID == id { + return &c.Redis[i] + } + } + + return nil +} + type S3 struct { Endpoint string `yaml:"endpoint"` AccessKeyID string `yaml:"access_key_id"` @@ -472,6 +501,7 @@ type Field struct { type SeqAPI struct { CacheID string `yaml:"cache_id"` + RedisID string `yaml:"redis_id"` GlobalOptions SeqAPIGlobalOptions `yaml:"global_options"` Options SeqAPIOptions `yaml:"options"` Envs map[string]SeqAPIEnv `yaml:"envs"` @@ -687,6 +717,24 @@ func Normalize(cfg *Config) error { return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) } } + + if cfg.Server.Auth != nil && cfg.Server.Auth.OIDC != nil { + oidc := cfg.Server.Auth.OIDC + hasCacheKey := oidc.CacheSecretKey != "" + hasCacheID := oidc.CacheID != "" + + switch { + case hasCacheKey && !hasCacheID: + return fmt.Errorf("auth.oidc.cache_secret_key is set but auth.oidc.cache_id is empty") + case !hasCacheKey && hasCacheID: + return fmt.Errorf("auth.oidc.cache_id is set but auth.oidc.cache_secret_key is empty") + case hasCacheID && hasCacheKey: + if cfg.Cache.InmemByID(oidc.CacheID) == nil && cfg.Cache.RedisByID(oidc.CacheID) == nil { + return fmt.Errorf("auth.oidc.cache_id %q not found", oidc.CacheID) + } + } + } + return nil } diff --git a/internal/app/mw/auth.go b/internal/app/mw/auth.go index ef8ee74..cf7c2ad 100644 --- a/internal/app/mw/auth.go +++ b/internal/app/mw/auth.go @@ -8,6 +8,7 @@ import ( "github.com/ozontech/seq-ui/internal/app/auth" config "github.com/ozontech/seq-ui/internal/app/config/v2" + "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/logger" ) @@ -31,7 +32,7 @@ type AuthProviders struct { func NewAuthProviders( ctx context.Context, jwtSecretKey string, oidcCfg *config.OIDC, - cacheCfg config.Cache, + oidcCache cache.Cache, ) (AuthProviders, error) { authPrvds := AuthProviders{} @@ -47,7 +48,7 @@ func NewAuthProviders( if oidcCfg != nil { logger.Info("initializing oidc provider") var err error - authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, oidcCfg, cacheCfg) + authPrvds.OidcProvider, err = auth.NewOIDCProvider(ctx, oidcCfg, oidcCache) if err != nil { return authPrvds, fmt.Errorf("failed to init oidc provider: %w", err) } diff --git a/internal/app/server/init.go b/internal/app/server/init.go index affc9c8..b3545a8 100644 --- a/internal/app/server/init.go +++ b/internal/app/server/init.go @@ -30,7 +30,7 @@ var defaultCORSAllowedMethods = []string{"HEAD", "GET", "POST", "PATCH", "DELETE func (s *Server) init(ctx context.Context, registrar *api.Registrar) error { var err error - s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.Auth.JWT.SecretKey, s.config.Auth.OIDC, s.Cache) + s.authPrvds, err = mw.NewAuthProviders(ctx, s.config.Auth.JWT.SecretKey, s.config.Auth.OIDC, s.oidcCache) if err != nil { return err } diff --git a/internal/app/server/server.go b/internal/app/server/server.go index 3d9e256..1b5a0ca 100644 --- a/internal/app/server/server.go +++ b/internal/app/server/server.go @@ -10,11 +10,13 @@ import ( "github.com/ozontech/seq-ui/internal/api" config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/mw" + "github.com/ozontech/seq-ui/internal/pkg/cache" ) // Server contains application dependencies. type Server struct { config *config.Server + oidcCache cache.Cache debugServer *http.Server grpcServer *grpc.Server httpServer *http.Server @@ -24,8 +26,8 @@ type Server struct { } // New returns a new Server. -func New(ctx context.Context, cfg *config.Server, registrar *api.Registrar) (*Server, error) { - s := &Server{config: cfg} +func New(ctx context.Context, cfg *config.Server, registrar *api.Registrar, oidcCache cache.Cache) (*Server, error) { + s := &Server{config: cfg, oidcCache: oidcCache} if err := s.init(ctx, registrar); err != nil { return nil, fmt.Errorf("init server: %w", err) diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index b9030c0..5cf2a1c 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -4,43 +4,77 @@ import ( "context" "fmt" - "go.uber.org/zap" - config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" + "go.uber.org/zap" ) -func NewInmemoryWithRedisOrInmemory(ctx context.Context, cfg config.Cache) (Cache, error) { - return newRedisBasedOrInmemory(ctx, cfg, true) -} +func FromConfig(ctx context.Context, cfg *config.Cache) (map[string]Cache, error) { + caches := make(map[string]Cache, len(cfg.Redis)+len(cfg.Inmemory)) -func NewRedisOrInmemory(ctx context.Context, cfg config.Cache) (Cache, error) { - return newRedisBasedOrInmemory(ctx, cfg, false) -} + for i := range cfg.Redis { + redis := &cfg.Redis[i] + var inmem *config.InmemoryCache + if redis.WithInmemID != "" { + inmem = cfg.InmemByID(redis.WithInmemID) + } -func newRedisBasedOrInmemory(ctx context.Context, cfg config.Cache, withInmem bool) (Cache, error) { - inmem, err := newInmemoryCache(cfg.Inmemory) - if err != nil { - return nil, fmt.Errorf("init inmemory cache: %w", err) + cache, err := New(ctx, inmem, redis) + if err != nil { + return nil, fmt.Errorf("new cache %q: %w", redis.ID, err) + } + + caches[redis.ID] = cache } - if cfg.Redis == nil { - logger.Warn("redis cache config is nil; inmemory cache will be used instead") - return inmem, nil + for i := range cfg.Inmemory { + inmem := &cfg.Inmemory[i] + if hasWithInmem(inmem.ID, cfg.Redis) { + continue + } + + cache, err := newInmemoryCache(*inmem) + if err != nil { + return nil, fmt.Errorf("new inmemory cache %q: %w", inmem.ID, err) + } + + caches[inmem.ID] = cache } - redis, err := newRedisCache(ctx, cfg.Redis) + return caches, nil +} + +func New(ctx context.Context, inmemCfg *config.InmemoryCache, redisCfg *config.Redis) (Cache, error) { + redis, redisErr := newRedisCache(ctx, redisCfg) + if inmemCfg == nil { + if redisErr != nil { + return nil, fmt.Errorf("init redis cache: %w", redisErr) + } + return redis, nil + } + + inmem, err := newInmemoryCache(*inmemCfg) if err != nil { + return nil, fmt.Errorf("init inmemory cache: %w", err) + } + + if redisErr != nil { logger.Warn("failed to init redis cache; inmemory cache will be used instead", zap.Error(err)) - return inmem, nil + return inmem, err } - if withInmem { - return &inmemWithRedis{ - inmem: inmem, - redis: redis, - }, nil + return &inmemWithRedis{ + inmem: inmem, + redis: redis, + }, nil +} + +func hasWithInmem(inmemID string, redisCfgs []config.Redis) bool { + for _, r := range redisCfgs { + if r.WithInmemID == inmemID { + return true + } } - return redis, nil + return false } From 7f3a924b353cd5c85c8443ed8c7ad5b8c3ecbcf6 Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Wed, 29 Jul 2026 14:48:12 +0300 Subject: [PATCH 7/8] refactor --- cmd/seq-ui/main.go | 29 +++++++++++++++++++++-------- internal/app/config/v2/config.go | 14 ++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index f75116d..4eb39bb 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -168,15 +168,15 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) - logger.Info("initializing clickhouse") - ch, err := initClickHouse(ctx, cfg.Server.CH) + logger.Info("initializing clickhouse clients") + chClients, err := initClickHouseClients(ctx, cfg) if err != nil { - logger.Fatal("failed to init clickhouse", zap.Error(err)) + logger.Fatal("failed to init clickhouse clients", zap.Error(err)) } var errorGroupsV1 *errorgroups_v1.ErrorGroups - if ch != nil { - repo := repositorych.New(ch, cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) + if chClients != nil { + repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) errorGroupsV1 = errorgroups_v1.New(svc) @@ -186,7 +186,7 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac } func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb.Client, error) { - clients := make(map[string]seqdb.Client) + clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { client, err := createSeqBDClient(ctx, clientCfg, cfg.Handlers.SeqAPI) if err != nil { @@ -230,7 +230,7 @@ func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI confi return seqdb.NewGRPCClient(ctx, clientParams) } -func initDb(ctx context.Context, cfg *config_v2.DB) (*pgxpool.Pool, error) { +func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { if cfg == nil { logger.Warn("db config is nil, running without db") return nil, nil @@ -270,7 +270,20 @@ func initExportService(ctx context.Context, cfg config.MassExport, client seqdb. return massexport.NewService(ctx, cfg, sessionStore, fileStore, client) } -func initClickHouse(ctx context.Context, cfg *config.CH) (driver.Conn, error) { +func initClickHouseClients(ctx context.Context, cfg config.Config) (map[string]driver.Conn, error) { + clients := make(map[string]driver.Conn, len(cfg.Clients.ClickHouse)) + for _, clientCfg := range cfg.Clients.ClickHouse { + client, err := createClickHouseClient(ctx, &clientCfg) + if err != nil { + return nil, fmt.Errorf("failed to create clickhouse client %s: %w", clientCfg.ID, err) + } + clients[clientCfg.ID] = client + } + + return clients, nil +} + +func createClickHouseClient(ctx context.Context, cfg *config.CHClient) (driver.Conn, error) { if cfg == nil { logger.Warn("clickhouse config is nil, running without clickhouse") return nil, nil diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index d9c7110..286e836 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -487,6 +487,20 @@ type Clients struct { ClickHouse []CHClient `yaml:"clickhouse"` } +func (c *Clients) ClickHouseByID(id string) *CHClient { + if c == nil { + return nil + } + + for i := range c.ClickHouse { + if c.ClickHouse[i].ID == id { + return &c.ClickHouse[i] + } + } + + return nil +} + type Handlers struct { SeqAPI SeqAPI `yaml:"seq_api"` ErrorGroups ErrorGroups `yaml:"error_groups"` From cb6d68cc76ee8fd64a932b6c384a0edf3acb169b Mon Sep 17 00:00:00 2001 From: Sergey Lazarenko Date: Thu, 30 Jul 2026 02:35:38 +0300 Subject: [PATCH 8/8] big update --- cmd/seq-ui/main.go | 62 +++++------ internal/api/seqapi/v1/grpc/aggregation.go | 4 +- internal/api/seqapi/v1/grpc/api.go | 103 ++++++++---------- internal/api/seqapi/v1/grpc/events.go | 10 +- internal/api/seqapi/v1/grpc/events_test.go | 6 +- internal/api/seqapi/v1/grpc/fields.go | 33 ++---- internal/api/seqapi/v1/grpc/fields_test.go | 12 +- internal/api/seqapi/v1/grpc/get_envs_test.go | 8 +- internal/api/seqapi/v1/grpc/limits_test.go | 4 +- internal/api/seqapi/v1/grpc/logs_lifespan.go | 6 +- .../api/seqapi/v1/grpc/logs_lifespan_test.go | 4 +- internal/api/seqapi/v1/grpc/search.go | 4 +- internal/api/seqapi/v1/grpc/search_test.go | 14 +-- internal/api/seqapi/v1/grpc/test_data.go | 10 +- internal/api/seqapi/v1/http/api.go | 8 +- internal/api/seqapi/v1/http/events_test.go | 6 +- internal/api/seqapi/v1/http/export_test.go | 14 +-- internal/api/seqapi/v1/http/fields_test.go | 12 +- internal/api/seqapi/v1/http/get_envs_test.go | 15 ++- internal/api/seqapi/v1/http/limits_test.go | 4 +- .../api/seqapi/v1/http/logs_lifespan_test.go | 4 +- internal/api/seqapi/v1/http/search_test.go | 26 ++--- internal/api/seqapi/v1/http/test_data.go | 10 +- internal/api/seqapi/v1/seqapi.go | 2 +- internal/app/config/migrate/v1_v2.go | 10 +- internal/app/config/v2/config.go | 93 ++++++++-------- internal/pkg/cache/ctor.go | 3 +- .../seqdb/seqproxyapi/v1/seq_proxy_api.pb.go | 2 +- .../seqproxyapi/v1/seq_proxy_api_grpc.pb.go | 2 +- .../massexport/seq_proxy_downloader.go | 4 +- internal/pkg/service/massexport/service.go | 2 +- .../service/massexport/sessionstore/redis.go | 16 +-- pkg/dashboards/v1/dashboards.pb.go | 2 +- pkg/dashboards/v1/dashboards_grpc.pb.go | 2 +- pkg/errorgroups/v1/errorgroups.pb.go | 2 +- pkg/errorgroups/v1/errorgroups_grpc.pb.go | 2 +- pkg/massexport/v1/massexport.pb.go | 2 +- pkg/massexport/v1/massexport_grpc.pb.go | 2 +- pkg/seqapi/v1/seq_api.pb.go | 2 +- pkg/seqapi/v1/seq_api_grpc.pb.go | 2 +- pkg/userprofile/v1/userprofile.pb.go | 2 +- pkg/userprofile/v1/userprofile_grpc.pb.go | 2 +- 42 files changed, 250 insertions(+), 283 deletions(-) diff --git a/cmd/seq-ui/main.go b/cmd/seq-ui/main.go index 4eb39bb..fdadaeb 100644 --- a/cmd/seq-ui/main.go +++ b/cmd/seq-ui/main.go @@ -120,20 +120,13 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac logger.Info("initializing seq-db clients") seqDBClients, err := initSeqDBClients(ctx, cfg) if err != nil { - logger.Fatal("failed to init seq-db client", zap.Error(err)) + logger.Fatal("failed to init seq-db clients", zap.Error(err)) } - var massExportV1 *massexport_v1.MassExport - if cfg.Handlers.MassExport != nil { - massExportClient, ok := seqDBClients[cfg.Handlers.MassExport.SeqDBID] - if !ok { - logger.Fatal("mass_export seq_db client not found", zap.String("seq_db_id", cfg.Handlers.MassExport.SeqDBID)) - } - exportServer, err := initExportService(ctx, *cfg.Handlers.MassExport, massExportClient) - if err != nil { - logger.Fatal("can't init export server", zap.Error(err)) - } - massExportV1 = massexport_v1.New(exportServer) + logger.Info("initializing clickhouse clients") + chClients, err := initClickHouseClients(ctx, cfg) + if err != nil { + logger.Fatal("failed to init clickhouse clients", zap.Error(err)) } logger.Info("initializing db") @@ -142,6 +135,18 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac logger.Fatal("failed to init db", zap.Error(err)) } + var massExportV1 *massexport_v1.MassExport + if cfg.Handlers.MassExport != nil { + me := cfg.Handlers.MassExport + redisCfg := cfg.Cache.RedisByID(me.SessionStore.RedisID) + + exportServer, err := initExportService(ctx, *me, redisCfg, seqDBClients[cfg.Handlers.MassExport.SeqDBID]) + if err != nil { + logger.Fatal("can't init export server", zap.Error(err)) + } + massExportV1 = massexport_v1.New(exportServer) + } + var ( asyncSearchesService asyncsearches.Service userProfileV1 *userprofile_v1.UserProfile @@ -156,27 +161,17 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac userProfileV1 = userprofile_v1.New(userProfilesSvc) dashboardsV1 = dashboards_v1.New(dashboardsSvc) - asyncSearchClient, ok := seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID] - if !ok { - logger.Fatal("async_search seq_db client not found", zap.String("seq_db_id", cfg.Handlers.AsyncSearch.SeqDBID)) - } - asyncSearchesService = asyncsearches.New(ctx, repo, asyncSearchClient, cfg.Handlers.AsyncSearch) + asyncSearchesService = asyncsearches.New(ctx, repo, seqDBClients[cfg.Handlers.AsyncSearch.SeqDBID], cfg.Handlers.AsyncSearch) } seqApiCache := caches[cfg.Handlers.SeqAPI.CacheID] seqApiRedisCache := caches[cfg.Handlers.SeqAPI.RedisID] - - seqApiV1 := seqapi_v1.New(cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) - - logger.Info("initializing clickhouse clients") - chClients, err := initClickHouseClients(ctx, cfg) - if err != nil { - logger.Fatal("failed to init clickhouse clients", zap.Error(err)) - } + seqApiV1 := seqapi_v1.New(&cfg.Handlers.SeqAPI, seqDBClients, seqApiCache, seqApiRedisCache, asyncSearchesService) var errorGroupsV1 *errorgroups_v1.ErrorGroups - if chClients != nil { - repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], cfg.Clients.ClickHouse.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) + if cfg.Handlers.ErrorGroups != nil { + chCfg := cfg.Clients.ClickHouseByID(cfg.Handlers.ErrorGroups.CHID) + repo := repositorych.New(chClients[cfg.Handlers.ErrorGroups.CHID], chCfg.Sharded, cfg.Handlers.ErrorGroups.QueryFilter) svc := errorgroups.New(repo, cfg.Handlers.ErrorGroups.LogTagsMapping) errorGroupsV1 = errorgroups_v1.New(svc) @@ -188,7 +183,7 @@ func initApp(ctx context.Context, cfg config.Config, caches map[string]cache.Cac func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb.Client, error) { clients := make(map[string]seqdb.Client, len(cfg.Clients.SeqDB)) for _, clientCfg := range cfg.Clients.SeqDB { - client, err := createSeqBDClient(ctx, clientCfg, cfg.Handlers.SeqAPI) + client, err := createSeqBDClient(ctx, &clientCfg, &cfg.Handlers.SeqAPI) if err != nil { return nil, fmt.Errorf("failed to create seq_db client %s: %w", clientCfg.ID, err) } @@ -198,7 +193,7 @@ func initSeqDBClients(ctx context.Context, cfg config.Config) (map[string]seqdb. return clients, nil } -func createSeqBDClient(ctx context.Context, cfg config.SeqDBClient, seqAPI config.SeqAPI) (seqdb.Client, error) { +func createSeqBDClient(ctx context.Context, cfg *config.SeqDBClient, seqAPI *config.SeqAPI) (seqdb.Client, error) { maxSearchLimit := int(seqAPI.Options.MaxSearchLimit) for _, env := range seqAPI.Envs { if env.SeqDBID != cfg.ID { @@ -254,8 +249,8 @@ func initDb(ctx context.Context, cfg *config.DB) (*pgxpool.Pool, error) { return pool, nil } -func initExportService(ctx context.Context, cfg config.MassExport, client seqdb.Client) (massexport.Service, error) { - sessionStore, err := sessionstore.NewRedisSessionStore(ctx, cfg.SessionStore) +func initExportService(ctx context.Context, cfg config.MassExport, redisCfg *config.Redis, client seqdb.Client) (massexport.Service, error) { + sessionStore, err := sessionstore.NewRedisSessionStore(ctx, redisCfg, cfg.SessionStore.ExportLifetime) if err != nil { return nil, fmt.Errorf("init session store: %w", err) } @@ -284,11 +279,6 @@ func initClickHouseClients(ctx context.Context, cfg config.Config) (map[string]d } func createClickHouseClient(ctx context.Context, cfg *config.CHClient) (driver.Conn, error) { - if cfg == nil { - logger.Warn("clickhouse config is nil, running without clickhouse") - return nil, nil - } - conn, err := clickhouse.Open(&clickhouse.Options{ Addr: cfg.Addrs, Auth: clickhouse.Auth{ diff --git a/internal/api/seqapi/v1/grpc/aggregation.go b/internal/api/seqapi/v1/grpc/aggregation.go index e8fd760..9603b88 100644 --- a/internal/api/seqapi/v1/grpc/aggregation.go +++ b/internal/api/seqapi/v1/grpc/aggregation.go @@ -80,7 +80,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ return nil, err } - if params.masker != nil { + if a.globalParams.masker != nil { buf := make([]string, 0) for i, agg := range resp.Aggregations { if agg == nil { @@ -98,7 +98,7 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ field = aggReq.GroupBy } - buf = params.masker.MaskAgg(field, buf) + buf = a.globalParams.masker.MaskAgg(field, buf) for j, key := range buf { if agg.Buckets[j] != nil { diff --git a/internal/api/seqapi/v1/grpc/api.go b/internal/api/seqapi/v1/grpc/api.go index da9a784..cc75ad0 100644 --- a/internal/api/seqapi/v1/grpc/api.go +++ b/internal/api/seqapi/v1/grpc/api.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" "google.golang.org/grpc/metadata" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/app/types" "github.com/ozontech/seq-ui/internal/pkg/cache" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" @@ -21,19 +21,25 @@ import ( ) type apiParams struct { - client seqdb.Client - options *config.SeqAPIOptions - fieldsCache *fieldsCache - masker *mask.Masker - pinnedFields []*seqapi.Field - systemFields []*seqapi.Field + client seqdb.Client + options *config.SeqAPIOptions +} + +type apiGlobalParams struct { + fieldsCache *fieldsCache + masker *mask.Masker + pinnedFields []*seqapi.Field + systemFields []*seqapi.Field + eventsCacheTTL time.Duration + logsLifespanCacheKey string + logsLifespanCacheTTL time.Duration } type API struct { seqapi.UnimplementedSeqAPIServiceServer - config config.SeqAPI - params apiParams + config *config.SeqAPI + globalParams apiGlobalParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache redisCache cache.Cache @@ -43,76 +49,57 @@ type API struct { } func New( - cfg config.SeqAPI, + cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, asyncSearches asyncsearches.Service, ) *API { - var globalfCache *fieldsCache - if cfg.FieldsCacheTTL > 0 { - globalfCache = newFieldsCache(cfg.FieldsCacheTTL) + globalParams := apiGlobalParams{ + pinnedFields: parseFields(cfg.GlobalOptions.PinnedFields), + systemFields: parseFields(cfg.GlobalOptions.SystemFields), + eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, + logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, + logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, + } + + if cfg.GlobalOptions.FieldsCacheTTL > 0 { + globalParams.fieldsCache = newFieldsCache(cfg.GlobalOptions.FieldsCacheTTL) } - globalMasker, err := mask.New(cfg.Masking) + globalMasker, err := mask.New(cfg.GlobalOptions.Masking) if err != nil { logger.Fatal("failed to init masking", zap.Error(err)) } + globalParams.masker = globalMasker - globalPinnedFields := parseFields(cfg.PinnedFields) - globalSystemFields := parseFields(cfg.SystemFields) - - var params apiParams - var paramsByEnv map[string]apiParams - + paramsByEnv := make(map[string]apiParams, max(1, len(cfg.Envs))) if len(cfg.Envs) > 0 { - paramsByEnv = make(map[string]apiParams) for envName, envConfig := range cfg.Envs { - client := seqDBСlients[envConfig.SeqDB] - options := envConfig.Options - - var envfCache *fieldsCache - if options.FieldsCacheTTL > 0 { - envfCache = newFieldsCache(options.FieldsCacheTTL) - } - - var envMasker, err = mask.New(options.Masking) - if err != nil { - logger.Fatal("failed to init env masking", zap.Error(err)) - } - - envPinnedFields := parseFields(options.PinnedFields) - envSystemFields := parseFields(options.SystemFields) - paramsByEnv[envName] = apiParams{ - client: client, - options: options, - fieldsCache: envfCache, - masker: envMasker, - pinnedFields: envPinnedFields, - systemFields: envSystemFields, + client: seqDBСlients[envConfig.SeqDBID], + options: envConfig.Options, } } } else { - client, exists := seqDBСlients[config.DefaultSeqDBClientID] - if !exists { - logger.Fatal("default client not found", - zap.String("clientID", config.DefaultSeqDBClientID)) + if len(seqDBСlients) != 1 { + logger.Fatal("seq_api.envs is empty, configure envs when clients.seq_db has more than one instance") + } + + var client seqdb.Client + for _, c := range seqDBСlients { + client = c } - params = apiParams{ - client: client, - options: cfg.SeqAPIOptions, - fieldsCache: globalfCache, - masker: globalMasker, - pinnedFields: globalPinnedFields, - systemFields: globalSystemFields, + paramsByEnv[""] = apiParams{ + client: client, + options: &cfg.Options, } } return &API{ config: cfg, - params: params, + globalParams: globalParams, paramsByEnv: paramsByEnv, inmemWithRedisCache: inmemWithRedisCache, redisCache: redisCache, @@ -133,7 +120,7 @@ func parseFields(fields []config.Field) []*seqapi.Field { return res } -func parseEnvs(cfg config.SeqAPI) *seqapi.GetEnvsResponse { +func parseEnvs(cfg *config.SeqAPI) *seqapi.GetEnvsResponse { var envs []*seqapi.GetEnvsResponse_Env if len(cfg.Envs) > 0 { // sort environment names to ensure deterministic output @@ -177,7 +164,7 @@ func parseEnvs(cfg config.SeqAPI) *seqapi.GetEnvsResponse { envs = append(envs, createEnvInfo(name, envConfig.Options)) } } else { - envs = []*seqapi.GetEnvsResponse_Env{createEnvInfo("", cfg.SeqAPIOptions)} + envs = []*seqapi.GetEnvsResponse_Env{createEnvInfo("", &cfg.Options)} } return &seqapi.GetEnvsResponse{Envs: envs} } @@ -193,7 +180,7 @@ func (a *API) GetEnvFromContext(ctx context.Context) string { func (a *API) GetParams(env string) (apiParams, error) { if len(a.config.Envs) == 0 { - return a.params, nil + return a.paramsByEnv[""], nil } if env == "" { diff --git a/internal/api/seqapi/v1/grpc/events.go b/internal/api/seqapi/v1/grpc/events.go index a4665b9..aea707b 100644 --- a/internal/api/seqapi/v1/grpc/events.go +++ b/internal/api/seqapi/v1/grpc/events.go @@ -41,8 +41,8 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap if cached, err := a.inmemWithRedisCache.Get(ctx, req.Id); err == nil { event := &seqapi.Event{} if err = proto.Unmarshal([]byte(cached), event); err == nil { - if params.masker != nil { - params.masker.Mask(event.Data) + if a.globalParams.masker != nil { + a.globalParams.masker.Mask(event.Data) } return &seqapi.GetEventResponse{Event: event}, nil } @@ -53,12 +53,12 @@ func (a *API) GetEvent(ctx context.Context, req *seqapi.GetEventRequest) (*seqap return nil, err } - if params.masker != nil && resp.Event != nil { - params.masker.Mask(resp.Event.Data) + if a.globalParams.masker != nil && resp.Event != nil { + a.globalParams.masker.Mask(resp.Event.Data) } if data, err := proto.Marshal(resp.Event); err == nil { - _ = a.inmemWithRedisCache.SetWithTTL(ctx, req.Id, string(data), params.options.EventsCacheTTL) + _ = a.inmemWithRedisCache.SetWithTTL(ctx, req.Id, string(data), a.globalParams.eventsCacheTTL) } else { logger.Error("failed to marshal event proto for caching", zap.String("id", req.Id), zap.Error(err)) } diff --git a/internal/api/seqapi/v1/grpc/events_test.go b/internal/api/seqapi/v1/grpc/events_test.go index 3f10c1f..b910470 100644 --- a/internal/api/seqapi/v1/grpc/events_test.go +++ b/internal/api/seqapi/v1/grpc/events_test.go @@ -14,7 +14,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -104,7 +104,7 @@ func TestGetEvent(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, }, }, @@ -369,7 +369,7 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/grpc/fields.go b/internal/api/seqapi/v1/grpc/fields.go index bae2836..f07109f 100644 --- a/internal/api/seqapi/v1/grpc/fields.go +++ b/internal/api/seqapi/v1/grpc/fields.go @@ -22,23 +22,23 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq return nil, status.Error(codes.InvalidArgument, err.Error()) } - if params.fieldsCache == nil { + if a.globalParams.fieldsCache == nil { resp, err := params.client.GetFields(ctx, req) if err != nil { return nil, err } - resp.SystemFields = params.systemFields - resp.PinnedFields = params.pinnedFields + resp.SystemFields = a.globalParams.systemFields + resp.PinnedFields = a.globalParams.pinnedFields return resp, nil } - fields, cached, isActual := params.fieldsCache.getFields() + fields, cached, isActual := a.globalParams.fieldsCache.getFields() if cached && isActual { return &seqapi.GetFieldsResponse{ Fields: fields, - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, }, nil } @@ -48,31 +48,22 @@ func (a *API) GetFields(ctx context.Context, req *seqapi.GetFieldsRequest) (*seq logger.Error("can't get fields; use cached fields", zap.Error(err)) return &seqapi.GetFieldsResponse{ Fields: fields, - SystemFields: params.systemFields, - PinnedFields: params.pinnedFields, + SystemFields: a.globalParams.systemFields, + PinnedFields: a.globalParams.pinnedFields, }, nil } return nil, err } - params.fieldsCache.setFields(resp.GetFields()) - resp.SystemFields = params.systemFields - resp.PinnedFields = params.pinnedFields + a.globalParams.fieldsCache.setFields(resp.GetFields()) + resp.SystemFields = a.globalParams.systemFields + resp.PinnedFields = a.globalParams.pinnedFields return resp, nil } func (a *API) GetPinnedFields(ctx context.Context, _ *seqapi.GetFieldsRequest) (*seqapi.GetFieldsResponse, error) { - ctx, span := tracing.StartSpan(ctx, "seqapi_v1_get_fields") - defer span.End() - - env := a.GetEnvFromContext(ctx) - params, err := a.GetParams(env) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - return &seqapi.GetFieldsResponse{ - Fields: params.pinnedFields, + Fields: a.globalParams.pinnedFields, }, nil } diff --git a/internal/api/seqapi/v1/grpc/fields_test.go b/internal/api/seqapi/v1/grpc/fields_test.go index f52149d..5df65b0 100644 --- a/internal/api/seqapi/v1/grpc/fields_test.go +++ b/internal/api/seqapi/v1/grpc/fields_test.go @@ -11,7 +11,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -20,7 +20,7 @@ func TestGetFields(t *testing.T) { tests := []struct { name string - cfg config.SeqAPIOptions + cfg config.SeqAPIGlobalOptions seqDBResp *seqapi.GetFieldsResponse wantResp *seqapi.GetFieldsResponse @@ -55,7 +55,7 @@ func TestGetFields(t *testing.T) { }, { name: "ok_with_system_and_pinned_fields", - cfg: config.SeqAPIOptions{ + cfg: config.SeqAPIGlobalOptions{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, @@ -121,7 +121,7 @@ func TestGetFields(t *testing.T) { SeqDB: seqDbMock, }, Cfg: config.SeqAPI{ - SeqAPIOptions: &tt.cfg, + GlobalOptions: tt.cfg, }, } @@ -173,7 +173,7 @@ func TestGetFieldsCached(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ FieldsCacheTTL: ttl, }, }, @@ -222,7 +222,7 @@ func TestGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/grpc/get_envs_test.go b/internal/api/seqapi/v1/grpc/get_envs_test.go index 5f5a1a4..4deb8be 100644 --- a/internal/api/seqapi/v1/grpc/get_envs_test.go +++ b/internal/api/seqapi/v1/grpc/get_envs_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -20,7 +20,7 @@ func TestGetEnvs(t *testing.T) { { name: "single_env", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, @@ -144,8 +144,8 @@ func TestGetEnvs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() api := API{ - config: tt.cfg, - envsResponse: parseEnvs(tt.cfg), + config: &tt.cfg, + envsResponse: parseEnvs(&tt.cfg), } resp, err := api.GetEnvs(context.TODO(), &seqapi.GetEnvsRequest{}) require.NoError(t, err) diff --git a/internal/api/seqapi/v1/grpc/limits_test.go b/internal/api/seqapi/v1/grpc/limits_test.go index 30892c6..f38efe3 100644 --- a/internal/api/seqapi/v1/grpc/limits_test.go +++ b/internal/api/seqapi/v1/grpc/limits_test.go @@ -8,7 +8,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -21,7 +21,7 @@ func TestGetLimits(t *testing.T) { { name: "ok", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan.go b/internal/api/seqapi/v1/grpc/logs_lifespan.go index d54eb89..09ed1fc 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan.go @@ -30,9 +30,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ return nil, status.Error(codes.InvalidArgument, err.Error()) } - cacheKey := params.options.LogsLifespanCacheKey - - if countStr, err := a.redisCache.Get(ctx, cacheKey); err == nil { + if countStr, err := a.redisCache.Get(ctx, a.globalParams.logsLifespanCacheKey); err == nil { count := 0 count, err = strconv.Atoi(countStr) if err == nil { @@ -58,7 +56,7 @@ func (a *API) GetLogsLifespan(ctx context.Context, _ *seqapi.GetLogsLifespanRequ count := int(a.nowFn().Sub(clientStatus.OldestStorageTime.AsTime()) / lifespan.MeasureUnit) res := time.Duration(count) * lifespan.MeasureUnit - err = a.redisCache.SetWithTTL(ctx, cacheKey, strconv.Itoa(count), params.options.LogsLifespanCacheTTL) + err = a.redisCache.SetWithTTL(ctx, a.globalParams.logsLifespanCacheKey, strconv.Itoa(count), a.globalParams.logsLifespanCacheTTL) if err != nil { logger.Error("can't set logs lifespan to cache", zap.Error(err)) } diff --git a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go index 9b2fc2f..a82cb88 100644 --- a/internal/api/seqapi/v1/grpc/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/grpc/logs_lifespan_test.go @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" @@ -106,7 +106,7 @@ func TestGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ LogsLifespanCacheKey: cacheKey, LogsLifespanCacheTTL: cacheTTL, }, diff --git a/internal/api/seqapi/v1/grpc/search.go b/internal/api/seqapi/v1/grpc/search.go index 7fb45fb..02149cf 100644 --- a/internal/api/seqapi/v1/grpc/search.go +++ b/internal/api/seqapi/v1/grpc/search.go @@ -123,9 +123,9 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se } } - if params.masker != nil { + if a.globalParams.masker != nil { for _, e := range resp.Events { - params.masker.Mask(e.Data) + a.globalParams.masker.Mask(e.Data) } } diff --git a/internal/api/seqapi/v1/grpc/search_test.go b/internal/api/seqapi/v1/grpc/search_test.go index b818531..ce1644c 100644 --- a/internal/api/seqapi/v1/grpc/search_test.go +++ b/internal/api/seqapi/v1/grpc/search_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -62,7 +62,7 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxAggregationsPerRequest: 5, }, @@ -81,7 +81,7 @@ func TestSearch(t *testing.T) { Limit: 10, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }, @@ -98,7 +98,7 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxAggregationsPerRequest: 2, }, @@ -115,7 +115,7 @@ func TestSearch(t *testing.T) { Offset: 11, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchOffsetLimit: 10, }, @@ -145,7 +145,7 @@ func TestSearch(t *testing.T) { }, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchTotalLimit: int64(limit), }, @@ -161,7 +161,7 @@ func TestSearch(t *testing.T) { Offset: 0, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), diff --git a/internal/api/seqapi/v1/grpc/test_data.go b/internal/api/seqapi/v1/grpc/test_data.go index a088897..ce8ed4e 100644 --- a/internal/api/seqapi/v1/grpc/test_data.go +++ b/internal/api/seqapi/v1/grpc/test_data.go @@ -5,7 +5,7 @@ import ( "time" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -18,15 +18,11 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - // when test cases don't explicitly provide configuration - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} - } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB for _, envConfig := range data.Cfg.Envs { - seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB + seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB } var asyncSvc asyncsearches.Service @@ -34,5 +30,5 @@ func setupTestAPI(data test.APITestData) *API { asyncSvc = data.Mocks.AsyncSearchesSvc } - return New(data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) } diff --git a/internal/api/seqapi/v1/http/api.go b/internal/api/seqapi/v1/http/api.go index b8c5f43..2f5b4b0 100644 --- a/internal/api/seqapi/v1/http/api.go +++ b/internal/api/seqapi/v1/http/api.go @@ -35,11 +35,10 @@ type apiGlobalParams struct { eventsCacheTTL time.Duration logsLifespanCacheKey string logsLifespanCacheTTL time.Duration - fieldsCacheTTL time.Duration } type API struct { - config config.SeqAPI + config *config.SeqAPI globalParams apiGlobalParams paramsByEnv map[string]apiParams inmemWithRedisCache cache.Cache @@ -50,7 +49,7 @@ type API struct { } func New( - cfg config.SeqAPI, + cfg *config.SeqAPI, seqDBСlients map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, @@ -61,7 +60,6 @@ func New( systemFields: parseFields(cfg.GlobalOptions.SystemFields), eventsCacheTTL: cfg.GlobalOptions.EventsCacheTTL, logsLifespanCacheKey: cfg.GlobalOptions.LogsLifespanCacheKey, - fieldsCacheTTL: cfg.GlobalOptions.FieldsCacheTTL, logsLifespanCacheTTL: cfg.GlobalOptions.LogsLifespanCacheTTL, } @@ -161,7 +159,7 @@ func parseFields(fields []config.Field) []field { return res } -func parseEnvs(cfg config.SeqAPI) getEnvsResponse { +func parseEnvs(cfg *config.SeqAPI) getEnvsResponse { var envs []envInfo if len(cfg.Envs) > 0 { // sort environment names to ensure deterministic output diff --git a/internal/api/seqapi/v1/http/events_test.go b/internal/api/seqapi/v1/http/events_test.go index 7b33966..619fe6d 100644 --- a/internal/api/seqapi/v1/http/events_test.go +++ b/internal/api/seqapi/v1/http/events_test.go @@ -14,7 +14,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" @@ -121,7 +121,7 @@ func TestServeGetEvent(t *testing.T) { ctrl := gomock.NewController(t) seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, }, }, @@ -381,7 +381,7 @@ func TestGetEventWithMasking(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ EventsCacheTTL: cacheTTL, Masking: tt.maskingCfg, }, diff --git a/internal/api/seqapi/v1/http/export_test.go b/internal/api/seqapi/v1/http/export_test.go index d203aa3..5c36e14 100644 --- a/internal/api/seqapi/v1/http/export_test.go +++ b/internal/api/seqapi/v1/http/export_test.go @@ -10,7 +10,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -40,7 +40,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, @@ -67,7 +67,7 @@ func TestServeExport(t *testing.T) { Fields: []string{"field1", "field2"}, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, @@ -94,7 +94,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxParallelExportRequests: 0, }, }, @@ -110,7 +110,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 5, MaxParallelExportRequests: 1, }, @@ -128,7 +128,7 @@ func TestServeExport(t *testing.T) { Format: efCSV, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, @@ -145,7 +145,7 @@ func TestServeExport(t *testing.T) { Offset: 0, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxExportLimit: 100, MaxParallelExportRequests: 1, }, diff --git a/internal/api/seqapi/v1/http/fields_test.go b/internal/api/seqapi/v1/http/fields_test.go index 3911f54..3d6a26b 100644 --- a/internal/api/seqapi/v1/http/fields_test.go +++ b/internal/api/seqapi/v1/http/fields_test.go @@ -9,7 +9,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -23,7 +23,7 @@ func TestServeGetFields(t *testing.T) { tests := []struct { name string - cfg config.SeqAPIOptions + cfg config.SeqAPIGlobalOptions want getFieldsResponse wantErr bool @@ -83,7 +83,7 @@ func TestServeGetFields(t *testing.T) { }, }, }, - cfg: config.Options{ + cfg: config.SeqAPIGlobalOptions{ SystemFields: []config.Field{ {Name: "field1", Type: "keyword"}, {Name: "field2", Type: "text"}, @@ -110,7 +110,7 @@ func TestServeGetFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &tt.cfg, + GlobalOptions: tt.cfg, }, } @@ -198,7 +198,7 @@ func TestServeGetFieldsCached(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ FieldsCacheTTL: ttl, }, }, @@ -256,7 +256,7 @@ func TestServeGetPinnedFields(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ PinnedFields: tt.fields, }, }, diff --git a/internal/api/seqapi/v1/http/get_envs_test.go b/internal/api/seqapi/v1/http/get_envs_test.go index 3ede5d5..53ddae3 100644 --- a/internal/api/seqapi/v1/http/get_envs_test.go +++ b/internal/api/seqapi/v1/http/get_envs_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetEnvs(t *testing.T) { @@ -18,7 +18,7 @@ func TestServeGetEnvs(t *testing.T) { { name: "single_env", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, @@ -42,10 +42,9 @@ func TestServeGetEnvs(t *testing.T) { { name: "ok_multiple_envs", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{}, Envs: map[string]config.SeqAPIEnv{ "cluster-220": { - SeqDB: "pro-seqdb", + SeqDBID: "pro-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 1000, MaxExportLimit: 500, @@ -55,7 +54,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "cluster-10": { - SeqDB: "prod-seqdb", + SeqDBID: "prod-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 1000, MaxExportLimit: 500, @@ -65,7 +64,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "cluster-102": { - SeqDB: "staging-seqdb", + SeqDBID: "staging-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 500, MaxExportLimit: 250, @@ -75,7 +74,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "prod": { - SeqDB: "stag-seqdb", + SeqDBID: "stag-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 500, MaxExportLimit: 250, @@ -85,7 +84,7 @@ func TestServeGetEnvs(t *testing.T) { }, }, "wyanki": { - SeqDB: "sta-seqdb", + SeqDBID: "sta-seqdb", Options: &config.SeqAPIOptions{ MaxSearchLimit: 500, MaxExportLimit: 250, diff --git a/internal/api/seqapi/v1/http/limits_test.go b/internal/api/seqapi/v1/http/limits_test.go index cf339e4..1a8dca9 100644 --- a/internal/api/seqapi/v1/http/limits_test.go +++ b/internal/api/seqapi/v1/http/limits_test.go @@ -6,7 +6,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" ) func TestServeGetLimits(t *testing.T) { @@ -21,7 +21,7 @@ func TestServeGetLimits(t *testing.T) { name: "ok", env: "default", cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 100, MaxExportLimit: 200, MaxParallelExportRequests: 2, diff --git a/internal/api/seqapi/v1/http/logs_lifespan_test.go b/internal/api/seqapi/v1/http/logs_lifespan_test.go index 0ed4b5c..692ef86 100644 --- a/internal/api/seqapi/v1/http/logs_lifespan_test.go +++ b/internal/api/seqapi/v1/http/logs_lifespan_test.go @@ -12,7 +12,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/cache" mock_cache "github.com/ozontech/seq-ui/internal/pkg/cache/mock" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" @@ -103,7 +103,7 @@ func TestServeGetLogsLifespan(t *testing.T) { seqData := test.APITestData{ Cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + GlobalOptions: config.SeqAPIGlobalOptions{ LogsLifespanCacheKey: cacheKey, LogsLifespanCacheTTL: cacheTTL, }, diff --git a/internal/api/seqapi/v1/http/search_test.go b/internal/api/seqapi/v1/http/search_test.go index c22272f..9e92656 100644 --- a/internal/api/seqapi/v1/http/search_test.go +++ b/internal/api/seqapi/v1/http/search_test.go @@ -11,7 +11,7 @@ import ( "github.com/ozontech/seq-ui/internal/api/httputil" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/api_error" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" mock_seqdb "github.com/ozontech/seq-ui/internal/pkg/client/seqdb/mock" "github.com/ozontech/seq-ui/pkg/seqapi/v1" ) @@ -51,7 +51,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -87,7 +87,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -124,7 +124,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -163,7 +163,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -211,7 +211,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -255,7 +255,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -289,7 +289,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: true, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), @@ -330,7 +330,7 @@ func TestServeSearch(t *testing.T) { Limit: 10, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }, @@ -346,7 +346,7 @@ func TestServeSearch(t *testing.T) { Aggregations: aggregationQueries{{}, {}, {}}, }, cfg: config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxAggregationsPerRequest: 2, }, @@ -363,7 +363,7 @@ func TestServeSearch(t *testing.T) { Offset: 11, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchOffsetLimit: 10, }, @@ -386,7 +386,7 @@ func TestServeSearch(t *testing.T) { PartialResponse: false, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, MaxSearchOffsetLimit: 10, }, @@ -419,7 +419,7 @@ func TestServeSearch(t *testing.T) { Limit: 3, }, cfg: test.SetCfgDefaults(config.SeqAPI{ - SeqAPIOptions: &config.SeqAPIOptions{ + Options: config.SeqAPIOptions{ MaxSearchLimit: 5, }, }), diff --git a/internal/api/seqapi/v1/http/test_data.go b/internal/api/seqapi/v1/http/test_data.go index 0f24d93..57431ed 100644 --- a/internal/api/seqapi/v1/http/test_data.go +++ b/internal/api/seqapi/v1/http/test_data.go @@ -9,7 +9,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/ozontech/seq-ui/internal/api/seqapi/v1/test" - "github.com/ozontech/seq-ui/internal/app/config" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/internal/pkg/client/seqdb" asyncsearches "github.com/ozontech/seq-ui/internal/pkg/service/async_searches" ) @@ -23,15 +23,11 @@ var ( ) func setupTestAPI(data test.APITestData) *API { - // when test cases don't explicitly provide configuration. - if data.Cfg.SeqAPIOptions == nil { - data.Cfg.SeqAPIOptions = &config.SeqAPIOptions{} - } seqDBClients := make(map[string]seqdb.Client) seqDBClients[config.DefaultSeqDBClientID] = data.Mocks.SeqDB for _, envConfig := range data.Cfg.Envs { - seqDBClients[envConfig.SeqDB] = data.Mocks.SeqDB + seqDBClients[envConfig.SeqDBID] = data.Mocks.SeqDB } var asyncSvc asyncsearches.Service @@ -39,7 +35,7 @@ func setupTestAPI(data test.APITestData) *API { asyncSvc = data.Mocks.AsyncSearchesSvc } - return New(data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) + return New(&data.Cfg, seqDBClients, data.Mocks.Cache, data.Mocks.Cache, asyncSvc) } func withQueryParamID(h http.HandlerFunc, id string) http.HandlerFunc { diff --git a/internal/api/seqapi/v1/seqapi.go b/internal/api/seqapi/v1/seqapi.go index f32d657..bd04cb3 100644 --- a/internal/api/seqapi/v1/seqapi.go +++ b/internal/api/seqapi/v1/seqapi.go @@ -17,7 +17,7 @@ type SeqAPI struct { } func New( - cfg config.SeqAPI, + cfg *config.SeqAPI, seqDB map[string]seqdb.Client, inmemWithRedisCache cache.Cache, redisCache cache.Cache, diff --git a/internal/app/config/migrate/v1_v2.go b/internal/app/config/migrate/v1_v2.go index f0b2bc4..b95a703 100644 --- a/internal/app/config/migrate/v1_v2.go +++ b/internal/app/config/migrate/v1_v2.go @@ -52,6 +52,9 @@ func migrateServer(src *v1.Server) *v2.Server { } if src.JWTSecretKey != "" { + if dst.Auth == nil { + dst.Auth = &v2.Auth{} + } dst.Auth.JWT = &v2.JWT{SecretKey: src.JWTSecretKey} } @@ -152,7 +155,7 @@ func migrateClients(src v1.Config) *v2.Clients { if src.Server != nil && src.Server.CH != nil { ch := src.Server.CH dst.ClickHouse = []v2.CHClient{{ - ID: v2.DefaultSeqDBClientID, + ID: v2.DefaultCHClientID, Addrs: ch.Addrs, Database: ch.Database, Username: ch.Username, @@ -381,8 +384,9 @@ func migrateFieldFilters(src *v1.FieldFilterSet) *v2.FieldFilterSet { return dst } -func migrateErrorGroups(eg v1.ErrorGroups) v2.ErrorGroups { - return v2.ErrorGroups{ +func migrateErrorGroups(eg v1.ErrorGroups) *v2.ErrorGroups { + return &v2.ErrorGroups{ + CHID: v2.DefaultCHClientID, LogTagsMapping: v2.LogTagsMapping{ Env: eg.LogTagsMapping.Env, Service: eg.LogTagsMapping.Service, diff --git a/internal/app/config/v2/config.go b/internal/app/config/v2/config.go index 286e836..8c237c2 100644 --- a/internal/app/config/v2/config.go +++ b/internal/app/config/v2/config.go @@ -28,6 +28,7 @@ import ( // addr: // auth: // oidc: +// cache_id: // skip_verify: // auth_urls: // root_ca: @@ -80,15 +81,9 @@ import ( // read_timeout: // handlers: // seq_api: -// options: -// max_search_limit: -// max_search_total_limit: -// max_search_offset_limit: -// max_export_limit: -// seq_cli_max_search_limit: -// max_parallel_export_requests: -// max_aggregations_per_request: -// max_buckets_per_aggregation_ts: +// cache_id: +// redis_id: +// global_options: // events_cache_ttl: // pinned_fields: // name: @@ -115,6 +110,15 @@ import ( // values: // process_fields: // ignore_fields: +// options: +// max_search_limit: +// max_search_total_limit: +// max_search_offset_limit: +// max_export_limit: +// seq_cli_max_search_limit: +// max_parallel_export_requests: +// max_aggregations_per_request: +// max_buckets_per_aggregation_ts: // envs: // : // seq_db_id: @@ -127,34 +131,9 @@ import ( // max_parallel_export_requests: // max_aggregations_per_request: // max_buckets_per_aggregation_ts: -// events_cache_ttl: -// pinned_fields: -// name: -// type: -// system_fields: -// name: -// type: -// logs_lifespan_cache_key: -// logs_lifespan_cache_ttl: -// fields_cache_ttl: -// masking: -// masks: -// re: -// groups: -// mode: -// replace_word: -// process_fields: -// ignore_fields: -// field_filters: -// condition: -// filters: -// field: -// mode: -// values: -// process_fields: -// ignore_fields: // default_env: // error_groups: +// ch_id: // log_tags_mapping: // env: // service: @@ -162,6 +141,7 @@ import ( // query_filter: // : // mass_export: +// seq_db_id: // batch_size: // workers_count: // tasks_channel_size: @@ -183,6 +163,7 @@ import ( // initial_retry_backoff: // max_retry_backoff: // async_search: +// seq_db_id: // admin_users: // list_query_length_limit: // db: @@ -213,6 +194,7 @@ import ( const ( DefaultSeqDBClientID = "default" + DefaultCHClientID = "default" DefaultInmemCacheID = "seqapi" DefaultRedisID = "default" DefaultMassExportRedisID = "mass_export" @@ -502,10 +484,10 @@ func (c *Clients) ClickHouseByID(id string) *CHClient { } type Handlers struct { - SeqAPI SeqAPI `yaml:"seq_api"` - ErrorGroups ErrorGroups `yaml:"error_groups"` - MassExport *MassExport `yaml:"mass_export"` - AsyncSearch AsyncSearch `yaml:"async_search"` + SeqAPI SeqAPI `yaml:"seq_api"` + ErrorGroups *ErrorGroups `yaml:"error_groups"` + MassExport *MassExport `yaml:"mass_export"` + AsyncSearch AsyncSearch `yaml:"async_search"` } type Field struct { @@ -723,12 +705,37 @@ func Normalize(cfg *Config) error { } if cfg.Handlers.MassExport != nil { - if cfg.Handlers.MassExport.SessionStore.RedisID == "" { - return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") + if cfg.Handlers.MassExport.SeqDBID == "" { + return fmt.Errorf("handlers.mass_export.seq_db_id cannot be empty") + } + if _, ok := seqDBIDs[cfg.Handlers.MassExport.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.seq_db_id %q", cfg.Handlers.MassExport.SeqDBID) + } + + if cfg.Handlers.MassExport.SessionStore != nil { + if cfg.Handlers.MassExport.SessionStore.RedisID == "" { + return fmt.Errorf("handlers.mass_export.session_store.redis_id cannot be empty") + } + if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { + return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) + } + } + } + + if cfg.Handlers.AsyncSearch.SeqDBID == "" { + return fmt.Errorf("handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + } + if _, ok := seqDBIDs[cfg.Handlers.AsyncSearch.SeqDBID]; !ok { + return fmt.Errorf("unknown handlers.async_search.seq_db_id %q", cfg.Handlers.AsyncSearch.SeqDBID) + } + + if cfg.Handlers.ErrorGroups != nil { + if cfg.Handlers.ErrorGroups.CHID == "" { + return fmt.Errorf("handlers.error_groups.ch_id cannot be empty") } - if _, ok := redisIDs[cfg.Handlers.MassExport.SessionStore.RedisID]; !ok { - return fmt.Errorf("unknown handlers.mass_export.session_store.redis_id %q", cfg.Handlers.MassExport.SessionStore.RedisID) + if _, ok := chIDs[cfg.Handlers.ErrorGroups.CHID]; !ok { + return fmt.Errorf("unknown handlers.error_groups.ch_id %q", cfg.Handlers.ErrorGroups.CHID) } } diff --git a/internal/pkg/cache/ctor.go b/internal/pkg/cache/ctor.go index 5cf2a1c..4c7f7b1 100644 --- a/internal/pkg/cache/ctor.go +++ b/internal/pkg/cache/ctor.go @@ -4,9 +4,10 @@ import ( "context" "fmt" + "go.uber.org/zap" + config "github.com/ozontech/seq-ui/internal/app/config/v2" "github.com/ozontech/seq-ui/logger" - "go.uber.org/zap" ) func FromConfig(ctx context.Context, cfg *config.Cache) (map[string]Cache, error) { diff --git a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go index eeba398..0fdab5f 100644 --- a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go +++ b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: v1/seq_proxy_api.proto package seqproxyapi diff --git a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go index ee491b3..2561cd9 100644 --- a/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go +++ b/internal/pkg/client/seqdb/seqproxyapi/v1/seq_proxy_api_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: v1/seq_proxy_api.proto package seqproxyapi diff --git a/internal/pkg/service/massexport/seq_proxy_downloader.go b/internal/pkg/service/massexport/seq_proxy_downloader.go index 4a92f3e..6bb6258 100644 --- a/internal/pkg/service/massexport/seq_proxy_downloader.go +++ b/internal/pkg/service/massexport/seq_proxy_downloader.go @@ -17,13 +17,13 @@ import ( ) type seqProxyDownloader struct { - cfg config.SeqProxyDownloader + cfg config.DownloadParams client seqdb.Client } const defaultDelay = 1 * time.Second -func newSeqProxyDownloader(client seqdb.Client, cfg config.SeqProxyDownloader) *seqProxyDownloader { +func newSeqProxyDownloader(client seqdb.Client, cfg config.DownloadParams) *seqProxyDownloader { if cfg.Delay <= 0 { cfg.Delay = defaultDelay } diff --git a/internal/pkg/service/massexport/service.go b/internal/pkg/service/massexport/service.go index e65ba6b..03256c9 100644 --- a/internal/pkg/service/massexport/service.go +++ b/internal/pkg/service/massexport/service.go @@ -93,7 +93,7 @@ func NewService( tasksChannelSize = cfg.TasksChannelSize } - downloader := newSeqProxyDownloader(client, *cfg.SeqProxyDownloader) + downloader := newSeqProxyDownloader(client, *cfg.DownloadParams) return &exportService{ sessionStore: sessionStore, diff --git a/internal/pkg/service/massexport/sessionstore/redis.go b/internal/pkg/service/massexport/sessionstore/redis.go index e5a7988..ebe503e 100644 --- a/internal/pkg/service/massexport/sessionstore/redis.go +++ b/internal/pkg/service/massexport/sessionstore/redis.go @@ -31,23 +31,23 @@ const ( maxLifetime = 30 * day ) -func NewRedisSessionStore(ctx context.Context, cfg *config.SessionStore) (SessionStore, error) { - client, err := redisclient.New(ctx, &cfg.Redis) +func NewRedisSessionStore(ctx context.Context, redisCfg *config.Redis, exportLifetimeCfg time.Duration) (SessionStore, error) { + client, err := redisclient.New(ctx, redisCfg) if err != nil { return nil, fmt.Errorf("create redis client: %w", err) } exportLifetime := defaultLifetime - if cfg.ExportLifetime != 0 { + if exportLifetimeCfg != 0 { switch { - case cfg.ExportLifetime < minLifetime: - logger.Warn("export lifetime from config is too low; set min lifetime", zap.String("export_lifetime", cfg.ExportLifetime.String())) + case exportLifetimeCfg < minLifetime: + logger.Warn("export lifetime from config is too low; set min lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) exportLifetime = minLifetime - case cfg.ExportLifetime > maxLifetime: - logger.Warn("export lifetime from config is too high; set max lifetime", zap.String("export_lifetime", cfg.ExportLifetime.String())) + case exportLifetimeCfg > maxLifetime: + logger.Warn("export lifetime from config is too high; set max lifetime", zap.String("export_lifetime", exportLifetimeCfg.String())) exportLifetime = maxLifetime default: - exportLifetime = cfg.ExportLifetime + exportLifetime = exportLifetimeCfg } } diff --git a/pkg/dashboards/v1/dashboards.pb.go b/pkg/dashboards/v1/dashboards.pb.go index 7f74181..d11a515 100644 --- a/pkg/dashboards/v1/dashboards.pb.go +++ b/pkg/dashboards/v1/dashboards.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: dashboards/v1/dashboards.proto package dashboards diff --git a/pkg/dashboards/v1/dashboards_grpc.pb.go b/pkg/dashboards/v1/dashboards_grpc.pb.go index 8217c47..9559155 100644 --- a/pkg/dashboards/v1/dashboards_grpc.pb.go +++ b/pkg/dashboards/v1/dashboards_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: dashboards/v1/dashboards.proto package dashboards diff --git a/pkg/errorgroups/v1/errorgroups.pb.go b/pkg/errorgroups/v1/errorgroups.pb.go index cc0e2c1..763796e 100644 --- a/pkg/errorgroups/v1/errorgroups.pb.go +++ b/pkg/errorgroups/v1/errorgroups.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: errorgroups/v1/errorgroups.proto package errorgroups diff --git a/pkg/errorgroups/v1/errorgroups_grpc.pb.go b/pkg/errorgroups/v1/errorgroups_grpc.pb.go index f8c6f27..a284026 100644 --- a/pkg/errorgroups/v1/errorgroups_grpc.pb.go +++ b/pkg/errorgroups/v1/errorgroups_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: errorgroups/v1/errorgroups.proto package errorgroups diff --git a/pkg/massexport/v1/massexport.pb.go b/pkg/massexport/v1/massexport.pb.go index 47c9007..3c5f3e9 100644 --- a/pkg/massexport/v1/massexport.pb.go +++ b/pkg/massexport/v1/massexport.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: massexport/v1/massexport.proto package massexport diff --git a/pkg/massexport/v1/massexport_grpc.pb.go b/pkg/massexport/v1/massexport_grpc.pb.go index eaaea70..a78e188 100644 --- a/pkg/massexport/v1/massexport_grpc.pb.go +++ b/pkg/massexport/v1/massexport_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: massexport/v1/massexport.proto package massexport diff --git a/pkg/seqapi/v1/seq_api.pb.go b/pkg/seqapi/v1/seq_api.pb.go index 2e5fa5b..c22da34 100644 --- a/pkg/seqapi/v1/seq_api.pb.go +++ b/pkg/seqapi/v1/seq_api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: seqapi/v1/seq_api.proto package seqapi diff --git a/pkg/seqapi/v1/seq_api_grpc.pb.go b/pkg/seqapi/v1/seq_api_grpc.pb.go index 83a451d..e6a5155 100644 --- a/pkg/seqapi/v1/seq_api_grpc.pb.go +++ b/pkg/seqapi/v1/seq_api_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: seqapi/v1/seq_api.proto package seqapi diff --git a/pkg/userprofile/v1/userprofile.pb.go b/pkg/userprofile/v1/userprofile.pb.go index d8e85f0..7f283ae 100644 --- a/pkg/userprofile/v1/userprofile.pb.go +++ b/pkg/userprofile/v1/userprofile.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.34.2 -// protoc v7.34.1 +// protoc v7.35.1 // source: userprofile/v1/userprofile.proto package userprofile diff --git a/pkg/userprofile/v1/userprofile_grpc.pb.go b/pkg/userprofile/v1/userprofile_grpc.pb.go index a2c1330..93ce310 100644 --- a/pkg/userprofile/v1/userprofile_grpc.pb.go +++ b/pkg/userprofile/v1/userprofile_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.4.0 -// - protoc v7.34.1 +// - protoc v7.35.1 // source: userprofile/v1/userprofile.proto package userprofile