diff --git a/.gitignore b/.gitignore index 272c8b0c..d6501bab 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ config/generated.go config/generat*.go config/initialization.dsl config/system_config.yml +plugin/enterprise +/plugin/enterprise/ diff --git a/Makefile b/Makefile index 04522ac4..aa8b445a 100755 --- a/Makefile +++ b/Makefile @@ -4,11 +4,12 @@ SHELL=/bin/bash APP_NAME := console APP_VERSION := 1.0.0_SNAPSHOT APP_CONFIG := $(APP_NAME).yml -APP_EOLDate ?= "2026-12-31T10:10:10Z" +APP_EOLDate ?= "2027-12-31T10:10:10Z" APP_STATIC_FOLDER := .public APP_STATIC_PACKAGE := public APP_UI_FOLDER := ui APP_PLUGIN_FOLDER := plugin +GOBUILD_FLAGS += -trimpath # easyjson -all domain.go include ../framework/Makefile @@ -45,6 +46,16 @@ web-lint: @(cd web && npx eslint . --ext .js,.jsx,.ts,.tsx) @echo "Linting complete." +.PHONY: format format-ci +format: format-ci + +format-ci: + @echo "formatting code" + @find . -type f -name '*.go' \ + -not -path './vendor/*' \ + -not -path './.git/*' \ + -print0 | xargs -0 gofmt -w + # Build the web app build-web: @echo "Building the web app..." diff --git a/common/info.go b/common/info.go new file mode 100644 index 00000000..197671e6 --- /dev/null +++ b/common/info.go @@ -0,0 +1,55 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package common + +import "infini.sh/framework/core/util" + +var allowedInstanceInfoKeys = []string{ + "id", + "name", + "application", + "labels", + "tags", + "description", + "status", +} + +func SanitizeInstanceInfoMap(payload util.MapStr) util.MapStr { + sanitized := util.MapStr{} + for _, key := range allowedInstanceInfoKeys { + if value, ok := payload[key]; ok { + sanitized[key] = value + } + } + return sanitized +} + +func SanitizeInstanceInfoBytes(body []byte) ([]byte, error) { + payload := util.MapStr{} + err := util.FromJSONBytes(body, &payload) + if err != nil { + return nil, err + } + return util.MustToJSONBytes(SanitizeInstanceInfoMap(payload)), nil +} diff --git a/common/info_test.go b/common/info_test.go new file mode 100644 index 00000000..7a3afafa --- /dev/null +++ b/common/info_test.go @@ -0,0 +1,37 @@ +package common + +import ( + "testing" + + "infini.sh/framework/core/util" +) + +func TestSanitizeInstanceInfoMap(t *testing.T) { + payload := util.MapStr{ + "id": "node-1", + "name": "console-a", + "application": util.MapStr{"name": "console"}, + "endpoint": "http://127.0.0.1:2900", + "host": util.MapStr{"name": "host-a"}, + "network": util.MapStr{"ip": []string{"10.0.0.1"}}, + "basic_auth": util.MapStr{"username": "admin"}, + } + + sanitized := SanitizeInstanceInfoMap(payload) + + if sanitized["id"] != "node-1" { + t.Fatalf("expected id to be preserved, got %v", sanitized["id"]) + } + if _, ok := sanitized["endpoint"]; ok { + t.Fatalf("expected endpoint to be removed") + } + if _, ok := sanitized["host"]; ok { + t.Fatalf("expected host to be removed") + } + if _, ok := sanitized["network"]; ok { + t.Fatalf("expected network to be removed") + } + if _, ok := sanitized["basic_auth"]; ok { + t.Fatalf("expected basic_auth to be removed") + } +} diff --git a/common/log_redaction.go b/common/log_redaction.go new file mode 100644 index 00000000..112a3924 --- /dev/null +++ b/common/log_redaction.go @@ -0,0 +1,81 @@ +package common + +import ( + "net" + "net/url" + "regexp" + "strings" +) + +var ( + // urlInStringRegex matches http(s)://host:port or http(s)://host embedded in a longer string. + urlInStringRegex = regexp.MustCompile(`(https?://)([^/:"\s]+)(:[0-9]+)?`) + // dialTCPRegex matches "dial tcp host:port" patterns in Go network error messages. + dialTCPRegex = regexp.MustCompile(`(dial tcp )([^:"\s]+)(:[0-9]+)`) +) + +// MaskLogError returns the error message with any embedded host/IP addresses redacted. +func MaskLogError(err error) string { + if err == nil { + return "" + } + msg := err.Error() + msg = urlInStringRegex.ReplaceAllString(msg, "${1}***${3}") + msg = dialTCPRegex.ReplaceAllString(msg, "${1}***${3}") + return msg +} + +func MaskLogToken(value string) string { + if value == "" { + return "" + } + + if len(value) <= 4 { + return "***" + } + + if len(value) <= 8 { + return value[:1] + "***" + value[len(value)-1:] + } + + return value[:2] + "***" + value[len(value)-2:] +} + +func MaskLogHost(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + + _, port, err := net.SplitHostPort(value) + if err == nil { + return "***:" + port + } + + return "***" +} + +func MaskLogEndpoint(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + + parsed, err := url.Parse(value) + if err == nil && parsed.Host != "" { + host := "***" + if port := parsed.Port(); port != "" { + host += ":" + port + } + if parsed.Scheme != "" { + return parsed.Scheme + "://" + host + } + return host + } + + if strings.Contains(value, "://") { + return "***" + } + + return MaskLogHost(value) +} diff --git a/common/log_redaction_test.go b/common/log_redaction_test.go new file mode 100644 index 00000000..2ecf530f --- /dev/null +++ b/common/log_redaction_test.go @@ -0,0 +1,113 @@ +package common + +import ( + "fmt" + "testing" +) + +func TestMaskLogError(t *testing.T) { + testCases := []struct { + name string + input string + expect string + }{ + { + name: "nil error", + input: "", + expect: "", + }, + { + name: "url in get error", + input: `request error: Get "http://192.168.3.8:8080/elasticsearch/node/_discovery": dial tcp 192.168.3.8:8080: connect: connection refused`, + expect: `request error: Get "http://***:8080/elasticsearch/node/_discovery": dial tcp ***:8080: connect: connection refused`, + }, + { + name: "https url", + input: `Post "https://10.0.0.1:9200/_bulk": connection refused`, + expect: `Post "https://***:9200/_bulk": connection refused`, + }, + { + name: "no sensitive data", + input: "index out of range", + expect: "index out of range", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.name == "nil error" { + if actual := MaskLogError(nil); actual != tc.expect { + t.Fatalf("unexpected result, got %q want %q", actual, tc.expect) + } + return + } + err := fmt.Errorf("%s", tc.input) + if actual := MaskLogError(err); actual != tc.expect { + t.Fatalf("unexpected masked error, got %q want %q", actual, tc.expect) + } + }) + } +} + +func TestMaskLogToken(t *testing.T) { + testCases := []struct { + name string + input string + expect string + }{ + {name: "empty", input: "", expect: ""}, + {name: "short", input: "abcd", expect: "***"}, + {name: "medium", input: "abcdefgh", expect: "a***h"}, + {name: "long", input: "d8de63bhalhmmus1n7k0", expect: "d8***k0"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := MaskLogToken(tc.input); actual != tc.expect { + t.Fatalf("unexpected masked token, got %q want %q", actual, tc.expect) + } + }) + } +} + +func TestMaskLogHost(t *testing.T) { + testCases := []struct { + name string + input string + expect string + }{ + {name: "empty", input: "", expect: ""}, + {name: "ipv4 hostport", input: "127.0.0.1:9200", expect: "***:9200"}, + {name: "ipv6 hostport", input: "[::1]:9200", expect: "***:9200"}, + {name: "plain host", input: "node-1.internal", expect: "***"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := MaskLogHost(tc.input); actual != tc.expect { + t.Fatalf("unexpected masked host, got %q want %q", actual, tc.expect) + } + }) + } +} + +func TestMaskLogEndpoint(t *testing.T) { + testCases := []struct { + name string + input string + expect string + }{ + {name: "empty", input: "", expect: ""}, + {name: "http endpoint", input: "http://192.168.3.185:8080", expect: "http://***:8080"}, + {name: "https ipv6 endpoint", input: "https://[2001:db8::1]:9200", expect: "https://***:9200"}, + {name: "host port", input: "127.0.0.1:9200", expect: "***:9200"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := MaskLogEndpoint(tc.input); actual != tc.expect { + t.Fatalf("unexpected masked endpoint, got %q want %q", actual, tc.expect) + } + }) + } +} diff --git a/common/probe.go b/common/probe.go new file mode 100644 index 00000000..3e1e4064 --- /dev/null +++ b/common/probe.go @@ -0,0 +1,199 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package common + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/segmentio/encoding/json" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic/adapter" +) + +const ClusterProbePathLabel = "console_probe_path" + +func NormalizeProbePath(path string) string { + path = strings.TrimSpace(path) + if path == "" || path == "/" { + return "" + } + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return path +} + +func GetProbePath(config *elastic.ElasticsearchConfig) string { + if config == nil || config.Labels == nil { + return "" + } + return NormalizeProbePath(util.ToString(config.Labels[ClusterProbePathLabel])) +} + +func SetProbePath(config *elastic.ElasticsearchConfig, path string) { + if config == nil { + return + } + path = NormalizeProbePath(path) + if config.Labels == nil { + config.Labels = util.MapStr{} + } + if path == "" { + delete(config.Labels, ClusterProbePathLabel) + return + } + config.Labels[ClusterProbePathLabel] = path +} + +func BuildEndpointWithPath(endpoint, path string) string { + path = NormalizeProbePath(path) + if path == "" { + return endpoint + } + + baseURL, err := url.Parse(endpoint) + if err != nil { + return strings.TrimRight(endpoint, "/") + path + } + + refURL, err := url.Parse(path) + if err != nil { + return strings.TrimRight(endpoint, "/") + path + } + + return baseURL.ResolveReference(refURL).String() +} + +func ClusterVersion(metadata *elastic.ElasticsearchMetadata) (*elastic.ClusterInformation, error) { + if metadata == nil || metadata.Config == nil { + return nil, errors.New("elasticsearch metadata is nil") + } + + probePath := GetProbePath(metadata.Config) + if probePath == "" { + return adapter.ClusterVersion(metadata) + } + + if metadata.Config.RequestTimeout <= 0 { + metadata.Config.RequestTimeout = 5 + } + + endpoint := fmt.Sprintf("%v://%v", metadata.GetSchema(), metadata.GetActiveHost()) + if err := probeClusterEndpoint(metadata.Config, endpoint, probePath); err != nil { + return nil, err + } + return loadClusterInformationWithoutRoot(metadata.Config, endpoint) +} + +func ClusterVersionWithConfig(config *elastic.ElasticsearchConfig) (*elastic.ClusterInformation, error) { + if config == nil { + return nil, errors.New("elasticsearch config is nil") + } + return ClusterVersion(&elastic.ElasticsearchMetadata{Config: config}) +} + +func probeClusterEndpoint(config *elastic.ElasticsearchConfig, endpoint, probePath string) error { + res, err := executeRequest(config, BuildEndpointWithPath(endpoint, probePath)) + if err != nil { + return err + } + if res.StatusCode != http.StatusOK { + return errors.New(string(res.Body)) + } + return nil +} + +func loadClusterInformationWithoutRoot(config *elastic.ElasticsearchConfig, endpoint string) (*elastic.ClusterInformation, error) { + stats := &elastic.ClusterStats{} + res, err := executeRequest(config, BuildEndpointWithPath(endpoint, "/_cluster/stats")) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + return nil, errors.New(string(res.Body)) + } + if err = json.Unmarshal(res.Body, stats); err != nil { + return nil, err + } + + nodes := &elastic.NodesResponse{} + res, err = executeRequest(config, BuildEndpointWithPath(endpoint, "/_nodes/_all/http")) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + return nil, errors.New(string(res.Body)) + } + if err = json.Unmarshal(res.Body, nodes); err != nil { + return nil, err + } + + info := &elastic.ClusterInformation{ + ClusterName: stats.ClusterName, + ClusterUUID: stats.ClusterUUID, + } + if info.ClusterName == "" { + info.ClusterName = nodes.ClusterName + } + for _, node := range nodes.Nodes { + if node.Version != "" { + info.Version.Number = node.Version + break + } + } + info.Version.Distribution = config.Distribution + if info.Version.Distribution == "" { + info.Version.Distribution = elastic.Elasticsearch + } + return info, nil +} + +func executeRequest(config *elastic.ElasticsearchConfig, requestURL string) (*util.Result, error) { + req := util.Request{ + Method: http.MethodGet, + Url: requestURL, + } + if config.BasicAuth != nil && strings.TrimSpace(config.BasicAuth.Username) != "" { + req.SetBasicAuth(config.BasicAuth.Username, config.BasicAuth.Password.Get()) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(getRequestTimeout(config))*time.Second) + req.Context = ctx + defer cancel() + + return util.ExecuteRequestWithCatchFlag(nil, &req, true) +} + +func getRequestTimeout(config *elastic.ElasticsearchConfig) int { + if config == nil || config.RequestTimeout <= 0 { + return 5 + } + return config.RequestTimeout +} diff --git a/common/probe_test.go b/common/probe_test.go new file mode 100644 index 00000000..c7373d6d --- /dev/null +++ b/common/probe_test.go @@ -0,0 +1,68 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package common + +import ( + "testing" + + "infini.sh/framework/core/elastic" +) + +func TestNormalizeProbePath(t *testing.T) { + tests := map[string]string{ + "": "", + " ": "", + "/": "", + "_cluster/health": "/_cluster/health", + " /_cat/nodes ": "/_cat/nodes", + } + + for input, expected := range tests { + if got := NormalizeProbePath(input); got != expected { + t.Fatalf("NormalizeProbePath(%q) = %q, want %q", input, got, expected) + } + } +} + +func TestProbePathStoredInLabels(t *testing.T) { + cfg := &elastic.ElasticsearchConfig{} + SetProbePath(cfg, "_cluster/health") + + if got := GetProbePath(cfg); got != "/_cluster/health" { + t.Fatalf("GetProbePath() = %q, want %q", got, "/_cluster/health") + } + + SetProbePath(cfg, "") + if got := GetProbePath(cfg); got != "" { + t.Fatalf("GetProbePath() = %q, want empty", got) + } +} + +func TestBuildEndpointWithPath(t *testing.T) { + got := BuildEndpointWithPath("https://example.com:9200", "/_cluster/health") + want := "https://example.com:9200/_cluster/health" + if got != want { + t.Fatalf("BuildEndpointWithPath() = %q, want %q", got, want) + } +} diff --git a/config/install_agent.tpl b/config/install_agent.tpl index e130b6c0..d4615c02 100644 --- a/config/install_agent.tpl +++ b/config/install_agent.tpl @@ -2,14 +2,18 @@ set -eo pipefail +DEFAULT_DOWNLOAD_URL="{{base_url}}" +DEFAULT_VERSION="{{version}}" + function print_usage() { - echo "Usage: curl -ksSL http://$[[CLOUD_ENDPOINT]]/instance/_get_install_script?token | sudo bash -s -- [-u url_for_download_program] [-v version_for_program ] [-t target_install_dir] [-o overwite_flag] [-s url_console_lan_adress]" + echo "Usage: curl -ksSL http://$[[CLOUD_ENDPOINT]]/instance/_get_install_script?token | bash -s -- [-u url_for_download_program] [-v version_for_program ] [-t target_install_dir] [-o overwite_flag] [-s url_console_lan_adress] [--no-service]" echo "Options:" - echo " -u, --url Install Agent download URL, format is schema://domain:port/stable/agent-platform-version.ext, can be manually specified" - echo " -v, --version Install Agent version, default is to get latest version online, can be manually specified" + echo " -u, --url Install Agent download URL, supports host, package directory, or direct package file URL" + echo " -v, --version Install Agent version, default is Console configured version, current Console build version, or the latest version from the download source" echo " -t, --target Install Agent target path, default is /opt/agent, can be manually specified" echo " -o, --overwrite Whether to overwrite existing files during Agent install, default is true, can be manually specified" echo " -s, --server Server address for Agent to communicate with INFINI Console after install, default is current Console address, can be manually specified" + echo " --no-service Skip service install/start and print foreground startup instructions for containers or non-sudo environments" exit 1 } @@ -68,7 +72,27 @@ function __catch() { } function get_latest_version() { - echo $(curl -m3 -s "https://release.infinilabs.com/.latest" |sed 's/",/"/;s/"//g;s/://1' |grep -Ev '^[{}]' |grep "$program_name" |awk '{print $NF}') + local input_url="${1%/}" + local latest_url + local latest_version="" + + case "$input_url" in + *.tar.gz|*.zip) + echo "" + return + ;; + esac + + for latest_url in "${input_url}/.latest" "${input_url%/agent/stable}/.latest"; do + [[ -z "$latest_url" ]] && continue + latest_version=$(curl -m3 -s "$latest_url" |sed 's/",/"/;s/"//g;s/://1' |grep -Ev '^[{}]' |grep "$program_name" |awk '{print $NF}') + if [[ -n "$latest_version" ]]; then + echo "$latest_version" + return + fi + done + + echo "" } function check_dir() { @@ -87,8 +111,10 @@ function check_dir() { if [[ "$(ls -A ${install_dir})" ]]; then if [ "$o" == "true" ]; then echo "WARN: Auto replace or upgrade exists agent files." - uninstall_service - rm -rf ${install_dir}/* + if [[ "${no_service}" != "true" ]]; then + uninstall_service + fi + cleanup_install_dir_preserving_runtime_data else echo "Error: Please manual clean exists agent files at ${install_dir}, reinstall again." exit 1 @@ -96,6 +122,21 @@ function check_dir() { fi } +function cleanup_install_dir_preserving_runtime_data() { + echo "[agent] preserving runtime data in ${install_dir}/data and ${install_dir}/log" + shopt -s dotglob nullglob + for item in "${install_dir}"/*; do + name="$(basename "${item}")" + case "${name}" in + data|log) + continue + ;; + esac + rm -rf "${item}" + done + shopt -u dotglob nullglob +} + function check_platform() { local platform=$(uname) local arch=$(uname -m) @@ -181,29 +222,51 @@ function check_platform() { } function install_binary() { - local download_url="$location/${program_name}-${version}-${file_ext}" + local archive_name="${program_name}-${version}-${file_ext}" + local download_url=$(resolve_download_url "$location" "$archive_name") + local downloaded_file="${download_url##*/}" + downloaded_file="${downloaded_file%%\?*}" echo "File: [$download_url]" tmp_dir="$(mktemp -d)" cd "$tmp_dir" if command -v curl >/dev/null 2>&1; then - curl -# -LO "$download_url" + curl -k -# -LO "$download_url" elif command -v wget >/dev/null 2>&1; then - wget -q -nc --show-progress --progress=bar:force:noscroll "$download_url" + wget --no-check-certificate -q -nc --show-progress --progress=bar:force:noscroll "$download_url" else echo "Error: Could not find curl or wget, Please install wget or curl in advance." >&2; exit 1; fi if [[ "${file_ext}" == *".tar.gz" ]]; then - tar -xzf "${program_name}-${version}-${file_ext}" -C "$install_dir" + tar -xzf "${downloaded_file}" -C "$install_dir" else - unzip -q "${program_name}-${version}-${file_ext}" -d "$install_dir" + unzip -q "${downloaded_file}" -d "$install_dir" fi cd "${install_dir}" && rm -rf "${tmp_dir}" && echo "" } +function resolve_download_url() { + local input_url="${1%/}" + local archive_name="$2" + + case "$input_url" in + *.tar.gz|*.zip) + echo "$input_url" + return + ;; + esac + + if [[ "$input_url" =~ ^[a-zA-Z][a-zA-Z0-9+.-]*://[^/]+$ ]]; then + echo "${input_url}/agent/stable/${archive_name}" + return + fi + + echo "${input_url}/${archive_name}" +} + function install_certs() { ca_crt="{{ca_crt}}" client_crt="{{client_crt}}" @@ -220,28 +283,44 @@ function install_config() { echo "[agent] waiting generate config" port={{port}} console_endpoint="{{console_endpoint}}" - - location_ep=$(echo "$console_endpoint" | sed -nE 's/(.*):\/\/([^/:]*):?([0-9]*).*/\1:\/\/\2:\3/p') - server=${register_server:-$location_ep} + remote_config_servers='{{remote_config_servers}}' + + server=${register_server:-$console_endpoint} + if [[ -n "${register_server}" ]]; then + remote_config_servers="[\"${register_server}\"]" + fi echo "[agent] agent listening port $port, will register to console endpoint [ $server ]" + echo "[agent] remote config servers: ${remote_config_servers}" cat < ${install_dir}/agent.yml -configs.auto_reload: true - env: - API_BINDING: "0.0.0.0:${port}" - -path.data: data -path.logs: log -path.configs: config + WEB_BINDING: "0.0.0.0:${port}" + MANAGED: true + REMOTE_CONFIG_SERVERS: ${remote_config_servers} + REVERSE_CHANNEL_ENDPOINTS: {{reverse_channel_endpoints}} + REMOTE_CONFIG_INTERVAL: "10s" + SECURITY_ENABLED: true + SECURITY_MANAGED_ENABLED: false + +path.data: "${install_dir}/data" +path.logs: "${install_dir}/log" +path.configs: "${install_dir}/config" +configs.auto_reload: true resource_limit.cpu.max_num_of_cpus: 1 -resource_limit.memory.max_in_bytes: 533708800 +resource_limit: + memory: + max_in_bytes: 533708800 #50MB + +task: + max_concurrent_tasks: 3 stats: - include_storage_stats_in_api: false + include_storage_stats_in_api: true elastic: skip_init_metadata_on_start: true + metadata_refresh: + enabled: false health_check: enabled: true interval: 60s @@ -252,45 +331,63 @@ elastic: disk_queue: max_msg_size: 20485760 max_bytes_per_file: 20485760 - max_used_bytes: 524288000 + max_used_bytes: 524288000 # 500MB retention.max_num_of_local_files: 1 compress: - idle_threshold: 0 + idle_threshold: 1 num_of_files_decompress_ahead: 0 segment: enabled: true api: + disable_api_directory: true + enabled: false + +web: + access_log_enabled: false + embedding_api: {{embedding_api}} enabled: true - tls: - enabled: false - cert_file: "config/client.crt" - key_file: "config/client.key" - ca_file: "config/ca.crt" - skip_insecure_verify: false + websocket: + enabled: {{websocket_enabled}} + base_path: /ws + skip_host_verify: true network: - binding: \$[[env.API_BINDING]] - -badger: - value_threshold: 1024 - mem_table_size: 1048576 - value_log_max_entries: 1000000 - value_log_file_size: 104857600 + binding: \$[[env.WEB_BINDING]] + ui: + vfs: true +# tls: +# enabled: true +# cert_file: /etc/ssl.crt +# key_file: /etc/ssl.key +# skip_insecure_verify: false + security: + enabled: \$[[env.SECURITY_ENABLED]] + managed: \$[[env.SECURITY_MANAGED_ENABLED]] + +agent: + setup: + reverse_channel_endpoints: \$[[env.REVERSE_CHANNEL_ENDPOINTS]] + +metrics: + enabled: false configs: #for managed client's setting - managed: true # managed by remote servers + managed: \$[[env.MANAGED]] # managed by remote servers panic_on_config_error: false #ignore config error - interval: "10s" - servers: # config servers - - "${server}" - soft_delete: false + allow_generated_metrics_tasks: false # allow auto-generated metrics tasks (e.g. k8s) + interval: \$[[env.REMOTE_CONFIG_INTERVAL]] + servers: \$[[env.REMOTE_CONFIG_SERVERS]] # config servers + manager: + access_token: '\$[[keystore.CONFIGS_MANAGER_ACCESS_TOKEN]]' max_backup_files: 5 + soft_delete: false tls: #for mTLS connection with config servers - enabled: false + enabled: true cert_file: "config/client.crt" key_file: "config/client.key" ca_file: "config/ca.crt" + default_domain: "{{console_domain}}" skip_insecure_verify: false node: @@ -298,6 +395,19 @@ node: EOF } +function install_manager_token() { + access_token="{{access_token}}" + agent_svc=${install_dir}/${program_name}-${file_ext%%.*} + + if [[ -z "${access_token}" ]]; then + echo "Error: access token is empty." >&2 + exit 1 + fi + + echo "[agent] waiting save console manager access token" + echo -n "${access_token}" | ${agent_svc} keystore add "CONFIGS_MANAGER_ACCESS_TOKEN" --stdin --force >/dev/null +} + function uninstall_service() { agent_svc=${install_dir}/${program_name}-${file_ext%%.*} chmod 755 $agent_svc @@ -307,8 +417,8 @@ function uninstall_service() { if [[ -f "$linux_svc" || -f "$macos_svc" ]]; then echo "[agent] waiting service stop & uninstall for exist agent" - $agent_svc -service stop &>/dev/null - $agent_svc -service uninstall &>/dev/null + (cd "${install_dir}" && $agent_svc -service stop &>/dev/null) + (cd "${install_dir}" && $agent_svc -service uninstall &>/dev/null) fi } @@ -316,8 +426,18 @@ function install_service() { agent_svc=${install_dir}/${program_name}-${file_ext%%.*} chmod 755 $agent_svc echo "[agent] waiting service install & start" - $agent_svc -service install &>/dev/null - $agent_svc -service start &>/dev/null + (cd "${install_dir}" && $agent_svc -service install &>/dev/null) + (cd "${install_dir}" && $agent_svc -service start &>/dev/null) +} + +function print_no_service_hint() { + local agent_bin="${program_name}-${file_ext%%.*}" + echo "[agent] service installation skipped (--no-service)" + echo "[agent] start agent in foreground:" + echo " (cd \"${install_dir}\" && exec ./${agent_bin} -config agent.yml)" + echo "[agent] container example:" + echo " ENTRYPOINT [\"sh\", \"-c\", \"cd ${install_dir} && exec ./agent-* -config agent.yml\"]" + echo " CMD [\"sh\", \"-c\", \"cd ${install_dir} && exec ./agent-* -config agent.yml\"]" } function main() { @@ -328,16 +448,21 @@ function main() { -t|--target) target_dir="$2"; shift 2 ;; -o|--overwrite) overwrite="$2"; shift 2 ;; -s|--server) register_server="$2"; shift 2 ;; + --no-service) no_service=true; shift ;; *) print_usage ;; esac done program_name=agent - location=${url_download:-https://release.infinilabs.com/agent/stable} + location=${url_download:-$DEFAULT_DOWNLOAD_URL} install_dir=${target_dir:-/opt/$program_name} - latest_version=$(get_latest_version) - version=${version:-$latest_version} + latest_version="" + if [[ -z "${version}" && -z "${DEFAULT_VERSION}" ]]; then + latest_version=$(get_latest_version "$location") + fi + version=${version:-${DEFAULT_VERSION:-$latest_version}} o=${overwrite:-true} + no_service=${no_service:-false} file_ext="" if [[ -z "${version}" ]]; then @@ -350,9 +475,14 @@ function main() { check_dir install_binary install_certs + install_manager_token install_config - uninstall_service - install_service + if [[ "${no_service}" == "true" ]]; then + print_no_service_hint + else + uninstall_service + install_service + fi echo "" echo "" diff --git a/config/install_gateway.tpl b/config/install_gateway.tpl new file mode 100644 index 00000000..92d5e82c --- /dev/null +++ b/config/install_gateway.tpl @@ -0,0 +1,529 @@ +#!/usr/bin/env bash + +set -eo pipefail + +DEFAULT_DOWNLOAD_URL="{{base_url}}" +DEFAULT_VERSION="{{version}}" + +function print_usage() { + echo "Usage: curl -ksSL http://console.local/instance/_get_gateway_install_script?token | sudo bash -s -- [-u url_for_download_program] [-v version_for_program] [-d target_install_dir] [--no-service]" + echo "Options:" + echo " -u, --url Install Gateway download URL, supports host, package directory, or direct package file URL" + echo " -v, --version Install Gateway version, default is Console configured version, current Console build version, or the latest version from the download source" + echo " -d, --install-dir Install Gateway target path, default is /opt/gateway" + echo " --no-service Install without system service (for containers or environments without sudo)" + exit 1 +} + +function print_header() { + echo " " + echo " @@@@@@@@@@@" + echo " @@@@@@@@@@@@" + echo " @@@@@@@@@@@@" + echo " @@@@@@@@@&@@@" + echo " #@@@@@@@@@@@@@" + echo " @@@ @@@@@@@@@@@@@ " + echo " &@@@@@@@ &@@@@@@@@@@@@@ " + echo " @&@@@@@@@&@ @@@&@@@@@@@&@ " + echo " @@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@ " + echo " @@@@@@@@@@@@@@@@@@& @@@@@@@@@@@@@ " + echo " %@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ " + echo " @@@@@@@@@@@@&@@@@@@@@@@@@@@@ " + echo " @@ ,@@@@@@@@@@@@@@@@@@@@@@@& " + echo " @@@@@. @@@@@&@@@@@@@@@@@@@@ " + echo " @@@@@@@@@@ @@@@@@@@@@@@@@@# " + echo " @&@@@&@@@&@@@ &@&@@@&@@@&@ " + echo " @@@@@@@@@@@@@. @@@@@@@* " + echo " @@@@@@@@@@@@@ %@@@ " + echo " @@@@@@@@@@@@@ " + echo "/@@@@@@@&@@@@@ " + echo "@@@@@@@@@@@@@ " + echo "@@@@@@@@@@@@@ " + echo "@@@@@@@@@@@@ Welcome to INFINI Labs!" + echo "" + echo "" + echo "Now attempting the Gateway installation... " + echo "" +} + +function print_footprint() { + echo " __ _ __ ____ __ _ __ __ " + echo " / // |/ // __// // |/ // / " + echo " / // || // _/ / // || // / " + echo "/_//_/|_//_/ /_//_/|_//_/ " + echo "" + echo "©INFINI.LTD, All Rights Reserved." + echo "" +} + +function __try() { + if [[ $try_status -eq 0 ]]; then + ! exception=$( $@ 2>&1 >/dev/null ) + try_status=${PIPESTATUS[0]} + fi +} + +function __catch() { + _old_try=$try_status + try_status=0 + [[ $_old_try -ne 0 ]] +} + +function get_latest_version() { + local input_url="${1:-$DEFAULT_DOWNLOAD_URL}" + input_url="${input_url%/}" + local latest_url + local latest_version="" + + case "$input_url" in + *.tar.gz|*.zip) + echo "" + return + ;; + esac + + for latest_url in "${input_url}/.latest" "${input_url%/gateway/stable}/.latest"; do + [[ -z "$latest_url" ]] && continue + latest_version=$(curl -m30 -s "$latest_url" |sed 's/",/"/;s/"//g;s/://1' |grep -Ev '^[{}]' |grep "$program_name" |awk '{print $NF}') + if [[ -n "$latest_version" ]]; then + echo "$latest_version" + return + fi + done + + echo "" +} + +function check_dir() { + if [[ "${install_dir}" != /* ]]; then + install_dir="$(pwd)/${install_dir}" + fi + if [[ ! -d "${install_dir}" ]]; then + __try mkdir -p "${install_dir}" + if __catch e; then + echo -e "Error: Unable to create installation directory, please manually create and reinstall.\nsudo mkdir -p ${install_dir} && sudo chown -R \$(whoami) ${install_dir}" >&2; exit 1; + fi + fi + install_dir=$(realpath "${install_dir}") + owner=$(ls -ld "${install_dir}" |awk '{print $3}') + if [[ "${owner}" != "$(whoami)" ]]; then + echo -e "Error: The installation directory ${install_dir} should be owner by current user.\nsudo chown -R \$(whoami) ${install_dir}" >&2; exit 1; + fi + + if [[ "$(ls -A ${install_dir})" ]]; then + echo "[gateway] found existing files in ${install_dir}, cleaning up while preserving runtime data" + if [[ "${install_dir}" == "/" ]]; then + echo "Error: refusing to clean root directory /" >&2; exit 1; + fi + cleanup_install_dir_preserving_runtime_data + fi +} + +function cleanup_install_dir_preserving_runtime_data() { + echo "[gateway] preserving runtime data in ${install_dir}/data and ${install_dir}/log" + shopt -s dotglob nullglob + for item in "${install_dir}"/*; do + name="$(basename "${item}")" + case "${name}" in + data|log) + continue + ;; + esac + rm -rf "${item}" + done + shopt -u dotglob nullglob +} + +function check_platform() { + local platform=$(uname) + local arch=$(uname -m) + + case $platform in + "Linux") + case $arch in + "i386"|"i686"|"x86") + file_ext="linux-386.tar.gz" + ;; + "x86_64"|"amd64") + file_ext="linux-amd64.tar.gz" + ;; + "aarch64"|"arm64") + file_ext="linux-arm64.tar.gz" + ;; + "armv5tel") + file_ext="linux-armv5.tar.gz" + ;; + "armv6l") + file_ext="linux-armv6.tar.gz" + ;; + "armv7"|"armv7l") + file_ext="linux-armv7.tar.gz" + ;; + "mips"|"mipsel") + file_ext="linux-mips.tar.gz" + ;; + "mips64") + file_ext="linux-mips64.tar.gz" + ;; + "mips64el") + file_ext="linux-mips64le.tar.gz" + ;; + "loong64"|"loongarch64") + file_ext="linux-loong64.tar.gz" + ;; + "sw_64") + file_ext="linux-sw64.tar.gz" + ;; + "riscv64") + file_ext="linux-riscv64.tar.gz" + ;; + *) + echo "Unsupported architecture: ${arch}" >&2 + exit 1 + ;; + esac + ;; + "Darwin") + case $arch in + "x86_64"|"amd64") + file_ext="mac-amd64.zip" + ;; + "arm64") + file_ext="mac-arm64.zip" + ;; + *) + echo "Unsupported architecture: ${arch}" >&2 + exit 1 + ;; + esac + ;; + "MINGW"*|"WSL"*|"Cygwin") + case $arch in + "i386"|"i686") + file_ext="windows-386.zip" + ;; + "x86_64"|"amd64") + file_ext="windows-amd64.zip" + ;; + *) + echo "Unsupported architecture: ${arch}" >&2 + exit 1 + ;; + esac + ;; + *) + echo "Unsupported platform: ${platform}" >&2 + exit 1 + ;; + esac +} + +function resolve_download_url() { + local input_url="${1:-$DEFAULT_DOWNLOAD_URL}" + input_url="${input_url%/}" + local archive_name="$2" + + case "$input_url" in + *.tar.gz|*.zip) + echo "$input_url" + return + ;; + esac + + if [[ "$input_url" =~ ^[a-zA-Z][a-zA-Z0-9+.-]*://[^/]+$ ]]; then + echo "${input_url}/gateway/stable/${archive_name}" + return + fi + + echo "${input_url}/${archive_name}" +} + +function install_binary() { + local archive_name="${program_name}-${version}-${file_ext}" + local download_url=$(resolve_download_url "$location" "$archive_name") + local downloaded_file="${download_url##*/}" + downloaded_file="${downloaded_file%%\?*}" + echo "File: [$download_url]" + + tmp_dir="$(mktemp -d)" + cd "$tmp_dir" + + if command -v curl >/dev/null 2>&1; then + curl -k -# -LO "$download_url" + elif command -v wget >/dev/null 2>&1; then + wget --no-check-certificate -q -nc --show-progress --progress=bar:force:noscroll "$download_url" + else + echo "Error: Could not find curl or wget, Please install wget or curl in advance." >&2; exit 1; + fi + + if [[ "${file_ext}" == *".tar.gz" ]]; then + tar -xzf "${downloaded_file}" -C "$install_dir" + else + unzip -q "${downloaded_file}" -d "$install_dir" + fi + + cd "${install_dir}" && rm -rf "${tmp_dir}" && echo "" +} + +function install_certs() { + ca_crt="{{ca_crt}}" + client_crt="{{client_crt}}" + client_key="{{client_key}}" + relay_server_crt="{{relay_server_crt}}" + relay_server_key="{{relay_server_key}}" + + mkdir -p ${install_dir}/config + echo "[gateway] waiting generate certs" + echo -e "${ca_crt}" > ${install_dir}/config/ca.crt + echo -e "${client_crt}" > ${install_dir}/config/client.crt + echo -e "${client_key}" > ${install_dir}/config/client.key + echo -e "${relay_server_crt}" > ${install_dir}/config/relay_server.crt + echo -e "${relay_server_key}" > ${install_dir}/config/relay_server.key +} + +function install_config() { + echo "[gateway] waiting generate config" + port={{port}} + console_endpoint="{{console_endpoint}}" + service_type="{{service_type}}" + relay_role="{{relay_role}}" + config_manager_server=${register_server:-$console_endpoint} + echo "[gateway] gateway api listening port $port" + echo "[gateway] relay config manager upstream: ${config_manager_server}" + echo "[gateway] service type: ${service_type}" + echo "[gateway] relay role: ${relay_role:-unset}" + cat < ${install_dir}/gateway.yml +configs.auto_reload: true + +env: + API_BINDING: "0.0.0.0:${port}" + SECURITY_ENABLED: true + CONFIG_MANAGER_SERVERS: ["${config_manager_server}"] + +path.data: "${install_dir}/data" +path.logs: "${install_dir}/log" +path.configs: "${install_dir}/config" + +api: + enabled: true + network: + binding: \$[[env.API_BINDING]] + websocket: + enabled: true + base_path: /ws + skip_host_verify: true + security: + enabled: \$[[env.SECURITY_ENABLED]] + username: '\$[[keystore.API_SECURITY_USERNAME]]' + password: '\$[[keystore.API_SECURITY_PASSWORD]]' + +web: + access_log_enabled: false + +badger: + mem_table_size: 10485760 + memory_mode: false + num_level0_tables: 1 + num_level0_tables_stall: 2 + num_mem_tables: 1 + path: "" + single_bucket_mode: true + sync_writes: false + value_log_file_size: 536870912 + value_log_max_entries: 1000000 + value_threshold: 1048576 + +disk_queue: + auto_skip_corrupted_file: true + cleanup_files_on_init: true + compress: + delete_after_compress: true + idle_threshold: 5 + message: + enabled: true + num_of_files_decompress_ahead: 3 + segment: + enabled: false + eof_retry_delay_in_ms: 500 + prepare_files_to_read: true + read_chan_buffer_size: 1000 + retention: + max_num_of_local_files: 2 + write_chan_buffer_size: 1000 + +elastic: + enabled: true + remote_configs: false + skip_init_metadata_on_start: false + availability_check: + enabled: true + interval: 30s + cluster_settings_check: + enabled: false + interval: 60s + health_check: + enabled: true + interval: 30s + metadata_refresh: + enabled: true + interval: 60s + +configs: + managed: true + panic_on_config_error: false + interval: "10s" + servers: \$[[env.CONFIG_MANAGER_SERVERS]] + manager: + access_token: '\$[[keystore.CONFIGS_MANAGER_ACCESS_TOKEN]]' + max_backup_files: 5 + soft_delete: false + tls: + enabled: true + cert_file: "config/client.crt" + key_file: "config/client.key" + ca_file: "config/ca.crt" + default_domain: "{{console_domain}}" + skip_insecure_verify: false + +node: + major_ip_pattern: ".*" + labels: + service_type: "${service_type}" + relay_role: "${relay_role}" +EOF +} + +function install_manager_token() { + access_token="{{access_token}}" + gateway_svc=${install_dir}/${program_name}-${file_ext%%.*} + + if [[ -z "${access_token}" ]]; then + echo "Error: access token is empty." >&2 + exit 1 + fi + + echo "[gateway] waiting save console manager access token" + echo -n "${access_token}" | ${gateway_svc} keystore add "CONFIGS_MANAGER_ACCESS_TOKEN" --stdin --force >/dev/null + echo -n "${access_token}" | ${gateway_svc} keystore add "configs_manager_bootstrap_token" --stdin --force >/dev/null +} + +function install_api_security_credentials() { + api_security_username="{{api_security_username}}" + api_security_password="{{api_security_password}}" + gateway_svc=${install_dir}/${program_name}-${file_ext%%.*} + + if [[ -z "${api_security_username}" ]]; then + echo "Error: api security username is empty." >&2 + exit 1 + fi + if [[ -z "${api_security_password}" ]]; then + echo "Error: api security password is empty." >&2 + exit 1 + fi + + echo "[gateway] waiting save local api security credentials" + echo -n "${api_security_username}" | ${gateway_svc} keystore add "API_SECURITY_USERNAME" --stdin --force >/dev/null + echo -n "${api_security_password}" | ${gateway_svc} keystore add "API_SECURITY_PASSWORD" --stdin --force >/dev/null +} + +function uninstall_service() { + gateway_svc=${install_dir}/${program_name}-${file_ext%%.*} + if [[ ! -f "$gateway_svc" ]]; then + return + fi + chmod 755 $gateway_svc + + macos_svc=/Library/LaunchDaemons/${service_name}.plist + linux_svc=/etc/systemd/system/${service_name}.service + + if [[ -f "$linux_svc" || -f "$macos_svc" ]]; then + echo "[gateway] waiting service stop & uninstall for exist ${service_name}" + (cd "${install_dir}" && SERVICE_NAME="${service_name}" $gateway_svc -service stop &>/dev/null || true) + (cd "${install_dir}" && SERVICE_NAME="${service_name}" $gateway_svc -service uninstall &>/dev/null || true) + fi +} + +function install_service() { + gateway_svc=${install_dir}/${program_name}-${file_ext%%.*} + chmod 755 $gateway_svc + echo "[gateway] waiting service install & start" + if ! (cd "${install_dir}" && SERVICE_NAME="${service_name}" $gateway_svc -service install &>/dev/null); then + echo "[gateway] failed to install service" >&2 + exit 1 + fi + if ! (cd "${install_dir}" && SERVICE_NAME="${service_name}" $gateway_svc -service start &>/dev/null); then + echo "[gateway] failed to start service" >&2 + exit 1 + fi +} + +function print_no_service_notice() { + gateway_svc=${install_dir}/${program_name}-${file_ext%%.*} + chmod 755 $gateway_svc + echo "[gateway] skip service install because --no-service is enabled" + echo "[gateway] start Gateway in foreground after installation:" + echo "cd \"${install_dir}\" && ./$(basename "${gateway_svc}") -config gateway.yml" +} + +function main() { + no_service="false" + while [[ $# -gt 0 ]]; do + case "$1" in + -u|--url) url_download="$2"; shift 2 ;; + -v|--version) version="$2"; shift 2 ;; + -d|--install-dir) target_dir="$2"; shift 2 ;; + --no-service) no_service="true"; shift ;; + *) print_usage ;; + esac + done + + program_name=gateway + service_name="{{service_name}}" + location=${url_download:-$DEFAULT_DOWNLOAD_URL} + install_dir=${target_dir:-{{install_dir}}} + latest_version="" + if [[ -z "${version}" && -z "${DEFAULT_VERSION}" ]]; then + latest_version=$(get_latest_version "$location") + fi + version=${version:-${DEFAULT_VERSION:-$latest_version}} + file_ext="" + if [[ -z "${service_name}" ]]; then + service_name="${program_name}" + fi + + if [[ -z "${version}" ]]; then + echo "Error: Could not obtain the latest version number. Please check the network and try again.">&2; exit 1; + else + echo "Name: [${program_name}], Service: [${service_name}], Version: [${version}], Path: [${install_dir}]" + fi + + check_platform + if [[ "$no_service" != "true" ]]; then + uninstall_service + fi + check_dir + install_binary + install_certs + install_config + install_manager_token + install_api_security_credentials + if [[ "$no_service" != "true" ]]; then + install_service + else + print_no_service_notice + fi + + echo "" + echo "" + echo "----------------------------------------------------------------" + echo "Congratulations, gateway install success!" + echo "----------------------------------------------------------------" + echo "" + echo "" + + print_footprint +} + +print_header + +main "$@" diff --git a/config/install_legacy_agent.tpl b/config/install_legacy_agent.tpl new file mode 100644 index 00000000..d4934509 --- /dev/null +++ b/config/install_legacy_agent.tpl @@ -0,0 +1,420 @@ +#!/usr/bin/env bash + +set -eo pipefail + +DEFAULT_DOWNLOAD_URL="{{base_url}}" +DEFAULT_VERSION="{{version}}" + +function print_usage() { + echo "Usage: curl -sSL http://get.infini.cloud/ | bash -s -- [-u url_for_download_program] [-v version_for_program ] [-t target_install_dir] [-s url_console_lan_adress] [--no-service]" + echo "Options:" + echo " -u, --url Download url of the program to install which default is http://localhost" + echo " -v, --version Version of the program to install, default is the Console configured version, current Console build version, or the latest version from the download source" + echo " -t, --target Target directory of the program install which default is /opt/agent" + echo " -s, --server Server address for Agent to communicate with INFINI Console after install, default is current Console address, can be manually specified" + echo " --no-service Skip service install/start and print foreground startup instructions for containers or non-sudo environments" + exit 1 +} + +function print_header() { + echo " " + echo " @@@@@@@@@@@" + echo " @@@@@@@@@@@@" + echo " @@@@@@@@@@@@" + echo " @@@@@@@@@&@@@" + echo " #@@@@@@@@@@@@@" + echo " @@@ @@@@@@@@@@@@@ " + echo " &@@@@@@@ &@@@@@@@@@@@@@ " + echo " @&@@@@@@@&@ @@@&@@@@@@@&@ " + echo " @@@@@@@@@@@@@@@@ @@@@@@@@@@@@@@ " + echo " @@@@@@@@@@@@@@@@@@& @@@@@@@@@@@@@ " + echo " %@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ " + echo " @@@@@@@@@@@@&@@@@@@@@@@@@@@@ " + echo " @@ ,@@@@@@@@@@@@@@@@@@@@@@@& " + echo " @@@@@. @@@@@&@@@@@@@@@@@@@@ " + echo " @@@@@@@@@@ @@@@@@@@@@@@@@@# " + echo " @&@@@&@@@&@@@ &@&@@@&@@@&@ " + echo " @@@@@@@@@@@@@. @@@@@@@* " + echo " @@@@@@@@@@@@@ %@@@ " + echo " @@@@@@@@@@@@@ " + echo "/@@@@@@@&@@@@@ " + echo "@@@@@@@@@@@@@ " + echo "@@@@@@@@@@@@@ " + echo "@@@@@@@@@@@@ Welcome to INFINI Labs!" + echo "" + echo "" + echo "Now attempting the installation... " + echo "" +} + +function print_footprint() { + echo " __ _ __ ____ __ _ __ __ " + echo " / // |/ // __// // |/ // / " + echo " / // || // _/ / // || // / " + echo "/_//_/|_//_/ /_//_/|_//_/ " + echo "" + echo "©INFINI.LTD, All Rights Reserved." + echo "" +} + +function __try() { + if [[ $try_status -eq 0 ]]; then + ! exception=$( $@ 2>&1 >/dev/null ) + try_status=${PIPESTATUS[0]} + fi +} + +function __catch() { + _old_try=$try_status + try_status=0 + [[ $_old_try -ne 0 ]] +} + +function confirm() { + display_str=$1 + default_ans=$2 + if [[ $default_ans == 'y/N' ]]; then + must_match='[yY]' + else + must_match='[nN]' + fi + read -p"${display_str} [${default_ans}]:" ans + [[ $ans == $must_match ]] +} + +function get_latest_version() { + local input_url="${1%/}" + local latest_url + local latest_version="" + + case "$input_url" in + *.tar.gz|*.zip) + echo "" + return + ;; + esac + + for latest_url in "${input_url}/.latest" "${input_url%/agent/stable}/.latest"; do + [[ -z "$latest_url" ]] && continue + latest_version=$(curl -m3 -s "$latest_url" |sed 's/",/"/;s/"//g;s/://1' |grep -Ev '^[{}]' |grep "$program_name" |awk '{print $NF}') + if [[ -n "$latest_version" ]]; then + echo "$latest_version" + return + fi + done + + echo "" +} + +function check_dir() { + if [[ ! -d "${install_dir}" ]]; then + __try mkdir -p "${install_dir}" + if __catch e; then + echo -e "Error: Unable to create installation directory, please manually create and reinstall.\nsudo mkdir -p ${install_dir} && sudo chown -R \$(whoami) ${install_dir}" >&2; exit 1; + fi + fi + + owner=$(ls -ld "${install_dir}" |awk '{print $3}') + if [[ "${owner}" != "$(whoami)" ]]; then + echo -e "Error: The installation directory ${install_dir} should be owner by current user.\nsudo chown -R \$(whoami) ${install_dir}" >&2; exit 1; + fi + + if [[ "$(ls -A ${install_dir})" ]]; then + confirm "RISK WARN: Replace or upgrade exists agent version, Proceed?" 'y/N' && echo || exit 1; + if [[ "${no_service}" != "true" ]]; then + uninstall_service + fi + rm -rf ${install_dir}/* + fi +} + +function check_platform() { + local platform=$(uname) + local arch=$(uname -m) + + case $platform in + "Linux") + case $arch in + "i386"|"i686"|"x86") + file_ext="linux-386.tar.gz" + ;; + "x86_64"|"amd64") + file_ext="linux-amd64.tar.gz" + ;; + "aarch64"|"arm64") + file_ext="linux-arm64.tar.gz" + ;; + "armv5tel") + file_ext="linux-armv5.tar.gz" + ;; + "armv6l") + file_ext="linux-armv6.tar.gz" + ;; + "armv7"|"armv7l") + file_ext="linux-armv7.tar.gz" + ;; + "mips"|"mipsel") + file_ext="linux-mips.tar.gz" + ;; + "mips64") + file_ext="linux-mips64.tar.gz" + ;; + "mips64el") + file_ext="linux-mips64le.tar.gz" + ;; + "loong64") + file_ext="linux-loong64.tar.gz" + ;; + "sw_64") + file_ext="linux-sw64.tar.gz" + ;; + "riscv64") + file_ext="linux-riscv64.tar.gz" + ;; + *) + echo "Unsupported architecture: ${arch}" >&2 + exit 1 + ;; + esac + ;; + "Darwin") + case $arch in + "x86_64"|"amd64") + file_ext="mac-amd64.zip" + ;; + "arm64") + file_ext="mac-arm64.zip" + ;; + *) + echo "Unsupported architecture: ${arch}" >&2 + exit 1 + ;; + esac + ;; + "MINGW"*|"WSL"*|"Cygwin") + case $arch in + "i386"|"i686") + file_ext="windows-386.zip" + ;; + "x86_64"|"amd64") + file_ext="windows-amd64.zip" + ;; + *) + echo "Unsupported architecture: ${arch}" >&2 + exit 1 + ;; + esac + ;; + *) + echo "Unsupported platform: ${platform}" >&2 + exit 1 + ;; + esac +} + +function install_binary() { + local download_url="$location/${program_name}-${version}-${file_ext}" + echo "File: [$download_url]" + + tmp_dir="$(mktemp -d)" + cd "$tmp_dir" + + if command -v curl >/dev/null 2>&1; then + curl -k -# -LO "$download_url" + elif command -v wget >/dev/null 2>&1; then + wget --no-check-certificate -q -nc --show-progress --progress=bar:force:noscroll "$download_url" + else + echo "Error: Could not find curl or wget, Please install wget or curl in advance." >&2; exit 1; + fi + + if [[ "${file_ext}" == *".tar.gz" ]]; then + tar -xzf "${program_name}-${version}-${file_ext}" -C "$install_dir" + else + unzip -q "${program_name}-${version}-${file_ext}" -d "$install_dir" + fi + + cd "${install_dir}" && rm -rf "${tmp_dir}" && echo "" +} + +function install_certs() { + ca_crt="{{ca_crt}}" + client_crt="{{client_crt}}" + client_key="{{client_key}}" + + mkdir -p ${install_dir}/config + echo "[agent] waiting generate certs" + echo -e "${ca_crt}" > ${install_dir}/config/ca.crt + echo -e "${client_crt}" > ${install_dir}/config/client.crt + echo -e "${client_key}" > ${install_dir}/config/client.key +} + +function install_config() { + echo "[agent] waiting generate config" + port={{port}} + console_endpoint="{{console_endpoint}}" + server=${register_server:-$console_endpoint} + echo "[agent] agent listening port $port, will register to console endpoint [ $server ]" + cat < ${install_dir}/agent.yml +configs.auto_reload: true + +env: + API_BINDING: "0.0.0.0:${port}" + +path.data: "${install_dir}/data" +path.logs: "${install_dir}/log" +path.configs: "${install_dir}/config" + +resource_limit.cpu.max_num_of_cpus: 1 +resource_limit.memory.max_in_bytes: 533708800 + +stats: + include_storage_stats_in_api: false + +elastic: + skip_init_metadata_on_start: true + health_check: + enabled: true + interval: 60s + availability_check: + enabled: false + interval: 60s + +disk_queue: + max_msg_size: 20485760 + max_bytes_per_file: 20485760 + max_used_bytes: 1024288000 + retention.max_num_of_local_files: 1 + compress: + idle_threshold: 0 + num_of_files_decompress_ahead: 0 + segment: + enabled: true + + api: + enabled: true + tls: + enabled: true + cert_file: "config/client.crt" + key_file: "config/client.key" + ca_file: "config/ca.crt" + skip_insecure_verify: false + network: + binding: \$[[env.API_BINDING]] + +badger: + value_threshold: 1024 + mem_table_size: 1048576 + value_log_max_entries: 1000000 + value_log_file_size: 104857600 + +configs: + #for managed client's setting + managed: true # managed by remote servers + panic_on_config_error: false #ignore config error + interval: "10s" + servers: # config servers + - "${server}" + soft_delete: false + max_backup_files: 5 + tls: #for mTLS connection with config servers + enabled: true + cert_file: "config/client.crt" + key_file: "config/client.key" + ca_file: "config/ca.crt" + default_domain: "{{console_domain}}" + skip_insecure_verify: false + +node: + major_ip_pattern: ".*" +EOF +} + +function uninstall_service() { + agent_svc=${install_dir}/${program_name}-${file_ext%%.*} + chmod 755 $agent_svc + + macos_svc=/Library/LaunchDaemons/agent.plist + linux_svc=/etc/systemd/system/agent.service + + if [[ -f "$linux_svc" || -f "$macos_svc" ]]; then + echo "[agent] waiting service stop & uninstall for exist agent" + (cd "${install_dir}" && $agent_svc -service stop &>/dev/null) + (cd "${install_dir}" && $agent_svc -service uninstall &>/dev/null) + fi + sleep 3 +} + +function install_service() { + agent_svc=${install_dir}/${program_name}-${file_ext%%.*} + chmod 755 $agent_svc + echo "[agent] waiting service install & start" + (cd "${install_dir}" && $agent_svc -service install &>/dev/null) + (cd "${install_dir}" && $agent_svc -service start &>/dev/null) + sleep 3 +} + +function print_no_service_hint() { + local agent_bin="${program_name}-${file_ext%%.*}" + echo "[agent] service installation skipped (--no-service)" + echo "[agent] start agent in foreground:" + echo " (cd \"${install_dir}\" && exec ./${agent_bin} -config agent.yml)" + echo "[agent] container example:" + echo " ENTRYPOINT [\"sh\", \"-c\", \"cd ${install_dir} && exec ./agent-* -config agent.yml\"]" + echo " CMD [\"sh\", \"-c\", \"cd ${install_dir} && exec ./agent-* -config agent.yml\"]" +} + +function main() { + while [[ $# -gt 0 ]]; do + case "$1" in + -u|--url) url_download="$2"; shift 2 ;; + -v|--version) version="$2"; shift 2 ;; + -t|--target) target_dir="$2"; shift 2 ;; + -s|--server) register_server="$2"; shift 2 ;; + --no-service) no_service=true; shift ;; + *) print_usage ;; + esac + done + + program_name=agent + location=${url_download:-$DEFAULT_DOWNLOAD_URL} + install_dir=${target_dir:-/opt/$program_name} + latest_version="" + if [[ -z "${version}" && -z "${DEFAULT_VERSION}" ]]; then + latest_version=$(get_latest_version "$location") + fi + version=${version:-${DEFAULT_VERSION:-$latest_version}} + file_ext="" + + if [[ -z "${version}" ]]; then + echo "Error: Could not obtain the latest version number. Please check the network and try again.">&2; exit 1; + else + echo "Name: [${program_name}], Version: [${version}], Path: [${install_dir}]" + fi + + no_service=${no_service:-false} + check_dir + check_platform + install_binary + install_certs + install_config + if [[ "${no_service}" == "true" ]]; then + print_no_service_hint + else + uninstall_service + install_service + fi + + echo "" + echo "" + echo "----------------------------------------------------------------" + echo "Congratulations, agent install success!" + echo "----------------------------------------------------------------" + echo "" + echo "" + + print_footprint +} + +print_header + +main "$@" diff --git a/config/self_hosted_packages.go b/config/self_hosted_packages.go new file mode 100644 index 00000000..36cfdeab --- /dev/null +++ b/config/self_hosted_packages.go @@ -0,0 +1,64 @@ +package config + +import ( + "os" + "path/filepath" + "strings" +) + +func EnsureSelfHostedPackageDirs(basePath string) error { + basePath, err := ResolveSelfHostedPackageBasePath(basePath) + if err != nil { + return err + } + basePath = strings.TrimSpace(basePath) + if basePath == "" { + return nil + } + + for _, relativeDir := range []string{ + filepath.Join("agent", "stable"), + filepath.Join("gateway", "stable"), + } { + if err := os.MkdirAll(filepath.Join(basePath, relativeDir), 0o755); err != nil { + return err + } + } + + return nil +} + +func ResolveSelfHostedPackageBasePath(basePath string) (string, error) { + basePath = strings.TrimSpace(basePath) + if basePath == "" || filepath.IsAbs(basePath) { + return basePath, nil + } + + executablePath, err := os.Executable() + if err != nil { + return "", err + } + return filepath.Join(filepath.Dir(executablePath), basePath), nil +} + +func HasSelfHostedPackageFiles(basePath, relativeDir string) (bool, error) { + basePath, err := ResolveSelfHostedPackageBasePath(basePath) + if err != nil { + return false, err + } + basePath = strings.TrimSpace(basePath) + if basePath == "" { + return false, nil + } + + targetDir := filepath.Join(basePath, relativeDir) + entries, err := os.ReadDir(targetDir) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return len(entries) > 0, nil +} diff --git a/config/self_hosted_packages_test.go b/config/self_hosted_packages_test.go new file mode 100644 index 00000000..6ca7bec9 --- /dev/null +++ b/config/self_hosted_packages_test.go @@ -0,0 +1,45 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestResolveSelfHostedPackageBasePathUsesExecutableDir(t *testing.T) { + executablePath, err := os.Executable() + if err != nil { + t.Fatalf("failed to get executable path: %v", err) + } + + got, err := ResolveSelfHostedPackageBasePath(".public") + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + + expected := filepath.Join(filepath.Dir(executablePath), ".public") + if got != expected { + t.Fatalf("expected %q, got %q", expected, got) + } +} + +func TestEnsureSelfHostedPackageDirsCreatesExpectedStructure(t *testing.T) { + baseDir := t.TempDir() + if err := EnsureSelfHostedPackageDirs(baseDir); err != nil { + t.Fatalf("expected nil error, got %v", err) + } + + for _, relativeDir := range []string{ + filepath.Join("agent", "stable"), + filepath.Join("gateway", "stable"), + } { + fullPath := filepath.Join(baseDir, relativeDir) + info, err := os.Stat(fullPath) + if err != nil { + t.Fatalf("expected %q to exist: %v", fullPath, err) + } + if !info.IsDir() { + t.Fatalf("expected %q to be a directory", fullPath) + } + } +} diff --git a/config/setup/common/agent.tpl b/config/setup/common/agent.tpl index e0212fe7..9184b9c8 100644 --- a/config/setup/common/agent.tpl +++ b/config/setup/common/agent.tpl @@ -11,7 +11,8 @@ POST $[[SETUP_INDEX_PREFIX]]configs/$[[SETUP_DOC_TYPE]]/system_ingest_config_yml "updated": "2023-10-18T14:49:56.768754+08:00", "metadata": { "labels": { - "instance": "_all" + "instance": "_all", + "service_type": "relay" }, "category": "app_settings", "name": "agent" @@ -31,7 +32,8 @@ POST $[[SETUP_INDEX_PREFIX]]configs/$[[SETUP_DOC_TYPE]]/task_config_tpl "updated": "2023-10-19T14:49:56.768754+08:00", "metadata": { "labels": { - "instance": "_all" + "instance": "_all", + "service_type": "migration" }, "category": "app_settings", "name": "agent" @@ -39,22 +41,44 @@ POST $[[SETUP_INDEX_PREFIX]]configs/$[[SETUP_DOC_TYPE]]/task_config_tpl "id": "task_config_tpl" } -#init_gateway_config.tpl +#init_gateway_relay_config.tpl POST $[[SETUP_INDEX_PREFIX]]configs/$[[SETUP_DOC_TYPE]]/agent_relay_gateway_config_yml { "payload": { - "content": "$[[SETUP_AGENT_RELAY_GATEWAY_CONFIG]]", + "content": "$[[SETUP_GATEWAY_RELAY_CONFIG]]", "version": 1, - "location": "agent_relay_gateway_config.yml", - "name": "agent_relay_gateway_config.yml" + "location": "relay.yml", + "name": "relay.yml" }, "updated": "2023-10-19T14:49:56.768754+08:00", "metadata": { "labels": { - "instance": "_all" + "instance": "_all", + "service_type": "relay" }, "category": "app_settings", "name": "gateway" }, "id": "agent_relay_gateway_config_yml" -} \ No newline at end of file +} + +#init_gateway_migration_config.tpl +POST $[[SETUP_INDEX_PREFIX]]configs/$[[SETUP_DOC_TYPE]]/gateway_migration_yml +{ + "payload": { + "content": "$[[SETUP_GATEWAY_MIGRATION_CONFIG]]", + "version": 1, + "location": "migration.yml", + "name": "migration.yml" + }, + "updated": "2023-10-19T14:49:56.768754+08:00", + "metadata": { + "labels": { + "instance": "_all", + "service_type": "migration" + }, + "category": "app_settings", + "name": "gateway" + }, + "id": "gateway_migration_yml" +} diff --git a/config/setup/common/alerting.en-US.tpl b/config/setup/common/alerting.en-US.tpl new file mode 100644 index 00000000..d629de8a --- /dev/null +++ b/config/setup/common/alerting.en-US.tpl @@ -0,0 +1,2391 @@ +PUT $[[SETUP_INDEX_PREFIX]]alert-rule +{ + "mappings": { + "properties": { + "bucket_conditions": { + "properties": { + "items": { + "properties": { + "bucket_count": { + "type": "long" + }, + "minimum_period_match": { + "type": "long" + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + }, + "priority": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + }, + "values": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "conditions": { + "properties": { + "items": { + "properties": { + "minimum_period_match": { + "type": "long" + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + }, + "priority": { + "type": "keyword", + "ignore_above": 256 + }, + "values": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "created": { + "type": "date" + }, + "creator": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "metrics": { + "properties": { + "bucket_label": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "bucket_size": { + "type": "keyword", + "ignore_above": 256 + }, + "expression": { + "type": "keyword", + "ignore_above": 256 + }, + "format_type": { + "type": "keyword", + "ignore_above": 256 + }, + "formula": { + "type": "keyword", + "ignore_above": 256 + }, + "groups": { + "properties": { + "field": { + "type": "keyword", + "ignore_above": 256 + }, + "limit": { + "type": "long" + } + } + }, + "items": { + "properties": { + "field": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "statistic": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "notification_config": { + "properties": { + "accept_time_range": { + "properties": { + "end": { + "type": "keyword", + "ignore_above": 256 + }, + "start": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "enabled": { + "type": "boolean" + }, + "escalation_throttle_period": { + "type": "keyword", + "ignore_above": 256 + }, + "message": { + "type": "keyword", + "ignore_above": 256 + }, + "normal": { + "properties": { + "created": { + "type": "date" + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "sub_type": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + }, + "updated": { + "type": "date" + }, + "webhook": { + "properties": { + "body": { + "type": "keyword", + "ignore_above": 256 + }, + "header_params": { + "properties": { + "Content-Type": { + "type": "keyword", + "ignore_above": 256 + }, + "Content-type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "throttle_period": { + "type": "keyword", + "ignore_above": 256 + }, + "title": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recovery_notification_config": { + "properties": { + "accept_time_range": { + "properties": { + "end": { + "type": "keyword", + "ignore_above": 256 + }, + "start": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "enabled": { + "type": "boolean" + }, + "event_enabled": { + "type": "boolean" + }, + "incremental_recovery_enabled": { + "type": "boolean" + }, + "message": { + "type": "keyword", + "ignore_above": 256 + }, + "normal": { + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "title": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "resource": { + "properties": { + "context": { + "type": "object" + }, + "filter": { + "type": "object", + "enabled": false + }, + "objects": { + "type": "keyword", + "ignore_above": 256 + }, + "raw_filter": { + "type": "object", + "enabled": false + }, + "resource_id": { + "type": "keyword", + "ignore_above": 256 + }, + "resource_name": { + "type": "keyword", + "ignore_above": 256 + }, + "time_field": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "schedule": { + "properties": { + "interval": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "updated": { + "type": "date" + } + } + } +} + +#alerting channel +#The `id` value is consistent with the `_id` value +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgnb2nt3q95nmusjl65g +{ + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-09T22:39:50.494915568+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing !*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8bq8d3q95ogankugqg +{ + "id": "cj8bq8d3q95ogankugqg", + "created": "2023-08-07T17:45:05.534408059+08:00", + "updated": "2023-08-09T22:39:56.489567891+08:00", + "name": "[Recovery] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*ResolveAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Target:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Duration:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-09T22:39:51.540172306+08:00", + "name": "[Alerting] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) is ongoing\\n**{{.title}}**\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgiospt3q95q49k3u00g +{ + "id": "cgiospt3q95q49k3u00g", + "created": "2023-03-30T13:28:07.531263747Z", + "updated": "2023-08-09T22:39:52.356059486+08:00", + "name": "[Alerting] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.cn/img/email/alert-header.png)\\n\\n🔥 Incident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) is ongoing\\n\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8ctat3q95l9ebbntlg +{ + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-09T22:39:58.967970184+08:00", + "name": "[Recovery] DingTalk Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.cn/img/email/recovery-header.png)\\n\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-09T22:40:04.665871275+08:00", + "name": "[Recovery] Wechat Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI Platform Alerting ]**\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgnb2r53q95nmusjl6vg +{ + "id": "cgnb2r53q95nmusjl6vg", + "created": "2023-04-06T11:47:56.652637309Z", + "updated": "2023-08-10T12:04:08.046781556+08:00", + "name": "[Alerting] Email Notification", + "type": "email", + "sub_type": "email", + "email": { + "server_id": "", + "recipients": { + "to": [], + "cc": [] + }, + "subject": "[INFINI Platform Alerting] 🔥 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8atf53q95lhahebg8g +{ + "id": "cj8atf53q95lhahebg8g", + "created": "2023-08-07T16:43:40.062389175+08:00", + "updated": "2023-08-10T12:04:42.842628127+08:00", + "name": "[Recovery] Email Notification", + "type": "email", + "sub_type": "email", + "email": { + "server_id": "", + "recipients": { + "to": [], + "cc": [] + }, + "subject": "[INFINI Platform Alerting] {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-10T17:18:38.592432088+08:00", + "name": "[Alerting] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) is ongoing\\n **{{.title}}**\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false + } +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-10T17:18:36.035896482+08:00", + "name": "[Recovery] Feishu Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"**{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"[View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj865st3q95rega919ig +{ + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-10T17:18:41.92016786+08:00", + "name": "[Alerting] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\"content\": \"**[ INFINI Platform Alerting ]**\\n🔥 Incident [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) is ongoing\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\nEventID: {{.event_id}}\\nTarget: {{.resource_name}}-{{.objects}}\\nTriggerAt: {{.trigger_at | datetime}}\\n{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj86l0l3q95rrpfea6ug +{ + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-10T17:18:44.422687739+08:00", + "name": "[Recovery] Discord Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"**[ INFINI Platform Alerting ]**\\n🌈 **{{.title}}**\\n\\n{{.message | str_replace \"\\n\" \"\\\\n\" }}\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n}" + }, + "sub_type": "discord", + "enabled": false +} + +#alerting +#The `id` value is consistent with the `_id` value +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710dpnoaps0 +{ + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T22:39:43.98598502+08:00", + "name": "Cluster Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(payload.elasticsearch.cluster_health.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 Health of Clusters ({{.total_results}} clusters in total, showing top {{len .results}}) Changed to Red{{else}}🔥 Health of Clusters ({{.total_results}} clusters in total) Changed to Red{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nCluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:02:17.165625799+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T22:20:17.864619426+08:00", + "name": "Index Health Change to Red", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics" + ], + "filter": {}, + "raw_filter": {"bool":{"must":[{"term":{"payload.elasticsearch.index_health.status":"red"}},{"term":{"metadata.name":{"value":"index_health"}}}]}}, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 Health of Indices ({{.total_results}} indices in total, showing top {{len .results}}) Changed to Red{{else}}🔥 Health of Indices ({{.total_results}} indices in total) Changed to Red{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D) is Red now\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}> is Red now\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈{{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T22:11:45.679048697+08:00", + "name": "Node left cluster", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "ignore_time_filter": true, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 Node left cluster", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Left: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Left: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cm4z6m5fv8l0qk2p9x7r +{ + "id": "builtin-cm4z6m5fv8l0qk2p9x7r", + "created": "2024-01-01T00:00:00Z", + "updated": "2024-01-01T00:00:00Z", + "name": "Cluster offline", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_cluster" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "labels.health_status": "unavailable" + } + }, + "ignore_time_filter": true, + "time_field": "updated", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "labels.health_status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(labels.health_status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 Cluster offline", + "message": "{{range .results}}\n{{$cid := index .group_values 0 }}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" $cid }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT $cid}}\nCluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Status: offline, Count: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cb34sfl6psfiqtovhpt4 +{ + "id": "builtin-cb34sfl6psfiqtovhpt4", + "created": "2022-07-07T03:08:46.297166036Z", + "updated": "2023-08-09T22:38:41.764325087+08:00", + "name": "Too Many Deleted Documents (Only Index>31GB)", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { "term": { "metadata.name": { "value": "index_stats" } } }, + { "range": { "payload.elasticsearch.index_stats.primaries.store.size_in_bytes": { "gte": 32212254720 } } } + ], + "must_not": [ + { "term": { "metadata.labels.index_name": { "value": "_all" } } } + ] + } +}, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 20 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "(a/(a+b))*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.docs.count", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "30" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "40" + ], + "priority": "high" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "55" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 Too Many Deleted Documents (>30%)", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Deleted ratio: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "24h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp2e4ianisjmu4giqs7 +{ + "id": "builtin-cbp2e4ianisjmu4giqs7", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T22:39:15.339913317+08:00", + "name": "Search latency is great than 500ms", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "index_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ], + "must_not": [ + { + "term": { + "metadata.labels.index_name": { + "value": "_all" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "a/b", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + "statistic": "rate" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.search.query_total", + "statistic": "rate" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "500" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "300" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 Search latency is great than 500ms", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Search latency: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calaqnh7h710dpnp2bm8 +{ + "id": "builtin-calaqnh7h710dpnp2bm8", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T22:38:55.677122718+08:00", + "name": "JVM utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + "statistic": "p90" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 JVM Usage of Nodes ({{.total_results}} nodes in total, showing top {{len .results}}) >= {{.first_threshold}}%{{else}}🔥 JVM Usage of Nodes ({{.total_results}} nodes in total) >= {{.first_threshold}}%{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, JVM Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calakp97h710dpnp1fa2 +{ + "id": "builtin-calakp97h710dpnp1fa2", + "created": "2022-06-16T03:58:29.437447113Z", + "updated": "2023-08-09T22:33:25.692835454+08:00", + "name": "CPU utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.process.cpu.percent", + "statistic": "avg" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 CPU Usage of Nodes ({{.total_results}} nodes in total, showing top {{len .results}}) >= {{.first_threshold}}%{{else}}🔥 CPU Usage of Nodes ({{.total_results}} nodes in total) >= {{.first_threshold}}%{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, CPU Usage: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T22:37:44.038127695+08:00", + "name": "Shard Storage >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 Shard Storage >55GB in ({{.total_results}} indices in total, showing top {{len .results}}){{else}}🔥 Shard Storage >55GB in ({{.total_results}} indices in total){{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[Alerting] Slack Notification", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Index: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0)}}%22} | {{index .group_values 1}}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Max shard storage: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at | datetime}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}{{if .recovery_context}}\n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710dpnogps1 +{ + "id": "builtin-cal8n7p7h710dpnogps1", + "created": "2022-06-16T03:11:01.445958361Z", + "updated": "2023-08-10T17:16:34.900352415+08:00", + "name": "Disk utilization is Too High", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 200 + } + ], + "formula": "((a-b)/a)*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 Disk Usage of Nodes ({{.total_results}} nodes in total, showing top {{len .results}}) >= {{.first_threshold}}%{{else}}🔥 Disk Usage of Nodes ({{.total_results}} nodes in total) >= {{.first_threshold}}%{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} Incident <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> is ongoing*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*TriggerAt:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Priority:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"Node: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}> of Cluster: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}>, Disk Usage: {{.result_value | to_fixed 2}}%, Free: {{.relation_values.b | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Incident\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n },\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"View Document\" \n },\n \"style\": \"primary\",\n \"url\": \"https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#disk-based-shard-allocation\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 [{{.rule_name}}] Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cujivv5ath26drn6bcl0 +{ + "id": "builtin-cujivv5ath26drn6bcl0", + "created": "2025-02-08T18:20:44.273334+08:00", + "updated": "2025-02-12T16:31:05.672771+08:00", + "name": "Cluster Metrics Collection Anomaly", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "terms": { + "metadata.name": [ + "cluster_health", + "cluster_stats", + "index_stats", + "node_stats", + "shard_stats" + ] + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.name", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "agent.id", + "statistic": "count" + } + ], + "bucket_label": { + "enabled": false + }, + "expression": "count(agent.id)" + }, + "bucket_conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "lt", + "values": [ + "0" + ], + "priority": "critical", + "type": "content", + "bucket_count": 10 + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 {{.rule_name}} Alerting", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nCluster [[{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D)] ({{index .group_values 1}}) metrics has dropped at {{.issue_timestamp | datetime}};\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} \ No newline at end of file diff --git a/config/setup/common/alerting.tpl b/config/setup/common/alerting.tpl index 1ae2cbcc..d629de8a 100644 --- a/config/setup/common/alerting.tpl +++ b/config/setup/common/alerting.tpl @@ -255,6 +255,9 @@ PUT $[[SETUP_INDEX_PREFIX]]alert-rule "event_enabled": { "type": "boolean" }, + "incremental_recovery_enabled": { + "type": "boolean" + }, "message": { "type": "keyword", "ignore_above": 256 @@ -395,7 +398,7 @@ POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgiospt3q95q49k3u00g }, "method": "POST", "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/alert-header.png)\\n\\n🔥 Incident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) is ongoing\\n\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.cn/img/email/alert-header.png)\\n\\n🔥 Incident [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) is ongoing\\n\\n**{{.title}}**\\n\\nPriority: {{.priority}}\\n\\nEventID: {{.event_id}}\\n\\nTarget: {{.resource_name}}-{{.objects}}\\n\\nTriggerAt: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" }, "sub_type": "dingtalk", "enabled": false @@ -413,7 +416,7 @@ POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8ctat3q95l9ebbntlg }, "method": "POST", "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.com/img/email/recovery-header.png)\\n\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n}\n" + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI Platform Alerting](https://infinilabs.cn/img/email/recovery-header.png)\\n\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n}\n" }, "sub_type": "dingtalk", "enabled": false @@ -451,7 +454,7 @@ POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgnb2r53q95nmusjl6vg "cc": [] }, "subject": "[INFINI Platform Alerting] 🔥 {{.title}}", - "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

Priority: {{.priority}}

\n

EventID: {{.event_id}}

\n

Target: {{.resource_name}}-{{.objects}}

\n

TriggerAt: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", "content_type": "text/html" }, "enabled": false @@ -471,7 +474,7 @@ POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8atf53q95lhahebg8g "cc": [] }, "subject": "[INFINI Platform Alerting] {{.title}}", - "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n View Detail\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", "content_type": "text/html" }, "enabled": false @@ -507,7 +510,7 @@ POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8e9s53q95gsdbb054g }, "method": "POST", "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", - "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"🌈 **{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"[View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n ]\n }\n}" + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI Platform Alerting ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"**{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"[View Incident]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n ]\n }\n}" }, "sub_type": "feishu", "enabled": false @@ -626,7 +629,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710d }, "notification_config": { "enabled": true, - "title": "Health of Clusters ({{len .results}} clusters in total) Changed to Red", + "title": "{{if gt .total_results (len .results)}}🔥 Health of Clusters ({{.total_results}} clusters in total, showing top {{len .results}}) Changed to Red{{else}}🔥 Health of Clusters ({{.total_results}} clusters in total) Changed to Red{{end}}", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nCluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) is Red now\n{{end}}", "normal": [ { @@ -675,8 +678,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710d "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8atf53q95lhahebg8g", @@ -738,11 +741,11 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calavvp7h710d "bucket_size": "1m", "groups": [ { - "field": "metadata.cluster_id", + "field": "metadata.labels.cluster_id", "limit": 50 }, { - "field": "metadata.index_name", + "field": "metadata.labels.index_name", "limit": 1000 } ], @@ -750,7 +753,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calavvp7h710d "items": [ { "name": "a", - "field": "metadata.index_name", + "field": "metadata.labels.index_name", "statistic": "count" } ], @@ -758,7 +761,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calavvp7h710d "bucket_label": { "enabled": false }, - "expression": "count(metadata.index_name)" + "expression": "count(metadata.labels.index_name)" }, "conditions": { "operator": "any", @@ -775,7 +778,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calavvp7h710d }, "notification_config": { "enabled": true, - "title": "Health of Indices ({{len .results}} indices in total) Changed to Red", + "title": "{{if gt .total_results (len .results)}}🔥 Health of Indices ({{.total_results}} indices in total, showing top {{len .results}}) Changed to Red{{else}}🔥 Health of Indices ({{.total_results}} indices in total) Changed to Red{{end}}", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D) is Red now\n{{end}}", "normal": [ { @@ -824,8 +827,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calavvp7h710d "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈{{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -867,7 +870,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp20n2anisjm "id": "builtin-cbp20n2anisjmu4gehc5", "created": "2022-08-09T08:52:44.63345561Z", "updated": "2023-08-09T22:11:45.679048697+08:00", - "name": "Elasticsearch node left cluster", + "name": "Node left cluster", "enabled": true, "resource": { "resource_id": "$[[SETUP_RESOURCE_ID]]", @@ -882,6 +885,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp20n2anisjm "metadata.labels.status": "unavailable" } }, + "ignore_time_filter": true, "time_field": "timestamp", "context": { "fields": null @@ -928,7 +932,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp20n2anisjm }, "notification_config": { "enabled": true, - "title": "Elasticsearch node left cluster", + "title": "🔥 Node left cluster", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Left: {{.result_value}}\n{{end}}", "normal": [ { @@ -977,8 +981,146 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp20n2anisjm "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cm4z6m5fv8l0qk2p9x7r +{ + "id": "builtin-cm4z6m5fv8l0qk2p9x7r", + "created": "2024-01-01T00:00:00Z", + "updated": "2024-01-01T00:00:00Z", + "name": "Cluster offline", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_cluster" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "labels.health_status": "unavailable" + } + }, + "ignore_time_filter": true, + "time_field": "updated", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "labels.health_status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(labels.health_status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 Cluster offline", + "message": "{{range .results}}\n{{$cid := index .group_values 0 }}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" $cid }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT $cid}}\nCluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Status: offline, Count: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1020,7 +1162,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cb34sfl6psfiq "id": "builtin-cb34sfl6psfiqtovhpt4", "created": "2022-07-07T03:08:46.297166036Z", "updated": "2023-08-09T22:38:41.764325087+08:00", - "name": "Too Many Deleted Documents", + "name": "Too Many Deleted Documents (Only Index>31GB)", "enabled": true, "resource": { "resource_id": "$[[SETUP_RESOURCE_ID]]", @@ -1031,12 +1173,16 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cb34sfl6psfiq ], "filter": {}, "raw_filter": { - "range": { - "payload.elasticsearch.cluster_stats.indices.store.size_in_bytes": { - "gte": 32212254720 - } - } - }, + "bool": { + "must": [ + { "term": { "metadata.name": { "value": "index_stats" } } }, + { "range": { "payload.elasticsearch.index_stats.primaries.store.size_in_bytes": { "gte": 32212254720 } } } + ], + "must_not": [ + { "term": { "metadata.labels.index_name": { "value": "_all" } } } + ] + } +}, "time_field": "timestamp", "context": { "fields": null @@ -1104,7 +1250,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cb34sfl6psfiq }, "notification_config": { "enabled": true, - "title": "Too Many Deleted Documents (>30%)", + "title": "🔥 Too Many Deleted Documents (>30%)", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Deleted: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { @@ -1151,8 +1297,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cb34sfl6psfiq "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1292,7 +1438,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp2e4ianisjm }, "notification_config": { "enabled": true, - "title": "Search latency is great than 500ms", + "title": "🔥 Search latency is great than 500ms", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Latency: {{.result_value | to_fixed 2}}ms\n{{end}}", "normal": [ { @@ -1341,8 +1487,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp2e4ianisjm "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1476,7 +1622,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calaqnh7h710d }, "notification_config": { "enabled": true, - "title": "JVM Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "title": "{{if gt .total_results (len .results)}}🔥 JVM Usage of Nodes ({{.total_results}} nodes in total, showing top {{len .results}}) >= {{.first_threshold}}%{{else}}🔥 JVM Usage of Nodes ({{.total_results}} nodes in total) >= {{.first_threshold}}%{{end}}", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), JVM Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { @@ -1525,8 +1671,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calaqnh7h710d "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1660,7 +1806,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calakp97h710d }, "notification_config": { "enabled": true, - "title": "CPU Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "title": "{{if gt .total_results (len .results)}}🔥 CPU Usage of Nodes ({{.total_results}} nodes in total, showing top {{len .results}}) >= {{.first_threshold}}%{{else}}🔥 CPU Usage of Nodes ({{.total_results}} nodes in total) >= {{.first_threshold}}%{{end}}", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), CPU Usage: {{.result_value | to_fixed 2}}%\n{{end}}", "normal": [ { @@ -1709,8 +1855,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calakp97h710d "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -1815,7 +1961,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calgapp7h710d }, "notification_config": { "enabled": true, - "title": "Shard Storage >55GB in ({{len .results}} indices in total)", + "title": "{{if gt .total_results (len .results)}}🔥 Shard Storage >55GB in ({{.total_results}} indices in total, showing top {{len .results}}){{else}}🔥 Shard Storage >55GB in ({{.total_results}} indices in total){{end}}", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nIndex: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Max Shard Storage: {{.result_value | format_bytes 2}}\n{{end}}", "normal": [ { @@ -1864,8 +2010,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calgapp7h710d "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at | datetime}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}", + "title": "🌈 {{.rule_name}} Resolved", + "message": "- EventID: {{.event_id}}\n- Target: {{.resource_name}}-{{.objects}}\n- TriggerAt: {{.trigger_at | datetime}}\n- ResolveAt: {{.timestamp | datetime}}\n- Duration: {{.duration}}{{if .recovery_context}}\n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -2004,7 +2150,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710d }, "notification_config": { "enabled": true, - "title": "Disk Usage of Nodes ({{len .results}} nodes in total) >= {{.first_threshold}}%", + "title": "{{if gt .total_results (len .results)}}🔥 Disk Usage of Nodes ({{.total_results}} nodes in total, showing top {{len .results}}) >= {{.first_threshold}}%{{else}}🔥 Disk Usage of Nodes ({{.total_results}} nodes in total) >= {{.first_threshold}}%{{end}}", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=N/A\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nNode: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) of Cluster: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), Usage: {{.result_value | to_fixed 2}}% / Free: {{.relation_values.b | format_bytes 2}}\n{{end}}", "normal": [ { @@ -2052,7 +2198,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710d "recovery_notification_config": { "enabled": true, "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", @@ -2168,7 +2314,7 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cujivv5ath26d }, "notification_config": { "enabled": true, - "title": "🔥 [{{.rule_name}}] Alerting", + "title": "🔥 {{.rule_name}} Alerting", "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=N/A\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\nCluster [[{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D)] ({{index .group_values 1}}) metrics has dropped at {{.issue_timestamp | datetime}};\n{{end}}", "normal": [ { @@ -2205,8 +2351,8 @@ POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cujivv5ath26d "category": "Platform", "recovery_notification_config": { "enabled": true, - "title": "🌈 [{{.rule_name}}] Resolved", - "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}} ", + "title": "🌈 {{.rule_name}} Resolved", + "message": "EventID: {{.event_id}} \nTarget: {{.resource_name}}-{{.objects}} \nTriggerAt: {{.trigger_at | datetime}} \nResolveAt: {{.timestamp | datetime}} \nDuration: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", "normal": [ { "id": "cj8bq8d3q95ogankugqg", diff --git a/config/setup/common/alerting.zh-CN.tpl b/config/setup/common/alerting.zh-CN.tpl new file mode 100644 index 00000000..6be45798 --- /dev/null +++ b/config/setup/common/alerting.zh-CN.tpl @@ -0,0 +1,2391 @@ +PUT $[[SETUP_INDEX_PREFIX]]alert-rule +{ + "mappings": { + "properties": { + "bucket_conditions": { + "properties": { + "items": { + "properties": { + "bucket_count": { + "type": "long" + }, + "minimum_period_match": { + "type": "long" + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + }, + "priority": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + }, + "values": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "category": { + "type": "keyword", + "ignore_above": 256 + }, + "conditions": { + "properties": { + "items": { + "properties": { + "minimum_period_match": { + "type": "long" + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + }, + "priority": { + "type": "keyword", + "ignore_above": 256 + }, + "values": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "operator": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "created": { + "type": "date" + }, + "creator": { + "properties": { + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "metrics": { + "properties": { + "bucket_label": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "bucket_size": { + "type": "keyword", + "ignore_above": 256 + }, + "expression": { + "type": "keyword", + "ignore_above": 256 + }, + "format_type": { + "type": "keyword", + "ignore_above": 256 + }, + "formula": { + "type": "keyword", + "ignore_above": 256 + }, + "groups": { + "properties": { + "field": { + "type": "keyword", + "ignore_above": 256 + }, + "limit": { + "type": "long" + } + } + }, + "items": { + "properties": { + "field": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "statistic": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "notification_config": { + "properties": { + "accept_time_range": { + "properties": { + "end": { + "type": "keyword", + "ignore_above": 256 + }, + "start": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "enabled": { + "type": "boolean" + }, + "escalation_throttle_period": { + "type": "keyword", + "ignore_above": 256 + }, + "message": { + "type": "keyword", + "ignore_above": 256 + }, + "normal": { + "properties": { + "created": { + "type": "date" + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword", + "ignore_above": 256 + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "sub_type": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + }, + "updated": { + "type": "date" + }, + "webhook": { + "properties": { + "body": { + "type": "keyword", + "ignore_above": 256 + }, + "header_params": { + "properties": { + "Content-Type": { + "type": "keyword", + "ignore_above": 256 + }, + "Content-type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "method": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "throttle_period": { + "type": "keyword", + "ignore_above": 256 + }, + "title": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "recovery_notification_config": { + "properties": { + "accept_time_range": { + "properties": { + "end": { + "type": "keyword", + "ignore_above": 256 + }, + "start": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "enabled": { + "type": "boolean" + }, + "event_enabled": { + "type": "boolean" + }, + "incremental_recovery_enabled": { + "type": "boolean" + }, + "message": { + "type": "keyword", + "ignore_above": 256 + }, + "normal": { + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "title": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "resource": { + "properties": { + "context": { + "type": "object" + }, + "filter": { + "type": "object", + "enabled": false + }, + "objects": { + "type": "keyword", + "ignore_above": 256 + }, + "raw_filter": { + "type": "object", + "enabled": false + }, + "resource_id": { + "type": "keyword", + "ignore_above": 256 + }, + "resource_name": { + "type": "keyword", + "ignore_above": 256 + }, + "time_field": { + "type": "keyword", + "ignore_above": 256 + }, + "type": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "schedule": { + "properties": { + "interval": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "updated": { + "type": "date" + } + } + } +} + +#alerting channel +#The `id` value is consistent with the `_id` value +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgnb2nt3q95nmusjl65g +{ + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-09T22:39:50.494915568+08:00", + "name": "[告警] Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生！*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*集群:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}|{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8bq8d3q95ogankugqg +{ + "id": "cj8bq8d3q95ogankugqg", + "created": "2023-08-07T17:45:05.534408059+08:00", + "updated": "2023-08-09T22:39:56.489567891+08:00", + "name": "[恢复] Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.SLACK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*恢复时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*目标:* {{.resource_name}}-{{.objects}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.trigger_at | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*持续时长:* {{.duration}}\"\n }\n },\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n }\n ]\n}" + }, + "sub_type": "slack", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgnb2kt3q95nmusjl64g +{ + "id": "cgnb2kt3q95nmusjl64g", + "created": "2023-04-06T11:47:31.161587662Z", + "updated": "2023-08-09T22:39:51.540172306+08:00", + "name": "[告警] 企业微信通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI 平台告警 ]**\\n🔥 告警事件 [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) 正在发生\\n**{{.title}}**\\n优先级: {{.priority}}\\n\\n事件编号: {{.event_id}}\\n\\n目标: {{.resource_name}}-{{.objects}}\\n\\n触发时间: {{.trigger_at | datetime}}\\n{{.message}}\"\n }\n}" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgiospt3q95q49k3u00g +{ + "id": "cgiospt3q95q49k3u00g", + "created": "2023-03-30T13:28:07.531263747Z", + "updated": "2023-08-09T22:39:52.356059486+08:00", + "name": "[告警] 钉钉通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI 平台告警](https://infinilabs.cn/img/email/alert-header.png)\\n\\n🔥 告警事件 [{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) 正在发生\\n\\n**{{.title}}**\\n\\n优先级: {{.priority}}\\n\\n事件编号: {{.event_id}}\\n\\n目标: {{.resource_name}}-{{.objects}}\\n\\n触发时间: {{.trigger_at | datetime}}\\n\\n---\\n\\n{{.message}}\"\n }\n}" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8ctat3q95l9ebbntlg +{ + "id": "cj8ctat3q95l9ebbntlg", + "created": "2023-08-07T18:59:55.28732241+08:00", + "updated": "2023-08-09T22:39:58.967970184+08:00", + "name": "[恢复] 钉钉通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DINGTALK_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"title\": \"{{.title}}\",\n \"text\": \"![INFINI 平台告警](https://infinilabs.cn/img/email/recovery-header.png)\\n\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [查看告警]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "dingtalk", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8e9gt3q95gsdbb0170 +{ + "id": "cj8e9gt3q95gsdbb0170", + "created": "2023-08-07T20:34:11.998953512+08:00", + "updated": "2023-08-09T22:40:04.665871275+08:00", + "name": "[恢复] 企业微信通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.WECOM_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msgtype\": \"markdown\",\n \"markdown\": {\n \"content\": \"**[ INFINI 平台告警 ]**\\n**{{.title}}**\\n\\n{{.message}}\\n\\n> [查看告警]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n}\n" + }, + "sub_type": "wechat", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cgnb2r53q95nmusjl6vg +{ + "id": "cgnb2r53q95nmusjl6vg", + "created": "2023-04-06T11:47:56.652637309Z", + "updated": "2023-08-10T12:04:08.046781556+08:00", + "name": "[告警] 邮件通知", + "type": "email", + "sub_type": "email", + "email": { + "server_id": "", + "recipients": { + "to": [], + "cc": [] + }, + "subject": "[INFINI 平台告警] 🔥 {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n \n

优先级: {{.priority}}

\n

事件编号: {{.event_id}}

\n

目标: {{.resource_name}}-{{.objects}}

\n

触发时间: {{.trigger_at | datetime}}

\n {{.message | md_to_html}}\n
\n

\n \n \n 查看明细\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8atf53q95lhahebg8g +{ + "id": "cj8atf53q95lhahebg8g", + "created": "2023-08-07T16:43:40.062389175+08:00", + "updated": "2023-08-10T12:04:42.842628127+08:00", + "name": "[恢复] 邮件通知", + "type": "email", + "sub_type": "email", + "email": { + "server_id": "", + "recipients": { + "to": [], + "cc": [] + }, + "subject": "[INFINI 平台告警恢复] {{.title}}", + "body": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n \n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n
\n \"email-header\"\n
\n
\n \n \n \n \n \n \n
\n
\n
\n {{.title}}\n

\n {{.message | md_to_html}}\n
\n

\n \n \n 查看明细\n \n

\n \n \n \n

\n \n

\n \"INFINI\n
\n
\n \n
\n

\n \n \n
\n
\n \n \n \n
\n
\n
\n \n
\n \n \n \n \n
\n
\n
\n
\n \n \n \n
\n
\n \n
\n \n \n
\n
\n \n ", + "content_type": "text/html" + }, + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/ch1os6t3q95lk6lepkq0 +{ + "id": "ch1os6t3q95lk6lepkq0", + "created": "2023-04-22T07:34:51.848540351Z", + "updated": "2023-08-10T17:18:38.592432088+08:00", + "name": "[告警] 飞书通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI 平台告警 ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"{{if eq .priority \"critical\"}}red{{else if eq .priority \"high\"}}orange{{else if eq .priority \"medium\"}}yellow{{else if eq .priority \"low\"}}grey{{else}}blue{{end}}\"\n },\n \"elements\": [{\n \"tag\": \"markdown\",\n \"content\": \"🔥 告警事件 [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) 正在发生\\n **{{.title}}**\\n优先级: {{.priority}}\\n事件编号: {{.event_id}}\\n目标: {{.resource_name}}-{{.objects}}\\n触发时间: {{.trigger_at | datetime}}\"\n },{\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n }\n ]\n}\n}" + }, + "sub_type": "feishu", + "enabled": false + } +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj8e9s53q95gsdbb054g +{ + "id": "cj8e9s53q95gsdbb054g", + "created": "2023-08-07T20:34:56.334695598+08:00", + "updated": "2023-08-10T17:18:36.035896482+08:00", + "name": "[恢复] 飞书通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.FEISHU_WEBHOOK_ENDPOINT}}", + "body": "{\n \"msg_type\": \"interactive\",\n \"card\": {\n \"header\": {\n \"title\": {\n \"content\": \"[ INFINI 平台告警 ]\",\n \"tag\": \"plain_text\"\n },\n \"template\":\"green\"\n },\n \"elements\": [\n {\n \"tag\": \"markdown\",\n \"content\": \"**{{.title}}**\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"\n },\n {\n \"tag\": \"hr\"\n },\n {\n \"tag\": \"markdown\",\n \"content\": \"[查看告警]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n }\n ]\n }\n}" + }, + "sub_type": "feishu", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj865st3q95rega919ig +{ + "id": "cj865st3q95rega919ig", + "created": "2023-08-07T11:20:19.223545026+08:00", + "updated": "2023-08-10T17:18:41.92016786+08:00", + "name": "[告警] Discord 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\"content\": \"**[ INFINI 平台告警 ]**\\n🔥 告警事件 [#{{.event_id}}]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}) 正在发生\\n**{{.title}}**\\n\\n优先级: {{.priority}}\\n事件编号: {{.event_id}}\\n目标: {{.resource_name}}-{{.objects}}\\n触发时间: {{.trigger_at | datetime}}\\n{{ .message | str_replace \"\\n\" \"\\\\n\" }}\"}" + }, + "sub_type": "discord", + "enabled": false +} +POST $[[SETUP_INDEX_PREFIX]]channel/$[[SETUP_DOC_TYPE]]/cj86l0l3q95rrpfea6ug +{ + "id": "cj86l0l3q95rrpfea6ug", + "created": "2023-08-07T11:52:34.192522006+08:00", + "updated": "2023-08-10T17:18:44.422687739+08:00", + "name": "[恢复] Discord 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "url": "{{$.env.DISCORD_WEBHOOK_ENDPOINT}}", + "body": "{\n \"content\": \"**[ INFINI 平台告警 ]**\\n🌈 **{{.title}}**\\n\\n{{.message | str_replace \"\\n\" \"\\\\n\" }}\\n> [查看告警]({{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}})\"\n}" + }, + "sub_type": "discord", + "enabled": false +} + +#alerting +#The `id` value is consistent with the `_id` value +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710dpnoaps0 +{ + "id": "builtin-cal8n7p7h710dpnoaps0", + "created": "2022-06-16T01:47:11.326727124Z", + "updated": "2023-08-09T22:39:43.98598502+08:00", + "name": "集群健康状态变为红色", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "payload.elasticsearch.cluster_health.status": "red" + } + }, + { + "term": { + "metadata.name": { + "value": "cluster_health" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.cluster_health.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(payload.elasticsearch.cluster_health.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 集群健康状态变为红色（共 {{.total_results}} 个集群，当前展示前 {{len .results}} 个）{{else}}🔥 集群健康状态变为红色（共 {{.total_results}} 个集群）{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) 当前为红色\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:02:17.165625799+08:00", + "name": "[告警] Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-Type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*Cluster:* <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{ index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}> 当前为红色\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calavvp7h710dpnp32r3 +{ + "id": "builtin-calavvp7h710dpnp32r3", + "created": "2022-06-16T04:22:23.001354546Z", + "updated": "2023-08-09T22:20:17.864619426+08:00", + "name": "索引健康状态变为红色", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics" + ], + "filter": {}, + "raw_filter": {"bool":{"must":[{"term":{"payload.elasticsearch.index_health.status":"red"}},{"term":{"metadata.name":{"value":"index_health"}}}]}}, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 1000 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.index_name", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.index_name)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 索引健康状态变为红色（共 {{.total_results}} 个索引，当前展示前 {{len .results}} 个）{{else}}🔥 索引健康状态变为红色（共 {{.total_results}} 个索引）{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n索引: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D) 当前为红色\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[告警] Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"索引: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0)}}%22} | {{index .group_values 1}}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}> 当前为红色\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp20n2anisjmu4gehc5 +{ + "id": "builtin-cbp20n2anisjmu4gehc5", + "created": "2022-08-09T08:52:44.63345561Z", + "updated": "2023-08-09T22:11:45.679048697+08:00", + "name": "节点离开集群", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_node" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "metadata.labels.status": "unavailable" + } + }, + "ignore_time_filter": true, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.cluster_id", + "limit": 5 + }, + { + "field": "metadata.node_id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "metadata.labels.status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(metadata.labels.status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 节点离开集群", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n节点: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), 离线数: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T10:42:17.686776304+08:00", + "name": "Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"节点: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>, 离线数: {{.result_value}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cm4z6m5fv8l0qk2p9x7r +{ + "id": "builtin-cm4z6m5fv8l0qk2p9x7r", + "created": "2024-01-01T00:00:00Z", + "updated": "2024-01-01T00:00:00Z", + "name": "集群离线", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_cluster" + ], + "filter": {}, + "raw_filter": { + "match_phrase": { + "labels.health_status": "unavailable" + } + }, + "ignore_time_filter": true, + "time_field": "updated", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "id", + "limit": 50 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "labels.health_status", + "statistic": "count" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "count(labels.health_status)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "1" + ], + "priority": "critical" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 集群离线", + "message": "{{range .results}}\n{{$cid := index .group_values 0 }}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" $cid }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT $cid}}\n集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), 状态: 离线, 数量: {{.result_value}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cb34sfl6psfiqtovhpt4 +{ + "id": "builtin-cb34sfl6psfiqtovhpt4", + "created": "2022-07-07T03:08:46.297166036Z", + "updated": "2023-08-09T22:38:41.764325087+08:00", + "name": "已删除文档占比过高(仅索引>31GB)", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { "term": { "metadata.name": { "value": "index_stats" } } }, + { "range": { "payload.elasticsearch.index_stats.primaries.store.size_in_bytes": { "gte": 32212254720 } } } + ], + "must_not": [ + { "term": { "metadata.labels.index_name": { "value": "_all" } } } + ] + } +}, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 20 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "(a/(a+b))*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.primaries.docs.deleted", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.docs.count", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "(max(payload.elasticsearch.index_stats.primaries.docs.deleted)/(max(payload.elasticsearch.index_stats.primaries.docs.deleted)+max(payload.elasticsearch.index_stats.primaries.docs.count)))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "30" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "40" + ], + "priority": "high" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "55" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 已删除文档占比过高 (>30%)", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n索引: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), 已删除占比: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"索引: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0)}}%22} | {{index .group_values 1}}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>, 删除占比: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "24h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cbp2e4ianisjmu4giqs7 +{ + "id": "builtin-cbp2e4ianisjmu4giqs7", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T22:39:15.339913317+08:00", + "name": "搜索延迟高于 500ms", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "index_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ], + "must_not": [ + { + "term": { + "metadata.labels.index_name": { + "value": "_all" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 50 + }, + { + "field": "metadata.labels.index_name", + "limit": 10 + } + ], + "formula": "a/b", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + "statistic": "rate" + }, + { + "name": "b", + "field": "payload.elasticsearch.index_stats.primaries.search.query_total", + "statistic": "rate" + } + ], + "format_type": "num", + "bucket_label": { + "enabled": false + }, + "expression": "rate(payload.elasticsearch.index_stats.total.search.query_time_in_millis)/rate(payload.elasticsearch.index_stats.primaries.search.query_total)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "500" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "300" + ], + "priority": "low" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 搜索延迟高于 500ms", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n索引: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), 延迟: {{.result_value | to_fixed 2}}ms\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "\n{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"索引: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0)}}%22} | {{index .group_values 1}}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>, 搜索延迟: {{.result_value | to_fixed 2}}ms\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calaqnh7h710dpnp2bm8 +{ + "id": "builtin-calaqnh7h710dpnp2bm8", + "created": "2022-06-16T04:11:10.242061032Z", + "updated": "2023-08-09T22:38:55.677122718+08:00", + "name": "JVM 使用率过高", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_percent", + "statistic": "p90" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "p90(payload.elasticsearch.node_stats.jvm.mem.heap_used_percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 节点 JVM 使用率过高（共 {{.total_results}} 个节点，当前展示前 {{len .results}} 个）>= {{.first_threshold}}%{{else}}🔥 节点 JVM 使用率过高（共 {{.total_results}} 个节点）>= {{.first_threshold}}%{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n节点: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), JVM 使用率: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-06T15:46:34.404507399+08:00", + "name": "Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"节点: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>, JVM 使用率: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calakp97h710dpnp1fa2 +{ + "id": "builtin-calakp97h710dpnp1fa2", + "created": "2022-06-16T03:58:29.437447113Z", + "updated": "2023-08-09T22:33:25.692835454+08:00", + "name": "CPU 使用率过高", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 300 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.process.cpu.percent", + "statistic": "avg" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "avg(payload.elasticsearch.node_stats.process.cpu.percent)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 节点 CPU 使用率过高（共 {{.total_results}} 个节点，当前展示前 {{len .results}} 个）>= {{.first_threshold}}%{{else}}🔥 节点 CPU 使用率过高（共 {{.total_results}} 个节点）>= {{.first_threshold}}%{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n节点: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), CPU 使用率: {{.result_value | to_fixed 2}}%\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T15:17:26.18861218+08:00", + "name": "[告警] Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"节点: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>, CPU 使用率: {{.result_value | to_fixed 2}}%\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-calgapp7h710dpnpbeb6 +{ + "id": "builtin-calgapp7h710dpnpbeb6", + "created": "2022-06-16T10:26:47.360988761Z", + "updated": "2023-08-09T22:37:44.038127695+08:00", + "name": "分片存储 >= 55G", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "range": { + "payload.elasticsearch.index_stats.shard_info.store_in_bytes": { + "gte": 59055800320 + } + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.index_name", + "limit": 500 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.index_stats.shard_info.store_in_bytes", + "statistic": "max" + } + ], + "format_type": "bytes", + "bucket_label": { + "enabled": false + }, + "expression": "max(payload.elasticsearch.index_stats.shard_info.store_in_bytes)" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "gte", + "values": [ + "59055800320" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 分片存储超过 55GB（共 {{.total_results}} 个索引，当前展示前 {{len .results}} 个）{{else}}🔥 分片存储超过 55GB（共 {{.total_results}} 个索引）{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$iu := printf \"%s/#/cluster/monitor/%s/indices/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n索引: [{{index .group_values 1}}]({{$iu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), 最大分片存储: {{.result_value | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "created": "2023-04-06T11:47:43.104108279Z", + "updated": "2023-08-07T14:02:53.734855705+08:00", + "name": "[告警] Slack 消息通知", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"索引: <{{$.env.INFINI_CONSOLE_ENDPOINT}}#/cluster/monitor/{{ index .group_values 0}}/indices/{{ index .group_values 1}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{ lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0)}}%22} | {{index .group_values 1}}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}}?_g={%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>, 最大分片存储: {{.result_value | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "slack", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "1h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "- 事件编号: {{.event_id}}\n- 目标: {{.resource_name}}-{{.objects}}\n- 触发时间: {{.trigger_at | datetime}}\n- 恢复时间: {{.timestamp | datetime}}\n- 持续时长: {{.duration}}{{if .recovery_context}}\n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cal8n7p7h710dpnogps1 +{ + "id": "builtin-cal8n7p7h710dpnogps1", + "created": "2022-06-16T03:11:01.445958361Z", + "updated": "2023-08-10T17:16:34.900352415+08:00", + "name": "磁盘使用率过高", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "term": { + "metadata.name": { + "value": "node_stats" + } + } + }, + { + "term": { + "metadata.category": { + "value": "elasticsearch" + } + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.labels.node_id", + "limit": 200 + } + ], + "formula": "((a-b)/a)*100", + "items": [ + { + "name": "a", + "field": "payload.elasticsearch.node_stats.fs.data.total_in_bytes", + "statistic": "max" + }, + { + "name": "b", + "field": "payload.elasticsearch.node_stats.fs.data.free_in_bytes", + "statistic": "max" + } + ], + "format_type": "ratio", + "bucket_label": { + "enabled": false + }, + "expression": "((max(payload.elasticsearch.node_stats.fs.data.total_in_bytes)-max(payload.elasticsearch.node_stats.fs.data.free_in_bytes))/max(payload.elasticsearch.node_stats.fs.data.total_in_bytes))*100" + }, + "conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "80" + ], + "priority": "low" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "90" + ], + "priority": "medium" + }, + { + "minimum_period_match": 5, + "operator": "gte", + "values": [ + "95" + ], + "priority": "high" + } + ] + }, + "notification_config": { + "enabled": true, + "title": "{{if gt .total_results (len .results)}}🔥 节点磁盘使用率过高（共 {{.total_results}} 个节点，当前展示前 {{len .results}} 个）>= {{.first_threshold}}%{{else}}🔥 节点磁盘使用率过高（共 {{.total_results}} 个节点）>= {{.first_threshold}}%{{end}}", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$nn := lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}\n{{$nu := printf \"%s/#/cluster/monitor/%s/nodes/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0) (index .group_values 1)}}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n节点: [{{$nn}}]({{$nu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%2C%22cluster_name%22:%22{{$cn | urlquery}}%22%2C%22node_name%22:%22{{$nn}}%22%7D) 所属集群: [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D), 使用率: {{.result_value | to_fixed 2}}% / 可用: {{.relation_values.b | format_bytes 2}}\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "name": "", + "type": "webhook", + "webhook": { + "header_params": { + "Content-type": "application/json" + }, + "method": "POST", + "body": "{\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*{{if eq .priority \"critical\"}} :fire: {{else if eq .priority \"error\"}} :rotating_light: {{else}} :warning: {{end}} 告警事件 <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}|#{{.event_id}}> 正在发生*\\n :point_right: *{{.rule_name}} - {{.title}}*\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*触发时间:* {{.timestamp | datetime}}\"\n }\n },\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"*优先级:* {{.priority}}\"\n }\n },\n {\n \"type\": \"divider\"\n }\n ]\n {{if gt (len .results) 0}}\n ,\"attachments\": [\n {{range .results}}\n {\n \"color\": {{if eq .priority \"critical\"}} \"#C91010\" {{else if eq .priority \"error\"}} \"#EB4C21\" {{else}} \"#FFB449\" {{end}},\n \"blocks\": [\n {\n \"type\": \"section\",\n \"text\": {\n \"type\": \"mrkdwn\",\n \"text\": \"节点: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/overview/{{index .group_values 0}}/nodes/{{index .group_values 1}}?_g={%22cluster_name%22:%22{{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}%22%2C%22node_name%22:%22{{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}%22} | {{lookup \"category=metadata, object=node, property=metadata.node_name, default=未知\" (index .group_values 1) }}> 所属集群: <{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/cluster/monitor/elasticsearch/{{index .group_values 0}} | {{lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}>, 磁盘使用率: {{.result_value | to_fixed 2}}%, 可用: {{.relation_values.b | format_bytes 2}}\"\n }\n }\n ]\n },\n {{end}}\n {\n \"blocks\": [\n {\n \"type\": \"divider\"\n },\n {\n \"type\": \"actions\",\n \"elements\": [\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看告警\" \n },\n \"url\": \"{{$.env.INFINI_CONSOLE_ENDPOINT}}/#/alerting/message/{{.event_id}}\"\n },\n {\n \"type\": \"button\",\n \"text\": {\n \"type\": \"plain_text\",\n \"text\": \"查看文档\" \n },\n \"style\": \"primary\",\n \"url\": \"https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cluster.html#disk-based-shard-allocation\"\n }\n ]\n },\n ]\n }\n ]\n {{end}}\n}" + }, + "sub_type": "", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 [{{.rule_name}}] 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} +POST $[[SETUP_INDEX_PREFIX]]alert-rule/$[[SETUP_DOC_TYPE]]/builtin-cujivv5ath26drn6bcl0 +{ + "id": "builtin-cujivv5ath26drn6bcl0", + "created": "2025-02-08T18:20:44.273334+08:00", + "updated": "2025-02-12T16:31:05.672771+08:00", + "name": "集群指标采集异常", + "enabled": true, + "resource": { + "resource_id": "$[[SETUP_RESOURCE_ID]]", + "resource_name": "$[[SETUP_RESOURCE_NAME]]", + "type": "elasticsearch", + "objects": [ + ".infini_metrics*" + ], + "filter": {}, + "raw_filter": { + "bool": { + "must": [ + { + "terms": { + "metadata.name": [ + "cluster_health", + "cluster_stats", + "index_stats", + "node_stats", + "shard_stats" + ] + } + } + ] + } + }, + "time_field": "timestamp", + "context": { + "fields": null + } + }, + "metrics": { + "bucket_size": "1m", + "groups": [ + { + "field": "metadata.labels.cluster_id", + "limit": 5 + }, + { + "field": "metadata.name", + "limit": 5 + } + ], + "formula": "a", + "items": [ + { + "name": "a", + "field": "agent.id", + "statistic": "count" + } + ], + "bucket_label": { + "enabled": false + }, + "expression": "count(agent.id)" + }, + "bucket_conditions": { + "operator": "any", + "items": [ + { + "minimum_period_match": 1, + "operator": "lt", + "values": [ + "0" + ], + "priority": "critical", + "type": "content", + "bucket_count": 10 + } + ] + }, + "notification_config": { + "enabled": true, + "title": "🔥 {{.rule_name}} 告警", + "message": "{{range .results}}\n{{$cn := lookup \"category=metadata, object=cluster, property=name, default=未知\" (index .group_values 0) }}\n{{$cu := printf \"%s/#/cluster/monitor/elasticsearch/%s\" $.env.INFINI_CONSOLE_ENDPOINT (index .group_values 0)}}\n集群 [{{$cn}}]({{$cu}}?_g=%7B%22timeRange%22:%7B%22min%22:%22{{$.min}}%22%2C%22max%22:%22{{$.max}}%22%7D%7D) ({{index .group_values 1}}) 指标在 {{.issue_timestamp | datetime}} 发生下降;\n{{end}}", + "normal": [ + { + "id": "cgnb2nt3q95nmusjl65g", + "enabled": true + }, + { + "id": "cgiospt3q95q49k3u00g", + "enabled": true + }, + { + "id": "cj865st3q95rega919ig", + "enabled": true + }, + { + "id": "cgnb2r53q95nmusjl6vg", + "enabled": true + }, + { + "id": "ch1os6t3q95lk6lepkq0", + "enabled": true + }, + { + "id": "cgnb2kt3q95nmusjl64g", + "enabled": true + } + ], + "throttle_period": "6h", + "accept_time_range": { + "start": "00:00", + "end": "23:59" + } + }, + "category": "Platform", + "recovery_notification_config": { + "enabled": true, + "title": "🌈 {{.rule_name}} 已恢复", + "message": "事件编号: {{.event_id}} \n目标: {{.resource_name}}-{{.objects}} \n触发时间: {{.trigger_at | datetime}} \n恢复时间: {{.timestamp | datetime}} \n持续时长: {{.duration}}{{if .recovery_context}} \n{{.recovery_context}}{{end}}", + "normal": [ + { + "id": "cj8bq8d3q95ogankugqg", + "enabled": true + }, + { + "id": "cj8ctat3q95l9ebbntlg", + "enabled": true + }, + { + "id": "cj8atf53q95lhahebg8g", + "enabled": true + }, + { + "id": "cj8e9s53q95gsdbb054g", + "enabled": true + }, + { + "id": "cj8e9gt3q95gsdbb0170", + "enabled": true + }, + { + "id": "cj86l0l3q95rrpfea6ug", + "enabled": true + } + ], + "event_enabled": true + }, + "schedule": { + "interval": "1m" + }, + "creator": { + "name": "$[[SETUP_USERNAME]]", + "id": "$[[SETUP_USER_ID]]" + } +} \ No newline at end of file diff --git a/config/setup/common/data/agent_relay_gateway_config.dat b/config/setup/common/data/agent_relay_gateway_config.dat deleted file mode 100644 index 048e8863..00000000 --- a/config/setup/common/data/agent_relay_gateway_config.dat +++ /dev/null @@ -1,123 +0,0 @@ -path.data: data -path.logs: log - -allow_multi_instance: true -configs.auto_reload: false - -entry: - - name: my_es_entry - enabled: true - router: my_router - max_concurrency: 200000 - network: - binding: 0.0.0.0:8081 -# tls: #for mTLS connection with config servers -# enabled: true -# ca_file: /xxx/ca.crt -# cert_file: /xxx/server.crt -# key_file: /xxx/server.key -# skip_insecure_verify: false - -flow: - - name: deny_flow - filter: - - set_response: - body: "request not allowed" - status: 500 - - name: ingest_flow - filter: - - basic_auth: - valid_users: - ingest: password - - request_body_json_del: - ignore_missing: true - path: - - payload.instance.pool.objects - - payload.host.network_sockets.tcp - - payload.host.network_sockets.udp - - payload.elasticsearch.index_routing_table - - payload.elasticsearch.cluster_stats.indices.mappings - - payload.elasticsearch.cluster_stats.nodes.plugins - - payload.elasticsearch.node_stats.rollup - - payload.elasticsearch.node_stats.http.routes - - payload.elasticsearch.node_stats.attributes - - payload.elasticsearch.node_stats.discovery - - payload.elasticsearch.node_stats.script - - payload.elasticsearch.node_stats.repositories - - payload.elasticsearch.node_stats.script_cache - - payload.elasticsearch.node_stats.http.clients - - payload.elasticsearch.node_stats.indices.translog - - payload.elasticsearch.node_stats.indices.warmer - - payload.elasticsearch.node_stats.indices.segments.file_sizes - - payload.elasticsearch.node_stats.transport.actions - - payload.elasticsearch.node_stats.discovery.cluster_applier_stats - - payload.elasticsearch.shard_stats.bulk - - payload.elasticsearch.shard_stats.commit - - payload.elasticsearch.shard_stats.retention_leases - - payload.elasticsearch.shard_stats.seq_no - - payload.elasticsearch.shard_stats.shard_path - - payload.elasticsearch.shard_stats.sparse_vector - - payload.elasticsearch.shard_stats.dense_vector - - payload.elasticsearch.shard_stats.translog - - payload.elasticsearch.shard_stats.warmer - - payload.elasticsearch.shard_stats.indices.segments.file_sizes - - rewrite_to_bulk: - type_removed: false - - bulk_request_mutate: - fix_null_id: true - generate_enhanced_id: true -# fix_null_type: true -# default_type: m-type -# default_index: m-index - index_rename: - metrics: "$[[SETUP_INDEX_PREFIX]]metrics" - logs: "$[[SETUP_INDEX_PREFIX]]logs" - - bulk_reshuffle: - when: - contains: - _ctx.request.path: /_bulk - elasticsearch: prod - level: node - partition_size: 1 - fix_null_id: true - -router: - - name: my_router - default_flow: deny_flow - rules: - - method: - - "POST" - enabled: true - pattern: - - "/{any_index}/_doc/" - flow: - - ingest_flow - -elasticsearch: - - name: prod - enabled: true - basic_auth: - username: $[[SETUP_AGENT_USERNAME]] - password: $[[SETUP_AGENT_PASSWORD]] - endpoints: $[[SETUP_ENDPOINTS]] - -pipeline: - - name: bulk_request_ingest - auto_start: true - keep_running: true - retry_delay_in_ms: 1000 - processor: - - bulk_indexing: - max_connection_per_node: 100 - num_of_slices: 1 - max_worker_size: 30 - idle_timeout_in_seconds: 10 - bulk: - compress: false - batch_size_in_mb: 10 - batch_size_in_docs: 10000 - consumer: - fetch_max_messages: 100 - queue_selector: - labels: - type: bulk_reshuffle \ No newline at end of file diff --git a/config/setup/common/data/gateway_migration.dat b/config/setup/common/data/gateway_migration.dat new file mode 100644 index 00000000..db6d6c5d --- /dev/null +++ b/config/setup/common/data/gateway_migration.dat @@ -0,0 +1,179 @@ +elastic: + enabled: true + availability_check: + enabled: true + interval: 30s + health_check: + enabled: true + interval: 30s + metadata_refresh: + enabled: true + interval: 60s + cluster_settings_check: + enabled: false + interval: 60s + remote_configs: false + +elasticsearch: + - name: gateway_migration_system + enabled: true + monitored: true + endpoints: $[[SETUP_ENDPOINTS]] + discovery: + enabled: false + basic_auth: + username: "$[[SETUP_AGENT_USERNAME]]" + password: "$[[keystore.$[[SETUP_AGENT_PASSWORD_KEY]]]]" + metadata_cache_enabled: false + - name: logging-server + enabled: true + endpoints: $[[SETUP_ENDPOINTS]] + discovery: + enabled: false + basic_auth: + username: "$[[SETUP_AGENT_USERNAME]]" + password: "$[[keystore.$[[SETUP_AGENT_PASSWORD_KEY]]]]" + metadata_cache_enabled: false + +entry: + - name: gateway_migration_entry + enabled: true + router: gateway_migration_router + max_concurrency: 10000 + network: + binding: 0.0.0.0:8082 + reuse_port: true + +router: + - name: gateway_migration_router + default_flow: gateway_migration_default_flow + tracing_flow: gateway_migration_logging_flow + rules: + - method: + - "*" + pattern: + - "/_bulk" + - "/{any_index}/_bulk" + flow: + - gateway_migration_async_bulk_flow + +flow: + - name: gateway_migration_default_flow + filter: + - elasticsearch: + elasticsearch: gateway_migration_system + max_connection_per_node: 1000 + - name: gateway_migration_logging_flow + filter: + - logging: + queue_name: gateway_migration_request_logging + max_request_body_size: 4096 + max_response_body_size: 4096 + - name: gateway_migration_async_bulk_flow + filter: + - bulk_reshuffle: + when: + contains: + _ctx.request.path: /_bulk + elasticsearch: gateway_migration_system + queue_name_prefix: gateway_migration_async_bulk + level: node + partition_size: 10 + continue_metadata_missing: true + fix_null_id: true + - elasticsearch: + elasticsearch: gateway_migration_system + max_connection_per_node: 1000 + +pipeline: + - name: gateway_migration_async_messages_merge + auto_start: true + keep_running: true + processor: + - indexing_merge: + input_queue: "gateway_migration_bulk_result_messages" + elasticsearch: "gateway_migration_system" + index_name: "$[[SETUP_INDEX_PREFIX]]async_bulk_results" + output_queue: + name: "gateway_migration_merged_requests" + label: + tag: "gateway_migration_merged" + worker_size: 1 + bulk_size_in_mb: 10 + - name: gateway_migration_request_logging_merge + auto_start: true + keep_running: true + processor: + - indexing_merge: + input_queue: "gateway_migration_request_logging" + elasticsearch: "gateway_migration_system" + index_name: "$[[SETUP_INDEX_PREFIX]]requests_logging" + output_queue: + name: "gateway_migration_merged_requests" + label: + tag: "gateway_migration_merged" + worker_size: 1 + bulk_size_in_mb: 10 + - name: gateway_migration_ingest_merged_requests + auto_start: true + keep_running: true + processor: + - bulk_indexing: + num_of_slices: 1 + bulk: + compress: false + batch_size_in_mb: 10 + batch_size_in_docs: 500 + invalid_queue: "gateway_migration_invalid_request" + response_handle: + bulk_result_message_queue: "gateway_migration_system_failure_messages" + max_request_body_size: 10240 + max_response_body_size: 10240 + save_success_results: false + max_error_details_count: 5 + consumer: + fetch_max_messages: 100 + queues: + type: indexing_merge + tag: "gateway_migration_merged" + when: + cluster_available: + - "gateway_migration_system" + - name: gateway_migration_async_ingest_bulk_requests + auto_start: true + keep_running: true + retry_delay_in_ms: 1000 + processor: + - bulk_indexing: + max_connection_per_node: 1000 + num_of_slices: 1 + max_worker_size: 200 + idle_timeout_in_seconds: 10 + bulk: + compress: false + batch_size_in_mb: 20 + batch_size_in_docs: 5000 + invalid_queue: "gateway_migration_bulk_invalid_requests" + dead_letter_queue: "gateway_migration_bulk_dead_requests" + response_handle: + bulk_result_message_queue: "gateway_migration_bulk_result_messages" + max_request_body_size: 1024 + max_response_body_size: 1024 + save_success_results: true + max_error_details_count: 5 + retry_rules: + default: true + retry_429: true + retry_4xx: false + denied: + status: [] + keyword: + - illegal_state_exception + consumer: + fetch_max_messages: 100 + eof_retry_delay_in_ms: 500 + queue_selector: + labels: + type: bulk_reshuffle + elasticsearch: gateway_migration_system + level: node diff --git a/config/setup/common/data/gateway_relay.dat b/config/setup/common/data/gateway_relay.dat new file mode 100644 index 00000000..6c161dbe --- /dev/null +++ b/config/setup/common/data/gateway_relay.dat @@ -0,0 +1,119 @@ +entry: + - name: gateway_relay_entry + enabled: true + router: gateway_relay_router + max_concurrency: 200000 + network: + binding: 0.0.0.0:8081 + tls: + enabled: true + cert_file: "config/relay_server.crt" + key_file: "config/relay_server.key" + ca_file: "config/ca.crt" + skip_insecure_verify: false + +flow: + - name: gateway_relay_deny_flow + filter: + - set_response: + body: "request not allowed" + status: 500 + - name: gateway_relay_ingest_flow + filter: + - basic_auth: + valid_users: + "$[[SETUP_AGENT_USERNAME]]": "$[[keystore.$[[SETUP_AGENT_PASSWORD_KEY]]]]" + - request_body_json_del: + ignore_missing: true + path: + - payload.instance.pool.objects + - payload.elasticsearch.cluster_stats.indices.mappings + - payload.elasticsearch.cluster_stats.nodes.plugins + - payload.elasticsearch.shard_stats.bulk + - payload.elasticsearch.shard_stats.commit + - payload.elasticsearch.shard_stats.retention_leases + - payload.elasticsearch.shard_stats.seq_no + - payload.elasticsearch.shard_stats.shard_path + - payload.elasticsearch.shard_stats.sparse_vector + - payload.elasticsearch.shard_stats.dense_vector + - payload.elasticsearch.shard_stats.translog + - payload.elasticsearch.shard_stats.warmer + - payload.elasticsearch.shard_stats.indices.segments.file_sizes + - payload.elasticsearch.node_stats.attributes + - payload.elasticsearch.node_stats.discovery + - payload.elasticsearch.node_stats.script + - payload.elasticsearch.node_stats.repositories + - payload.elasticsearch.node_stats.script_cache + - payload.elasticsearch.node_stats.http.clients + - payload.elasticsearch.node_stats.indices.translog + - payload.elasticsearch.node_stats.indices.warmer + - payload.elasticsearch.node_stats.indices.segments.file_sizes + - payload.elasticsearch.node_stats.transport.actions + - payload.elasticsearch.node_stats.discovery.cluster_applier_stats + - rewrite_to_bulk: + type_removed: false + - bulk_request_mutate: + fix_null_id: true + generate_enhanced_id: true + index_rename: + metrics: "$[[SETUP_INDEX_PREFIX]]metrics" + logs: "$[[SETUP_INDEX_PREFIX]]logs" + - bulk_reshuffle: + when: + contains: + _ctx.request.path: /_bulk + elasticsearch: gateway_relay_system + queue_name_prefix: gateway_relay_async_bulk + level: node + partition_size: $[[SETUP_RELAY_PARTITION_SIZE]] + continue_metadata_missing: true + fix_null_id: true + - elasticsearch: + elasticsearch: gateway_relay_system + max_connection_per_node: 1000 + +router: + - name: gateway_relay_router + default_flow: gateway_relay_deny_flow + rules: + - method: + - "POST" + enabled: true + pattern: + - "/{any_index}/_doc/" + flow: + - gateway_relay_ingest_flow + +elasticsearch: + - name: gateway_relay_system + enabled: true + monitored: true + basic_auth: + username: "$[[SETUP_AGENT_USERNAME]]" + password: "$[[keystore.$[[SETUP_AGENT_PASSWORD_KEY]]]]" + discovery: + enabled: false + metadata_cache_enabled: false + endpoints: $[[SETUP_ENDPOINTS]] + +pipeline: + - name: gateway_relay_bulk_request_ingest + auto_start: true + keep_running: true + processor: + - bulk_indexing: + max_connection_per_node: 100 + num_of_slices: 1 + max_worker_size: 30 + idle_timeout_in_seconds: 10 + bulk: + compress: false + batch_size_in_mb: 10 + batch_size_in_docs: 10000 + consumer: + fetch_max_messages: 100 + queue_selector: + labels: + type: bulk_reshuffle + elasticsearch: gateway_relay_system + level: node diff --git a/config/setup/common/data/system_ingest_config.dat b/config/setup/common/data/system_ingest_config.dat index 440a1f94..50308888 100644 --- a/config/setup/common/data/system_ingest_config.dat +++ b/config/setup/common/data/system_ingest_config.dat @@ -24,6 +24,8 @@ metrics: - load instance: enabled: true + overall: + enabled: true elastic: availability_check: @@ -54,8 +56,8 @@ pipeline: Content-Type: application/json body: $[[message]] basic_auth: - username: '$[[SETUP_AGENT_USERNAME]]' - password: '$[[SETUP_AGENT_PASSWORD]]' + username: "$[[SETUP_AGENT_USERNAME]]" + password: "$[[keystore.$[[SETUP_AGENT_PASSWORD_KEY]]]]" # tls: #for mTLS connection with config servers # enabled: true # ca_file: /xxx/ca.crt @@ -64,5 +66,5 @@ pipeline: # skip_insecure_verify: false schema: "$[[SETUP_SCHEME]]" # receiver endpoint, fallback in order - hosts: "$[[SETUP_HOSTS]]" - valid_status_code: [200,201] #panic on other status code \ No newline at end of file + hosts: $[[SETUP_HOSTS]] + valid_status_code: [200,201] #panic on other status code diff --git a/config/setup/common/data/task_config_tpl.dat b/config/setup/common/data/task_config_tpl.dat index ac57a85a..4ff9ab3a 100644 --- a/config/setup/common/data/task_config_tpl.dat +++ b/config/setup/common/data/task_config_tpl.dat @@ -6,9 +6,10 @@ elasticsearch: name: "$[[TASK_ID]]" cluster_uuid: "$[[CLUSTER_UUID]]" enabled: true + monitored: true distribution: "$[[CLUSTER_DISTRIBUTION]]" version: "$[[CLUSTER_VERSION]]" - endpoints: "$[[CLUSTER_ENDPOINT]]" + endpoints: ["$[[CLUSTER_ENDPOINT]]"] discovery: enabled: false basic_auth: @@ -54,4 +55,4 @@ pipeline: logs_path: "$[[NODE_LOGS_PATH]]" queue_name: logs when: - cluster_available: ["$[[TASK_ID]]"] \ No newline at end of file + cluster_available: ["$[[TASK_ID]]"] diff --git a/config/setup/easysearch/template_ilm.tpl b/config/setup/easysearch/template_ilm.tpl index d6a45954..45316558 100644 --- a/config/setup/easysearch/template_ilm.tpl +++ b/config/setup/easysearch/template_ilm.tpl @@ -24,7 +24,8 @@ PUT _template/$[[SETUP_TEMPLATE_NAME]] }, "codec": "ZSTD", "source_reuse": false, - "number_of_shards": "1" + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]" } }, "mappings": { @@ -171,7 +172,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]metrics-rollover }, "codec" : "ZSTD", "source_reuse": true, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "mapping.coerce": false, "mapping.ignore_malformed": true @@ -240,7 +242,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]logs-rollover }, "codec": "ZSTD", "source_reuse": false, - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -299,7 +302,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]requests_logging-rollover }, "codec": "ZSTD", "source_reuse": true, - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -369,7 +373,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]async_bulk_results-rollover }, "codec": "ZSTD", "source_reuse": false, - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -439,7 +444,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]alert-history-rollover }, "codec" : "ZSTD", "source_reuse": false, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -606,7 +612,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]activities-rollover }, "codec" : "ZSTD", "source_reuse": false, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -711,7 +718,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]audit-logs-rollover }, "codec" : "ZSTD", "source_reuse": false, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { diff --git a/config/setup/easysearch/template_ilm_1_12_1.tpl b/config/setup/easysearch/template_ilm_1_12_1.tpl index 37d40a36..9001532e 100644 --- a/config/setup/easysearch/template_ilm_1_12_1.tpl +++ b/config/setup/easysearch/template_ilm_1_12_1.tpl @@ -24,7 +24,8 @@ PUT _template/$[[SETUP_TEMPLATE_NAME]] }, "codec": "ZSTD", "source_reuse": false, - "number_of_shards": "1" + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]" } }, "mappings": { @@ -110,7 +111,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]metrics-rollover }, "codec" : "ZSTD", "source_reuse": false, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "mapping.coerce": false, "mapping.ignore_malformed": true @@ -197,7 +199,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]logs-rollover }, "codec": "ZSTD", "source_reuse": false, - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -256,7 +259,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]requests_logging-rollover }, "codec": "ZSTD", "source_reuse": true, - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -326,7 +330,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]async_bulk_results-rollover }, "codec": "ZSTD", "source_reuse": false, - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -396,7 +401,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]alert-history-rollover }, "codec" : "ZSTD", "source_reuse": false, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -563,7 +569,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]activities-rollover }, "codec" : "ZSTD", "source_reuse": false, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -668,7 +675,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]audit-logs-rollover }, "codec" : "ZSTD", "source_reuse": false, - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { diff --git a/config/setup/easysearch/template_rollup.tpl b/config/setup/easysearch/template_rollup.tpl index 06133f73..6d6d99d9 100644 --- a/config/setup/easysearch/template_rollup.tpl +++ b/config/setup/easysearch/template_rollup.tpl @@ -6,7 +6,7 @@ PUT /_rollup/jobs/rollup_index_stats "target_index": "rollup_index_stats_{{ctx.source_index}}", "timestamp": "timestamp", "continuous": true, - "page_size": 600, + "page_size": 100, "cron": "*/5 0-23 * * *", "timezone": "UTC", "stats": [ @@ -51,7 +51,7 @@ PUT /_rollup/jobs/rollup_index_health "target_index": "rollup_index_health_{{ctx.source_index}}", "timestamp": "timestamp", "continuous": true, - "page_size": 600, + "page_size": 100, "cron": "*/5 0-23 * * *", "timezone": "UTC", "stats": [ @@ -90,7 +90,7 @@ PUT /_rollup/jobs/rollup_cluster_stats "rollup": { "source_index": "$[[SETUP_INDEX_PREFIX]]metrics", "target_index": "rollup_cluster_stats_{{ctx.source_index}}", - "page_size": 600, + "page_size": 100, "continuous": true, "cron": "*/5 0-23 * * *", "timezone": "UTC", @@ -132,7 +132,7 @@ PUT /_rollup/jobs/rollup_cluster_health "source_index": "$[[SETUP_INDEX_PREFIX]]metrics", "target_index": "rollup_cluster_health_{{ctx.source_index}}", "continuous": true, - "page_size": 600, + "page_size": 100, "cron": "*/5 0-23 * * *", "timezone": "UTC", "stats": [ @@ -173,7 +173,7 @@ PUT /_rollup/jobs/rollup_node_stats "target_index": "rollup_node_stats_{{ctx.source_index}}", "timestamp": "timestamp", "continuous": true, - "page_size": 600, + "page_size": 100, "cron": "*/5 0-23 * * *", "timezone": "UTC", "stats": [ @@ -256,7 +256,7 @@ PUT /_rollup/jobs/rollup_shard_stats_metrics "target_index": "rollup_shard_stats_metrics_{{ctx.source_index}}", "timestamp": "timestamp", "continuous": true, - "page_size": 600, + "page_size": 100, "cron": "*/5 0-23 * * *", "timezone": "UTC", "stats": [ @@ -302,7 +302,7 @@ PUT /_rollup/jobs/rollup_shard_stats_state "target_index": "rollup_shard_stats_state_{{ctx.source_index}}", "timestamp": "timestamp", "continuous": true, - "page_size": 600, + "page_size": 100, "cron": "*/5 0-23 * * *", "timezone": "UTC", "stats": [ @@ -375,7 +375,8 @@ PUT _template/rollup_policy_template "order": 1, "index_patterns": ["rollup*"], "settings": { - "index.lifecycle.name": "ilm_$[[SETUP_INDEX_PREFIX]]rollup-30days-retention" + "index.lifecycle.name": "ilm_$[[SETUP_INDEX_PREFIX]]rollup-30days-retention", + "index.auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]" } } diff --git a/config/setup/elasticsearch/template_ilm.tpl b/config/setup/elasticsearch/template_ilm.tpl index ea7069a8..87ffbe84 100644 --- a/config/setup/elasticsearch/template_ilm.tpl +++ b/config/setup/elasticsearch/template_ilm.tpl @@ -23,7 +23,8 @@ PUT _template/$[[SETUP_TEMPLATE_NAME]] } } }, - "number_of_shards": "1" + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]" } }, "mappings": { @@ -105,7 +106,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]metrics-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]metrics" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "mapping.coerce": false, "mapping.ignore_malformed": true @@ -173,7 +175,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]logs-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]logs" }, "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -231,7 +234,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]requests_logging-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]requests_logging" }, "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -300,7 +304,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]async_bulk_results-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]async_bulk_results" }, "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -369,7 +374,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]alert-history-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]alert-history" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -534,7 +540,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]activities-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]activities" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -638,7 +645,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]audit-logs-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]audit-logs" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async" } }, diff --git a/config/setup/elasticsearch/v5/template_ilm.tpl b/config/setup/elasticsearch/v5/template_ilm.tpl index 8e2377b7..0f1d335b 100644 --- a/config/setup/elasticsearch/v5/template_ilm.tpl +++ b/config/setup/elasticsearch/v5/template_ilm.tpl @@ -21,7 +21,8 @@ PUT _template/$[[SETUP_TEMPLATE_NAME]] } } }, - "number_of_shards": "1" + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]" } }, "mappings": { @@ -59,7 +60,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]metrics-rollover "index" : { "format" : "7", "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async" } }, @@ -121,7 +123,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]logs-rollover "index": { "format": "7", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -174,7 +177,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]requests_logging-rollover "index": { "format": "7", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -238,7 +242,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]async_bulk_results-rollover "index": { "format": "7", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -302,7 +307,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]alert-history-rollover "index" : { "format" : "7", "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -457,7 +463,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]activities-rollover "index" : { "format" : "7", "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -556,7 +563,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]audit-logs-rollover "index" : { "format" : "7", "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { diff --git a/config/setup/elasticsearch/v6/template_ilm.tpl b/config/setup/elasticsearch/v6/template_ilm.tpl index 80e78f1e..d2327632 100644 --- a/config/setup/elasticsearch/v6/template_ilm.tpl +++ b/config/setup/elasticsearch/v6/template_ilm.tpl @@ -23,7 +23,8 @@ PUT _template/$[[SETUP_TEMPLATE_NAME]] } } }, - "number_of_shards": "1" + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]" } }, "mappings": { @@ -94,7 +95,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]metrics-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]metrics" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async" } }, @@ -164,7 +166,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]logs-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]logs" }, "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -224,7 +227,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]requests_logging-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]requests_logging" }, "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -295,7 +299,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]async_bulk_results-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]async_bulk_results" }, "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -366,7 +371,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]alert-history-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]alert-history" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -533,7 +539,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]activities-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]activities" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { @@ -639,7 +646,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]audit-logs-rollover "rollover_alias" : "$[[SETUP_INDEX_PREFIX]]audit-logs" }, "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async", "analysis": { "analyzer": { diff --git a/config/setup/opensearch/template_ilm.tpl b/config/setup/opensearch/template_ilm.tpl index 9c6fa964..16974fa0 100644 --- a/config/setup/opensearch/template_ilm.tpl +++ b/config/setup/opensearch/template_ilm.tpl @@ -23,7 +23,8 @@ PUT _template/$[[SETUP_TEMPLATE_NAME]] } } }, - "number_of_shards": "1" + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]" } }, "mappings": { @@ -117,7 +118,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]metrics-rollover "settings": { "index":{ "codec" : "best_compression", - "number_of_shards" : "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog.durability":"async" }, "plugins.index_state_management.rollover_alias": "$[[SETUP_INDEX_PREFIX]]metrics" @@ -167,7 +169,8 @@ PUT /_template/$[[SETUP_INDEX_PREFIX]]logs-rollover "settings": { "plugins.index_state_management.rollover_alias": "$[[SETUP_INDEX_PREFIX]]logs", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -222,7 +225,8 @@ PUT _template/$[[SETUP_INDEX_PREFIX]]requests_logging-rollover "settings": { "plugins.index_state_management.rollover_alias": "$[[SETUP_INDEX_PREFIX]]requests_logging", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -288,7 +292,8 @@ PUT /_template/$[[SETUP_INDEX_PREFIX]]async_bulk_results-rollover "settings": { "plugins.index_state_management.rollover_alias": "$[[SETUP_INDEX_PREFIX]]async_bulk_results", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" } @@ -353,7 +358,8 @@ PUT /_template/$[[SETUP_INDEX_PREFIX]]alert-history-rollover "settings": { "plugins.index_state_management.rollover_alias": "$[[SETUP_INDEX_PREFIX]]alert-history", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" }, @@ -516,7 +522,8 @@ PUT /_template/$[[SETUP_INDEX_PREFIX]]activities-rollover "settings": { "plugins.index_state_management.rollover_alias": "$[[SETUP_INDEX_PREFIX]]activities", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" }, @@ -620,7 +627,8 @@ PUT /_template/$[[SETUP_INDEX_PREFIX]]audit-logs-rollover "settings": { "plugins.index_state_management.rollover_alias": "$[[SETUP_INDEX_PREFIX]]audit-logs", "codec": "best_compression", - "number_of_shards": "1", + "number_of_shards": "$[[SETUP_PRIMARY_SHARDS]]", + "auto_expand_replicas": "$[[SETUP_AUTO_EXPAND_REPLICAS]]", "translog": { "durability": "async" }, diff --git a/config/system_config.tpl b/config/system_config.tpl index 6b69cb08..96acdcb4 100644 --- a/config/system_config.tpl +++ b/config/system_config.tpl @@ -7,7 +7,7 @@ elasticsearch: enabled: true monitored: true reserved: true - endpoint: $[[CLUSTER_ENDPOINT]] + endpoints: $[[CLUSTER_ENDPOINT]] discovery: enabled: false basic_auth: @@ -113,4 +113,4 @@ pipeline: queues: type: indexing_merge when: - cluster_available: ["$[[CLUSTER_ID]]"] \ No newline at end of file + cluster_available: ["$[[CLUSTER_ID]]"] diff --git a/console.yml b/console.yml index 1b700f01..fbf72394 100644 --- a/console.yml +++ b/console.yml @@ -1,160 +1,206 @@ -#env: -# INFINI_CONSOLE_ENDPOINT: "http://127.0.0.1:9000" -# INGEST_CLUSTER_ENDPOINT: "https://127.0.0.1:9200" -# INGEST_CLUSTER_CREDENTIAL_ID: chjkp9dath21f1ae9tq0 -# SLACK_WEBHOOK_ENDPOINT: -# DISCORD_WEBHOOK_ENDPOINT: -# DINGTALK_WEBHOOK_ENDPOINT: -# WECOM_WEBHOOK_ENDPOINT: -# FEISHU_WEBHOOK_ENDPOINT: - - -# must in major config file -path.configs: "config" -configs: - managed: false - auto_reload: true - manager: - local_configs_repo_path: ./config_repo/ - tls: # for mTLS connection with config servers - enabled: true - ca_file: config/certs/ca.crt - cert_file: config/certs/ca.crt - key_file: config/certs/ca.key - skip_insecure_verify: false -web: - enabled: true - embedding_api: true - # base_path: /console - security: - enabled: true - ui: - enabled: true - path: .public - vfs: true - local: true - network: - binding: 0.0.0.0:9000 - skip_occupied_port: true - gzip: - enabled: true - -elastic: - enabled: true - remote_configs: true - health_check: - enabled: true - interval: 30s - availability_check: - enabled: true - interval: 60s - metadata_refresh: - enabled: true - interval: 30s - cluster_settings_check: - enabled: true - interval: 20s - store: - enabled: false - orm: - enabled: true - init_template: true - template_name: ".infini" - index_prefix: ".infini_" - build_template_for_object: false - # override_exists_template: true - -metrics: - enabled: true - queue: metrics -# event_queue: -# cluster_health: "cluster_metrics" - elasticsearch: - enabled: true - cluster_stats: true - node_stats: true - index_stats: true - -## badger kv storage configuration -badger: - enabled: true - single_bucket_mode: true - path: '' - memory_mode: false - sync_writes: false - mem_table_size: 10485760 - num_mem_tables: 1 - # lsm tuning options - value_log_max_entries: 1000000 - value_log_file_size: 536870912 - value_threshold: 1048576 - num_level0_tables: 1 - num_level0_tables_stall: 2 - -security: - enabled: true -# authc: -# realms: -# ldap: -# test: #setup guide: https://github.com/infinilabs/testing/blob/main/setup/gateway/cases/elasticsearch/elasticsearch-with-ldap.yml -# enabled: true -# host: "localhost" -# port: 3893 -# bind_dn: "cn=serviceuser,ou=svcaccts,dc=glauth,dc=com" -# bind_password: "mysecret" -# base_dn: "dc=glauth,dc=com" -# user_filter: "(cn=%s)" -# group_attribute: "ou" -# bypass_api_key: true -# cache_ttl: "10s" -# default_roles: ["ReadonlyUI","DATA"] #default for all ldap users if no specify roles was defined -# role_mapping: -# group: -# superheros: [ "Administrator" ] -## uid: -## hackers: [ "Administrator" ] -# testing: -# enabled: true -# host: "ldap.forumsys.com" -# port: 389 -# bind_dn: "cn=read-only-admin,dc=example,dc=com" -# bind_password: "password" -# base_dn: "dc=example,dc=com" -# user_filter: "(uid=%s)" -# cache_ttl: "10s" -# default_roles: ["ReadonlyUI","DATA"] #default for all ldap users if no specify roles was defined -# role_mapping: -# uid: -# tesla: [ "readonly","data" ] -# oauth: -# enabled: true -# client_id: "850d747174ace88ce889" -# client_secret: "3d437b64e06371d6f62769320438d3dfc95a8d8e" -## default_roles: ["ReadonlyUI","DATA"] #default for all sso users if no specify roles was defined -# role_mapping: -# medcl: ["Administrator"] -# authorize_url: "https://github.com/login/oauth/authorize" -# token_url: "https://github.com/login/oauth/access_token" -# redirect_url: "" -# scopes: [] - -#agent: -# setup: -# download_url: "https://release.infinilabs.com/agent/stable" -# version: 0.5.0-214 -# ca_cert: "config/certs/ca.crt" -# ca_key: "config/certs/ca.key" -# console_endpoint: $[[env.INFINI_CONSOLE_ENDPOINT]] -# ingest_cluster_endpoint: $[[env.INGEST_CLUSTER_ENDPOINT]] -# ingest_cluster_credential_id: $[[env.INGEST_CLUSTER_CREDENTIAL_ID]] - - -http_client: - default: - tls: - skip_insecure_verify: true - skip_domain_verify: true - proxy: - enabled: true - default_config: - using_proxy_env: true +# env: +# INFINI_CONSOLE_ENDPOINT: "http://127.0.0.1:9000" +# INGEST_CLUSTER_ENDPOINT: "https://127.0.0.1:9200" +# INGEST_CLUSTER_CREDENTIAL_ID: chjkp9dath21f1ae9tq0 +# SLACK_WEBHOOK_ENDPOINT: +# DISCORD_WEBHOOK_ENDPOINT: +# DINGTALK_WEBHOOK_ENDPOINT: +# WECOM_WEBHOOK_ENDPOINT: +# FEISHU_WEBHOOK_ENDPOINT: https://open.feishu.cn/open-apis/bot/v2/hook/ + + +# must in major config file +path.configs: "config" +configs: + managed: false + auto_reload: true + manager: + local_configs_repo_path: ./config_repo/ + tls: # for mTLS connection with config servers + enabled: true + ca_file: config/certs/ca.crt + cert_file: config/certs/ca.crt + key_file: config/certs/ca.key + skip_insecure_verify: false +api: + enabled: false + network: + binding: 0.0.0.0:9443 + skip_occupied_port: false + websocket: + # Agent reverse channel can use /ws on the active Console listener. + # If web.embedding_api or web.websocket is enabled, install scripts prefer the web endpoint. + enabled: true + base_path: /ws + skip_host_verify: true + tls: + # Keep API listener TLS optional so agents can use the embedded /ws on web :9000. + enabled: false + skip_insecure_verify: false + default_domain: localhost +web: + enabled: true + access_log_enabled: false + embedding_api: false + websocket: + enabled: true + base_path: /ws + skip_host_verify: true + tls: + enabled: true + skip_insecure_verify: true + default_domain: "localhost" + # base_path: /console + security: + enabled: true + ui: + enabled: true + # Relative paths are resolved from the Console executable directory. + path: .public + vfs: true + local: true + network: + binding: 0.0.0.0:9000 + skip_occupied_port: true + gzip: + enabled: true + +elastic: + enabled: true + remote_configs: true + health_check: + enabled: true + interval: 30s + availability_check: + enabled: true + interval: 60s + metadata_refresh: + enabled: true + interval: 30s + cluster_settings_check: + enabled: true + interval: 20s + store: + enabled: false + orm: + enabled: true + init_template: true + template_name: ".infini" + index_prefix: ".infini_" + build_template_for_object: false + override_exists_template: true + +metrics: + enabled: true + queue: metrics +# event_queue: +# cluster_health: "cluster_metrics" + elasticsearch: + enabled: true + cluster_stats: true + node_stats: true + index_stats: true + +disk_queue: + cleanup_files_on_init: true + compress: + delete_after_compress: true + +## badger kv storage configuration +badger: + enabled: true + single_bucket_mode: true + path: '' + memory_mode: false + sync_writes: false + mem_table_size: 10485760 + num_mem_tables: 1 + # lsm tuning options + value_log_max_entries: 1000000 + value_log_file_size: 536870912 + value_threshold: 1048576 + num_level0_tables: 1 + num_level0_tables_stall: 2 + +security: + enabled: true +# authc: +# realms: +# ldap: +# test: #setup guide: https://github.com/infinilabs/testing/blob/main/setup/gateway/cases/elasticsearch/elasticsearch-with-ldap.yml +# enabled: true +# host: "localhost" +# port: 3893 +# bind_dn: "cn=serviceuser,ou=svcaccts,dc=glauth,dc=com" +# bind_password: "mysecret" +# base_dn: "dc=glauth,dc=com" +# user_filter: "(cn=%s)" +# group_attribute: "ou" +# bypass_api_key: true +# cache_ttl: "10s" +# default_roles: ["ReadonlyUI","DATA"] #default for all ldap users if no specify roles was defined +# role_mapping: +# group: +# superheros: [ "Administrator" ] +## uid: +## hackers: [ "Administrator" ] +# testing: +# enabled: true +# host: "ldap.forumsys.com" +# port: 389 +# bind_dn: "cn=read-only-admin,dc=example,dc=com" +# bind_password: "password" +# base_dn: "dc=example,dc=com" +# user_filter: "(uid=%s)" +# cache_ttl: "10s" +# default_roles: ["ReadonlyUI","DATA"] #default for all ldap users if no specify roles was defined +# role_mapping: +# uid: +# tesla: [ "readonly","data" ] +# oauth: +# enabled: true +# client_id: "850d747174ace88ce889" +# client_secret: "3d437b64e06371d6f62769320438d3dfc95a8d8e" +## default_roles: ["ReadonlyUI","DATA"] #default for all sso users if no specify roles was defined +# role_mapping: +# medcl: ["Administrator"] +# authorize_url: "https://github.com/login/oauth/authorize" +# token_url: "https://github.com/login/oauth/access_token" +# redirect_url: "" +# scopes: [] + +#agent: +# setup: +# # Optional package mirror. If files exist under web.ui.path/agent/stable, +# # Console automatically uses its own web endpoint as the download source. +# # Otherwise it falls back to the official release site. For custom mirrors, +# # set an explicit URL such as https://mirror.local/agent/stable. +# download_url: "https://release.infinilabs.com/agent/stable" +# # Optional install directory. Default is /infini/agent. +# install_dir: "/infini/agent" +# version: 0.5.0-214 +# ca_cert: "config/certs/ca.crt" +# ca_key: "config/certs/ca.key" +# # Optional when the target host must use a different Console address than the current page. +# console_endpoint: $[[env.INFINI_CONSOLE_ENDPOINT]] +# ingest_cluster_endpoint: $[[env.INGEST_CLUSTER_ENDPOINT]] +# ingest_cluster_credential_id: $[[env.INGEST_CLUSTER_CREDENTIAL_ID]] + +# setting version for console select install agent tpl and download agent version +agent: + setup: + install_dir: /infini/agent + +gateway: + setup: + install_dir: /infini/gateway + +http_client: + default: + tls: + skip_insecure_verify: true + skip_domain_verify: true + proxy: + enabled: false + default_config: + using_proxy_env: true diff --git a/core/auth.go b/core/auth.go index e88d89fe..f1a127bd 100644 --- a/core/auth.go +++ b/core/auth.go @@ -57,7 +57,7 @@ func (handler Handler) RequireLogin(h httprouter.Handle) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if api.IsAuthEnable() { - claims, err := security.ValidateLogin(r.Header.Get("Authorization")) + claims, err := security.ValidateLoginFromRequest(r) if err != nil { handler.WriteError(w, err.Error(), http.StatusUnauthorized) return @@ -77,7 +77,7 @@ func (handler Handler) RequirePermission(h httprouter.Handle, permissions ...str } if api.IsAuthEnable() { - claims, err := security.ValidateLogin(r.Header.Get("Authorization")) + claims, err := security.ValidateLoginFromRequest(r) if err != nil { handler.WriteError(w, err.Error(), http.StatusUnauthorized) return @@ -94,12 +94,32 @@ func (handler Handler) RequirePermission(h httprouter.Handle, permissions ...str } } +func RequestUsesSecureTransport(req *http.Request) bool { + return api.RequestUsesSecureTransport(req, api.SecureTransportOptions{TrustForwardHeaders: true}) +} + +func (handler Handler) RequireSecureTransport(h httprouter.Handle) httprouter.Handle { + return handler.Handler.RequireSecureTransport(h, api.SecureTransportOptions{TrustForwardHeaders: true}) +} + +func RequireSecureTransport(h httprouter.Handle) httprouter.Handle { + return api.RequireSecureTransport(h, api.SecureTransportOptions{TrustForwardHeaders: true}) +} + +func (handler Handler) RequireReplayProtection(h httprouter.Handle) httprouter.Handle { + return handler.Handler.RequireReplayProtection(h) +} + +func RequireReplayProtection(h httprouter.Handle) httprouter.Handle { + return api.RequireReplayProtection(h) +} + func (handler Handler) RequireClusterPermission(h httprouter.Handle, permissions ...string) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if api.IsAuthEnable() { id := ps.ByName("id") - claims, err := security.ValidateLogin(r.Header.Get("Authorization")) + claims, err := security.ValidateLoginFromRequest(r) if err != nil { handler.WriteError(w, err.Error(), http.StatusUnauthorized) return @@ -119,9 +139,9 @@ func (handler Handler) RequireClusterPermission(h httprouter.Handle, permissions func (handler Handler) GetCurrentUser(req *http.Request) string { if api.IsAuthEnable() { - claims, ok := req.Context().Value("user").(*security.UserClaims) - if ok { - return claims.Username + user, err := security.FromUserContext(req.Context()) + if err == nil && user != nil { + return user.Username } } return "" diff --git a/core/auth_test.go b/core/auth_test.go new file mode 100644 index 00000000..5fca2a71 --- /dev/null +++ b/core/auth_test.go @@ -0,0 +1,78 @@ +package core + +import ( + "crypto/tls" + "net/http" + "net/http/httptest" + "testing" + + httprouter "infini.sh/framework/core/api/router" +) + +func TestRequestUsesSecureTransport(t *testing.T) { + tests := []struct { + name string + setup func(req *http.Request) + secure bool + }{ + { + name: "tls request", + setup: func(req *http.Request) { + req.TLS = &tls.ConnectionState{} + }, + secure: true, + }, + { + name: "forwarded proto", + setup: func(req *http.Request) { + req.Header.Set("X-Forwarded-Proto", "https") + }, + secure: true, + }, + { + name: "forwarded header", + setup: func(req *http.Request) { + req.Header.Set("Forwarded", `for=127.0.0.1;proto=https;host=console.local`) + }, + secure: true, + }, + { + name: "plain http", + setup: func(req *http.Request) { + }, + secure: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "http://console.local/account/login", nil) + tt.setup(req) + + if RequestUsesSecureTransport(req) != tt.secure { + t.Fatalf("expected secure=%v", tt.secure) + } + }) + } +} + +func TestRequireSecureTransport(t *testing.T) { + handler := Handler{} + called := false + protected := handler.RequireSecureTransport(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + called = true + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "http://console.local/account/login", nil) + resp := httptest.NewRecorder() + + protected(resp, req, nil) + + if called { + t.Fatal("expected insecure request to be blocked") + } + if resp.Code != http.StatusUpgradeRequired { + t.Fatalf("expected status %d, got %d", http.StatusUpgradeRequired, resp.Code) + } +} diff --git a/core/elastic.go b/core/elastic.go index 39c9cb34..532e10d3 100644 --- a/core/elastic.go +++ b/core/elastic.go @@ -30,13 +30,15 @@ import ( "infini.sh/framework/core/radix" "infini.sh/framework/core/util" "net/http" + "sort" + "strings" ) func (handler Handler) IndexRequired(h httprouter.Handle, route ...string) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if api.IsAuthEnable() { - claims, err := rbac.ValidateLogin(r.Header.Get("Authorization")) + claims, err := rbac.ValidateLoginFromRequest(r) if err != nil { handler.WriteError(w, err.Error(), http.StatusUnauthorized) return @@ -61,7 +63,7 @@ func (handler Handler) ClusterRequired(h httprouter.Handle, route ...string) htt return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if api.IsAuthEnable() { - claims, err := rbac.ValidateLogin(r.Header.Get("Authorization")) + claims, err := rbac.ValidateLoginFromRequest(r) if err != nil { handler.WriteError(w, err.Error(), http.StatusUnauthorized) return @@ -80,6 +82,43 @@ func (handler Handler) ClusterRequired(h httprouter.Handle, route ...string) htt } } +func normalizeClusterIDs(clusterIDs []string) []string { + if len(clusterIDs) == 0 { + return nil + } + + seen := map[string]struct{}{} + normalized := make([]string, 0, len(clusterIDs)) + for _, clusterID := range clusterIDs { + clusterID = strings.TrimSpace(clusterID) + if clusterID == "" { + continue + } + if _, ok := seen[clusterID]; ok { + continue + } + seen[clusterID] = struct{}{} + normalized = append(normalized, clusterID) + } + if len(normalized) == 0 { + return nil + } + sort.Strings(normalized) + return normalized +} + +func buildClusterFilter(field string, clusterIDs []string) util.MapStr { + clusterIDs = normalizeClusterIDs(clusterIDs) + if len(clusterIDs) == 0 { + return nil + } + return util.MapStr{ + "terms": util.MapStr{ + field: clusterIDs, + }, + } +} + func (handler Handler) GetClusterFilter(r *http.Request, field string) (util.MapStr, bool) { if !api.IsAuthEnable() { return nil, true @@ -88,21 +127,18 @@ func (handler Handler) GetClusterFilter(r *http.Request, field string) (util.Map if hasAllPrivilege { return nil, true } + clusterIds = normalizeClusterIDs(clusterIds) if len(clusterIds) == 0 { return nil, false } - return util.MapStr{ - "terms": util.MapStr{ - field: clusterIds, - }, - }, false + return buildClusterFilter(field, clusterIds), false } func (handler Handler) GetAllowedClusters(r *http.Request) ([]string, bool) { if !api.IsAuthEnable() { return nil, true } hasAllPrivilege, clusterIds := rbac.GetCurrentUserCluster(r) - return clusterIds, hasAllPrivilege + return normalizeClusterIDs(clusterIds), hasAllPrivilege } func (handler Handler) GetAllowedIndices(r *http.Request, clusterID string) ([]string, bool) { @@ -134,7 +170,7 @@ func (handler Handler) ValidateProxyRequest(req *http.Request, clusterID string) if !api.IsAuthEnable() { return false, "", nil } - claims, err := rbac.ValidateLogin(req.Header.Get("Authorization")) + claims, err := rbac.ValidateLoginFromRequest(req) if err != nil { return false, "", err } @@ -176,9 +212,9 @@ func (handler Handler) GetCurrentUserIndex(req *http.Request) (bool, map[string] if !api.IsAuthEnable() { return true, nil } - ctxVal := req.Context().Value("user") - if userClaims, ok := ctxVal.(*rbac.UserClaims); ok { - roles := userClaims.Roles + user, err := rbac.FromUserContext(req.Context()) + if err == nil && user != nil { + roles := user.Roles var realIndex = map[string][]string{} for _, roleName := range roles { role, ok := rbac.RoleMap[roleName] @@ -199,10 +235,9 @@ func (handler Handler) GetCurrentUserIndex(req *http.Request) (bool, map[string] } func (handler Handler) GetCurrentUserClusterIndex(req *http.Request, clusterID string) (bool, []string) { - ctxVal := req.Context().Value("user") - if userClaims, ok := ctxVal.(*rbac.UserClaims); ok { - return rbac.GetRoleIndex(userClaims.Roles, clusterID) - } else { - panic("user context value not found") + user, err := rbac.FromUserContext(req.Context()) + if err == nil && user != nil { + return rbac.GetRoleIndex(user.Roles, clusterID) } + return false, nil } diff --git a/core/elastic_test.go b/core/elastic_test.go new file mode 100644 index 00000000..caccc87e --- /dev/null +++ b/core/elastic_test.go @@ -0,0 +1,100 @@ +package core + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "infini.sh/console/core/security" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" +) + +func TestGetClusterFilterReturnsAllPrivilegeForAdministrator(t *testing.T) { + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + originalRole, hadRole := security.RoleMap[security.RoleAdminName] + security.RoleMap[security.RoleAdminName] = security.BuiltinRoles[security.RoleAdminName] + t.Cleanup(func() { + if hadRole { + security.RoleMap[security.RoleAdminName] = originalRole + return + } + delete(security.RoleMap, security.RoleAdminName) + }) + + req := newRequestWithRoles(security.RoleAdminName) + + clusterFilter, hasAllPrivilege := Handler{}.GetClusterFilter(req, "id") + if !hasAllPrivilege { + t.Fatalf("expected administrator to have all cluster privilege") + } + if clusterFilter != nil { + t.Fatalf("expected no cluster filter for administrator, got %#v", clusterFilter) + } +} + +func TestGetClusterFilterNormalizesRestrictedClusterIDs(t *testing.T) { + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + roleName := "cluster-filter-normalization-test" + security.RoleMap[roleName] = security.Role{ + Name: roleName, + Privilege: security.RolePrivilege{ + Elasticsearch: security.ElasticsearchPrivilege{ + Cluster: security.ClusterPrivilege{ + Resources: []security.InnerCluster{ + {ID: " cluster-b "}, + {ID: ""}, + {ID: "cluster-a"}, + {ID: "cluster-a"}, + }, + }, + }, + }, + } + t.Cleanup(func() { + delete(security.RoleMap, roleName) + }) + + req := newRequestWithRoles(roleName) + + clusterFilter, hasAllPrivilege := Handler{}.GetClusterFilter(req, "id") + if hasAllPrivilege { + t.Fatalf("expected restricted role to require cluster filter") + } + expected := util.MapStr{ + "terms": util.MapStr{ + "id": []string{"cluster-a", "cluster-b"}, + }, + } + if !reflect.DeepEqual(clusterFilter, expected) { + t.Fatalf("expected normalized cluster filter %#v, got %#v", expected, clusterFilter) + } +} + +func TestBuildClusterFilterSkipsEmptyClusterIDs(t *testing.T) { + clusterFilter := buildClusterFilter("id", []string{"", " ", ""}) + if clusterFilter != nil { + t.Fatalf("expected empty cluster IDs to produce nil filter, got %#v", clusterFilter) + } +} + +func newRequestWithRoles(roles ...string) *http.Request { + req := httptest.NewRequest(http.MethodGet, "/_test", nil) + return req.WithContext(security.NewUserContext(req.Context(), &security.UserClaims{ + ShortUser: &security.ShortUser{ + Username: "tester", + Roles: roles, + }, + })) +} diff --git a/core/insight/metric_data.go b/core/insight/metric_data.go index 2cc5dcba..3cda0abd 100644 --- a/core/insight/metric_data.go +++ b/core/insight/metric_data.go @@ -87,8 +87,9 @@ type GroupSort struct { } type MetricGroupItem struct { - Field string `json:"field"` - Limit int `json:"limit"` + Field string `json:"field"` + Limit int `json:"limit"` + Missing string `json:"missing,omitempty"` } func (m *Metric) GenerateExpression() (string, error) { diff --git a/core/insight/metric_util.go b/core/insight/metric_util.go index b6aead76..29ac5b55 100644 --- a/core/insight/metric_util.go +++ b/core/insight/metric_util.go @@ -35,7 +35,6 @@ import ( log "github.com/cihub/seelog" "infini.sh/console/core/insight/function" - "infini.sh/framework/core/elastic" "infini.sh/framework/core/util" ) @@ -196,11 +195,15 @@ func GenerateQuery(metric *Metric) (interface{}, error) { ) if metric.BucketSize != "" && metric.TimeField != "" { useDateHistogram = true - if metric.BucketSize == "auto" { + bucketSize := strings.TrimSpace(metric.BucketSize) + if bucketSize == "" { + bucketSize = "auto" + } + if strings.EqualFold(bucketSize, "auto") { dateHistogramAggName = "auto_date_histogram" buckets := metric.Buckets if buckets == 0 { - buckets = 2 + buckets = 30 } dateHistogramAgg = util.MapStr{ "field": metric.TimeField, @@ -208,19 +211,10 @@ func GenerateQuery(metric *Metric) (interface{}, error) { } } else { dateHistogramAggName = "date_histogram" - verInfo := elastic.GetClient(metric.ClusterId).GetVersion() - - if verInfo.Number == "" { - panic("invalid version") - } - - intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, metric.BucketSize) - if err != nil { - return nil, fmt.Errorf("get interval field error: %w", err) - } dateHistogramAgg = util.MapStr{ - "field": metric.TimeField, - intervalField: metric.BucketSize, + "field": metric.TimeField, + "interval": bucketSize, + "min_doc_count": 0, } } } @@ -254,6 +248,9 @@ func GenerateQuery(metric *Metric) (interface{}, error) { "field": groups[i].Field, "size": limit, } + if groups[i].Missing != "" { + termsCfg["missing"] = groups[i].Missing + } if i == grpLength-1 && len(metric.Sort) > 0 { //use bucket sort instead of terms order when time after group if metric.UseBucketSort() && len(metric.Sort) > 0 { @@ -610,6 +607,7 @@ func MergeGroupValues(metricData []MetricData) []MetricData { return metricData } grpMd := map[string]MetricData{} + groupOrder := make([]string, 0, len(metricData)) for _, md := range metricData { if len(md.Groups) == 0 { continue @@ -629,8 +627,10 @@ func MergeGroupValues(metricData []MetricData) []MetricData { existingMd.Data[k] = v } } + grpMd[groupKey] = existingMd } else { grpMd[groupKey] = md + groupOrder = append(groupOrder, groupKey) } } // sort the merged metric data by timestamp @@ -648,8 +648,10 @@ func MergeGroupValues(metricData []MetricData) []MetricData { } // Convert map to slice mergedMetricData := make([]MetricData, 0, len(grpMd)) - for _, md := range grpMd { - mergedMetricData = append(mergedMetricData, md) + for _, groupKey := range groupOrder { + if md, ok := grpMd[groupKey]; ok { + mergedMetricData = append(mergedMetricData, md) + } } return mergedMetricData } diff --git a/core/insight/metric_util_test.go b/core/insight/metric_util_test.go index 5c3402dc..686f8249 100644 --- a/core/insight/metric_util_test.go +++ b/core/insight/metric_util_test.go @@ -374,3 +374,37 @@ func TestCollectMetricDataWithPercentage(t *testing.T) { } } + +func TestMergeGroupValuesPreservesFirstSeenGroupOrder(t *testing.T) { + metricData := []MetricData{ + { + Groups: []MetricDataGroup{{Value: "node_stats"}}, + Data: map[string][]MetricDataItem{ + "a": {{Timestamp: float64(2000), Value: float64(2)}}, + }, + }, + { + Groups: []MetricDataGroup{{Value: "cluster_health"}}, + Data: map[string][]MetricDataItem{ + "a": {{Timestamp: float64(2000), Value: float64(8)}}, + }, + }, + { + Groups: []MetricDataGroup{{Value: "node_stats"}}, + Data: map[string][]MetricDataItem{ + "a": {{Timestamp: float64(1000), Value: float64(1)}}, + }, + }, + } + + merged := MergeGroupValues(metricData) + if assert.Len(t, merged, 2) { + assert.Equal(t, "node_stats", merged[0].Groups[0].Value) + assert.Equal(t, "cluster_health", merged[1].Groups[0].Value) + } + + if assert.Len(t, merged[0].Data["a"], 2) { + assert.Equal(t, float64(1000), merged[0].Data["a"][0].Timestamp) + assert.Equal(t, float64(2000), merged[0].Data["a"][1].Timestamp) + } +} diff --git a/core/replay_test.go b/core/replay_test.go new file mode 100644 index 00000000..c5c7aa66 --- /dev/null +++ b/core/replay_test.go @@ -0,0 +1,81 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package core + +import ( + "net/http" + "net/http/httptest" + "testing" + + replaysecurity "infini.sh/framework/core/security/replay" +) + +func TestReplayNonceCanOnlyBeUsedOnce(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + + nonce, _, err := replaysecurity.IssueReplayNonce(req, http.MethodPost, "/account/login") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + req.Header.Set(replaysecurity.HeaderName, nonce) + if err := replaysecurity.ValidateAndConsumeReplayNonce(req); err != nil { + t.Fatalf("expected first nonce use to succeed: %v", err) + } + if err := replaysecurity.ValidateAndConsumeReplayNonce(req); err == nil { + t.Fatal("expected second nonce use to be rejected") + } +} + +func TestReplayNonceBindsToAuthorizationHeader(t *testing.T) { + issueReq := httptest.NewRequest(http.MethodPut, "https://console.local/credential/test", nil) + issueReq.Header.Set("Authorization", "Bearer token-a") + + nonce, _, err := replaysecurity.IssueReplayNonce(issueReq, http.MethodPut, "/credential/test") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + useReq := httptest.NewRequest(http.MethodPut, "https://console.local/credential/test", nil) + useReq.Header.Set(replaysecurity.HeaderName, nonce) + useReq.Header.Set("Authorization", "Bearer token-b") + if err := replaysecurity.ValidateAndConsumeReplayNonce(useReq); err == nil { + t.Fatal("expected nonce bound to a different authorization header to fail") + } +} + +func TestReplayNonceBindsToPathAndMethod(t *testing.T) { + issueReq := httptest.NewRequest(http.MethodPost, "https://console.local/setup/_initialize", nil) + + nonce, _, err := replaysecurity.IssueReplayNonce(issueReq, http.MethodPost, "/setup/_initialize") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + useReq := httptest.NewRequest(http.MethodPut, "https://console.local/setup/_initialize", nil) + useReq.Header.Set(replaysecurity.HeaderName, nonce) + if err := replaysecurity.ValidateAndConsumeReplayNonce(useReq); err == nil { + t.Fatal("expected nonce with mismatched method to fail") + } +} diff --git a/core/security/access_token.go b/core/security/access_token.go index bad08095..0632aba5 100644 --- a/core/security/access_token.go +++ b/core/security/access_token.go @@ -35,10 +35,13 @@ import ( "infini.sh/framework/core/util" ) +const accessTokenTTL = 2 * time.Hour + func GenerateAccessToken(user *User) (map[string]interface{}, error) { var data map[string]interface{} roles, privilege := user.GetPermissions() + expireAt := time.Now().Add(accessTokenTTL) token1 := jwt.NewWithClaims(jwt.SigningMethodHS256, UserClaims{ ShortUser: &ShortUser{ @@ -48,7 +51,7 @@ func GenerateAccessToken(user *User) (map[string]interface{}, error) { Roles: roles, }, RegisteredClaims: &jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(time.Now().Add(24 * time.Hour)), + ExpiresAt: jwt.NewNumericDate(expireAt), }, }) @@ -57,14 +60,19 @@ func GenerateAccessToken(user *User) (map[string]interface{}, error) { return nil, errors.Errorf("failed to generate access_token for user: %v", user.Username) } - token := Token{ExpireIn: time.Now().Unix() + 86400} + token := Token{ + JwtStr: tokenString, + Value: tokenString, + ExpireIn: expireAt.Unix(), + } SetUserToken(user.ID, token) data = util.MapStr{ "access_token": tokenString, + "expires_at": expireAt.Unix(), "username": user.Username, "id": user.ID, - "expire_in": 86400, + "expire_in": int64(accessTokenTTL / time.Second), "roles": roles, "privilege": privilege, } diff --git a/core/security/access_token_test.go b/core/security/access_token_test.go new file mode 100644 index 00000000..f07929ca --- /dev/null +++ b/core/security/access_token_test.go @@ -0,0 +1,85 @@ +package security + +import ( + "fmt" + "infini.sh/framework/core/kv" + "testing" + "time" +) + +type memoryKVStore struct { + values map[string][]byte +} + +func (m *memoryKVStore) Open() error { return nil } + +func (m *memoryKVStore) Close() error { return nil } + +func (m *memoryKVStore) GetValue(bucket string, key []byte) ([]byte, error) { + value, ok := m.values[fmt.Sprintf("%s:%s", bucket, string(key))] + if !ok { + return nil, nil + } + return value, nil +} + +func (m *memoryKVStore) GetCompressedValue(bucket string, key []byte) ([]byte, error) { + return m.GetValue(bucket, key) +} + +func (m *memoryKVStore) AddValueCompress(bucket string, key []byte, value []byte) error { + return m.AddValue(bucket, key, value) +} + +func (m *memoryKVStore) AddValue(bucket string, key []byte, value []byte) error { + m.values[fmt.Sprintf("%s:%s", bucket, string(key))] = value + return nil +} + +func (m *memoryKVStore) ExistsKey(bucket string, key []byte) (bool, error) { + _, ok := m.values[fmt.Sprintf("%s:%s", bucket, string(key))] + return ok, nil +} + +func (m *memoryKVStore) DeleteKey(bucket string, key []byte) error { + delete(m.values, fmt.Sprintf("%s:%s", bucket, string(key))) + return nil +} + +func TestGenerateAccessTokenIncludesAbsoluteExpiry(t *testing.T) { + kv.Register("access-token-test", &memoryKVStore{ + values: map[string][]byte{}, + }) + + user := &User{ + AuthProvider: "native", + Username: "tester", + } + user.ID = "user-access-token-expiry" + t.Cleanup(func() { + DeleteUserToken(user.ID) + }) + + token, err := GenerateAccessToken(user) + if err != nil { + t.Fatalf("generate access token: %v", err) + } + + expireIn, ok := token["expire_in"].(int64) + if !ok { + t.Fatalf("expected expire_in int64, got %T", token["expire_in"]) + } + if expireIn != int64(accessTokenTTL/time.Second) { + t.Fatalf("expected expire_in %d, got %d", int64(accessTokenTTL/time.Second), expireIn) + } + + expiresAt, ok := token["expires_at"].(int64) + if !ok { + t.Fatalf("expected expires_at int64, got %T", token["expires_at"]) + } + + now := time.Now().Unix() + if expiresAt < now+int64(accessTokenTTL/time.Second)-5 || expiresAt > now+int64(accessTokenTTL/time.Second)+5 { + t.Fatalf("expected expires_at near %d, got %d", now+int64(accessTokenTTL/time.Second), expiresAt) + } +} diff --git a/core/security/context.go b/core/security/context.go index 5dd10b45..30429db3 100644 --- a/core/security/context.go +++ b/core/security/context.go @@ -30,6 +30,7 @@ package security import ( "context" "fmt" + frameworksecurity "infini.sh/framework/core/security" "github.com/golang-jwt/jwt/v4" ) @@ -39,6 +40,7 @@ const ctxUserKey = "user" type UserClaims struct { *jwt.RegisteredClaims *ShortUser + PermissionKeys []string `json:"permission_keys,omitempty"` } type ShortUser struct { @@ -50,18 +52,128 @@ type ShortUser struct { const Secret = "console" +var frameworkDefaultPermissions = []frameworksecurity.PermissionKey{ + frameworksecurity.GetOrInitPermission("generic", "license", "info"), +} + func NewUserContext(ctx context.Context, clam *UserClaims) context.Context { + if clam != nil { + ctx = frameworksecurity.AddUserToContext(ctx, clam.ToSessionInfo()) + } return context.WithValue(ctx, ctxUserKey, clam) } func FromUserContext(ctx context.Context) (*ShortUser, error) { ctxUser := ctx.Value(ctxUserKey) - if ctxUser == nil { - return nil, fmt.Errorf("user not found") + if ctxUser != nil { + switch reqUser := ctxUser.(type) { + case *UserClaims: + return reqUser.ShortUser, nil + case *ShortUser: + return reqUser, nil + } + } + + sessionUser, err := frameworksecurity.GetUserFromContext(ctx) + if err == nil && sessionUser != nil { + return NewShortUserFromSession(sessionUser), nil + } + return nil, fmt.Errorf("user not found") +} + +func NewShortUserFromSession(sessionUser *frameworksecurity.UserSessionInfo) *ShortUser { + if sessionUser == nil { + return nil + } + return &ShortUser{ + Provider: sessionUser.Provider, + Username: sessionUser.Login, + UserId: sessionUser.UserID, + Roles: append([]string(nil), sessionUser.Roles...), + } +} + +func NewUserClaimsFromSession(sessionUser *frameworksecurity.UserSessionInfo) *UserClaims { + shortUser := NewShortUserFromSession(sessionUser) + if shortUser == nil { + return nil + } + var permissionKeys []string + if sessionUser != nil && sessionUser.UserAssignedPermission != nil { + keys := sessionUser.UserAssignedPermission.GetPermissionKeys() + permissionKeys = make([]string, 0, len(keys)) + for _, key := range keys { + permissionKeys = append(permissionKeys, string(key)) + } + } + return &UserClaims{ + ShortUser: shortUser, + PermissionKeys: permissionKeys, + } +} + +func (u *UserClaims) ToSessionInfo() *frameworksecurity.UserSessionInfo { + if u == nil { + return nil + } + sessionUser := u.ShortUser.ToSessionInfo() + if sessionUser == nil { + return nil } - reqUser, ok := ctxUser.(*UserClaims) - if !ok { - return nil, fmt.Errorf("invalid context user") + permissionKeys := make([]frameworksecurity.PermissionKey, 0, len(u.PermissionKeys)) + for _, permissionKey := range u.PermissionKeys { + if permissionKey == "" { + continue + } + permissionKeys = append(permissionKeys, frameworksecurity.PermissionKey(permissionKey)) + } + if len(permissionKeys) > 0 { + sessionUser.UserAssignedPermission = frameworksecurity.NewUserAssignedPermission(permissionKeys, nil) + } + return EnsureFrameworkDefaultPermissions(sessionUser) +} + +func (u *ShortUser) ToSessionInfo() *frameworksecurity.UserSessionInfo { + if u == nil { + return nil + } + sessionUser := &frameworksecurity.UserSessionInfo{ + Provider: u.Provider, + Login: u.Username, + Roles: append([]string(nil), u.Roles...), + } + sessionUser.SetUserID(u.UserId) + return EnsureFrameworkDefaultPermissions(sessionUser) +} + +func EnsureFrameworkDefaultPermissions(sessionUser *frameworksecurity.UserSessionInfo) *frameworksecurity.UserSessionInfo { + if sessionUser == nil { + return nil + } + + permissions := getFrameworkPermissionKeys(sessionUser) + for _, permission := range frameworkDefaultPermissions { + if !hasFrameworkPermission(permissions, permission) { + permissions = append(permissions, permission) + } + } + sessionUser.UserAssignedPermission = frameworksecurity.NewUserAssignedPermission(permissions, nil) + + return sessionUser +} + +func getFrameworkPermissionKeys(sessionUser *frameworksecurity.UserSessionInfo) []frameworksecurity.PermissionKey { + if sessionUser == nil || sessionUser.UserAssignedPermission == nil { + return nil + } + return append([]frameworksecurity.PermissionKey(nil), sessionUser.UserAssignedPermission.GetPermissionKeys()...) +} + +func hasFrameworkPermission(permissions []frameworksecurity.PermissionKey, permission frameworksecurity.PermissionKey) bool { + for _, existing := range permissions { + if existing == permission { + return true + } } - return reqUser.ShortUser, nil + return false } diff --git a/core/security/context_test.go b/core/security/context_test.go new file mode 100644 index 00000000..1a3d8fe4 --- /dev/null +++ b/core/security/context_test.go @@ -0,0 +1,98 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import ( + "context" + "testing" + + frameworksecurity "infini.sh/framework/core/security" +) + +// Console handlers should still be able to read the current user after middleware +// starts writing framework-native session users into request contexts. +func TestFromUserContextReadsFrameworkSessionUser(t *testing.T) { + sessionUser := &frameworksecurity.UserSessionInfo{ + Provider: "native", + Login: "admin@example.org", + Roles: []string{"admin"}, + } + sessionUser.SetUserID("user-1") + + ctx := frameworksecurity.AddUserToContext(context.Background(), sessionUser) + user, err := FromUserContext(ctx) + if err != nil { + t.Fatalf("from user context: %v", err) + } + if user.Username != "admin@example.org" { + t.Fatalf("expected username to map from framework session, got %q", user.Username) + } + if user.UserId != "user-1" { + t.Fatalf("expected user id to map from framework session, got %q", user.UserId) + } +} + +// While console auth middleware is still in place it should populate both the legacy +// console context key and the framework-native session context used by shared helpers. +func TestNewUserContextAlsoSeedsFrameworkContext(t *testing.T) { + ctx := NewUserContext(context.Background(), &UserClaims{ + ShortUser: &ShortUser{ + Provider: "native", + Username: "admin@example.org", + UserId: "user-1", + Roles: []string{"admin"}, + }, + }) + + sessionUser, err := frameworksecurity.GetUserFromContext(ctx) + if err != nil { + t.Fatalf("framework get user from context: %v", err) + } + if sessionUser.Login != "admin@example.org" { + t.Fatalf("expected framework session login, got %q", sessionUser.Login) + } + if sessionUser.UserID != "user-1" { + t.Fatalf("expected framework session user id, got %q", sessionUser.UserID) + } +} + +func TestEnsureFrameworkDefaultPermissionsAddsLicenseInfoOnce(t *testing.T) { + sessionUser := &frameworksecurity.UserSessionInfo{ + UserAssignedPermission: frameworksecurity.NewUserAssignedPermission([]frameworksecurity.PermissionKey{ + frameworksecurity.GetSimplePermission("cluster", "unit", "read"), + frameworksecurity.GetSimplePermission("generic", "license", "info"), + }, nil), + } + + EnsureFrameworkDefaultPermissions(sessionUser) + EnsureFrameworkDefaultPermissions(sessionUser) + + if sessionUser.UserAssignedPermission == nil { + t.Fatalf("expected user assigned permissions to be initialized") + } + permissions := sessionUser.UserAssignedPermission.GetPermissionKeys() + if len(permissions) != 2 { + t.Fatalf("expected default permissions to be added once, got %v", permissions) + } +} diff --git a/core/security/enum/const.go b/core/security/enum/const.go index 9f9968f2..2c34eecc 100644 --- a/core/security/enum/const.go +++ b/core/security/enum/const.go @@ -181,8 +181,8 @@ var ( AliasReadPermission = []string{"alias:read", "alias:write"} ViewsAllPermission = []string{PermissionViewRead, PermissionViewWrite, PermissionLayoutRead, PermissionLayoutWrite} ViewsReadPermission = []string{PermissionViewRead, PermissionLayoutRead} - DiscoverReadPermission = []string{PermissionViewRead} - DiscoverAllPermission = []string{PermissionViewRead} + DiscoverReadPermission = []string{PermissionViewRead, PermissionLayoutRead} + DiscoverAllPermission = []string{PermissionViewRead, PermissionLayoutRead, PermissionLayoutWrite} RuleReadPermission = []string{PermissionAlertRuleRead, PermissionAlertHistoryRead} RuleAllPermission = []string{PermissionAlertRuleRead, PermissionAlertRuleWrite, PermissionAlertHistoryRead, PermissionElasticsearchClusterRead} diff --git a/core/security/enum/const_test.go b/core/security/enum/const_test.go new file mode 100644 index 00000000..0d3c2180 --- /dev/null +++ b/core/security/enum/const_test.go @@ -0,0 +1,28 @@ +package enum + +import ( + "testing" +) + +func TestDiscoverPermissionsIncludeLayoutAccess(t *testing.T) { + has := func(permissions []string, target string) bool { + for _, permission := range permissions { + if permission == target { + return true + } + } + return false + } + + if !has(DiscoverReadPermission, PermissionLayoutRead) { + t.Fatalf("expected discover read permissions to include %q", PermissionLayoutRead) + } + + if !has(DiscoverAllPermission, PermissionLayoutRead) { + t.Fatalf("expected discover all permissions to include %q", PermissionLayoutRead) + } + + if !has(DiscoverAllPermission, PermissionLayoutWrite) { + t.Fatalf("expected discover all permissions to include %q", PermissionLayoutWrite) + } +} diff --git a/core/security/framework_permissions.go b/core/security/framework_permissions.go new file mode 100644 index 00000000..6b70dbbb --- /dev/null +++ b/core/security/framework_permissions.go @@ -0,0 +1,50 @@ +package security + +import ( + "sort" + + "infini.sh/console/core/security/enum" + frameworksecurity "infini.sh/framework/core/security" +) + +func ExpandFrameworkPermissionKeysForPlatformPrivileges(privileges []string) []frameworksecurity.PermissionKey { + permissionSet := map[frameworksecurity.PermissionKey]struct{}{} + add := func(keys ...frameworksecurity.PermissionKey) { + for _, key := range keys { + if key == "" { + continue + } + permissionSet[key] = struct{}{} + } + } + + for _, privilege := range privileges { + add(frameworksecurity.PermissionKey(privilege)) + switch privilege { + case enum.SecurityAll: + add( + frameworksecurity.GetOrInitPermission("generic", "security:auth:api-token", frameworksecurity.Create), + frameworksecurity.GetOrInitPermission("generic", "security:auth:api-token", frameworksecurity.Update), + frameworksecurity.GetOrInitPermission("generic", "security:auth:api-token", frameworksecurity.Delete), + frameworksecurity.GetOrInitPermission("generic", "security:auth:api-token", frameworksecurity.Search), + ) + case enum.SecurityRead: + add( + frameworksecurity.GetOrInitPermission("generic", "security:auth:api-token", frameworksecurity.Search), + ) + } + } + + if len(permissionSet) == 0 { + return nil + } + + keys := make([]frameworksecurity.PermissionKey, 0, len(permissionSet)) + for key := range permissionSet { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + return keys +} diff --git a/core/security/framework_permissions_test.go b/core/security/framework_permissions_test.go new file mode 100644 index 00000000..feed1c2c --- /dev/null +++ b/core/security/framework_permissions_test.go @@ -0,0 +1,33 @@ +package security + +import ( + "testing" + + "infini.sh/console/core/security/enum" + frameworksecurity "infini.sh/framework/core/security" +) + +func TestExpandFrameworkPermissionKeysForPlatformPrivileges(t *testing.T) { + keys := ExpandFrameworkPermissionKeysForPlatformPrivileges([]string{enum.SecurityAll}) + + has := func(target frameworksecurity.PermissionKey) bool { + for _, key := range keys { + if key == target { + return true + } + } + return false + } + + if !has(frameworksecurity.PermissionKey(enum.SecurityAll)) { + t.Fatalf("expected platform privilege %q to be preserved", enum.SecurityAll) + } + + if !has(frameworksecurity.GetOrInitPermission("generic", "security:auth:api-token", frameworksecurity.Create)) { + t.Fatal("expected create access token permission to be included for system.security:all") + } + + if !has(frameworksecurity.GetOrInitPermission("generic", "security:auth:api-token", frameworksecurity.Search)) { + t.Fatal("expected search access token permission to be included for system.security:all") + } +} diff --git a/core/security/user.go b/core/security/user.go index 0d4177e1..92e1d97a 100644 --- a/core/security/user.go +++ b/core/security/user.go @@ -34,19 +34,36 @@ import ( type User struct { orm.ORMObjectBase - AuthProvider string `json:"auth_provider" elastic_mapping:"auth_provider: { type: keyword }"` - Username string `json:"name" elastic_mapping:"name: { type: keyword }"` - Nickname string `json:"nick_name" elastic_mapping:"nick_name: { type: keyword }"` - Password string `json:"password" elastic_mapping:"password: { type: keyword }"` - Email string `json:"email" elastic_mapping:"email: { type: keyword }"` - Phone string `json:"phone" elastic_mapping:"phone: { type: keyword }"` - Tags []string `json:"tags" elastic_mapping:"mobile: { type: keyword }"` + AuthProvider string `json:"auth_provider" elastic_mapping:"auth_provider: { type: keyword }"` + Username string `json:"name" elastic_mapping:"name: { type: keyword }"` + Nickname string `json:"nick_name" elastic_mapping:"nick_name: { type: keyword }"` + Password string `json:"password" elastic_mapping:"password: { type: keyword }"` + PasswordSalt string `json:"password_salt,omitempty" elastic_mapping:"password_salt: { type: keyword }"` + PasswordVerifier string `json:"password_verifier,omitempty" elastic_mapping:"password_verifier: { type: keyword }"` + Email string `json:"email" elastic_mapping:"email: { type: keyword }"` + Phone string `json:"phone" elastic_mapping:"phone: { type: keyword }"` + Tags []string `json:"tags" elastic_mapping:"mobile: { type: keyword }"` + Enabled *bool `json:"enabled,omitempty" elastic_mapping:"enabled: { type: boolean }"` AvatarUrl string `json:"avatar_url" elastic_mapping:"avatar_url: { type: keyword }"` Roles []UserRole `json:"roles" elastic_mapping:"roles: { type: object }"` Payload interface{} `json:"-"` //used for storing additional data derived from auth provider } +func (user *User) IsEnabled() bool { + if user == nil || user.Enabled == nil { + return true + } + return *user.Enabled +} + +func (user *User) SetEnabled(enabled bool) { + if user == nil { + return + } + user.Enabled = &enabled +} + func (user *User) GetPermissions() (roles []string, privileges []string) { for _, v := range user.Roles { role, ok := RoleMap[v.Name] diff --git a/core/security/user_test.go b/core/security/user_test.go new file mode 100644 index 00000000..7e52d013 --- /dev/null +++ b/core/security/user_test.go @@ -0,0 +1,23 @@ +package security + +import "testing" + +func TestUserIsEnabledDefaultsToTrue(t *testing.T) { + var user User + if !user.IsEnabled() { + t.Fatalf("expected user without enabled flag to be enabled by default") + } +} + +func TestUserSetEnabled(t *testing.T) { + var user User + user.SetEnabled(false) + if user.IsEnabled() { + t.Fatalf("expected user to be disabled after SetEnabled(false)") + } + + user.SetEnabled(true) + if !user.IsEnabled() { + t.Fatalf("expected user to be enabled after SetEnabled(true)") + } +} diff --git a/core/security/validate.go b/core/security/validate.go index 44f94fba..80d544cb 100644 --- a/core/security/validate.go +++ b/core/security/validate.go @@ -31,6 +31,7 @@ import ( "errors" "fmt" "net/http" + "net/http/httptest" "strings" "time" @@ -38,6 +39,7 @@ import ( "infini.sh/console/core/security/enum" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/radix" + frameworksecurity "infini.sh/framework/core/security" "infini.sh/framework/core/util" ) @@ -60,6 +62,13 @@ type IndexRequest struct { type ElasticsearchAPIPrivilege map[string]map[string]struct{} +var legacyPermissionAliases = map[string][]string{ + "template.delete": {"indices.delete_template"}, + "template.exists": {"indices.exists_template"}, + "template.get": {"indices.get_template"}, + "template.put": {"indices.put_template"}, +} + func (ep ElasticsearchAPIPrivilege) Merge(epa ElasticsearchAPIPrivilege) { for k, permissions := range epa { if _, ok := ep[k]; ok { @@ -98,6 +107,18 @@ func NewClusterRequest(ps httprouter.Params, privilege []string) ClusterRequest } } +func hasPermissionOrAlias(permissions map[string]struct{}, permission string) bool { + if _, ok := permissions[permission]; ok { + return true + } + for _, alias := range legacyPermissionAliases[permission] { + if _, ok := permissions[alias]; ok { + return true + } + } + return false +} + func validateApiPermission(apiPrivileges map[string]struct{}, permissions map[string]struct{}) { if _, ok := permissions["*"]; ok { for privilege := range apiPrivileges { @@ -105,8 +126,8 @@ func validateApiPermission(apiPrivileges map[string]struct{}, permissions map[st } return } - for permission := range permissions { - if _, ok := apiPrivileges[permission]; ok { + for permission := range apiPrivileges { + if hasPermissionOrAlias(permissions, permission) { delete(apiPrivileges, permission) } } @@ -117,7 +138,7 @@ func validateApiPermission(apiPrivileges map[string]struct{}, permissions map[st } prefix := privilege[:position] - if _, ok := permissions[prefix+".*"]; ok { + if hasPermissionOrAlias(permissions, prefix+".*") { delete(apiPrivileges, privilege) } } @@ -284,12 +305,11 @@ func GetRoleCluster(roles []string) (bool, []string) { // GetCurrentUserCluster get cluster id by current login user // return true when has all cluster privilege, otherwise return cluster id list func GetCurrentUserCluster(req *http.Request) (bool, []string) { - ctxVal := req.Context().Value("user") - if userClaims, ok := ctxVal.(*UserClaims); ok { - return GetRoleCluster(userClaims.Roles) - } else { - panic("user context value not found") + user, err := FromUserContext(req.Context()) + if err == nil && user != nil { + return GetRoleCluster(user.Roles) } + return false, nil } func GetRoleIndex(roles []string, clusterID string) (bool, []string) { @@ -313,18 +333,22 @@ func GetRoleIndex(roles []string, clusterID string) (bool, []string) { return false, realIndex } -func ValidateLogin(authorizationHeader string) (clams *UserClaims, err error) { - - if authorizationHeader == "" { - err = errors.New("authorization header is empty") - return +func ParseBearerToken(authorizationHeader string) (string, error) { + if strings.TrimSpace(authorizationHeader) == "" { + return "", errors.New("authorization header is empty") } fields := strings.Fields(authorizationHeader) - if fields[0] != "Bearer" || len(fields) != 2 { - err = errors.New("authorization header is invalid") - return + if len(fields) != 2 || !strings.EqualFold(fields[0], "Bearer") { + return "", errors.New("authorization header is invalid") + } + return fields[1], nil +} + +func ValidateLogin(authorizationHeader string) (clams *UserClaims, err error) { + tokenString, err := ParseBearerToken(authorizationHeader) + if err != nil { + return nil, err } - tokenString := fields[1] token, err := jwt.ParseWithClaims(tokenString, &UserClaims{}, func(token *jwt.Token) (interface{}, error) { if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { @@ -333,7 +357,11 @@ func ValidateLogin(authorizationHeader string) (clams *UserClaims, err error) { return []byte(Secret), nil }) if err != nil { - return + sessionUser, frameworkErr := frameworksecurity.ValidateAuthorizationHeader(authorizationHeader) + if frameworkErr == nil { + return NewUserClaimsFromSession(sessionUser), nil + } + return nil, err } clams, ok := token.Claims.(*UserClaims) @@ -352,6 +380,14 @@ func ValidateLogin(authorizationHeader string) (clams *UserClaims, err error) { DeleteUserToken(clams.UserId) return } + activeToken := tokenVal.Value + if activeToken == "" { + activeToken = tokenVal.JwtStr + } + if activeToken != "" && activeToken != tokenString { + err = errors.New("token is invalid") + return + } if ok && token.Valid { return clams, nil } @@ -359,6 +395,51 @@ func ValidateLogin(authorizationHeader string) (clams *UserClaims, err error) { } +func ValidateLoginFromRequest(req *http.Request) (claims *UserClaims, err error) { + if req == nil { + return nil, errors.New("request is nil") + } + + sessionUser, frameworkErr := frameworksecurity.ValidateLogin(httptest.NewRecorder(), req) + if frameworkErr == nil && sessionUser != nil && sessionUser.IsValid() { + claims = NewUserClaimsFromSession(sessionUser) + if claims == nil || claims.ShortUser == nil { + return nil, errors.New("invalid user info") + } + if err = enrichClaimsFromNativeUser(claims); err != nil { + return nil, err + } + return claims, nil + } + + claims, err = ValidateLogin(req.Header.Get("Authorization")) + if err != nil { + return nil, err + } + if err = enrichClaimsFromNativeUser(claims); err != nil { + return nil, err + } + return claims, nil +} + +func enrichClaimsFromNativeUser(claims *UserClaims) error { + if claims == nil || claims.ShortUser == nil || claims.UserId == "" { + return nil + } + user, err := GetAdapter("native").User.Get(claims.UserId) + if err != nil || user.ID == "" { + return nil + } + if !user.IsEnabled() { + return fmt.Errorf("user account [%s] is disabled", claims.Username) + } + if len(claims.Roles) == 0 { + roles, _ := user.GetPermissions() + claims.Roles = roles + } + return nil +} + func ValidatePermission(claims *UserClaims, permissions []string) (err error) { user := claims.ShortUser @@ -367,9 +448,20 @@ func ValidatePermission(claims *UserClaims, permissions []string) (err error) { err = errors.New("user id is empty") return } + if len(claims.PermissionKeys) > 0 { + userPermissionMap := make(map[string]struct{}, len(claims.PermissionKeys)) + for _, permission := range claims.PermissionKeys { + userPermissionMap[permission] = struct{}{} + } + for _, permission := range permissions { + if _, ok := userPermissionMap[permission]; !ok { + return errors.New("permission denied") + } + } + return nil + } if user.Roles == nil { - err = errors.New("api permission is empty") - return + return errors.New("api permission is empty") } // 权限校验 diff --git a/core/security/validate_test.go b/core/security/validate_test.go new file mode 100644 index 00000000..38b69621 --- /dev/null +++ b/core/security/validate_test.go @@ -0,0 +1,163 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import ( + "github.com/golang-jwt/jwt" + "testing" + "time" +) + +func TestValidateAPIPermissionSupportsLegacyTemplateAliases(t *testing.T) { + tests := []struct { + name string + privilege string + permission string + }{ + { + name: "template delete", + privilege: "template.delete", + permission: "indices.delete_template", + }, + { + name: "template exists", + privilege: "template.exists", + permission: "indices.exists_template", + }, + { + name: "template get", + privilege: "template.get", + permission: "indices.get_template", + }, + { + name: "template put", + privilege: "template.put", + permission: "indices.put_template", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + apiPrivileges := map[string]struct{}{ + tt.privilege: {}, + } + permissions := map[string]struct{}{ + tt.permission: {}, + } + + validateApiPermission(apiPrivileges, permissions) + + if len(apiPrivileges) != 0 { + t.Fatalf("expected legacy permission %q to satisfy %q", tt.permission, tt.privilege) + } + }) + } +} + +func issueTestToken(t *testing.T, userID string) string { + t.Helper() + + expireAt := time.Now().Add(time.Hour) + token := jwt.NewWithClaims(jwt.SigningMethodHS256, UserClaims{ + ShortUser: &ShortUser{ + Provider: "native", + Username: "tester", + UserId: userID, + }, + RegisteredClaims: &jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(expireAt), + }, + }) + + tokenString, err := token.SignedString([]byte(Secret)) + if err != nil { + t.Fatalf("sign token: %v", err) + } + + userTokenLocker.Lock() + tokenMap[userID] = Token{ + Value: tokenString, + ExpireIn: expireAt.Unix(), + } + userTokenLocker.Unlock() + + t.Cleanup(func() { + userTokenLocker.Lock() + delete(tokenMap, userID) + userTokenLocker.Unlock() + }) + + return tokenString +} + +func TestValidateLoginRejectsReplacedActiveToken(t *testing.T) { + userID := "user-revoked" + authorizationHeader := "Bearer " + issueTestToken(t, userID) + if _, err := ValidateLogin(authorizationHeader); err != nil { + t.Fatalf("expected token to validate before revoke, got %v", err) + } + + userTokenLocker.Lock() + tokenMap[userID] = Token{ + Value: "replaced-token", + ExpireIn: time.Now().Add(time.Hour).Unix(), + } + userTokenLocker.Unlock() + + if _, err := ValidateLogin(authorizationHeader); err == nil { + t.Fatal("expected replaced token to be rejected") + } +} + +func TestValidateLoginRejectsStaleTokenValue(t *testing.T) { + userID := "user-stale" + authorizationHeader := "Bearer " + issueTestToken(t, userID) + + userTokenLocker.Lock() + tokenMap[userID] = Token{ + Value: "replacement-token", + ExpireIn: time.Now().Add(time.Hour).Unix(), + } + userTokenLocker.Unlock() + + if _, err := ValidateLogin(authorizationHeader); err == nil { + t.Fatal("expected stale token to be rejected") + } +} + +func TestValidateLoginSupportsLegacyJwtField(t *testing.T) { + userID := "user-legacy" + tokenString := issueTestToken(t, userID) + + userTokenLocker.Lock() + tokenMap[userID] = Token{ + JwtStr: tokenString, + ExpireIn: time.Now().Add(time.Hour).Unix(), + } + userTokenLocker.Unlock() + + if _, err := ValidateLogin("Bearer " + tokenString); err != nil { + t.Fatalf("expected legacy jwt field to remain valid, got %v", err) + } +} diff --git a/docs/content.en/docs/reference/agent/docker.md b/docs/content.en/docs/reference/agent/docker.md deleted file mode 100644 index d522adcf..00000000 --- a/docs/content.en/docs/reference/agent/docker.md +++ /dev/null @@ -1,179 +0,0 @@ ---- -weight: 35 -title: Container Deployment -asciinema: true ---- - -# Container Deployment - -INFINI Agent supports container deployment. - -## Download Image - -The images of INFINI Agent are published at the official repository of Docker. The URL is as follows: -[https://hub.docker.com/r/infinilabs/agent](https://hub.docker.com/r/infinilabs/agent) - -Run the following command: - -``` -docker pull infinilabs/agent:{{< globaldata "agent" "version" >}} -``` - -## Verifying the Image - -After downloading the image locally, you will notice that the container image of INFINI Agent is very small, with a size less than 25 MB. So, the downloading is very fast. - -``` -✗ docker images |grep "agent" |grep "{{< globaldata "agent" "version" >}}" -REPOSITORY TAG IMAGE ID CREATED SIZE -infinilabs/agent {{< globaldata "agent" "version" >}} c7bd9ad063d9 4 days ago 13.8MB -``` - -## Configuration - -Create a configuration file `agent.yml` to perform basic configuration as follows: - -``` -api: - enabled: true - network: - binding: 0.0.0.0:8080 - -metrics: - enabled: true - queue: metrics - network: - enabled: true - summary: true - details: true - memory: - metrics: - - swap - - memory - disk: - metrics: - - ioqs - - usage - cpu: - metrics: - - idle - - system - - user - - iowait - - load - elasticsearch: - enabled: true - agent_mode: true - node_stats: true - index_stats: true - cluster_stats: true - -elasticsearch: - - name: default - enabled: true - endpoint: http://192.168.3.4:9200 - monitored: false - discovery: - enabled: true - -pipeline: - - name: metrics_ingest - auto_start: true - keep_running: true - processor: - - json_indexing: - index_name: ".infini_metrics" - elasticsearch: "default" - input_queue: "metrics" - output_queue: - name: "metrics_requests" - label: - tag: "metrics" - worker_size: 1 - bulk_size_in_mb: 10 - - name: consume-metrics_requests - auto_start: true - keep_running: true - processor: - - bulk_indexing: - bulk: - compress: true - batch_size_in_mb: 10 - batch_size_in_docs: 5000 - consumer: - fetch_max_messages: 100 - queues: - type: indexing_merge - when: - cluster_available: [ "default" ] - -agent: - major_ip_pattern: "192.*" - labels: - env: dev - tags: - - linux - - x86 - - es7 - - v7.5 - -path.data: data -path.logs: log - -agent.manager.endpoint: http://192.168.3.4:9000 -``` - -Note: In the above configuration, replace the Elasticsearch configuration with the actual server connection address and authentication information. - -## Starting - -Run the following command: - -``` -docker run -p 8080:8080 -v=`pwd`/agent.yml:/agent.yml infinilabs/agent:{{< globaldata "agent" "version" >}} -``` - -## Docker Compose - -You can also use docker compose to manage container instances. Create one docker-compose.yml file as follows: - -``` -version: "3.5" - -services: - infini-agent: - image: infinilabs/agent:{{< globaldata "agent" "version" >}} - ports: - - 8080:8080 - container_name: "infini-agent" - volumes: - - ./agent.yml:/agent.yml - -volumes: - dist: -``` - -Run the following command to start INFINI Agent. - -``` -➜ docker-compose up -Recreating infini-agent ... done -Attaching to infini-agent -infini-agent | _ ___ __ __ _____ -infini-agent | /_\ / _ \ /__\/\ \ \/__ \ -infini-agent | //_\\ / /_\//_\ / \/ / / /\/ -infini-agent | / _ \/ /_\\//__/ /\ / / / -infini-agent | \_/ \_/\____/\__/\_\ \/ \/ -infini-agent | -infini-agent | [AGENT] A light-weight, powerful and high-performance elasticsearch agent. -infini-agent | [AGENT] 0.1.0_SNAPSHOT#15, 2022-08-26 15:05:43, 2025-12-31 10:10:10, 164bd8a0d74cfd0ba5607352e125d72b46a1079e -infini-agent | [08-31 09:11:45] [INF] [app.go:164] initializing agent. -infini-agent | [08-31 09:11:45] [INF] [app.go:165] using config: /agent.yml. -infini-agent | [08-31 09:11:45] [INF] [instance.go:72] workspace: /data/agent/nodes/cc7ibke5epac7314bf9g -infini-agent | [08-31 09:11:45] [INF] [metrics.go:63] ip:172.18.0.2, host:bd9f43490911, labels:, tags: -infini-agent | [08-31 09:11:45] [INF] [api.go:261] api listen at: http://0.0.0.0:8080 -infini-agent | [08-31 09:11:45] [INF] [actions.go:367] elasticsearch [default] is available -infini-agent | [08-31 09:11:45] [INF] [module.go:116] all modules are started -infini-agent | [08-31 09:11:45] [INF] [manage.go:180] register agent to console -infini-agent | [08-31 09:11:45] [INF] [app.go:334] agent is up and running now. -``` diff --git a/docs/content.en/docs/reference/agent/install.md b/docs/content.en/docs/reference/agent/install.md deleted file mode 100644 index 77fef216..00000000 --- a/docs/content.en/docs/reference/agent/install.md +++ /dev/null @@ -1,287 +0,0 @@ ---- -weight: 20 -title: Installing Agent -asciinema: true ---- - -# Installing The Agent - -## Before You Begin - -Install and keep [INFINI Console](../../getting-started/install) running. - -## Install by Console generated script - -``` -curl -sSL http://localhost:9000/agent/install.sh?token=cjctdrms4us1c6fu04ag |sudo bash -s -- -u https://release.infinilabs.com/agent/stable -v 0.6.0-262 -t /opt/agent -``` - -> The -u and -v parameters indicate that the specified version of the Agent is downloaded from the specified URL, and the -t parameter indicates the installation path. In a networked environment, the -- and subsequent parameters can be ignored, and by default, the latest version of the Agent will be downloaded from the official website for installation. - -## Container Deployment - -INFINI Agent also supports Docker container deployment. - -{{< button relref="./docker" >}}Learn More{{< /button >}} - -## Configuration - -Most of the configuration of INFINI Agent can be completed using `agent.yml`. After the configuration is modified, the agent program needs to be restarted to make the configuration take effect. - -After unzip the file and open `agent.yml`, you will see this: - -``` -env: - LOGGING_ES_ENDPOINT: http://localhost:9200 - LOGGING_ES_USER: admin - LOGGING_ES_PASS: admin - API_BINDING: "0.0.0.0:2900" - -path.data: data -path.logs: log - -api: - enabled: true - network: - binding: $[[env.API_BINDING]] - -# omitted ... -agent.manager.endpoint: http://192.168.3.4:9000 -``` - -In most cases, you only need to config the `LOGGING_ES_ENDPOINT`, but if Elasticsearch has security authentication enabled, then configure the `LOGGING_ES_USER` and `LOGGING_ES_PASS`. - -The user must have access to the cluster metadata, index metadata, and all indexes with `.infini` prefix. - -## Starting the Agent - -Run the agent program to start INFINI Agent, as follows: - -``` - _ ___ __ __ _____ - /_\ / _ \ /__\/\ \ \/__ \ - //_\\ / /_\//_\ / \/ / / /\/ -/ _ \/ /_\\//__/ /\ / / / -\_/ \_/\____/\__/\_\ \/ \/ - -[AGENT] A light-weight, powerful and high-performance elasticsearch agent. -[AGENT] 0.1.0#14, 2022-08-26 14:09:29, 2025-12-31 10:10:10, 4489a8dff2b68501a0dd9ae15276cf5751d50e19 -[08-31 15:52:07] [INF] [app.go:164] initializing agent. -[08-31 15:52:07] [INF] [app.go:165] using config: /Users/INFINI/agent/agent-0.1.0-14-mac-arm64/agent.yml. -[08-31 15:52:07] [INF] [instance.go:72] workspace: /Users/INFINI/agent/agent-0.1.0-14-mac-arm64/data/agent/nodes/cc7h5qitoaj25p2g9t20 -[08-31 15:52:07] [INF] [metrics.go:63] ip:192.168.3.22, host:INFINI-MacBook.local, labels:, tags: -[08-31 15:52:07] [INF] [api.go:261] api listen at: http://0.0.0.0:8080 -[08-31 15:52:07] [INF] [module.go:116] all modules are started -[08-31 15:52:07] [INF] [manage.go:180] register agent to console -[08-31 15:52:07] [INF] [actions.go:367] elasticsearch [default] is available -[08-31 15:52:07] [INF] [manage.go:203] registering, waiting for review -[08-31 15:52:07] [INF] [app.go:334] agent is up and running now. -``` - -If the above startup information is displayed, the agent is running successfully and listening on the responding port. - -But now agent can't work normally util it's being added to INFINI Console. See [Agent Manage](./manage/manage) - -## Shutting Down the Agent - -To shut down INFINI Agent, hold down Ctrl+C. The following information will be displayed: - -``` -^C -[AGENT] got signal: interrupt, start shutting down -[08-31 15:57:13] [INF] [module.go:145] all modules are stopped -[08-31 15:57:13] [INF] [app.go:257] agent now terminated. -[AGENT] 0.1.0, uptime: 5m6.240314s - - __ _ __ ____ __ _ __ __ - / // |/ // __// // |/ // / - / // || // _/ / // || // / -/_//_/|_//_/ /_//_/|_//_/ - -©INFINI.LTD, All Rights Reserved. -``` - -## System Service - -To run the INFINI Agent as a system service, run the following commands: - -``` -➜ ./agent -service install -Success -➜ ./agent -service start -Success -``` - -Uninstall service: - -``` -➜ ./agent -service stop -Success -➜ ./agent -service uninstall -Success -``` - -## Manual Configuration - -If you want to manually configure the INFINI Agent to collect Elasticsearch logs and metrics, you can refer to the `agent.yml`. If you want to collect metrics and logs for other Elasticsearch clusters, you need to add the corresponding configuraiton under `elasticsearch` and `pipeline` configuration. - -If you want to toggle off some metrics/logs collecting, set the corresponding `pipeline.enabled` to `false. - -### Collect Elasticsearch Metrics - -Collect node stats: - -``` - - name: collect_default_node_stats - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_node_stats: - elasticsearch: default -``` - -Collect index stats: - -``` - - name: collect_default_index_stats - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_index_stats: - elasticsearch: default -``` - -Collect cluster stats: - -``` - - name: collect_default_cluster_stats - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_cluster_stats: - elasticsearch: default -``` - -Collect cluster health info: - -``` - - name: collect_default_cluster_health - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_cluster_health: - elasticsearch: default -``` - -### Collect Elasticsearch Logs - -Collect the logs from the specified nodes, set the `endpoint` to the specified node in the `elasticsearch` configuration: - -``` - - name: collect_default_es_logs - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 3000 - processor: - - es_logs_processor: - queue_name: "logs" - elasticsearch: default -``` - -If you have multiple nodes running on the local host, add more `elasticsearch` and `pipeline` configurations: - -``` -elasticsearch: - # omitted ... - - name: cluster-a-node-1 - enabled: true - endpoint: http://localhost:9202 - monitored: false - discovery: - enabled: true - -# omitted ... - - - name: collect_node_1_es_logs - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 3000 - processor: - - es_logs_processor: - queue_name: "logs" - elasticsearch: cluster-a-node-1 -``` - -### Collect Other Logs - -If `es_logs_processor` can't provide the flexibility you need, or you want to collect other services' logs on the local host, you can use `logs_processor` to collect them. There's a sample configuration to collect Elasticsearch logs in the `agent.yml`, you can modify it or add new configurations, and update the `metadata` and `labels` for better investigations later. - -``` - - name: log_collect - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 3000 - processor: - - logs_processor: - queue_name: "logs" - logs_path: "/opt/es/elasticsearch-7.7.1/logs" - # metadata for all log items - metadata: - category: elasticsearch - # patterns are matched in order - patterns: - - pattern: ".*_server.json$" # file name pattern to match - # log type, json/text/multiline - type: json - # metadata for matched files - metadata: - name: server - # (json) timestamp fields in json message, match the first one - timestamp_fields: ["timestamp", "@timestamp"] - # (json) remove fields with specified key path - remove_fields: - [ - "type", - "cluster.name", - "cluster.uuid", - "node.name", - "node.id", - "timestamp", - "@timestamp", - ] - - pattern: "gc.log$" # file name pattern to match - # log type, json/text/multiline - type: json - # metadata for matched files - metadata: - name: gc - # (text) regex to match timestamp in the log entries - timestamp_patterns: - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2}.\\d{3}\\+\\d{4}" - - "\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" - - pattern: ".*.log$" # file name pattern to match - # log type, json/text/multiline - type: multiline - # (multiline) the pattern to match a new line - line_pattern: '^\[' - # metadata for matched files - metadata: - name: server - # (text) regex to match timestamp in the log entries - timestamp_patterns: - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2}.\\d{3}\\+\\d{4}" - - "\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" -``` diff --git a/docs/content.en/docs/reference/alerting/variables.md b/docs/content.en/docs/reference/alerting/variables.md index 74243f12..ec67290d 100644 --- a/docs/content.en/docs/reference/alerting/variables.md +++ b/docs/content.en/docs/reference/alerting/variables.md @@ -28,6 +28,7 @@ The syntax for rendering fields is `{{ .fieldname }}`, and the variable fields t | title | string | event title | Node (`{{.first_group_value}}`) disk used >= 90% | | message | string | event content | EventID：`{{.event_id}}`; Cluster：`{{.resource_name}}` | | results | array | result of groups | | +| total_results | number | total matched results before truncation | 18 | | ┗ threshold | array | | ["90"] | | ┗ priority | string | | high | | ┗ group_values | array | | ["cluster-xxx", "node-xxx"] | diff --git a/docs/content.en/docs/reference/setup/_index.md b/docs/content.en/docs/reference/setup/_index.md index 3d61986e..d6f51596 100644 --- a/docs/content.en/docs/reference/setup/_index.md +++ b/docs/content.en/docs/reference/setup/_index.md @@ -12,7 +12,7 @@ After the initial install, it will enter the initialization guide page where you ## Configuration -Connecting to system cluster (elasticsearch required version 5.3 or above). +Connecting to system cluster (Easysearch required version 2.3 or above). {{% load-img "/img/screenshot/initialization/configuration.png" %}} diff --git a/docs/content.en/docs/reference/system/security/_index.md b/docs/content.en/docs/reference/system/security/_index.md index 57d2ecbc..8e18a080 100644 --- a/docs/content.en/docs/reference/system/security/_index.md +++ b/docs/content.en/docs/reference/system/security/_index.md @@ -23,7 +23,7 @@ INFINI Console Security is enabled by default，and we can disable it by configu ```aidl web: enabled: true - embedding_api: true + embedding_api: false auth: enabled: false ui: diff --git a/docs/content.en/docs/reference/system/security/token.md b/docs/content.en/docs/reference/system/security/token.md new file mode 100644 index 00000000..87398a2c --- /dev/null +++ b/docs/content.en/docs/reference/system/security/token.md @@ -0,0 +1,54 @@ +--- +weight: 36 +title: Token +--- + +# Token Management + +## Introduction + +The token created on this page is a **managed API token** for accessing protected INFINI Console HTTP APIs. It is intended for scripts, automation jobs, and external system integrations. + +After creation, the full token value is shown only once, so copy and store it securely right away. + +> This token is different from the user session token issued after web login. Session tokens are for Console login state, while this managed token is for long-running third-party or automation API calls. + +## How to use + +Include the following header when calling Console APIs: + +```bash +X-API-TOKEN: +``` + +Example: + +```bash +curl -X GET "http://localhost:9000/account/profile" \ + -H "X-API-TOKEN: " +``` + +If the request succeeds, Console returns the account profile for the user associated with the token. + +## Common use cases + +- Calling protected Console APIs from scripts +- Accessing Console APIs in CI/CD or operations jobs +- Integrating external systems with Console over HTTP + +## Permission model + +A token is bound to the user who created it and uses that user's permissions. The APIs it can access depend on the creator's platform and data privileges. + +## Expiration and revocation + +- You can configure the expiration time when creating the token +- Tokens can expire at a specific time or be set to **never expire** +- If no expiration is specified in the create request, the default is **1 year** +- Deleting a token revokes it immediately + +## Notes + +- The full token value is displayed only once after creation +- Do not commit tokens into source control or expose them in logs, screenshots, or chats +- Use tokens only where needed and rotate them regularly diff --git a/docs/content.en/docs/troubleshooting/_index.md b/docs/content.en/docs/troubleshooting/_index.md index 2b3e6e50..992499fc 100644 --- a/docs/content.en/docs/troubleshooting/_index.md +++ b/docs/content.en/docs/troubleshooting/_index.md @@ -72,6 +72,24 @@ POST /.infini_cluster/_update_by_query?conflicts=proceed } ``` +### Console cannot start after the system cluster password is changed externally + +#### Fault Description + +If the system cluster password is rotated directly in Easysearch / Elasticsearch, the locally stored `SYSTEM_CLUSTER_PASS` used by Console will no longer match, and Console will fail to connect during startup. + +#### Solution + +Run the recovery command in the Console installation directory to refresh the local system cluster password. If the username changed too, provide it together: + +```bash +./console recovery -pass 'new-password' +./console recovery -user admin -pass 'new-password' +printf '%s' 'new-password' | ./console recovery -stdin +``` + +Restart Console after the command finishes. + ### Startup Error ``` diff --git a/docs/content.zh/docs/reference/agent/docker.md b/docs/content.zh/docs/reference/agent/docker.md deleted file mode 100644 index b0205580..00000000 --- a/docs/content.zh/docs/reference/agent/docker.md +++ /dev/null @@ -1,180 +0,0 @@ ---- -weight: 15 -title: 容器部署 -asciinema: true ---- - -# 容器部署 - -探针 (_INFINI Agent_) 支持容器方式部署。 - -## 下载镜像 - -探针(_INFINI Agent_) 的镜像发布在 Docker 的官方仓库，地址如下： - -[https://hub.docker.com/r/infinilabs/agent](https://hub.docker.com/r/infinilabs/agent) - -使用下面的命令即可获取最新的容器镜像： - -``` -docker pull infinilabs/agent:{{< globaldata "agent" "version" >}} -``` - -## 验证镜像 - -将镜像下载到本地之后，可以看到 探针 (_INFINI Agent_) 的容器镜像非常小，只有不到 20MB，所以下载是非常快的。 - -``` -✗ docker images |grep "agent" |grep "{{< globaldata "agent" "version" >}}" -REPOSITORY TAG IMAGE ID CREATED SIZE -infinilabs/agent latest {{< globaldata "agent" "version" >}} 4 days ago 13.8MB -``` - -## 创建配置 - -现在需要创建一个配置文件 `agent.yml`，来进行基本的配置，如下： - -``` -api: - enabled: true - network: - binding: 0.0.0.0:8080 - -metrics: - enabled: true - queue: metrics - network: - enabled: true - summary: true - details: true - memory: - metrics: - - swap - - memory - disk: - metrics: - - ioqs - - usage - cpu: - metrics: - - idle - - system - - user - - iowait - - load - elasticsearch: - enabled: true - agent_mode: true - node_stats: true - index_stats: true - cluster_stats: true - -elasticsearch: - - name: default - enabled: true - endpoint: http://192.168.3.4:9200 - monitored: false - discovery: - enabled: true - -pipeline: - - name: metrics_ingest - auto_start: true - keep_running: true - processor: - - json_indexing: - index_name: ".infini_metrics" - elasticsearch: "default" - input_queue: "metrics" - output_queue: - name: "metrics_requests" - label: - tag: "metrics" - worker_size: 1 - bulk_size_in_mb: 10 - - name: consume-metrics_requests - auto_start: true - keep_running: true - processor: - - bulk_indexing: - bulk: - compress: true - batch_size_in_mb: 10 - batch_size_in_docs: 5000 - consumer: - fetch_max_messages: 100 - queues: - type: indexing_merge - when: - cluster_available: [ "default" ] - -agent: - major_ip_pattern: "192.*" - labels: - env: dev - tags: - - linux - - x86 - - es7 - - v7.5 - -path.data: data -path.logs: log - -agent.manager.endpoint: http://192.168.3.4:9000 -``` - -Note: 上面配置里面的 Elasticsearch 的相关配置，请改成实际的服务器连接地址和认证信息，需要版本 v7.3 及以上。 - -## 启动 Agent - -使用如下命令启动 Agent 容器： - -``` -docker run -p 8080:8080 -v=`pwd`/agent.yml:/agent.yml infinilabs/agent:{{< globaldata "agent" "version" >}} -``` - -## Docker Compose - -还可以使用 docker compose 来管理容器实例，新建一个 `docker-compose.yml` 文件如下： - -``` -version: "3.5" - -services: - infini-agent: - image: infinilabs/agent:latest - ports: - - 8080:8080 - container_name: "infini-agent" - volumes: - - ./agent.yml:/agent.yml - -volumes: - dist: -``` - -在配置文件所在目录，执行如下命令即可启动，如下： - -``` -➜ docker-compose up -Recreating infini-agent ... done -Attaching to infini-agent -infini-agent | _ ___ __ __ _____ -infini-agent | /_\ / _ \ /__\/\ \ \/__ \ -infini-agent | //_\\ / /_\//_\ / \/ / / /\/ -infini-agent | / _ \/ /_\\//__/ /\ / / / -infini-agent | \_/ \_/\____/\__/\_\ \/ \/ -infini-agent | -infini-agent | [AGENT] A light-weight, powerful and high-performance elasticsearch agent. -infini-agent | [AGENT] 0.1.0_SNAPSHOT#15, 2022-08-26 15:05:43, 2025-12-31 10:10:10, 164bd8a0d74cfd0ba5607352e125d72b46a1079e -infini-agent | [08-31 09:11:45] [INF] [app.go:164] initializing agent. -infini-agent | [08-31 09:11:45] [INF] [app.go:165] using config: /agent.yml. -infini-agent | [08-31 09:11:45] [INF] [instance.go:72] workspace: /data/agent/nodes/cc7ibke5epac7314bf9g -infini-agent | [08-31 09:11:45] [INF] [metrics.go:63] ip:172.18.0.2, host:bd9f43490911, labels:, tags: -infini-agent | [08-31 09:11:45] [INF] [api.go:261] api listen at: http://0.0.0.0:8080 -infini-agent | [08-31 09:11:45] [INF] [actions.go:367] elasticsearch [default] is available -infini-agent | [08-31 09:11:45] [INF] [module.go:116] all modules are started -infini-agent | [08-31 09:11:45] [INF] [manage.go:180] register agent to console -infini-agent | [08-31 09:11:45] [INF] [app.go:334] agent is up and running now. -``` diff --git a/docs/content.zh/docs/reference/agent/install.md b/docs/content.zh/docs/reference/agent/install.md deleted file mode 100644 index 90381c84..00000000 --- a/docs/content.zh/docs/reference/agent/install.md +++ /dev/null @@ -1,298 +0,0 @@ ---- -weight: 15 -title: 下载安装 -asciinema: true ---- - -# 安装探针 - -探针支持两种方式安装，一种是手动下载安装配置，还有一种是结合新版本的 Console (>=1.3.0)，生成一键安装脚本。 -只要执行一键安装脚本即可在主机上完成探针的安装。我们推荐使用结合 Console 来安装探针，简单和方便管理。 - -## 一键安装 - -### 安装前准备 - -安装并运行 [INFINI Console](../../getting-started/install.md) - -### 复制一键安装脚本 - -在 INFINI Console 左侧菜单 `资源管理>探针管理`，进入页面之后点击 `Install Agent` 按钮，即可复制类似如下一键安装脚本： - -``` -curl -sSL http://localhost:9000/agent/install.sh?token=cjctdrms4us1c6fu04ag |sudo bash -s -- -u https://release.infinilabs.com/agent/stable -v 0.6.0-262 -t /opt/agent -``` - -> -u和-v参数表示从指定的 URL 下载指定版本的 Agent，-t参数表示安装的路径，在联网的环境中，-- 及后面的参数都可以忽略，默认情况下将从官网下载最新的 Agent 版本进行安装。 - -将一键安装脚本粘贴到终端执行即可完成安装，安装之后该探针实例会被自动注册到 INFINI Console。具体操作步骤参考 [Agent 快速安装](manage/manage/#快速安装探针) - -## 下载安装 - -根据您所在的操作系统和平台选择下面相应的下载地址： - -[https://release.infinilabs.com/agent/](https://release.infinilabs.com/agent/) - -## 容器部署 - -探针(_INFINI Agent_) 也支持 Docker 容器方式部署。 - -{{< button relref="./docker" >}}了解更多{{< /button >}} - -## 配置 - -下载安装包解压之后，打开 `agent.yml` 配置文件，我们可以看到以下配置： - -``` -env: - LOGGING_ES_ENDPOINT: http://localhost:9200 - LOGGING_ES_USER: admin - LOGGING_ES_PASS: admin - API_BINDING: "0.0.0.0:2900" - -path.data: data -path.logs: log - -api: - enabled: true - network: - binding: $[[env.API_BINDING]] - -# omitted ... -``` - -通常，我们只需要修改 `LOGGING_ES_ENDPOINT` 环境变量配置，若 Elasticsearch 开启了安全验证，则需要修改 `LOGGING_ES_USER` 和 `LOGGING_ES_PASS` 配置。 - -这里的用户要求具备集群的元数据、索引的元数据以及 `.infini*` 索引的完全访问权限，以及索引模板的创建权限。 - -## 启动 INFINI Agent - -直接运行程序即可启动 探针(_INFINI Agent_) 了(这里使用的是 Mac 版本的，不同平台的程序文件名称略有不同)，如下： - -``` - _ ___ __ __ _____ - /_\ / _ \ /__\/\ \ \/__ \ - //_\\ / /_\//_\ / \/ / / /\/ -/ _ \/ /_\\//__/ /\ / / / -\_/ \_/\____/\__/\_\ \/ \/ - -[AGENT] A light-weight, powerful and high-performance elasticsearch agent. -[AGENT] 0.1.0#14, 2022-08-26 14:09:29, 2025-12-31 10:10:10, 4489a8dff2b68501a0dd9ae15276cf5751d50e19 -[08-31 15:52:07] [INF] [app.go:164] initializing agent. -[08-31 15:52:07] [INF] [app.go:165] using config: /Users/INFINI/agent/agent-0.1.0-14-mac-arm64/agent.yml. -[08-31 15:52:07] [INF] [instance.go:72] workspace: /Users/INFINI/agent/agent-0.1.0-14-mac-arm64/data/agent/nodes/cc7h5qitoaj25p2g9t20 -[08-31 15:52:07] [INF] [metrics.go:63] ip:192.168.3.22, host:INFINI-MacBook.local, labels:, tags: -[08-31 15:52:07] [INF] [api.go:261] api listen at: http://0.0.0.0:8080 -[08-31 15:52:07] [INF] [module.go:116] all modules are started -[08-31 15:52:07] [INF] [manage.go:180] register agent to console -[08-31 15:52:07] [INF] [actions.go:367] elasticsearch [default] is available -[08-31 15:52:07] [INF] [manage.go:203] registering, waiting for review -[08-31 15:52:07] [INF] [app.go:334] agent is up and running now. -``` - -看到上面的启动信息，说明 探针 (_INFINI Agent_) 已经成功运行了! - -## 停止 INFINI Agent - -如果需要停止 探针(_INFINI Agent_) ，按 `Ctrl+C` 即可停止 探针(_INFINI Agent_)，如下： - -``` -^C -[AGENT] got signal: interrupt, start shutting down -[08-31 15:57:13] [INF] [module.go:145] all modules are stopped -[08-31 15:57:13] [INF] [app.go:257] agent now terminated. -[AGENT] 0.1.0, uptime: 5m6.240314s - - __ _ __ ____ __ _ __ __ - / // |/ // __// // |/ // / - / // || // _/ / // || // / -/_//_/|_//_/ /_//_/|_//_/ - -©INFINI.LTD, All Rights Reserved. -``` - -## 配置服务后台运行 - -如果希望将 探针(_INFINI Agent_) 以后台任务的方式运行，如下： - -``` -➜ ./agent -service install -Success -➜ ./agent -service start -Success -``` - -卸载服务也很简单，如下： - -``` -➜ ./agent -service stop -Success -➜ ./agent -service uninstall -Success -``` - -## 手动配置 Agent 采集功能 - -如果希望手动配置 Elasticsearch 日志和指标采集，可以参考 `agent.yml` 提供的默认参考配置。如果需要添加其他 Elasticsearch 集群的采集，需要在 `elasticsearch` 增加相应的集群配置信息，并配置对应的 `pipeline` 来采集该集群的数据。 - -如果你需要手动关闭某一项日志采集，把对应的采集 pipeline `enabled` 选项设置为 `false`。 - -### 采集 Elasticsearch 指标 - -配置采集节点 stats： - -``` - - name: collect_default_node_stats - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_node_stats: - elasticsearch: default -``` - -配置采集集群索引 stats： - -``` - - name: collect_default_index_stats - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_index_stats: - elasticsearch: default -``` - -配置采集集群 stats： - -``` - - name: collect_default_cluster_stats - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_cluster_stats: - elasticsearch: default -``` - -配置采集集群健康信息： - -``` - - name: collect_default_cluster_health - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 10000 - processor: - - es_cluster_health: - elasticsearch: default -``` - -### 采集 Elasticsearch 日志 - -配置采集节点日志，`elasticsearch` 需要配置采集节点的 `endpoint`： - -``` - - name: collect_default_es_logs - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 3000 - processor: - - es_logs_processor: - queue_name: "logs" - elasticsearch: default -``` - -如果有多个 Elasticsearch 节点在当前主机运行，每个 Elasticsearch 需要配置的对应的集群信息和 `pipeline`： - -``` -elasticsearch: - # omitted ... - - name: cluster-a-node-1 - enabled: true - endpoint: http://localhost:9202 - monitored: false - discovery: - enabled: true - -# omitted ... - - - name: collect_node_1_es_logs - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 3000 - processor: - - es_logs_processor: - queue_name: "logs" - elasticsearch: cluster-a-node-1 - -``` - -### 采集本地日志文件 - -如果 `es_logs_processor` 提供的配置选项不够灵活，或者你想采集主机上其他日志文件，也可以通过 `logs_processor` 来配置任意目录下的日志采集。`agent.yml` 默认提供了一个采集 Elasticsearch 日志的配置来作为参考，你可以修改这个配置或者增加新的配置来适配本地的日志文件，并添加对应的标签和 metadata 信息来方便过滤筛选。 - -``` - - name: log_collect - enabled: false - auto_start: true - keep_running: true - retry_delay_in_ms: 3000 - processor: - - logs_processor: - queue_name: "logs" - logs_path: "/opt/es/elasticsearch-7.7.1/logs" - # metadata for all log items - metadata: - category: elasticsearch - # patterns are matched in order - patterns: - - pattern: ".*_server.json$" # file name pattern to match - # log type, json/text/multiline - type: json - # metadata for matched files - metadata: - name: server - # (json) timestamp fields in json message, match the first one - timestamp_fields: ["timestamp", "@timestamp"] - # (json) remove fields with specified key path - remove_fields: - [ - "type", - "cluster.name", - "cluster.uuid", - "node.name", - "node.id", - "timestamp", - "@timestamp", - ] - - pattern: "gc.log$" # file name pattern to match - # log type, json/text/multiline - type: json - # metadata for matched files - metadata: - name: gc - # (text) regex to match timestamp in the log entries - timestamp_patterns: - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2}.\\d{3}\\+\\d{4}" - - "\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" - - pattern: ".*.log$" # file name pattern to match - # log type, json/text/multiline - type: multiline - # (multiline) the pattern to match a new line - line_pattern: '^\[' - # metadata for matched files - metadata: - name: server - # (text) regex to match timestamp in the log entries - timestamp_patterns: - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2}.\\d{3}\\+\\d{4}" - - "\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" - - "\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2},\\d{3}" -``` diff --git a/docs/content.zh/docs/reference/alerting/variables.md b/docs/content.zh/docs/reference/alerting/variables.md index 237ac64f..787b401d 100644 --- a/docs/content.zh/docs/reference/alerting/variables.md +++ b/docs/content.zh/docs/reference/alerting/variables.md @@ -31,6 +31,7 @@ asciinema: true | title | string | event title | Node (`{{.first_group_value}}`) disk used >= 90% | | message | string | event content | EventID：`{{.event_id}}`; Cluster：`{{.resource_name}}` | | results | array | result of groups | | +| total_results | number | total matched results before truncation | 18 | | results[0].threshold | array | | ["90"] | | results[0].priority | string | | high | | results[0].group_values | array | | ["cluster-xxx", "node-xxx"] | diff --git a/docs/content.zh/docs/reference/system/security/_index.md b/docs/content.zh/docs/reference/system/security/_index.md index c9a7a8b5..1ec51e0c 100644 --- a/docs/content.zh/docs/reference/system/security/_index.md +++ b/docs/content.zh/docs/reference/system/security/_index.md @@ -23,7 +23,7 @@ INFINI Console Security 默认是开启的，如果需要关闭，可以修改 c ```aidl web: enabled: true - embedding_api: true + embedding_api: false auth: enabled: false ui: diff --git a/docs/content.zh/docs/reference/system/security/token.md b/docs/content.zh/docs/reference/system/security/token.md new file mode 100644 index 00000000..b971cd8e --- /dev/null +++ b/docs/content.zh/docs/reference/system/security/token.md @@ -0,0 +1,54 @@ +--- +weight: 36 +title: Token 管理 +--- + +# Token 管理 + +## 简介 + +这里创建的 Token 是 **管理型 API Token**，用于访问受保护的 INFINI Console HTTP API，适合脚本、自动化任务或外部系统集成场景。 + +创建成功后，系统只会展示一次完整 Token，请立即复制并妥善保存。 + +> 这类 Token 与用户登录后产生的会话 Token 不同。会话 Token 主要用于 Console Web 登录态；这里创建的 Token 主要用于第三方系统或自动化程序长期调用 API。 + +## 使用方式 + +调用 Console API 时，在请求头中带上 `X-API-TOKEN`： + +```bash +X-API-TOKEN: +``` + +示例： + +```bash +curl -X GET "http://localhost:9000/account/profile" \ + -H "X-API-TOKEN: " +``` + +如果请求成功，将返回当前 Token 对应用户的账户信息。 + +## 使用场景 + +- 通过脚本调用 Console 的受保护接口 +- 在 CI/CD 或运维任务中访问 Console API +- 供外部系统以 HTTP 方式集成 Console 能力 + +## 权限说明 + +Token 绑定到创建它的用户身份，使用时继承该用户的访问权限。也就是说，Token 能访问哪些接口，取决于该用户本身拥有哪些平台权限和数据权限。 + +## 有效期和失效规则 + +- 创建时可以自定义有效期 +- 可以设置为指定到期时间，或设置为 **永不过期** +- 如果创建请求里未显式指定有效期，默认有效期为 **1 年** +- 删除对应 Token 后，将无法继续使用 + +## 注意事项 + +- Token 只在创建成功时完整展示一次，关闭弹窗后无法再次查看原文 +- 不要将 Token 提交到代码仓库、日志、截图或聊天记录中 +- 建议仅在必要的自动化场景中使用，并按需定期轮换 diff --git a/docs/content.zh/docs/troubleshooting/_index.md b/docs/content.zh/docs/troubleshooting/_index.md index 16fe11b2..0f728bda 100644 --- a/docs/content.zh/docs/troubleshooting/_index.md +++ b/docs/content.zh/docs/troubleshooting/_index.md @@ -75,6 +75,24 @@ POST /.infini_cluster/_update_by_query?conflicts=proceed } ``` +### 外部修改系统集群密码后 Console 无法启动 + +#### 问题描述 + +如果在 Easysearch / Elasticsearch 外部直接修改了 Console 系统集群账号密码，Console 本地保存的 `SYSTEM_CLUSTER_PASS` 会失效，启动时将无法连接系统集群。 + +#### 解决方案 + +在 Console 安装目录执行恢复命令，更新本地系统集群密码；如果用户名也一起变更，可以同时指定： + +```bash +./console recovery -pass 'new-password' +./console recovery -user admin -pass 'new-password' +printf '%s' 'new-password' | ./console recovery -stdin +``` + +执行完成后重启 Console 即可。 + ### 启动报错 ``` diff --git a/go.mod b/go.mod index d8bd6fd6..251ebc59 100644 --- a/go.mod +++ b/go.mod @@ -122,4 +122,7 @@ require ( gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + infini.sh/license v0.0.0 ) + +replace infini.sh/license => ../license diff --git a/info_api.go b/info_api.go new file mode 100644 index 00000000..5c1e611e --- /dev/null +++ b/info_api.go @@ -0,0 +1,52 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + console_common "infini.sh/console/common" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + framework_model "infini.sh/framework/core/model" + "infini.sh/framework/core/util" + "net/http" +) + +func init() { + api.HandleAPIMethod(api.GET, "/_info", sanitizedInfoAPIHandler) + api.HandleUIMethod(api.GET, "/_info", sanitizedInfoAPIHandler, api.RequireLogin()) +} + +func sanitizedInfoAPIHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + info := framework_model.GetInstanceInfo() + payload := console_common.SanitizeInstanceInfoMap(util.MapStr{ + "id": info.ID, + "name": info.Name, + "application": info.Application, + "labels": info.Labels, + "tags": info.Tags, + "description": info.Description, + "status": info.Status, + }) + api.DefaultAPI.WriteJSON(w, payload, http.StatusOK) +} diff --git a/main.go b/main.go index 5e5d2b01..75bb520d 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,8 @@ import ( _ "expvar" "fmt" api3 "infini.sh/console/modules/agent/api" + common2 "infini.sh/console/modules/agent/common" + uiapi "infini.sh/console/plugin/api" "infini.sh/console/plugin/api/email" "infini.sh/console/plugin/audit_log" "infini.sh/framework/core/api" @@ -36,6 +38,8 @@ import ( model2 "infini.sh/framework/core/model" "infini.sh/framework/core/util" elastic2 "infini.sh/framework/modules/elastic" + "os" + "strings" _ "time/tzdata" log "github.com/cihub/seelog" @@ -55,7 +59,7 @@ import ( "infini.sh/framework/core/module" "infini.sh/framework/core/orm" task1 "infini.sh/framework/core/task" - _ "infini.sh/framework/modules/api" + apimodule "infini.sh/framework/modules/api" "infini.sh/framework/modules/metrics" "infini.sh/framework/modules/pipeline" queue2 "infini.sh/framework/modules/queue/disk_queue" @@ -69,6 +73,80 @@ import ( var appConfig *config.AppConfig var appUI *UI +func lookupSystemClusterID() (string, bool) { + value := global.Lookup(elastic.GlobalSystemElasticsearchID) + systemID, ok := value.(string) + if !ok || systemID == "" { + return "", false + } + return systemID, true +} + +func getSystemClusterClient() (elastic.API, bool) { + systemID, ok := lookupSystemClusterID() + if !ok { + return nil, false + } + + client := elastic.GetClientNoPanic(systemID) + if client == nil { + return nil, false + } + return client, true +} + +func isSystemClusterRollupSupported(cfg *elastic.ElasticsearchConfig) bool { + if cfg == nil || cfg.Distribution != elastic.Easysearch || cfg.Version == "" { + return false + } + + version, err := util.ParseSemantic(cfg.Version) + if err != nil { + log.Warnf("failed to parse system cluster version [%s] for rollup support: %v", cfg.Version, err) + return false + } + + return version.AtLeast(util.MustParseSemantic("1.12.1")) +} + +func getSystemClusterAppSetting() interface{} { + client, ok := getSystemClusterClient() + if !ok { + return nil + } + + systemID, ok := lookupSystemClusterID() + if !ok { + return nil + } + + cfg := elastic.GetConfigNoPanic(systemID) + if cfg == nil { + return nil + } + + settings, err := client.GetClusterSettings(nil) + if err != nil { + log.Errorf("failed to get cluster settings with system cluster: %v", err) + return nil + } + + rollupEnabled, _ := util.GetMapValueByKeys([]string{"persistent", "rollup", "search", "enabled"}, settings) + rollupEnabledValue := false + switch v := rollupEnabled.(type) { + case string: + rollupEnabledValue = strings.EqualFold(v, "true") + case bool: + rollupEnabledValue = v + } + return map[string]interface{}{ + "distribution": cfg.Distribution, + "version": cfg.Version, + "rollup_enabled": rollupEnabledValue, + "rollup_supported": isSystemClusterRollupSupported(cfg), + } +} + func main() { terminalHeader := ("\n") terminalHeader += (" ___ ___ __ __ ___ __ __ \n") @@ -84,6 +162,15 @@ func main() { config.Version, config.BuildNumber, config.LastCommitLog, config.BuildDate, config.EOLDate, terminalHeader, terminalFooter) app.Init(nil) + if len(os.Args) > 1 && os.Args[1] == "recovery" { + if err := setup1.RunRecoveryCmd(os.Args[2:]); err != nil { + fmt.Println(err.Error()) + app.Shutdown() + os.Exit(1) + } + app.Shutdown() + return + } defer app.Shutdown() modules := []module.ModuleItem{} @@ -105,6 +192,7 @@ func main() { //load core modules first module.RegisterSystemModule(&setup1.Module{}) module.RegisterSystemModule(uiModule) + module.RegisterSystemModule(&apimodule.APIModule{}) if !global.Env().SetupRequired() { for _, v := range modules { @@ -157,76 +245,78 @@ func main() { orm.RegisterSchemaWithIndexName(model.Notification{}, "notification") orm.RegisterSchemaWithIndexName(model.EmailServer{}, "email-server") orm.RegisterSchemaWithIndexName(model2.Instance{}, "instance") + orm.RegisterSchemaWithIndexName(common2.PendingRegistrationToken{}, "agent-registration-token") orm.RegisterSchemaWithIndexName(api3.RemoteConfig{}, "configs") orm.RegisterSchemaWithIndexName(model.AuditLog{}, "audit-logs") orm.RegisterSchemaWithIndexName(host.HostInfo{}, "host") module.Start() + uiapi.RefreshConsoleSelfAPIProxyUIRoutes() - var initFunc = func() { + var initFunc = func(startDeferredModules bool) { // check cluster health before initialization, refuse to start if status is red - sysClusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) - client := elastic.GetClient(sysClusterID) - health, err := client.ClusterHealth(context.Background()) - if err != nil { - panic(fmt.Errorf("failed to get system cluster health: %v", err)) - } - if health != nil && health.Status == "red" { - panic(fmt.Errorf("system cluster health status is [red], please fix the cluster before starting")) + if err := setup1.EnsureSystemClusterBasicAuth(); err != nil { + panic(fmt.Errorf("failed to hydrate system cluster auth: %v", err)) } + client, ok := getSystemClusterClient() + if !ok { + log.Warn("skip system cluster post-initialization, system cluster is not available") + } else { + health, err := client.ClusterHealth(context.Background()) + if err != nil { + panic(fmt.Errorf("failed to get system cluster health: %v", err)) + } + if health != nil && health.Status == "red" { + panic(fmt.Errorf("system cluster health status is [red], please fix the cluster before starting")) + } - elastic2.InitTemplate(false) + elastic2.InitTemplate(false) + } - if global.Env().SetupRequired() { + if startDeferredModules { for k, v := range modules { log.Debugf("start module: %v", k) v.Value.Start() } } - task1.RunWithinGroup("initialize_alerting", func(ctx context.Context) error { - err := alerting2.InitTasks() - if err != nil { - log.Errorf("init alerting task error: %v", err) - } - return err - }) - task1.RunWithinGroup("initialize_email_server", func(ctx context.Context) error { - err := email.InitEmailServer() - if err != nil { - log.Errorf("init email server error: %v", err) - } - return err - }) - api.RegisterAppSetting("system_cluster", func() interface{} { - client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) - settings, err := client.GetClusterSettings(nil) - if err != nil { - log.Errorf("failed to get cluster settings with system cluster: %v", err) - return nil - } - - rollupEnabled, _ := util.GetMapValueByKeys([]string{"persistent", "rollup", "search", "enabled"}, settings) - rollupEnabledValue := false - if v, ok := rollupEnabled.(string); ok && v == "true" { - rollupEnabledValue = true - } - return map[string]interface{}{ - "rollup_enabled": rollupEnabledValue, - } - }) + if orm.HasHandler() { + task1.RunWithinGroup("initialize_alerting", func(ctx context.Context) error { + err := alerting2.InitTasks() + if err != nil { + log.Errorf("init alerting task error: %v", err) + } + return err + }) + task1.RunWithinGroup("initialize_email_server", func(ctx context.Context) error { + err := email.InitEmailServer() + if err != nil { + log.Errorf("init email server error: %v", err) + } + return err + }) + } else { + log.Warn("skip alerting and email initialization, ORM handler is not registered") + } + api.RegisterAppSetting("system_cluster", getSystemClusterAppSetting) } if !global.Env().SetupRequired() { - initFunc() + initFunc(false) } else { - setup1.RegisterSetupCallback(initFunc) + setup1.RegisterSetupCallback(func() { + initFunc(true) + }) } if !global.Env().SetupRequired() { - err := bootstrapRequirementCheck() - if err != nil { - panic(err) + if _, ok := lookupSystemClusterID(); !ok { + log.Warn("skip bootstrap requirement check, system cluster is not available") + } else { + err := bootstrapRequirementCheck() + if err != nil { + panic(err) + } } } diff --git a/model/alerting/alert.go b/model/alerting/alert.go index 38040650..e1d13db7 100644 --- a/model/alerting/alert.go +++ b/model/alerting/alert.go @@ -85,6 +85,7 @@ type AlertMessage struct { ID string `json:"id,omitempty" elastic_meta:"_id" elastic_mapping:"id: { type: keyword }"` Created time.Time `json:"created,omitempty" elastic_mapping:"created: { type: date }"` Updated time.Time `json:"updated,omitempty" elastic_mapping:"updated: { type: date }"` + RecoveredAt time.Time `json:"recovered_at,omitempty" elastic_mapping:"recovered_at: { type: date }"` RuleID string `json:"rule_id" elastic_mapping:"rule_id: { type: keyword,copy_to:search_text }"` ResourceID string `json:"resource_id" elastic_mapping:"resource_id: { type: keyword,copy_to:search_text }"` ResourceName string `json:"resource_name" elastic_mapping:"resource_name: { type: keyword,copy_to:search_text }"` diff --git a/model/alerting/resource.go b/model/alerting/resource.go index a8a1a66b..2ab37053 100644 --- a/model/alerting/resource.go +++ b/model/alerting/resource.go @@ -32,14 +32,15 @@ import ( ) type Resource struct { - ID string `json:"resource_id" elastic_mapping:"resource_id:{type:keyword}"` - Name string `json:"resource_name" elastic_mapping:"resource_name:{type:keyword}"` - Type string `json:"type" elastic_mapping:"type:{type:keyword}"` - Objects []string `json:"objects" elastic_mapping:"objects:{type:keyword,copy_to:search_text}"` - Filter FilterQuery `json:"filter,omitempty" elastic_mapping:"-"` - RawFilter map[string]interface{} `json:"raw_filter,omitempty" elastic_mapping:"raw_filter:{type:object,enabled:false}"` - TimeField string `json:"time_field,omitempty" elastic_mapping:"id:{type:keyword}"` - Context Context `json:"context"` + ID string `json:"resource_id" elastic_mapping:"resource_id:{type:keyword}"` + Name string `json:"resource_name" elastic_mapping:"resource_name:{type:keyword}"` + Type string `json:"type" elastic_mapping:"type:{type:keyword}"` + Objects []string `json:"objects" elastic_mapping:"objects:{type:keyword,copy_to:search_text}"` + Filter FilterQuery `json:"filter,omitempty" elastic_mapping:"-"` + RawFilter map[string]interface{} `json:"raw_filter,omitempty" elastic_mapping:"raw_filter:{type:object,enabled:false}"` + TimeField string `json:"time_field,omitempty" elastic_mapping:"id:{type:keyword}"` + IgnoreTimeFilter bool `json:"ignore_time_filter,omitempty" elastic_mapping:"ignore_time_filter:{type:boolean}"` + Context Context `json:"context"` } func (r Resource) Validate() error { diff --git a/model/alerting/rule.go b/model/alerting/rule.go index c5c327ca..760a7287 100644 --- a/model/alerting/rule.go +++ b/model/alerting/rule.go @@ -110,12 +110,13 @@ type NotificationConfig struct { } type RecoveryNotificationConfig struct { - Enabled bool `json:"enabled"` // channel enabled - Title string `json:"title"` //text template - Message string `json:"message"` // text template - AcceptTimeRange TimeRange `json:"accept_time_range,omitempty"` - Normal []Channel `json:"normal,omitempty"` - EventEnabled bool `json:"event_enabled"` + Enabled bool `json:"enabled"` // channel enabled + Title string `json:"title"` //text template + Message string `json:"message"` // text template + AcceptTimeRange TimeRange `json:"accept_time_range,omitempty"` + Normal []Channel `json:"normal,omitempty"` + EventEnabled bool `json:"event_enabled"` + IncrementalRecoveryEnabled bool `json:"incremental_recovery_enabled"` } type MessageTemplate struct { diff --git a/model/email_server.go b/model/email_server.go index 4f0179e0..5c8ebb71 100644 --- a/model/email_server.go +++ b/model/email_server.go @@ -41,6 +41,7 @@ type EmailServer struct { Host string `json:"host" elastic_mapping:"host:{type:keyword}"` Port int `json:"port" elastic_mapping:"port:{type:keyword}"` TLS bool `json:"tls" elastic_mapping:"tls:{type:keyword}"` + Sender string `json:"sender" elastic_mapping:"sender:{type:keyword}"` Auth *model.BasicAuth `json:"auth" elastic_mapping:"auth:{type:object}"` Enabled bool `json:"enabled" elastic_mapping:"enabled:{type:boolean}"` CredentialID string `json:"credential_id" elastic_mapping:"credential_id:{type:keyword}"` diff --git a/model/instance.go b/model/instance.go index 80b26b58..bc5e230d 100644 --- a/model/instance.go +++ b/model/instance.go @@ -34,6 +34,7 @@ import ( "net/http" "time" + agent_common "infini.sh/console/modules/agent/common" "infini.sh/framework/core/model" "infini.sh/framework/core/util" "infini.sh/framework/modules/pipeline" @@ -157,8 +158,8 @@ func (inst *TaskWorker) TryConnectWithTimeout(duration time.Duration) error { } func (inst *TaskWorker) doRequest(req *util.Request, resBody interface{}) error { - if inst.BasicAuth != nil && inst.BasicAuth.Username != "" { - req.SetBasicAuth(inst.BasicAuth.Username, inst.BasicAuth.Password.Get()) + if err := agent_common.ApplyInstanceRequestAuth(req, &inst.Instance); err != nil { + return err } result, err := util.ExecuteRequest(req) if err != nil { diff --git a/model/instance_test.go b/model/instance_test.go new file mode 100644 index 00000000..a8f3d4a2 --- /dev/null +++ b/model/instance_test.go @@ -0,0 +1,60 @@ +package model + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + agent_common "infini.sh/console/modules/agent/common" + framework_model "infini.sh/framework/core/model" + ucfg "infini.sh/framework/lib/go-ucfg" +) + +func TestTaskWorkerDoRequestUsesAccessCredentialToken(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer access-token" { + t.Fatalf("unexpected authorization header: %q", got) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + agent_common.RememberPreviousToken("cred-1", "access-token") + + inst := &TaskWorker{ + Instance: framework_model.Instance{ + Endpoint: server.URL, + AccessCredentialID: "cred-1", + }, + } + + if err := inst.TryConnectWithTimeout(time.Second); err != nil { + t.Fatalf("expected token auth to work, got %v", err) + } +} + +func TestTaskWorkerDoRequestFallsBackToBasicAuth(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, password, ok := r.BasicAuth() + if !ok || user != "managed_gateway" || password != "secret" { + t.Fatalf("unexpected basic auth credentials: ok=%v user=%q password=%q", ok, user, password) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + inst := &TaskWorker{ + Instance: framework_model.Instance{ + Endpoint: server.URL, + BasicAuth: &framework_model.BasicAuth{ + Username: "managed_gateway", + Password: ucfg.SecretString("secret"), + }, + }, + } + + if err := inst.TryConnectWithTimeout(time.Second); err != nil { + t.Fatalf("expected basic auth to work, got %v", err) + } +} diff --git a/modules/agent/api/elasticsearch.go b/modules/agent/api/elasticsearch.go index e6c32c74..f592c1b8 100644 --- a/modules/agent/api/elasticsearch.go +++ b/modules/agent/api/elasticsearch.go @@ -31,13 +31,21 @@ import ( "context" "errors" "fmt" + "net" "net/http" + "path/filepath" + "regexp" "runtime" + "sort" + "strings" "sync/atomic" "time" "github.com/buger/jsonparser" log "github.com/cihub/seelog" + console_common "infini.sh/console/common" + agent_common "infini.sh/console/modules/agent/common" + elasticapi "infini.sh/console/modules/elastic/api" "infini.sh/console/plugin/managed/server" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" @@ -45,7 +53,6 @@ import ( "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" - "infini.sh/framework/modules/elastic/adapter" "infini.sh/framework/modules/elastic/common" "infini.sh/framework/modules/elastic/metadata" ) @@ -83,6 +90,13 @@ func GetEnrolledNodesByAgent(instanceID string) (map[string]BindingItem, error) item.ClusterUUID = util.ToString(f["cluster_uuid"]) item.NodeUUID = nodeID + item.PublishAddress = strings.TrimSpace(util.ToString(f["publish_address"])) + item.EndpointSchema = strings.TrimSpace(strings.ToLower(util.ToString(f["endpoint_schema"]))) + item.PathLogs = util.ToString(f["path_logs"]) + item.LogsPaths = extractStringSlice(f["logs_paths"]) + if v, ok := f["collection_interval"]; ok { + item.CollectionInterval = util.InterfaceToInt(v) + } t, ok := v["updated"] if ok { @@ -102,7 +116,17 @@ func GetEnrolledNodesByAgent(instanceID string) (map[string]BindingItem, error) return ids, nil } -func refreshNodesInfo(instanceID, instanceEndpoint string) (*elastic.DiscoveryResult, error) { +func refreshNodesInfo(instanceID string) (*elastic.DiscoveryResult, error) { + instance := model.Instance{} + instance.ID = instanceID + exists, err := orm.GetV2(orm.NewContext(), &instance) + if err != nil { + return nil, fmt.Errorf("error on get agent instance: %w", err) + } + if !exists { + return nil, fmt.Errorf("agent instance [%s] not found", instanceID) + } + enrolledNodesByAgent, err := GetEnrolledNodesByAgent(instanceID) if err != nil { return nil, fmt.Errorf("error on get binding nodes info: %w", err) @@ -110,7 +134,7 @@ func refreshNodesInfo(instanceID, instanceEndpoint string) (*elastic.DiscoveryRe ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - nodesInfo, err := GetElasticsearchNodesViaAgent(ctxTimeout, instanceEndpoint) + nodesInfo, err := GetElasticsearchNodesViaAgent(ctxTimeout, &instance) if err != nil { //TODO return already biding nodes info ?? return nil, fmt.Errorf("error on get nodes info from agent: %w", err) @@ -140,7 +164,7 @@ func refreshNodesInfo(instanceID, instanceEndpoint string) (*elastic.DiscoveryRe if _, ok := newNodes[k]; !ok { client := elastic.GetClientNoPanic(v.ClusterID) if client == nil { - log.Error("client not found:", v.ClusterID) + log.Errorf("agent client not found for cluster [%s]", v.ClusterID) continue } status := "online" @@ -152,27 +176,39 @@ func refreshNodesInfo(instanceID, instanceEndpoint string) (*elastic.DiscoveryRe //get nodes information nodeInfos, err := metadata.GetNodeInformation(v.ClusterID, []string{v.NodeUUID}) if err != nil || len(nodeInfos) == 0 { - log.Error("node info not found:", v.ClusterID, ",", []string{v.NodeUUID}, ",", err, err != nil, len(nodeInfos) == 0) + if err != nil { + log.Errorf("failed to get fallback node info for cluster [%s], node [%s]: %v", v.ClusterID, v.NodeUUID, err) + } else { + log.Errorf("fallback node info not found for cluster [%s], node [%s]", v.ClusterID, v.NodeUUID) + } continue } //get node information nodeInfo, ok = nodeInfos[v.NodeUUID] if !ok { - log.Error("node info not found:", v.ClusterID, ",", v.NodeUUID, ",", err) + log.Errorf("fallback node info map missing cluster [%s], node [%s]", v.ClusterID, v.NodeUUID) continue } //get cluster information clusterInfo, err = metadata.GetClusterInformation(v.ClusterID) if err != nil || clusterInfo == nil { - log.Error("cluster info not found:", v.ClusterID, ",", err, clusterInfo == nil) + if err != nil { + log.Errorf("failed to get cluster info for cluster [%s]: %v", v.ClusterID, err) + } else { + log.Errorf("cluster info not found for cluster [%s]", v.ClusterID) + } continue } } else { - clusterInfo, err = adapter.ClusterVersion(elastic.GetMetadata(v.ClusterID)) + clusterInfo, err = console_common.ClusterVersion(elastic.GetMetadata(v.ClusterID)) if err != nil || clusterInfo == nil { - log.Error(err) + if err != nil { + log.Errorf("failed to resolve cluster version for cluster [%s]: %v", v.ClusterID, err) + } else { + log.Errorf("cluster version not found for cluster [%s]", v.ClusterID) + } continue } } @@ -203,7 +239,11 @@ func refreshNodesInfo(instanceID, instanceEndpoint string) (*elastic.DiscoveryRe } // get nodes info via agent -func GetElasticsearchNodesViaAgent(ctx context.Context, endpoint string) (*elastic.DiscoveryResult, error) { +func GetElasticsearchNodesViaAgent(ctx context.Context, instance *model.Instance) (*elastic.DiscoveryResult, error) { + if instance == nil || instance.ID == "" { + return nil, errors.New("invalid agent instance") + } + req := &util.Request{ Method: http.MethodGet, Path: "/elasticsearch/node/_discovery", @@ -211,7 +251,15 @@ func GetElasticsearchNodesViaAgent(ctx context.Context, endpoint string) (*elast } obj := elastic.DiscoveryResult{} - _, err := server.ProxyAgentRequest("elasticsearch", endpoint, req, &obj) + if shouldUseReverseChannelOnlyForInstance(instance) { + _, err := ProxyAgentRequestViaChannel(instance.ID, req, &obj) + if err != nil { + return nil, err + } + return &obj, nil + } + + _, err := proxyAgentRequest(instance, req, &obj) if err != nil { return nil, err } @@ -219,21 +267,71 @@ func GetElasticsearchNodesViaAgent(ctx context.Context, endpoint string) (*elast return &obj, nil } +func shouldFallbackToDirectAgentDiscovery(err error) bool { + return isAgentReverseChannelRecoverableError(err) +} + type BindingItem struct { //infini system assigned id - ClusterID string `json:"cluster_id"` - - ClusterUUID string `json:"cluster_uuid"` - NodeUUID string `json:"node_uuid"` + ClusterID string `json:"cluster_id"` + ClusterName string `json:"cluster_name,omitempty"` + + ClusterUUID string `json:"cluster_uuid"` + NodeUUID string `json:"node_uuid"` + PublishAddress string `json:"publish_address,omitempty"` + EndpointSchema string `json:"endpoint_schema,omitempty"` + NodeName string `json:"node_name,omitempty"` + PathHome string `json:"path_home,omitempty"` + PathLogs string `json:"path_logs"` + LogsPaths []string `json:"logs_paths"` + // CollectionInterval is the metrics collection interval in seconds; 0 means use the agent default (10s). + CollectionInterval int `json:"collection_interval,omitempty"` Updated int64 `json:"updated"` } -func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath string) (interface{}, error) { +type ClusterBinding struct { + ClusterID string `json:"cluster_id"` + LogsPaths []string `json:"logs_paths,omitempty"` +} - reqBody := util.MustToJSONBytes(util.MapStr{ - "logs_path": logsPath, - }) +// isLegacyLogsPathAgent returns true when the agent predates multi-path logs_path support. +func isLegacyLogsPathAgent(instance *model.Instance) bool { + if instance == nil { + return false + } + version := strings.TrimSpace(instance.Application.Version.VersionNumber) + if version == "" { + return false + } + parsed, err := util.ParseSemantic(version) + if err != nil { + parsed, err = util.ParseGeneric(version) + if err != nil { + return false + } + } + cmp, err := parsed.Compare(agent_common.LegacyAgentMaxVersion) + if err != nil { + return false + } + return cmp <= 0 +} + +func GetSearchLogFiles(ctx context.Context, instance *model.Instance, logsPaths []string) (interface{}, error) { + if len(logsPaths) == 0 { + return nil, fmt.Errorf("logs_path is not configured for this node") + } + + body := util.MapStr{} + if len(logsPaths) == 1 || isLegacyLogsPathAgent(instance) { + // Always send a plain string for single-path or legacy agents (≤1.31.0). + body["logs_path"] = logsPaths[0] + } else { + body["logs_path"] = logsPaths + } + + reqBody := util.MustToJSONBytes(body) req := &util.Request{ Method: http.MethodPost, @@ -243,18 +341,18 @@ func GetElasticLogFiles(ctx context.Context, instance *model.Instance, logsPath } resBody := map[string]interface{}{} - _, err := server.ProxyAgentRequest("elasticsearch", instance.GetEndpoint(), req, &resBody) + _, err := proxyAgentRequest(instance, req, &resBody) if err != nil { return nil, err } if resBody["success"] != true { - return nil, fmt.Errorf("get elasticsearch log files error: %v", resBody) + return nil, fmt.Errorf("get search log files error: %v", resBody) } return resBody["result"], nil } -func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, body interface{}) (interface{}, error) { +func GetSearchLogFileContent(ctx context.Context, instance *model.Instance, body interface{}) (interface{}, error) { req := &util.Request{ Method: http.MethodPost, Path: "/elasticsearch/logs/_read", @@ -262,12 +360,12 @@ func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, bod Body: util.MustToJSONBytes(body), } resBody := map[string]interface{}{} - _, err := server.ProxyAgentRequest("elasticsearch", instance.GetEndpoint(), req, &resBody) + _, err := proxyAgentRequest(instance, req, &resBody) if err != nil { return nil, err } if resBody["success"] != true { - return nil, fmt.Errorf("get elasticsearch log files error: %v", resBody["error"]) + return nil, fmt.Errorf("get search log files error: %v", resBody) } var hasMore bool if v, ok := resBody["EOF"].(bool); ok && !v { @@ -282,24 +380,24 @@ func GetElasticLogFileContent(ctx context.Context, instance *model.Instance, bod func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { clusterID := ps.MustGetParameter("id") nodeID := ps.MustGetParameter("node_id") - inst, pathLogs, err := getAgentByNodeID(clusterID, nodeID) + inst, logsPaths, err := getAgentByNodeID(clusterID, nodeID) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("failed to resolve agent for cluster [%s], node [%s]: %v", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(nodeID), err) return } if inst == nil { - log.Error(fmt.Sprintf("can not find agent by node [%s]", nodeID)) + log.Warnf("no agent associated with cluster [%s], node [%s]", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(nodeID)) h.WriteJSON(w, util.MapStr{ "success": false, "reason": "AGENT_NOT_FOUND", }, http.StatusOK) return } - logFiles, err := GetElasticLogFiles(nil, inst, pathLogs) + logFiles, err := GetSearchLogFiles(nil, inst, logsPaths) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("failed to get log files for cluster [%s], node [%s]: %v", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(nodeID), err) return } h.WriteJSON(w, util.MapStr{ @@ -311,10 +409,10 @@ func (h *APIHandler) getLogFilesByNode(w http.ResponseWriter, req *http.Request, func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { clusterID := ps.MustGetParameter("id") nodeID := ps.MustGetParameter("node_id") - inst, pathLogs, err := getAgentByNodeID(clusterID, nodeID) + inst, logsPaths, err := getAgentByNodeID(clusterID, nodeID) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("failed to resolve agent for cluster [%s], node [%s]: %v", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(nodeID), err) return } if inst == nil { @@ -324,28 +422,34 @@ func (h *APIHandler) getLogFileContent(w http.ResponseWriter, req *http.Request, reqBody := struct { FileName string `json:"file_name"` LogsPath string `json:"logs_path"` - Offset int `json:"offset"` + Offset int64 `json:"offset"` Lines int `json:"lines"` StartLineNumber int64 `json:"start_line_number"` + TailLines int `json:"tail_lines"` }{} err = h.DecodeJSON(req, &reqBody) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("failed to decode log content request for cluster [%s], node [%s]: %v", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(nodeID), err) + return + } + reqBody.LogsPath, err = pickAllowedLogsPath(logsPaths, reqBody.LogsPath) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("invalid logs path for cluster [%s], node [%s]: %v", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(nodeID), err) return } - reqBody.LogsPath = pathLogs - res, err := GetElasticLogFileContent(nil, inst, reqBody) + res, err := GetSearchLogFileContent(nil, inst, reqBody) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("failed to get log content for cluster [%s], node [%s]: %v", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(nodeID), err) return } h.WriteJSON(w, res, http.StatusOK) } -// instance, pathLogs -func getAgentByNodeID(clusterID, nodeID string) (*model.Instance, string, error) { +// instance, logsPaths +func getAgentByNodeID(clusterID, nodeID string) (*model.Instance, []string, error) { q := orm.Query{ Size: 1000, @@ -358,20 +462,30 @@ func getAgentByNodeID(clusterID, nodeID string) (*model.Instance, string, error) err, result := orm.Search(model.Setting{}, &q) if err != nil { - return nil, "", err + return nil, nil, err } nodeInfo, err := metadata.GetNodeConfig(clusterID, nodeID) if err != nil || nodeInfo == nil { - log.Error("node info is nil") - return nil, "", err + if err != nil { + log.Errorf("failed to get node config for cluster [%s], node [%s]: %v", clusterID, nodeID, err) + } else { + log.Errorf("node config not found for cluster [%s], node [%s]", clusterID, nodeID) + } + return nil, nil, err } - pathLogs := nodeInfo.Payload.NodeInfo.GetPathLogs() + logsPaths := normalizeLogsPaths(nil, nodeInfo.Payload.NodeInfo.GetPathLogs()) for _, row := range result.Result { v, ok := row.(map[string]interface{}) if ok { + if payload, ok := v["payload"].(map[string]interface{}); ok { + logsPaths = normalizeLogsPaths(extractStringSlice(payload["logs_paths"]), util.ToString(payload["path_logs"])) + if len(logsPaths) == 0 { + logsPaths = normalizeLogsPaths(nil, nodeInfo.Payload.NodeInfo.GetPathLogs()) + } + } x, ok := v["metadata"] if ok { @@ -383,119 +497,237 @@ func getAgentByNodeID(clusterID, nodeID string) (*model.Instance, string, error) if ok { inst := &model.Instance{} inst.ID = util.ToString(id) - _, err = orm.Get(inst) + _, err = orm.GetV2(orm.NewContext(), inst) if err != nil { - return nil, pathLogs, err + return nil, logsPaths, err } if inst.Name == "" { - return nil, pathLogs, nil + return nil, logsPaths, nil } - return inst, pathLogs, nil + return inst, logsPaths, nil } } } } } } - return nil, "", nil + return nil, nil, nil } type ClusterInfo struct { - ClusterIDs []string `json:"cluster_id"` + ClusterIDs []string `json:"cluster_id"` + Clusters []ClusterBinding `json:"clusters,omitempty"` } var autoEnrollRunning = atomic.Bool{} -func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - //{"cluster_id":["infini_default_system_cluster"]} - clusterInfo := ClusterInfo{} - if req.Method == "POST" { - bytes, err := h.GetRawBody(req) - if err != nil { - panic(err) +func normalizeClusterInfo(info ClusterInfo) ClusterInfo { + clusterIDs := make([]string, 0, len(info.ClusterIDs)+len(info.Clusters)) + seen := map[string]int{} + existingClusters := info.Clusters + + appendCluster := func(clusterID string, logsPaths []string) { + clusterID = strings.TrimSpace(clusterID) + if clusterID == "" { + return } - if len(bytes) > 0 { - util.FromJSONBytes(bytes, &clusterInfo) + logsPaths = normalizeLogsPaths(logsPaths, "") + if idx, exists := seen[clusterID]; exists { + if len(logsPaths) > 0 { + info.Clusters[idx].LogsPaths = logsPaths + } + return } + seen[clusterID] = len(info.Clusters) + clusterIDs = append(clusterIDs, clusterID) + info.Clusters = append(info.Clusters, ClusterBinding{ + ClusterID: clusterID, + LogsPaths: logsPaths, + }) } - if len(clusterInfo.ClusterIDs) <= 0 { - panic(errors.New("please select cluster to enroll")) + info.Clusters = make([]ClusterBinding, 0, len(info.Clusters)) + for _, clusterID := range info.ClusterIDs { + appendCluster(clusterID, nil) + } + for _, item := range existingClusters { + appendCluster(item.ClusterID, item.LogsPaths) + } + info.ClusterIDs = clusterIDs + return info +} + +func (info ClusterInfo) GetLogsPaths(clusterID string) []string { + for _, item := range info.Clusters { + if item.ClusterID == clusterID { + return normalizeLogsPaths(item.LogsPaths, "") + } + } + return nil +} + +func hydrateAutoEnrollClusterInfo(clusterInfo ClusterInfo) (ClusterInfo, error) { + if len(clusterInfo.ClusterIDs) == 0 { + return clusterInfo, nil + } + + clusters := make([]ClusterBinding, 0, len(clusterInfo.ClusterIDs)) + for _, clusterID := range clusterInfo.ClusterIDs { + clusters = append(clusters, ClusterBinding{ + ClusterID: clusterID, + }) + } + clusterInfo.Clusters = clusters + return clusterInfo, nil +} + +func getAutoEnrollClusterInfo(clusterInfo ClusterInfo) (ClusterInfo, error) { + clusterInfo = normalizeClusterInfo(clusterInfo) + if len(clusterInfo.ClusterIDs) > 0 { + return hydrateAutoEnrollClusterInfo(clusterInfo) + } + + q := &orm.Query{ + Size: 1000, + Conds: orm.And( + orm.Eq("metric_collection_mode", elastic.ModeAgent), + orm.Eq("enabled", true), + ), + } + err, res := orm.Search(&elastic.ElasticsearchConfig{}, q) + if err != nil { + return ClusterInfo{}, err + } + + ids := make([]string, 0, len(res.Result)) + for _, row := range res.Result { + item, ok := row.(map[string]interface{}) + if !ok { + continue + } + id := util.ToString(item["id"]) + if id != "" { + ids = append(ids, id) + } } + clusterInfo.ClusterIDs = ids + clusterInfo = normalizeClusterInfo(clusterInfo) + return hydrateAutoEnrollClusterInfo(clusterInfo) +} +func startAutoEnroll(clusterInfo ClusterInfo) error { + clusterInfo, err := getAutoEnrollClusterInfo(clusterInfo) + if err != nil { + return err + } + if len(clusterInfo.ClusterIDs) <= 0 { + return nil + } if autoEnrollRunning.Load() { - panic(errors.New("auto_enroll is already running in background")) + return errors.New("auto_enroll is already running in background") } autoEnrollRunning.Swap(true) - go func(clusterInfo ClusterInfo) { - defer func() { - autoEnrollRunning.Swap(false) - if !global.Env().IsDebug { - if r := recover(); r != nil { - var v string - switch r.(type) { - case error: - v = r.(error).Error() - case runtime.Error: - v = r.(runtime.Error).Error() - case string: - v = r.(string) - } - if v != "" { - log.Error(v) - } + go runAutoEnroll(clusterInfo) + return nil +} + +func runAutoEnroll(clusterInfo ClusterInfo) { + defer func() { + autoEnrollRunning.Swap(false) + if !global.Env().IsDebug { + if r := recover(); r != nil { + var v string + switch r.(type) { + case error: + v = r.(error).Error() + case runtime.Error: + v = r.(runtime.Error).Error() + case string: + v = r.(string) + } + if v != "" { + log.Errorf("auto enroll panic recovered: %s", v) } } - log.Debug("finish auto enroll") - }() - - log.Debug("start auto enroll") - //get instances - q := &orm.Query{Conds: orm.And(orm.Eq("application.name", "agent"))} - q.From = 0 - q.Size = 50000 - err, res := orm.Search(&model.Instance{}, q) - if err != nil { - log.Error(err) - return } + log.Trace("finish auto enroll") + }() + + log.Trace("start auto enroll") + q := &orm.Query{Conds: orm.And(orm.Eq("application.name", "agent"))} + q.From = 0 + q.Size = 50000 + err, res := orm.Search(&model.Instance{}, q) + if err != nil { + log.Errorf("failed to search agent instances during auto enroll: %v", err) + return + } - for _, v := range res.Result { - f, ok := v.(map[string]interface{}) - if ok { - instanceIDObj, ok1 := f["id"] - instanceEndpointObj, ok2 := f["endpoint"] - if ok1 && ok2 { - instanceID, ok1 := instanceIDObj.(string) - instanceEndpoint, ok2 := instanceEndpointObj.(string) - if ok1 && ok2 { - nodes, err := refreshNodesInfo(instanceID, instanceEndpoint) - if err != nil { - log.Error(err) - continue - } - log.Debugf("instance:%v,%v, has: %v nodes, %v unknown nodes", instanceID, instanceEndpoint, len(nodes.Nodes), len(nodes.UnknownProcess)) - if len(nodes.UnknownProcess) > 0 { - pids := h.bindInstanceToCluster(clusterInfo, nodes, instanceID, instanceEndpoint) - log.Infof("instance:%v,%v, success enroll %v nodes", instanceID, instanceEndpoint, len(pids)) - } + for _, v := range res.Result { + f, ok := v.(map[string]interface{}) + if !ok { + continue + } + instanceIDObj, ok1 := f["id"] + instanceEndpointObj, ok2 := f["endpoint"] + if !ok1 || !ok2 { + continue + } + instanceID, ok1 := instanceIDObj.(string) + instanceEndpoint, ok2 := instanceEndpointObj.(string) + if !ok1 || !ok2 { + continue + } + nodes, err := refreshNodesInfo(instanceID) + if err != nil { + log.Errorf("failed to refresh nodes for agent instance [%s] at [%s]: %s", instanceID, console_common.MaskLogEndpoint(instanceEndpoint), console_common.MaskLogError(err)) + continue + } + log.Tracef("instance:%v,%v, has: %v nodes, %v unknown nodes", instanceID, console_common.MaskLogEndpoint(instanceEndpoint), len(nodes.Nodes), len(nodes.UnknownProcess)) + if len(nodes.UnknownProcess) > 0 { + pids := bindInstanceToCluster(clusterInfo, nodes, instanceID, instanceEndpoint) + if len(pids) > 0 { + log.Infof("instance:%v,%v, success enroll %v nodes", instanceID, console_common.MaskLogEndpoint(instanceEndpoint), len(pids)) + } + } - if len(nodes.Nodes) > 0 { - for k, v := range nodes.Nodes { - log.Debug(k, v.Status, v.Enrolled) - if !v.Enrolled { - pids := h.bindInstanceToCluster(clusterInfo, nodes, instanceID, instanceEndpoint) - log.Infof("instance:%v,%v, success enroll %v nodes", instanceID, instanceEndpoint, len(pids)) - } - } - } + if len(nodes.Nodes) > 0 { + for k, v := range nodes.Nodes { + log.Tracef("node status, id=%s, status=%v, enrolled=%v", console_common.MaskLogToken(k), v.Status, v.Enrolled) + if !v.Enrolled { + pids := bindInstanceToCluster(clusterInfo, nodes, instanceID, instanceEndpoint) + if len(pids) > 0 { + log.Infof("instance:%v,%v, success enroll %v nodes", instanceID, console_common.MaskLogEndpoint(instanceEndpoint), len(pids)) } + break } } } + } +} - }(clusterInfo) +func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + //{"cluster_id":["infini_default_system_cluster"]} + clusterInfo := ClusterInfo{} + if req.Method == "POST" { + bytes, err := h.GetRawBody(req) + if err != nil { + panic(err) + } + if len(bytes) > 0 { + util.FromJSONBytes(bytes, &clusterInfo) + } + } + + clusterInfo = normalizeClusterInfo(clusterInfo) + if len(clusterInfo.ClusterIDs) <= 0 { + panic(errors.New("please select cluster to enroll")) + } + + if err := startAutoEnroll(clusterInfo); err != nil { + panic(err) + } //get all unknown nodes //check each process with cluster id @@ -510,7 +742,7 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque id := ps.MustGetParameter("instance_id") instance := model.Instance{} instance.ID = id - exists, err := orm.Get(&instance) + exists, err := orm.GetV2(orm.NewContext(), &instance) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -519,7 +751,7 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque return } - nodes, err := refreshNodesInfo(instance.ID, instance.GetEndpoint()) + nodes, err := refreshNodesInfo(instance.ID) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return @@ -536,7 +768,8 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque if len(bytes) > 0 { clusterInfo := ClusterInfo{} util.FromJSONBytes(bytes, &clusterInfo) - discoveredPIDs = h.bindInstanceToCluster(clusterInfo, nodes, instance.ID, instance.GetEndpoint()) + clusterInfo = normalizeClusterInfo(clusterInfo) + discoveredPIDs = bindInstanceToCluster(clusterInfo, nodes, instance.ID, instance.GetEndpoint()) } } @@ -553,67 +786,126 @@ func (h *APIHandler) discoveryESNodesInfo(w http.ResponseWriter, req *http.Reque } } + // Enrich enrolled nodes with collection_interval from binding settings. + enrolledNodes, _ := GetEnrolledNodesByAgent(id) + if len(enrolledNodes) > 0 { + // Build a response map so we can add the extra field per node. + enrichedNodes := make(map[string]util.MapStr, len(nodes.Nodes)) + for nodeID, nodeInfo := range nodes.Nodes { + m := util.MapStr{} + b := util.MustToJSONBytes(nodeInfo) + util.FromJSONBytes(b, &m) + if binding, ok := enrolledNodes[nodeID]; ok && binding.CollectionInterval > 0 { + m["collection_interval"] = binding.CollectionInterval + } + enrichedNodes[nodeID] = m + } + h.WriteJSON(w, util.MapStr{ + "nodes": enrichedNodes, + "unknown_process": nodes.UnknownProcess, + }, http.StatusOK) + return + } + h.WriteJSON(w, nodes, http.StatusOK) } -func (h *APIHandler) bindInstanceToCluster(clusterInfo ClusterInfo, nodes *elastic.DiscoveryResult, instanceID, instanceEndpoint string) map[int]*elastic.LocalNodeInfo { +// updateClusterCollectionInterval updates the metrics collection interval (in seconds) +// for all nodes of a specific cluster on the given agent instance. +func (h *APIHandler) updateClusterCollectionInterval(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + instanceID := ps.MustGetParameter("instance_id") + clusterID := ps.MustGetParameter("cluster_id") + + body := struct { + CollectionInterval int `json:"collection_interval"` + }{} + if err := h.DecodeJSON(req, &body); err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if body.CollectionInterval < 0 { + h.WriteError(w, "collection_interval must be >= 0", http.StatusBadRequest) + return + } + + enrolledNodes, err := GetEnrolledNodesByAgent(instanceID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + updated := 0 + for _, item := range enrolledNodes { + if item.ClusterID != clusterID { + continue + } + item.CollectionInterval = body.CollectionInterval + settings := NewNodeAgentSettings(instanceID, &item) + if err := orm.Save(&orm.Context{Refresh: "wait_for"}, settings); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + updated++ + } + + h.WriteJSON(w, util.MapStr{"acknowledged": true, "updated": updated}, http.StatusOK) +} + +func bindInstanceToCluster(clusterInfo ClusterInfo, nodes *elastic.DiscoveryResult, instanceID, instanceEndpoint string) map[int]*elastic.LocalNodeInfo { discoveredPIDs := map[int]*elastic.LocalNodeInfo{} if len(clusterInfo.ClusterIDs) > 0 { //try connect this node to cluster by using this cluster's agent credential for _, clusterID := range clusterInfo.ClusterIDs { + preparedConf, err := elasticapi.PrepareClusterForAgentCollection(clusterID) + if err != nil { + log.Errorf("failed to prepare cluster [%s] for agent collection: %v", console_common.MaskLogToken(clusterID), err) + continue + } meta := elastic.GetMetadata(clusterID) if meta != nil { states, err := elastic.GetClient(clusterID).GetClusterState() if err != nil || states == nil { - log.Error(err) + if err != nil { + log.Errorf("failed to get cluster state for cluster [%s]: %v", console_common.MaskLogToken(clusterID), err) + } else { + log.Errorf("cluster state is empty for cluster [%s]", console_common.MaskLogToken(clusterID)) + } continue } clusterUUID := states.ClusterUUID + auth, err := resolveEnrollAgentAuth(preparedConf) + if err != nil { + log.Errorf("failed to get agent credential for cluster [%s]: %v", console_common.MaskLogToken(clusterID), err) + continue + } + if !hasUsableAgentBasicAuth(auth) { + log.Errorf("cluster [%s] has no available agent credential", console_common.MaskLogToken(clusterID)) + continue + } - //no auth or agent auth configured - if meta.Config.AgentCredentialID != "" || meta.Config.CredentialID == "" { - auth, err := common.GetAgentBasicAuth(meta.Config) - if err != nil { - panic(err) - } - - for _, v := range nodes.Nodes { - if !v.Enrolled { - if v.NodeInfo != nil { - pid := v.NodeInfo.Process.Id - nodeHost := v.NodeInfo.GetHttpPublishHost() - nodeInfo := h.internalProcessBind(clusterID, clusterUUID, instanceID, instanceEndpoint, pid, nodeHost, auth) - if nodeInfo != nil { - discoveredPIDs[pid] = nodeInfo - } + for _, v := range nodes.Nodes { + if !v.Enrolled { + if v.NodeInfo != nil { + pid := v.NodeInfo.Process.Id + nodeHost := v.NodeInfo.GetHttpPublishHost() + nodeInfo := (&APIHandler{}).internalProcessBind(clusterID, clusterUUID, instanceID, instanceEndpoint, pid, nodeHost, auth, "") + if nodeInfo != nil { + discoveredPIDs[pid] = nodeInfo } } } + } - //try connect - for _, node := range nodes.UnknownProcess { - - pid := node.PID - - for _, v := range node.ListenAddresses { - - ip := v.IP - port := v.Port - - if util.ContainStr(ip, "::") { - ip = fmt.Sprintf("[%s]", ip) - } - - if util.ContainStr(ip, "*") { - ip = util.LocalAddress - } + for _, node := range nodes.UnknownProcess { + pid := node.PID - nodeHost := fmt.Sprintf("%s:%d", ip, port) - nodeInfo := h.internalProcessBind(clusterID, clusterUUID, instanceID, instanceEndpoint, pid, nodeHost, auth) - if nodeInfo != nil { - discoveredPIDs[pid] = nodeInfo - } + for _, v := range prioritizeListenAddresses(node.ListenAddresses) { + nodeHost := fmt.Sprintf("%s:%d", normalizeListenHostIP(v.IP), v.Port) + nodeInfo := (&APIHandler{}).internalProcessBind(clusterID, clusterUUID, instanceID, instanceEndpoint, pid, nodeHost, auth, node.Cmdline) + if nodeInfo != nil { + discoveredPIDs[pid] = nodeInfo + break } } } @@ -623,28 +915,88 @@ func (h *APIHandler) bindInstanceToCluster(clusterInfo ClusterInfo, nodes *elast return discoveredPIDs } -func (h *APIHandler) internalProcessBind(clusterID, clusterUUID, instanceID, instanceEndpoint string, pid int, nodeHost string, auth *model.BasicAuth) *elastic.LocalNodeInfo { - success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "http", auth, instanceEndpoint) - if !success && tryAgain { - //try https again - success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "https", auth, instanceEndpoint) +func normalizeListenHostIP(ip string) string { + ip = strings.TrimSpace(ip) + if ip == "" || util.ContainStr(ip, "*") || ip == util.LocalIpv6Address { + return util.LocalAddress } - log.Debug(clusterUUID, nodeHost, instanceEndpoint, success, tryAgain, nodeInfo) + rawIP := strings.Trim(ip, "[]") + if parsed := net.ParseIP(rawIP); parsed != nil { + if parsed.IsUnspecified() || parsed.IsLoopback() { + return util.LocalAddress + } + if parsed.To4() == nil { + return fmt.Sprintf("[%s]", rawIP) + } + return rawIP + } + + if strings.Contains(ip, ":") && !strings.HasPrefix(ip, "[") { + return fmt.Sprintf("[%s]", ip) + } + return ip +} + +func prioritizeListenAddresses(addresses []model.ListenAddr) []model.ListenAddr { + if len(addresses) <= 1 { + return addresses + } + + items := append([]model.ListenAddr(nil), addresses...) + sort.SliceStable(items, func(i, j int) bool { + left := listenAddressPriority(items[i]) + right := listenAddressPriority(items[j]) + if left != right { + return left < right + } + return items[i].Port < items[j].Port + }) + return items +} + +func listenAddressPriority(addr model.ListenAddr) int { + switch addr.Port { + case 9200: + return 0 + case 443, 80: + return 1 + case 9300: + return 3 + default: + return 2 + } +} + +func (h *APIHandler) internalProcessBind(clusterID, clusterUUID, instanceID, instanceEndpoint string, pid int, nodeHost string, auth *model.BasicAuth, cmdline string) *elastic.LocalNodeInfo { + endpointSchema := "https" + success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "https", auth, instanceID) + if !success && tryAgain { + // fallback to http for clusters that expose plain-text HTTP only. + success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "http", auth, instanceID) + if success { + endpointSchema = "http" + } + } if success { - log.Debug("connect to es node success:", nodeHost, ", pid: ", pid) + log.Tracef("connect to es node success: cluster_uuid=%s, node=%s, instance=%s", console_common.MaskLogToken(clusterUUID), console_common.MaskLogHost(nodeHost), console_common.MaskLogEndpoint(instanceEndpoint)) if nodeInfo.ClusterInfo.ClusterUUID != clusterUUID { - log.Info("cluster uuid not match, cluster id: ", clusterID, ", cluster uuid: ", clusterUUID, ", node cluster uuid: ", nodeInfo.ClusterInfo.ClusterUUID) + log.Debugf("cluster uuid not match, cluster=%s, cluster_uuid=%s, node_cluster_uuid=%s", console_common.MaskLogToken(clusterID), console_common.MaskLogToken(clusterUUID), console_common.MaskLogToken(nodeInfo.ClusterInfo.ClusterUUID)) return nil } //enroll this node item := BindingItem{ - ClusterID: clusterID, - ClusterUUID: nodeInfo.ClusterInfo.ClusterUUID, - NodeUUID: nodeInfo.NodeUUID, + ClusterID: clusterID, + ClusterUUID: nodeInfo.ClusterInfo.ClusterUUID, + NodeUUID: nodeInfo.NodeUUID, + PublishAddress: strings.TrimSpace(nodeHost), + EndpointSchema: endpointSchema, + PathHome: extractNodePathHome(nodeInfo.NodeInfo), + LogsPaths: deriveLogsPathsFromCmdline(cmdline, extractNodePathHome(nodeInfo.NodeInfo)), } + item.PathLogs = firstString(item.LogsPaths) settings := NewNodeAgentSettings(instanceID, &item) err := orm.Save(&orm.Context{ @@ -657,15 +1009,17 @@ func (h *APIHandler) internalProcessBind(clusterID, clusterUUID, instanceID, ins } return nodeInfo } + + log.Tracef("failed to connect to es node via agent proxy: cluster_uuid=%s, node=%s, instance=%s, retry_https=%v", console_common.MaskLogToken(clusterUUID), console_common.MaskLogHost(nodeHost), console_common.MaskLogEndpoint(instanceEndpoint), tryAgain) return nil } -func (h *APIHandler) getESNodeInfoViaProxy(esHost string, esSchema string, auth *model.BasicAuth, endpoint string) (success, tryAgain bool, info *elastic.LocalNodeInfo) { +func (h *APIHandler) getESNodeInfoViaProxy(esHost string, esSchema string, auth *model.BasicAuth, instanceID string) (success, tryAgain bool, info *elastic.LocalNodeInfo) { esConfig := elastic.ElasticsearchConfig{Host: esHost, Schema: esSchema, BasicAuth: auth} - return h.getESNodeInfoViaProxyWithConfig(&esConfig, auth, endpoint) + return h.getESNodeInfoViaProxyWithConfig(&esConfig, instanceID) } -func (h *APIHandler) getESNodeInfoViaProxyWithConfig(cfg *elastic.ElasticsearchConfig, auth *model.BasicAuth, endpoint string) (success, tryAgain bool, info *elastic.LocalNodeInfo) { +func (h *APIHandler) getESNodeInfoViaProxyWithConfig(cfg *elastic.ElasticsearchConfig, instanceID string) (success, tryAgain bool, info *elastic.LocalNodeInfo) { body := util.MustToJSONBytes(cfg) if cfg.BasicAuth != nil { body, _ = jsonparser.Set(body, []byte(`"`+cfg.BasicAuth.Password.Get()+`"`), "basic_auth", "password") @@ -678,23 +1032,27 @@ func (h *APIHandler) getESNodeInfoViaProxyWithConfig(cfg *elastic.ElasticsearchC Context: ctx, Body: body, } - if auth != nil { - req.SetBasicAuth(auth.Username, auth.Password.Get()) + + exists, instance, err := server.GetRuntimeInstanceByID(instanceID) + if err != nil || !exists || instance == nil { + if global.Env().IsDebug && err != nil { + log.Errorf("failed to load agent instance [%s] for node info proxy: %v", instanceID, err) + } + return false, true, nil } obj := elastic.LocalNodeInfo{} - res, err := server.ProxyAgentRequest("elasticsearch", endpoint, req, &obj) + res, err := proxyAgentRequest(instance, req, &obj) + if isForbiddenAgentReverseResult(res) { + return false, false, nil + } if err != nil { if global.Env().IsDebug { - log.Error(err) + log.Errorf("failed to proxy elasticsearch node info via agent [%s]: %v", instanceID, err) } return false, true, nil } - if res != nil && res.StatusCode == http.StatusForbidden { - return false, false, nil - } - if res != nil && res.StatusCode == http.StatusOK { node := elastic.LocalNodeInfo{} err := util.FromJSONBytes(res.Body, &node) @@ -707,6 +1065,14 @@ func (h *APIHandler) getESNodeInfoViaProxyWithConfig(cfg *elastic.ElasticsearchC return false, true, nil } +func shouldFallbackToDirectAgentNodeInfo(err error) bool { + return isAgentReverseChannelRecoverableError(err) +} + +func isForbiddenAgentReverseResult(res *util.Result) bool { + return res != nil && res.StatusCode == http.StatusForbidden +} + func NewClusterSettings(clusterID string) *model.Setting { settings := model.Setting{ Metadata: model.Metadata{ @@ -723,6 +1089,7 @@ func NewClusterSettings(clusterID string) *model.Setting { } func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { + logsPaths := normalizeLogsPaths(item.LogsPaths, item.PathLogs) settings := model.Setting{ Metadata: model.Metadata{ @@ -737,14 +1104,288 @@ func NewNodeAgentSettings(instanceID string, item *BindingItem) *model.Setting { } settings.Payload = util.MapStr{ - "cluster_id": item.ClusterID, - "cluster_uuid": item.ClusterUUID, - "node_uuid": item.NodeUUID, + "cluster_id": item.ClusterID, + "cluster_uuid": item.ClusterUUID, + "node_uuid": item.NodeUUID, + "publish_address": strings.TrimSpace(item.PublishAddress), + "endpoint_schema": normalizeSchema(item.EndpointSchema), + "path_logs": firstString(logsPaths), + "logs_paths": logsPaths, + "collection_interval": item.CollectionInterval, } return &settings } +func extractStringSlice(value interface{}) []string { + switch v := value.(type) { + case nil: + return nil + case []string: + return normalizeLogsPaths(v, "") + case []interface{}: + items := make([]string, 0, len(v)) + for _, item := range v { + items = append(items, util.ToString(item)) + } + return normalizeLogsPaths(items, "") + default: + return nil + } +} + +func normalizeLogsPaths(paths []string, fallback string) []string { + items := paths + if len(items) == 0 && fallback != "" { + items = []string{fallback} + } + + seen := map[string]struct{}{} + result := make([]string, 0, len(items)) + for _, item := range items { + item = strings.TrimSpace(item) + if item == "" { + continue + } + if _, exists := seen[item]; exists { + continue + } + seen[item] = struct{}{} + result = append(result, item) + } + return result +} + +var ( + cmdlinePathHomeRegx = regexp.MustCompile(`(?:^|\s)-D(?:es|opensearch)\.path\.home=([^\s]+)`) + cmdlinePathLogsRegx = regexp.MustCompile(`(?:^|\s)-D(?:es|opensearch)\.path\.logs=([^\s]+)`) + cmdlineGCFileRegx = regexp.MustCompile(`(?:^|\s)-Xlog:[^\s]*?file=([^\s]+)`) +) + +func deriveLogsPathsFromCmdline(cmdline, fallbackHome string) []string { + pathHome := extractCmdlineValue(cmdlinePathHomeRegx, cmdline) + if pathHome == "" { + pathHome = strings.TrimSpace(fallbackHome) + } + + currentLogsPath := extractCmdlineValue(cmdlinePathLogsRegx, cmdline) + if currentLogsPath == "" && pathHome != "" { + currentLogsPath = filepath.Join(pathHome, "logs") + } + + result := make([]string, 0, 2) + result = appendLogsDir(result, currentLogsPath, pathHome) + result = appendLogsFileDir(result, trimGCLogFileValue(extractCmdlineValue(cmdlineGCFileRegx, cmdline)), pathHome) + return result +} + +func extractCmdlineValue(reg *regexp.Regexp, cmdline string) string { + matches := reg.FindStringSubmatch(cmdline) + if len(matches) > 1 { + return trimCmdlinePathValue(matches[1]) + } + return "" +} + +func appendLogsDir(paths []string, value, base string) []string { + if len(paths) >= 2 { + return paths + } + resolved := resolveCmdlinePath(value, base) + if resolved == "" { + return paths + } + for _, item := range paths { + if item == resolved { + return paths + } + } + return append(paths, resolved) +} + +func appendLogsFileDir(paths []string, value, base string) []string { + resolved := resolveCmdlinePath(value, base) + if resolved == "" { + return paths + } + return appendLogsDir(paths, filepath.Dir(resolved), "") +} + +func resolveCmdlinePath(value, base string) string { + value = trimCmdlinePathValue(value) + if value == "" { + return "" + } + if !filepath.IsAbs(value) { + if base == "" { + return "" + } + value = filepath.Join(base, value) + } + return filepath.Clean(value) +} + +func trimCmdlinePathValue(value string) string { + return strings.Trim(strings.TrimSpace(value), `"'`) +} + +func trimGCLogFileValue(value string) string { + value = trimCmdlinePathValue(value) + if value == "" { + return "" + } + searchFrom := 0 + if len(value) > 1 && value[1] == ':' { + searchFrom = 2 + } + if idx := strings.Index(value[searchFrom:], ":"); idx >= 0 { + value = value[:searchFrom+idx] + } + return value +} + +func extractNodePathHome(nodeInfo *elastic.NodesInfo) string { + if nodeInfo == nil { + return "" + } + path, ok := nodeInfo.Settings["path"] + if !ok { + return "" + } + pathObj, ok := path.(map[string]interface{}) + if !ok { + return "" + } + return strings.TrimSpace(util.ToString(pathObj["home"])) +} + +func firstString(items []string) string { + if len(items) == 0 { + return "" + } + return items[0] +} + +func hasUsableAgentBasicAuth(auth *model.BasicAuth) bool { + return auth == nil || auth.Username != "" +} + +func resolveEnrollClusterID(clusterID, clusterUUID, clusterName string) (string, error) { + clusterID = strings.TrimSpace(clusterID) + clusterUUID = strings.TrimSpace(clusterUUID) + clusterName = strings.TrimSpace(clusterName) + + // Prefer the explicit cluster id from UI when it still exists and is enabled. + if clusterID != "" { + conf := &elastic.ElasticsearchConfig{} + conf.ID = clusterID + exists, err := orm.GetV2(orm.NewContext(), conf) + if err != nil { + return "", err + } + if exists && conf.Enabled { + if clusterUUID == "" || strings.TrimSpace(conf.ClusterUUID) == "" || strings.TrimSpace(conf.ClusterUUID) == clusterUUID { + return clusterID, nil + } + } + } + + if clusterUUID == "" { + return clusterID, nil + } + + q := orm.Query{ + Size: 100, + Conds: orm.And( + orm.Eq("cluster_uuid", clusterUUID), + orm.Eq("enabled", true), + ), + } + clusters := []elastic.ElasticsearchConfig{} + if err, _ := orm.SearchWithJSONMapper(&clusters, &q); err != nil { + return "", err + } + if len(clusters) == 0 { + return clusterID, nil + } + if clusterName != "" { + for _, item := range clusters { + if strings.TrimSpace(item.Name) == clusterName || strings.TrimSpace(item.RawName) == clusterName { + return item.ID, nil + } + } + } + return clusters[0].ID, nil +} + +func resolveEnrollAgentAuth(conf *elastic.ElasticsearchConfig) (*model.BasicAuth, error) { + auth, err := common.GetAgentBasicAuth(conf) + if err == nil { + return auth, nil + } + if strings.Contains(strings.ToLower(err.Error()), "record not found") { + // Missing credential record should not block no-auth clusters from enrollment. + // Continue with anonymous access and let node probe decide reachability. + return nil, nil + } + return nil, err +} + +func normalizeSchema(schema string) string { + switch strings.ToLower(strings.TrimSpace(schema)) { + case "https": + return "https" + case "http": + return "http" + default: + return "" + } +} + +func (h *APIHandler) getEnrollNodeInfo(item BindingItem, auth *model.BasicAuth, preparedConf *elastic.ElasticsearchConfig, instanceID string) (bool, string, *elastic.LocalNodeInfo) { + nodeHost := strings.TrimSpace(item.PublishAddress) + if nodeHost != "" { + success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "https", auth, instanceID) + if success { + return true, "https", nodeInfo + } + if !success && tryAgain { + success, _, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "http", auth, instanceID) + if success { + return true, "http", nodeInfo + } + } + return false, "", nodeInfo + } + success, _, nodeInfo := h.getESNodeInfoViaProxyWithConfig(preparedConf, instanceID) + if !success { + return false, "", nodeInfo + } + if preparedConf != nil { + if schema := normalizeSchema(preparedConf.Schema); schema != "" { + return true, schema, nodeInfo + } + } + return true, "http", nodeInfo +} + +func pickAllowedLogsPath(allowed []string, requested string) (string, error) { + allowed = normalizeLogsPaths(allowed, "") + if len(allowed) == 0 { + return "", fmt.Errorf("no logs path configured") + } + if requested == "" { + return allowed[0], nil + } + requested = strings.TrimSpace(requested) + for _, item := range allowed { + if item == requested { + return item, nil + } + } + return "", fmt.Errorf("invalid logs path: %s", requested) +} + func NewIndexSettings(clusterID, nodeID, agentID, indexName, indexID string) *model.Setting { settings := model.Setting{ @@ -812,23 +1453,64 @@ func (h *APIHandler) enrollESNode(w http.ResponseWriter, req *http.Request, ps h h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + resolvedClusterID, err := resolveEnrollClusterID(item.ClusterID, item.ClusterUUID, item.ClusterName) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if resolvedClusterID != "" { + item.ClusterID = resolvedClusterID + } + + if strings.TrimSpace(item.PublishAddress) == "" && item.ClusterID != "" && item.NodeUUID != "" { + if nodeCfg, nodeErr := metadata.GetNodeConfig(item.ClusterID, item.NodeUUID); nodeErr == nil && nodeCfg != nil && nodeCfg.Payload.NodeInfo != nil { + item.PublishAddress = strings.TrimSpace(nodeCfg.Payload.NodeInfo.GetHttpPublishHost()) + } + } //check if the cluster's agent credential is valid - meta := elastic.GetMetadata(item.ClusterID) - if meta == nil { - h.WriteError(w, "cluster not found", http.StatusInternalServerError) + preparedConf, err := elasticapi.PrepareClusterForAgentCollection(item.ClusterID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) return } //use agent credential to access the node - meta.Config.BasicAuth, _ = common.GetAgentBasicAuth(meta.Config) + preparedConf.BasicAuth, err = resolveEnrollAgentAuth(preparedConf) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !hasUsableAgentBasicAuth(preparedConf.BasicAuth) { + h.WriteError(w, "cluster has no available agent credential", http.StatusInternalServerError) + return + } - success, _, _ := h.getESNodeInfoViaProxyWithConfig(meta.Config, meta.Config.BasicAuth, instance.GetEndpoint()) + success, endpointSchema, nodeInfo := h.getEnrollNodeInfo(item, preparedConf.BasicAuth, preparedConf, instance.ID) - if success { - //update node's setting + if success && nodeInfo != nil { + if item.ClusterUUID != "" && nodeInfo.ClusterInfo.ClusterUUID != "" && item.ClusterUUID != nodeInfo.ClusterInfo.ClusterUUID { + h.WriteError(w, "cluster uuid not match", http.StatusInternalServerError) + return + } + if item.NodeUUID != "" && nodeInfo.NodeUUID != "" && item.NodeUUID != nodeInfo.NodeUUID { + h.WriteError(w, "node uuid not match", http.StatusInternalServerError) + return + } + item.ClusterUUID = nodeInfo.ClusterInfo.ClusterUUID + item.NodeUUID = nodeInfo.NodeUUID + if item.PublishAddress == "" && nodeInfo.NodeInfo != nil { + item.PublishAddress = strings.TrimSpace(nodeInfo.NodeInfo.GetHttpPublishHost()) + } + item.EndpointSchema = endpointSchema + if item.PathHome == "" { + item.PathHome = extractNodePathHome(nodeInfo.NodeInfo) + } + item.LogsPaths = deriveLogsPathsFromCmdline("", item.PathHome) + item.PathLogs = firstString(item.LogsPaths) + // Save will create the binding on first manual enroll and update it on subsequent enrolls. settings := NewNodeAgentSettings(instID, &item) - err = orm.Update(&orm.Context{ + err = orm.Save(&orm.Context{ Refresh: "wait_for", }, settings) if err != nil { diff --git a/modules/agent/api/elasticsearch_test.go b/modules/agent/api/elasticsearch_test.go new file mode 100644 index 00000000..37a47f90 --- /dev/null +++ b/modules/agent/api/elasticsearch_test.go @@ -0,0 +1,192 @@ +package api + +import ( + "testing" + + console_common "infini.sh/console/common" + agentservice "infini.sh/console/service/agent" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" +) + +func TestNormalizeClusterInfo(t *testing.T) { + info := normalizeClusterInfo(ClusterInfo{ + ClusterIDs: []string{"cluster-a", "cluster-a", "cluster-b"}, + Clusters: []ClusterBinding{ + {ClusterID: "cluster-a", LogsPaths: []string{" /var/log/es ", "/var/log/es"}}, + {ClusterID: "cluster-c", LogsPaths: []string{"/srv/logs"}}, + }, + }) + + if len(info.ClusterIDs) != 3 { + t.Fatalf("expected 3 unique cluster ids, got %d", len(info.ClusterIDs)) + } + if info.ClusterIDs[0] != "cluster-a" || info.ClusterIDs[1] != "cluster-b" || info.ClusterIDs[2] != "cluster-c" { + t.Fatalf("unexpected cluster id order: %#v", info.ClusterIDs) + } + if got := info.GetLogsPaths("cluster-a"); len(got) != 1 || got[0] != "/var/log/es" { + t.Fatalf("expected normalized logs path for cluster-a, got %#v", got) + } + if got := info.GetLogsPaths("cluster-b"); len(got) != 0 { + t.Fatalf("expected no logs paths for cluster-b, got %#v", got) + } +} + +func TestNewClusterAgentSettings(t *testing.T) { + settings := agentservice.NewClusterAgentSettings("cluster-a", []string{" /var/log/es ", "/srv/logs"}) + if settings.Metadata.Name != "agent" { + t.Fatalf("expected agent metadata name, got %q", settings.Metadata.Name) + } + if got := settings.Payload["path_logs"]; got != "/var/log/es" { + t.Fatalf("expected first logs path to be saved as path_logs, got %#v", got) + } + logsPaths, ok := settings.Payload["logs_paths"].([]string) + if !ok { + t.Fatalf("expected logs_paths slice, got %#v", settings.Payload["logs_paths"]) + } + if len(logsPaths) != 2 || logsPaths[1] != "/srv/logs" { + t.Fatalf("unexpected logs_paths payload: %#v", logsPaths) + } +} + +func TestHydrateAutoEnrollClusterInfoReturnsEmptyWithoutClusterIDs(t *testing.T) { + info, err := hydrateAutoEnrollClusterInfo(ClusterInfo{}) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if len(info.ClusterIDs) != 0 { + t.Fatalf("expected no cluster ids, got %#v", info.ClusterIDs) + } + if len(info.Clusters) != 0 { + t.Fatalf("expected no clusters, got %#v", info.Clusters) + } +} + +func TestDeriveLogsPathsFromCmdlineSinglePathWhenGCMatchesCurrent(t *testing.T) { + cmdline := `/usr/share/elasticsearch/jdk/bin/java -Xlog:gc*,gc+age=trace:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m -Des.path.home=/usr/share/elasticsearch` + got := deriveLogsPathsFromCmdline(cmdline, "") + if len(got) != 1 || got[0] != "/usr/share/elasticsearch/logs" { + t.Fatalf("unexpected logs paths: %#v", got) + } +} + +func TestDeriveLogsPathsFromCmdlineTwoPathsWhenGCDiffers(t *testing.T) { + cmdline := `/usr/share/elasticsearch/jdk/bin/java -Xlog:gc*,gc+age=trace:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m -Des.path.home=/usr/share/elasticsearch` + got := deriveLogsPathsFromCmdline(cmdline, "") + if len(got) != 2 { + t.Fatalf("expected 2 logs paths, got %#v", got) + } + if got[0] != "/usr/share/elasticsearch/logs" || got[1] != "/var/log/elasticsearch" { + t.Fatalf("unexpected logs paths: %#v", got) + } +} + +func TestShouldFallbackToDirectAgentDiscovery(t *testing.T) { + if !shouldFallbackToDirectAgentDiscovery(errAgentReverseChannelDisconnected) { + t.Fatal("expected disconnected reverse channel to fall back to direct discovery") + } + if !shouldFallbackToDirectAgentDiscovery(errAgentReverseChannelNotConnected) { + t.Fatal("expected not connected reverse channel to fall back to direct discovery") + } + if shouldFallbackToDirectAgentDiscovery(assertDiscoveryError("boom")) { + t.Fatal("did not expect non-recoverable errors to fall back to direct discovery") + } +} + +func TestShouldFallbackToDirectAgentNodeInfo(t *testing.T) { + if !shouldFallbackToDirectAgentNodeInfo(errAgentReverseChannelDisconnected) { + t.Fatal("expected disconnected reverse channel to fall back to direct node info") + } + if !shouldFallbackToDirectAgentNodeInfo(errAgentReverseChannelNotConnected) { + t.Fatal("expected not connected reverse channel to fall back to direct node info") + } + if shouldFallbackToDirectAgentNodeInfo(assertDiscoveryError("boom")) { + t.Fatal("did not expect non-recoverable errors to fall back to direct node info") + } +} + +func TestIsForbiddenAgentReverseResult(t *testing.T) { + if !isForbiddenAgentReverseResult(&util.Result{StatusCode: 403}) { + t.Fatal("expected forbidden reverse result to be detected") + } + if isForbiddenAgentReverseResult(&util.Result{StatusCode: 404}) { + t.Fatal("did not expect 404 reverse result to be treated as forbidden") + } + if isForbiddenAgentReverseResult(nil) { + t.Fatal("did not expect nil result to be treated as forbidden") + } +} + +func TestPrioritizeListenAddressesPrefersHTTPPorts(t *testing.T) { + got := prioritizeListenAddresses([]model.ListenAddr{ + {IP: "::", Port: 9300}, + {IP: "::", Port: 9200}, + {IP: "::", Port: 443}, + {IP: "::", Port: 8080}, + }) + + if len(got) != 4 { + t.Fatalf("unexpected listen address count: %#v", got) + } + + expectedPorts := []int{9200, 443, 8080, 9300} + for idx, port := range expectedPorts { + if got[idx].Port != port { + t.Fatalf("unexpected listen address order: %#v", got) + } + } +} + +func TestNormalizeListenHostIP(t *testing.T) { + testCases := []struct { + name string + input string + expect string + }{ + {name: "wildcard star", input: "*", expect: "127.0.0.1"}, + {name: "ipv4 unspecified", input: "0.0.0.0", expect: "127.0.0.1"}, + {name: "ipv6 unspecified", input: "::", expect: "127.0.0.1"}, + {name: "ipv6 loopback", input: "::1", expect: "127.0.0.1"}, + {name: "normal ipv4", input: "192.168.1.10", expect: "192.168.1.10"}, + {name: "normal ipv6", input: "2001:db8::10", expect: "[2001:db8::10]"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := normalizeListenHostIP(tc.input); actual != tc.expect { + t.Fatalf("unexpected normalized host, got %q want %q", actual, tc.expect) + } + }) + } +} + +func TestMaskLogHelpersRemainAvailable(t *testing.T) { + if got := console_common.MaskLogEndpoint("http://192.168.3.185:8080"); got != "http://***:8080" { + t.Fatalf("unexpected masked endpoint: %q", got) + } +} + +func TestHasUsableAgentBasicAuth(t *testing.T) { + if !hasUsableAgentBasicAuth(nil) { + t.Fatal("expected nil auth to be allowed for no-auth clusters") + } + if !hasUsableAgentBasicAuth(&model.BasicAuth{Username: "infini-agent"}) { + t.Fatal("expected normal auth to be allowed") + } + if hasUsableAgentBasicAuth(&model.BasicAuth{}) { + t.Fatal("expected empty username auth to be rejected") + } +} + +func TestGetEnrolledNodeInfoUsesPublishAddress(t *testing.T) { + item := BindingItem{PublishAddress: "172.25.0.2:9200"} + if item.PublishAddress != "172.25.0.2:9200" { + t.Fatalf("expected publish address to be preserved, got %q", item.PublishAddress) + } +} + +type assertDiscoveryError string + +func (e assertDiscoveryError) Error() string { + return string(e) +} diff --git a/modules/agent/api/host.go b/modules/agent/api/host.go index 2491013c..ee5e1933 100644 --- a/modules/agent/api/host.go +++ b/modules/agent/api/host.go @@ -65,14 +65,14 @@ func (h *APIHandler) enrollHost(w http.ResponseWriter, req *http.Request, ps htt case "agent": obj := model.Instance{} obj.ID = hi.AgentID - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { continue } hostInfo = &host.HostInfo{} hostInfo.IP = hi.IP hostInfo.AgentID = hi.AgentID - err = orm.Create(nil, hostInfo) + err = orm.Create(orm.NewContext(), hostInfo) if err != nil { errors[hi.IP] = util.MapStr{ "error": err.Error(), @@ -158,7 +158,7 @@ func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, obj := model.Instance{} obj.ID = hostInfo.AgentID - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": hostInfo.AgentID, @@ -179,7 +179,7 @@ func (h *APIHandler) GetHostAgentInfo(w http.ResponseWriter, req *http.Request, func getHost(hostID string) (*host.HostInfo, error) { hostInfo := &host.HostInfo{} hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) + exists, err := orm.GetV2(orm.NewContext(), hostInfo) if err != nil { return nil, fmt.Errorf("get host info error: %w", err) } @@ -193,7 +193,7 @@ func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Requ hostID := ps.MustGetParameter("host_id") hostInfo := &host.HostInfo{} hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) + exists, err := orm.GetV2(orm.NewContext(), hostInfo) if err != nil { log.Error(err) h.WriteError(w, err.Error(), http.StatusInternalServerError) @@ -210,7 +210,7 @@ func (h *APIHandler) GetHostElasticProcess(w http.ResponseWriter, req *http.Requ obj := model.Instance{} obj.ID = hostInfo.AgentID - exists, err = orm.Get(&obj) + exists, err = orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": hostInfo.AgentID, diff --git a/modules/agent/api/init.go b/modules/agent/api/init.go index c5eaa74f..23c0a45a 100644 --- a/modules/agent/api/init.go +++ b/modules/agent/api/init.go @@ -28,10 +28,13 @@ package api import ( + "context" "infini.sh/console/core" "infini.sh/console/core/security/enum" "infini.sh/console/plugin/managed/server" + agentservice "infini.sh/console/service/agent" "infini.sh/framework/core/api" + "infini.sh/framework/core/task" ) type APIHandler struct { @@ -40,23 +43,44 @@ type APIHandler struct { func Init() { handler := APIHandler{} + registerAgentReverseChannel() api.HandleAPIMethod(api.POST, "/host/_enroll", handler.enrollHost) api.HandleAPIMethod(api.GET, "/host/:host_id/agent/info", handler.GetHostAgentInfo) api.HandleAPIMethod(api.GET, "/host/:host_id/processes", handler.GetHostElasticProcess) api.HandleAPIMethod(api.DELETE, "/host/:host_id", handler.deleteHost) + api.HandleAPIMethod(api.POST, "/agent/instance/stats", handler.RequirePermission(handler.getAgentInstanceStatus, enum.PermissionAgentInstanceRead)) //bind agent with nodes api.HandleAPIMethod(api.GET, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_discovery", handler.RequirePermission(handler.discoveryESNodesInfo, enum.PermissionAgentInstanceRead)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_enroll", handler.RequirePermission(handler.enrollESNode, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/:instance_id/node/_revoke", handler.RequirePermission(handler.revokeESNode, enum.PermissionAgentInstanceWrite)) + api.HandleAPIMethod(api.POST, "/instance/:instance_id/cluster/:cluster_id/_collection_interval", handler.RequirePermission(handler.updateClusterCollectionInterval, enum.PermissionAgentInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance/node/_auto_enroll", handler.RequirePermission(handler.autoEnrollESNode, enum.PermissionAgentInstanceWrite)) //get elasticsearch node logs, direct fetch or via stored logs(TODO) - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/logs/_list", handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionAgentInstanceRead)) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/node/:node_id/logs/_read", handler.RequirePermission(handler.getLogFileContent, enum.PermissionAgentInstanceRead)) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/node/:node_id/logs/_list", handler.RequireClusterPermission(handler.RequirePermission(handler.getLogFilesByNode, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchNodeRead))) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/node/:node_id/logs/_read", handler.RequireClusterPermission(handler.RequirePermission(handler.getLogFileContent, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchNodeRead))) server.RegisterConfigProvider(remoteConfigProvider) server.RegisterConfigProvider(dynamicAgentConfigProvider) + server.RegisterSecretProvider(agentSecretProvider) + agentservice.RegisterAutoEnrollCallback(func(clusterIDs []string) { + if err := startAutoEnroll(ClusterInfo{ClusterIDs: clusterIDs}); err != nil { + // ignore concurrent trigger errors here; manual/scheduled runs already cover the next pass + } + }) + task.RegisterScheduleTask(task.ScheduleTask{ + ID: "agent-auto-enroll-clusters", + Description: "auto enroll agent clusters", + Type: "interval", + Interval: "24h", + Singleton: true, + Task: func(ctx context.Context) { + if err := startAutoEnroll(ClusterInfo{}); err != nil { + // ignore concurrent trigger errors; the running task will complete the current scan + } + }, + }) } diff --git a/modules/agent/api/instance.go b/modules/agent/api/instance.go new file mode 100644 index 00000000..598e1241 --- /dev/null +++ b/modules/agent/api/instance.go @@ -0,0 +1,148 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +var proxyAgentRequestViaChannelFn = ProxyAgentRequestViaChannel +var proxyAgentRequestDirectFn = proxyAgentRequestDirect + +func (h *APIHandler) getAgentInstanceStatus(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + var instanceIDs []string + if err := h.DecodeJSON(req, &instanceIDs); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(instanceIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + result := util.MapStr{} + for _, instanceID := range instanceIDs { + result[instanceID] = util.MapStr{} + } + + q := orm.Query{} + q.RawQuery = util.MustToJSONBytes(util.MapStr{ + "size": len(instanceIDs), + "query": util.MapStr{ + "terms": util.MapStr{ + "_id": instanceIDs, + }, + }, + }) + + instances := []model.Instance{} + if err, _ := orm.SearchWithJSONMapper(&instances, &q); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + var ( + wg sync.WaitGroup + mu sync.Mutex + ) + + for i := range instances { + instance := instances[i] + wg.Add(1) + go func(inst model.Instance) { + defer wg.Done() + stats := util.MapStr{} + if fetchAgentInstanceStats(inst, &stats) { + mu.Lock() + result[inst.ID] = stats + mu.Unlock() + } + }(instance) + } + + wg.Wait() + h.WriteJSON(w, result, http.StatusOK) +} + +func fetchAgentInstanceStats(instance model.Instance, stats *util.MapStr) bool { + if IsAgentReverseChannelConnected(instance.ID) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + req := &util.Request{ + Method: http.MethodGet, + Path: "/stats", + Context: ctx, + } + if _, err := proxyAgentRequestViaChannelFn(instance.ID, req, stats); err == nil { + return true + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req := &util.Request{ + Method: http.MethodGet, + Path: "/stats", + Context: ctx, + } + if _, err := proxyAgentRequestDirectFn(&instance, req, stats); err == nil { + return true + } + + info, err := fetchAgentInstanceInfoDirect(&instance) + if err != nil { + return false + } + if stats != nil { + (*stats)["system"] = util.MapStr{} + if info != nil { + (*stats)["application"] = info.Application + } + } + return true +} + +func fetchAgentInstanceInfoDirect(instance *model.Instance) (*model.Instance, error) { + if instance == nil { + return nil, fmt.Errorf("instance is nil") + } + + for _, infoPath := range []string{"/agent/_info", "/_info"} { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + req := &util.Request{ + Method: http.MethodGet, + Path: infoPath, + Context: ctx, + } + obj := &model.Instance{} + res, err := proxyAgentRequestDirectFn(instance, req, obj) + cancel() + if err == nil { + return obj, nil + } + if !shouldFallbackAgentInfoPath(infoPath, res, err) { + return nil, err + } + } + + return nil, fmt.Errorf("agent info unavailable") +} + +func shouldFallbackAgentInfoPath(path string, res *util.Result, err error) bool { + if path != "/agent/_info" || err == nil { + return false + } + if res != nil { + return res.StatusCode == http.StatusNotFound + } + return true +} diff --git a/modules/agent/api/instance_test.go b/modules/agent/api/instance_test.go new file mode 100644 index 00000000..1f437de9 --- /dev/null +++ b/modules/agent/api/instance_test.go @@ -0,0 +1,62 @@ +package api + +import ( + "fmt" + "net/http" + "testing" + + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" +) + +func TestFetchAgentInstanceStatsFallsBackToAgentInfo(t *testing.T) { + originalChannelProxy := proxyAgentRequestViaChannelFn + originalDirectProxy := proxyAgentRequestDirectFn + t.Cleanup(func() { + proxyAgentRequestViaChannelFn = originalChannelProxy + proxyAgentRequestDirectFn = originalDirectProxy + }) + + proxyAgentRequestViaChannelFn = func(instanceID string, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + return nil, fmt.Errorf("reverse unavailable") + } + proxyAgentRequestDirectFn = func(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + switch req.Path { + case "/stats": + return &util.Result{StatusCode: http.StatusNotFound}, fmt.Errorf("stats not exposed on web listener") + case "/agent/_info": + obj, ok := responseObjectToUnMarshall.(*model.Instance) + if !ok { + t.Fatalf("expected model.Instance response object, got %T", responseObjectToUnMarshall) + } + obj.Application.Name = "agent" + return &util.Result{StatusCode: http.StatusOK}, nil + default: + return nil, fmt.Errorf("unexpected path %s", req.Path) + } + } + + stats := util.MapStr{} + instance := model.Instance{} + instance.ID = "agent-1" + instance.Application.Name = "agent" + ok := fetchAgentInstanceStats(instance, &stats) + if !ok { + t.Fatal("expected agent info fallback to mark instance available") + } + if _, exists := stats["system"]; !exists { + t.Fatal("expected system marker when agent info fallback succeeds") + } +} + +func TestShouldFallbackAgentInfoPath(t *testing.T) { + if !shouldFallbackAgentInfoPath("/agent/_info", &util.Result{StatusCode: http.StatusNotFound}, fmt.Errorf("not found")) { + t.Fatal("expected 404 on /agent/_info to trigger fallback") + } + if shouldFallbackAgentInfoPath("/_info", &util.Result{StatusCode: http.StatusNotFound}, fmt.Errorf("not found")) { + t.Fatal("did not expect /_info fallback") + } + if shouldFallbackAgentInfoPath("/agent/_info", &util.Result{StatusCode: http.StatusUnauthorized}, fmt.Errorf("unauthorized")) { + t.Fatal("did not expect non-404 response to trigger fallback") + } +} diff --git a/modules/agent/api/proxy.go b/modules/agent/api/proxy.go new file mode 100644 index 00000000..483b07f1 --- /dev/null +++ b/modules/agent/api/proxy.go @@ -0,0 +1,77 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + + agent_common "infini.sh/console/modules/agent/common" + "infini.sh/console/plugin/managed/server" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" +) + +func shouldAttemptAgentReverseProxy(instance *model.Instance, req *util.Request, connected bool) bool { + return instance != nil && + req != nil && + connected && + strings.EqualFold(instance.Application.Name, "agent") +} + +func agentInstanceProxyProvider(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, bool, error) { + if !shouldAttemptAgentReverseProxy(instance, req, IsAgentReverseChannelConnected(instance.ID)) { + return nil, false, nil + } + + res, err := ProxyAgentRequestViaChannel(instance.ID, req, responseObjectToUnMarshall) + if shouldFallbackToDirectAgentProxy(res, err) { + return nil, false, nil + } + return res, true, err +} + +func shouldFallbackToDirectAgentProxy(res *util.Result, err error) bool { + if err == nil { + return false + } + if res != nil && res.StatusCode == http.StatusNotFound { + return true + } + return isAgentReverseChannelRecoverableError(err) +} + +func proxyAgentRequestDirect(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + if instance == nil { + return nil, fmt.Errorf("instance is nil") + } + if req == nil { + return nil, fmt.Errorf("request is nil") + } + if err := agent_common.ApplyInstanceRequestAuth(req, instance); err != nil { + return nil, err + } + endpoint := agent_common.ResolveInstanceRequestEndpoint(instance, req.Path) + return server.ProxyAgentRequest("runtime", endpoint, req, responseObjectToUnMarshall) +} + +func proxyAgentRequest(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + if instance == nil { + return nil, fmt.Errorf("instance is nil") + } + if req == nil { + return nil, fmt.Errorf("request is nil") + } + + if shouldAttemptAgentReverseProxy(instance, req, IsAgentReverseChannelConnected(instance.ID)) { + res, err := ProxyAgentRequestViaChannel(instance.ID, req, responseObjectToUnMarshall) + if !shouldFallbackToDirectAgentProxy(res, err) { + return res, err + } + } + + return proxyAgentRequestDirect(instance, req, responseObjectToUnMarshall) +} + +func init() { + server.RegisterInstanceProxyProvider(agentInstanceProxyProvider) +} diff --git a/modules/agent/api/proxy_test.go b/modules/agent/api/proxy_test.go new file mode 100644 index 00000000..572af81d --- /dev/null +++ b/modules/agent/api/proxy_test.go @@ -0,0 +1,107 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "infini.sh/framework/core/env" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" +) + +func TestShouldAttemptAgentReverseProxy(t *testing.T) { + agentInstance := &model.Instance{Application: env.Application{Name: "agent"}} + otherInstance := &model.Instance{Application: env.Application{Name: "gateway"}} + + testCases := []struct { + name string + instance *model.Instance + req *util.Request + connected bool + expect bool + }{ + {name: "connected agent uses reverse for arbitrary path", instance: agentInstance, req: &util.Request{Method: http.MethodGet, Path: "/any/path"}, connected: true, expect: true}, + {name: "path no longer gates reverse", instance: agentInstance, req: &util.Request{Method: http.MethodPost, Path: "/totally/custom"}, connected: true, expect: true}, + {name: "disconnected agent skips reverse", instance: agentInstance, req: &util.Request{Method: http.MethodGet, Path: "/queue/stats"}, connected: false, expect: false}, + {name: "non agent skips reverse", instance: otherInstance, req: &util.Request{Method: http.MethodGet, Path: "/queue/stats"}, connected: true, expect: false}, + {name: "nil request skips reverse", instance: agentInstance, connected: true, expect: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := shouldAttemptAgentReverseProxy(tc.instance, tc.req, tc.connected); actual != tc.expect { + t.Fatalf("unexpected match result: %v", actual) + } + }) + } +} + +func TestShouldFallbackToDirectAgentProxy(t *testing.T) { + testCases := []struct { + name string + res *util.Result + err error + expect bool + }{ + {name: "no error", err: nil, expect: false}, + {name: "not found", res: &util.Result{StatusCode: http.StatusNotFound}, err: assertError("not found"), expect: true}, + {name: "disconnected", err: errAgentReverseChannelDisconnected, expect: true}, + {name: "not connected", err: errAgentReverseChannelNotConnected, expect: true}, + {name: "other error", err: assertError("boom"), expect: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := shouldFallbackToDirectAgentProxy(tc.res, tc.err); actual != tc.expect { + t.Fatalf("unexpected fallback result: %v", actual) + } + }) + } +} + +func TestProxyAgentRequestFallsBackToDirectWhenReverseChannelIsNotConnected(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/elasticsearch/logs/_list" { + t.Fatalf("unexpected request path: %s", r.URL.Path) + } + if r.Method != http.MethodPost { + t.Fatalf("unexpected request method: %s", r.Method) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"success":true,"result":["app.log"]}`)) + })) + defer server.Close() + + instance := &model.Instance{ + Endpoint: "http://agent-api.local:2900", + Services: []model.ServiceInfo{ + {Name: "api", Endpoint: "http://agent-api.local:2900"}, + {Name: "web", Endpoint: server.URL}, + }, + } + instance.ID = "agent-direct-only" + instance.Application.Name = "agent" + + resBody := map[string]interface{}{} + res, err := proxyAgentRequest(instance, &util.Request{ + Method: http.MethodPost, + Path: "/elasticsearch/logs/_list", + Body: []byte(`{"logs_path":"/var/log/elasticsearch"}`), + }, &resBody) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res == nil || res.StatusCode != http.StatusOK { + t.Fatalf("unexpected response: %#v", res) + } + if resBody["success"] != true { + t.Fatalf("expected success body, got %#v", resBody) + } +} + +type assertError string + +func (e assertError) Error() string { + return string(e) +} diff --git a/modules/agent/api/remote_config.go b/modules/agent/api/remote_config.go index ac5c10b6..3daa7dcd 100644 --- a/modules/agent/api/remote_config.go +++ b/modules/agent/api/remote_config.go @@ -31,18 +31,36 @@ import ( "bytes" "fmt" log "github.com/cihub/seelog" + console_common "infini.sh/console/common" + agent_common "infini.sh/console/modules/agent/common" "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" + "infini.sh/framework/core/keystore" "infini.sh/framework/core/kv" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" + keystore2 "infini.sh/framework/lib/keystore" "infini.sh/framework/modules/configs/common" common2 "infini.sh/framework/modules/elastic/common" metadata2 "infini.sh/framework/modules/elastic/metadata" + "net" + "net/url" + "regexp" + "sort" + "strings" "time" ) +const systemClusterPassKey = "SYSTEM_CLUSTER_PASS" +const systemClusterIngestPasswordKey = "SYSTEM_CLUSTER_INGEST_PASSWORD" + +var systemIngestSchemaLineRegexp = regexp.MustCompile(`(?m)^(\s*schema:\s*).*$`) +var systemIngestHostsLineRegexp = regexp.MustCompile(`(?m)^(\s*hosts:\s*).*$`) +var systemIngestTLSBlockRegexp = regexp.MustCompile(`(?ms)^\s*#\s*tls:\s*#for mTLS connection with config servers\s*\n^\s*#\s*enabled:\s*true\s*\n^\s*#\s*ca_file:\s*/xxx/ca\.crt\s*\n^\s*#\s*cert_file:\s*/xxx/client\.crt\s*\n^\s*#\s*key_file:\s*/xxx/client\.key\s*\n^\s*#\s*skip_insecure_verify:\s*false\s*$`) +var relayEntryBindingRegexp = regexp.MustCompile(`(?m)^\s*binding:\s*(.+?)\s*$`) +var relayIngestDialTimeout = net.DialTimeout + type RemoteConfig struct { orm.ORMObjectBase Metadata model.Metadata `json:"metadata" elastic_mapping:"metadata: { type: object }"` @@ -67,10 +85,17 @@ func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { } result := []*common.ConfigFile{} + var relayIngestHosts []string + if strings.EqualFold(strings.TrimSpace(instance.Application.Name), "agent") { + relayIngestHosts = listRelayGatewayIngestHosts() + } for _, row := range searchResult.Result { v, ok := row.(map[string]interface{}) if ok { + if shouldSkipGatewayConfigByType(instance, v) { + continue + } x, ok := v["payload"] if ok { f, ok := x.(map[string]interface{}) @@ -81,6 +106,8 @@ func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { item.Name = util.ToString(name) item.Location = util.ToString(f["location"]) item.Content = util.ToString(f["content"]) + item.Content = rewriteLegacyAgentConfigContent(instance, item.Content) + item.Content = rewriteAgentRelayIngestContent(instance, item.Location, item.Content, relayIngestHosts) item.Version, _ = util.ToInt64(util.ToString(f["version"])) item.Size = int64(len(item.Content)) item.Managed = true @@ -102,6 +129,330 @@ func remoteConfigProvider(instance model.Instance) []*common.ConfigFile { return result } +func listRelayGatewayIngestHosts() []string { + relayPort := discoverRelayGatewayIngestPort() + q := buildRelayGatewayInstancesQuery() + instances := []model.Instance{} + if err, _ := orm.SearchWithJSONMapper(&instances, &q); err != nil { + log.Errorf("failed to search relay gateways for ingest hosts: %v", err) + return nil + } + endpoints := make([]string, 0, len(instances)) + roleByHost := map[string]string{} + for _, instance := range instances { + endpoint := strings.TrimSpace(strings.TrimRight(instance.GetEndpoint(), "/")) + if endpoint == "" { + continue + } + endpoints = append(endpoints, endpoint) + host := relayGatewayIngestHostFromEndpoint(endpoint, relayPort) + if host == "" { + continue + } + roleByHost[host] = preferredRelayRole(roleByHost[host], normalizeRelayRoleLabel(instance.Labels["relay_role"])) + } + relayIngestHosts := normalizeRelayGatewayIngestHosts(endpoints, relayPort) + sortRelayHosts(relayIngestHosts, roleByHost) + reachableHosts := filterReachableRelayIngestHosts(relayIngestHosts, 800*time.Millisecond) + sortRelayHosts(reachableHosts, roleByHost) + if len(reachableHosts) == 0 && len(relayIngestHosts) > 0 { + log.Warnf("none of relay ingest hosts are reachable from console, fallback to discovered hosts: %v", relayIngestHosts) + return relayIngestHosts + } + return reachableHosts +} + +func normalizeRelayRoleLabel(role string) string { + switch strings.ToLower(strings.TrimSpace(role)) { + case "primary": + return "primary" + case "secondary": + return "secondary" + default: + return "" + } +} + +func relayRolePriority(role string) int { + switch normalizeRelayRoleLabel(role) { + case "primary": + return 0 + case "secondary": + return 1 + default: + return 2 + } +} + +func preferredRelayRole(current, next string) string { + if relayRolePriority(next) < relayRolePriority(current) { + return next + } + return current +} + +func sortRelayHosts(hosts []string, roleByHost map[string]string) { + sort.SliceStable(hosts, func(i, j int) bool { + iPriority := relayRolePriority(roleByHost[hosts[i]]) + jPriority := relayRolePriority(roleByHost[hosts[j]]) + if iPriority != jPriority { + return iPriority < jPriority + } + return hosts[i] < hosts[j] + }) +} + +func filterReachableRelayIngestHosts(hosts []string, timeout time.Duration) []string { + if len(hosts) == 0 { + return nil + } + if timeout <= 0 { + timeout = 800 * time.Millisecond + } + reachable := make([]string, 0, len(hosts)) + for _, host := range hosts { + host = strings.TrimSpace(host) + if host == "" { + continue + } + conn, err := relayIngestDialTimeout("tcp", host, timeout) + if err != nil { + if global.Env().IsDebug { + log.Debugf("relay ingest host is not reachable from console: host=%s, err=%v", host, err) + } + continue + } + _ = conn.Close() + reachable = append(reachable, host) + } + return reachable +} + +func buildRelayGatewayInstancesQuery() orm.Query { + queryDSL := util.MapStr{ + "size": 1000, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + {"term": util.MapStr{"application.name": "gateway"}}, + }, + "should": []util.MapStr{ + {"term": util.MapStr{"labels.service_type": "relay"}}, + {"term": util.MapStr{"metadata.labels.service_type": "relay"}}, + }, + "minimum_should_match": 1, + }, + }, + } + return orm.Query{ + RawQuery: util.MustToJSONBytes(queryDSL), + } +} + +func discoverRelayGatewayIngestPort() string { + q := orm.Query{ + Size: 100, + Conds: orm.And( + orm.Eq("metadata.category", "app_settings"), + orm.Eq("metadata.name", "gateway"), + orm.Eq("metadata.labels.service_type", "relay"), + ), + } + err, searchResult := orm.Search(RemoteConfig{}, &q) + if err != nil { + log.Errorf("failed to search relay gateway config docs for ingest port: %v", err) + return "8081" + } + for _, row := range searchResult.Result { + v, ok := row.(map[string]interface{}) + if !ok { + continue + } + payload, ok := v["payload"].(map[string]interface{}) + if !ok { + continue + } + if port := extractRelayGatewayIngestPort(util.ToString(payload["content"])); port != "" { + return port + } + } + return "8081" +} + +func extractRelayGatewayIngestPort(content string) string { + if content == "" { + return "" + } + matches := relayEntryBindingRegexp.FindAllStringSubmatch(content, -1) + for _, match := range matches { + if len(match) < 2 { + continue + } + binding := strings.TrimSpace(strings.Trim(match[1], `"'`)) + if binding == "" { + continue + } + if _, port, err := net.SplitHostPort(binding); err == nil && port != "" { + return port + } + if idx := strings.LastIndex(binding, ":"); idx > 0 && idx < len(binding)-1 { + return binding[idx+1:] + } + } + return "" +} + +func normalizeRelayGatewayIngestHosts(endpoints []string, relayPort string) []string { + if len(endpoints) == 0 { + return nil + } + if relayPort == "" { + relayPort = "8081" + } + result := make([]string, 0, len(endpoints)) + seen := map[string]struct{}{} + for _, endpoint := range endpoints { + host := relayGatewayIngestHostFromEndpoint(endpoint, relayPort) + if host == "" { + continue + } + if _, exists := seen[host]; exists { + continue + } + seen[host] = struct{}{} + result = append(result, host) + } + return result +} + +func relayGatewayIngestHostFromEndpoint(endpoint, relayPort string) string { + normalized := strings.TrimSpace(strings.TrimRight(endpoint, "/")) + if normalized == "" { + return "" + } + parseTarget := normalized + if !strings.Contains(parseTarget, "://") { + parseTarget = "http://" + parseTarget + } + parsed, err := url.Parse(parseTarget) + if err != nil || parsed == nil { + return "" + } + host := strings.TrimSpace(parsed.Hostname()) + if host == "" { + return "" + } + return net.JoinHostPort(host, relayPort) +} + +func rewriteAgentRelayIngestContent(instance model.Instance, location, content string, relayIngestHosts []string) string { + if content == "" || len(relayIngestHosts) == 0 { + return content + } + if !strings.EqualFold(strings.TrimSpace(instance.Application.Name), "agent") { + return content + } + if !strings.EqualFold(strings.TrimSpace(location), "system_ingest_config.yml") { + return content + } + rewritten := systemIngestSchemaLineRegexp.ReplaceAllString(content, `${1}`+util.MustToJSON("https")) + rewritten = systemIngestHostsLineRegexp.ReplaceAllString(rewritten, `${1}`+string(util.MustToJSONBytes(relayIngestHosts))) + rewritten = systemIngestTLSBlockRegexp.ReplaceAllString(rewritten, strings.Join([]string{ + " tls: #for mTLS connection with config servers", + " enabled: true", + " ca_file: config/ca.crt", + " cert_file: config/client.crt", + " key_file: config/client.key", + " skip_domain_verify: true", + " skip_insecure_verify: false", + }, "\n")) + return rewritten +} + +func shouldSkipGatewayConfigByType(instance model.Instance, configDoc map[string]interface{}) bool { + if !strings.EqualFold(strings.TrimSpace(instance.Application.Name), "gateway") { + return false + } + instanceType := resolveGatewayServiceType(instance.Labels) + if instanceType == "" { + return false + } + configType := extractGatewayTypeFromConfigDoc(configDoc) + if configType == "" { + configType = inferGatewayTypeFromPayloadLocation(configDoc) + } + if configType == "" || configType == "_all" { + return false + } + return configType != instanceType +} + +func resolveGatewayServiceType(labels map[string]string) string { + if labels == nil { + return "" + } + return normalizeGatewayTypeLabel(labels["service_type"]) +} + +func normalizeGatewayTypeLabel(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "relay": + return "relay" + case "migration": + return "migration" + default: + return "" + } +} + +func extractGatewayTypeFromConfigDoc(configDoc map[string]interface{}) string { + metadata, ok := configDoc["metadata"].(map[string]interface{}) + if !ok { + return "" + } + labels, ok := metadata["labels"].(map[string]interface{}) + if !ok { + return "" + } + return normalizeGatewayTypeLabel(util.ToString(labels["service_type"])) +} + +func inferGatewayTypeFromPayloadLocation(configDoc map[string]interface{}) string { + payload, ok := configDoc["payload"].(map[string]interface{}) + if !ok { + return "" + } + location := strings.ToLower(strings.TrimSpace(util.ToString(payload["location"]))) + switch location { + case "relay.yml": + return "relay" + case "migration.yml": + return "migration" + default: + return "" + } +} + +func rewriteLegacyAgentConfigContent(instance model.Instance, content string) string { + if content == "" { + return content + } + + if instance.Application.Name != "agent" && instance.Application.Name != "gateway" { + return content + } + + if !strings.Contains(content, "$[[SETUP_AGENT_PASSWORD]]") { + return content + } + + return strings.ReplaceAll( + content, + "$[[SETUP_AGENT_PASSWORD]]", + fmt.Sprintf("$[[keystore.%s]]", getSystemClusterIngestSecretKey()), + ) +} + func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { if instance.Application.Name != "agent" { @@ -143,7 +494,7 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { panic(err) } latestTimestamp = time.Now().Unix() - log.Infof("hash: %v vs %v, update version to current timestamp: %v", string(v), hash, latestTimestamp) + log.Tracef("agent config hash changed for [%s]: %s -> %s, version=%v", instance.ID, console_common.MaskLogToken(string(v)), console_common.MaskLogToken(hash), latestTimestamp) } cfg.Size = int64(len(cfg.Content)) @@ -156,13 +507,129 @@ func dynamicAgentConfigProvider(instance model.Instance) []*common.ConfigFile { return result } +func agentSecretProvider(instance model.Instance) *common.Secrets { + if instance.Application.Name != "agent" && instance.Application.Name != "gateway" { + return nil + } + + secrets := &common.Secrets{Keystore: map[string]common.KeystoreValue{}} + appendKeystoreSecret(secrets, getSystemClusterIngestSecretKey()) + appendTokenCredentialSecret(secrets, instance.ManagerCredentialID, agent_common.AgentManagerTokenKey()) + if instance.Application.Name == "agent" { + appendTokenCredentialSecret(secrets, instance.AccessCredentialID, agent_common.AgentAccessTokenKey()) + } + + if instance.Application.Name == "agent" { + ids, err := GetEnrolledNodesByAgent(instance.ID) + if err != nil { + panic(err) + } + + for _, v := range ids { + auth, err := getAgentBasicAuth(v.ClusterID) + if err != nil { + log.Error(err) + continue + } + if auth == nil { + continue + } + secrets.Keystore[getAgentPasswordKey(v.ClusterID)] = common.KeystoreValue{ + Type: "plaintext", + Value: auth.Password.Get(), + } + } + } + + if len(secrets.Keystore) == 0 { + return nil + } + return secrets +} + +func getSystemClusterIngestSecretKey() string { + systemClusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + + if metadata := elastic.GetMetadata(systemClusterID); metadata != nil && metadata.Config != nil { + if metadata.Config.Distribution == elastic.Easysearch { + return systemClusterIngestPasswordKey + } + return systemClusterPassKey + } + + if cfg := elastic.GetConfigNoPanic(systemClusterID); cfg != nil { + if cfg.Distribution == elastic.Easysearch { + return systemClusterIngestPasswordKey + } + return systemClusterPassKey + } + + return systemClusterPassKey +} + +func appendKeystoreSecret(secrets *common.Secrets, key string) { + if secrets == nil || key == "" { + return + } + value, err := keystore.GetValue(key) + if err == keystore2.ErrKeyDoesntExists { + return + } + if err != nil { + log.Error(err) + return + } + secrets.Keystore[key] = common.KeystoreValue{ + Type: "plaintext", + Value: string(value), + } +} + +func appendTokenCredentialSecret(secrets *common.Secrets, credentialID, keystoreKey string) { + if secrets == nil || credentialID == "" || keystoreKey == "" { + return + } + value, err := agent_common.GetTokenCredentialValue(credentialID) + if err != nil { + log.Error(err) + return + } + if value == "" { + return + } + secrets.Keystore[keystoreKey] = common.KeystoreValue{ + Type: "plaintext", + Value: value, + } +} + +func getAgentPasswordKey(clusterID string) string { + // One agent can bind multiple clusters, so the synced keystore entry must stay cluster-scoped. + return fmt.Sprintf("%s_password", clusterID) +} + +func getAgentBasicAuth(clusterID string) (*model.BasicAuth, error) { + metadata := elastic.GetMetadata(clusterID) + if metadata == nil || metadata.Config == nil || metadata.Config.AgentCredentialID == "" { + return nil, nil + } + + credential, err := common2.GetCredential(metadata.Config.AgentCredentialID) + if err != nil { + return nil, err + } + + return credential.DecodeBasicAuth() +} + func getAgentIngestConfigs(instance string, items map[string]BindingItem) (string, string) { if instance == "" { panic("instance id is empty") } - buffer := bytes.NewBuffer([]byte("configs.template: ")) + elasticBuffer := bytes.NewBufferString("elasticsearch:") + pipelineBuffer := bytes.NewBufferString("\npipeline:") //sort items newItems := []util.KeyValue{} @@ -208,22 +675,12 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin distribution = metadata.Config.Distribution clusterName = metadata.Config.Name - if metadata.Config.AgentCredentialID != "" { - credential, err := common2.GetCredential(metadata.Config.AgentCredentialID) - if err != nil { - log.Error(err) - continue - } - var dv interface{} - dv, err = credential.Decode() - if err != nil { - log.Error(err) - continue - } - if auth, ok := dv.(model.BasicAuth); ok { - username = auth.Username - password = auth.Password.Get() - } + if auth, err := getAgentBasicAuth(v.ClusterID); err != nil { + log.Error(err) + continue + } else if auth != nil { + username = auth.Username + password = fmt.Sprintf("$[[keystore.%s]]", getAgentPasswordKey(v.ClusterID)) } } @@ -240,40 +697,168 @@ func getAgentIngestConfigs(instance string, items map[string]BindingItem) (strin continue } - nodeEndPoint := metadata.PrepareEndpoint(publishAddress) + effectiveSchema := normalizeSchema(v.EndpointSchema) + if effectiveSchema == "" { + effectiveSchema = probeAndBackfillNodeEndpointSchema(instance, &v, publishAddress) + } + nodeEndPoint := prepareAgentNodeEndpoint(metadata, publishAddress, effectiveSchema) - pathLogs := nodeInfo.Payload.NodeInfo.GetPathLogs() + logsPaths := normalizeLogsPaths(v.LogsPaths, v.PathLogs) + if len(logsPaths) == 0 { + logsPaths = normalizeLogsPaths(nil, nodeInfo.Payload.NodeInfo.GetPathLogs()) + } if v.Updated > latestVersion { latestVersion = v.Updated } taskID := v.ClusterID + "_" + v.NodeUUID - buffer.Write([]byte(fmt.Sprintf("\n - name: \"%v\"\n path: ./config/task_config.tpl\n "+ - "variable:\n "+ - "TASK_ID: %v\n "+ - "CLUSTER_ID: %v\n "+ - "CLUSTER_NAME: %v\n "+ - "CLUSTER_UUID: %v\n "+ - "NODE_UUID: %v\n "+ - "CLUSTER_VERSION: %v\n "+ - "CLUSTER_DISTRIBUTION: %v\n "+ - "CLUSTER_ENDPOINT: [\"%v\"]\n "+ - "CLUSTER_USERNAME: \"%v\"\n "+ - "CLUSTER_PASSWORD: \"%v\"\n "+ - "CLUSTER_LEVEL_TASKS_ENABLED: %v\n "+ - "NODE_LEVEL_TASKS_ENABLED: %v\n "+ - "NODE_LOGS_PATH: \"%v\"\n\n\n", taskID, taskID, - v.ClusterID, clusterName, v.ClusterUUID, v.NodeUUID, version, distribution, nodeEndPoint, username, password, clusterLevelEnabled, nodeLevelEnabled, pathLogs))) + // Resolve effective collection interval: per-node override > cluster default > agent default (0) + collectionInterval := v.CollectionInterval + if collectionInterval == 0 { + collectionInterval = metadata.Config.AgentCollectionInterval + } + elasticBuffer.WriteString(renderAgentTaskElasticsearchConfig( + taskID, + v.ClusterUUID, + version, + distribution, + nodeEndPoint, + username, + password, + )) + pipelineBuffer.WriteString(renderAgentTaskPipelineConfig( + taskID, + v.ClusterID, + clusterName, + v.ClusterUUID, + clusterLevelEnabled, + nodeLevelEnabled, + logsPaths, + collectionInterval, + )) } + buffer := bytes.NewBufferString(elasticBuffer.String()) + buffer.WriteString(pipelineBuffer.String()) + hash := util.MD5digest(buffer.String()) - //password: $[[keystore.$[[CLUSTER_ID]]_password]] buffer.WriteString("\n") buffer.WriteString(fmt.Sprintf("#MANAGED_CONFIG_VERSION: %v\n#MANAGED: true\n", latestVersion)) return buffer.String(), hash } +func prepareAgentNodeEndpoint(_ *elastic.ElasticsearchMetadata, publishAddress, endpointSchema string) string { + host := strings.TrimSpace(publishAddress) + if host == "" { + return "" + } + if strings.Contains(host, "://") { + if parsed, err := url.Parse(host); err == nil && parsed != nil && parsed.Host != "" { + return fmt.Sprintf("%s://%s", strings.ToLower(parsed.Scheme), parsed.Host) + } + return host + } + schema := normalizeSchema(endpointSchema) + if schema == "" { + // Keep backward compatibility for legacy bindings that were created before endpoint_schema existed. + schema = "http" + } + return fmt.Sprintf("%s://%s", schema, host) +} + +func probeAndBackfillNodeEndpointSchema(instanceID string, item *BindingItem, publishAddress string) string { + if item == nil { + return "" + } + host := strings.TrimSpace(publishAddress) + if host == "" { + return "" + } + auth, err := getAgentBasicAuth(item.ClusterID) + if err != nil { + log.Errorf("failed to get agent basic auth while probing node schema: cluster=%s, node=%s, err=%v", item.ClusterID, item.NodeUUID, err) + return "" + } + if !hasUsableAgentBasicAuth(auth) { + return "" + } + + schema := "" + handler := &APIHandler{} + if success, tryAgain, _ := handler.getESNodeInfoViaProxy(host, "https", auth, instanceID); success { + schema = "https" + } else if tryAgain { + if success, _, _ := handler.getESNodeInfoViaProxy(host, "http", auth, instanceID); success { + schema = "http" + } + } + if schema == "" { + return "" + } + + item.EndpointSchema = schema + if strings.TrimSpace(item.PublishAddress) == "" { + item.PublishAddress = host + } + settings := NewNodeAgentSettings(instanceID, item) + if err := orm.Save(&orm.Context{Refresh: "wait_for"}, settings); err != nil { + log.Errorf("failed to backfill node endpoint schema: node=%s, cluster=%s, schema=%s, err=%v", item.NodeUUID, item.ClusterID, schema, err) + } + return schema +} + +func renderAgentTaskElasticsearchConfig(taskID, clusterUUID, version, distribution, nodeEndpoint, username, password string) string { + return fmt.Sprintf( + "\n - id: %s\n name: %s\n cluster_uuid: %s\n enabled: true\n monitored: true\n distribution: %s\n version: %s\n endpoints: [%s]\n discovery:\n enabled: false\n basic_auth:\n username: %s\n password: %s\n traffic_control:\n enabled: true\n max_qps_per_node: 100\n max_bytes_per_node: 10485760\n max_connection_per_node: 5\n", + util.MustToJSON(taskID), + util.MustToJSON(taskID), + util.MustToJSON(clusterUUID), + util.MustToJSON(distribution), + util.MustToJSON(version), + util.MustToJSON(nodeEndpoint), + util.MustToJSON(username), + util.MustToJSON(password), + ) +} + +func renderAgentTaskPipelineConfig(taskID, clusterID, clusterName, clusterUUID string, clusterLevelEnabled, nodeLevelEnabled bool, logsPaths []string, collectionInterval int) string { + logsPathValue := `""` + switch len(logsPaths) { + case 0: + case 1: + logsPathValue = util.MustToJSON(logsPaths[0]) + default: + logsPathValue = util.MustToJSON(logsPaths) + } + intervalLine := "" + if collectionInterval > 0 { + intervalLine = fmt.Sprintf("\n interval: %ds", collectionInterval) + } + return fmt.Sprintf( + "\n - auto_start: %t\n enabled: %t\n keep_running: true%s\n name: collect_%s_es_node_stats\n retry_delay_in_ms: 10000\n processor:\n - es_node_stats:\n elasticsearch: %s\n labels:\n cluster_id: %s\n cluster_uuid: %s\n cluster_name: %s\n when:\n cluster_available:\n - %s\n\n - auto_start: %t\n enabled: %t\n keep_running: true%s\n name: collect_%s_es_logs\n retry_delay_in_ms: 10000\n processor:\n - es_logs_processor:\n elasticsearch: %s\n labels:\n cluster_id: %s\n cluster_uuid: %s\n cluster_name: %s\n logs_path: %s\n queue_name: logs\n when:\n cluster_available:\n - %s\n", + nodeLevelEnabled, + nodeLevelEnabled, + intervalLine, + taskID, + util.MustToJSON(taskID), + util.MustToJSON(clusterID), + util.MustToJSON(clusterUUID), + util.MustToJSON(clusterName), + util.MustToJSON(taskID), + nodeLevelEnabled, + nodeLevelEnabled, + intervalLine, + taskID, + util.MustToJSON(taskID), + util.MustToJSON(clusterID), + util.MustToJSON(clusterUUID), + util.MustToJSON(clusterName), + logsPathValue, + util.MustToJSON(taskID), + ) +} + const LastAgentHash = "last_agent_hash" diff --git a/modules/agent/api/remote_config_test.go b/modules/agent/api/remote_config_test.go new file mode 100644 index 00000000..8287caab --- /dev/null +++ b/modules/agent/api/remote_config_test.go @@ -0,0 +1,462 @@ +package api + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + config2 "infini.sh/framework/core/config" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/env" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" +) + +func TestRenderAgentTaskTemplateConfigProducesValidYAML(t *testing.T) { + content := "elasticsearch:" + renderAgentTaskElasticsearchConfig( + "cluster-1_node-1", + "uuid:cluster", + "8.13.4", + "easysearch", + "https://192.168.3.8:9200", + "agent-user", + "$[[keystore.cluster-1_password]]", + ) + "\npipeline:" + renderAgentTaskPipelineConfig( + "cluster-1_node-1", + "cluster-1", + "Quartz: primary #1", + "uuid:cluster", + false, + true, + []string{`C:\Program Files\Agent\logs`}, + 0, + ) + + cfg, err := config2.NewConfigWithYAML([]byte(content), "generated_metrics_tasks.yml") + if err != nil { + t.Fatalf("expected generated yaml to parse, got error: %v\n%s", err, content) + } + + var parsed struct { + Elasticsearch []struct { + ID string `config:"id"` + ClusterUUID string `config:"cluster_uuid"` + Monitored bool `config:"monitored"` + Endpoints []string `config:"endpoints"` + BasicAuth struct { + Password string `config:"password"` + } `config:"basic_auth"` + } `config:"elasticsearch"` + Pipeline []struct { + Name string `config:"name"` + Processor []struct { + NodeStats struct { + Labels struct { + ClusterName string `config:"cluster_name"` + } `config:"labels"` + } `config:"es_node_stats"` + Logs struct { + LogsPath string `config:"logs_path"` + } `config:"es_logs_processor"` + } `config:"processor"` + } `config:"pipeline"` + } + if err := cfg.Unpack(&parsed); err != nil { + t.Fatalf("expected generated yaml to unpack, got error: %v", err) + } + + if len(parsed.Elasticsearch) != 1 { + t.Fatalf("expected 1 elasticsearch config, got %d", len(parsed.Elasticsearch)) + } + if got := parsed.Elasticsearch[0].ID; got != "cluster-1_node-1" { + t.Fatalf("expected elasticsearch id to round-trip, got %#v", got) + } + if got := parsed.Elasticsearch[0].ClusterUUID; got != "uuid:cluster" { + t.Fatalf("expected cluster uuid to round-trip, got %#v", got) + } + if !parsed.Elasticsearch[0].Monitored { + t.Fatalf("expected generated elasticsearch config to enable monitoring") + } + if got := parsed.Elasticsearch[0].BasicAuth.Password; got != "$[[keystore.cluster-1_password]]" { + t.Fatalf("expected password placeholder to round-trip, got %#v", got) + } + if len(parsed.Elasticsearch[0].Endpoints) != 1 || parsed.Elasticsearch[0].Endpoints[0] != "https://192.168.3.8:9200" { + t.Fatalf("expected single endpoint array, got %#v", parsed.Elasticsearch[0].Endpoints) + } + if len(parsed.Pipeline) != 2 { + t.Fatalf("expected 2 pipelines, got %d", len(parsed.Pipeline)) + } + if got := parsed.Pipeline[0].Processor[0].NodeStats.Labels.ClusterName; got != "Quartz: primary #1" { + t.Fatalf("expected cluster name to round-trip, got %#v", got) + } + if got := parsed.Pipeline[1].Processor[0].Logs.LogsPath; got != `C:\Program Files\Agent\logs` { + t.Fatalf("expected logs path to round-trip, got %#v", got) + } + if !strings.Contains(content, `endpoints: ["https://192.168.3.8:9200"]`) { + t.Fatalf("expected endpoint string to be quoted safely, got: %s", content) + } + if strings.Contains(content, "configs.template:") { + t.Fatalf("expected generated config to be fully rendered, got: %s", content) + } +} + +func TestRenderAgentTaskTemplateConfigSupportsMultipleLogsPaths(t *testing.T) { + content := "elasticsearch:" + renderAgentTaskElasticsearchConfig( + "cluster-1_node-1", + "uuid:cluster", + "8.13.4", + "easysearch", + "https://192.168.3.8:9200", + "agent-user", + "$[[keystore.cluster-1_password]]", + ) + "\npipeline:" + renderAgentTaskPipelineConfig( + "cluster-1_node-1", + "cluster-1", + "Quartz: primary #1", + "uuid:cluster", + false, + true, + []string{"/infini/easysearch/logs", "/infini/easysearch/gc"}, + 0, + ) + + cfg, err := config2.NewConfigWithYAML([]byte(content), "generated_metrics_tasks.yml") + if err != nil { + t.Fatalf("expected generated yaml to parse, got error: %v\n%s", err, content) + } + + var parsed struct { + Pipeline []struct { + Processor []struct { + Logs struct { + LogsPath []string `config:"logs_path"` + } `config:"es_logs_processor"` + } `config:"processor"` + } `config:"pipeline"` + } + if err := cfg.Unpack(&parsed); err != nil { + t.Fatalf("expected generated yaml to unpack, got error: %v", err) + } + + if len(parsed.Pipeline) != 2 { + t.Fatalf("expected 2 pipelines, got %d", len(parsed.Pipeline)) + } + if got := parsed.Pipeline[1].Processor[0].Logs.LogsPath; len(got) != 2 || got[0] != "/infini/easysearch/logs" || got[1] != "/infini/easysearch/gc" { + t.Fatalf("expected logs paths to round-trip, got %#v", got) + } +} + +func TestRenderAgentTaskPipelineConfigWithCustomInterval(t *testing.T) { + content := "elasticsearch:" + renderAgentTaskElasticsearchConfig( + "cluster-1_node-1", + "uuid:cluster", + "8.13.4", + "easysearch", + "https://192.168.3.8:9200", + "agent-user", + "$[[keystore.cluster-1_password]]", + ) + "\npipeline:" + renderAgentTaskPipelineConfig( + "cluster-1_node-1", + "cluster-1", + "Cluster 1", + "uuid:cluster", + false, + true, + []string{"/infini/easysearch/logs"}, + 30, + ) + + cfg, err := config2.NewConfigWithYAML([]byte(content), "generated_metrics_tasks.yml") + if err != nil { + t.Fatalf("expected generated yaml to parse with interval, got error: %v\n%s", err, content) + } + + var parsed struct { + Pipeline []struct { + Interval string `config:"interval"` + } `config:"pipeline"` + } + if err := cfg.Unpack(&parsed); err != nil { + t.Fatalf("expected generated yaml to unpack with interval, got error: %v", err) + } + if len(parsed.Pipeline) != 2 { + t.Fatalf("expected 2 pipelines, got %d", len(parsed.Pipeline)) + } + if got := parsed.Pipeline[0].Interval; got != "30s" { + t.Fatalf("expected interval to be 30s, got %q", got) + } + if got := parsed.Pipeline[1].Interval; got != "30s" { + t.Fatalf("expected logs pipeline interval to be 30s, got %q", got) + } +} + +func TestTaskConfigTemplateRendersEndpointArrayForLegacyAgent(t *testing.T) { + templatePath := filepath.Join("..", "..", "..", "config", "setup", "common", "data", "task_config_tpl.dat") + if _, err := os.Stat(templatePath); err != nil { + t.Fatalf("failed to stat task template: %v", err) + } + + cfg, err := config2.NewConfigWithTemplate(config2.ConfigTemplate{ + Path: templatePath, + Variable: util.MapStr{ + "TASK_ID": "cluster-1_node-1", + "CLUSTER_ID": "cluster-1", + "CLUSTER_NAME": "Cluster 1", + "CLUSTER_UUID": "uuid:cluster", + "NODE_UUID": "node:uuid", + "CLUSTER_VERSION": "8.13.4", + "CLUSTER_DISTRIBUTION": "easysearch", + "CLUSTER_ENDPOINT": "http://192.168.3.8:9200", + "CLUSTER_USERNAME": "agent-user", + "CLUSTER_PASSWORD": "$[[keystore.cluster-1_password]]", + "CLUSTER_LEVEL_TASKS_ENABLED": false, + "NODE_LEVEL_TASKS_ENABLED": true, + "NODE_LOGS_PATH": "/var/log/elasticsearch", + }, + }) + if err != nil { + t.Fatalf("expected task template to render, got %v", err) + } + + var parsed struct { + Elasticsearch []struct { + Monitored bool `config:"monitored"` + Endpoints []string `config:"endpoints"` + } `config:"elasticsearch"` + } + if err := cfg.Unpack(&parsed); err != nil { + t.Fatalf("expected rendered task config to unpack, got %v", err) + } + if len(parsed.Elasticsearch) != 1 || !parsed.Elasticsearch[0].Monitored { + t.Fatalf("expected rendered task config to enable monitoring, got %#v", parsed.Elasticsearch) + } + if len(parsed.Elasticsearch) != 1 || len(parsed.Elasticsearch[0].Endpoints) != 1 || parsed.Elasticsearch[0].Endpoints[0] != "http://192.168.3.8:9200" { + t.Fatalf("expected single endpoint array, got %#v", parsed.Elasticsearch) + } +} + +func TestShouldSkipGatewayConfigByType(t *testing.T) { + instance := model.Instance{ + Application: env.Application{Name: "gateway"}, + Labels: map[string]string{ + "service_type": "relay", + }, + } + + relayDoc := map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "service_type": "relay", + }, + }, + "payload": map[string]interface{}{ + "location": "relay.yml", + }, + } + if shouldSkipGatewayConfigByType(instance, relayDoc) { + t.Fatal("expected relay gateway to keep relay config") + } + + migrationDoc := map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "service_type": "migration", + }, + }, + "payload": map[string]interface{}{ + "location": "migration.yml", + }, + } + if !shouldSkipGatewayConfigByType(instance, migrationDoc) { + t.Fatal("expected relay gateway to skip migration config") + } +} + +func TestShouldSkipGatewayConfigByTypeWithLegacyConfigDoc(t *testing.T) { + instance := model.Instance{ + Application: env.Application{Name: "gateway"}, + Labels: map[string]string{ + "service_type": "migration", + }, + } + + legacyRelayDoc := map[string]interface{}{ + "payload": map[string]interface{}{ + "location": "relay.yml", + }, + } + if !shouldSkipGatewayConfigByType(instance, legacyRelayDoc) { + t.Fatal("expected migration gateway to skip legacy relay config") + } + + legacyMigrationDoc := map[string]interface{}{ + "payload": map[string]interface{}{ + "location": "migration.yml", + }, + } + if shouldSkipGatewayConfigByType(instance, legacyMigrationDoc) { + t.Fatal("expected migration gateway to keep legacy migration config") + } +} + +func TestNormalizeRelayGatewayIngestHosts(t *testing.T) { + got := normalizeRelayGatewayIngestHosts([]string{ + "https://relay-1.local:2900", + "http://relay-1.local:2900/", + "relay-2.local:2900", + "", + "https://[2001:db8::1]:2900", + }, "9443") + expected := []string{ + "relay-1.local:9443", + "relay-2.local:9443", + "[2001:db8::1]:9443", + } + if strings.Join(got, ",") != strings.Join(expected, ",") { + t.Fatalf("expected %v, got %v", expected, got) + } +} + +func TestExtractRelayGatewayIngestPort(t *testing.T) { + content := strings.Join([]string{ + "entry:", + " - name: gateway_relay_entry", + " network:", + " binding: 0.0.0.0:9443", + }, "\n") + + if got := extractRelayGatewayIngestPort(content); got != "9443" { + t.Fatalf("expected port 9443, got %q", got) + } +} + +func TestBuildRelayGatewayInstancesQueryMatchesBothServiceTypeFields(t *testing.T) { + q := buildRelayGatewayInstancesQuery() + raw := string(q.RawQuery) + if !strings.Contains(raw, `"labels.service_type"`) { + t.Fatalf("expected query to match labels.service_type, got %s", raw) + } + if !strings.Contains(raw, `"metadata.labels.service_type"`) { + t.Fatalf("expected query to match metadata.labels.service_type, got %s", raw) + } + if !strings.Contains(raw, `"minimum_should_match":1`) { + t.Fatalf("expected query to require one relay service_type match, got %s", raw) + } +} + +func TestFilterReachableRelayIngestHosts(t *testing.T) { + originalDial := relayIngestDialTimeout + defer func() { relayIngestDialTimeout = originalDial }() + + relayIngestDialTimeout = func(network, address string, timeout time.Duration) (net.Conn, error) { + if address == "relay-1.local:8081" { + c1, c2 := net.Pipe() + _ = c2.Close() + return c1, nil + } + return nil, fmt.Errorf("connection refused") + } + + filtered := filterReachableRelayIngestHosts([]string{ + "relay-1.local:8081", + "relay-2.local:8081", + }, 500*time.Millisecond) + if len(filtered) != 1 || filtered[0] != "relay-1.local:8081" { + t.Fatalf("expected only reachable relay host, got %v", filtered) + } +} + +func TestSortRelayHostsByRole(t *testing.T) { + hosts := []string{ + "relay-b.local:8081", + "relay-a.local:8081", + "relay-c.local:8081", + } + roleByHost := map[string]string{ + "relay-a.local:8081": "secondary", + "relay-b.local:8081": "primary", + } + sortRelayHosts(hosts, roleByHost) + expected := []string{ + "relay-b.local:8081", + "relay-a.local:8081", + "relay-c.local:8081", + } + if strings.Join(hosts, ",") != strings.Join(expected, ",") { + t.Fatalf("expected hosts sorted by role then host, got %v", hosts) + } +} + +func TestRewriteAgentRelayIngestContent(t *testing.T) { + instance := model.Instance{Application: env.Application{Name: "agent"}} + content := strings.Join([]string{ + " basic_auth:", + " username: \"infini-ingest\"", + " password: \"$[[keystore.SYSTEM_CLUSTER_INGEST_PASSWORD]]\"", + "# tls: #for mTLS connection with config servers", + "# enabled: true", + "# ca_file: /xxx/ca.crt", + "# cert_file: /xxx/client.crt", + "# key_file: /xxx/client.key", + "# skip_insecure_verify: false", + " schema: \"https\"", + " hosts: [\"192.168.3.8:9200\"]", + }, "\n") + + rewritten := rewriteAgentRelayIngestContent( + instance, + "system_ingest_config.yml", + content, + []string{"relay-1.local:8081", "relay-2.local:8081"}, + ) + + if !strings.Contains(rewritten, `schema: "https"`) { + t.Fatalf("expected schema to be rewritten to https, got: %s", rewritten) + } + if !strings.Contains(rewritten, `hosts: ["relay-1.local:8081","relay-2.local:8081"]`) { + t.Fatalf("expected hosts to be rewritten to relay gateways, got: %s", rewritten) + } + if !strings.Contains(rewritten, `tls: #for mTLS connection with config servers`) || + !strings.Contains(rewritten, `cert_file: config/client.crt`) || + !strings.Contains(rewritten, `key_file: config/client.key`) { + t.Fatalf("expected tls block to be enabled for relay ingest, got: %s", rewritten) + } + + unchanged := rewriteAgentRelayIngestContent( + instance, + "generated_metrics_tasks.yml", + content, + []string{"relay-1.local:8081"}, + ) + if unchanged != content { + t.Fatalf("expected non-system-ingest config to remain unchanged") + } +} + +func TestPrepareAgentNodeEndpointUsesBindingSchema(t *testing.T) { + meta := &elastic.ElasticsearchMetadata{ + Config: &elastic.ElasticsearchConfig{ + Schema: "http", + }, + } + got := prepareAgentNodeEndpoint(meta, "10.244.0.10:9200", "https") + if got != "https://10.244.0.10:9200" { + t.Fatalf("expected binding schema to win, got %q", got) + } +} + +func TestPrepareAgentNodeEndpointKeepsExplicitURL(t *testing.T) { + meta := &elastic.ElasticsearchMetadata{ + Config: &elastic.ElasticsearchConfig{ + Schema: "http", + }, + } + got := prepareAgentNodeEndpoint(meta, "https://10.244.0.10:9200", "") + if got != "https://10.244.0.10:9200" { + t.Fatalf("expected explicit endpoint scheme to be preserved, got %q", got) + } +} diff --git a/modules/agent/api/reverse_channel.go b/modules/agent/api/reverse_channel.go new file mode 100644 index 00000000..1387c487 --- /dev/null +++ b/modules/agent/api/reverse_channel.go @@ -0,0 +1,152 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + "sync" + + log "github.com/cihub/seelog" + agent_common "infini.sh/console/modules/agent/common" + framework_api "infini.sh/framework/core/api" + framework_ws "infini.sh/framework/core/api/websocket" + framework_reverse "infini.sh/framework/core/api/websocket/reverse" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + elastic "infini.sh/framework/modules/elastic" +) + +var errAgentReverseChannelDisconnected = framework_reverse.ErrDisconnected +var errAgentReverseChannelNotConnected = framework_reverse.ErrNotConnected + +const reverseChannelRegistrationLabelKey = "registered_via_reverse_channel" +const reverseChannelRegistrationLabelValue = "true" + +var agentReverseChannel = framework_reverse.NewSessionManager(framework_reverse.ManagerOptions{}) +var agentReverseChannelRegisterOnce sync.Once + +func registerAgentReverseChannel() { + agentReverseChannelRegisterOnce.Do(func() { + framework_ws.RegisterConnectCallback(onAgentReverseConnect) + framework_ws.RegisterDisconnectCallback(onAgentReverseDisconnect) + framework_api.HandleWebSocketCommand(framework_reverse.HelloCommand, "agent reverse hello", handleAgentReverseHelloCommand) + framework_api.HandleWebSocketCommand(framework_reverse.ResponseCommand, "agent reverse response", handleAgentReverseResponseCommand) + }) +} + +func onAgentReverseConnect(sessionID string, w http.ResponseWriter, r *http.Request) error { + instanceID := strings.TrimSpace(r.Header.Get(framework_reverse.HeaderPeerID)) + if instanceID == "" { + return nil + } + + instance := model.Instance{} + instance.ID = instanceID + exists, err := orm.GetV2(orm.NewContext(), &instance) + if err == elastic.ErrNotFound { + err = nil + exists = false + } + if err != nil { + return err + } + if !exists { + return fmt.Errorf("instance not registered") + } + if !strings.EqualFold(instance.Application.Name, "agent") { + return fmt.Errorf("instance [%s] is not agent", instanceID) + } + if err := agent_common.ValidateManagerRequestAuth(r, &instance, (*model.BasicAuth)(&global.Env().SystemConfig.Configs.ManagerConfig.BasicAuth)); err != nil { + return err + } + if err := ensureReverseChannelRegistrationMarker(&instance); err != nil { + log.Warnf("failed to persist reverse channel registration marker for agent [%s]: %v", instanceID, err) + } + agentReverseChannel.RegisterPendingSession(sessionID, instanceID) + return nil +} + +func onAgentReverseDisconnect(sessionID string) { + agentReverseChannel.OnDisconnect(sessionID) +} + +func handleAgentReverseHelloCommand(c *framework_ws.WebsocketConnection, array []string) { + if len(array) < 2 { + return + } + payload := strings.Join(array[1:], " ") + if err := agentReverseChannel.HandleHelloPayload(payload); err != nil { + log.Errorf("failed to activate agent reverse channel: %v", err) + } +} + +func handleAgentReverseResponseCommand(c *framework_ws.WebsocketConnection, array []string) { + if len(array) < 2 { + return + } + payload := strings.Join(array[1:], " ") + if err := agentReverseChannel.HandleResponsePayload(payload); err != nil { + log.Errorf("failed to parse agent reverse response: %v", err) + } +} + +func isAgentReverseChannelRecoverableError(err error) bool { + return framework_reverse.IsRecoverableError(err) +} + +func buildReverseRequestHeaders(instanceID string) (http.Header, error) { + accessToken, err := loadReverseAccessToken(instanceID) + if err != nil { + return nil, err + } + headers := http.Header{} + if accessToken != "" { + headers.Set("Authorization", "Bearer "+accessToken) + } + return headers, nil +} + +func ProxyAgentRequestViaChannel(instanceID string, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + headers, err := buildReverseRequestHeaders(instanceID) + if err != nil { + return nil, err + } + return agentReverseChannel.ProxyRequest(instanceID, req, headers, framework_ws.SendPrivateMessage, responseObjectToUnMarshall) +} + +func IsAgentReverseChannelConnected(instanceID string) bool { + return agentReverseChannel.IsConnected(instanceID) +} + +func shouldUseReverseChannelOnlyForInstance(instance *model.Instance) bool { + if instance == nil || instance.Labels == nil { + return false + } + return strings.EqualFold(strings.TrimSpace(instance.Labels[reverseChannelRegistrationLabelKey]), reverseChannelRegistrationLabelValue) +} + +func ensureReverseChannelRegistrationMarker(instance *model.Instance) error { + if instance == nil { + return fmt.Errorf("instance is nil") + } + if shouldUseReverseChannelOnlyForInstance(instance) { + return nil + } + if instance.Labels == nil { + instance.Labels = map[string]string{} + } + instance.Labels[reverseChannelRegistrationLabelKey] = reverseChannelRegistrationLabelValue + return orm.Save(&orm.Context{Refresh: orm.WaitForRefresh}, instance) +} + +func loadReverseAccessToken(instanceID string) (string, error) { + instance := model.Instance{} + instance.ID = instanceID + exists, err := orm.GetV2(orm.NewContext(), &instance) + if err != nil || !exists { + return "", err + } + return agent_common.GetPreferredTokenCredentialValue(instance.AccessCredentialID) +} diff --git a/modules/agent/api/reverse_channel_test.go b/modules/agent/api/reverse_channel_test.go new file mode 100644 index 00000000..e6f77bcc --- /dev/null +++ b/modules/agent/api/reverse_channel_test.go @@ -0,0 +1,62 @@ +package api + +import ( + "net/http/httptest" + "testing" + + "infini.sh/framework/core/model" +) + +func TestAgentReverseChannelOnConnectAllowsGenericWebsocket(t *testing.T) { + req := httptest.NewRequest("GET", "/ws", nil) + + if err := onAgentReverseConnect("session-1", nil, req); err != nil { + t.Fatalf("expected generic websocket connection to pass, got %v", err) + } +} + +func TestShouldUseReverseChannelOnlyForInstance(t *testing.T) { + cases := []struct { + name string + instance *model.Instance + expect bool + }{ + { + name: "nil instance", + expect: false, + }, + { + name: "empty labels", + instance: &model.Instance{ + Labels: map[string]string{}, + }, + expect: false, + }, + { + name: "marker enabled", + instance: &model.Instance{ + Labels: map[string]string{ + reverseChannelRegistrationLabelKey: reverseChannelRegistrationLabelValue, + }, + }, + expect: true, + }, + { + name: "marker mixed case", + instance: &model.Instance{ + Labels: map[string]string{ + reverseChannelRegistrationLabelKey: "TRUE", + }, + }, + expect: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := shouldUseReverseChannelOnlyForInstance(tc.instance); got != tc.expect { + t.Fatalf("unexpected reverse-only result, got=%v want=%v", got, tc.expect) + } + }) + } +} diff --git a/modules/agent/common/config.go b/modules/agent/common/config.go index 1f4b7936..744eea41 100644 --- a/modules/agent/common/config.go +++ b/modules/agent/common/config.go @@ -37,9 +37,7 @@ import ( func GetAgentConfig() *model.AgentConfig { agentCfg := &model.AgentConfig{ Enabled: true, - Setup: &model.SetupConfig{ - DownloadURL: "https://release.infinilabs.com/agent/stable", - }, + Setup: &model.SetupConfig{}, } _, err := env.ParseConfig("agent", agentCfg) if err != nil { diff --git a/modules/agent/common/endpoint.go b/modules/agent/common/endpoint.go new file mode 100644 index 00000000..96ba87e2 --- /dev/null +++ b/modules/agent/common/endpoint.go @@ -0,0 +1,72 @@ +package common + +import ( + "net/url" + "strings" + + "infini.sh/framework/core/model" +) + +const ( + agentAPIServiceName = "api" + agentWebServiceName = "web" +) + +func ResolveInstanceRequestEndpoint(instance *model.Instance, requestPath string) string { + if instance == nil { + return "" + } + + normalizedPath := normalizeRequestPath(requestPath) + if strings.EqualFold(instance.Application.Name, "agent") && isAgentDirectAccessPath(normalizedPath) { + if endpoint := getInstanceServiceEndpoint(instance, agentWebServiceName); endpoint != "" { + return endpoint + } + } + + if endpoint := getInstanceServiceEndpoint(instance, agentAPIServiceName); endpoint != "" && !isAgentDirectAccessPath(normalizedPath) { + return endpoint + } + + return instance.GetEndpoint() +} + +func getInstanceServiceEndpoint(instance *model.Instance, serviceName string) string { + if instance == nil || serviceName == "" { + return "" + } + for _, service := range instance.Services { + if strings.EqualFold(service.Name, serviceName) && strings.TrimSpace(service.Endpoint) != "" { + return strings.TrimSpace(service.Endpoint) + } + } + return "" +} + +func isAgentDirectAccessPath(rawPath string) bool { + switch normalizeRequestPath(rawPath) { + case "/agent/_info", + "/elasticsearch/node/_discovery", + "/elasticsearch/node/_info", + "/elasticsearch/logs/_list", + "/elasticsearch/logs/_read": + return true + default: + return false + } +} + +func normalizeRequestPath(rawPath string) string { + rawPath = strings.TrimSpace(rawPath) + if rawPath == "" { + return "" + } + parsed, err := url.Parse(rawPath) + if err != nil { + return rawPath + } + if parsed.Path != "" { + return parsed.Path + } + return rawPath +} diff --git a/modules/agent/common/endpoint_test.go b/modules/agent/common/endpoint_test.go new file mode 100644 index 00000000..956a51bc --- /dev/null +++ b/modules/agent/common/endpoint_test.go @@ -0,0 +1,39 @@ +package common + +import ( + "testing" + + "infini.sh/framework/core/model" +) + +func TestResolveInstanceRequestEndpointUsesWebForAgentDirectAccess(t *testing.T) { + instance := &model.Instance{ + Endpoint: "http://agent-api.local:2900", + Services: []model.ServiceInfo{ + {Name: "api", Endpoint: "http://agent-api.local:2900"}, + {Name: "web", Endpoint: "http://agent-web.local:9000"}, + }, + } + instance.Application.Name = "agent" + + endpoint := ResolveInstanceRequestEndpoint(instance, "/agent/_info") + if endpoint != "http://agent-web.local:9000" { + t.Fatalf("unexpected endpoint: %s", endpoint) + } +} + +func TestResolveInstanceRequestEndpointUsesAPIForStats(t *testing.T) { + instance := &model.Instance{ + Endpoint: "http://agent-api.local:2900", + Services: []model.ServiceInfo{ + {Name: "api", Endpoint: "http://agent-api.local:2900"}, + {Name: "web", Endpoint: "http://agent-web.local:9000"}, + }, + } + instance.Application.Name = "agent" + + endpoint := ResolveInstanceRequestEndpoint(instance, "/stats") + if endpoint != "http://agent-api.local:2900" { + t.Fatalf("unexpected endpoint: %s", endpoint) + } +} diff --git a/modules/agent/common/token.go b/modules/agent/common/token.go new file mode 100644 index 00000000..65cd1a94 --- /dev/null +++ b/modules/agent/common/token.go @@ -0,0 +1,523 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package common + +import ( + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "infini.sh/framework/core/credential" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + configcommon "infini.sh/framework/modules/configs/common" +) + +const ( + agentManagerTokenPurpose = "manager" + agentAccessTokenPurpose = "access" + AgentPendingTokenSourceUI = "manual_registration" + AgentPendingTokenSourceVM = "install_script" + defaultPendingTokenTTL = 24 * time.Hour + tokenGracePeriod = time.Hour +) + +var rotatedTokenCache = util.NewCacheWithExpireOnAdd(tokenGracePeriod, 1024) +var getManagedTokenSeedFunc = credential.GetOrInitSecret + +var ( + ErrInvalidManagerToken = errors.New("invalid agent manager token") + ErrInvalidManagerBasicAuth = errors.New("invalid manager basic auth") + ErrManagerAuthNotConfigured = errors.New("agent manager auth is not configured") +) + +type PendingRegistrationToken struct { + orm.ORMObjectBase + CredentialID string `json:"credential_id" elastic_mapping:"credential_id:{type:keyword}"` + TokenHash string `json:"token_hash" elastic_mapping:"token_hash:{type:keyword}"` + Purpose string `json:"purpose" elastic_mapping:"purpose:{type:keyword}"` + Source string `json:"source,omitempty" elastic_mapping:"source:{type:keyword}"` + InstanceID string `json:"instance_id,omitempty" elastic_mapping:"instance_id:{type:keyword}"` + Consumed bool `json:"consumed" elastic_mapping:"consumed:{type:boolean}"` + ExpiresAt int64 `json:"expires_at" elastic_mapping:"expires_at:{type:date,format:epoch_millis}"` +} + +func CreatePendingManagerToken(source string) (*PendingRegistrationToken, string, error) { + tokenValue, err := GenerateManagedTokenValue() + if err != nil { + return nil, "", err + } + + record, err := findPendingManagerToken(tokenValue) + if err != nil { + return nil, "", err + } + if record != nil { + return record, tokenValue, nil + } + + credentialID, err := SaveTokenCredential(BuildPendingManagerCredentialName(), BuildPendingManagerCredentialTags(), tokenValue) + if err != nil { + return nil, "", err + } + + pendingRecord := &PendingRegistrationToken{ + CredentialID: credentialID, + TokenHash: HashAgentToken(tokenValue), + Purpose: agentManagerTokenPurpose, + Source: source, + Consumed: false, + ExpiresAt: time.Now().Add(defaultPendingTokenTTL).UnixMilli(), + } + pendingRecord.ID = util.GetUUID() + if err := orm.Create(&orm.Context{Refresh: orm.WaitForRefresh}, pendingRecord); err != nil { + return nil, "", err + } + return pendingRecord, tokenValue, nil +} + +func GenerateManagedTokenValue() (string, error) { + seed, err := getManagedTokenSeedFunc() + if err != nil { + return "", err + } + sum := sha256.Sum256(append([]byte("managed-bootstrap-token:v1:"), seed...)) + return hex.EncodeToString(sum[:]), nil +} + +func GetPendingRegistrationTokenByID(id string) (*PendingRegistrationToken, error) { + if strings.TrimSpace(id) == "" { + return nil, nil + } + record := &PendingRegistrationToken{ORMObjectBase: orm.ORMObjectBase{ID: id}} + exists, err := orm.GetV2(orm.NewContext(), record) + if err != nil || !exists { + if err == nil { + return nil, fmt.Errorf("pending registration token not found") + } + return nil, err + } + return record, nil +} + +func FindPendingManagerTokenByValue(tokenValue string) (*PendingRegistrationToken, error) { + tokenValue = strings.TrimSpace(tokenValue) + if tokenValue == "" { + return nil, nil + } + record, err := findPendingManagerToken(tokenValue) + if err != nil || record != nil { + return record, err + } + + expectedToken, err := GenerateManagedTokenValue() + if err != nil { + return nil, err + } + if subtle.ConstantTimeCompare([]byte(expectedToken), []byte(tokenValue)) != 1 { + return nil, nil + } + return createPendingManagerToken(tokenValue, AgentPendingTokenSourceVM) +} + +func MarkPendingRegistrationTokenConsumed(record *PendingRegistrationToken, instanceID string) error { + if record == nil { + return nil + } + record.Consumed = true + record.InstanceID = strings.TrimSpace(instanceID) + return orm.Update(&orm.Context{Refresh: orm.WaitForRefresh}, record) +} + +func findPendingManagerToken(tokenValue string) (*PendingRegistrationToken, error) { + query := orm.Query{ + Size: 1, + Conds: orm.And( + orm.Eq("token_hash", HashAgentToken(tokenValue)), + orm.Eq("purpose", agentManagerTokenPurpose), + ), + } + records := []PendingRegistrationToken{} + if err, _ := orm.SearchWithJSONMapper(&records, &query); err != nil { + return nil, err + } + if len(records) == 0 { + return nil, nil + } + record := records[0] + return &record, nil +} + +func createPendingManagerToken(tokenValue, source string) (*PendingRegistrationToken, error) { + credentialID, err := SaveTokenCredential(BuildPendingManagerCredentialName(), BuildPendingManagerCredentialTags(), tokenValue) + if err != nil { + return nil, err + } + + record := &PendingRegistrationToken{ + CredentialID: credentialID, + TokenHash: HashAgentToken(tokenValue), + Purpose: agentManagerTokenPurpose, + Source: source, + Consumed: false, + ExpiresAt: time.Now().Add(defaultPendingTokenTTL).UnixMilli(), + } + record.ID = util.GetUUID() + if err := orm.Create(&orm.Context{Refresh: orm.WaitForRefresh}, record); err != nil { + return nil, err + } + return record, nil +} + +func SaveTokenCredential(name string, tags []string, tokenValue string) (string, error) { + cred := credential.Credential{ + Name: name, + Type: credential.Token, + Tags: normalizeCredentialTags(tags), + Payload: map[string]interface{}{ + credential.Token: map[string]interface{}{ + "value": strings.TrimSpace(tokenValue), + }, + }, + } + cred.ID = util.GetUUID() + if err := cred.Encode(); err != nil { + return "", err + } + if err := orm.Create(&orm.Context{Refresh: orm.WaitForRefresh}, &cred); err != nil { + return "", err + } + return cred.ID, nil +} + +func UpdateTokenCredential(credentialID, name string, tags []string, tokenValue string) error { + if strings.TrimSpace(credentialID) == "" { + return fmt.Errorf("credential id is empty") + } + cred := credential.Credential{} + cred.ID = credentialID + exists, err := orm.GetV2(orm.NewContext(), &cred) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("credential not found") + } + cred.Name = name + cred.Type = credential.Token + cred.Tags = normalizeCredentialTags(tags) + cred.Payload = map[string]interface{}{ + credential.Token: map[string]interface{}{ + "value": strings.TrimSpace(tokenValue), + }, + } + if err := cred.Encode(); err != nil { + return err + } + return orm.Update(&orm.Context{Refresh: orm.WaitForRefresh}, &cred) +} + +func GetTokenCredentialValue(credentialID string) (string, error) { + if strings.TrimSpace(credentialID) == "" { + return "", nil + } + cred := credential.Credential{} + cred.ID = credentialID + exists, err := orm.GetV2(orm.NewContext(), &cred) + if err != nil { + return "", err + } + if !exists { + return "", fmt.Errorf("credential not found") + } + return cred.DecodeToken() +} + +func GetPreferredTokenCredentialValue(credentialID string) (string, error) { + if previous := getRotatedTokenValue(credentialID); previous != "" { + return previous, nil + } + return GetTokenCredentialValue(credentialID) +} + +func RememberPreviousToken(credentialID, tokenValue string) { + credentialID = strings.TrimSpace(credentialID) + tokenValue = strings.TrimSpace(tokenValue) + if credentialID == "" || tokenValue == "" { + return + } + rotatedTokenCache.Put(credentialID, tokenValue) +} + +func ApplyInstanceRequestAuth(req *util.Request, instance *model.Instance) error { + if req == nil || instance == nil { + return nil + } + if tokenValue, err := GetPreferredTokenCredentialValue(instance.AccessCredentialID); err != nil { + return err + } else if tokenValue != "" { + req.AddHeader("Authorization", "Bearer "+tokenValue) + return nil + } + if instance.AccessToken != nil && strings.TrimSpace(instance.AccessToken.Value) != "" { + req.AddHeader("Authorization", "Bearer "+strings.TrimSpace(instance.AccessToken.Value)) + return nil + } + if instance.BasicAuth != nil { + req.SetBasicAuth(instance.BasicAuth.Username, instance.BasicAuth.Password.Get()) + } + return nil +} + +func ApplyInstanceHTTPRequestAuth(req *http.Request, instance *model.Instance) error { + if req == nil || instance == nil { + return nil + } + if tokenValue, err := GetPreferredTokenCredentialValue(instance.AccessCredentialID); err != nil { + return err + } else if tokenValue != "" { + req.Header.Set("Authorization", "Bearer "+tokenValue) + return nil + } + if instance.AccessToken != nil && strings.TrimSpace(instance.AccessToken.Value) != "" { + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(instance.AccessToken.Value)) + return nil + } + if instance.BasicAuth != nil { + req.SetBasicAuth(instance.BasicAuth.Username, instance.BasicAuth.Password.Get()) + } + return nil +} + +func ApplyBearerToken(req *util.Request, tokenValue string) { + if req == nil { + return + } + tokenValue = strings.TrimSpace(tokenValue) + if tokenValue == "" { + return + } + req.AddHeader("Authorization", "Bearer "+tokenValue) +} + +func ExtractBearerToken(req *http.Request) string { + if req == nil { + return "" + } + value := strings.TrimSpace(req.Header.Get("Authorization")) + if !strings.HasPrefix(strings.ToLower(value), "bearer ") { + return "" + } + return strings.TrimSpace(value[7:]) +} + +func ExtractAPIToken(req *http.Request) string { + if req == nil { + return "" + } + return strings.TrimSpace(req.Header.Get(model.API_TOKEN)) +} + +func ExtractManagerToken(req *http.Request) string { + if tokenValue := ExtractAPIToken(req); tokenValue != "" { + return tokenValue + } + return ExtractBearerToken(req) +} + +func ValidateManagerRequestAuth(req *http.Request, instance *model.Instance, fallbackBasicAuth *model.BasicAuth) error { + return validateManagerRequestAuth(req, instance, fallbackBasicAuth, ValidateManagerToken, false) +} + +func ValidateLegacyCompatibleManagerRequestAuth(req *http.Request, instance *model.Instance, fallbackBasicAuth *model.BasicAuth) error { + return validateManagerRequestAuth(req, instance, fallbackBasicAuth, ValidateManagerToken, true) +} + +func IsManagerAuthFailure(err error) bool { + return errors.Is(err, ErrInvalidManagerToken) || + errors.Is(err, ErrInvalidManagerBasicAuth) || + errors.Is(err, ErrManagerAuthNotConfigured) +} + +func validateManagerRequestAuth( + req *http.Request, + instance *model.Instance, + fallbackBasicAuth *model.BasicAuth, + validateToken func(instance *model.Instance, tokenValue string) (bool, error), + allowLegacyWithoutManagerAuth bool, +) error { + if instance == nil { + return fmt.Errorf("instance is nil") + } + if strings.TrimSpace(instance.ManagerCredentialID) != "" { + ok, err := validateToken(instance, ExtractManagerToken(req)) + if err != nil { + return err + } + if !ok { + return ErrInvalidManagerToken + } + return nil + } + if fallbackBasicAuth == nil || strings.TrimSpace(fallbackBasicAuth.Username) == "" { + if allowLegacyWithoutManagerAuth { + return nil + } + return ErrManagerAuthNotConfigured + } + if req == nil { + return ErrInvalidManagerBasicAuth + } + user, password, ok := req.BasicAuth() + if !ok || user != fallbackBasicAuth.Username || password != fallbackBasicAuth.Password.Get() { + return ErrInvalidManagerBasicAuth + } + return nil +} + +func ValidateManagerToken(instance *model.Instance, tokenValue string) (bool, error) { + tokenValue = strings.TrimSpace(tokenValue) + if instance == nil || tokenValue == "" || strings.TrimSpace(instance.ManagerCredentialID) == "" { + return false, nil + } + expected, err := GetTokenCredentialValue(instance.ManagerCredentialID) + if err != nil { + return false, err + } + if subtle.ConstantTimeCompare([]byte(expected), []byte(tokenValue)) == 1 { + return true, nil + } + previous := getRotatedTokenValue(instance.ManagerCredentialID) + if previous == "" { + return false, nil + } + return subtle.ConstantTimeCompare([]byte(previous), []byte(tokenValue)) == 1, nil +} + +func getRotatedTokenValue(credentialID string) string { + credentialID = strings.TrimSpace(credentialID) + if credentialID == "" { + return "" + } + value := rotatedTokenCache.Get(credentialID) + if value == nil { + return "" + } + tokenValue, _ := value.(string) + tokenValue = strings.TrimSpace(tokenValue) + if tokenValue == "" { + return "" + } + return tokenValue +} + +func HashAgentToken(tokenValue string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(tokenValue))) + return hex.EncodeToString(sum[:]) +} + +func BuildManagerCredentialName(instance *model.Instance) string { + return fmt.Sprintf("%s (%s)", getAgentCredentialDisplayName(instance), getManagedCredentialProduct(instance)) +} + +func BuildAccessCredentialName(instance *model.Instance) string { + return fmt.Sprintf("%s (%s Access)", getAgentCredentialDisplayName(instance), getManagedCredentialProduct(instance)) +} + +func BuildPendingManagerCredentialName() string { + return fmt.Sprintf("%s (Managed)", util.PickRandomName()) +} + +func BuildManagerCredentialTags() []string { + return []string{"agent", "token", agentManagerTokenPurpose} +} + +func BuildPendingManagerCredentialTags() []string { + return []string{"agent", "token", agentManagerTokenPurpose, "pending"} +} + +func BuildAccessCredentialTags() []string { + return []string{"agent", "token", agentAccessTokenPurpose} +} + +func AgentManagerTokenKey() string { + return configcommon.ManagerTokenKeystoreKey +} + +func AgentAccessTokenKey() string { + return configcommon.AgentAccessTokenKeystoreKey +} + +func normalizeCredentialTags(tags []string) []string { + seen := map[string]struct{}{} + items := make([]string, 0, len(tags)) + for _, tag := range tags { + tag = strings.TrimSpace(tag) + if tag == "" { + continue + } + if _, ok := seen[tag]; ok { + continue + } + seen[tag] = struct{}{} + items = append(items, tag) + } + return items +} + +func getInstanceDisplayName(instance *model.Instance) string { + if instance == nil { + return "" + } + if name := strings.TrimSpace(instance.Name); name != "" { + return name + } + return strings.TrimSpace(instance.ID) +} + +func getAgentCredentialDisplayName(instance *model.Instance) string { + if displayName := getInstanceDisplayName(instance); displayName != "" { + return displayName + } + return util.PickRandomName() +} + +func getManagedCredentialProduct(instance *model.Instance) string { + if instance != nil { + if name := strings.TrimSpace(instance.Application.Name); name != "" { + return strings.ToUpper(name[:1]) + name[1:] + } + } + return "Managed" +} diff --git a/modules/agent/common/token_test.go b/modules/agent/common/token_test.go new file mode 100644 index 00000000..11130671 --- /dev/null +++ b/modules/agent/common/token_test.go @@ -0,0 +1,162 @@ +package common + +import ( + "net/http/httptest" + "strings" + "testing" + + "infini.sh/framework/core/model" + ucfg "infini.sh/framework/lib/go-ucfg" +) + +func TestGenerateManagedTokenValueIsDeterministic(t *testing.T) { + originalSeedFunc := getManagedTokenSeedFunc + t.Cleanup(func() { + getManagedTokenSeedFunc = originalSeedFunc + }) + + getManagedTokenSeedFunc = func() ([]byte, error) { + return []byte("credential-secret"), nil + } + + first, err := GenerateManagedTokenValue() + if err != nil { + t.Fatalf("generate first token: %v", err) + } + second, err := GenerateManagedTokenValue() + if err != nil { + t.Fatalf("generate second token: %v", err) + } + if first != second { + t.Fatalf("expected deterministic token, got %q and %q", first, second) + } +} + +func TestValidateManagerRequestAuth(t *testing.T) { + t.Run("accepts valid manager token", func(t *testing.T) { + req := httptest.NewRequest("GET", "/instance/_register", nil) + req.Header.Set("Authorization", "Bearer manager-token") + instance := &model.Instance{ManagerCredentialID: "cred-1"} + err := validateManagerRequestAuth(req, instance, nil, func(instance *model.Instance, tokenValue string) (bool, error) { + return tokenValue == "manager-token", nil + }, false) + if err != nil { + t.Fatalf("expected token auth to pass, got %v", err) + } + }) + + t.Run("accepts valid api token header", func(t *testing.T) { + req := httptest.NewRequest("GET", "/instance/_register", nil) + req.Header.Set(model.API_TOKEN, "manager-token") + instance := &model.Instance{ManagerCredentialID: "cred-1"} + err := validateManagerRequestAuth(req, instance, nil, func(instance *model.Instance, tokenValue string) (bool, error) { + return tokenValue == "manager-token", nil + }, false) + if err != nil { + t.Fatalf("expected api token auth to pass, got %v", err) + } + }) + + t.Run("rejects invalid manager token", func(t *testing.T) { + req := httptest.NewRequest("GET", "/instance/_register", nil) + instance := &model.Instance{ManagerCredentialID: "cred-1"} + err := validateManagerRequestAuth(req, instance, nil, func(instance *model.Instance, tokenValue string) (bool, error) { + return false, nil + }, false) + if err != ErrInvalidManagerToken { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("accepts valid manager basic auth", func(t *testing.T) { + req := httptest.NewRequest("GET", "/configs/_sync", nil) + req.SetBasicAuth("manager", "secret") + instance := &model.Instance{} + basicAuth := &model.BasicAuth{Username: "manager"} + basicAuth.Password = ucfg.SecretString("secret") + err := validateManagerRequestAuth(req, instance, basicAuth, func(instance *model.Instance, tokenValue string) (bool, error) { + t.Fatal("token validator should not be called when manager credential id is empty") + return false, nil + }, false) + if err != nil { + t.Fatalf("expected basic auth to pass, got %v", err) + } + }) + + t.Run("rejects when manager auth is not configured", func(t *testing.T) { + req := httptest.NewRequest("GET", "/ws", nil) + err := validateManagerRequestAuth(req, &model.Instance{}, nil, func(instance *model.Instance, tokenValue string) (bool, error) { + t.Fatal("token validator should not be called when manager credential id is empty") + return false, nil + }, false) + if err != ErrManagerAuthNotConfigured { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("allows legacy instance when manager auth is not configured", func(t *testing.T) { + req := httptest.NewRequest("GET", "/configs/_sync", nil) + err := validateManagerRequestAuth(req, &model.Instance{}, nil, func(instance *model.Instance, tokenValue string) (bool, error) { + t.Fatal("token validator should not be called when manager credential id is empty") + return false, nil + }, true) + if err != nil { + t.Fatalf("expected legacy compatibility to pass, got %v", err) + } + }) +} + +func TestApplyInstanceHTTPRequestAuth(t *testing.T) { + t.Run("applies bearer token from access credential", func(t *testing.T) { + req := httptest.NewRequest("GET", "/ws", nil) + instance := &model.Instance{AccessCredentialID: "cred-1"} + RememberPreviousToken("cred-1", "access-token") + + if err := ApplyInstanceHTTPRequestAuth(req, instance); err != nil { + t.Fatalf("expected auth to apply, got %v", err) + } + + if got := req.Header.Get("Authorization"); got != "Bearer access-token" { + t.Fatalf("unexpected authorization header: %q", got) + } + }) + + t.Run("falls back to direct access token", func(t *testing.T) { + req := httptest.NewRequest("GET", "/ws", nil) + instance := &model.Instance{ + AccessToken: &model.Token{ + Value: "direct-token", + }, + } + + if err := ApplyInstanceHTTPRequestAuth(req, instance); err != nil { + t.Fatalf("expected access token to apply, got %v", err) + } + + if got := req.Header.Get("Authorization"); got != "Bearer direct-token" { + t.Fatalf("unexpected authorization header: %q", got) + } + }) + + t.Run("falls back to basic auth", func(t *testing.T) { + req := httptest.NewRequest("GET", "/ws", nil) + instance := &model.Instance{ + BasicAuth: &model.BasicAuth{ + Username: "managed_gateway", + Password: ucfg.SecretString("secret"), + }, + } + + if err := ApplyInstanceHTTPRequestAuth(req, instance); err != nil { + t.Fatalf("expected basic auth to apply, got %v", err) + } + + user, password, ok := req.BasicAuth() + if !ok || user != "managed_gateway" || password != "secret" { + t.Fatalf("unexpected basic auth credentials: ok=%v user=%q password=%q", ok, user, password) + } + if auth := req.Header.Get("Authorization"); !strings.HasPrefix(auth, "Basic ") { + t.Fatalf("expected basic authorization header, got %q", auth) + } + }) +} diff --git a/modules/agent/common/version.go b/modules/agent/common/version.go new file mode 100644 index 00000000..00b1426f --- /dev/null +++ b/modules/agent/common/version.go @@ -0,0 +1,33 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* Copyright © INFINI Ltd. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package common + +// LegacyAgentMaxVersion is the last agent version that predates several new +// protocol features (multi-path logs_path, token-based managed auth, new install +// script). Agents with a version number <= this value are treated as legacy. +const LegacyAgentMaxVersion = "1.31.0" diff --git a/modules/agent/model/config.go b/modules/agent/model/config.go index a5bae4d1..3ae0f46f 100644 --- a/modules/agent/model/config.go +++ b/modules/agent/model/config.go @@ -33,9 +33,12 @@ type AgentConfig struct { } type SetupConfig struct { - DownloadURL string `config:"download_url"` - CACertFile string `config:"ca_cert"` - CAKeyFile string `config:"ca_key"` - ConsoleEndpoint string `config:"console_endpoint"` - Port string `config:"port"` + DownloadURL string `config:"download_url"` + InstallDir string `config:"install_dir"` + Version string `config:"version"` + CACertFile string `config:"ca_cert"` + CAKeyFile string `config:"ca_key"` + ConsoleEndpoint string `config:"console_endpoint"` + ReverseChannelEndpoints []string `config:"reverse_channel_endpoints"` + Port string `config:"port"` } diff --git a/modules/elastic/api/alias.go b/modules/elastic/api/alias.go index 9203f2d5..61568dda 100644 --- a/modules/elastic/api/alias.go +++ b/modules/elastic/api/alias.go @@ -38,14 +38,14 @@ func (h *APIHandler) HandleAliasAction(w http.ResponseWriter, req *http.Request, exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleAliasAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } if !exists { errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(errStr) + log.Errorf("HandleAliasAction failed: %v", errStr) h.WriteError(w, errStr, http.StatusInternalServerError) return } @@ -54,12 +54,13 @@ func (h *APIHandler) HandleAliasAction(w http.ResponseWriter, req *http.Request, err = h.DecodeJSON(req, aliasReq) if err != nil { - log.Error(err) + log.Errorf("HandleAliasAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + esDistribution := elastic.GetMetadata(targetClusterID).Config.Distribution esVersion := elastic.GetMetadata(targetClusterID).Config.Version - if r, _ := util.VersionCompare(esVersion, "6.4"); r == -1 { + if r, _ := util.VersionCompare(esVersion, "6.4"); r == -1 && esDistribution == "elasticsearch" { for i := range aliasReq.Actions { for k, v := range aliasReq.Actions[i] { if v != nil && v["is_write_index"] != nil { @@ -71,10 +72,9 @@ func (h *APIHandler) HandleAliasAction(w http.ResponseWriter, req *http.Request, } bodyBytes, _ := json.Marshal(aliasReq) - err = client.Alias(bodyBytes) if err != nil { - log.Error(err) + log.Errorf("HandleAliasAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -87,20 +87,20 @@ func (h *APIHandler) HandleGetAliasAction(w http.ResponseWriter, req *http.Reque exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleGetAliasAction failed: %v", err) h.WriteJSON(w, err.Error(), http.StatusInternalServerError) return } if !exists { errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(errStr) + log.Errorf("HandleGetAliasAction failed: %v", errStr) h.WriteError(w, errStr, http.StatusInternalServerError) return } res, err := client.GetAliasesDetail() if err != nil { - log.Error(err) + log.Errorf("HandleGetAliasAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/cluster_overview.go b/modules/elastic/api/cluster_overview.go index 0519646a..613e58f6 100644 --- a/modules/elastic/api/cluster_overview.go +++ b/modules/elastic/api/cluster_overview.go @@ -54,9 +54,6 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, h.WriteJSON(w, util.MapStr{}, http.StatusOK) return } - //only query the first cluster info - clusterIDs = clusterIDs[0:1] - cids := make([]interface{}, 0, len(clusterIDs)) for _, clusterID := range clusterIDs { cids = append(cids, clusterID) @@ -68,7 +65,7 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, q1.Conds = orm.And( orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.name", "cluster_stats"), - orm.Eq("metadata.labels.cluster_id", cids[0]), + orm.In("metadata.labels.cluster_id", cids), ) q1.Collapse("metadata.labels.cluster_id") q1.AddSort("timestamp", orm.DESC) @@ -126,7 +123,7 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, for _, cid := range clusterIDs { clusterUUID, err := adapter.GetClusterUUID(cid) if err != nil { - log.Error(err) + log.Errorf("FetchClusterInfo failed: %v", err) continue } clusterUUIDs = append(clusterUUIDs, clusterUUID) @@ -178,8 +175,8 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, "bool": util.MapStr{ "must": []util.MapStr{ { - "term": util.MapStr{ - "metadata.labels.cluster_uuid": clusterUUIDs[0], + "terms": util.MapStr{ + "metadata.labels.cluster_uuid": clusterUUIDs, }, }, { @@ -261,7 +258,7 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("FetchClusterInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -269,7 +266,7 @@ func (h *APIHandler) FetchClusterInfo(w http.ResponseWriter, req *http.Request, defer cancel() indexMetrics, err := h.getMetrics(ctx, term_level, query, indexMetricItems, bucketSize) if err != nil { - log.Error(err) + log.Errorf("FetchClusterInfo failed: %v", err) if errors.Is(err, context.DeadlineExceeded) { h.WriteError(w, cerr.New(cerr.ErrTypeRequestTimeout, "", err).Error(), http.StatusRequestTimeout) return @@ -537,9 +534,6 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p clusterUUID, err := adapter.GetClusterUUID(id) query := util.MapStr{ "size": 1000, - "collapse": util.MapStr{ - "field": "metadata.labels.node_id", - }, "sort": []util.MapStr{ { "timestamp": util.MapStr{ @@ -606,8 +600,18 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p h.WriteJSON(w, resBody, http.StatusInternalServerError) } nodeInfos := map[string]util.MapStr{} + seenNodeInfos := map[string]struct{}{} for _, hit := range searchResult.Result { if hitM, ok := hit.(map[string]interface{}); ok { + nodeID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, hitM) + nodeIDStr := util.ToString(nodeID) + if nodeIDStr == "" { + continue + } + if _, exists := seenNodeInfos[nodeIDStr]; exists { + continue + } + seenNodeInfos[nodeIDStr] = struct{}{} shardInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, hitM) var totalShards float64 if v, ok := shardInfo.(map[string]interface{}); ok { @@ -626,7 +630,6 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p heapUsage, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "mem", "heap_used_percent"}, hitM) availDisk, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total", "available_in_bytes"}, hitM) totalDisk, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total", "total_in_bytes"}, hitM) - nodeID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, hitM) var usedDisk string if v, ok := availDisk.(float64); ok { availDisk = util.ByteSize(uint64(v)) @@ -635,8 +638,8 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p } } - if v, ok := nodeID.(string); ok { - nodeInfos[v] = util.MapStr{ + if nodeIDStr != "" { + nodeInfos[nodeIDStr] = util.MapStr{ "timestamp": hitM["timestamp"], "shards": totalShards, "cpu": cpu, @@ -718,7 +721,7 @@ func (h *APIHandler) GetRealtimeClusterNodes(w http.ResponseWriter, req *http.Re } catShardsInfo, err := esClient.CatShards() if err != nil { - log.Error(err) + log.Errorf("GetRealtimeClusterNodes failed: %v", err) } shardCounts := map[string]int{} nodeM := map[string]string{} @@ -1042,9 +1045,13 @@ func (h *APIHandler) getShardQPS(clusterID string, nodeUUID string, indexName st } if nodeUUID != "" { must = append(must, util.MapStr{ - "term": util.MapStr{ - "metadata.labels.node_id": util.MapStr{ - "value": nodeUUID, + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": []util.MapStr{ + {"term": util.MapStr{"metadata.labels.node_id": util.MapStr{"value": nodeUUID}}}, + {"term": util.MapStr{"metadata.labels.node_uuid": util.MapStr{"value": nodeUUID}}}, + {"term": util.MapStr{"payload.elasticsearch.shard_stats.routing.node": util.MapStr{"value": nodeUUID}}}, + {"term": util.MapStr{"payload.elasticsearch.shard_stats.routing.current_node": util.MapStr{"value": nodeUUID}}}, }, }, }) @@ -1326,7 +1333,7 @@ func (h *APIHandler) SearchClusterMetadata(w http.ResponseWriter, req *http.Requ } } - clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "id") if !hasAllPrivilege && clusterFilter == nil { h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) return @@ -1444,7 +1451,7 @@ func (h *APIHandler) getClusterMonitorState(w http.ResponseWriter, req *http.Req dsl := util.MustToJSONBytes(queryDSL) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), dsl) if err != nil { - log.Error(err) + log.Errorf("getClusterMonitorState failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/discover.go b/modules/elastic/api/discover.go index f30626f1..c102bac7 100644 --- a/modules/elastic/api/discover.go +++ b/modules/elastic/api/discover.go @@ -34,22 +34,50 @@ import ( "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "net/http" + "strings" "time" ) +const kueryWildcardSymbol = "@kuery-wildcard@" + +func escapeSuggestionQuery(query string) string { + replacer := strings.NewReplacer( + ".", "\\.", + "?", "\\?", + "+", "\\+", + "*", "\\*", + "|", "\\|", + "{", "\\{", + "}", "\\}", + "[", "\\[", + "]", "\\]", + "(", "\\(", + ")", "\\)", + "\"", "\\\"", + "\\", "\\\\", + "#", "\\#", + "@", "\\@", + "&", "\\&", + "<", "\\<", + ">", "\\>", + "~", "\\~", + ) + return replacer.Replace(query) +} + func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { targetClusterID := ps.ByName("id") exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleEseSearchAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } if !exists { errStr := fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(errStr) + log.Errorf("HandleEseSearchAction failed: %v", errStr) h.WriteError(w, errStr, http.StatusNotFound) return } @@ -62,14 +90,15 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ err = h.DecodeJSON(req, &reqParams) if err != nil { - log.Error(err) + log.Errorf("HandleEseSearchAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + sanitizeAutoRangeInMap(reqParams.Body) //validate index search api permission reqUser, err := security.FromUserContext(req.Context()) if err != nil { - log.Error(err) + log.Errorf("HandleEseSearchAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -82,7 +111,7 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ err = security.ValidateIndex(indexReq, newRole) if err != nil { - log.Error(err) + log.Errorf("HandleEseSearchAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusForbidden) return } @@ -121,7 +150,7 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ vr, err := util.VersionCompare(ver.Number, "7.2") if err != nil { errStr := fmt.Sprintf("version compare error: %v", err) - log.Error(errStr) + log.Errorf("HandleEseSearchAction failed: %v", errStr) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -207,13 +236,102 @@ func (h *APIHandler) HandleEseSearchAction(w http.ResponseWriter, req *http.Requ h.Write(w, searchRes.RawResult.Body) } +func sanitizeAutoRangeInMap(data map[string]interface{}) bool { + if data == nil { + return false + } + changed := false + for key, val := range data { + switch typed := val.(type) { + case map[string]interface{}: + if key == "range" { + if sanitizeAutoRangeNode(typed) { + changed = true + } + if len(typed) == 0 { + delete(data, key) + changed = true + } + continue + } + if sanitizeAutoRangeInMap(typed) { + changed = true + } + if len(typed) == 0 { + delete(data, key) + changed = true + } + case []interface{}: + newList := make([]interface{}, 0, len(typed)) + listChanged := false + for _, item := range typed { + if itemMap, ok := item.(map[string]interface{}); ok { + if sanitizeAutoRangeInMap(itemMap) { + listChanged = true + } + if len(itemMap) == 0 { + listChanged = true + continue + } + newList = append(newList, itemMap) + continue + } + newList = append(newList, item) + } + if listChanged { + data[key] = newList + changed = true + } + } + } + return changed +} + +func sanitizeAutoRangeNode(rangeNode map[string]interface{}) bool { + changed := false + for field, condVal := range rangeNode { + cond, ok := condVal.(map[string]interface{}) + if !ok { + continue + } + for _, boundKey := range []string{"gte", "lte", "gt", "lt", "from", "to"} { + if isAutoRangeValue(cond[boundKey]) { + delete(cond, boundKey) + changed = true + } + } + if !hasRangeBounds(cond) { + delete(cond, "format") + } + if len(cond) == 0 || !hasRangeBounds(cond) { + delete(rangeNode, field) + changed = true + } + } + return changed +} + +func hasRangeBounds(rangeCond map[string]interface{}) bool { + for _, key := range []string{"gte", "lte", "gt", "lt", "from", "to"} { + if _, ok := rangeCond[key]; ok { + return true + } + } + return false +} + +func isAutoRangeValue(value interface{}) bool { + v, ok := value.(string) + return ok && strings.EqualFold(strings.TrimSpace(v), "auto") +} + func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { resBody := map[string]interface{}{} targetClusterID := ps.ByName("id") exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleValueSuggestionAction failed: %v", err) resBody["error"] = err.Error() h.WriteError(w, err.Error(), http.StatusInternalServerError) return @@ -226,21 +344,34 @@ func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *htt } var reqParams = struct { - BoolFilter interface{} `json:"boolFilter"` - FieldName string `json:"field"` - Query string `json:"query"` + BoolFilter interface{} `json:"boolFilter"` + BoolFilterAlt interface{} `json:"bool_filter"` + FieldName string `json:"field"` + Query string `json:"query"` }{} err = h.DecodeJSON(req, &reqParams) if err != nil { - log.Error(err) + log.Errorf("HandleValueSuggestionAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } indexName := ps.ByName("index") - boolQ := util.MapStr{ - "filter": reqParams.BoolFilter, + boolFilter := reqParams.BoolFilter + if boolFilter == nil { + boolFilter = reqParams.BoolFilterAlt + } + if boolFilter == nil { + boolFilter = []interface{}{} } var values = []interface{}{} + query := strings.TrimSpace(reqParams.Query) + if query == "" || query == kueryWildcardSymbol || reqParams.FieldName == "" { + h.WriteJSON(w, values, http.StatusOK) + return + } + boolQ := util.MapStr{ + "filter": boolFilter, + } indices, hasAll := h.GetAllowedIndices(req, targetClusterID) if !hasAll { if len(indices) == 0 { @@ -255,19 +386,22 @@ func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *htt }, } } + termsAgg := util.MapStr{ + "field": reqParams.FieldName, + "include": escapeSuggestionQuery(query) + ".*", + "execution_hint": "map", + "shard_size": 10, + } queryBody := util.MapStr{ - "size": 0, + "size": 0, + "timeout": "1500ms", + "terminate_after": 10000, "query": util.MapStr{ "bool": boolQ, }, "aggs": util.MapStr{ "suggestions": util.MapStr{ - "terms": util.MapStr{ - "field": reqParams.FieldName, - "include": reqParams.Query + ".*", - "execution_hint": "map", - "shard_size": 10, - }, + "terms": termsAgg, }, }, } @@ -275,12 +409,17 @@ func (h *APIHandler) HandleValueSuggestionAction(w http.ResponseWriter, req *htt searchRes, err := client.SearchWithRawQueryDSL(indexName, queryBodyBytes) if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Warnf("HandleValueSuggestionAction fallback to empty suggestions: %v", err) + h.WriteJSON(w, values, http.StatusOK) return } - for _, bucket := range searchRes.Aggregations["suggestions"].Buckets { + suggestionAgg, ok := searchRes.Aggregations["suggestions"] + if !ok { + h.WriteJSON(w, values, http.StatusOK) + return + } + for _, bucket := range suggestionAgg.Buckets { values = append(values, bucket["key"]) } h.WriteJSON(w, values, http.StatusOK) @@ -294,7 +433,7 @@ func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http. exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleTraceIDSearchAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -324,7 +463,7 @@ func (h *APIHandler) HandleTraceIDSearchAction(w http.ResponseWriter, req *http. } searchRes, err := client.SearchWithRawQueryDSL(traceIndex, util.MustToJSONBytes(queryDSL)) if err != nil { - log.Error(err) + log.Errorf("HandleTraceIDSearchAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/host.go b/modules/elastic/api/host.go index 7eb380ac..9717623b 100644 --- a/modules/elastic/api/host.go +++ b/modules/elastic/api/host.go @@ -156,7 +156,7 @@ func (h *APIHandler) updateHost(w http.ResponseWriter, req *http.Request, ps htt obj := host.HostInfo{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -169,7 +169,7 @@ func (h *APIHandler) updateHost(w http.ResponseWriter, req *http.Request, ps htt err = h.DecodeJSON(req, &toUpObj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("updateHost failed: %v", err) return } @@ -181,10 +181,10 @@ func (h *APIHandler) updateHost(w http.ResponseWriter, req *http.Request, ps htt if toUpObj.IP != "" { obj.IP = toUpObj.IP } - err = orm.Save(nil, &obj) + err = orm.Save(&orm.Context{Refresh: "wait_for"}, &obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("updateHost failed: %v", err) return } @@ -197,7 +197,7 @@ func (h *APIHandler) updateHost(w http.ResponseWriter, req *http.Request, ps htt func (h *APIHandler) getDiscoverHosts(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { hosts, err := discoverHost() if err != nil { - log.Error(err) + log.Errorf("getDiscoverHosts failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -295,9 +295,6 @@ func getHostSummaryFromNode(nodeIDs []string) (map[string]util.MapStr, error) { }, }, }, - "collapse": util.MapStr{ - "field": "metadata.labels.node_id", - }, "query": util.MapStr{ "bool": util.MapStr{ "must": []util.MapStr{ @@ -331,12 +328,17 @@ func getHostSummaryFromNode(nodeIDs []string) (map[string]util.MapStr, error) { return nil, err } summary := map[string]util.MapStr{} + seenNodeIDs := map[string]struct{}{} for _, v := range results.Result { result, ok := v.(map[string]interface{}) if ok { nodeID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, result) if ok { strNodeID := util.ToString(nodeID) + if _, exists := seenNodeIDs[strNodeID]; exists { + continue + } + seenNodeIDs[strNodeID] = struct{}{} summary[strNodeID] = util.MapStr{} osCPUPercent, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "cpu", "percent"}, result) if ok { @@ -409,7 +411,7 @@ func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps } err, result := orm.Search(host.HostInfo{}, q) if err != nil { - log.Error(err) + log.Errorf("FetchHostInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -427,7 +429,7 @@ func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps buf := util.MustToJSONBytes(row) err = util.FromJSONBytes(buf, &tempHost) if err != nil { - log.Error(err) + log.Errorf("FetchHostInfo failed: %v", err) continue } if tempHost.AgentID != "" { @@ -443,7 +445,7 @@ func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps summaryFromAgent, err := getHostSummaryFromAgent(agentIDs) if err != nil { - log.Error(err) + log.Errorf("FetchHostInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -452,7 +454,7 @@ func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps if len(nodeIDs) > 0 { summaryFromNode, err = getHostSummaryFromNode(nodeIDs) if err != nil { - log.Error(err) + log.Errorf("FetchHostInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -460,7 +462,7 @@ func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps statusMetric, err := getAgentOnlineStatusOfRecentDay(hostIDs) if err != nil { - log.Error(err) + log.Errorf("FetchHostInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -495,7 +497,7 @@ func (h *APIHandler) FetchHostInfo(w http.ResponseWriter, req *http.Request, ps } hostMetrics, err := h.getGroupHostMetric(context.Background(), agentIDs, min, max, bucketSize, hostMetricItems, "agent.id") if err != nil { - log.Error(err) + log.Errorf("FetchHostInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -552,9 +554,9 @@ func (h *APIHandler) GetHostInfo(w http.ResponseWriter, req *http.Request, ps ht hostID := ps.MustGetParameter("host_id") hostInfo := &host.HostInfo{} hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) + exists, err := orm.GetV2(orm.NewContext(), hostInfo) if err != nil { - log.Error(err) + log.Errorf("GetHostInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -697,9 +699,9 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque hostID := ps.MustGetParameter("host_id") hostInfo := &host.HostInfo{} hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) + exists, err := orm.GetV2(orm.NewContext(), hostInfo) if err != nil { - log.Error(err) + log.Errorf("GetSingleHostMetrics failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -711,7 +713,7 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque resBody := map[string]interface{}{} bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, "", "", 60) if err != nil { - log.Error(err) + log.Errorf("GetSingleHostMetrics failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -720,7 +722,7 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("GetSingleHostMetrics failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -729,7 +731,7 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque if hostInfo.AgentID == "" { resBody["metrics"], err = h.getSingleHostMetricFromNode(ctx, hostInfo.NodeID, min, max, bucketSize, key) if err != nil { - log.Error(err) + log.Errorf("GetSingleHostMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -797,7 +799,7 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque case DiskPartitionUsageMetricKey, NetworkInterfaceOutputRateMetricKey: resBody["metrics"], err = h.getGroupHostMetrics(ctx, hostInfo.AgentID, min, max, bucketSize, key) if err != nil { - log.Error(err) + log.Errorf("GetSingleHostMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -806,7 +808,7 @@ func (h *APIHandler) GetSingleHostMetrics(w http.ResponseWriter, req *http.Reque } hostMetrics, err := h.getSingleHostMetric(ctx, hostInfo.AgentID, min, max, bucketSize, metricItems) if err != nil { - log.Error(err) + log.Errorf("GetSingleHostMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -915,7 +917,7 @@ func (h *APIHandler) getGroupHostMetric(ctx context.Context, agentIDs []string, func getHost(hostID string) (*host.HostInfo, error) { hostInfo := &host.HostInfo{} hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) + exists, err := orm.GetV2(orm.NewContext(), hostInfo) if err != nil { return nil, fmt.Errorf("get host info error: %w", err) } @@ -929,7 +931,7 @@ func (h *APIHandler) GetHostMetricStats(w http.ResponseWriter, req *http.Request hostID := ps.MustGetParameter("host_id") hostInfo, err := getHost(hostID) if err != nil { - log.Error(err) + log.Errorf("GetHostMetricStats failed: %v", err) h.WriteJSON(w, util.MapStr{}, http.StatusOK) return } @@ -1016,9 +1018,9 @@ func (h *APIHandler) GetHostOverviewInfo(w http.ResponseWriter, req *http.Reques hostID := ps.MustGetParameter("host_id") hostInfo := &host.HostInfo{} hostInfo.ID = hostID - exists, err := orm.Get(hostInfo) + exists, err := orm.GetV2(orm.NewContext(), hostInfo) if err != nil { - log.Error(err) + log.Errorf("GetHostOverviewInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -1034,20 +1036,18 @@ func (h *APIHandler) GetHostOverviewInfo(w http.ResponseWriter, req *http.Reques summary util.MapStr ) if hostInfo.AgentID != "" { - summaries, err := getHostSummaryFromAgent([]string{hostID}) + summaries, err := getHostSummaryFromAgent([]string{hostInfo.AgentID}) if err != nil { - log.Error(err) + log.Errorf("GetHostOverviewInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - if v, ok := summaries[hostID]; ok { - summary = v - } + summary = resolveHostOverviewAgentSummary(hostInfo, summaries) } else if hostInfo.NodeID != "" { summaries, err := getHostSummaryFromNode([]string{hostInfo.NodeID}) if err != nil { - log.Error(err) + log.Errorf("GetHostOverviewInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -1066,6 +1066,23 @@ func (h *APIHandler) GetHostOverviewInfo(w http.ResponseWriter, req *http.Reques } +func resolveHostOverviewAgentSummary(hostInfo *host.HostInfo, summaries map[string]util.MapStr) util.MapStr { + if hostInfo == nil || len(summaries) == 0 { + return nil + } + if hostInfo.AgentID != "" { + if summary, ok := summaries[hostInfo.AgentID]; ok { + return summary + } + } + if hostInfo.ID != "" { + if summary, ok := summaries[hostInfo.ID]; ok { + return summary + } + } + return nil +} + // discoverHost auto discover host ip from elasticsearch node metadata and agent ips func discoverHost() (map[string]interface{}, error) { queryDsl := util.MapStr{ diff --git a/modules/elastic/api/host_test.go b/modules/elastic/api/host_test.go new file mode 100644 index 00000000..52f06be5 --- /dev/null +++ b/modules/elastic/api/host_test.go @@ -0,0 +1,28 @@ +package api + +import ( + "testing" + + "infini.sh/framework/core/host" + "infini.sh/framework/core/util" +) + +func TestResolveHostOverviewAgentSummaryPrefersAgentID(t *testing.T) { + hostInfo := &host.HostInfo{ + AgentID: "agent-1", + } + hostInfo.ID = "host-1" + + summary := resolveHostOverviewAgentSummary(hostInfo, map[string]util.MapStr{ + "agent-1": {"cpu": util.MapStr{"used_percent": 42}}, + "host-1": {"cpu": util.MapStr{"used_percent": 1}}, + }) + + if summary == nil { + t.Fatal("expected agent summary") + } + cpu, _ := summary.GetValue("cpu.used_percent") + if cpu != 42 { + t.Fatalf("unexpected summary selected: %#v", summary) + } +} diff --git a/modules/elastic/api/ilm.go b/modules/elastic/api/ilm.go index 49925214..cd06a45a 100644 --- a/modules/elastic/api/ilm.go +++ b/modules/elastic/api/ilm.go @@ -28,19 +28,302 @@ package api import ( + "context" + "encoding/json" + "fmt" log "github.com/cihub/seelog" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/util" "io" "net/http" + "net/url" + "strconv" + "strings" ) +type rawRequester interface { + Request(ctx context.Context, method, url string, body []byte) (*util.Result, error) +} + +const ( + elasticsearchWaitForSnapshotMinVersion = "8.1.0" + elasticsearchDeleteSearchableSnapshotVersion = "8.13.0" +) + +func sanitizeILMPolicyForTarget(requester rawRequester, cfg *elastic.ElasticsearchConfig, policyConfig []byte) ([]byte, error) { + if len(policyConfig) == 0 { + return policyConfig, nil + } + payload := map[string]interface{}{} + if err := util.FromJSONBytes(policyConfig, &payload); err != nil { + return nil, err + } + changed, err := sanitizeILMPolicyPayload(requester, cfg, payload) + if err != nil { + return nil, err + } + if !changed { + return policyConfig, nil + } + return util.MustToJSONBytes(payload), nil +} + +func sanitizeILMPolicyPayload(requester rawRequester, cfg *elastic.ElasticsearchConfig, payload map[string]interface{}) (bool, error) { + switch { + case strings.EqualFold(cfg.Distribution, elastic.Opensearch): + return removeUnsupportedDeleteActionsFromISM(payload), nil + case strings.EqualFold(cfg.Distribution, elastic.Easysearch): + return removeUnsupportedDeleteActionsFromPhases(payload, false, false), nil + default: + return removeMissingSLMReferences(requester, cfg, payload) + } +} + +func removeMissingSLMReferences(requester rawRequester, cfg *elastic.ElasticsearchConfig, payload map[string]interface{}) (bool, error) { + waitForSnapshotSupported := supportsElasticsearchDeleteAction(cfg, elasticsearchWaitForSnapshotMinVersion) + deleteSearchableSnapshotSupported := supportsElasticsearchDeleteAction(cfg, elasticsearchDeleteSearchableSnapshotVersion) + changed := removeUnsupportedDeleteActionsFromPhases(payload, waitForSnapshotSupported, deleteSearchableSnapshotSupported) + phases := getILMPhases(payload) + if len(phases) == 0 { + return changed, nil + } + existsCache := map[string]bool{} + if !waitForSnapshotSupported { + return changed, nil + } + for _, phaseValue := range phases { + phase, ok := phaseValue.(map[string]interface{}) + if !ok || phase == nil { + continue + } + actions, ok := phase["actions"].(map[string]interface{}) + if !ok || actions == nil { + continue + } + waitForSnapshot, ok := actions["wait_for_snapshot"].(map[string]interface{}) + if !ok || waitForSnapshot == nil { + continue + } + policyName := strings.TrimSpace(util.ToString(waitForSnapshot["policy"])) + if policyName == "" { + delete(actions, "wait_for_snapshot") + changed = true + continue + } + exists, cached := existsCache[policyName] + if !cached { + statusCode := http.StatusOK + _, statusCode, err := rawJSONRequestWithRequester(requester, cfg, util.Verb_GET, "/_slm/policy/"+url.PathEscape(policyName), nil) + if err != nil && statusCode != http.StatusNotFound { + return changed, err + } + exists = statusCode != http.StatusNotFound + existsCache[policyName] = exists + } + if !exists { + delete(actions, "wait_for_snapshot") + changed = true + } + } + return changed, nil +} + +func removeUnsupportedDeleteActionsFromPhases(payload map[string]interface{}, waitForSnapshotSupported bool, deleteSearchableSnapshotSupported bool) bool { + phases := getILMPhases(payload) + if len(phases) == 0 { + return false + } + changed := false + for _, phaseValue := range phases { + phase, ok := phaseValue.(map[string]interface{}) + if !ok || phase == nil { + continue + } + actions, ok := phase["actions"].(map[string]interface{}) + if !ok || actions == nil { + continue + } + if !waitForSnapshotSupported { + if _, exists := actions["wait_for_snapshot"]; exists { + delete(actions, "wait_for_snapshot") + changed = true + } + } + deleteAction, ok := actions["delete"].(map[string]interface{}) + if !ok || deleteAction == nil { + continue + } + if !deleteSearchableSnapshotSupported { + if _, exists := deleteAction["delete_searchable_snapshot"]; exists { + delete(deleteAction, "delete_searchable_snapshot") + changed = true + } + } + } + return changed +} + +func removeUnsupportedDeleteActionsFromISM(payload map[string]interface{}) bool { + policy, _ := payload["policy"].(map[string]interface{}) + states, _ := policy["states"].([]interface{}) + if len(states) == 0 { + return false + } + changed := false + for _, stateValue := range states { + state, ok := stateValue.(map[string]interface{}) + if !ok || state == nil { + continue + } + actions, _ := state["actions"].([]interface{}) + if len(actions) == 0 { + continue + } + filtered := actions[:0] + for _, actionValue := range actions { + action, ok := actionValue.(map[string]interface{}) + if !ok || action == nil { + filtered = append(filtered, actionValue) + continue + } + if _, exists := action["wait_for_snapshot"]; exists { + changed = true + continue + } + deleteAction, ok := action["delete"].(map[string]interface{}) + if ok && deleteAction != nil { + if _, exists := deleteAction["delete_searchable_snapshot"]; exists { + delete(deleteAction, "delete_searchable_snapshot") + changed = true + } + } + filtered = append(filtered, actionValue) + } + state["actions"] = filtered + } + return changed +} + +func supportsElasticsearchDeleteAction(cfg *elastic.ElasticsearchConfig, minVersion string) bool { + if cfg == nil || !strings.EqualFold(cfg.Distribution, elastic.Elasticsearch) { + return false + } + if strings.TrimSpace(cfg.Version) == "" { + return false + } + cr, err := util.VersionCompare(cfg.Version, minVersion) + if err != nil { + return false + } + return cr >= 0 +} + +func getILMPhases(payload map[string]interface{}) map[string]interface{} { + policy, _ := payload["policy"].(map[string]interface{}) + phases, _ := policy["phases"].(map[string]interface{}) + return phases +} + +func rawJSONRequest(clusterID, method, path string, body []byte) (map[string]interface{}, int, error) { + cfg := elastic.GetConfig(clusterID) + client := elastic.GetClient(clusterID) + requester, ok := client.(rawRequester) + if !ok { + return nil, 0, fmt.Errorf("cluster client does not support raw requests") + } + return rawJSONRequestWithRequester(requester, cfg, method, path, body) +} + +func rawJSONRequestWithRequester(requester rawRequester, cfg *elastic.ElasticsearchConfig, method, path string, body []byte) (map[string]interface{}, int, error) { + requestURL := fmt.Sprintf("%s%s", strings.TrimRight(cfg.GetAnyEndpoint(), "/"), path) + resp, err := requester.Request(context.Background(), method, requestURL, body) + if err != nil { + return nil, 0, err + } + if resp.StatusCode == http.StatusNotFound { + return nil, resp.StatusCode, nil + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, resp.StatusCode, fmt.Errorf("%s", resp.Body) + } + if len(resp.Body) == 0 { + return map[string]interface{}{}, resp.StatusCode, nil + } + result := map[string]interface{}{} + if err := util.FromJSONBytes(resp.Body, &result); err != nil { + return nil, resp.StatusCode, err + } + return result, resp.StatusCode, nil +} + +func parseInt64(value interface{}) int64 { + switch v := value.(type) { + case json.Number: + i, _ := v.Int64() + return i + case float64: + return int64(v) + case int64: + return v + case int: + return int64(v) + case string: + i, _ := strconv.ParseInt(v, 10, 64) + return i + default: + return 0 + } +} + +func putVersionedILMPolicy(requester rawRequester, cfg *elastic.ElasticsearchConfig, path string, policyConfig []byte) error { + current, statusCode, err := rawJSONRequestWithRequester(requester, cfg, util.Verb_GET, path, nil) + if err != nil && statusCode != http.StatusNotFound { + return err + } + if statusCode != http.StatusNotFound { + seqNo := parseInt64(current["_seq_no"]) + primaryTerm := parseInt64(current["_primary_term"]) + path += fmt.Sprintf("?if_seq_no=%d&if_primary_term=%d", seqNo, primaryTerm) + } + _, _, err = rawJSONRequestWithRequester(requester, cfg, util.Verb_PUT, path, policyConfig) + return err +} + +func putEasysearchILMPolicy(clusterID, policy string, policyConfig []byte) error { + cfg := elastic.GetConfig(clusterID) + client := elastic.GetClient(clusterID) + requester, ok := client.(rawRequester) + if !ok { + return fmt.Errorf("cluster client does not support raw requests") + } + sanitizedPolicyConfig, err := sanitizeILMPolicyForTarget(requester, cfg, policyConfig) + if err != nil { + return err + } + return putVersionedILMPolicy(requester, cfg, "/_ilm/policy/"+url.PathEscape(policy), sanitizedPolicyConfig) +} + +func putOpensearchILMPolicy(clusterID, policy string, policyConfig []byte) error { + cfg := elastic.GetConfig(clusterID) + client := elastic.GetClient(clusterID) + requester, ok := client.(rawRequester) + if !ok { + return fmt.Errorf("cluster client does not support raw requests") + } + sanitizedPolicyConfig, err := sanitizeILMPolicyForTarget(requester, cfg, policyConfig) + if err != nil { + return err + } + return putVersionedILMPolicy(requester, cfg, "/_plugins/_ism/policies/"+url.PathEscape(policy), sanitizedPolicyConfig) +} + func (h *APIHandler) HandleGetILMPolicyAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { clusterID := ps.MustGetParameter("id") esClient := elastic.GetClient(clusterID) policies, err := esClient.GetILMPolicy("") if err != nil { - log.Error(err) + log.Errorf("HandleGetILMPolicyAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -51,15 +334,31 @@ func (h *APIHandler) HandleSaveILMPolicyAction(w http.ResponseWriter, req *http. clusterID := ps.MustGetParameter("id") policy := ps.MustGetParameter("policy") esClient := elastic.GetClient(clusterID) + cfg := elastic.GetConfig(clusterID) reqBody, err := io.ReadAll(req.Body) if err != nil { - log.Error(err) + log.Errorf("HandleSaveILMPolicyAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - err = esClient.PutILMPolicy(policy, reqBody) + requester, requesterOK := esClient.(rawRequester) + if requesterOK { + reqBody, err = sanitizeILMPolicyForTarget(requester, cfg, reqBody) + if err != nil { + log.Errorf("HandleSaveILMPolicyAction failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + if strings.EqualFold(cfg.Distribution, elastic.Easysearch) { + err = putEasysearchILMPolicy(clusterID, policy, reqBody) + } else if strings.EqualFold(cfg.Distribution, elastic.Opensearch) { + err = putOpensearchILMPolicy(clusterID, policy, reqBody) + } else { + err = esClient.PutILMPolicy(policy, reqBody) + } if err != nil { - log.Error(err) + log.Errorf("HandleSaveILMPolicyAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -72,7 +371,7 @@ func (h *APIHandler) HandleDeleteILMPolicyAction(w http.ResponseWriter, req *htt esClient := elastic.GetClient(clusterID) err := esClient.DeleteILMPolicy(policy) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteILMPolicyAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/ilm_test.go b/modules/elastic/api/ilm_test.go new file mode 100644 index 00000000..afea310c --- /dev/null +++ b/modules/elastic/api/ilm_test.go @@ -0,0 +1,192 @@ +package api + +import ( + "context" + "net/http" + "strings" + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/util" +) + +func TestPutVersionedILMPolicyUsesExistingSeqNoAndPrimaryTerm(t *testing.T) { + requester := &mockILMRequester{ + responses: map[string]*util.Result{ + "GET http://example.com/_plugins/_ism/policies/metrics": { + StatusCode: http.StatusOK, + Body: []byte(`{"_id":"metrics","_seq_no":7,"_primary_term":3,"policy":{"policy_id":"metrics"}}`), + }, + "PUT http://example.com/_plugins/_ism/policies/metrics?if_seq_no=7&if_primary_term=3": { + StatusCode: http.StatusOK, + Body: []byte(`{"_id":"metrics","_seq_no":8,"_primary_term":3}`), + }, + }, + } + + err := putVersionedILMPolicy( + requester, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com"}, + "/_plugins/_ism/policies/metrics", + []byte(`{"policy":{"policy_id":"metrics"}}`), + ) + if err != nil { + t.Fatalf("putVersionedILMPolicy returned error: %v", err) + } + + if len(requester.calls) != 2 { + t.Fatalf("expected 2 requests, got %#v", requester.calls) + } + if requester.calls[1] != "PUT http://example.com/_plugins/_ism/policies/metrics?if_seq_no=7&if_primary_term=3" { + t.Fatalf("expected versioned put request, got %s", requester.calls[1]) + } +} + +func TestPutVersionedILMPolicyCreatesWhenPolicyMissing(t *testing.T) { + requester := &mockILMRequester{ + responses: map[string]*util.Result{ + "GET http://example.com/_plugins/_ism/policies/metrics": { + StatusCode: http.StatusNotFound, + Body: []byte(`{"error":"not found"}`), + }, + "PUT http://example.com/_plugins/_ism/policies/metrics": { + StatusCode: http.StatusCreated, + Body: []byte(`{"_id":"metrics","_seq_no":1,"_primary_term":1}`), + }, + }, + } + + err := putVersionedILMPolicy( + requester, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com"}, + "/_plugins/_ism/policies/metrics", + []byte(`{"policy":{"policy_id":"metrics"}}`), + ) + if err != nil { + t.Fatalf("putVersionedILMPolicy returned error: %v", err) + } + + if len(requester.calls) != 2 { + t.Fatalf("expected 2 requests, got %#v", requester.calls) + } + if requester.calls[1] != "PUT http://example.com/_plugins/_ism/policies/metrics" { + t.Fatalf("expected create put request without version params, got %s", requester.calls[1]) + } +} + +func TestSanitizeILMPolicyForTargetRemovesWaitForSnapshotWhenSLMMissing(t *testing.T) { + requester := &mockILMRequester{ + responses: map[string]*util.Result{ + "GET http://example.com/_slm/policy/daily-backup": { + StatusCode: http.StatusNotFound, + Body: []byte(`{"error":"not found"}`), + }, + }, + } + + body, err := sanitizeILMPolicyForTarget( + requester, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com", Distribution: elastic.Elasticsearch, Version: "8.13.0"}, + []byte(`{"policy":{"phases":{"delete":{"min_age":"90d","actions":{"delete":{"delete_searchable_snapshot":true},"wait_for_snapshot":{"policy":"daily-backup"}}}}}}`), + ) + if err != nil { + t.Fatalf("sanitizeILMPolicyForTarget returned error: %v", err) + } + + output := string(body) + if strings.Contains(output, "wait_for_snapshot") { + t.Fatalf("expected wait_for_snapshot to be removed, got %s", output) + } + if !strings.Contains(output, "delete_searchable_snapshot") { + t.Fatalf("expected delete_searchable_snapshot to be preserved, got %s", output) + } +} + +func TestSanitizeILMPolicyForTargetPreservesWaitForSnapshotWhenSLMExists(t *testing.T) { + requester := &mockILMRequester{ + responses: map[string]*util.Result{ + "GET http://example.com/_slm/policy/daily-backup": { + StatusCode: http.StatusOK, + Body: []byte(`{"policies":{"daily-backup":{"version":1}}}`), + }, + }, + } + + body, err := sanitizeILMPolicyForTarget( + requester, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com", Distribution: elastic.Elasticsearch, Version: "8.13.0"}, + []byte(`{"policy":{"phases":{"delete":{"min_age":"90d","actions":{"wait_for_snapshot":{"policy":"daily-backup"}}}}}}`), + ) + if err != nil { + t.Fatalf("sanitizeILMPolicyForTarget returned error: %v", err) + } + + if !strings.Contains(string(body), "wait_for_snapshot") { + t.Fatalf("expected wait_for_snapshot to be preserved, got %s", string(body)) + } +} + +func TestSanitizeILMPolicyForTargetRemovesWaitForSnapshotBeforeEightOne(t *testing.T) { + body, err := sanitizeILMPolicyForTarget( + &mockILMRequester{}, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com", Distribution: elastic.Elasticsearch, Version: "8.0.0"}, + []byte(`{"policy":{"phases":{"delete":{"actions":{"wait_for_snapshot":{"policy":"daily-backup"}}}}}}`), + ) + if err != nil { + t.Fatalf("sanitizeILMPolicyForTarget returned error: %v", err) + } + + if strings.Contains(string(body), "wait_for_snapshot") { + t.Fatalf("expected wait_for_snapshot to be removed before 8.1, got %s", string(body)) + } +} + +func TestSanitizeILMPolicyForTargetRemovesDeleteSearchableSnapshotBeforeEightThirteen(t *testing.T) { + body, err := sanitizeILMPolicyForTarget( + &mockILMRequester{}, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com", Distribution: elastic.Elasticsearch, Version: "8.12.0"}, + []byte(`{"policy":{"phases":{"delete":{"actions":{"delete":{"delete_searchable_snapshot":true}}}}}}`), + ) + if err != nil { + t.Fatalf("sanitizeILMPolicyForTarget returned error: %v", err) + } + + if strings.Contains(string(body), "delete_searchable_snapshot") { + t.Fatalf("expected delete_searchable_snapshot to be removed before 8.13, got %s", string(body)) + } +} + +func TestSanitizeILMPolicyForTargetRemovesWaitForSnapshotForOpensearch(t *testing.T) { + body, err := sanitizeILMPolicyForTarget( + &mockILMRequester{}, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com", Distribution: elastic.Opensearch}, + []byte(`{"policy":{"states":[{"name":"delete","actions":[{"wait_for_snapshot":{"policy":"daily-backup"}},{"delete":{}}]}]}}`), + ) + if err != nil { + t.Fatalf("sanitizeILMPolicyForTarget returned error: %v", err) + } + + if strings.Contains(string(body), "wait_for_snapshot") { + t.Fatalf("expected wait_for_snapshot to be removed for opensearch payload, got %s", string(body)) + } + if strings.Contains(string(body), "delete_searchable_snapshot") { + t.Fatalf("expected delete_searchable_snapshot to be removed for opensearch payload, got %s", string(body)) + } +} + +type mockILMRequester struct { + responses map[string]*util.Result + calls []string +} + +func (m *mockILMRequester) Request(_ context.Context, method, requestURL string, _ []byte) (*util.Result, error) { + key := method + " " + requestURL + m.calls = append(m.calls, key) + if response, ok := m.responses[key]; ok { + return response, nil + } + return &util.Result{ + StatusCode: http.StatusNotFound, + Body: []byte(`{"error":"not found"}`), + }, nil +} diff --git a/modules/elastic/api/index_metrics.go b/modules/elastic/api/index_metrics.go index 2961b993..8302d5db 100644 --- a/modules/elastic/api/index_metrics.go +++ b/modules/elastic/api/index_metrics.go @@ -122,7 +122,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu top = len(indexNames) } else { - indexNames, err = h.getTopIndexName(req, clusterID, top, min, max) + indexNames, err = h.getTopIndexName(req, clusterID, top, min, max, bucketSize) if err != nil { return nil, err } @@ -713,7 +713,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu } intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) if err != nil { - log.Error(err) + log.Errorf("getIndexMetrics failed: %v", err) panic(err) } @@ -804,7 +804,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu } -func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, min, max int64) ([]string, error) { +func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, min, max int64, bucketSize int) ([]string, error) { ver := h.Client().GetVersion() cr, _ := util.VersionCompare(ver.Number, "6.1") if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { @@ -859,12 +859,6 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in }, }) } - bucketSizeStr := "60s" - intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) - if err != nil { - return nil, err - } - partition_num := 10 indexCount := v1.GetIndicesCount(clusterID) if indexCount < 40 { @@ -872,6 +866,13 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in } else { partition_num = indexCount / 20 } + estimatedIndexBuckets := estimateTopIndexBuckets(indexCount, partition_num) + bucketSize = normalizeTopIndexBucketSize(bucketSize, min, max, estimatedIndexBuckets) + bucketSizeStr := fmt.Sprintf("%ds", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } term_index := util.MapStr{ "field": "metadata.labels.index_name", @@ -1012,7 +1013,7 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in } response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) if err != nil { - log.Error(err) + log.Errorf("getTopIndexName failed: %v", err) return nil, err } var maxQpsKVS = map[string]float64{} @@ -1065,3 +1066,55 @@ func (t TopTermOrder) Less(i, j int) bool { func (t TopTermOrder) Swap(i, j int) { t[i], t[j] = t[j], t[i] } + +func normalizeTopIndexBucketSize(bucketSize int, min, max int64, indexBuckets int) int { + if bucketSize <= 0 { + bucketSize = 60 + } + if max <= min { + return bucketSize + } + if indexBuckets <= 0 { + indexBuckets = 1 + } + const ( + esMaxBuckets = int64(65535) + aggGroupCount = int64(2) // group_by_index + group_by_index1 + estShardBuckets = int64(2) // estimated shard terms fan-out + basePerDateBucket = int64(1) + ) + durationSeconds := (max - min) / 1000 + if durationSeconds <= 0 { + return bucketSize + } + perDateCost := aggGroupCount * int64(indexBuckets) * (basePerDateBucket + estShardBuckets) + if perDateCost <= 0 { + perDateCost = 1 + } + maxDateBuckets := esMaxBuckets / perDateCost + if maxDateBuckets < 1 { + maxDateBuckets = 1 + } + minRequired := int((durationSeconds + maxDateBuckets - 1) / maxDateBuckets) + if minRequired > bucketSize { + bucketSize = minRequired + } + return bucketSize +} + +func estimateTopIndexBuckets(indexCount, partitionNum int) int { + if indexCount <= 0 { + return 1 + } + if partitionNum <= 0 { + partitionNum = 1 + } + estimated := (indexCount + partitionNum - 1) / partitionNum + if estimated < 1 { + estimated = 1 + } + if estimated > 10000 { + estimated = 10000 + } + return estimated +} diff --git a/modules/elastic/api/index_overview.go b/modules/elastic/api/index_overview.go index edda7e97..167116c0 100644 --- a/modules/elastic/api/index_overview.go +++ b/modules/elastic/api/index_overview.go @@ -284,18 +284,16 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps h.WriteJSON(w, util.MapStr{}, http.StatusOK) return } - indexIDs = indexIDs[0:1] // map indexIDs(cluster_id:index_name => cluster_uuid:indexName) var ( indexIDM = map[string]string{} newIndexIDs []interface{} clusterIndexNames = map[string][]string{} ) - indexID := indexIDs[0] timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("FetchIndexInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -305,21 +303,26 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps firstClusterID string firstIndexName string ) - if v, ok := indexID.(string); ok { + for i, indexID := range indexIDs { + v, ok := indexID.(string) + if !ok { + h.WriteError(w, fmt.Sprintf("invalid index_id: %v", indexID), http.StatusInternalServerError) + return + } parts := strings.Split(v, ":") if len(parts) != 2 { h.WriteError(w, fmt.Sprintf("invalid index_id: %s", indexID), http.StatusInternalServerError) return } - firstClusterID, firstIndexName = parts[0], parts[1] - if GetMonitorState(firstClusterID) == elastic.ModeAgentless { - h.APIHandler.FetchIndexInfo(w, ctx, indexIDs) - return + clusterID, indexName := parts[0], parts[1] + if i == 0 { + firstClusterID, firstIndexName = clusterID, indexName + if GetMonitorState(firstClusterID) == elastic.ModeAgentless { + h.APIHandler.FetchIndexInfo(w, ctx, indexIDs) + return + } } - clusterIndexNames[firstClusterID] = append(clusterIndexNames[firstClusterID], firstIndexName) - } else { - h.WriteError(w, fmt.Sprintf("invalid index_id: %v", indexID), http.StatusInternalServerError) - return + clusterIndexNames[clusterID] = append(clusterIndexNames[clusterID], indexName) } for clusterID, indexNames := range clusterIndexNames { clusterUUID, err := adapter.GetClusterUUID(clusterID) @@ -333,11 +336,15 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps indexIDM[fmt.Sprintf("%s:%s", clusterID, indexName)] = newIndexID } } + if len(newIndexIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } q1 := orm.Query{WildcardIndex: true} q1.Conds = orm.And( orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.name", "shard_stats"), - orm.Eq("metadata.labels.index_id", newIndexIDs[0]), + orm.In("metadata.labels.index_id", newIndexIDs), ) q1.Collapse("metadata.labels.shard_id") q1.AddSort("timestamp", orm.DESC) @@ -359,6 +366,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps indexID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_id"}, hitM) indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, hitM) + isPrimary, _ := parseBoolValue(primary) if v, ok := indexID.(string); ok { if _, ok = summaryMap[v]; !ok { summaryMap[v] = &ShardsSummary{} @@ -367,19 +375,19 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps if iv, ok := indexName.(string); ok { indexInfo.Index = iv } - if count, ok := shardDocCount.(float64); ok && primary == true { - indexInfo.DocsCount += int64(count) + if count, ok := parseInt64Value(shardDocCount); ok && isPrimary { + indexInfo.DocsCount += count } - if deleted, ok := shardDocDeleted.(float64); ok && primary == true { - indexInfo.DocsDeleted += int64(deleted) + if deleted, ok := parseInt64Value(shardDocDeleted); ok && isPrimary { + indexInfo.DocsDeleted += deleted } - if storeSize, ok := storeInBytes.(float64); ok { - indexInfo.StoreInBytes += int64(storeSize) - if primary == true { - indexInfo.PriStoreInBytes += int64(storeSize) + if storeSize, ok := parseInt64Value(storeInBytes); ok { + indexInfo.StoreInBytes += storeSize + if isPrimary { + indexInfo.PriStoreInBytes += storeSize } } - if primary == true { + if isPrimary { indexInfo.Shards++ } else { indexInfo.Replicas++ @@ -388,10 +396,88 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps } } } + missingIndexIDs := make([]interface{}, 0) + for _, idx := range newIndexIDs { + if idxStr, ok := idx.(string); ok { + if _, exists := summaryMap[idxStr]; !exists { + missingIndexIDs = append(missingIndexIDs, idxStr) + } + } + } + if len(missingIndexIDs) > 0 { + q2 := orm.Query{WildcardIndex: true} + q2.Conds = orm.And( + orm.Eq("metadata.category", "elasticsearch"), + orm.Eq("metadata.name", "index_stats"), + orm.In("metadata.labels.index_id", missingIndexIDs), + ) + q2.Collapse("metadata.labels.index_id") + q2.AddSort("timestamp", orm.DESC) + q2.Size = len(missingIndexIDs) + err, indexResults := orm.Search(&event.Event{}, &q2) + if err != nil { + h.WriteJSON(w, util.MapStr{ + "error": err.Error(), + }, http.StatusInternalServerError) + return + } + for _, v := range indexResults.Result { + result, ok := v.(map[string]interface{}) + if !ok { + continue + } + indexIDVal, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "index_id"}, result) + if !ok { + continue + } + indexIDStr, ok := indexIDVal.(string) + if !ok { + continue + } + if _, exists := summaryMap[indexIDStr]; !exists { + summaryMap[indexIDStr] = &ShardsSummary{} + } + summary := summaryMap[indexIDStr] + if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "total", "docs"}, result); ok { + if docsM, ok := docs.(map[string]interface{}); ok { + if count, ok := parseInt64Value(docsM["count"]); ok { + summary.DocsCount = count + } + if deleted, ok := parseInt64Value(docsM["deleted"]); ok { + summary.DocsDeleted = deleted + } + } + } + if indexInfo, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "index_info"}, result); ok { + if infoM, ok := indexInfo.(map[string]interface{}); ok { + if idxName, ok := infoM["index"].(string); ok { + summary.Index = idxName + } + if shards, ok := parseInt64Value(infoM["shards"]); ok { + summary.Shards = int(shards) + } + if replicas, ok := parseInt64Value(infoM["replicas"]); ok { + summary.Replicas = int(replicas) + } + if storeSize, ok := infoM["store_size"].(string); ok { + if storeInBytes, err := util.ToBytes(storeSize); err == nil { + summary.StoreInBytes = int64(storeInBytes) + } + } + if priStoreSize, ok := infoM["pri_store_size"].(string); ok { + if priStoreInBytes, err := util.ToBytes(priStoreSize); err == nil { + summary.PriStoreInBytes = int64(priStoreInBytes) + } + } + } + } + summary.Timestamp = result["timestamp"] + } + } statusMetric, err := h.GetIndexStatusOfRecentDay(firstClusterID, firstIndexName) if err != nil { - log.Error(err) + log.Errorf("FetchIndexInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -544,7 +630,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, req *http.Request, ps } metrics, err := h.getMetrics(ctx, term_level, query, nodeMetricItems, bucketSize) if err != nil { - log.Error(err) + log.Errorf("FetchIndexInfo failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) } indexMetrics := map[string]util.MapStr{} @@ -630,7 +716,7 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h } clusterUUID, err := adapter.GetClusterUUID(clusterID) if err != nil { - log.Error(err) + log.Errorf("GetIndexInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -680,20 +766,21 @@ func (h *APIHandler) GetIndexInfo(w http.ResponseWriter, req *http.Request, ps h resultM, ok := row.(map[string]interface{}) if ok { primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, resultM) + isPrimary, _ := parseBoolValue(primary) storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, resultM) if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, resultM); ok { //summary["docs"] = docs - if v, ok := docs.(float64); ok && primary == true { - shardSum.DocsCount += int64(v) + if v, ok := parseInt64Value(docs); ok && isPrimary { + shardSum.DocsCount += v } } - if storeSize, ok := storeInBytes.(float64); ok { - shardSum.StoreInBytes += int64(storeSize) - if primary == true { - shardSum.PriStoreInBytes += int64(storeSize) + if storeSize, ok := parseInt64Value(storeInBytes); ok { + shardSum.StoreInBytes += storeSize + if isPrimary { + shardSum.PriStoreInBytes += storeSize } } - if primary == true { + if isPrimary { shardSum.Shards++ } else { shardSum.Replicas++ @@ -726,7 +813,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps } clusterUUID, err := adapter.GetClusterUUID(clusterID) if err != nil { - log.Error(err) + log.Errorf("GetIndexShards failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -741,7 +828,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps q1.AddSort("timestamp", orm.DESC) err, result := orm.Search(&event.Event{}, &q1) if err != nil { - log.Error(err) + log.Errorf("GetIndexShards failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -755,7 +842,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps ) err, nodesResult := orm.Search(elastic.NodeConfig{}, q) if err != nil { - log.Error(err) + log.Errorf("GetIndexShards failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -771,7 +858,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps } qps, err := h.getShardQPS(clusterID, "", indexName, 20) if err != nil { - log.Error(err) + log.Errorf("GetIndexShards failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -789,7 +876,7 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps } shardV, err := source.GetValue("payload.elasticsearch.shard_stats") if err != nil { - log.Error(err) + log.Errorf("GetIndexShards failed: %v", err) continue } shardInfo["id"], _ = source.GetValue("metadata.labels.node_id") @@ -828,8 +915,8 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ h.APIHandler.GetSingleIndexMetrics(w, req, ps) return } - indexName := ps.MustGetParameter("index") - if !h.IsIndexAllowed(req, clusterID, indexName) { + rawIndexName := ps.MustGetParameter("index") + if !h.IsIndexAllowed(req, clusterID, rawIndexName) { h.WriteJSON(w, util.MapStr{ "error": http.StatusText(http.StatusForbidden), }, http.StatusForbidden) @@ -837,7 +924,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ } clusterUUID, err := h.getClusterUUID(clusterID) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -861,7 +948,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ { "term": util.MapStr{ "metadata.labels.index_name": util.MapStr{ - "value": indexName, + "value": rawIndexName, }, }, }, @@ -886,7 +973,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ } bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, metricType, 60) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -897,7 +984,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -940,17 +1027,17 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ metricItems := []*common.MetricItem{} metrics := map[string]*common.MetricItem{} if metricKey == ShardStateMetricKey { - shardStateMetric, err := h.getIndexShardsMetric(ctx, clusterID, indexName, min, max, bucketSize, shardID) + shardStateMetric, err := h.getIndexShardsMetric(ctx, clusterID, rawIndexName, min, max, bucketSize, shardID) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } metrics["shard_state"] = shardStateMetric } else if metricKey == v1.IndexHealthMetricKey { - healthMetric, err := h.GetIndexHealthMetric(ctx, clusterID, indexName, min, max, bucketSize) + healthMetric, err := h.GetIndexHealthMetric(ctx, clusterID, rawIndexName, min, max, bucketSize) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -1028,7 +1115,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ } metrics, err = h.getSingleIndexMetrics(context.Background(), metricItems, query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) } } @@ -1036,7 +1123,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ if metricItem.HitsTotal > 0 && metricItem.MinBucketSize == 0 { minBucketSize, err := v1.GetMetricMinBucketSize(clusterID, metricType) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) } else { metricItem.MinBucketSize = int64(minBucketSize) } @@ -1128,7 +1215,7 @@ func (h *APIHandler) getIndexShardsMetric(ctx context.Context, id, indexName str queryDSL := util.MustToJSONBytes(query) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL) if err != nil { - log.Error(err) + log.Errorf("getIndexShardsMetric failed: %v", err) return nil, err } diff --git a/modules/elastic/api/init.go b/modules/elastic/api/init.go index 59cc41e1..391ed5c4 100644 --- a/modules/elastic/api/init.go +++ b/modules/elastic/api/init.go @@ -43,23 +43,25 @@ func init() { api.HandleAPIMethod(api.GET, "/elasticsearch/:id/queue_metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleQueueMetricsAction)) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/storage_metrics", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetStorageMetricAction)) - api.HandleAPIMethod(api.POST, "/elasticsearch/", clusterAPI.RequirePermission(clusterAPI.HandleCreateClusterAction, enum.PermissionElasticsearchClusterWrite)) + api.HandleAPIMethod(api.POST, "/elasticsearch/", clusterAPI.RequireSecureTransport(clusterAPI.RequireReplayProtection(clusterAPI.RequirePermission(clusterAPI.HandleCreateClusterAction, enum.PermissionElasticsearchClusterWrite)))) api.HandleAPIMethod(api.GET, "/elasticsearch/indices", clusterAPI.RequireLogin(clusterAPI.ListIndex)) api.HandleAPIMethod(api.GET, "/elasticsearch/status", clusterAPI.RequireLogin(clusterAPI.GetClusterStatusAction)) api.HandleAPIMethod(api.GET, "/elasticsearch/:id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleGetClusterAction, enum.PermissionElasticsearchClusterRead))) - api.HandleAPIMethod(api.PUT, "/elasticsearch/:id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleUpdateClusterAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id", clusterAPI.RequireSecureTransport(clusterAPI.RequireReplayProtection(clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleUpdateClusterAction, enum.PermissionElasticsearchClusterWrite))))) api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleDeleteClusterAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.POST, "/elasticsearch/_enable", clusterAPI.RequirePermission(clusterAPI.HandleEnableClusterMonitoringAction, enum.PermissionElasticsearchClusterWrite)) + api.HandleAPIMethod(api.POST, "/elasticsearch/_disable", clusterAPI.RequirePermission(clusterAPI.HandleDisableClusterMonitoringAction, enum.PermissionElasticsearchClusterWrite)) api.HandleAPIMethod(api.GET, "/elasticsearch/_search", clusterAPI.RequirePermission(clusterAPI.HandleSearchClusterAction, enum.PermissionElasticsearchClusterRead)) api.HandleAPIMethod(api.POST, "/elasticsearch/_search", clusterAPI.RequirePermission(clusterAPI.HandleSearchClusterAction, enum.PermissionElasticsearchClusterRead)) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/search_template", clusterAPI.HandleCreateSearchTemplateAction) - api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/search_template/:template_id", clusterAPI.HandleUpdateSearchTemplateAction) - api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/search_template/:template_id", clusterAPI.HandleDeleteSearchTemplateAction) - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template", clusterAPI.HandleSearchSearchTemplateAction) - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template/:template_id", clusterAPI.HandleGetSearchTemplateAction) - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template_history/_search", clusterAPI.HandleSearchSearchTemplateHistoryAction) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/_render/template", clusterAPI.HandleRenderTemplateAction) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/_search/template", clusterAPI.HandleSearchTemplateAction) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/search_template", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleCreateSearchTemplateAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/search_template/:template_id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleUpdateSearchTemplateAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/search_template/:template_id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleDeleteSearchTemplateAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleSearchSearchTemplateAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template/:template_id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleGetSearchTemplateAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search_template_history/_search", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleSearchSearchTemplateHistoryAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/_render/template", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleRenderTemplateAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/_search/template", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleSearchTemplateAction, enum.PermissionElasticsearchClusterRead))) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/alias", clusterAPI.RequireClusterPermission(clusterAPI.HandleAliasAction)) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/alias", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetAliasAction)) @@ -75,7 +77,7 @@ func init() { api.HandleAPIMethod(api.POST, "/elasticsearch/:id/view/:view_id/_set_default_layout", clusterAPI.RequireClusterPermission(clusterAPI.SetDefaultLayout)) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/search/ese", clusterAPI.RequireLogin(clusterAPI.HandleEseSearchAction)) - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search/trace_id", clusterAPI.HandleTraceIDSearchAction) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/search/trace_id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleTraceIDSearchAction, enum.PermissionElasticsearchMetricRead))) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/suggestions/values/:index", clusterAPI.RequireClusterPermission(clusterAPI.HandleValueSuggestionAction)) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/setting", clusterAPI.RequireClusterPermission(clusterAPI.HandleSettingAction)) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/setting/:key", clusterAPI.RequireClusterPermission(clusterAPI.HandleGetSettingAction)) @@ -104,29 +106,29 @@ func init() { api.HandleAPIMethod(api.GET, "/elasticsearch/:id/index/:index/nodes", clusterAPI.RequirePermission(clusterAPI.getIndexNodes, enum.PermissionElasticsearchMetricRead, enum.PermissionElasticsearchNodeRead)) api.HandleAPIMethod(api.POST, "/elasticsearch/index/info", clusterAPI.RequirePermission(clusterAPI.FetchIndexInfo, enum.PermissionElasticsearchMetricRead)) - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/trace_template", clusterAPI.HandleSearchTraceTemplateAction) - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.HandleGetTraceTemplateAction) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/trace_template", clusterAPI.HandleCrateTraceTemplateAction) - api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.HandleSaveTraceTemplateAction) - api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.HandleDeleteTraceTemplateAction) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/trace_template", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleSearchTraceTemplateAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleGetTraceTemplateAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/trace_template", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleCrateTraceTemplateAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleSaveTraceTemplateAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/trace_template/:template_id", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleDeleteTraceTemplateAction, enum.PermissionElasticsearchClusterWrite))) api.HandleAPIMethod(api.POST, "/elasticsearch/activity/_search", clusterAPI.RequirePermission(clusterAPI.HandleSearchActivityAction, enum.PermissionActivityRead)) - api.HandleAPIMethod(api.GET, "/host/_discover", clusterAPI.getDiscoverHosts) - api.HandleAPIMethod(api.POST, "/host/_search", clusterAPI.SearchHostMetadata) - api.HandleAPIMethod(api.POST, "/host/info", clusterAPI.FetchHostInfo) - api.HandleAPIMethod(api.GET, "/host/:host_id/metrics", clusterAPI.GetSingleHostMetrics) - api.HandleAPIMethod(api.GET, "/host/:host_id/metric/_stats", clusterAPI.GetHostMetricStats) - api.HandleAPIMethod(api.GET, "/host/:host_id", clusterAPI.GetHostInfo) - api.HandleAPIMethod(api.PUT, "/host/:host_id", clusterAPI.updateHost) - api.HandleAPIMethod(api.GET, "/host/:host_id/info", clusterAPI.GetHostOverviewInfo) - - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/_ilm/policy", clusterAPI.HandleGetILMPolicyAction) - api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/_ilm/policy/:policy", clusterAPI.HandleSaveILMPolicyAction) - api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/_ilm/policy/:policy", clusterAPI.HandleDeleteILMPolicyAction) - - api.HandleAPIMethod(api.GET, "/elasticsearch/:id/_template", clusterAPI.HandleGetTemplateAction) - api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/_template/:template_name", clusterAPI.HandleSaveTemplateAction) + api.HandleAPIMethod(api.GET, "/host/_discover", clusterAPI.RequirePermission(clusterAPI.getDiscoverHosts, enum.PermissionElasticsearchNodeRead, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.POST, "/host/_search", clusterAPI.RequirePermission(clusterAPI.SearchHostMetadata, enum.PermissionElasticsearchNodeRead, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.POST, "/host/info", clusterAPI.RequirePermission(clusterAPI.FetchHostInfo, enum.PermissionElasticsearchNodeRead, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.GET, "/host/:host_id/metrics", clusterAPI.RequirePermission(clusterAPI.GetSingleHostMetrics, enum.PermissionElasticsearchNodeRead, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.GET, "/host/:host_id/metric/_stats", clusterAPI.RequirePermission(clusterAPI.GetHostMetricStats, enum.PermissionElasticsearchNodeRead, enum.PermissionElasticsearchMetricRead)) + api.HandleAPIMethod(api.GET, "/host/:host_id", clusterAPI.RequirePermission(clusterAPI.GetHostInfo, enum.PermissionElasticsearchNodeRead)) + api.HandleAPIMethod(api.PUT, "/host/:host_id", clusterAPI.RequirePermission(clusterAPI.updateHost, enum.PermissionElasticsearchClusterWrite)) + api.HandleAPIMethod(api.GET, "/host/:host_id/info", clusterAPI.RequirePermission(clusterAPI.GetHostOverviewInfo, enum.PermissionElasticsearchNodeRead, enum.PermissionElasticsearchMetricRead)) + + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/_ilm/policy", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleGetILMPolicyAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/_ilm/policy/:policy", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleSaveILMPolicyAction, enum.PermissionElasticsearchClusterWrite))) + api.HandleAPIMethod(api.DELETE, "/elasticsearch/:id/_ilm/policy/:policy", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleDeleteILMPolicyAction, enum.PermissionElasticsearchClusterWrite))) + + api.HandleAPIMethod(api.GET, "/elasticsearch/:id/_template", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleGetTemplateAction, enum.PermissionElasticsearchClusterRead))) + api.HandleAPIMethod(api.PUT, "/elasticsearch/:id/_template/:template_name", clusterAPI.RequireClusterPermission(clusterAPI.RequirePermission(clusterAPI.HandleSaveTemplateAction, enum.PermissionElasticsearchClusterWrite))) api.HandleAPIMethod(api.GET, "/elasticsearch/:id/shard/:shard_id/info", clusterAPI.RequirePermission(clusterAPI.GetShardInfo, enum.PermissionElasticsearchMetricRead)) api.HandleAPIMethod(api.GET, "/elasticsearch/metadata", clusterAPI.RequireLogin(clusterAPI.GetMetadata)) diff --git a/modules/elastic/api/manage.go b/modules/elastic/api/manage.go index 0928e00e..c7a56ad0 100644 --- a/modules/elastic/api/manage.go +++ b/modules/elastic/api/manage.go @@ -35,18 +35,23 @@ import ( "time" "infini.sh/framework/core/queue" + ucfg "infini.sh/framework/lib/go-ucfg" log "github.com/cihub/seelog" + console_common "infini.sh/console/common" "infini.sh/console/core" v1 "infini.sh/console/modules/elastic/api/v1" + agentservice "infini.sh/console/service/agent" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/credential" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" + elasticmodule "infini.sh/framework/modules/elastic" "infini.sh/framework/modules/elastic/common" ) @@ -55,19 +60,30 @@ type APIHandler struct { v1.APIHandler } +const ( + clusterCredentialKindPlatform = "platform" + clusterCredentialKindAgent = "agent" + autoAgentCollectionUsername = "infini-agent" + autoAgentCollectionFallbackUsername = "infini-console-agent" +) + func (h *APIHandler) Client() elastic.API { return elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) } func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var conf = &elastic.ElasticsearchConfig{} - err := h.DecodeJSON(req, conf) + payload := &elasticConfigPayload{} + err := h.DecodeJSON(req, payload) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + conf := &payload.ElasticsearchConfig + console_common.SetProbePath(conf, payload.ProbePath) + applyExplicitPlatformAuthPreference(conf, payload.AuthEnabled) conf.Enabled = true + conf.Monitored = true if len(conf.Hosts) > 0 && conf.Host == "" { conf.Host = conf.Hosts[0] } @@ -85,9 +101,9 @@ func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http. Refresh: "wait_for", } if conf.CredentialID == "" && conf.BasicAuth != nil && conf.BasicAuth.Username != "" { - credentialID, err := saveBasicAuthToCredential(conf.Name+"_platform("+conf.ID+")", conf.BasicAuth) + credentialID, err := saveClusterBasicAuthToCredential(conf.Name, conf.ID, clusterCredentialKindPlatform, conf.BasicAuth) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -96,9 +112,9 @@ func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http. conf.BasicAuth = nil if conf.AgentCredentialID == "" && conf.AgentBasicAuth != nil && conf.AgentBasicAuth.Username != "" { - credentialID, err := saveBasicAuthToCredential(conf.Name+"_agent("+conf.ID+")", conf.AgentBasicAuth) + credentialID, err := saveClusterBasicAuthToCredential(conf.Name, conf.ID, clusterCredentialKindAgent, conf.AgentBasicAuth) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -112,28 +128,138 @@ func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http. if conf.MetricCollectionMode == "" { conf.MetricCollectionMode = elastic.ModeAgentless } - err = orm.Create(ctx, conf) + err = ensureManagedAgentCollectionCredential(conf, "") if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - basicAuth, err := common.GetBasicAuth(conf) + err = orm.Create(ctx, conf) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if err := hydrateRuntimeBasicAuth(conf); err != nil { + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - conf.BasicAuth = basicAuth conf.Source = elastic.ElasticsearchConfigSourceElasticsearch _, err = common.InitElasticInstance(*conf) if err != nil { log.Warn("error on init elasticsearch:", err) + } else { + elasticmodule.SyncClusterHealthStatus(conf.ID) + } + if conf.MetricCollectionMode == elastic.ModeAgent { + agentservice.TriggerAutoEnroll([]string{conf.ID}) } h.WriteCreatedOKJSON(w, conf.ID) } +func PrepareClusterForAgentCollection(clusterID string) (*elastic.ElasticsearchConfig, error) { + if strings.TrimSpace(clusterID) == "" { + return nil, fmt.Errorf("cluster id is required") + } + + conf := &elastic.ElasticsearchConfig{} + conf.ID = clusterID + exists, err := orm.GetV2(orm.NewContext(), conf) + if err != nil { + return nil, err + } + if !exists { + return nil, fmt.Errorf("cluster not found") + } + + oldMode := conf.MetricCollectionMode + oldAgentCredentialID := conf.AgentCredentialID + oldNoDefaultAuthForAgent := conf.NoDefaultAuthForAgent + + conf.MetricCollectionMode = elastic.ModeAgent + if err := ensureManagedAgentCollectionCredential(conf, conf.Name); err != nil { + return nil, err + } + + changed := oldMode != conf.MetricCollectionMode || + oldAgentCredentialID != conf.AgentCredentialID || + oldNoDefaultAuthForAgent != conf.NoDefaultAuthForAgent + + if changed { + if err := orm.Save(&orm.Context{Refresh: "wait_for"}, conf); err != nil { + return nil, err + } + if err := syncAutoGeneratedClusterCredentialName(conf.AgentCredentialID, conf.ID, conf.Name, conf.Name, clusterCredentialKindAgent); err != nil { + return nil, err + } + if err := hydrateRuntimeBasicAuth(conf); err != nil { + return nil, err + } + conf.Source = elastic.ElasticsearchConfigSourceElasticsearch + if _, err := common.InitElasticInstance(*conf); err != nil { + log.Warn("error on init elasticsearch:", err) + } + if oldMode != conf.MetricCollectionMode { + recordCollectionModeChangeActivity(conf.ID, conf.Name, oldMode, conf.MetricCollectionMode) + } + } + + return conf, nil +} + +// EnsureManagedAgentCredential initializes or refreshes the auto-generated infini-agent credential when supported. +func EnsureManagedAgentCredential(conf *elastic.ElasticsearchConfig, previousClusterName string) error { + return ensureManagedAgentCollectionCredential(conf, previousClusterName) +} + +func applyExplicitPlatformAuthPreference(conf *elastic.ElasticsearchConfig, authEnabled *bool) { + if conf == nil || authEnabled == nil { + return + } + + if !*authEnabled { + conf.CredentialID = "" + conf.BasicAuth = nil + conf.NoDefaultAuthForAgent = true + return + } + + conf.NoDefaultAuthForAgent = false +} + +func applyExplicitPlatformAuthPreferenceToSource(source map[string]interface{}, authEnabled *bool) { + if source == nil || authEnabled == nil { + return + } + + if !*authEnabled { + source["credential_id"] = "" + source["basic_auth"] = nil + source["no_default_auth_for_agent"] = true + return + } + + source["no_default_auth_for_agent"] = false +} + +func hydrateRuntimeBasicAuth(conf *elastic.ElasticsearchConfig) error { + return hydrateRuntimeBasicAuthWithGetter(conf, common.GetBasicAuth) +} + +func hydrateRuntimeBasicAuthWithGetter(conf *elastic.ElasticsearchConfig, getBasicAuth func(*elastic.ElasticsearchConfig) (*model.BasicAuth, error)) error { + if conf == nil { + return nil + } + basicAuth, err := getBasicAuth(conf) + if err != nil { + return err + } + conf.BasicAuth = basicAuth + return nil +} + func saveBasicAuthToCredential(name string, auth *model.BasicAuth) (string, error) { cred := credential.Credential{ Name: name, @@ -151,44 +277,500 @@ func saveBasicAuthToCredential(name string, auth *model.BasicAuth) (string, erro if err != nil { return "", err } - err = orm.Create(nil, &cred) + err = orm.Create(&orm.Context{Refresh: "wait_for"}, &cred) if err != nil { return "", err } return cred.ID, nil } +func saveClusterBasicAuthToCredential(clusterName, clusterID, kind string, auth *model.BasicAuth) (string, error) { + return saveBasicAuthToCredential(getClusterCredentialDisplayName(clusterName, kind), auth) +} + +func getClusterCredentialDisplayName(clusterName, kind string) string { + clusterName = strings.TrimSpace(clusterName) + switch kind { + case clusterCredentialKindAgent: + return fmt.Sprintf("%s (Agent)", clusterName) + default: + return fmt.Sprintf("%s (Platform)", clusterName) + } +} + +func getLegacyClusterCredentialDisplayName(clusterName, clusterID, kind string) string { + clusterName = strings.TrimSpace(clusterName) + return fmt.Sprintf("%s_%s(%s)", clusterName, kind, clusterID) +} + +func isAutoGeneratedClusterCredentialName(name, clusterName, clusterID, kind string) bool { + if name == "" { + return false + } + return name == getClusterCredentialDisplayName(clusterName, kind) || + (clusterID != "" && name == getLegacyClusterCredentialDisplayName(clusterName, clusterID, kind)) +} + +func syncAutoGeneratedClusterCredentialName(credentialID, clusterID, oldClusterName, newClusterName, kind string) error { + if credentialID == "" { + return nil + } + + cred := credential.Credential{} + cred.ID = credentialID + exists, err := orm.GetV2(orm.NewContext(), &cred) + if err != nil { + return err + } + if !exists { + return nil + } + + if !isAutoGeneratedClusterCredentialName(cred.Name, oldClusterName, clusterID, kind) && + !isAutoGeneratedClusterCredentialName(cred.Name, newClusterName, clusterID, kind) { + return nil + } + + newName := getClusterCredentialDisplayName(newClusterName, kind) + if cred.Name == newName { + return nil + } + + cred.Name = newName + return orm.Update(&orm.Context{Refresh: "wait_for"}, &cred) +} + +func ensureManagedAgentCollectionCredential(conf *elastic.ElasticsearchConfig, previousClusterName string) error { + if conf == nil { + return nil + } + if conf.Distribution != elastic.Easysearch { + return nil + } + + existingCredential, existingAuth, shouldManage, err := getManagedAgentCredential(conf, previousClusterName) + if err != nil { + return err + } + if !shouldManage { + if conf.AgentCredentialID != "" { + conf.NoDefaultAuthForAgent = true + } + return nil + } + + platformAuth, err := common.GetBasicAuth(conf) + if err != nil { + return err + } + if platformAuth == nil || platformAuth.Username == "" { + if conf.NoDefaultAuthForAgent { + conf.AgentCredentialID = "" + conf.AgentBasicAuth = nil + return nil + } + if conf.MetricCollectionMode == elastic.ModeAgent { + return fmt.Errorf("platform credential is required to create agent collection user") + } + return nil + } + + username := autoAgentCollectionUsername + password := "" + if existingAuth != nil { + if strings.TrimSpace(existingAuth.Username) != "" { + username = strings.TrimSpace(existingAuth.Username) + } + if existingAuth.Password.Get() != "" { + password = existingAuth.Password.Get() + } + } + if password == "" { + password = util.GenerateSecureString(20) + } + + client, cleanup, err := newManagedClusterSecurityClient(conf, platformAuth) + if err != nil { + return err + } + defer cleanup() + username, err = initAgentCollectionUser(client, conf.Distribution, username, password) + if err != nil { + if shouldFallbackToPlatformCredentialForManagedAgentProvision(err) { + if existingCredential != nil { + log.Warnf("keep existing managed agent credential for cluster [%s] because auto-provision refresh is unavailable: %v", conf.Name, err) + conf.AgentCredentialID = existingCredential.ID + conf.AgentBasicAuth = nil + conf.NoDefaultAuthForAgent = true + return nil + } + + log.Warnf("skip managed agent credential auto-provision for cluster [%s], falling back to platform credential: %v", conf.Name, err) + conf.AgentCredentialID = "" + conf.AgentBasicAuth = nil + conf.NoDefaultAuthForAgent = false + return nil + } + return err + } + + if existingCredential != nil { + existingCredential.Name = getClusterCredentialDisplayName(conf.Name, clusterCredentialKindAgent) + existingCredential.Type = credential.BasicAuth + existingCredential.Tags = []string{"ES"} + existingCredential.Payload = map[string]interface{}{ + "basic_auth": map[string]interface{}{ + "username": username, + "password": password, + }, + } + if err := existingCredential.Encode(); err != nil { + return err + } + if err := orm.Update(&orm.Context{Refresh: "wait_for"}, existingCredential); err != nil { + return err + } + conf.AgentCredentialID = existingCredential.ID + } else { + auth := &model.BasicAuth{ + Username: username, + Password: ucfg.SecretString(password), + } + credentialID, err := saveClusterBasicAuthToCredential(conf.Name, conf.ID, clusterCredentialKindAgent, auth) + if err != nil { + return err + } + conf.AgentCredentialID = credentialID + } + + conf.AgentBasicAuth = nil + conf.NoDefaultAuthForAgent = true + return nil +} + +func getManagedAgentCredential(conf *elastic.ElasticsearchConfig, previousClusterName string) (*credential.Credential, *model.BasicAuth, bool, error) { + if conf == nil || conf.AgentCredentialID == "" { + return nil, nil, true, nil + } + + cred, err := common.GetCredential(conf.AgentCredentialID) + if err != nil { + // Agent credential can be deleted manually while cluster config still keeps + // the old credential id. Treat it as absent and let follow-up logic + // re-provision or fall back based on current auth settings. + if strings.Contains(strings.ToLower(err.Error()), "record not found") { + conf.AgentCredentialID = "" + conf.AgentBasicAuth = nil + return nil, nil, true, nil + } + return nil, nil, false, err + } + auth, err := cred.DecodeBasicAuth() + if err != nil { + return nil, nil, false, err + } + + if isAutoGeneratedClusterCredentialName(cred.Name, conf.Name, conf.ID, clusterCredentialKindAgent) || + (previousClusterName != "" && isAutoGeneratedClusterCredentialName(cred.Name, previousClusterName, conf.ID, clusterCredentialKindAgent)) { + return cred, auth, true, nil + } + + return cred, auth, false, nil +} + +func managedClusterSecurityRuntimeID(clusterID string) string { + clusterID = strings.TrimSpace(clusterID) + if clusterID == "" { + clusterID = util.GetUUID() + } + return fmt.Sprintf("%s-managed-security", clusterID) +} + +func newManagedClusterSecurityClient(conf *elastic.ElasticsearchConfig, auth *model.BasicAuth) (elastic.API, func(), error) { + tempConf := *conf + tempConf.ID = managedClusterSecurityRuntimeID(conf.ID) + tempConf.BasicAuth = auth + tempConf.CredentialID = "" + tempConf.AgentBasicAuth = nil + tempConf.AgentCredentialID = "" + client, err := common.InitClientWithConfig(tempConf) + if err != nil { + return nil, nil, err + } + // The managed Easysearch security bootstrap runs before the new cluster config + // is registered globally, so the temporary client must carry its own metadata + // under an isolated runtime ID to avoid polluting the real cluster state. + elastic.GetOrInitMetadata(&tempConf) + return client, func() { + elastic.RemoveInstance(tempConf.ID) + }, nil +} + +func initAgentCollectionUser(client elastic.API, distribution, username, password string) (string, error) { + roleBody, err := buildAgentCollectionRoleBody(distribution) + if err != nil { + return username, err + } + if err := client.PutRole(username, roleBody); err != nil { + fallbackUsername, fallback := getManagedAgentCollectionFallbackUsername(distribution, username, err) + if !fallback { + return username, wrapAgentCollectionProvisionError(distribution, "role", err) + } + username = fallbackUsername + if err := client.PutRole(username, roleBody); err != nil { + return username, wrapAgentCollectionProvisionError(distribution, "role", err) + } + } + + userBody, err := buildAgentCollectionUserBody(distribution, username, password) + if err != nil { + return username, err + } + if err := client.PutUser(username, userBody); err != nil { + return username, wrapAgentCollectionProvisionError(distribution, "user", err) + } + return username, nil +} + +func buildAgentCollectionRoleBody(distribution string) ([]byte, error) { + switch distribution { + case elastic.Easysearch: + return util.MustToJSONBytes(util.MapStr{ + "cluster": []string{ + "cluster_monitor", + }, + "description": "Provide the minimum permissions for INFINI AGENT to collect metrics", + "indices": []util.MapStr{ + { + "names": []string{ + "*", + }, + "query": "", + "field_security": []string{}, + "field_mask": []string{}, + "privileges": []string{ + "indices_monitor", + }, + }, + }, + }), nil + default: + return nil, fmt.Errorf("unsupported distribution for agent collection role: %s", distribution) + } +} + +func buildAgentCollectionUserBody(distribution, username, password string) ([]byte, error) { + switch distribution { + case elastic.Easysearch: + return util.MustToJSONBytes(util.MapStr{ + "roles": []string{username}, + "password": password, + }), nil + default: + return nil, fmt.Errorf("unsupported distribution for agent collection user: %s", distribution) + } +} + +func wrapAgentCollectionProvisionError(distribution, resource string, err error) error { + if err == nil { + return nil + } + + if distribution == elastic.Easysearch && isUnsupportedEasysearchSecurityAPIError(err) { + return fmt.Errorf( + "failed to create agent collection %s: target Easysearch cluster does not allow modifying _security/%s via the current credential; check security.restapi.roles_enabled and security.restapi.endpoints_disabled for %s access: %w", + resource, + resource, + strings.ToUpper(resource), + err, + ) + } + + return fmt.Errorf("failed to create agent collection %s: %w", resource, err) +} + +func shouldFallbackToPlatformCredentialForManagedAgentProvision(err error) bool { + if err == nil { + return false + } + + lower := strings.ToLower(err.Error()) + return strings.Contains(lower, "does not allow modifying _security/role") || + strings.Contains(lower, "does not allow modifying _security/user") +} + +func applyClusterRuntimeConfig(conf *elastic.ElasticsearchConfig) error { + if conf == nil { + return nil + } + + conf.Source = elastic.ElasticsearchConfigSourceElasticsearch + elastic.UpdateConfig(*conf) + + meta := elastic.GetMetadata(conf.ID) + if meta == nil { + if _, err := common.InitElasticInstance(*conf); err != nil { + log.Warn("error on init elasticsearch:", err) + } + return nil + } + + updatedMeta := *meta + cfgCopy := *conf + updatedMeta.Config = &cfgCopy + elastic.SetMetadata(conf.ID, &updatedMeta) + return nil +} + +func setClustersMonitored(monitored bool, clusterIDs []string) error { + if len(clusterIDs) == 0 { + return nil + } + + for _, clusterID := range clusterIDs { + clusterID = strings.TrimSpace(clusterID) + if clusterID == "" { + continue + } + + conf := &elastic.ElasticsearchConfig{} + conf.ID = clusterID + exists, err := orm.GetV2(orm.NewContext(), conf) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("cluster [%s] not found", clusterID) + } + if conf.Monitored == monitored { + continue + } + + conf.Monitored = monitored + if err := orm.Save(&orm.Context{Refresh: "wait_for"}, conf); err != nil { + return err + } + if err := hydrateRuntimeBasicAuth(conf); err != nil { + return err + } + if err := applyClusterRuntimeConfig(conf); err != nil { + return err + } + } + + return nil +} + +func getManagedAgentCollectionFallbackUsername(distribution, username string, err error) (string, bool) { + if distribution != elastic.Easysearch { + return "", false + } + if username != autoAgentCollectionUsername { + return "", false + } + if !isUnavailableSecurityResourceError(err) { + return "", false + } + return autoAgentCollectionFallbackUsername, true +} + +func isUnavailableSecurityResourceError(err error) bool { + if err == nil { + return false + } + + lowerRaw := strings.ToLower(err.Error()) + return strings.Contains(lowerRaw, `"status":"not_found"`) && + strings.Contains(lowerRaw, "resource") && + strings.Contains(lowerRaw, "not available") +} + +func isUnsupportedEasysearchSecurityAPIError(err error) bool { + if err == nil { + return false + } + if isUnavailableSecurityResourceError(err) { + return true + } + + lowerRaw := strings.ToLower(err.Error()) + return strings.Contains(lowerRaw, "invalid_index_name_exception") && + strings.Contains(lowerRaw, `"_security"`) +} + +func extractOptionalBool(value interface{}) *bool { + boolValue, ok := value.(bool) + if !ok { + return nil + } + return &boolValue +} + func (h *APIHandler) HandleGetClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") clusterConf := elastic.ElasticsearchConfig{} clusterConf.ID = id - exists, err := orm.Get(&clusterConf) + exists, err := orm.GetV2(orm.NewContext(), &clusterConf) if err != nil || !exists { - log.Error(err) + log.Errorf("HandleGetClusterAction failed: %v", err) h.Error404(w) return } - h.WriteGetOKJSON(w, id, clusterConf) + source := map[string]interface{}{} + util.MustFromJSONBytes(util.MustToJSONBytes(clusterConf), &source) + h.WriteGetOKJSON(w, id, source) +} + +func (h *APIHandler) HandleEnableClusterMonitoringAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + h.handleSetClusterMonitoringAction(w, req, true) +} + +func (h *APIHandler) HandleDisableClusterMonitoringAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + h.handleSetClusterMonitoringAction(w, req, false) +} + +func (h *APIHandler) handleSetClusterMonitoringAction(w http.ResponseWriter, req *http.Request, monitored bool) { + clusterIDs := []string{} + if err := h.DecodeJSON(req, &clusterIDs); err != nil { + log.Errorf("handleSetClusterMonitoringAction failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if len(clusterIDs) == 0 { + h.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + + if err := setClustersMonitored(monitored, clusterIDs); err != nil { + log.Errorf("handleSetClusterMonitoringAction failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteAckOKJSON(w) } func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var conf = map[string]interface{}{} err := h.DecodeJSON(req, &conf) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } id := ps.MustGetParameter("id") originConf := elastic.ElasticsearchConfig{} originConf.ID = id - exists, err := orm.Get(&originConf) + exists, err := orm.GetV2(orm.NewContext(), &originConf) if err != nil || !exists { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.Error404(w) return } var oldCollectionMode = originConf.MetricCollectionMode + authEnabled := extractOptionalBool(conf["is_auth"]) + delete(conf, "is_auth") + delete(conf, "agent_logs_paths") buf := util.MustToJSONBytes(originConf) source := map[string]interface{}{} util.MustFromJSONBytes(buf, &source) @@ -207,6 +789,7 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. } source[k] = v } + applyExplicitPlatformAuthPreferenceToSource(source, authEnabled) // convert hosts array to string to get first if hosts, ok := conf["hosts"].([]interface{}); ok && len(hosts) > 0 { @@ -225,12 +808,16 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. newConf := &elastic.ElasticsearchConfig{} json.Unmarshal(confBytes, newConf) newConf.ID = id + if probePath, ok := conf["probe_path"]; ok { + console_common.SetProbePath(newConf, util.ToString(probePath)) + delete(conf, "probe_path") + } if conf["credential_id"] == nil { if newConf.BasicAuth != nil && newConf.BasicAuth.Username != "" { - credentialID, err := saveBasicAuthToCredential(newConf.Name+"_platform("+newConf.ID+")", newConf.BasicAuth) + credentialID, err := saveClusterBasicAuthToCredential(newConf.Name, newConf.ID, clusterCredentialKindPlatform, newConf.BasicAuth) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -243,9 +830,9 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. if conf["agent_credential_id"] == nil { if newConf.AgentBasicAuth != nil && newConf.AgentBasicAuth.Username != "" { - credentialID, err := saveBasicAuthToCredential(newConf.Name+"_agent("+newConf.ID+")", newConf.AgentBasicAuth) + credentialID, err := saveClusterBasicAuthToCredential(newConf.Name, newConf.ID, clusterCredentialKindAgent, newConf.AgentBasicAuth) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -255,10 +842,28 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. newConf.AgentCredentialID = "" } } + err = ensureManagedAgentCollectionCredential(newConf, originConf.Name) + if err != nil { + log.Errorf("HandleUpdateClusterAction failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } err = orm.Save(ctx, newConf) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + err = syncAutoGeneratedClusterCredentialName(newConf.CredentialID, newConf.ID, originConf.Name, newConf.Name, clusterCredentialKindPlatform) + if err != nil { + log.Errorf("HandleUpdateClusterAction failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + err = syncAutoGeneratedClusterCredentialName(newConf.AgentCredentialID, newConf.ID, originConf.Name, newConf.Name, clusterCredentialKindAgent) + if err != nil { + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -266,12 +871,12 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. if oldCollectionMode != newConf.MetricCollectionMode { recordCollectionModeChangeActivity(newConf.ID, newConf.Name, oldCollectionMode, newConf.MetricCollectionMode) } - basicAuth, err := common.GetBasicAuth(newConf) - if err != nil { + if err := hydrateRuntimeBasicAuth(newConf); err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - newConf.BasicAuth = basicAuth + + elastic.RemoveHostsByClusterID(id) //update config in heap newConf.Source = elastic.ElasticsearchConfigSourceElasticsearch @@ -279,10 +884,30 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. if err != nil { log.Warn("error on init elasticsearch:", err) } + if oldCollectionMode != elastic.ModeAgent && newConf.MetricCollectionMode == elastic.ModeAgent { + agentservice.TriggerAutoEnroll([]string{newConf.ID}) + } h.WriteUpdatedOKJSON(w, id) } +func extractLogsPathsFromValue(value interface{}) []string { + switch v := value.(type) { + case nil: + return nil + case []string: + return agentservice.NormalizeLogsPaths(v) + case []interface{}: + items := make([]string, 0, len(v)) + for _, item := range v { + items = append(items, util.ToString(item)) + } + return agentservice.NormalizeLogsPaths(items) + default: + return nil + } +} + func recordCollectionModeChangeActivity(clusterID, clusterName, oldMode, newMode string) { activityInfo := &event.Activity{ ID: util.GetUUID(), @@ -320,7 +945,7 @@ func recordCollectionModeChangeActivity(clusterID, clusterName, oldMode, newMode "activity": activityInfo, }})) if err != nil { - log.Error(err) + log.Errorf("recordCollectionModeChangeActivity failed: %v", err) } } @@ -330,9 +955,9 @@ func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http. esConfig := elastic.ElasticsearchConfig{} esConfig.ID = id - ok, err := orm.Get(&esConfig) + ok, err := orm.GetV2(orm.NewContext(), &esConfig) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -350,7 +975,7 @@ func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http. err = orm.Delete(ctx, &esConfig) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -363,15 +988,27 @@ func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http. } err = orm.DeleteBy(elastic.NodeConfig{}, util.MustToJSONBytes(delDsl)) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) } err = orm.DeleteBy(elastic.IndexConfig{}, util.MustToJSONBytes(delDsl)) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) } elastic.RemoveInstance(id) elastic.RemoveHostsByClusterID(id) + err = kv.DeleteKey(elastic.KVElasticNodeMetadata, []byte(id)) + if err != nil { + log.Errorf("failed to delete node metadata for cluster [%s]: %v", id, err) + } + err = kv.DeleteKey(elastic.KVElasticIndexMetadata, []byte(id)) + if err != nil { + log.Errorf("failed to delete index metadata for cluster [%s]: %v", id, err) + } + err = kv.DeleteKey(elastic.KVElasticClusterSettings, []byte(id)) + if err != nil { + log.Errorf("failed to delete cluster settings metadata for cluster [%s]: %v", id, err) + } h.WriteDeletedOKJSON(w, id) } @@ -380,7 +1017,7 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. name = h.GetParameterOrDefault(req, "name", "") sortField = h.GetParameterOrDefault(req, "sort_field", "") sortOrder = h.GetParameterOrDefault(req, "sort_order", "") - queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d%s}` + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d%s%s}` strSize = h.GetParameterOrDefault(req, "size", "20") strFrom = h.GetParameterOrDefault(req, "from", "0") mustBuilder = &strings.Builder{} @@ -388,7 +1025,7 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. if name != "" { mustBuilder.WriteString(fmt.Sprintf(`{"prefix":{"name.text": "%s"}}`, name)) } - clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "id") if !hasAllPrivilege && clusterFilter == nil { h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) return @@ -399,6 +1036,9 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. } mustBuilder.Write(util.MustToJSONBytes(clusterFilter)) } + if mustBuilder.Len() == 0 { + mustBuilder.WriteString(`{"match_all":{}}`) + } size, _ := strconv.Atoi(strSize) if size <= 0 { @@ -408,23 +1048,38 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. if from < 0 { from = 0 } - var sort = "" + var ( + functions = "" + sort = "" + trackScore = "" + ) if sortField != "" && sortOrder != "" { sort = fmt.Sprintf(`,"sort":[{"%s":{"order":"%s"}}]`, sortField, sortOrder) + } else { + functions = `,"functions":[{"filter":{"term":{"labels.health_status":"red"}},"weight":300},{"filter":{"term":{"labels.health_status":"yellow"}},"weight":200},{"filter":{"term":{"labels.health_status":"unavailable"}},"weight":100},{"filter":{"term":{"labels.health_status":"green"}},"weight":1}]` + sort = `,"sort":[{"_score":{"order":"desc"}},{"name.keyword":{"order":"asc","unmapped_type":"keyword"}}]` + trackScore = `,"track_scores":true` + queryDSL = `{"query":{"function_score":{"query":{"bool":{"must":[%s]}}%s,"score_mode":"sum","boost_mode":"replace"}}, "size": %d, "from": %d%s%s}` } - queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from, sort) + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), functions, size, from, sort, trackScore) q := orm.Query{ RawQuery: []byte(queryDSL), } err, result := orm.Search(elastic.ElasticsearchConfig{}, &q) if err != nil { - log.Error(err) + if global.Env().IsDebug { + log.Errorf("cluster search failed, name=%q, from=%d, size=%d, dsl=%s, err=%v", name, from, size, queryDSL, err) + } + log.Errorf("HandleSearchClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } searchRes := elastic.SearchResponse{} util.MustFromJSONBytes(result.Raw, &searchRes) + if global.Env().IsDebug && len(searchRes.Hits.Hits) == 0 { + log.Debugf("cluster search returned zero hits, name=%q, from=%d, size=%d, dsl=%s", name, from, size, queryDSL) + } if len(searchRes.Hits.Hits) > 0 { for _, hit := range searchRes.Hits.Hits { if basicAuth, ok := hit.Source["basic_auth"]; ok { @@ -485,7 +1140,7 @@ func (h *APIHandler) HandleMetricsSummaryAction(w http.ResponseWriter, req *http err, result := orm.Search(event.Event{}, &q) if err != nil { resBody["error"] = err.Error() - log.Error("MetricsSummary search error: ", err) + log.Errorf("HandleMetricsSummaryAction metrics summary search failed: %v", err) h.WriteJSON(w, resBody, http.StatusInternalServerError) return } @@ -566,7 +1221,7 @@ func (h *APIHandler) HandleMetricsSummaryAction(w http.ResponseWriter, req *http q.RawQuery = util.MustToJSONBytes(query) err, result = orm.Search(event.Event{}, &q) if err != nil { - log.Error("MetricsSummary search error: ", err) + log.Errorf("HandleMetricsSummaryAction metrics summary search failed: %v", err) } else { if len(result.Result) > 0 { if v, ok := result.Result[0].(map[string]interface{}); ok { @@ -622,7 +1277,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -633,7 +1288,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http } else if key == ShardStateMetricKey { clusterUUID, err := h.getClusterUUID(id) if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -689,7 +1344,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http } shardStateMetric, err := getNodeShardStateMetric(ctx, query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -700,7 +1355,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http metrics, err = h.GetClusterMetrics(ctx, id, bucketSize, min, max, key) } if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -708,7 +1363,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http if metrics[key].HitsTotal > 0 && metrics[key].MinBucketSize == 0 { minBucketSize, err := v1.GetMetricMinBucketSize(id, metricType) if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) } else { metrics[key].MinBucketSize = int64(minBucketSize) } @@ -725,7 +1380,7 @@ func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Re id := ps.ByName("id") bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, id, v1.MetricTypeNodeStats, 90) if err != nil { - log.Error(err) + log.Errorf("HandleNodeMetricsAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -739,7 +1394,7 @@ func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Re timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("HandleNodeMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -747,7 +1402,7 @@ func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Re defer cancel() metrics, err := h.getNodeMetrics(ctx, id, bucketSize, min, max, nodeName, top, key) if err != nil { - log.Error(err) + log.Errorf("HandleNodeMetricsAction failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -755,7 +1410,7 @@ func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Re if metrics[key].HitsTotal > 0 { minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats) if err != nil { - log.Error(err) + log.Errorf("HandleNodeMetricsAction failed: %v", err) } else { metrics[key].MinBucketSize = int64(minBucketSize) } @@ -766,7 +1421,7 @@ func (h *APIHandler) HandleNodeMetricsAction(w http.ResponseWriter, req *http.Re if ver.Distribution == "" { cr, err := util.VersionCompare(ver.Number, "6.1") if err != nil { - log.Error(err) + log.Errorf("HandleNodeMetricsAction failed: %v", err) } if cr < 0 { resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate" @@ -786,7 +1441,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R } bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, id, v1.MetricTypeNodeStats, 90) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -801,7 +1456,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -811,13 +1466,13 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R if key == v1.DocPercentMetricKey { metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, v1.DocCountMetricKey) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } docsDeletedMetrics, err := h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, v1.DocsDeletedMetricKey) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -874,7 +1529,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R } else { metrics, err = h.getIndexMetrics(ctx, req, id, bucketSize, min, max, indexName, top, shardID, key) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -883,7 +1538,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R if metrics[key].HitsTotal > 0 { minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) } else { metrics[key].MinBucketSize = int64(minBucketSize) } @@ -894,7 +1549,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R if ver.Distribution == "" { cr, err := util.VersionCompare(ver.Number, "6.1") if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) } if cr < 0 { resBody["tips"] = "The system cluster version is lower than 6.1, the top index may be inaccurate" @@ -909,7 +1564,7 @@ func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.R id := ps.ByName("id") bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, id, v1.MetricTypeNodeStats, 90) if err != nil { - log.Error(err) + log.Errorf("HandleQueueMetricsAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -923,7 +1578,7 @@ func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.R timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("HandleQueueMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -931,7 +1586,7 @@ func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.R defer cancel() metrics, err := h.getThreadPoolMetrics(ctx, id, bucketSize, min, max, nodeName, top, key) if err != nil { - log.Error(err) + log.Errorf("HandleQueueMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -939,7 +1594,7 @@ func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.R if metrics[key].HitsTotal > 0 { minBucketSize, err := v1.GetMetricMinBucketSize(id, v1.MetricTypeNodeStats) if err != nil { - log.Error(err) + log.Errorf("HandleQueueMetricsAction failed: %v", err) } else { metrics[key].MinBucketSize = int64(minBucketSize) } @@ -950,7 +1605,7 @@ func (h *APIHandler) HandleQueueMetricsAction(w http.ResponseWriter, req *http.R if ver.Distribution == "" { cr, err := util.VersionCompare(ver.Number, "6.1") if err != nil { - log.Error(err) + log.Errorf("HandleQueueMetricsAction failed: %v", err) } if cr < 0 { resBody["tips"] = "The system cluster version is lower than 6.1, the top node may be inaccurate" @@ -1014,7 +1669,7 @@ func (h *APIHandler) GetClusterHealth(w http.ResponseWriter, req *http.Request, exists, client, err := h.GetClusterClient(id) if err != nil { - log.Error(err) + log.Errorf("GetClusterHealth failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -1507,7 +2162,7 @@ func (h *APIHandler) getClusterStatusMetric(ctx context.Context, id string, min, queryDSL := util.MustToJSONBytes(query) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL) if err != nil { - log.Error(err) + log.Errorf("getClusterStatusMetric failed: %v", err) return nil, err } metricData := []interface{}{} diff --git a/modules/elastic/api/manage_test.go b/modules/elastic/api/manage_test.go new file mode 100644 index 00000000..f05bcaf6 --- /dev/null +++ b/modules/elastic/api/manage_test.go @@ -0,0 +1,402 @@ +package api + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" +) + +func TestGetClusterCredentialDisplayName(t *testing.T) { + tests := []struct { + name string + cluster string + kind string + wantName string + }{ + { + name: "platform credential name", + cluster: "easysearch-test-cluster", + kind: clusterCredentialKindPlatform, + wantName: "easysearch-test-cluster (Platform)", + }, + { + name: "agent credential name", + cluster: "easysearch-test-cluster", + kind: clusterCredentialKindAgent, + wantName: "easysearch-test-cluster (Agent)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getClusterCredentialDisplayName(tt.cluster, tt.kind); got != tt.wantName { + t.Fatalf("expected %q, got %q", tt.wantName, got) + } + }) + } +} + +func TestIsAutoGeneratedClusterCredentialName(t *testing.T) { + const clusterID = "d7vd5sr0ebik06gvpjd0" + tests := []struct { + name string + credName string + cluster string + kind string + want bool + }{ + { + name: "matches new platform format", + credName: "easysearch-test-cluster (Platform)", + cluster: "easysearch-test-cluster", + kind: clusterCredentialKindPlatform, + want: true, + }, + { + name: "matches legacy agent format", + credName: "easysearch-test-cluster_agent(d7vd5sr0ebik06gvpjd0)", + cluster: "easysearch-test-cluster", + kind: clusterCredentialKindAgent, + want: true, + }, + { + name: "ignores custom names", + credName: "production readonly credential", + cluster: "easysearch-test-cluster", + kind: clusterCredentialKindPlatform, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isAutoGeneratedClusterCredentialName(tt.credName, tt.cluster, clusterID, tt.kind); got != tt.want { + t.Fatalf("expected %v, got %v", tt.want, got) + } + }) + } +} + +func TestEnsureManagedAgentCollectionCredentialSkipsAgentlessWithoutPlatformAuth(t *testing.T) { + conf := &elastic.ElasticsearchConfig{ + Distribution: elastic.Easysearch, + MetricCollectionMode: elastic.ModeAgentless, + } + + if err := ensureManagedAgentCollectionCredential(conf, ""); err != nil { + t.Fatalf("expected agentless cluster without platform auth to be skipped, got %v", err) + } +} + +func TestEnsureManagedAgentCollectionCredentialRequiresPlatformAuthInAgentMode(t *testing.T) { + conf := &elastic.ElasticsearchConfig{ + Distribution: elastic.Easysearch, + MetricCollectionMode: elastic.ModeAgent, + } + + err := ensureManagedAgentCollectionCredential(conf, "") + if err == nil { + t.Fatal("expected error when agent mode cluster has no platform auth") + } + if !strings.Contains(err.Error(), "platform credential is required") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestEnsureManagedAgentCollectionCredentialAllowsNoAuthAgentMode(t *testing.T) { + conf := &elastic.ElasticsearchConfig{ + Distribution: elastic.Easysearch, + MetricCollectionMode: elastic.ModeAgent, + NoDefaultAuthForAgent: true, + AgentBasicAuth: &model.BasicAuth{Username: "infini-agent"}, + } + + if err := ensureManagedAgentCollectionCredential(conf, ""); err != nil { + t.Fatalf("expected explicit no-auth agent mode to be allowed, got %v", err) + } + if conf.AgentBasicAuth != nil { + t.Fatalf("expected stale auto agent basic auth to be cleared, got %#v", conf.AgentBasicAuth) + } +} + +func TestEnsureManagedAgentCollectionCredentialSkipsUnsupportedDistribution(t *testing.T) { + conf := &elastic.ElasticsearchConfig{ + Distribution: elastic.Elasticsearch, + MetricCollectionMode: elastic.ModeAgent, + } + + if err := ensureManagedAgentCollectionCredential(conf, ""); err != nil { + t.Fatalf("expected unsupported distribution to be skipped, got %v", err) + } +} + +func TestHydrateRuntimeBasicAuthSetsDecodedCredential(t *testing.T) { + conf := &elastic.ElasticsearchConfig{CredentialID: "cred-1"} + auth := &model.BasicAuth{Username: "admin"} + + err := hydrateRuntimeBasicAuthWithGetter(conf, func(cfg *elastic.ElasticsearchConfig) (*model.BasicAuth, error) { + if cfg.CredentialID != "cred-1" { + t.Fatalf("expected credential-backed config, got %#v", cfg) + } + return auth, nil + }) + if err != nil { + t.Fatalf("expected auth hydration to succeed, got %v", err) + } + if conf.BasicAuth != auth { + t.Fatalf("expected hydrated auth to be applied to runtime config, got %#v", conf.BasicAuth) + } +} + +func TestHydrateRuntimeBasicAuthPropagatesGetterError(t *testing.T) { + wantErr := errors.New("decode credential failed") + + err := hydrateRuntimeBasicAuthWithGetter(&elastic.ElasticsearchConfig{}, func(*elastic.ElasticsearchConfig) (*model.BasicAuth, error) { + return nil, wantErr + }) + if !errors.Is(err, wantErr) { + t.Fatalf("expected getter error %v, got %v", wantErr, err) + } +} + +func TestApplyExplicitPlatformAuthPreferenceDisablesPlatformAuth(t *testing.T) { + authEnabled := false + conf := &elastic.ElasticsearchConfig{ + CredentialID: "cred-1", + BasicAuth: &model.BasicAuth{Username: "admin"}, + NoDefaultAuthForAgent: false, + } + + applyExplicitPlatformAuthPreference(conf, &authEnabled) + + if conf.CredentialID != "" { + t.Fatalf("expected credential to be cleared, got %q", conf.CredentialID) + } + if conf.BasicAuth != nil { + t.Fatalf("expected basic auth to be cleared, got %#v", conf.BasicAuth) + } + if !conf.NoDefaultAuthForAgent { + t.Fatal("expected no-default-auth flag to be enabled") + } +} + +func TestApplyExplicitPlatformAuthPreferenceEnablesPlatformAuth(t *testing.T) { + authEnabled := true + conf := &elastic.ElasticsearchConfig{ + NoDefaultAuthForAgent: true, + } + + applyExplicitPlatformAuthPreference(conf, &authEnabled) + + if conf.NoDefaultAuthForAgent { + t.Fatal("expected no-default-auth flag to be reset when auth is enabled") + } +} + +func TestApplyExplicitPlatformAuthPreferenceToSourceDisablesPlatformAuth(t *testing.T) { + authEnabled := false + source := map[string]interface{}{ + "credential_id": "cred-1", + "basic_auth": map[string]interface{}{ + "username": "admin", + }, + } + + applyExplicitPlatformAuthPreferenceToSource(source, &authEnabled) + + if source["credential_id"] != "" { + t.Fatalf("expected credential_id to be cleared, got %#v", source["credential_id"]) + } + if value, ok := source["basic_auth"]; !ok || value != nil { + t.Fatalf("expected basic_auth to be cleared, got %#v", source["basic_auth"]) + } + if source["no_default_auth_for_agent"] != true { + t.Fatalf("expected no_default_auth_for_agent to be true, got %#v", source["no_default_auth_for_agent"]) + } +} + +func TestNewManagedClusterSecurityClientUsesTemporaryMetadataForUnregisteredCluster(t *testing.T) { + conf := &elastic.ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: "cluster-under-test"}, + Name: "cluster-under-test", + Distribution: elastic.Easysearch, + Version: "2.2.0", + Enabled: true, + Schema: "http", + Host: "127.0.0.1:9200", + Endpoint: "http://127.0.0.1:9200", + } + auth := &model.BasicAuth{Username: "admin"} + + tempRuntimeID := managedClusterSecurityRuntimeID(conf.ID) + _, cleanup, err := newManagedClusterSecurityClient(conf, auth) + if err != nil { + t.Fatalf("expected managed client to initialize, got %v", err) + } + defer cleanup() + + if meta := elastic.GetMetadata(conf.ID); meta != nil { + t.Fatalf("expected real cluster metadata %q to stay untouched, got %#v", conf.ID, meta.Config) + } + if meta := elastic.GetMetadata(tempRuntimeID); meta == nil { + t.Fatal("expected temporary client metadata to be initialized") + } else if meta.Config == nil || meta.Config.ID != tempRuntimeID { + t.Fatalf("expected metadata for %q, got %#v", tempRuntimeID, meta.Config) + } +} + +func TestBuildAgentCollectionRoleBody(t *testing.T) { + var body struct { + Cluster []string `json:"cluster"` + Indices []struct { + Names []string `json:"names"` + Privileges []string `json:"privileges"` + } `json:"indices"` + } + + roleBody, err := buildAgentCollectionRoleBody(elastic.Easysearch) + if err != nil { + t.Fatalf("expected easysearch role body, got %v", err) + } + if err := json.Unmarshal(roleBody, &body); err != nil { + t.Fatalf("expected valid json body, got %v", err) + } + + if len(body.Cluster) != 1 || body.Cluster[0] != "cluster_monitor" { + t.Fatalf("expected cluster_monitor permission, got %#v", body.Cluster) + } + if len(body.Indices) != 1 { + t.Fatalf("expected one indices permission block, got %#v", body.Indices) + } + if len(body.Indices[0].Names) != 1 || body.Indices[0].Names[0] != "*" { + t.Fatalf("expected wildcard indices scope, got %#v", body.Indices[0].Names) + } + if len(body.Indices[0].Privileges) != 1 || body.Indices[0].Privileges[0] != "indices_monitor" { + t.Fatalf("expected indices_monitor privilege, got %#v", body.Indices[0].Privileges) + } +} + +func TestBuildAgentCollectionRoleBodyRejectsUnsupportedDistribution(t *testing.T) { + if _, err := buildAgentCollectionRoleBody(elastic.Elasticsearch); err == nil { + t.Fatal("expected unsupported distribution error") + } +} + +func TestBuildAgentCollectionUserBodyRejectsUnsupportedDistribution(t *testing.T) { + if _, err := buildAgentCollectionUserBody(elastic.Elasticsearch, "infini-agent", "secret"); err == nil { + t.Fatal("expected unsupported distribution error") + } +} + +func TestWrapAgentCollectionProvisionErrorForEasysearchRoleAPI(t *testing.T) { + err := wrapAgentCollectionProvisionError( + elastic.Easysearch, + "role", + errors.New(`{"status":"NOT_FOUND","message":"Resource 'infini-agent' is not available."}`), + ) + + if err == nil { + t.Fatal("expected wrapped error") + } + if !strings.Contains(err.Error(), "does not allow modifying _security/role") { + t.Fatalf("expected actionable role api error, got %v", err) + } + if !strings.Contains(err.Error(), "security.restapi.roles_enabled") { + t.Fatalf("expected config hint in error, got %v", err) + } +} + +func TestWrapAgentCollectionProvisionErrorForUnsupportedSecurityAPI(t *testing.T) { + err := wrapAgentCollectionProvisionError( + elastic.Easysearch, + "role", + errors.New(`{"error":{"root_cause":[{"type":"invalid_index_name_exception","reason":"Invalid index name [_security], must not start with '_', '-', or '+'","index_uuid":"_na_","index":"_security"}],"type":"invalid_index_name_exception","reason":"Invalid index name [_security], must not start with '_', '-', or '+'","index_uuid":"_na_","index":"_security"},"status":400}`), + ) + + if err == nil { + t.Fatal("expected wrapped error") + } + if !strings.Contains(err.Error(), "does not allow modifying _security/role") { + t.Fatalf("expected unsupported security api hint, got %v", err) + } +} + +func TestShouldFallbackToPlatformCredentialForManagedAgentProvision(t *testing.T) { + err := errors.New("failed to create agent collection user: target Easysearch cluster does not allow modifying _security/user via the current credential") + if !shouldFallbackToPlatformCredentialForManagedAgentProvision(err) { + t.Fatalf("expected unsupported managed-user provisioning to fall back, got %v", err) + } +} + +func TestShouldFallbackToPlatformCredentialForManagedAgentProvisionWithUnsupportedSecurityAPI(t *testing.T) { + err := errors.New("failed to create agent collection role: target Easysearch cluster does not allow modifying _security/role via the current credential") + if !shouldFallbackToPlatformCredentialForManagedAgentProvision(err) { + t.Fatalf("expected unsupported security api provisioning to fall back, got %v", err) + } +} + +func TestShouldFallbackToPlatformCredentialForManagedAgentProvisionSkipsOtherErrors(t *testing.T) { + err := errors.New("failed to create agent collection user: connection refused") + if shouldFallbackToPlatformCredentialForManagedAgentProvision(err) { + t.Fatalf("expected ordinary provisioning errors to remain blocking, got %v", err) + } +} + +func TestApplyClusterRuntimeConfigUpdatesExistingMetadataConfig(t *testing.T) { + conf := &elastic.ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: "cluster-runtime-config"}, + Name: "cluster-runtime-config", + Monitored: true, + } + oldMeta := elastic.InitMetadata(conf, true) + t.Cleanup(func() { + elastic.RemoveInstance(conf.ID) + }) + + nextConf := *conf + nextConf.Monitored = false + + if err := applyClusterRuntimeConfig(&nextConf); err != nil { + t.Fatalf("expected runtime config update to succeed, got %v", err) + } + + meta := elastic.GetMetadata(conf.ID) + if meta == nil || meta.Config == nil { + t.Fatal("expected metadata to remain available after runtime config update") + } + if meta.Config.Monitored { + t.Fatalf("expected monitored flag to be updated to false, got %#v", meta.Config) + } + if oldMeta == meta { + t.Fatal("expected metadata instance to be replaced to trigger update listeners") + } +} + +func TestGetManagedAgentCollectionFallbackUsername(t *testing.T) { + username, ok := getManagedAgentCollectionFallbackUsername( + elastic.Easysearch, + autoAgentCollectionUsername, + errors.New(`{"status":"NOT_FOUND","message":"Resource 'infini-agent' is not available."}`), + ) + if !ok { + t.Fatal("expected fallback username for reserved infini-agent resource") + } + if username != autoAgentCollectionFallbackUsername { + t.Fatalf("expected fallback username %q, got %q", autoAgentCollectionFallbackUsername, username) + } +} + +func TestGetManagedAgentCollectionFallbackUsernameSkipsCustomUser(t *testing.T) { + if _, ok := getManagedAgentCollectionFallbackUsername( + elastic.Easysearch, + "custom-agent-user", + errors.New(`{"status":"NOT_FOUND","message":"Resource 'custom-agent-user' is not available."}`), + ); ok { + t.Fatal("expected no fallback for custom managed usernames") + } +} diff --git a/modules/elastic/api/metrics_util.go b/modules/elastic/api/metrics_util.go index 0125e3dd..2e850d49 100644 --- a/modules/elastic/api/metrics_util.go +++ b/modules/elastic/api/metrics_util.go @@ -268,9 +268,19 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m var rangeFrom, rangeTo time.Time var err error var useMinMax = bucketSize == 0 + if strings.EqualFold(strings.TrimSpace(minStr), "auto") { + minStr = "" + } + if strings.EqualFold(strings.TrimSpace(maxStr), "auto") { + maxStr = "" + } + effectiveBucketSize := bucketSize + if effectiveBucketSize <= 0 { + effectiveBucketSize = GetMinBucketSize() + } now := time.Now() if minStr == "" { - rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + rangeFrom = now.Add(-time.Second * time.Duration(effectiveBucketSize*metricCount+1)) } else { //try 2021-08-21T14:06:04.818Z rangeFrom, err = util.ParseStandardTime(minStr) @@ -278,8 +288,8 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m //try 1629637500000 v, err := util.ToInt64(minStr) if err != nil { - log.Error("invalid timestamp:", minStr, err) - rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + log.Errorf("GetMetricRangeAndBucketSize invalid min timestamp [%s]: %v", minStr, err) + rangeFrom = now.Add(-time.Second * time.Duration(effectiveBucketSize*metricCount+1)) } else { rangeFrom = util.FromUnixTimestamp(v / 1000) } @@ -287,14 +297,14 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m } if maxStr == "" { - rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + rangeTo = now.Add(-time.Second * time.Duration(effectiveBucketSize)) } else { rangeTo, err = util.ParseStandardTime(maxStr) if err != nil { v, err := util.ToInt64(maxStr) if err != nil { - log.Error("invalid timestamp:", maxStr, err) - rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + log.Errorf("GetMetricRangeAndBucketSize invalid max timestamp [%s]: %v", maxStr, err) + rangeTo = now.Add(-time.Second * time.Duration(effectiveBucketSize)) } else { rangeTo = util.FromUnixTimestamp(int64(v) / 1000) } @@ -311,6 +321,91 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m return bucketSize, min, max, nil } +func buildDateHistogramParams(query map[string]interface{}, intervalField, bucketSizeStr string) util.MapStr { + params := util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + "min_doc_count": 0, + } + if bounds, ok := extractDateHistogramBounds(query); ok { + params["extended_bounds"] = bounds + } + return params +} + +func extractDateHistogramBounds(query map[string]interface{}) (util.MapStr, bool) { + queryMap, ok := asMap(query["query"]) + if !ok { + return nil, false + } + boolMap, ok := asMap(queryMap["bool"]) + if !ok { + return nil, false + } + filters, ok := asSlice(boolMap["filter"]) + if !ok { + return nil, false + } + for _, filterItem := range filters { + filterMap, ok := asMap(filterItem) + if !ok { + continue + } + rangeMap, ok := asMap(filterMap["range"]) + if !ok { + continue + } + timestampRange, ok := asMap(rangeMap["timestamp"]) + if !ok { + continue + } + min := firstNotNil(timestampRange["gte"], timestampRange["from"], timestampRange["gt"]) + max := firstNotNil(timestampRange["lte"], timestampRange["to"], timestampRange["lt"]) + if min != nil && max != nil { + return util.MapStr{ + "min": min, + "max": max, + }, true + } + } + return nil, false +} + +func asMap(value interface{}) (map[string]interface{}, bool) { + switch v := value.(type) { + case map[string]interface{}: + return v, true + case util.MapStr: + return map[string]interface{}(v), true + default: + return nil, false + } +} + +func asSlice(value interface{}) ([]interface{}, bool) { + switch v := value.(type) { + case []interface{}: + return v, true + case []util.MapStr: + result := make([]interface{}, 0, len(v)) + for _, item := range v { + result = append(result, item) + } + return result, true + default: + return nil, false + } +} + +func firstNotNil(values ...interface{}) interface{} { + for _, value := range values { + if value != nil { + return value + } + } + return nil +} + // 获取单个指标，可以包含多条曲线 func (h *APIHandler) getSingleMetrics(ctx context.Context, metricItems []*common.MetricItem, query map[string]interface{}, bucketSize int) (map[string]*common.MetricItem, error) { metricData := map[string][][]interface{}{} @@ -363,11 +458,8 @@ func (h *APIHandler) getSingleMetrics(ctx context.Context, metricItems []*common query["size"] = 0 query["aggs"] = util.MapStr{ "dates": util.MapStr{ - "date_histogram": util.MapStr{ - "field": "timestamp", - intervalField: bucketSizeStr, - }, - "aggs": aggs, + "date_histogram": buildDateHistogramParams(query, intervalField, bucketSizeStr), + "aggs": aggs, }, } queryDSL := util.MustToJSONBytes(query) @@ -458,7 +550,7 @@ func (h *APIHandler) getBucketMetrics(query map[string]interface{}, bucketItems //bucketSizeStr := fmt.Sprintf("%vs", bucketSize) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) if err != nil { - log.Error(err) + log.Errorf("getBucketMetrics failed: %v", err) panic(err) } //grpMetricItemsIndex := map[string]int{} @@ -584,8 +676,18 @@ func ConvertBucketItemsToAggQuery(bucketItems []*common.BucketItem, metricItems switch bucketItem.Type { case "terms": + termsParams := util.MapStr{} + for k, v := range bucketItem.Parameters { + termsParams[k] = v + } + // Some Elasticsearch-compatible engines return UnmappedTerms runtime + // errors on terms aggregation. Provide a default missing value so the + // aggregation can still be planned when the field is absent. + if _, ok := termsParams["missing"]; !ok { + termsParams["missing"] = "" + } bucketAgg = util.MapStr{ - "terms": bucketItem.Parameters, + "terms": termsParams, } break case "date_histogram": @@ -901,7 +1003,7 @@ func parseGroupMetricData(buckets []elastic.BucketBase, isPercent bool) ([]inter for _, bucket := range buckets { v, ok := bucket["key"].(float64) if !ok { - log.Error("invalid bucket key") + log.Errorf("parseGroupMetricData invalid bucket key in aggregation response") return nil, fmt.Errorf("invalid bucket key") } dateTime := int64(v) @@ -1011,11 +1113,8 @@ func (h *APIHandler) getSingleIndexMetricsByNodeStats(ctx context.Context, metri query["size"] = 0 query["aggs"] = util.MapStr{ "dates": util.MapStr{ - "date_histogram": util.MapStr{ - "field": "timestamp", - intervalField: bucketSizeStr, - }, - "aggs": sumAggs, + "date_histogram": buildDateHistogramParams(query, intervalField, bucketSizeStr), + "aggs": sumAggs, }, } return parseSingleIndexMetrics(ctx, term_level, clusterID, metricItems, query, bucketSize, metricData, metricItemsMap) @@ -1106,11 +1205,8 @@ func (h *APIHandler) getSingleIndexMetrics(ctx context.Context, metricItems []*c query["size"] = 0 query["aggs"] = util.MapStr{ "dates": util.MapStr{ - "date_histogram": util.MapStr{ - "field": "timestamp", - intervalField: bucketSizeStr, - }, - "aggs": sumAggs, + "date_histogram": buildDateHistogramParams(query, intervalField, bucketSizeStr), + "aggs": sumAggs, }, } return parseSingleIndexMetrics(ctx, term_level, clusterID, metricItems, query, bucketSize, metricData, metricItemsMap) diff --git a/modules/elastic/api/monitor_state.go b/modules/elastic/api/monitor_state.go index e83ffebc..4b63c72e 100644 --- a/modules/elastic/api/monitor_state.go +++ b/modules/elastic/api/monitor_state.go @@ -28,14 +28,13 @@ package api import ( - "fmt" "infini.sh/framework/core/elastic" ) func GetMonitorState(clusterID string) string { conf := elastic.GetConfig(clusterID) if conf == nil { - panic(fmt.Errorf("config of cluster [%s] is not found", clusterID)) + return elastic.ModeAgentless } if conf.MetricCollectionMode == "" { if conf.MonitorConfigs != nil && !conf.MonitorConfigs.NodeStats.Enabled && !conf.MonitorConfigs.IndexStats.Enabled { diff --git a/modules/elastic/api/node_metrics.go b/modules/elastic/api/node_metrics.go index 0784436d..0b2df64e 100644 --- a/modules/elastic/api/node_metrics.go +++ b/modules/elastic/api/node_metrics.go @@ -168,7 +168,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke } else { nodeNames, err = h.getTopNodeName(clusterID, top, 15) if err != nil { - log.Error(err) + log.Errorf("getNodeMetrics failed: %v", err) } } if len(nodeNames) > 0 { @@ -1146,7 +1146,7 @@ func (h *APIHandler) getNodeMetrics(ctx context.Context, clusterID string, bucke aggs := generateGroupAggs(nodeMetricItems) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) if err != nil { - log.Error(err) + log.Errorf("getNodeMetrics failed: %v", err) panic(err) } @@ -1310,7 +1310,7 @@ func (h *APIHandler) getTopNodeName(clusterID string, top int, lastMinutes int) } response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) if err != nil { - log.Error(err) + log.Errorf("getTopNodeName failed: %v", err) return nil, err } var maxQpsKVS = map[string]float64{} diff --git a/modules/elastic/api/node_overview.go b/modules/elastic/api/node_overview.go index 6a13ddc9..f78c651a 100644 --- a/modules/elastic/api/node_overview.go +++ b/modules/elastic/api/node_overview.go @@ -199,12 +199,10 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps h.WriteJSON(w, util.MapStr{}, http.StatusOK) return } - //only query one node info for fast response - nodeIDs = nodeIDs[0:1] timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("invalid fetch node info timeout [%s]: %v", timeout, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -213,6 +211,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps q1 := orm.Query{WildcardIndex: true} query := util.MapStr{ + "size": 1000, "sort": []util.MapStr{ { "timestamp": util.MapStr{ @@ -220,9 +219,6 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps }, }, }, - "collapse": util.MapStr{ - "field": "metadata.labels.node_id", - }, "query": util.MapStr{ "bool": util.MapStr{ "must": []util.MapStr{ @@ -241,8 +237,8 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps }, }, { - "term": util.MapStr{ - "metadata.labels.node_id": nodeIDs[0], + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, }, }, }, @@ -253,16 +249,22 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps err, results := orm.Search(&event.Event{}, &q1) if err != nil { - log.Error(err) + log.Errorf("failed to search node overview info for nodes %v: %v", nodeIDs, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } statusMap := map[string]interface{}{} + seenNodeIDs := map[string]struct{}{} for _, v := range results.Result { result, ok := v.(map[string]interface{}) if ok { nodeID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, result) if ok { + nodeIDStr := util.ToString(nodeID) + if _, exists := seenNodeIDs[nodeIDStr]; exists { + continue + } + seenNodeIDs[nodeIDStr] = struct{}{} source := map[string]interface{}{} //timestamp, ok := result["timestamp"].(string) uptime, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "uptime_in_millis"}, result) @@ -301,13 +303,13 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps } } - statusMap[util.ToString(nodeID)] = source + statusMap[nodeIDStr] = source } } } statusMetric, err := getNodeOnlineStatusOfRecentDay(nodeIDs) if err != nil { - log.Error(err) + log.Errorf("failed to get node online status for nodes %v: %v", nodeIDs, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -361,8 +363,8 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps }, }, { - "term": util.MapStr{ - "metadata.labels.node_id": nodeIDs[0], + "terms": util.MapStr{ + "metadata.labels.node_id": nodeIDs, }, }, }, @@ -418,7 +420,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps } metrics, err := h.getMetrics(ctx, "", query, nodeMetricItems, bucketSize) if err != nil { - log.Error(err) + log.Errorf("failed to get node overview metrics for nodes %v: %v", nodeIDs, err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -495,7 +497,6 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht orm.Eq("metadata.name", "node_stats"), orm.Eq("metadata.labels.node_id", nodeID), ) - q1.Collapse("metadata.labels.node_id") q1.AddSort("timestamp", orm.DESC) err, result := orm.Search(&event.Event{}, &q1) kvs := util.MapStr{} @@ -628,7 +629,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque resBody := map[string]interface{}{} bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, v1.MetricTypeNodeStats, 60) if err != nil { - log.Error(err) + log.Errorf("failed to get node metric range for cluster [%s], node [%s]: %v", clusterID, nodeID, err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -658,7 +659,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("invalid node metric timeout [%s] for cluster [%s], node [%s], key [%s]: %v", timeout, clusterID, nodeID, metricKey, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -668,12 +669,47 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque if metricKey == NodeHealthMetricKey { healthMetric, err := getNodeHealthMetric(ctx, query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("failed to get node health metric for cluster [%s], node [%s]: %v", clusterID, nodeID, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } metrics["node_health"] = healthMetric } else if metricKey == ShardStateMetricKey { + nodeIdentityFilter := util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.labels.node_id": util.MapStr{ + "value": nodeID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.node_uuid": util.MapStr{ + "value": nodeID, + }, + }, + }, + { + "term": util.MapStr{ + "payload.elasticsearch.shard_stats.routing.node": util.MapStr{ + "value": nodeID, + }, + }, + }, + { + "term": util.MapStr{ + "payload.elasticsearch.shard_stats.routing.current_node": util.MapStr{ + "value": nodeID, + }, + }, + }, + }, + }, + } query = util.MapStr{ "size": 0, "query": util.MapStr{ @@ -695,13 +731,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque }, }, }, - { - "term": util.MapStr{ - "metadata.labels.node_id": util.MapStr{ - "value": nodeID, - }, - }, - }, + nodeIdentityFilter, }, "filter": []util.MapStr{ { @@ -718,7 +748,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque } shardStateMetric, err := getNodeShardStateMetric(ctx, query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("failed to get shard state metric for cluster [%s], node [%s]: %v", clusterID, nodeID, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -792,7 +822,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque metrics, err = h.getSingleMetrics(ctx, metricItems, query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("failed to get node metrics for cluster [%s], node [%s], key [%s]: %v", clusterID, nodeID, metricKey, err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -801,7 +831,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque if metrics[metricKey].HitsTotal > 0 { minBucketSize, err := v1.GetMetricMinBucketSize(clusterID, v1.MetricTypeNodeStats) if err != nil { - log.Error(err) + log.Errorf("failed to get node metric min bucket size for cluster [%s], key [%s]: %v", clusterID, metricKey, err) } else { metrics[metricKey].MinBucketSize = int64(minBucketSize) } @@ -838,7 +868,7 @@ func getNodeShardStateMetric(ctx context.Context, query util.MapStr, bucketSize queryDSL := util.MustToJSONBytes(query) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL) if err != nil { - log.Error(err) + log.Errorf("failed to query shard state metric data: %v", err) return nil, err } @@ -882,7 +912,7 @@ func getNodeHealthMetric(ctx context.Context, query util.MapStr, bucketSize int) queryDSL := util.MustToJSONBytes(query) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, queryDSL) if err != nil { - log.Error(err) + log.Errorf("failed to query node health metric data: %v", err) return nil, err } @@ -894,7 +924,7 @@ func getNodeHealthMetric(ctx context.Context, query util.MapStr, bucketSize int) for _, bucket := range response.Aggregations["dates"].Buckets { v, ok := bucket["key"].(float64) if !ok { - log.Error("invalid bucket key") + log.Errorf("invalid bucket key in node health metric aggregation: %#v", bucket["key"]) return nil, fmt.Errorf("invalid bucket key") } dateTime := int64(v) @@ -1069,71 +1099,209 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string) (map[string][]interface{}, } func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.ByName("id") + if GetMonitorState(id) == elastic.ModeAgentless { + h.APIHandler.GetNodeIndices(w, req, ps) + return + } + var ( min = h.GetParameterOrDefault(req, "min", "now-15m") max = h.GetParameterOrDefault(req, "max", "now") ) resBody := map[string]interface{}{} - id := ps.ByName("id") nodeUUID := ps.ByName("node_id") - q := &orm.Query{Size: 1} - q.AddSort("timestamp", orm.DESC) - q.Conds = orm.And( - orm.Eq("metadata.category", "elasticsearch"), - orm.Eq("metadata.labels.cluster_id", id), - orm.Eq("metadata.labels.node_id", nodeUUID), - orm.Eq("metadata.name", "node_routing_table"), - ) - err, result := orm.Search(event.Event{}, q) + indices, err := h.getNodeLatestIndicesAgent(req, min, max, id, nodeUUID) if err != nil { resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) + return } - namesM := util.MapStr{} - if len(result.Result) > 0 { - if data, ok := result.Result[0].(map[string]interface{}); ok { - if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists { - if rows, ok := routingTable.([]interface{}); ok { - for _, row := range rows { - if v, ok := row.(map[string]interface{}); ok { - if indexName, ok := v["index"].(string); ok { - namesM[indexName] = true - } - } - } + // Agent mode primarily depends on shard_stats. When shard_stats is absent + // (e.g. newly created/idle index), fall back to routing-table based index + // listing to avoid returning an empty result. + if len(indices) == 0 { + h.APIHandler.GetNodeIndices(w, req, ps) + return + } + h.WriteJSON(w, indices, http.StatusOK) +} + +// getNodeLatestIndicesAgent builds the index list for a single node in agent mode. +// It uses shard_stats (filtered to this node) as the primary source, enriched by IndexConfig. +func (h *APIHandler) getNodeLatestIndicesAgent(req *http.Request, min, max, clusterID, nodeUUID string) ([]interface{}, error) { + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return []interface{}{}, nil + } + + clusterUUID, err := h.getClusterUUID(clusterID) + if err != nil { + return nil, err + } + + nodeIdentityFilter := util.MapStr{ + "bool": util.MapStr{ + "minimum_should_match": 1, + "should": []util.MapStr{ + {"term": util.MapStr{"metadata.labels.node_id": util.MapStr{"value": nodeUUID}}}, + {"term": util.MapStr{"metadata.labels.node_uuid": util.MapStr{"value": nodeUUID}}}, + {"term": util.MapStr{"payload.elasticsearch.shard_stats.routing.node": util.MapStr{"value": nodeUUID}}}, + {"term": util.MapStr{"payload.elasticsearch.shard_stats.routing.current_node": util.MapStr{"value": nodeUUID}}}, + }, + }, + } + + // Query shard_stats for this node in the time range. + query := util.MapStr{ + "size": 10000, + "_source": []string{ + "metadata.labels.index_name", + "metadata.labels.shard", + "metadata.labels.shard_id", + "payload.elasticsearch.shard_stats.docs", + "payload.elasticsearch.shard_stats.store", + "payload.elasticsearch.shard_stats.routing", + "timestamp", + }, + "sort": []util.MapStr{ + {"timestamp": util.MapStr{"order": "desc"}}, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + {"range": util.MapStr{"timestamp": util.MapStr{"gte": min, "lte": max}}}, + }, + "must": []util.MapStr{ + {"term": util.MapStr{"metadata.category": util.MapStr{"value": "elasticsearch"}}}, + {"term": util.MapStr{"metadata.labels.cluster_uuid": util.MapStr{"value": clusterUUID}}}, + {"term": util.MapStr{"metadata.name": util.MapStr{"value": "shard_stats"}}}, + nodeIdentityFilter, + }, + }, + }, + } + q := &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} + indexInfos := map[string]*ShardsSummary{} + seenShards := map[string]struct{}{} + _, searchResult := orm.Search(event.Event{}, q) + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + shardDocCount, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, hitM) + storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, hitM) + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) + shardID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "shard_id"}, hitM) + shardNum, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "shard"}, hitM) + primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, hitM) + isPrimary, _ := parseBoolValue(primary) + if v, ok := indexName.(string); ok { + dedupeKey := util.ToString(shardID) + if dedupeKey == "" { + dedupeKey = fmt.Sprintf("%s:%s:%v", v, util.ToString(shardNum), primary) + } + if _, exists := seenShards[dedupeKey]; exists { + continue + } + seenShards[dedupeKey] = struct{}{} + if _, ok = indexInfos[v]; !ok { + indexInfos[v] = &ShardsSummary{Index: v} + } + info := indexInfos[v] + if count, ok := parseInt64Value(shardDocCount); ok && isPrimary { + info.DocsCount += count + } + if storeSize, ok := parseInt64Value(storeInBytes); ok { + info.StoreInBytes += storeSize + } + if isPrimary { + info.Shards++ + } else { + info.Replicas++ + } + if info.Timestamp == nil { + info.Timestamp = hitM["timestamp"] } } } } - indexNames := make([]interface{}, 0, len(namesM)) - for name, _ := range namesM { - indexNames = append(indexNames, name) + if len(indexInfos) == 0 { + return []interface{}{}, nil } - q1 := &orm.Query{Size: 100} + // Enrich with IndexConfig (state, health, configured shards/replicas). + // Query all IndexConfig for the cluster (same approach as GetClusterIndices) + // to avoid issues with orm.In on large index lists. + q1 := &orm.Query{Size: 2000} q1.AddSort("timestamp", orm.DESC) q1.Conds = orm.And( orm.Eq("metadata.category", "elasticsearch"), - orm.Eq("metadata.cluster_id", id), - orm.In("metadata.index_name", indexNames), - orm.NotEq("metadata.labels.index_status", "deleted"), + orm.Eq("metadata.cluster_id", clusterID), ) - err, result = orm.Search(elastic.IndexConfig{}, q1) - if err != nil { - resBody["error"] = err.Error() - h.WriteJSON(w, resBody, http.StatusInternalServerError) + _, indexConfigResult := orm.Search(elastic.IndexConfig{}, q1) + indexConfigMap := map[string]map[string]interface{}{} + for _, hit := range indexConfigResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + nameV, _ := util.GetMapValueByKeys([]string{"metadata", "index_name"}, hitM) + if name, ok2 := nameV.(string); ok2 { + if _, exists := indexConfigMap[name]; !exists { + indexConfigMap[name] = hitM + } + } + } } - indices, err := h.getLatestIndices(req, min, max, id, &result) - if err != nil { - resBody["error"] = err.Error() - h.WriteJSON(w, resBody, http.StatusInternalServerError) + indices := []interface{}{} + var indexPattern *radix.Pattern + if !hasAllPrivilege { + indexPattern = radix.Compile(allowedIndices...) } - h.WriteJSON(w, indices, http.StatusOK) + for indexName, info := range indexInfos { + if indexPattern != nil && !indexPattern.Match(indexName) { + continue + } + state := "" + health := "" + shardsNum := 0 + replicasNum := info.Replicas + if cfg := indexConfigMap[indexName]; cfg != nil { + sv, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, cfg) + hv, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, cfg) + state, _ = sv.(string) + health, _ = hv.(string) + if sn, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_shards"}, cfg); sn != nil { + shardsNum, _ = util.ToInt(util.ToString(sn)) + } + if rn, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_replicas"}, cfg); rn != nil { + replicasNum, _ = util.ToInt(util.ToString(rn)) + } + } + if state == "delete" { + health = "N/A" + } + unassigned := 0 + if shardsNum > 0 { + unassigned = (replicasNum+1)*shardsNum - info.Shards - info.Replicas + if unassigned < 0 { + unassigned = 0 + } + } + indices = append(indices, util.MapStr{ + "index": indexName, + "status": state, + "health": health, + "timestamp": info.Timestamp, + "docs_count": info.DocsCount, + "shards": info.Shards, + "replicas": replicasNum, + "unassigned_shards": unassigned, + "store_size": util.FormatBytes(float64(info.StoreInBytes), 1), + }) + } + return indices, nil } type ShardsSummary struct { @@ -1159,10 +1327,15 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, } query := util.MapStr{ - "size": 10000, - "_source": []string{"metadata.labels.index_name", "payload.elasticsearch.shard_stats.docs", "payload.elasticsearch.shard_stats.store", "payload.elasticsearch.shard_stats.routing", "timestamp"}, - "collapse": util.MapStr{ - "field": "metadata.labels.shard_id", + "size": 10000, + "_source": []string{ + "metadata.labels.index_name", + "metadata.labels.shard", + "metadata.labels.shard_id", + "payload.elasticsearch.shard_stats.docs", + "payload.elasticsearch.shard_stats.store", + "payload.elasticsearch.shard_stats.routing", + "timestamp", }, "sort": []util.MapStr{ { @@ -1210,35 +1383,50 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, }, } q := &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} + indexInfos := map[string]*ShardsSummary{} err, searchResult := orm.Search(event.Event{}, q) if err != nil { - return nil, err - } - indexInfos := map[string]*ShardsSummary{} - for _, hit := range searchResult.Result { - if hitM, ok := hit.(map[string]interface{}); ok { - shardDocCount, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, hitM) - storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, hitM) - indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) - primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, hitM) - if v, ok := indexName.(string); ok { - if _, ok = indexInfos[v]; !ok { - indexInfos[v] = &ShardsSummary{} - } - indexInfo := indexInfos[v] - indexInfo.Index = v - if count, ok := shardDocCount.(float64); ok && primary == true { - indexInfo.DocsCount += int64(count) - } - if storeSize, ok := storeInBytes.(float64); ok { - indexInfo.StoreInBytes += int64(storeSize) - } - if primary == true { - indexInfo.Shards++ - } else { - indexInfo.Replicas++ + log.Warnf("failed to enrich latest indices for cluster [%s], fallback to base index state only: %v", clusterID, err) + } else { + seenShards := map[string]struct{}{} + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + shardDocCount, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "docs", "count"}, hitM) + storeInBytes, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "store", "size_in_bytes"}, hitM) + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) + shardID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "shard_id"}, hitM) + shardNum, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "shard"}, hitM) + primary, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "shard_stats", "routing", "primary"}, hitM) + isPrimary, _ := parseBoolValue(primary) + if v, ok := indexName.(string); ok { + dedupeKey := util.ToString(shardID) + if dedupeKey == "" { + dedupeKey = fmt.Sprintf("%s:%s:%v", v, util.ToString(shardNum), primary) + } + if _, exists := seenShards[dedupeKey]; exists { + continue + } + seenShards[dedupeKey] = struct{}{} + if _, ok = indexInfos[v]; !ok { + indexInfos[v] = &ShardsSummary{} + } + indexInfo := indexInfos[v] + indexInfo.Index = v + if count, ok := parseInt64Value(shardDocCount); ok && isPrimary { + indexInfo.DocsCount += count + } + if storeSize, ok := parseInt64Value(storeInBytes); ok { + indexInfo.StoreInBytes += storeSize + } + if isPrimary { + indexInfo.Shards++ + } else { + indexInfo.Replicas++ + } + if indexInfo.Timestamp == nil { + indexInfo.Timestamp = hitM["timestamp"] + } } - indexInfo.Timestamp = hitM["timestamp"] } } } @@ -1256,10 +1444,6 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, if state == "delete" { health = "N/A" } - shards, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_shards"}, hitM) - replicas, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_replicas"}, hitM) - shardsNum, _ := util.ToInt(shards.(string)) - replicasNum, _ := util.ToInt(replicas.(string)) if v, ok := indexName.(string); ok { if indexPattern != nil { if !indexPattern.Match(v) { @@ -1267,6 +1451,10 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, } } if indexInfos[v] != nil { + shards, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_shards"}, hitM) + replicas, _ := util.GetMapValueByKeys([]string{"payload", "index_state", "settings", "index", "number_of_replicas"}, hitM) + shardsNum, _ := util.ToInt(util.ToString(shards)) + replicasNum, _ := util.ToInt(util.ToString(replicas)) indices = append(indices, util.MapStr{ "index": v, "status": state, @@ -1306,7 +1494,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps } clusterUUID, err := h.getClusterUUID(clusterID) if err != nil { - log.Error(err) + log.Errorf("failed to get cluster UUID for cluster [%s] when listing node shards for node [%s]: %v", clusterID, nodeID, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -1320,7 +1508,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps q1.AddSort("timestamp", orm.DESC) err, result := orm.Search(&event.Event{}, &q1) if err != nil { - log.Error(err) + log.Errorf("failed to search node shards for cluster [%s], node [%s]: %v", clusterID, nodeID, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -1328,7 +1516,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps if len(result.Result) > 0 { qps, err := h.getShardQPS(clusterID, nodeID, "", 20) if err != nil { - log.Error(err) + log.Errorf("failed to get shard QPS for cluster [%s], node [%s]: %v", clusterID, nodeID, err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -1338,7 +1526,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps source := util.MapStr(row) shardV, err := source.GetValue("payload.elasticsearch.shard_stats") if err != nil { - log.Error(err) + log.Errorf("failed to read shard stats from node shard result for cluster [%s], node [%s]: %v", clusterID, nodeID, err) continue } shardInfo := util.MapStr{} diff --git a/modules/elastic/api/proxy.go b/modules/elastic/api/proxy.go index 7add726a..c69a6552 100644 --- a/modules/elastic/api/proxy.go +++ b/modules/elastic/api/proxy.go @@ -56,7 +56,7 @@ func (h *APIHandler) HandleProxyAction(w http.ResponseWriter, req *http.Request, exists, esClient, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleProxyAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -64,7 +64,7 @@ func (h *APIHandler) HandleProxyAction(w http.ResponseWriter, req *http.Request, if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleProxyAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -73,7 +73,7 @@ func (h *APIHandler) HandleProxyAction(w http.ResponseWriter, req *http.Request, var realPath = authPath newURL, err := url.Parse(realPath) if err != nil { - log.Error(err) + log.Errorf("HandleProxyAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -126,7 +126,7 @@ func (h *APIHandler) HandleProxyAction(w http.ResponseWriter, req *http.Request, newReq.Method = method isSuperAdmin, permission, err := h.ValidateProxyRequest(newReq, targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleProxyAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusForbidden) return @@ -153,7 +153,7 @@ func (h *APIHandler) HandleProxyAction(w http.ResponseWriter, req *http.Request, metadata := elastic.GetMetadata(targetClusterID) if metadata == nil { resBody["error"] = fmt.Sprintf("cluster [%s] metadata not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleProxyAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -192,7 +192,7 @@ func (h *APIHandler) HandleProxyAction(w http.ResponseWriter, req *http.Request, } else { body, err := io.ReadAll(req.Body) if err != nil { - log.Error(err) + log.Errorf("HandleProxyAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/search.go b/modules/elastic/api/search.go index b9bc975b..83ccacc3 100644 --- a/modules/elastic/api/search.go +++ b/modules/elastic/api/search.go @@ -44,7 +44,7 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleCreateSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -52,7 +52,7 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleCreateSearchTemplateAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -61,7 +61,7 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req err = h.DecodeJSON(req, template) if err != nil { - log.Error(err) + log.Errorf("HandleCreateSearchTemplateAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -77,7 +77,7 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req //fmt.Println(client) err = client.SetSearchTemplate(template.Name, bodyBytes) if err != nil { - log.Error(err) + log.Errorf("HandleCreateSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -91,7 +91,7 @@ func (h *APIHandler) HandleCreateSearchTemplateAction(w http.ResponseWriter, req index := orm.GetIndexName(elastic.SearchTemplate{}) insertRes, err := esClient.Index(index, "", id, template, "wait_for") if err != nil { - log.Error(err) + log.Errorf("HandleCreateSearchTemplateAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -110,7 +110,7 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -118,7 +118,7 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -127,7 +127,7 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req err = h.DecodeJSON(req, template) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -137,14 +137,14 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req index := orm.GetIndexName(elastic.SearchTemplate{}) getRes, err := esClient.Get(index, "", templateID) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return } if getRes.Found == false { resBody["error"] = fmt.Sprintf("template %s can not be found", templateID) - log.Error(resBody["error"]) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -157,7 +157,7 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req if template.Name != "" && template.Name != targetName { err = client.DeleteSearchTemplate(targetName) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -178,7 +178,7 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req err = client.SetSearchTemplate(targetName, bodyBytes) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -187,7 +187,7 @@ func (h *APIHandler) HandleUpdateSearchTemplateAction(w http.ResponseWriter, req targetTemplate["updated"] = time.Now() insertRes, err := esClient.Index(index, "", templateID, targetTemplate, "wait_for") if err != nil { - log.Error(err) + log.Errorf("HandleUpdateSearchTemplateAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -213,7 +213,7 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req targetClusterID := ps.ByName("id") exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -221,7 +221,7 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleDeleteSearchTemplateAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -232,7 +232,7 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) res, err := esClient.Get(index, "", templateID) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -240,14 +240,14 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req err = client.DeleteSearchTemplate(res.Source["name"].(string)) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return } delRes, err := esClient.Delete(index, "", res.ID) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteSearchTemplateAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -261,7 +261,7 @@ func (h *APIHandler) HandleDeleteSearchTemplateAction(w http.ResponseWriter, req } _, err = esClient.Index(orm.GetIndexName(ht), "", util.GetUUID(), ht, "wait_for") if err != nil { - log.Error(err) + log.Errorf("HandleDeleteSearchTemplateAction failed: %v", err) } resBody["_id"] = templateID @@ -292,7 +292,7 @@ func (h *APIHandler) HandleSearchSearchTemplateAction(w http.ResponseWriter, req res, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.SearchTemplate{}), []byte(queryDSL)) if err != nil { - log.Error(err) + log.Errorf("HandleSearchSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -308,7 +308,7 @@ func (h *APIHandler) HandleGetSearchTemplateAction(w http.ResponseWriter, req *h indexName := orm.GetIndexName(elastic.SearchTemplate{}) getResponse, err := h.Client().Get(indexName, "", id) if err != nil { - log.Error(err) + log.Errorf("HandleGetSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() if getResponse != nil { h.WriteJSON(w, resBody, getResponse.StatusCode) @@ -342,7 +342,7 @@ func (h *APIHandler) HandleSearchSearchTemplateHistoryAction(w http.ResponseWrit res, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.SearchTemplateHistory{}), []byte(queryDSL)) if err != nil { - log.Error(err) + log.Errorf("HandleSearchSearchTemplateHistoryAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -356,7 +356,7 @@ func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http targetClusterID := ps.ByName("id") exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleRenderTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -364,14 +364,14 @@ func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleRenderTemplateAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } reqBody := map[string]interface{}{} err = h.DecodeJSON(req, &reqBody) if err != nil { - log.Error(err) + log.Errorf("HandleRenderTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -379,7 +379,7 @@ func (h *APIHandler) HandleRenderTemplateAction(w http.ResponseWriter, req *http res, err := client.RenderTemplate(reqBody) if err != nil { - log.Error(err) + log.Errorf("HandleRenderTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -393,7 +393,7 @@ func (h *APIHandler) HandleSearchTemplateAction(w http.ResponseWriter, req *http targetClusterID := ps.ByName("id") exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -401,14 +401,14 @@ func (h *APIHandler) HandleSearchTemplateAction(w http.ResponseWriter, req *http if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleSearchTemplateAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } reqBody := map[string]interface{}{} err = h.DecodeJSON(req, &reqBody) if err != nil { - log.Error(err) + log.Errorf("HandleSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -416,7 +416,7 @@ func (h *APIHandler) HandleSearchTemplateAction(w http.ResponseWriter, req *http res, err := client.SearchTemplate(reqBody) if err != nil { - log.Error(err) + log.Errorf("HandleSearchTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return diff --git a/modules/elastic/api/setting.go b/modules/elastic/api/setting.go index 941b84f4..10dcb8ac 100644 --- a/modules/elastic/api/setting.go +++ b/modules/elastic/api/setting.go @@ -47,7 +47,7 @@ func (h *APIHandler) HandleSettingAction(w http.ResponseWriter, req *http.Reques err := h.DecodeJSON(req, &reqParams) if err != nil { - log.Error(err) + log.Errorf("HandleSettingAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -63,7 +63,7 @@ func (h *APIHandler) HandleSettingAction(w http.ResponseWriter, req *http.Reques } if err != nil { - log.Error(err) + log.Errorf("HandleSettingAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -83,7 +83,7 @@ func (h *APIHandler) HandleGetSettingAction(w http.ResponseWriter, req *http.Req searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.Setting{}), []byte(queryDSL)) if err != nil { - log.Error(err) + log.Errorf("HandleGetSettingAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return diff --git a/modules/elastic/api/shard.go b/modules/elastic/api/shard.go index 54784fc0..40762290 100644 --- a/modules/elastic/api/shard.go +++ b/modules/elastic/api/shard.go @@ -41,7 +41,7 @@ func (h *APIHandler) GetShardInfo(w http.ResponseWriter, req *http.Request, ps h shardID := ps.MustGetParameter("shard_id") clusterUUID, err := adapter.GetClusterUUID(clusterID) if err != nil { - log.Error(err) + log.Errorf("GetShardInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -57,7 +57,7 @@ func (h *APIHandler) GetShardInfo(w http.ResponseWriter, req *http.Request, ps h err, res := orm.Search(&event.Event{}, &q) if err != nil { - log.Error(err) + log.Errorf("GetShardInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/template.go b/modules/elastic/api/template.go index eb6177ba..10de16f2 100644 --- a/modules/elastic/api/template.go +++ b/modules/elastic/api/template.go @@ -41,7 +41,7 @@ func (h *APIHandler) HandleGetTemplateAction(w http.ResponseWriter, req *http.Re esClient := elastic.GetClient(clusterID) templates, err := esClient.GetTemplate("") if err != nil { - log.Error(err) + log.Errorf("HandleGetTemplateAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -54,13 +54,13 @@ func (h *APIHandler) HandleSaveTemplateAction(w http.ResponseWriter, req *http.R esClient := elastic.GetClient(clusterID) reqBody, err := io.ReadAll(req.Body) if err != nil { - log.Error(err) + log.Errorf("HandleSaveTemplateAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } esResBody, err := esClient.PutTemplate(templateName, reqBody) if err != nil { - log.Error(err) + log.Errorf("HandleSaveTemplateAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/test_connection.go b/modules/elastic/api/test_connection.go index d37bc02c..31c2f679 100644 --- a/modules/elastic/api/test_connection.go +++ b/modules/elastic/api/test_connection.go @@ -28,8 +28,10 @@ package api import ( + "context" "fmt" "github.com/segmentio/encoding/json" + console_common "infini.sh/console/common" "infini.sh/console/core" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" @@ -47,33 +49,54 @@ type TestAPI struct { core.Handler } +const ( + tryConnectErrorKeyHealthRed = "cluster.connect.error.health_red" + tryConnectErrorKeyNonESEndpoint = "cluster.connect.error.non_es_endpoint" + tryConnectErrorKeyTLSMismatch = "cluster.connect.error.tls_mismatch" + tryConnectErrorKeyAuthRequired = "cluster.connect.error.auth_required" + tryConnectErrorKeyEndpointUnreachable = "cluster.connect.error.endpoint_unreachable" + tryConnectErrorKeyUnexpectedStatus = "cluster.connect.error.unexpected_status" + tryConnectErrorKeyDefault = "cluster.regist.try_connect.failed" +) + +type elasticConfigPayload struct { + elastic.ElasticsearchConfig + ProbePath string `json:"probe_path,omitempty"` + AuthEnabled *bool `json:"is_auth,omitempty"` + RejectRed *bool `json:"reject_red,omitempty"` +} + var testAPI = TestAPI{} var testInited bool +func RegisterPublicUITestAPI() { + api.HandleUIMethod( + api.POST, + "/elasticsearch/try_connect", + testAPI.RequireSecureTransport(testAPI.HandleTestConnectionAction), + api.AllowPublicAccess(), + ) +} + func InitTestAPI() { if !testInited { - api.HandleAPIMethod(api.POST, "/elasticsearch/try_connect", testAPI.HandleTestConnectionAction) + api.HandleAPIMethod(api.POST, "/elasticsearch/try_connect", testAPI.RequireSecureTransport(testAPI.HandleTestConnectionAction)) testInited = true } } func (h TestAPI) HandleTestConnectionAction(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ( - freq = httpPool.AcquireRequest() - fres = httpPool.AcquireResponse() - resBody = map[string]interface{}{} - ) - defer func() { - httpPool.ReleaseRequest(freq) - httpPool.ReleaseResponse(fres) - }() - var config = &elastic.ElasticsearchConfig{} - err := h.DecodeJSON(req, &config) + var resBody = map[string]interface{}{} + payload := &elasticConfigPayload{} + err := h.DecodeJSON(req, payload) if err != nil { panic(err) } defer req.Body.Close() + config := &payload.ElasticsearchConfig + console_common.SetProbePath(config, payload.ProbePath) + applyExplicitPlatformAuthPreference(config, payload.AuthEnabled) var url string if config.Endpoint != "" { url = config.Endpoint @@ -123,38 +146,20 @@ func (h TestAPI) HandleTestConnectionAction(w http.ResponseWriter, req *http.Req clusterUUID string ) for i, url = range config.Endpoints { - if !util.SuffixStr(url, "/") { - url = fmt.Sprintf("%s/", url) - } - - freq.SetRequestURI(url) - freq.Header.SetMethod("GET") - - if config.BasicAuth != nil && strings.TrimSpace(config.BasicAuth.Username) != "" { - freq.SetBasicAuth(config.BasicAuth.Username, config.BasicAuth.Password.Get()) - } - - const testClientName = "elasticsearch_test_connection" - err = api.GetFastHttpClient(testClientName).DoTimeout(freq, fres, 10*time.Second) - + clusterInfo, err := console_common.ClusterVersionWithConfig(&elastic.ElasticsearchConfig{ + Schema: config.Schema, + Endpoint: url, + Endpoints: []string{url}, + Distribution: config.Distribution, + BasicAuth: config.BasicAuth, + RequestTimeout: 10, + Labels: config.Labels, + }) if err != nil { - panic(err) - } - - var statusCode = fres.StatusCode() - if statusCode > 300 || statusCode == 0 { - resBody["error"] = fmt.Sprintf("invalid status code: %d", statusCode) - h.WriteJSON(w, resBody, 500) + writeTryConnectError(h, w, err) return } - b := fres.Body() - clusterInfo := &elastic.ClusterInformation{} - err = json.Unmarshal(b, clusterInfo) - if err != nil { - panic(err) - } - resBody["version"] = clusterInfo.Version.Number resBody["cluster_uuid"] = clusterInfo.ClusterUUID resBody["cluster_name"] = clusterInfo.ClusterName @@ -173,19 +178,9 @@ func (h TestAPI) HandleTestConnectionAction(w http.ResponseWriter, req *http.Req break } //fetch cluster health info - freq.SetRequestURI(fmt.Sprintf("%s/_cluster/health", url)) - fres.Reset() - err = api.GetFastHttpClient(testClientName).Do(freq, fres) - if err != nil { - resBody["error"] = fmt.Sprintf("error on get cluster health: %v", err) - h.WriteJSON(w, resBody, http.StatusInternalServerError) - return - } - - healthInfo := &elastic.ClusterHealth{} - err = json.Unmarshal(fres.Body(), &healthInfo) + healthInfo, err := fetchClusterHealth(url, config) if err != nil { - resBody["error"] = fmt.Sprintf("error on decode cluster health info : %v", err) + resBody["error"] = buildTryConnectErrorPayload(err) h.WriteJSON(w, resBody, http.StatusInternalServerError) return } @@ -194,16 +189,142 @@ func (h TestAPI) HandleTestConnectionAction(w http.ResponseWriter, req *http.Req resBody["number_of_data_nodes"] = healthInfo.NumberOf_data_nodes resBody["active_shards"] = healthInfo.ActiveShards - if healthInfo.Status == "red" { - resBody["error"] = "cluster health status is red, please fix the cluster before connecting" + if payload.RejectRed != nil && *payload.RejectRed && healthInfo.Status == "red" { + resBody["error"] = buildTryConnectErrorPayload(errors.New("cluster health status is red, please fix the cluster before connecting")) h.WriteJSON(w, resBody, http.StatusInternalServerError) return } - - freq.Reset() - fres.Reset() } h.WriteJSON(w, resBody, http.StatusOK) } + +func writeTryConnectError(h TestAPI, w http.ResponseWriter, err error) { + h.WriteJSON(w, map[string]interface{}{ + "error": buildTryConnectErrorPayload(err), + }, http.StatusInternalServerError) +} + +func buildTryConnectErrorPayload(err error) map[string]interface{} { + reason, key := resolveTryConnectError(err) + return map[string]interface{}{ + "reason": reason, + "key": key, + } +} + +func sanitizeTryConnectError(err error) string { + reason, _ := resolveTryConnectError(err) + return reason +} + +func sanitizeTryConnectErrorKey(err error) string { + _, key := resolveTryConnectError(err) + return key +} + +func resolveTryConnectError(err error) (string, string) { + raw := extractTryConnectReason(err) + lowerRaw := strings.ToLower(raw) + + switch { + case strings.Contains(lowerRaw, "cluster health status is red"): + return "cluster health status is red, please fix the cluster before connecting", tryConnectErrorKeyHealthRed + case strings.Contains(lowerRaw, "invalid character '<' looking for beginning of value"), + strings.Contains(lowerRaw, ""), + strings.Contains(lowerRaw, " 300 || res.StatusCode == 0 { + return nil, errors.New(fmt.Sprintf("invalid status code: %d", res.StatusCode)) + } + + healthInfo := &elastic.ClusterHealth{} + err = json.Unmarshal(res.Body, healthInfo) + if err != nil { + return nil, err + } + return healthInfo, nil +} diff --git a/modules/elastic/api/test_connection_test.go b/modules/elastic/api/test_connection_test.go new file mode 100644 index 00000000..5316c0b9 --- /dev/null +++ b/modules/elastic/api/test_connection_test.go @@ -0,0 +1,98 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package api + +import ( + "errors" + "testing" +) + +func TestSanitizeTryConnectError(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + { + name: "connection refused", + err: errors.New(`Get "https://127.0.0.1:9200": dial tcp 127.0.0.1:9200: connect: connection refused`), + want: "unable to connect to the cluster endpoint, please check the address, network accessibility, and TLS setting", + }, + { + name: "tls mismatch", + err: errors.New(`Get "https://127.0.0.1:9200": http: server gave HTTP response to HTTPS client`), + want: "TLS setting does not match the cluster endpoint, please check whether HTTPS is enabled", + }, + { + name: "security exception json", + err: errors.New(`{"error":{"root_cause":[{"type":"security_exception","reason":"Missing authentication information for REST request [/]"}],"type":"security_exception","reason":"Missing authentication information for REST request [/]"},"status":401}`), + want: "authentication is required or invalid, please check the credential", + }, + { + name: "html response", + err: errors.New(`json: invalid character '<' looking for beginning of value: `), + want: "the endpoint did not return an Elasticsearch-compatible API response, please check the address and port", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sanitizeTryConnectError(tt.err); got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} + +func TestSanitizeTryConnectErrorKey(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + { + name: "connection refused", + err: errors.New(`Get "https://127.0.0.1:9200": dial tcp 127.0.0.1:9200: connect: connection refused`), + want: tryConnectErrorKeyEndpointUnreachable, + }, + { + name: "tls mismatch", + err: errors.New(`Get "https://127.0.0.1:9200": http: server gave HTTP response to HTTPS client`), + want: tryConnectErrorKeyTLSMismatch, + }, + { + name: "html response", + err: errors.New(`json: invalid character '<' looking for beginning of value: `), + want: tryConnectErrorKeyNonESEndpoint, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sanitizeTryConnectErrorKey(tt.err); got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} diff --git a/modules/elastic/api/threadpool_metrics.go b/modules/elastic/api/threadpool_metrics.go index e2fe84a4..0f4f6dbf 100644 --- a/modules/elastic/api/threadpool_metrics.go +++ b/modules/elastic/api/threadpool_metrics.go @@ -111,7 +111,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string, } else { nodeNames, err = h.getTopNodeName(clusterID, top, 15) if err != nil { - log.Error(err) + log.Errorf("getThreadPoolMetrics failed: %v", err) } } if len(nodeNames) > 0 { @@ -622,7 +622,7 @@ func (h *APIHandler) getThreadPoolMetrics(ctx context.Context, clusterID string, } intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) if err != nil { - log.Error(err) + log.Errorf("getThreadPoolMetrics failed: %v", err) panic(err) } diff --git a/modules/elastic/api/trace_template.go b/modules/elastic/api/trace_template.go index 54c79993..47f25af3 100644 --- a/modules/elastic/api/trace_template.go +++ b/modules/elastic/api/trace_template.go @@ -43,7 +43,7 @@ func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req * exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleCrateTraceTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -51,7 +51,7 @@ func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req * if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleCrateTraceTemplateAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -60,7 +60,7 @@ func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req * err = h.DecodeJSON(req, traceReq) if err != nil { - log.Error(err) + log.Errorf("HandleCrateTraceTemplateAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -72,7 +72,7 @@ func (h *APIHandler) HandleCrateTraceTemplateAction(w http.ResponseWriter, req * var id = util.GetUUID() insertRes, err := client.Index(orm.GetIndexName(elastic.TraceTemplate{}), "", id, traceReq, "wait_for") if err != nil { - log.Error(err) + log.Errorf("HandleCrateTraceTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -112,7 +112,7 @@ func (h *APIHandler) HandleSearchTraceTemplateAction(w http.ResponseWriter, req res, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.TraceTemplate{}), []byte(queryDSL)) if err != nil { - log.Error(err) + log.Errorf("HandleSearchTraceTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -127,7 +127,7 @@ func (h *APIHandler) HandleSaveTraceTemplateAction(w http.ResponseWriter, req *h reqParams := elastic.TraceTemplate{} err := h.DecodeJSON(req, &reqParams) if err != nil { - log.Error(err) + log.Errorf("HandleSaveTraceTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -137,7 +137,7 @@ func (h *APIHandler) HandleSaveTraceTemplateAction(w http.ResponseWriter, req *h esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) _, err = esClient.Index(orm.GetIndexName(reqParams), "", reqParams.ID, reqParams, "wait_for") if err != nil { - log.Error(err) + log.Errorf("HandleSaveTraceTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusOK) return @@ -157,7 +157,7 @@ func (h *APIHandler) HandleGetTraceTemplateAction(w http.ResponseWriter, req *ht indexName := orm.GetIndexName(elastic.TraceTemplate{}) getResponse, err := h.Client().Get(indexName, "", id) if err != nil { - log.Error(err) + log.Errorf("HandleGetTraceTemplateAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) } @@ -170,7 +170,7 @@ func (h *APIHandler) HandleDeleteTraceTemplateAction(w http.ResponseWriter, req esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) delRes, err := esClient.Delete(orm.GetIndexName(elastic.TraceTemplate{}), "", id, "wait_for") if err != nil { - log.Error(err) + log.Errorf("HandleDeleteTraceTemplateAction failed: %v", err) resBody["error"] = err.Error() if delRes != nil { h.WriteJSON(w, resBody, delRes.StatusCode) diff --git a/modules/elastic/api/v1/cluster_overview.go b/modules/elastic/api/v1/cluster_overview.go index 53955259..3fffd90e 100644 --- a/modules/elastic/api/v1/cluster_overview.go +++ b/modules/elastic/api/v1/cluster_overview.go @@ -431,9 +431,6 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p } query := util.MapStr{ "size": 1000, - "collapse": util.MapStr{ - "field": "metadata.labels.node_id", - }, "sort": []util.MapStr{ { "timestamp": util.MapStr{ @@ -486,8 +483,18 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p h.WriteJSON(w, resBody, http.StatusInternalServerError) } nodeInfos := map[string]util.MapStr{} + seenNodeInfos := map[string]struct{}{} for _, hit := range searchResult.Result { if hitM, ok := hit.(map[string]interface{}); ok { + nodeID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, hitM) + nodeIDStr := util.ToString(nodeID) + if nodeIDStr == "" { + continue + } + if _, exists := seenNodeInfos[nodeIDStr]; exists { + continue + } + seenNodeInfos[nodeIDStr] = struct{}{} shardInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info"}, hitM) var totalShards float64 if v, ok := shardInfo.(map[string]interface{}); ok { @@ -505,22 +512,18 @@ func (h *APIHandler) GetClusterNodes(w http.ResponseWriter, req *http.Request, p load, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "os", "cpu", "load_average", "1m"}, hitM) heapUsage, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "mem", "heap_used_percent"}, hitM) freeDisk, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "fs", "total", "free_in_bytes"}, hitM) - nodeID, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, hitM) if v, ok := freeDisk.(float64); ok { freeDisk = util.ByteSize(uint64(v)) } - if v, ok := nodeID.(string); ok { - nodeInfos[v] = util.MapStr{ - "timestamp": hitM["timestamp"], - "shards": totalShards, - "cpu": cpu, - "load_1m": load, - "heap.percent": heapUsage, - "disk.avail": freeDisk, - "uptime": uptime, - } - + nodeInfos[nodeIDStr] = util.MapStr{ + "timestamp": hitM["timestamp"], + "shards": totalShards, + "cpu": cpu, + "load_1m": load, + "heap.percent": heapUsage, + "disk.avail": freeDisk, + "uptime": uptime, } } } @@ -973,7 +976,7 @@ func (h *APIHandler) SearchClusterMetadata(w http.ResponseWriter, req *http.Requ } } - clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "id") if !hasAllPrivilege && clusterFilter == nil { h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) return diff --git a/modules/elastic/api/v1/index_metrics.go b/modules/elastic/api/v1/index_metrics.go index b4728873..03ea769e 100644 --- a/modules/elastic/api/v1/index_metrics.go +++ b/modules/elastic/api/v1/index_metrics.go @@ -130,9 +130,9 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu top = len(indexNames) } else { - indexNames, err = h.getTopIndexName(req, clusterID, top, min, max) + indexNames, err = h.getTopIndexName(req, clusterID, top, min, max, bucketSize) if err != nil { - log.Error(err) + log.Errorf("getIndexMetrics failed: %v", err) } } @@ -747,7 +747,7 @@ func (h *APIHandler) getIndexMetrics(ctx context.Context, req *http.Request, clu } -func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, min, max int64) ([]string, error) { +func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top int, min, max int64, bucketSize int) ([]string, error) { ver := h.Client().GetVersion() cr, _ := util.VersionCompare(ver.Number, "6.1") if (ver.Distribution == "" || ver.Distribution == elastic.Elasticsearch) && cr == -1 { @@ -789,12 +789,6 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in }, }) } - bucketSizeStr := "60s" - intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) - if err != nil { - return nil, err - } - partition_num := 10 indexCount := GetIndicesCount(clusterID) if indexCount < 40 { @@ -802,6 +796,13 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in } else { partition_num = indexCount / 20 } + estimatedIndexBuckets := estimateTopIndexBuckets(indexCount, partition_num) + bucketSize = normalizeTopIndexBucketSize(bucketSize, min, max, estimatedIndexBuckets) + bucketSizeStr := fmt.Sprintf("%ds", bucketSize) + intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) + if err != nil { + return nil, err + } term_index := util.MapStr{ "field": "metadata.labels.index_name", @@ -914,7 +915,7 @@ func (h *APIHandler) getTopIndexName(req *http.Request, clusterID string, top in } response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) if err != nil { - log.Error(err) + log.Errorf("getTopIndexName failed: %v", err) return nil, err } var maxQpsKVS = map[string]float64{} @@ -967,3 +968,55 @@ func (t TopTermOrder) Less(i, j int) bool { func (t TopTermOrder) Swap(i, j int) { t[i], t[j] = t[j], t[i] } + +func normalizeTopIndexBucketSize(bucketSize int, min, max int64, indexBuckets int) int { + if bucketSize <= 0 { + bucketSize = 60 + } + if max <= min { + return bucketSize + } + if indexBuckets <= 0 { + indexBuckets = 1 + } + const ( + esMaxBuckets = int64(65535) + aggGroupCount = int64(2) // group_by_index + group_by_index1 + estShardBuckets = int64(2) // estimated shard terms fan-out + basePerDateBucket = int64(1) + ) + durationSeconds := (max - min) / 1000 + if durationSeconds <= 0 { + return bucketSize + } + perDateCost := aggGroupCount * int64(indexBuckets) * (basePerDateBucket + estShardBuckets) + if perDateCost <= 0 { + perDateCost = 1 + } + maxDateBuckets := esMaxBuckets / perDateCost + if maxDateBuckets < 1 { + maxDateBuckets = 1 + } + minRequired := int((durationSeconds + maxDateBuckets - 1) / maxDateBuckets) + if minRequired > bucketSize { + bucketSize = minRequired + } + return bucketSize +} + +func estimateTopIndexBuckets(indexCount, partitionNum int) int { + if indexCount <= 0 { + return 1 + } + if partitionNum <= 0 { + partitionNum = 1 + } + estimated := (indexCount + partitionNum - 1) / partitionNum + if estimated < 1 { + estimated = 1 + } + if estimated > 10000 { + estimated = 10000 + } + return estimated +} diff --git a/modules/elastic/api/v1/index_overview.go b/modules/elastic/api/v1/index_overview.go index b33229c5..5adfd309 100644 --- a/modules/elastic/api/v1/index_overview.go +++ b/modules/elastic/api/v1/index_overview.go @@ -52,17 +52,15 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context, h.WriteJSON(w, util.MapStr{}, http.StatusOK) return } - //only query first index - indexIDs = indexIDs[0:1] q1 := orm.Query{WildcardIndex: true} q1.Conds = orm.And( orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.name", "index_stats"), - orm.Eq("metadata.labels.index_id", indexIDs[0]), + orm.In("metadata.labels.index_id", indexIDs), ) - //q1.Collapse("metadata.labels.index_id") + q1.Collapse("metadata.labels.index_id") q1.AddSort("timestamp", orm.DESC) - q1.Size = 1 + q1.Size = len(indexIDs) err, results := orm.Search(&event.Event{}, &q1) if err != nil { @@ -79,7 +77,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context, summary := map[string]interface{}{} if docs, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "total", "docs"}, result); ok { if docsM, ok := docs.(map[string]interface{}); ok { - summary["docs_deleted"] = docsM["docs_deleted"] + summary["docs_deleted"] = docsM["deleted"] summary["docs_count"] = docsM["count"] } } @@ -130,7 +128,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context, } statusMetric, err := h.GetIndexStatusOfRecentDay(firstClusterID, firstIndexName) if err != nil { - log.Error(err) + log.Errorf("FetchIndexInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -241,7 +239,7 @@ func (h *APIHandler) FetchIndexInfo(w http.ResponseWriter, ctx context.Context, } metrics, err := h.getMetrics(ctx, query, nodeMetricItems, bucketSize) if err != nil { - log.Error(err) + log.Errorf("FetchIndexInfo failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -431,6 +429,35 @@ func (h *APIHandler) GetIndexShards(w http.ResponseWriter, req *http.Request, ps const IndexHealthMetricKey = "index_health" +func normalizeIndexHealthBucketSize(bucketSize int, min, max int64) int { + if bucketSize <= 0 { + bucketSize = 60 + } + durationSeconds := (max - min) / 1000 + if durationSeconds <= 0 { + return bucketSize + } + const ( + maxBuckets = int64(65535) + minBuckets = int64(24) + ) + minIntervalForMaxBuckets := int((durationSeconds + maxBuckets - 1) / maxBuckets) + if minIntervalForMaxBuckets < 1 { + minIntervalForMaxBuckets = 1 + } + maxIntervalForMinBuckets := int(durationSeconds / minBuckets) + if bucketSize < minIntervalForMaxBuckets { + bucketSize = minIntervalForMaxBuckets + } + if maxIntervalForMinBuckets >= 1 && bucketSize > maxIntervalForMinBuckets { + bucketSize = maxIntervalForMinBuckets + } + if bucketSize < 1 { + bucketSize = 1 + } + return bucketSize +} + func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { clusterID := ps.MustGetParameter("id") indexName := ps.MustGetParameter("index") @@ -473,7 +500,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ resBody := map[string]interface{}{} bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, MetricTypeIndexStats, 60) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -501,7 +528,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -575,7 +602,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ } metrics, err = h.getSingleMetrics(ctx, metricItems, query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -584,7 +611,7 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ if metricItem.HitsTotal > 0 && metricItem.MinBucketSize == 0 { minBucketSize, err := GetMetricMinBucketSize(clusterID, MetricTypeIndexStats) if err != nil { - log.Error(err) + log.Errorf("GetSingleIndexMetrics failed: %v", err) } else { metricItem.MinBucketSize = int64(minBucketSize) } @@ -595,7 +622,8 @@ func (h *APIHandler) GetSingleIndexMetrics(w http.ResponseWriter, req *http.Requ } func (h *APIHandler) GetIndexHealthMetric(ctx context.Context, id, indexName string, min, max int64, bucketSize int) (*common.MetricItem, error) { - bucketSizeStr := fmt.Sprintf("%vs", bucketSize) + healthBucketSize := normalizeIndexHealthBucketSize(bucketSize, min, max) + bucketSizeStr := fmt.Sprintf("%vs", healthBucketSize) intervalField, err := getDateHistogramIntervalField(global.MustLookupString(elastic.GlobalSystemElasticsearchID), bucketSizeStr) if err != nil { return nil, err @@ -668,7 +696,7 @@ func (h *APIHandler) GetIndexHealthMetric(ctx context.Context, id, indexName str if errors.Is(err, context.DeadlineExceeded) { return nil, cerr.New(cerr.ErrTypeRequestTimeout, "", err) } - log.Error(err) + log.Errorf("GetIndexHealthMetric failed: %v", err) return nil, err } @@ -757,6 +785,7 @@ func (h *APIHandler) GetIndexStatusOfRecentDay(clusterID, indexName string) (map "term_health": util.MapStr{ "terms": util.MapStr{ "field": "payload.elasticsearch.index_health.status", + "size": 10, }, }, }, diff --git a/modules/elastic/api/v1/manage.go b/modules/elastic/api/v1/manage.go index ce2eb654..9bb5c787 100644 --- a/modules/elastic/api/v1/manage.go +++ b/modules/elastic/api/v1/manage.go @@ -34,8 +34,10 @@ import ( "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" "infini.sh/framework/core/orm" "infini.sh/framework/core/util" + elasticmodule "infini.sh/framework/modules/elastic" "infini.sh/framework/modules/elastic/common" "math" "net/http" @@ -58,12 +60,13 @@ func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http. var conf = &elastic.ElasticsearchConfig{} err := h.DecodeJSON(req, conf) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } // TODO validate data format conf.Enabled = true + conf.Monitored = true conf.Host = strings.TrimSpace(conf.Host) conf.Endpoint = fmt.Sprintf("%s://%s", conf.Schema, conf.Host) conf.ID = util.GetUUID() @@ -73,7 +76,7 @@ func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http. if conf.CredentialID == "" && conf.BasicAuth != nil && conf.BasicAuth.Username != "" { credentialID, err := saveBasicAuthToCredential(conf) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -85,13 +88,13 @@ func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http. } err = orm.Create(ctx, conf) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } basicAuth, err := common.GetBasicAuth(conf) if err != nil { - log.Error(err) + log.Errorf("HandleCreateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -100,6 +103,8 @@ func (h *APIHandler) HandleCreateClusterAction(w http.ResponseWriter, req *http. _, err = common.InitElasticInstance(*conf) if err != nil { log.Warn("error on init elasticsearch:", err) + } else { + elasticmodule.SyncClusterHealthStatus(conf.ID) } h.WriteCreatedOKJSON(w, conf.ID) @@ -125,7 +130,7 @@ func saveBasicAuthToCredential(conf *elastic.ElasticsearchConfig) (string, error if err != nil { return "", err } - err = orm.Create(nil, &cred) + err = orm.Create(&orm.Context{Refresh: "wait_for"}, &cred) if err != nil { return "", err } @@ -136,9 +141,9 @@ func (h *APIHandler) HandleGetClusterAction(w http.ResponseWriter, req *http.Req id := ps.MustGetParameter("id") clusterConf := elastic.ElasticsearchConfig{} clusterConf.ID = id - exists, err := orm.Get(&clusterConf) + exists, err := orm.GetV2(orm.NewContext(), &clusterConf) if err != nil || !exists { - log.Error(err) + log.Errorf("HandleGetClusterAction failed: %v", err) h.Error404(w) return } @@ -149,16 +154,16 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. var conf = map[string]interface{}{} err := h.DecodeJSON(req, &conf) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } id := ps.MustGetParameter("id") originConf := elastic.ElasticsearchConfig{} originConf.ID = id - exists, err := orm.Get(&originConf) + exists, err := orm.GetV2(orm.NewContext(), &originConf) if err != nil || !exists { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.Error404(w) return } @@ -201,7 +206,7 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. if newConf.BasicAuth != nil && newConf.BasicAuth.Username != "" { credentialID, err := saveBasicAuthToCredential(newConf) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -213,19 +218,21 @@ func (h *APIHandler) HandleUpdateClusterAction(w http.ResponseWriter, req *http. } err = orm.Save(ctx, newConf) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } basicAuth, err := common.GetBasicAuth(newConf) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } newConf.BasicAuth = basicAuth + elastic.RemoveHostsByClusterID(id) + //update config in heap newConf.Source = elastic.ElasticsearchConfigSourceElasticsearch _, err = common.InitElasticInstance(*newConf) @@ -242,9 +249,9 @@ func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http. esConfig := elastic.ElasticsearchConfig{} esConfig.ID = id - ok, err := orm.Get(&esConfig) + ok, err := orm.GetV2(orm.NewContext(), &esConfig) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -262,7 +269,7 @@ func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http. err = orm.Delete(ctx, &esConfig) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -275,15 +282,27 @@ func (h *APIHandler) HandleDeleteClusterAction(w http.ResponseWriter, req *http. } err = orm.DeleteBy(elastic.NodeConfig{}, util.MustToJSONBytes(delDsl)) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) } err = orm.DeleteBy(elastic.IndexConfig{}, util.MustToJSONBytes(delDsl)) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteClusterAction failed: %v", err) } elastic.RemoveInstance(id) elastic.RemoveHostsByClusterID(id) + err = kv.DeleteKey(elastic.KVElasticNodeMetadata, []byte(id)) + if err != nil { + log.Errorf("failed to delete node metadata for cluster [%s]: %v", id, err) + } + err = kv.DeleteKey(elastic.KVElasticIndexMetadata, []byte(id)) + if err != nil { + log.Errorf("failed to delete index metadata for cluster [%s]: %v", id, err) + } + err = kv.DeleteKey(elastic.KVElasticClusterSettings, []byte(id)) + if err != nil { + log.Errorf("failed to delete cluster settings metadata for cluster [%s]: %v", id, err) + } h.WriteDeletedOKJSON(w, id) } @@ -292,7 +311,7 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. name = h.GetParameterOrDefault(req, "name", "") sortField = h.GetParameterOrDefault(req, "sort_field", "") sortOrder = h.GetParameterOrDefault(req, "sort_order", "") - queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d%s}` + queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d%s%s}` strSize = h.GetParameterOrDefault(req, "size", "20") strFrom = h.GetParameterOrDefault(req, "from", "0") mustBuilder = &strings.Builder{} @@ -300,7 +319,7 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. if name != "" { mustBuilder.WriteString(fmt.Sprintf(`{"prefix":{"name.text": "%s"}}`, name)) } - clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "_id") + clusterFilter, hasAllPrivilege := h.GetClusterFilter(req, "id") if !hasAllPrivilege && clusterFilter == nil { h.WriteJSON(w, elastic.SearchResponse{}, http.StatusOK) return @@ -311,6 +330,9 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. } mustBuilder.Write(util.MustToJSONBytes(clusterFilter)) } + if mustBuilder.Len() == 0 { + mustBuilder.WriteString(`{"match_all":{}}`) + } size, _ := strconv.Atoi(strSize) if size <= 0 { @@ -320,23 +342,38 @@ func (h *APIHandler) HandleSearchClusterAction(w http.ResponseWriter, req *http. if from < 0 { from = 0 } - var sort = "" + var ( + functions = "" + sort = "" + trackScore = "" + ) if sortField != "" && sortOrder != "" { sort = fmt.Sprintf(`,"sort":[{"%s":{"order":"%s"}}]`, sortField, sortOrder) + } else { + functions = `,"functions":[{"filter":{"term":{"labels.health_status":"red"}},"weight":300},{"filter":{"term":{"labels.health_status":"yellow"}},"weight":200},{"filter":{"term":{"labels.health_status":"unavailable"}},"weight":100},{"filter":{"term":{"labels.health_status":"green"}},"weight":1}]` + sort = `,"sort":[{"_score":{"order":"desc"}},{"name.keyword":{"order":"asc","unmapped_type":"keyword"}}]` + trackScore = `,"track_scores":true` + queryDSL = `{"query":{"function_score":{"query":{"bool":{"must":[%s]}}%s,"score_mode":"sum","boost_mode":"replace"}}, "size": %d, "from": %d%s%s}` } - queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), size, from, sort) + queryDSL = fmt.Sprintf(queryDSL, mustBuilder.String(), functions, size, from, sort, trackScore) q := orm.Query{ RawQuery: []byte(queryDSL), } err, result := orm.Search(elastic.ElasticsearchConfig{}, &q) if err != nil { - log.Error(err) + if global.Env().IsDebug { + log.Errorf("cluster search failed, name=%q, from=%d, size=%d, dsl=%s, err=%v", name, from, size, queryDSL, err) + } + log.Errorf("HandleSearchClusterAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } searchRes := elastic.SearchResponse{} util.MustFromJSONBytes(result.Raw, &searchRes) + if global.Env().IsDebug && len(searchRes.Hits.Hits) == 0 { + log.Debugf("cluster search returned zero hits, name=%q, from=%d, size=%d, dsl=%s", name, from, size, queryDSL) + } if len(searchRes.Hits.Hits) > 0 { for _, hit := range searchRes.Hits.Hits { if basicAuth, ok := hit.Source["basic_auth"]; ok { @@ -397,7 +434,7 @@ func (h *APIHandler) HandleMetricsSummaryAction(w http.ResponseWriter, req *http err, result := orm.Search(event.Event{}, &q) if err != nil { resBody["error"] = err.Error() - log.Error("MetricsSummary search error: ", err) + log.Errorf("HandleMetricsSummaryAction metrics summary search failed: %v", err) h.WriteJSON(w, resBody, http.StatusInternalServerError) return } @@ -478,7 +515,7 @@ func (h *APIHandler) HandleMetricsSummaryAction(w http.ResponseWriter, req *http q.RawQuery = util.MustToJSONBytes(query) err, result = orm.Search(event.Event{}, &q) if err != nil { - log.Error("MetricsSummary search error: ", err) + log.Errorf("HandleMetricsSummaryAction metrics summary search failed: %v", err) } else { if len(result.Result) > 0 { if v, ok := result.Result[0].(map[string]interface{}); ok { @@ -525,7 +562,7 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -541,14 +578,14 @@ func (h *APIHandler) HandleClusterMetricsAction(w http.ResponseWriter, req *http if metrics[key].HitsTotal > 0 { minBucketSize, err := GetMetricMinBucketSize(id, metricType) if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) } else { metrics[key].MinBucketSize = int64(minBucketSize) } } } if err != nil { - log.Error(err) + log.Errorf("HandleClusterMetricsAction failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -564,7 +601,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R id := ps.ByName("id") bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, id, MetricTypeIndexStats, 90) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -575,7 +612,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R timeout := h.GetParameterOrDefault(req, "timeout", "60s") du, err := time.ParseDuration(timeout) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -653,7 +690,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R if metrics[key].HitsTotal > 0 { minBucketSize, err := GetMetricMinBucketSize(id, MetricTypeNodeStats) if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) } else { metrics[key].MinBucketSize = int64(minBucketSize) } @@ -665,7 +702,7 @@ func (h *APIHandler) HandleIndexMetricsAction(w http.ResponseWriter, req *http.R if ver.Distribution == "" { cr, err := util.VersionCompare(ver.Number, "6.1") if err != nil { - log.Error(err) + log.Errorf("HandleIndexMetricsAction failed: %v", err) } if cr < 0 { resBody["tips"] = "The system cluster version is lower than 6.1, the top index may be inaccurate" @@ -728,7 +765,7 @@ func (h *APIHandler) GetClusterHealth(w http.ResponseWriter, req *http.Request, exists, client, err := h.GetClusterClient(id) if err != nil { - log.Error(err) + log.Errorf("GetClusterHealth failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -1278,7 +1315,7 @@ func (h *APIHandler) getClusterStatusMetric(ctx context.Context, id string, min, queryDSL := util.MustToJSONBytes(query) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).QueryDSL(ctx, getAllMetricsIndex(), nil, util.MustToJSONBytes(query)) if err != nil { - log.Error(err) + log.Errorf("getClusterStatusMetric failed: %v", err) return nil, err } metricData := []interface{}{} diff --git a/modules/elastic/api/v1/metrics_util.go b/modules/elastic/api/v1/metrics_util.go index 6f5b1868..8b4f38b5 100644 --- a/modules/elastic/api/v1/metrics_util.go +++ b/modules/elastic/api/v1/metrics_util.go @@ -289,6 +289,9 @@ func (h *APIHandler) GetMetricRangeAndBucketSize(req *http.Request, clusterID, m bucketSize := 0 bucketSizeStr := h.GetParameterOrDefault(req, "bucket_size", "") //默认 10，每个 bucket 的时间范围，单位秒 + if strings.EqualFold(strings.TrimSpace(bucketSizeStr), "auto") { + bucketSizeStr = "" + } if bucketSizeStr != "" { du, err := util.ParseDuration(bucketSizeStr) if err != nil { @@ -324,9 +327,22 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m var rangeFrom, rangeTo time.Time var err error var useMinMax = bucketSize == 0 + if strings.EqualFold(strings.TrimSpace(minStr), "auto") { + minStr = "" + } + if strings.EqualFold(strings.TrimSpace(maxStr), "auto") { + maxStr = "" + } + effectiveBucketSize := bucketSize + if effectiveBucketSize <= 0 { + effectiveBucketSize = minBucketSize + } + if effectiveBucketSize <= 0 { + effectiveBucketSize = GetMinBucketSize() + } now := time.Now() if minStr == "" { - rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + rangeFrom = now.Add(-time.Second * time.Duration(effectiveBucketSize*metricCount+1)) } else { //try 2021-08-21T14:06:04.818Z rangeFrom, err = util.ParseStandardTime(minStr) @@ -334,8 +350,8 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m //try 1629637500000 v, err := util.ToInt64(minStr) if err != nil { - log.Error("invalid timestamp:", minStr, err) - rangeFrom = now.Add(-time.Second * time.Duration(bucketSize*metricCount+1)) + log.Errorf("GetMetricRangeAndBucketSize invalid min timestamp [%s]: %v", minStr, err) + rangeFrom = now.Add(-time.Second * time.Duration(effectiveBucketSize*metricCount+1)) } else { rangeFrom = util.FromUnixTimestamp(v / 1000) } @@ -343,14 +359,14 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m } if maxStr == "" { - rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + rangeTo = now.Add(-time.Second * time.Duration(effectiveBucketSize)) } else { rangeTo, err = util.ParseStandardTime(maxStr) if err != nil { v, err := util.ToInt64(maxStr) if err != nil { - log.Error("invalid timestamp:", maxStr, err) - rangeTo = now.Add(-time.Second * time.Duration(int(1*(float64(bucketSize))))) + log.Errorf("GetMetricRangeAndBucketSize invalid max timestamp [%s]: %v", maxStr, err) + rangeTo = now.Add(-time.Second * time.Duration(effectiveBucketSize)) } else { rangeTo = util.FromUnixTimestamp(int64(v) / 1000) } @@ -371,6 +387,91 @@ func GetMetricRangeAndBucketSize(minStr string, maxStr string, bucketSize int, m return bucketSize, min, max, nil } +func buildDateHistogramParams(query map[string]interface{}, intervalField, bucketSizeStr string) util.MapStr { + params := util.MapStr{ + "field": "timestamp", + intervalField: bucketSizeStr, + "min_doc_count": 0, + } + if bounds, ok := extractDateHistogramBounds(query); ok { + params["extended_bounds"] = bounds + } + return params +} + +func extractDateHistogramBounds(query map[string]interface{}) (util.MapStr, bool) { + queryMap, ok := asMap(query["query"]) + if !ok { + return nil, false + } + boolMap, ok := asMap(queryMap["bool"]) + if !ok { + return nil, false + } + filters, ok := asSlice(boolMap["filter"]) + if !ok { + return nil, false + } + for _, filterItem := range filters { + filterMap, ok := asMap(filterItem) + if !ok { + continue + } + rangeMap, ok := asMap(filterMap["range"]) + if !ok { + continue + } + timestampRange, ok := asMap(rangeMap["timestamp"]) + if !ok { + continue + } + min := firstNotNil(timestampRange["gte"], timestampRange["from"], timestampRange["gt"]) + max := firstNotNil(timestampRange["lte"], timestampRange["to"], timestampRange["lt"]) + if min != nil && max != nil { + return util.MapStr{ + "min": min, + "max": max, + }, true + } + } + return nil, false +} + +func asMap(value interface{}) (map[string]interface{}, bool) { + switch v := value.(type) { + case map[string]interface{}: + return v, true + case util.MapStr: + return map[string]interface{}(v), true + default: + return nil, false + } +} + +func asSlice(value interface{}) ([]interface{}, bool) { + switch v := value.(type) { + case []interface{}: + return v, true + case []util.MapStr: + result := make([]interface{}, 0, len(v)) + for _, item := range v { + result = append(result, item) + } + return result, true + default: + return nil, false + } +} + +func firstNotNil(values ...interface{}) interface{} { + for _, value := range values { + if value != nil { + return value + } + } + return nil +} + // findClosestNiceInterval finds the interval in sortedNiceIntervals that is closest to targetSize. func findClosestNiceInterval(targetSize float64, sortedNiceIntervals []float64) float64 { if len(sortedNiceIntervals) == 0 { @@ -513,11 +614,8 @@ func (h *APIHandler) getSingleMetrics(ctx context.Context, metricItems []*common query["size"] = 0 query["aggs"] = util.MapStr{ "dates": util.MapStr{ - "date_histogram": util.MapStr{ - "field": "timestamp", - intervalField: bucketSizeStr, - }, - "aggs": aggs, + "date_histogram": buildDateHistogramParams(query, intervalField, bucketSizeStr), + "aggs": aggs, }, } queryDSL := util.MustToJSONBytes(query) @@ -615,7 +713,7 @@ func (h *APIHandler) getBucketMetrics(query map[string]interface{}, bucketItems //bucketSizeStr := fmt.Sprintf("%vs", bucketSize) response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) if err != nil { - log.Error(err) + log.Errorf("getBucketMetrics failed: %v", err) panic(err) } //grpMetricItemsIndex := map[string]int{} @@ -741,8 +839,18 @@ func ConvertBucketItemsToAggQuery(bucketItems []*common.BucketItem, metricItems switch bucketItem.Type { case "terms": + termsParams := util.MapStr{} + for k, v := range bucketItem.Parameters { + termsParams[k] = v + } + // Some Elasticsearch-compatible engines return UnmappedTerms runtime + // errors on terms aggregation. Provide a default missing value so the + // aggregation can still be planned when the field is absent. + if _, ok := termsParams["missing"]; !ok { + termsParams["missing"] = "" + } bucketAgg = util.MapStr{ - "terms": bucketItem.Parameters, + "terms": termsParams, } break case "date_histogram": @@ -1058,7 +1166,7 @@ func parseHealthMetricData(buckets []elastic.BucketBase) ([]interface{}, error) for _, bucket := range buckets { v, ok := bucket["key"].(float64) if !ok { - log.Error("invalid bucket key") + log.Errorf("parseHealthMetricData invalid bucket key in aggregation response") return nil, fmt.Errorf("invalid bucket key") } dateTime := int64(v) @@ -1091,10 +1199,15 @@ func GetIndicesCount(clusterID string) int { if meta != nil && meta.ClusterState != nil && meta.ClusterState.Metadata != nil && meta.ClusterState.Metadata.Indices != nil { indexCount = len(meta.ClusterState.Metadata.Indices) } else { - log.Warnf("Can't get indices from metadata with %s", clusterID) esClient := elastic.GetClient(clusterID) - indexInfos, _ := esClient.GetIndices("") - indexCount = len(*indexInfos) + indexInfos, err := esClient.GetIndices("") + if err != nil { + log.Warnf("Can't get indices count with %s from metadata or indices API: %v", clusterID, err) + return indexCount + } + if indexInfos != nil { + indexCount = len(*indexInfos) + } } log.Debugf("Get cluster id %s indices count %d", clusterID, indexCount) return indexCount diff --git a/modules/elastic/api/v1/metrics_util_test.go b/modules/elastic/api/v1/metrics_util_test.go index f6ea477f..180a812b 100644 --- a/modules/elastic/api/v1/metrics_util_test.go +++ b/modules/elastic/api/v1/metrics_util_test.go @@ -2,10 +2,30 @@ package v1 import ( "math" + "net/http" "testing" "time" ) +func TestGetMetricRangeAndBucketSize_AutoBucketSize(t *testing.T) { + handler := APIHandler{} + req, err := http.NewRequest("GET", "https://infinilabs.com/api/?bucket_size=auto&min=auto&max=auto", nil) + if err != nil { + t.Fatal(err) + } + + bucketSize, min, max, err := handler.GetMetricRangeAndBucketSize(req, "", "", 15) + if err != nil { + t.Fatalf("expected no error for bucket_size=auto, got: %v", err) + } + if bucketSize <= 0 { + t.Fatalf("expected positive bucket size, got: %d", bucketSize) + } + if max < min { + t.Fatalf("expected max >= min, got min=%d max=%d", min, max) + } +} + var defaultActualTargetPoints = 120 // Default target points for bucket size calculation var maxBucketSizeGlobal = 24 * time.Hour.Seconds() // 1 day in seconds var minBucketSizeGlobal = 20 // Minimum bucket size in seconds diff --git a/modules/elastic/api/v1/node_overview.go b/modules/elastic/api/v1/node_overview.go index 65a5c8a8..db7b179a 100644 --- a/modules/elastic/api/v1/node_overview.go +++ b/modules/elastic/api/v1/node_overview.go @@ -202,6 +202,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps q1 := orm.Query{WildcardIndex: true} query := util.MapStr{ + "size": 1000, "sort": []util.MapStr{ { "timestamp": util.MapStr{ @@ -209,9 +210,6 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps }, }, }, - "collapse": util.MapStr{ - "field": "metadata.labels.node_id", - }, "query": util.MapStr{ "bool": util.MapStr{ "must": []util.MapStr{ @@ -242,7 +240,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps err, results := orm.Search(&event.Event{}, &q1) if err != nil { - log.Error(err) + log.Errorf("FetchNodeInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -252,11 +250,17 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps } var clusterID string statusMap := map[string]interface{}{} + seenNodeIDs := map[string]struct{}{} for _, v := range results.Result { result, ok := v.(map[string]interface{}) if ok { nodeID, ok := util.GetMapValueByKeys([]string{"metadata", "labels", "node_id"}, result) if ok { + nodeIDStr := util.ToString(nodeID) + if _, exists := seenNodeIDs[nodeIDStr]; exists { + continue + } + seenNodeIDs[nodeIDStr] = struct{}{} source := map[string]interface{}{} //timestamp, ok := result["timestamp"].(string) uptime, ok := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "jvm", "uptime_in_millis"}, result) @@ -295,13 +299,13 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps } } - statusMap[util.ToString(nodeID)] = source + statusMap[nodeIDStr] = source } } } statusMetric, err := getNodeOnlineStatusOfRecentDay(nodeIDs) if err != nil { - log.Error(err) + log.Errorf("FetchNodeInfo failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -412,7 +416,7 @@ func (h *APIHandler) FetchNodeInfo(w http.ResponseWriter, req *http.Request, ps } metrics, err := h.getMetrics(context.Background(), query, nodeMetricItems, bucketSize) if err != nil { - log.Error(err) + log.Errorf("FetchNodeInfo failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -488,7 +492,6 @@ func (h *APIHandler) GetNodeInfo(w http.ResponseWriter, req *http.Request, ps ht orm.Eq("metadata.name", "node_stats"), orm.Eq("metadata.labels.node_id", nodeID), ) - q1.Collapse("metadata.labels.node_id") q1.AddSort("timestamp", orm.DESC) err, result := orm.Search(&event.Event{}, &q1) kvs := util.MapStr{} @@ -569,7 +572,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque clusterID := ps.MustGetParameter("id") clusterUUID, err := adapter.GetClusterUUID(clusterID) if err != nil { - log.Error(err) + log.Errorf("GetSingleNodeMetrics failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -607,7 +610,7 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque resBody := map[string]interface{}{} bucketSize, min, max, err := h.GetMetricRangeAndBucketSize(req, clusterID, MetricTypeNodeStats, 60) if err != nil { - log.Error(err) + log.Errorf("GetSingleNodeMetrics failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -691,13 +694,13 @@ func (h *APIHandler) GetSingleNodeMetrics(w http.ResponseWriter, req *http.Reque metricItems = append(metricItems, metricItem) metrics, err := h.getSingleMetrics(context.Background(), metricItems, query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("GetSingleNodeMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } healthMetric, err := getNodeHealthMetric(query, bucketSize) if err != nil { - log.Error(err) + log.Errorf("GetSingleNodeMetrics failed: %v", err) h.WriteError(w, err, http.StatusInternalServerError) return } @@ -729,7 +732,7 @@ func getNodeHealthMetric(query util.MapStr, bucketSize int) (*common.MetricItem, } response, err := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)).SearchWithRawQueryDSL(getAllMetricsIndex(), util.MustToJSONBytes(query)) if err != nil { - log.Error(err) + log.Errorf("getNodeHealthMetric failed: %v", err) return nil, err } @@ -741,7 +744,7 @@ func getNodeHealthMetric(query util.MapStr, bucketSize int) (*common.MetricItem, for _, bucket := range response.Aggregations["dates"].Buckets { v, ok := bucket["key"].(float64) if !ok { - log.Error("invalid bucket key") + log.Errorf("getNodeHealthMetric invalid bucket key in aggregation response") return nil, fmt.Errorf("invalid bucket key") } dateTime := int64(v) @@ -914,7 +917,7 @@ func getNodeOnlineStatusOfRecentDay(nodeIDs []string) (map[string][]interface{}, return recentStatus, nil } -func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (h *APIHandler) GetNodeIndices(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var ( min = h.GetParameterOrDefault(req, "min", "now-15m") max = h.GetParameterOrDefault(req, "max", "now") @@ -923,6 +926,8 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps resBody := map[string]interface{}{} id := ps.ByName("id") nodeUUID := ps.ByName("node_id") + + // Step 1: routing table is the primary source of index names for this node. q := &orm.Query{Size: 1} q.AddSort("timestamp", orm.DESC) q.Conds = orm.And( @@ -936,8 +941,9 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps if err != nil { resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) + return } - namesM := util.MapStr{} + indexNames := map[string]bool{} if len(result.Result) > 0 { if data, ok := result.Result[0].(map[string]interface{}); ok { if routingTable, exists := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_routing_table"}, data); exists { @@ -945,7 +951,7 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps for _, row := range rows { if v, ok := row.(map[string]interface{}); ok { if indexName, ok := v["index"].(string); ok { - namesM[indexName] = true + indexNames[indexName] = true } } } @@ -954,41 +960,52 @@ func (h *APIHandler) getNodeIndices(w http.ResponseWriter, req *http.Request, ps } } - indexNames := make([]interface{}, 0, len(namesM)) - for name, _ := range namesM { - indexNames = append(indexNames, name) + if len(indexNames) == 0 { + h.WriteJSON(w, []interface{}{}, http.StatusOK) + return } - q1 := &orm.Query{Size: 100} + // Step 2: IndexConfig provides health and status enrichment (optional). + // Query all IndexConfig for the cluster to avoid issues with orm.In. + q1 := &orm.Query{Size: 2000} q1.AddSort("timestamp", orm.DESC) q1.Conds = orm.And( orm.Eq("metadata.category", "elasticsearch"), orm.Eq("metadata.cluster_id", id), - orm.In("metadata.index_name", indexNames), - orm.NotEq("metadata.labels.index_status", "deleted"), ) - err, result = orm.Search(elastic.IndexConfig{}, q1) - if err != nil { - resBody["error"] = err.Error() - h.WriteJSON(w, resBody, http.StatusInternalServerError) + _, indexConfigResult := orm.Search(elastic.IndexConfig{}, q1) + indexConfigMap := map[string]map[string]interface{}{} + for _, hit := range indexConfigResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + nameV, _ := util.GetMapValueByKeys([]string{"metadata", "index_name"}, hitM) + if name, ok := nameV.(string); ok { + if _, exists := indexConfigMap[name]; !exists { + indexConfigMap[name] = hitM + } + } + } } - indices, err := h.getLatestIndices(req, min, max, id, &result) + indices, err := h.getLatestNodeIndices(req, min, max, id, indexNames, indexConfigMap) if err != nil { resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) + return } h.WriteJSON(w, indices, http.StatusOK) } -func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, clusterID string, result *orm.Result) ([]interface{}, error) { +// getLatestNodeIndices builds the index list for a single node using the routing-table +// index names as the primary source of truth, enriched by IndexConfig and index_stats. +func (h *APIHandler) getLatestNodeIndices(req *http.Request, min string, max string, clusterID string, indexNames map[string]bool, indexConfigMap map[string]map[string]interface{}) ([]interface{}, error) { //filter indices allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) if !hasAllPrivilege && len(allowedIndices) == 0 { return []interface{}{}, nil } + // Query index_stats for metric enrichment (docs_count, store_size, etc.). query := util.MapStr{ "size": 2000, "_source": []string{"metadata", "payload.elasticsearch.index_stats.index_info", "timestamp"}, @@ -1041,20 +1058,146 @@ func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, }, } q := &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} + indexInfos := map[string]util.MapStr{} err, searchResult := orm.Search(event.Event{}, q) if err != nil { - return nil, err + log.Warnf("failed to enrich latest indices for cluster [%s] in v1 API, fallback to base index state only: %v", clusterID, err) + } else { + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + indexInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "index_info"}, hitM) + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) + if v, ok := indexName.(string); ok { + if infoM, ok := indexInfo.(map[string]interface{}); ok { + if _, ok = infoM["index"].(string); ok { + infoM["timestamp"] = hitM["timestamp"] + indexInfos[v] = infoM + } + } + } + } + } + } + + indices := []interface{}{} + var indexPattern *radix.Pattern + if !hasAllPrivilege { + indexPattern = radix.Compile(allowedIndices...) + } + + // Iterate over routing-table index names as the primary source so we always + // return results even when IndexConfig or index_stats data is absent. + for indexName := range indexNames { + if indexPattern != nil && !indexPattern.Match(indexName) { + continue + } + if info := indexInfos[indexName]; info != nil { + // Rich data from index_stats: apply any deletion status from IndexConfig. + if cfg := indexConfigMap[indexName]; cfg != nil { + state, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, cfg) + if state == "delete" { + info["status"] = "delete" + info["health"] = "N/A" + } + } + indices = append(indices, info) + } else if cfg := indexConfigMap[indexName]; cfg != nil { + // No index_stats but IndexConfig exists: return state and health from config. + state, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "state"}, cfg) + health, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "health_status"}, cfg) + if state == "delete" { + health = "N/A" + } + indices = append(indices, util.MapStr{ + "index": indexName, + "status": state, + "health": health, + "timestamp": cfg["timestamp"], + }) + } else { + // Routing table only: return the index name so the UI shows it exists. + indices = append(indices, util.MapStr{ + "index": indexName, + }) + } + } + return indices, nil +} + +// getLatestIndices is used by GetClusterIndices where IndexConfig is the primary source. +func (h *APIHandler) getLatestIndices(req *http.Request, min string, max string, clusterID string, result *orm.Result) ([]interface{}, error) { + allowedIndices, hasAllPrivilege := h.GetAllowedIndices(req, clusterID) + if !hasAllPrivilege && len(allowedIndices) == 0 { + return []interface{}{}, nil } + + query := util.MapStr{ + "size": 2000, + "_source": []string{"metadata", "payload.elasticsearch.index_stats.index_info", "timestamp"}, + "collapse": util.MapStr{ + "field": "metadata.labels.index_name", + }, + "sort": []util.MapStr{ + { + "timestamp": util.MapStr{ + "order": "desc", + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "filter": []util.MapStr{ + { + "range": util.MapStr{ + "timestamp": util.MapStr{ + "gte": min, + "lte": max, + }, + }, + }, + }, + "must": []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.name": util.MapStr{ + "value": "index_stats", + }, + }, + }, + }, + }, + }, + } + q := &orm.Query{RawQuery: util.MustToJSONBytes(query), WildcardIndex: true} indexInfos := map[string]util.MapStr{} - for _, hit := range searchResult.Result { - if hitM, ok := hit.(map[string]interface{}); ok { - indexInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "index_info"}, hitM) - indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) - if v, ok := indexName.(string); ok { - if infoM, ok := indexInfo.(map[string]interface{}); ok { - if _, ok = infoM["index"].(string); ok { - infoM["timestamp"] = hitM["timestamp"] - indexInfos[v] = infoM + err, searchResult := orm.Search(event.Event{}, q) + if err != nil { + log.Warnf("failed to enrich latest indices for cluster [%s] in v1 API, fallback to base index state only: %v", clusterID, err) + } else { + for _, hit := range searchResult.Result { + if hitM, ok := hit.(map[string]interface{}); ok { + indexInfo, _ := util.GetMapValueByKeys([]string{"payload", "elasticsearch", "index_stats", "index_info"}, hitM) + indexName, _ := util.GetMapValueByKeys([]string{"metadata", "labels", "index_name"}, hitM) + if v, ok := indexName.(string); ok { + if infoM, ok := indexInfo.(map[string]interface{}); ok { + if _, ok = infoM["index"].(string); ok { + infoM["timestamp"] = hitM["timestamp"] + indexInfos[v] = infoM + } } } } @@ -1123,7 +1266,7 @@ func (h *APIHandler) GetNodeShards(w http.ResponseWriter, req *http.Request, ps shardInfo, ok = util.GetMapValueByKeys([]string{"payload", "elasticsearch", "node_stats", "shard_info", "shards"}, row) qps, err := h.getIndexQPS(clusterID, 20) if err != nil { - log.Error(err) + log.Errorf("GetNodeShards failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } diff --git a/modules/elastic/api/value_parse.go b/modules/elastic/api/value_parse.go new file mode 100644 index 00000000..8042cb40 --- /dev/null +++ b/modules/elastic/api/value_parse.go @@ -0,0 +1,101 @@ +package api + +import ( + "encoding/json" + "strings" + + "infini.sh/framework/core/util" +) + +func parseInt64Value(value interface{}) (int64, bool) { + if value == nil { + return 0, false + } + switch v := value.(type) { + case int: + return int64(v), true + case int8: + return int64(v), true + case int16: + return int64(v), true + case int32: + return int64(v), true + case int64: + return v, true + case uint: + return int64(v), true + case uint8: + return int64(v), true + case uint16: + return int64(v), true + case uint32: + return int64(v), true + case uint64: + return int64(v), true + case float32: + return int64(v), true + case float64: + return int64(v), true + case json.Number: + fv, err := v.Float64() + if err != nil { + return 0, false + } + return int64(fv), true + case string: + trimmed := strings.TrimSpace(v) + if trimmed == "" { + return 0, false + } + iv, err := util.ToInt64(trimmed) + if err != nil { + return 0, false + } + return iv, true + default: + iv, err := util.ToInt64(strings.TrimSpace(util.ToString(v))) + if err != nil { + return 0, false + } + return iv, true + } +} + +func parseBoolValue(value interface{}) (bool, bool) { + if value == nil { + return false, false + } + switch v := value.(type) { + case bool: + return v, true + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "true", "1", "yes", "y": + return true, true + case "false", "0", "no", "n": + return false, true + default: + return false, false + } + case int, int8, int16, int32, int64: + iv, ok := parseInt64Value(v) + if !ok { + return false, false + } + return iv != 0, true + case uint, uint8, uint16, uint32, uint64: + iv, ok := parseInt64Value(v) + if !ok { + return false, false + } + return iv != 0, true + case float32, float64, json.Number: + iv, ok := parseInt64Value(v) + if !ok { + return false, false + } + return iv != 0, true + default: + return false, false + } +} diff --git a/modules/elastic/api/view.go b/modules/elastic/api/view.go index 334ad991..d83d7370 100644 --- a/modules/elastic/api/view.go +++ b/modules/elastic/api/view.go @@ -46,7 +46,7 @@ func (h *APIHandler) HandleCreateViewAction(w http.ResponseWriter, req *http.Req exists, _, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleCreateViewAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -54,7 +54,7 @@ func (h *APIHandler) HandleCreateViewAction(w http.ResponseWriter, req *http.Req if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleCreateViewAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -63,7 +63,7 @@ func (h *APIHandler) HandleCreateViewAction(w http.ResponseWriter, req *http.Req err = h.DecodeJSON(req, viewReq) if err != nil { - log.Error(err) + log.Errorf("HandleCreateViewAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -75,7 +75,7 @@ func (h *APIHandler) HandleCreateViewAction(w http.ResponseWriter, req *http.Req viewReq.Attributes.ClusterID = targetClusterID _, err = esClient.Index(orm.GetIndexName(viewReq.Attributes), "", id, viewReq.Attributes, "wait_for") if err != nil { - log.Error(err) + log.Errorf("HandleCreateViewAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -107,7 +107,7 @@ func (h *APIHandler) HandleGetViewListAction(w http.ResponseWriter, req *http.Re searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.View{}), queryDSL) if err != nil { - log.Error(err) + log.Errorf("HandleGetViewListAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -146,9 +146,9 @@ func (h *APIHandler) HandleDeleteViewAction(w http.ResponseWriter, req *http.Req view := elastic.View{ ID: viewID, } - _, err := orm.Get(&view) + _, err := orm.GetV2(orm.NewContext(), &view) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteViewAction failed: %v", err) h.WriteJSON(w, err.Error(), http.StatusInternalServerError) return } @@ -162,7 +162,7 @@ func (h *APIHandler) HandleDeleteViewAction(w http.ResponseWriter, req *http.Req } err = orm.Delete(ctx, &view) if err != nil { - log.Error(err) + log.Errorf("HandleDeleteViewAction failed: %v", err) h.WriteJSON(w, err.Error(), http.StatusInternalServerError) return } @@ -179,7 +179,7 @@ func (h *APIHandler) HandleResolveIndexAction(w http.ResponseWriter, req *http.R exists, client, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleResolveIndexAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -187,7 +187,7 @@ func (h *APIHandler) HandleResolveIndexAction(w http.ResponseWriter, req *http.R if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleResolveIndexAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -214,7 +214,7 @@ func (h *APIHandler) HandleResolveIndexAction(w http.ResponseWriter, req *http.R } searchRes, err := client.SearchWithRawQueryDSL(wild, util.MustToJSONBytes(q)) if err != nil { - log.Error(err) + log.Errorf("HandleResolveIndexAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -252,7 +252,7 @@ func (h *APIHandler) HandleResolveIndexAction(w http.ResponseWriter, req *http.R res, err := client.GetAliasesAndIndices() if err != nil || res == nil { - log.Error(err) + log.Errorf("HandleResolveIndexAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -301,7 +301,7 @@ func (h *APIHandler) HandleBulkGetViewAction(w http.ResponseWriter, req *http.Re err := h.DecodeJSON(req, &reqIDs) if err != nil { - log.Error(err) + log.Errorf("HandleBulkGetViewAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -321,7 +321,7 @@ func (h *APIHandler) HandleBulkGetViewAction(w http.ResponseWriter, req *http.Re {"match": {"cluster_id": "%s"}}]}}}`, strings.Join(strIDs, ","), targetClusterID)) searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.View{}), queryDSL) if err != nil { - log.Error(err) + log.Errorf("HandleBulkGetViewAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -352,7 +352,7 @@ func (h *APIHandler) HandleBulkGetViewAction(w http.ResponseWriter, req *http.Re for _, indexName := range indexNames { fields, err := elastic.GetFieldCaps(esTragertClient, indexName, []string{"_source", "_id", "_type", "_index"}) if err != nil { - log.Error(err) + log.Errorf("HandleBulkGetViewAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -386,7 +386,7 @@ func (h *APIHandler) HandleUpdateViewAction(w http.ResponseWriter, req *http.Req exists, _, err := h.GetClusterClient(targetClusterID) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateViewAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -394,7 +394,7 @@ func (h *APIHandler) HandleUpdateViewAction(w http.ResponseWriter, req *http.Req if !exists { resBody["error"] = fmt.Sprintf("cluster [%s] not found", targetClusterID) - log.Error(resBody["error"]) + log.Errorf("HandleUpdateViewAction failed: %v", resBody["error"]) h.WriteJSON(w, resBody, http.StatusNotFound) return } @@ -403,7 +403,7 @@ func (h *APIHandler) HandleUpdateViewAction(w http.ResponseWriter, req *http.Req err = h.DecodeJSON(req, viewReq) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateViewAction failed: %v", err) resBody["error"] = err h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -420,9 +420,9 @@ func (h *APIHandler) HandleUpdateViewAction(w http.ResponseWriter, req *http.Req oldView := &elastic.View{ ID: id, } - _, err = orm.Get(oldView) + _, err = orm.GetV2(orm.NewContext(), oldView) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateViewAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -433,9 +433,9 @@ func (h *APIHandler) HandleUpdateViewAction(w http.ResponseWriter, req *http.Req ctx := &orm.Context{ Refresh: "wait_for", } - err = orm.Save(ctx, viewReq.Attributes) + err = orm.Save(ctx, &viewReq.Attributes) if err != nil { - log.Error(err) + log.Errorf("HandleUpdateViewAction failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -457,7 +457,7 @@ func (h *APIHandler) HandleGetFieldCapsAction(w http.ResponseWriter, req *http.R esClient := elastic.GetClient(targetClusterID) kbnFields, err := elastic.GetFieldCaps(esClient, pattern, metaFields) if err != nil { - log.Error(err) + log.Errorf("HandleGetFieldCapsAction failed: %v", err) resBody["error"] = err.Error() h.WriteJSON(w, resBody, http.StatusInternalServerError) return @@ -497,7 +497,7 @@ func (h *APIHandler) HandleGetViewAction(w http.ResponseWriter, req *http.Reques obj := elastic.View{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -507,7 +507,7 @@ func (h *APIHandler) HandleGetViewAction(w http.ResponseWriter, req *http.Reques } if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("HandleGetViewAction failed: %v", err) return } @@ -519,7 +519,7 @@ func (h *APIHandler) SetDefaultLayout(w http.ResponseWriter, req *http.Request, err := h.DecodeJSON(req, viewReq) if err != nil { - log.Error(err) + log.Errorf("SetDefaultLayout failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } @@ -527,7 +527,7 @@ func (h *APIHandler) SetDefaultLayout(w http.ResponseWriter, req *http.Request, id := ps.MustGetParameter("view_id") viewObj := elastic.View{} viewObj.ID = id - exists, err := orm.Get(&viewObj) + exists, err := orm.GetV2(orm.NewContext(), &viewObj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -543,7 +543,7 @@ func (h *APIHandler) SetDefaultLayout(w http.ResponseWriter, req *http.Request, err = orm.Update(ctx, &viewObj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) - log.Error(err) + log.Errorf("SetDefaultLayout failed: %v", err) return } diff --git a/modules/security/access_token_route_test.go b/modules/security/access_token_route_test.go new file mode 100644 index 00000000..1cdbda58 --- /dev/null +++ b/modules/security/access_token_route_test.go @@ -0,0 +1,61 @@ +package security + +import ( + "bytes" + "net" + "net/http" + "net/http/httptest" + "testing" + + api2 "infini.sh/framework/core/api" + "infini.sh/framework/core/config" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" +) + +func newSecurityTestBinding(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on random port: %v", err) + } + defer listener.Close() + + return listener.Addr().String() +} + +func TestAccessTokenUIRoutesAreRegistered(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + testEnv.SystemConfig.WebAppConfig.Security.Enabled = false + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + for _, callback := range global.GetFuncBeforeSetup() { + callback() + } + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newSecurityTestBinding(t) + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + req := httptest.NewRequest(http.MethodPost, "http://console.local/auth/access_token", bytes.NewBufferString(`{}`)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + + func() { + defer func() { + _ = recover() + }() + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve ui request: %v", err) + } + }() + + if resp.Code == http.StatusNotFound { + t.Fatal("expected /auth/access_token to be registered on the UI router, got 404") + } +} diff --git a/modules/security/api/account.go b/modules/security/api/account.go deleted file mode 100644 index f6a9522e..00000000 --- a/modules/security/api/account.go +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Console is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -/* Copyright © INFINI Ltd. All rights reserved. - * web: https://infinilabs.com - * mail: hello#infini.ltd */ - -package api - -import ( - "fmt" - log "github.com/cihub/seelog" - "golang.org/x/crypto/bcrypt" - rbac "infini.sh/console/core/security" - "infini.sh/console/modules/security/realm" - "infini.sh/framework/core/api" - httprouter "infini.sh/framework/core/api/router" - "infini.sh/framework/core/util" - "net/http" -) - -const userInSession = "user_session:" - -// const SSOProvider = "sso" -const NativeProvider = "native" - -//const LDAPProvider = "ldap" - -func (h APIHandler) Logout(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - reqUser, err := rbac.FromUserContext(r.Context()) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - - rbac.DeleteUserToken(reqUser.UserId) - h.WriteOKJSON(w, util.MapStr{ - "status": "ok", - }) -} - -func (h APIHandler) Profile(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - reqUser, err := rbac.FromUserContext(r.Context()) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - - if reqUser.Provider == NativeProvider { - user, err := h.User.Get(reqUser.UserId) - if err != nil { - log.Error(err) - h.ErrorInternalServer(w, err.Error()) - return - } - if user.Nickname == "" { - user.Nickname = user.Username - } - - u := util.MapStr{ - "user_id": user.ID, - "name": user.Username, - "email": user.Email, - "nick_name": user.Nickname, - "phone": user.Phone, - } - - h.WriteOKJSON(w, api.FoundResponse(reqUser.UserId, u)) - } else { - - //TODO fetch external profile - - u := util.MapStr{ - "user_id": reqUser.UserId, - "name": reqUser.Username, - "email": "", //TOOD, save user profile come from SSO - "nick_name": reqUser.Username, //TODO - "phone": "", //TODO - } - h.WriteOKJSON(w, api.FoundResponse(reqUser.UserId, u)) - } - -} - -func (h APIHandler) UpdatePassword(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - reqUser, err := rbac.FromUserContext(r.Context()) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - var req struct { - OldPassword string `json:"old_password"` - NewPassword string `json:"new_password"` - } - err = h.DecodeJSON(r, &req) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - - user, err := h.User.Get(reqUser.UserId) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(req.OldPassword)) - if err == bcrypt.ErrMismatchedHashAndPassword { - h.ErrorInternalServer(w, "old password is not correct") - return - } - hash, err := bcrypt.GenerateFromPassword([]byte(req.NewPassword), bcrypt.DefaultCost) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - user.Password = string(hash) - err = h.User.Update(&user) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - h.WriteOKJSON(w, api.UpdateResponse(reqUser.UserId)) - return -} - -func (h APIHandler) UpdateProfile(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - reqUser, err := rbac.FromUserContext(r.Context()) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - var req struct { - Name string `json:"name"` - Phone string `json:"phone"` - Email string `json:"email"` - } - err = h.DecodeJSON(r, &req) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - user, err := h.User.Get(reqUser.UserId) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - user.Username = req.Name - user.Email = req.Email - user.Phone = req.Phone - err = h.User.Update(&user) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - h.WriteOKJSON(w, api.UpdateResponse(reqUser.UserId)) - return -} - -func (h APIHandler) Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - var req struct { - Username string `json:"username"` - Password string `json:"password"` - } - err := h.DecodeJSON(r, &req) - if err != nil { - h.ErrorInternalServer(w, err.Error()) - return - } - - var user *rbac.User - - //check user validation - ok, user, err := realm.Authenticate(req.Username, req.Password) - if err != nil { - h.WriteError(w, err.Error(), 500) - return - } - - if !ok { - h.WriteError(w, "invalid username or password", 403) - return - } - - if user == nil { - h.ErrorInternalServer(w, fmt.Sprintf("failed to authenticate user: %v", req.Username)) - return - } - - //check permissions - ok, err = realm.Authorize(user) - if err != nil || !ok { - h.ErrorInternalServer(w, fmt.Sprintf("failed to authorize user: %v", req.Username)) - return - } - - //fetch user profile - //TODO - if user.Nickname == "" { - user.Nickname = user.Username - } - - //generate access token - token, err := rbac.GenerateAccessToken(user) - if err != nil { - h.ErrorInternalServer(w, fmt.Sprintf("failed to authorize user: %v", req.Username)) - return - } - - //api.SetSession(w, r, userInSession+req.Username, req.Username) - h.WriteOKJSON(w, token) -} diff --git a/modules/security/api/account_password.go b/modules/security/api/account_password.go new file mode 100644 index 00000000..3b18cd9f --- /dev/null +++ b/modules/security/api/account_password.go @@ -0,0 +1,91 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package api + +import ( + "net/http" + + "golang.org/x/crypto/bcrypt" + "infini.sh/console/common" + rbac "infini.sh/console/core/security" + "infini.sh/console/model" + "infini.sh/console/service" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + frameworksecurity "infini.sh/framework/core/security" +) + +func (h APIHandler) UpdatePassword(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + reqUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + var req struct { + OldPassword string `json:"old_password"` + NewPassword string `json:"new_password"` + } + err = h.DecodeJSON(r, &req) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + + user, err := h.User.Get(reqUser.UserId) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(req.OldPassword)) + if err == bcrypt.ErrMismatchedHashAndPassword { + h.ErrorInternalServer(w, "old password is not correct") + return + } + + material, err := frameworksecurity.GeneratePasswordMaterial(req.NewPassword) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + user.Password = material.Hash + user.PasswordSalt = material.Salt + user.PasswordVerifier = material.Verifier + + err = h.User.Update(&user) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + + if r.Header.Get("Referer") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(reqUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("update password").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(reqUser.Username).WithOperationTypeModification(). + WithEventRecord("user updated password").Build() + _ = service.LogAuditLog(auditLog) + } + + h.WriteOKJSON(w, api.UpdateResponse(reqUser.UserId)) +} diff --git a/modules/security/api/account_profile.go b/modules/security/api/account_profile.go new file mode 100644 index 00000000..a97d1964 --- /dev/null +++ b/modules/security/api/account_profile.go @@ -0,0 +1,134 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package api + +import ( + "net/http" + "sort" + + rbac "infini.sh/console/core/security" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/orm" + frameworksecurity "infini.sh/framework/core/security" +) + +type accountProfileResponse struct { + frameworksecurity.UserProfile + Privilege []string `json:"privilege,omitempty"` +} + +func (h APIHandler) Profile(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + reqUser, err := rbac.FromUserContext(r.Context()) + if err != nil || reqUser == nil { + h.WriteError(w, "invalid user", http.StatusUnauthorized) + return + } + + profile := accountProfileResponse{ + UserProfile: frameworksecurity.UserProfile{ + Roles: append([]string(nil), reqUser.Roles...), + Preferences: frameworksecurity.Preferences{}, + }, + } + profile.ID = reqUser.UserId + profile.Name = reqUser.Username + profile.Privilege = collectPlatformPrivileges(profile.Roles) + + if orm.HasHandler() { + if user, err := h.User.Get(reqUser.UserId); err == nil && user.ID != "" { + if user.Username != "" { + profile.Name = user.Username + } + profile.Email = user.Email + profile.Phone = user.Phone + profile.Avatar = user.AvatarUrl + + if roles, privilege := user.GetPermissions(); len(roles) > 0 || len(privilege) > 0 { + if len(roles) > 0 { + profile.Roles = append([]string(nil), roles...) + } + if len(privilege) > 0 { + profile.Privilege = normalizeStringList(privilege) + } + } + } + } + + profile.Permissions = privilegesToPermissionKeys(profile.Privilege) + h.WriteJSON(w, profile, http.StatusOK) +} + +func collectPlatformPrivileges(roles []string) []string { + privileges := normalizeStringList(rbac.CombineUserRoles(roles).Platform) + if len(privileges) > 0 { + return privileges + } + + var builtinPrivileges []string + for _, roleName := range roles { + role, ok := rbac.BuiltinRoles[roleName] + if !ok { + continue + } + builtinPrivileges = append(builtinPrivileges, role.Privilege.Platform...) + } + return normalizeStringList(builtinPrivileges) +} + +func privilegesToPermissionKeys(privileges []string) []frameworksecurity.PermissionKey { + permissions := rbac.ExpandFrameworkPermissionKeysForPlatformPrivileges(normalizeStringList(privileges)) + sessionUser := &frameworksecurity.UserSessionInfo{ + UserAssignedPermission: frameworksecurity.NewUserAssignedPermission(permissions, nil), + } + rbac.EnsureFrameworkDefaultPermissions(sessionUser) + if sessionUser.UserAssignedPermission == nil { + return nil + } + return sessionUser.UserAssignedPermission.GetPermissionKeys() +} + +func normalizeStringList(values []string) []string { + if len(values) == 0 { + return nil + } + + unique := make(map[string]struct{}, len(values)) + for _, value := range values { + if value == "" { + continue + } + unique[value] = struct{}{} + } + + if len(unique) == 0 { + return nil + } + + out := make([]string, 0, len(unique)) + for value := range unique { + out = append(out, value) + } + sort.Strings(out) + return out +} diff --git a/modules/security/api/init.go b/modules/security/api/init.go index 47ceaa3c..cf4ce04c 100644 --- a/modules/security/api/init.go +++ b/modules/security/api/init.go @@ -32,6 +32,8 @@ import ( rbac "infini.sh/console/core/security" "infini.sh/console/core/security/enum" "infini.sh/framework/core/api" + frameworkaccount "infini.sh/framework/modules/security/account" + frameworkrbac "infini.sh/framework/modules/security/native" ) type APIHandler struct { @@ -43,29 +45,43 @@ const adapterType = "native" var apiHandler APIHandler +func permissionKeys(keys []string) []api.PermissionKey { + result := make([]api.PermissionKey, 0, len(keys)) + for _, key := range keys { + result = append(result, api.PermissionKey(key)) + } + return result +} + func Init() { apiHandler = APIHandler{Adapter: rbac.GetAdapter(adapterType)} //TODO handle hard coded + frameworkrbac.RegisterPublicUIAuthRoutes() - api.HandleAPIMethod(api.GET, "/permission/:type", apiHandler.RequireLogin(apiHandler.ListPermission)) + api.HandleAPIMethod(api.GET, "/permission/:type", apiHandler.RequireLogin(apiHandler.ListPermission), api.RequireLogin()) - api.HandleAPIMethod(api.POST, "/role/:type", apiHandler.RequirePermission(apiHandler.CreateRole, enum.RoleAllPermission...)) - api.HandleAPIMethod(api.GET, "/role/:id", apiHandler.RequirePermission(apiHandler.GetRole, enum.RoleReadPermission...)) - api.HandleAPIMethod(api.DELETE, "/role/:id", apiHandler.RequirePermission(apiHandler.DeleteRole, enum.RoleAllPermission...)) - api.HandleAPIMethod(api.PUT, "/role/:id", apiHandler.RequirePermission(apiHandler.UpdateRole, enum.RoleAllPermission...)) - api.HandleAPIMethod(api.GET, "/role/_search", apiHandler.RequirePermission(apiHandler.SearchRole, enum.RoleReadPermission...)) + api.HandleAPIMethod(api.POST, "/role/:type", apiHandler.RequirePermission(apiHandler.CreateRole, enum.RoleAllPermission...), api.RequirePermission(permissionKeys(enum.RoleAllPermission)...)) + api.HandleAPIMethod(api.GET, "/role/:id", apiHandler.RequirePermission(apiHandler.GetRole, enum.RoleReadPermission...), api.RequirePermission(permissionKeys(enum.RoleReadPermission)...)) + api.HandleAPIMethod(api.DELETE, "/role/:id", apiHandler.RequirePermission(apiHandler.DeleteRole, enum.RoleAllPermission...), api.RequirePermission(permissionKeys(enum.RoleAllPermission)...)) + api.HandleAPIMethod(api.PUT, "/role/:id", apiHandler.RequirePermission(apiHandler.UpdateRole, enum.RoleAllPermission...), api.RequirePermission(permissionKeys(enum.RoleAllPermission)...)) + api.HandleAPIMethod(api.GET, "/role/_search", apiHandler.RequirePermission(apiHandler.SearchRole, enum.RoleReadPermission...), api.RequirePermission(permissionKeys(enum.RoleReadPermission)...)) - api.HandleAPIMethod(api.POST, "/user", apiHandler.RequirePermission(apiHandler.CreateUser, enum.UserAllPermission...)) - api.HandleAPIMethod(api.GET, "/user/:id", apiHandler.RequirePermission(apiHandler.GetUser, enum.UserReadPermission...)) - api.HandleAPIMethod(api.DELETE, "/user/:id", apiHandler.RequirePermission(apiHandler.DeleteUser, enum.UserAllPermission...)) - api.HandleAPIMethod(api.PUT, "/user/:id", apiHandler.RequirePermission(apiHandler.UpdateUser, enum.UserAllPermission...)) - api.HandleAPIMethod(api.GET, "/user/_search", apiHandler.RequirePermission(apiHandler.SearchUser, enum.UserReadPermission...)) - api.HandleAPIMethod(api.PUT, "/user/:id/password", apiHandler.RequirePermission(apiHandler.UpdateUserPassword, enum.UserAllPermission...)) + api.HandleAPIMethod(api.POST, "/user", apiHandler.RequireSecureTransport(apiHandler.RequireReplayProtection(apiHandler.RequirePermission(apiHandler.CreateUser, enum.UserAllPermission...))), api.RequirePermission(permissionKeys(enum.UserAllPermission)...)) + api.HandleAPIMethod(api.GET, "/user/:id", apiHandler.RequirePermission(apiHandler.GetUser, enum.UserReadPermission...), api.RequirePermission(permissionKeys(enum.UserReadPermission)...)) + api.HandleAPIMethod(api.DELETE, "/user/:id", apiHandler.RequirePermission(apiHandler.DeleteUser, enum.UserAllPermission...), api.RequirePermission(permissionKeys(enum.UserAllPermission)...)) + api.HandleAPIMethod(api.PUT, "/user/:id", apiHandler.RequirePermission(apiHandler.UpdateUser, enum.UserAllPermission...), api.RequirePermission(permissionKeys(enum.UserAllPermission)...)) + api.HandleAPIMethod(api.GET, "/user/_search", apiHandler.RequirePermission(apiHandler.SearchUser, enum.UserReadPermission...), api.RequirePermission(permissionKeys(enum.UserReadPermission)...)) + api.HandleAPIMethod(api.POST, "/user/_enable", apiHandler.RequirePermission(apiHandler.EnableUser, enum.UserAllPermission...), api.RequirePermission(permissionKeys(enum.UserAllPermission)...)) + api.HandleAPIMethod(api.POST, "/user/_disable", apiHandler.RequirePermission(apiHandler.DisableUser, enum.UserAllPermission...), api.RequirePermission(permissionKeys(enum.UserAllPermission)...)) + api.HandleAPIMethod(api.PUT, "/user/:id/password", apiHandler.RequireSecureTransport(apiHandler.RequireReplayProtection(apiHandler.RequirePermission(apiHandler.UpdateUserPassword, enum.UserAllPermission...))), api.RequirePermission(permissionKeys(enum.UserAllPermission)...)) - api.HandleAPIMethod(api.POST, "/account/login", apiHandler.Login) - api.HandleAPIMethod(api.POST, "/account/logout", apiHandler.Logout) - api.HandleAPIMethod(api.DELETE, "/account/logout", apiHandler.Logout) + api.HandleAPIMethod(api.POST, "/account/replay_nonce", apiHandler.RequireSecureTransport(frameworkrbac.IssueReplayNonce)) + api.HandleAPIMethod(api.POST, "/account/login/challenge", apiHandler.RequireSecureTransport(frameworkrbac.LoginChallenge)) + api.HandleAPIMethod(api.POST, "/account/login", apiHandler.RequireSecureTransport(frameworkrbac.Login)) + api.HandleAPIMethod(api.POST, "/account/refresh", apiHandler.RequireSecureTransport(apiHandler.RequireLogin(frameworkaccount.Refresh)), api.RequireLogin()) + api.HandleAPIMethod(api.POST, "/account/logout", apiHandler.RequireLogin(frameworkaccount.Logout), api.RequireLogin()) + api.HandleAPIMethod(api.DELETE, "/account/logout", apiHandler.RequireLogin(frameworkaccount.Logout), api.RequireLogin()) - api.HandleAPIMethod(api.GET, "/account/profile", apiHandler.RequireLogin(apiHandler.Profile)) - api.HandleAPIMethod(api.PUT, "/account/password", apiHandler.RequireLogin(apiHandler.UpdatePassword)) + api.HandleAPIMethod(api.GET, "/account/profile", apiHandler.RequireLogin(apiHandler.Profile), api.RequireLogin()) + api.HandleAPIMethod(api.PUT, "/account/password", apiHandler.RequireSecureTransport(apiHandler.RequireReplayProtection(apiHandler.RequireLogin(apiHandler.UpdatePassword))), api.RequireLogin()) } diff --git a/modules/security/api/init_test.go b/modules/security/api/init_test.go new file mode 100644 index 00000000..5ddf838c --- /dev/null +++ b/modules/security/api/init_test.go @@ -0,0 +1,80 @@ +package api + +import ( + "bytes" + "net" + "net/http" + "net/http/httptest" + "testing" + + _ "infini.sh/console/modules/security/realm/authc/native" + api2 "infini.sh/framework/core/api" + config2 "infini.sh/framework/core/config" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" +) + +func newTestBinding(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on random port: %v", err) + } + defer listener.Close() + + return listener.Addr().String() +} + +func TestInitRegistersPublicLoginUIRoutes(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + Init() + + webCfg := config2.WebAppConfig{} + webCfg.NetworkConfig.Binding = newTestBinding(t) + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + tests := []struct { + name string + path string + body string + }{ + { + name: "replay nonce", + path: "/account/replay_nonce", + body: `{"method":"POST","path":"/account/login"}`, + }, + { + name: "login challenge", + path: "/account/login/challenge", + body: `{"login":""}`, + }, + { + name: "login", + path: "/account/login", + body: `{"login":""}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "https://console.local"+tt.path, bytes.NewBufferString(tt.body)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve ui request: %v", err) + } + + if resp.Code == http.StatusNotFound { + t.Fatalf("expected %s to be registered on the UI router, got 404", tt.path) + } + }) + } +} diff --git a/modules/security/api/role.go b/modules/security/api/role.go index ff642066..060a330b 100644 --- a/modules/security/api/role.go +++ b/modules/security/api/role.go @@ -28,26 +28,33 @@ package api import ( + "encoding/json" log "github.com/cihub/seelog" + "infini.sh/console/common" rbac "infini.sh/console/core/security" + "infini.sh/console/model" + "infini.sh/console/service" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "net/http" "time" ) +const errRoleAssignedToUsers = "role is still assigned to users" + func (h APIHandler) CreateRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { roleType := ps.MustGetParameter("type") - //localUser, err := rbac.FromUserContext(r.Context()) - //if err != nil { - // log.Error(err.Error()) - // h.ErrorInternalServer(w, err.Error()) - // return - //} - err := rbac.IsAllowRoleType(roleType) + localUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } + err = rbac.IsAllowRoleType(roleType) if err != nil { h.ErrorInternalServer(w, err.Error()) return @@ -77,6 +84,16 @@ func (h APIHandler) CreateRole(w http.ResponseWriter, r *http.Request, ps httpro return } rbac.RoleMap[role.Name] = *role + + if r.Header.Get("Referer") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(localUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("create role").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(role.Name).WithOperationTypeNew(). + WithEventRecord(util.MustToJSON(role)).Build() + _ = service.LogAuditLog(auditLog) + } + h.WriteOKJSON(w, api.CreateResponse(id)) return @@ -143,15 +160,26 @@ func (h APIHandler) GetRole(w http.ResponseWriter, r *http.Request, ps httproute func (h APIHandler) DeleteRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") - //localUser, err := biz.FromUserContext(r.Context()) - //if err != nil { - // log.Error(err.Error()) - // h.ErrorInternalServer(w, err.Error()) - // return - //} + localUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } oldRole, err := h.Role.Get(id) if err != nil { h.ErrorInternalServer(w, err.Error()) + return + } + + assignedUsers, err := countUsersByRoleID(id) + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + if assignedUsers > 0 { + h.WriteError(w, errRoleAssignedToUsers, http.StatusConflict) + return } err = h.Adapter.Role.Delete(id) @@ -161,20 +189,30 @@ func (h APIHandler) DeleteRole(w http.ResponseWriter, r *http.Request, ps httpro return } delete(rbac.RoleMap, oldRole.Name) + + if r.Header.Get("Referer") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(localUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("delete role").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(oldRole.Name).WithOperationTypeDeletion(). + WithEventRecord(util.MustToJSON(oldRole)).Build() + _ = service.LogAuditLog(auditLog) + } + h.WriteOKJSON(w, api.DeleteResponse(id)) return } func (h APIHandler) UpdateRole(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") - //localUser, err := biz.FromUserContext(r.Context()) - //if err != nil { - // log.Error(err.Error()) - // h.ErrorInternalServer(w, err.Error()) - // return - //} + localUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } role := &rbac.Role{} - err := h.DecodeJSON(r, role) + err = h.DecodeJSON(r, role) if err != nil { h.Error400(w, err.Error()) return @@ -196,14 +234,40 @@ func (h APIHandler) UpdateRole(w http.ResponseWriter, r *http.Request, ps httpro role.Updated = &now role.Created = oldRole.Created err = h.Role.Update(role) - delete(rbac.RoleMap, oldRole.Name) - rbac.RoleMap[role.Name] = *role if err != nil { _ = log.Error(err.Error()) h.ErrorInternalServer(w, err.Error()) return } + delete(rbac.RoleMap, oldRole.Name) + rbac.RoleMap[role.Name] = *role + + if r.Header.Get("Referer") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(localUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("update role").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(role.Name).WithOperationTypeModification(). + WithEventRecord(util.MustToJSON(role)).Build() + _ = service.LogAuditLog(auditLog) + } + h.WriteOKJSON(w, api.UpdateResponse(id)) return } + +func countUsersByRoleID(roleID string) (int64, error) { + roleIDJSON, err := json.Marshal(roleID) + if err != nil { + return 0, err + } + + query := orm.Query{ + RawQuery: []byte(`{"query":{"term":{"roles.id":` + string(roleIDJSON) + `}},"size":0}`), + } + err, result := orm.Search(&rbac.User{}, &query) + if err != nil { + return 0, err + } + return result.Total, nil +} diff --git a/modules/security/api/user.go b/modules/security/api/user.go index 5ce04202..7835ad32 100644 --- a/modules/security/api/user.go +++ b/modules/security/api/user.go @@ -32,14 +32,18 @@ import ( "errors" "github.com/buger/jsonparser" log "github.com/cihub/seelog" - "golang.org/x/crypto/bcrypt" + "infini.sh/console/common" rbac "infini.sh/console/core/security" + "infini.sh/console/model" + "infini.sh/console/service" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" + frameworksecurity "infini.sh/framework/core/security" "infini.sh/framework/core/util" "infini.sh/framework/modules/elastic" "net/http" "sort" + "strings" "time" ) @@ -54,21 +58,25 @@ func (h APIHandler) CreateUser(w http.ResponseWriter, r *http.Request, ps httpro h.Error400(w, "username is required") return } - //localUser, err := biz.FromUserContext(r.Context()) - //if err != nil { - // log.Error(err.Error()) - // h.ErrorInternalServer(w, err.Error()) - // return - //} + localUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } if h.userNameExists(w, user.Username) { return } randStr := util.GenerateSecureString(16) - hash, err := bcrypt.GenerateFromPassword([]byte(randStr), bcrypt.DefaultCost) + material, err := frameworksecurity.GeneratePasswordMaterial(randStr) if err != nil { + h.ErrorInternalServer(w, err.Error()) return } - user.Password = string(hash) + user.Password = material.Hash + user.PasswordSalt = material.Salt + user.PasswordVerifier = material.Verifier + user.SetEnabled(true) now := time.Now() user.Created = &now @@ -81,6 +89,16 @@ func (h APIHandler) CreateUser(w http.ResponseWriter, r *http.Request, ps httpro h.ErrorInternalServer(w, err.Error()) return } + + if r.Header.Get("Referer") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(localUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("create user").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(user.Username).WithOperationTypeNew(). + WithEventRecord(util.MustToJSON(user)).Build() + _ = service.LogAuditLog(auditLog) + } + h.WriteOKJSON(w, util.MapStr{ "_id": id, "password": randStr, @@ -117,6 +135,9 @@ func (h APIHandler) GetUser(w http.ResponseWriter, r *http.Request, ps httproute h.ErrorInternalServer(w, err.Error()) return } + user.Password = "" + user.PasswordSalt = "" + user.PasswordVerifier = "" h.WriteOKJSON(w, api.FoundResponse(id, user)) return } @@ -130,12 +151,12 @@ func (h APIHandler) UpdateUser(w http.ResponseWriter, r *http.Request, ps httpro h.Error400(w, err.Error()) return } - //localUser, err := biz.FromUserContext(r.Context()) - //if err != nil { - // log.Error(err.Error()) - // h.ErrorInternalServer(w, err.Error()) - // return - //} + localUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } oldUser, err := h.User.Get(id) if err != nil { _ = log.Error(err.Error()) @@ -150,6 +171,12 @@ func (h APIHandler) UpdateUser(w http.ResponseWriter, r *http.Request, ps httpro user.Updated = &now user.Created = oldUser.Created user.ID = id + user.Password = oldUser.Password + user.PasswordSalt = oldUser.PasswordSalt + user.PasswordVerifier = oldUser.PasswordVerifier + if user.Enabled == nil { + user.SetEnabled(oldUser.IsEnabled()) + } err = h.User.Update(&user) if err != nil { @@ -168,10 +195,106 @@ func (h APIHandler) UpdateUser(w http.ResponseWriter, r *http.Request, ps httpro if len(changeLog) > 0 { rbac.DeleteUserToken(id) } + + if r.Header.Get("Referer") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(localUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("update user").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(user.Username).WithOperationTypeModification(). + WithEventRecord(util.MustToJSON(user)).Build() + _ = service.LogAuditLog(auditLog) + } + h.WriteOKJSON(w, api.UpdateResponse(id)) return } +func (h APIHandler) EnableUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + h.batchSetUserEnabled(w, r, true) +} + +func (h APIHandler) DisableUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + h.batchSetUserEnabled(w, r, false) +} + +func (h APIHandler) batchSetUserEnabled(w http.ResponseWriter, r *http.Request, enabled bool) { + var userIDs []string + if err := h.DecodeJSON(r, &userIDs); err != nil { + h.Error400(w, err.Error()) + return + } + if len(userIDs) == 0 { + h.WriteAckOKJSON(w) + return + } + + reqUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error("failed to get user from context, err: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + var updatedUsernames []string + for _, userID := range userIDs { + user, err := h.User.Get(userID) + if errors.Is(err, elastic.ErrNotFound) { + h.WriteJSON(w, api.NotFoundResponse(userID), http.StatusNotFound) + return + } + if err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + + if reqUser != nil && reqUser.UserId == userID && !enabled { + h.Error400(w, "can not disable yourself") + return + } + if !enabled && isAdministratorUser(user) { + h.Error400(w, "can not disable administrator") + return + } + + if user.IsEnabled() == enabled { + continue + } + user.SetEnabled(enabled) + if err = h.User.Update(&user); err != nil { + h.ErrorInternalServer(w, err.Error()) + return + } + if !enabled { + rbac.DeleteUserToken(userID) + } + updatedUsernames = append(updatedUsernames, user.Username) + } + + if len(updatedUsernames) > 0 && r.Header.Get("Referer") != "" { + eventName := "enable user" + if !enabled { + eventName = "disable user" + } + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(reqUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName(eventName).WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(strings.Join(updatedUsernames, ",")).WithOperationTypeModification(). + WithEventRecord(util.MustToJSON(updatedUsernames)).Build() + _ = service.LogAuditLog(auditLog) + } + + h.WriteAckOKJSON(w) +} + +func isAdministratorUser(user rbac.User) bool { + for _, role := range user.Roles { + if role.ID == rbac.RoleAdminName || role.Name == rbac.RoleAdminName { + return true + } + } + return false +} + func (h APIHandler) DeleteUser(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") user, err := rbac.FromUserContext(r.Context()) @@ -184,6 +307,9 @@ func (h APIHandler) DeleteUser(w http.ResponseWriter, r *http.Request, ps httpro h.WriteError(w, "can not delete yourself", http.StatusInternalServerError) return } + + oldUser, getErr := h.User.Get(id) + err = h.User.Delete(id) if errors.Is(err, elastic.ErrNotFound) { h.WriteJSON(w, api.NotFoundResponse(id), http.StatusNotFound) @@ -195,6 +321,20 @@ func (h APIHandler) DeleteUser(w http.ResponseWriter, r *http.Request, ps httpro return } rbac.DeleteUserToken(id) + + if r.Header.Get("Referer") != "" { + resourceName := id + if getErr == nil { + resourceName = oldUser.Username + } + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(user.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("delete user").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(resourceName).WithOperationTypeDeletion(). + WithEventRecord(resourceName).Build() + _ = service.LogAuditLog(auditLog) + } + h.WriteOKJSON(w, api.DeleteResponse(id)) return } @@ -217,6 +357,8 @@ func (h APIHandler) SearchUser(w http.ResponseWriter, r *http.Request, ps httpro hitsBuf.Write([]byte("[")) jsonparser.ArrayEach(res.Raw, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { value = jsonparser.Delete(value, "_source", "password") + value = jsonparser.Delete(value, "_source", "password_salt") + value = jsonparser.Delete(value, "_source", "password_verifier") hitsBuf.Write(value) hitsBuf.Write([]byte(",")) }, "hits", "hits") @@ -249,23 +391,26 @@ func (h APIHandler) UpdateUserPassword(w http.ResponseWriter, r *http.Request, p h.Error400(w, err.Error()) return } - //localUser, err := biz.FromUserContext(r.Context()) - //if err != nil { - // log.Error(err.Error()) - // h.ErrorInternalServer(w, err.Error()) - // return - //} + localUser, err := rbac.FromUserContext(r.Context()) + if err != nil { + log.Error(err.Error()) + h.ErrorInternalServer(w, err.Error()) + return + } user, err := h.User.Get(id) if err != nil { _ = log.Error(err.Error()) h.ErrorInternalServer(w, err.Error()) return } - hash, err := bcrypt.GenerateFromPassword([]byte(req.Password), bcrypt.DefaultCost) + material, err := frameworksecurity.GeneratePasswordMaterial(req.Password) if err != nil { + h.ErrorInternalServer(w, err.Error()) return } - user.Password = string(hash) + user.Password = material.Hash + user.PasswordSalt = material.Salt + user.PasswordVerifier = material.Verifier //t:=time.Now() //user.Updated =&t err = h.User.Update(&user) @@ -277,6 +422,15 @@ func (h APIHandler) UpdateUserPassword(w http.ResponseWriter, r *http.Request, p //disable old token to let user login rbac.DeleteUserToken(id) + if r.Header.Get("Referer") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(localUser.Username). + WithLogTypeOperation().WithResourceTypeAccountCenter(). + WithEventName("reset user password").WithEventSourceIP(common.GetClientIP(r)). + WithResourceName(user.Username).WithOperationTypeModification(). + WithEventRecord("password reset for user: " + user.Username).Build() + _ = service.LogAuditLog(auditLog) + } + h.WriteOKJSON(w, api.UpdateResponse(id)) return diff --git a/modules/security/credential/api/credential.go b/modules/security/credential/api/credential.go index 6f5c0f10..01027fec 100644 --- a/modules/security/credential/api/credential.go +++ b/modules/security/credential/api/credential.go @@ -32,14 +32,17 @@ import ( "fmt" log "github.com/cihub/seelog" "infini.sh/console/core" + agent_common "infini.sh/console/modules/agent/common" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/credential" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/model" "infini.sh/framework/core/orm" "infini.sh/framework/core/task" "infini.sh/framework/core/util" "net/http" "strconv" + "strings" ) type APIHandler struct { @@ -54,12 +57,23 @@ func (h *APIHandler) createCredential(w http.ResponseWriter, req *http.Request, http.Error(w, err.Error(), http.StatusInternalServerError) return } + cred.Name = strings.TrimSpace(cred.Name) err = cred.Validate() if err != nil { log.Error(err) http.Error(w, err.Error(), http.StatusInternalServerError) return } + exists, err := credentialNameExists(cred.Name, "") + if err != nil { + log.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if exists { + h.WriteError(w, "credential name already exists", http.StatusConflict) + return + } err = cred.Encode() if err != nil { log.Error(err) @@ -84,7 +98,7 @@ func (h *APIHandler) updateCredential(w http.ResponseWriter, req *http.Request, obj := credential.Credential{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -100,42 +114,71 @@ func (h *APIHandler) updateCredential(w http.ResponseWriter, req *http.Request, log.Error(err) return } + newObj.Name = strings.TrimSpace(newObj.Name) err = newObj.Validate() if err != nil { log.Error(err) http.Error(w, err.Error(), http.StatusInternalServerError) return } + exists, err = credentialNameExists(newObj.Name, obj.ID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + if exists { + h.WriteError(w, "credential name already exists", http.StatusConflict) + return + } + err = validateCredentialUpdate(&obj, &newObj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } encodeChanged := false - if obj.Type != newObj.Type { - encodeChanged = true - } else { - switch newObj.Type { - case credential.BasicAuth: - var oldPwd string - if oldParams, ok := obj.Payload[newObj.Type].(map[string]interface{}); ok { - if pwd, ok := oldParams["password"].(string); ok { - oldPwd = pwd - } else { - http.Error(w, fmt.Sprintf("invalid password of credential [%s]", obj.ID), http.StatusInternalServerError) - return - } + rotatedTokenValue := "" + switch newObj.Type { + case credential.BasicAuth: + var oldPwd string + if oldParams, ok := obj.Payload[newObj.Type].(map[string]interface{}); ok { + if pwd, ok := oldParams["password"].(string); ok { + oldPwd = pwd + } else { + http.Error(w, fmt.Sprintf("invalid password of credential [%s]", obj.ID), http.StatusInternalServerError) + return } - if params, ok := newObj.Payload[newObj.Type].(map[string]interface{}); ok { - if pwd, ok := params["password"].(string); ok && pwd != oldPwd { - obj.Payload = newObj.Payload - encodeChanged = true - } else { - if oldParams, ok := obj.Payload[obj.Type].(map[string]interface{}); ok { - oldParams["username"] = params["username"] - } - } + } + if params, ok := newObj.Payload[newObj.Type].(map[string]interface{}); ok { + if pwd, ok := params["password"].(string); ok && pwd != oldPwd { + obj.Payload = newObj.Payload + encodeChanged = true + } + if oldParams, ok := obj.Payload[obj.Type].(map[string]interface{}); ok { + oldParams["username"] = params["username"] } - default: - h.WriteError(w, fmt.Sprintf("unsupport credential type [%s]", newObj.Type), http.StatusInternalServerError) - return } + case credential.Token: + var oldValue string + if oldParams, ok := obj.Payload[newObj.Type].(map[string]interface{}); ok { + if value, ok := oldParams["value"].(string); ok { + oldValue = value + } else { + http.Error(w, fmt.Sprintf("invalid token value of credential [%s]", obj.ID), http.StatusInternalServerError) + return + } + } + if params, ok := newObj.Payload[newObj.Type].(map[string]interface{}); ok { + if value, ok := params["value"].(string); ok && value != oldValue { + obj.Payload = newObj.Payload + encodeChanged = true + rotatedTokenValue = oldValue + } + } + default: + h.WriteError(w, fmt.Sprintf("unsupport credential type [%s]", newObj.Type), http.StatusInternalServerError) + return } obj.Name = newObj.Name obj.Type = newObj.Type @@ -158,6 +201,9 @@ func (h *APIHandler) updateCredential(w http.ResponseWriter, req *http.Request, log.Error(err) return } + if rotatedTokenValue != "" { + agent_common.RememberPreviousToken(obj.ID, rotatedTokenValue) + } task.RunWithinGroup("credential_callback", func(ctx context.Context) error { credential.TriggerChangeEvent(&obj) return nil @@ -166,13 +212,23 @@ func (h *APIHandler) updateCredential(w http.ResponseWriter, req *http.Request, h.WriteUpdatedOKJSON(w, id) } +func validateCredentialUpdate(current, next *credential.Credential) error { + if current == nil || next == nil { + return fmt.Errorf("credential update input can not be nil") + } + if current.Type != next.Type { + return fmt.Errorf("credential type cannot be changed") + } + return nil +} + func (h *APIHandler) deleteCredential(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") obj := credential.Credential{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -204,6 +260,40 @@ func (h *APIHandler) deleteCredential(w http.ResponseWriter, req *http.Request, h.WriteDeletedOKJSON(w, id) } +func credentialNameExists(name, excludeID string) (bool, error) { + name = strings.TrimSpace(name) + if name == "" { + return false, nil + } + + existing := []credential.Credential{} + query := orm.Query{ + Size: 10, + Conds: orm.And( + orm.Eq("name", name), + ), + } + err, _ := orm.SearchWithJSONMapper(&existing, &query) + if err != nil { + return false, err + } + if len(existing) == 0 { + return false, nil + } + return hasCredentialNameConflict(existing, excludeID), nil +} + +func hasCredentialNameConflict(existing []credential.Credential, excludeID string) bool { + excludeID = strings.TrimSpace(excludeID) + for _, item := range existing { + if excludeID != "" && item.ID == excludeID { + continue + } + return true + } + return false +} + func canDelete(cred *credential.Credential) (bool, error) { if cred == nil { return false, fmt.Errorf("parameter cred can not be nil") @@ -215,6 +305,19 @@ func canDelete(cred *credential.Credential) (bool, error) { if err != nil { return false, fmt.Errorf("query elasticsearch config error: %w", err) } + if result.Total > 0 { + return false, nil + } + q = orm.Query{ + Conds: orm.Or( + orm.Eq("manager_credential_id", cred.ID), + orm.Eq("access_credential_id", cred.ID), + ), + } + err, result = orm.Search(model.Instance{}, &q) + if err != nil { + return false, fmt.Errorf("query instance config error: %w", err) + } return result.Total == 0, nil } @@ -223,16 +326,7 @@ func (h *APIHandler) searchCredential(w http.ResponseWriter, req *http.Request, keyword = h.GetParameterOrDefault(req, "keyword", "") strSize = h.GetParameterOrDefault(req, "size", "20") strFrom = h.GetParameterOrDefault(req, "from", "0") - mustQ []interface{} ) - if keyword != "" { - mustQ = append(mustQ, util.MapStr{ - "query_string": util.MapStr{ - "default_field": "*", - "query": keyword, - }, - }) - } size, _ := strconv.Atoi(strSize) if size <= 0 { size = 20 @@ -242,24 +336,7 @@ func (h *APIHandler) searchCredential(w http.ResponseWriter, req *http.Request, from = 0 } - queryDSL := util.MapStr{ - "size": size, - "from": from, - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "desc", - }, - }, - }, - } - if len(mustQ) > 0 { - queryDSL["query"] = util.MapStr{ - "bool": util.MapStr{ - "must": mustQ, - }, - } - } + queryDSL := buildCredentialSearchQueryDSL(keyword, from, size) q := orm.Query{} q.RawQuery = util.MustToJSONBytes(queryDSL) @@ -276,18 +353,71 @@ func (h *APIHandler) searchCredential(w http.ResponseWriter, req *http.Request, for _, hit := range searchRes.Hits.Hits { delete(hit.Source, "encrypt") util.MapStr(hit.Source).Delete("payload.basic_auth.password") + util.MapStr(hit.Source).Delete("payload.token.value") } } h.WriteJSON(w, searchRes, http.StatusOK) } +func buildCredentialSearchQueryDSL(keyword string, from, size int) util.MapStr { + mustQ := []interface{}{} + if keyword != "" { + mustQ = append(mustQ, util.MapStr{ + "query_string": util.MapStr{ + "default_field": "*", + "query": keyword, + }, + }) + } + + queryDSL := util.MapStr{ + "size": size, + "from": from, + "sort": []util.MapStr{ + { + "created": util.MapStr{ + "order": "desc", + }, + }, + }, + } + boolQuery := util.MapStr{ + "must_not": []interface{}{ + buildManagedPendingCredentialExclusion(), + }, + } + if len(mustQ) > 0 { + boolQuery["must"] = mustQ + } + queryDSL["query"] = util.MapStr{ + "bool": boolQuery, + } + return queryDSL +} + +func buildManagedPendingCredentialExclusion() util.MapStr { + mustQ := make([]interface{}, 0, len(agent_common.BuildPendingManagerCredentialTags())) + for _, tag := range agent_common.BuildPendingManagerCredentialTags() { + mustQ = append(mustQ, util.MapStr{ + "term": util.MapStr{ + "tags": tag, + }, + }) + } + return util.MapStr{ + "bool": util.MapStr{ + "must": mustQ, + }, + } +} + func (h *APIHandler) getCredential(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") obj := credential.Credential{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -296,5 +426,6 @@ func (h *APIHandler) getCredential(w http.ResponseWriter, req *http.Request, ps return } util.MapStr(obj.Payload).Delete("basic_auth.password") + util.MapStr(obj.Payload).Delete("token.value") h.WriteGetOKJSON(w, id, obj) } diff --git a/modules/security/credential/api/credential_test.go b/modules/security/credential/api/credential_test.go new file mode 100644 index 00000000..0597821a --- /dev/null +++ b/modules/security/credential/api/credential_test.go @@ -0,0 +1,83 @@ +package api + +import ( + "testing" + + agent_common "infini.sh/console/modules/agent/common" + "infini.sh/framework/core/credential" + "infini.sh/framework/core/util" +) + +func TestCredentialNameConflictIgnoresExcludedID(t *testing.T) { + items := []credential.Credential{ + {Name: "shared-name"}, + } + items[0].ID = "cred-1" + + if hasCredentialNameConflict(items, "cred-1") { + t.Fatalf("expected excluded credential id to be ignored") + } +} + +func TestCredentialNameConflictDetectsDifferentID(t *testing.T) { + items := []credential.Credential{ + {Name: "shared-name"}, + } + items[0].ID = "cred-2" + + if !hasCredentialNameConflict(items, "cred-1") { + t.Fatalf("expected different credential id to be treated as conflict") + } +} + +func TestValidateCredentialUpdateRejectsTypeChange(t *testing.T) { + current := &credential.Credential{Type: credential.BasicAuth} + next := &credential.Credential{Type: credential.Token} + + if err := validateCredentialUpdate(current, next); err == nil { + t.Fatalf("expected credential type change to be rejected") + } +} + +func TestValidateCredentialUpdateAllowsSameType(t *testing.T) { + current := &credential.Credential{Type: credential.BasicAuth} + next := &credential.Credential{Type: credential.BasicAuth} + + if err := validateCredentialUpdate(current, next); err != nil { + t.Fatalf("expected same credential type to be allowed, got %v", err) + } +} + +func TestBuildManagedPendingCredentialExclusion(t *testing.T) { + clause := buildManagedPendingCredentialExclusion() + boolClause, ok := clause["bool"].(util.MapStr) + if !ok { + t.Fatalf("expected bool clause, got %#v", clause["bool"]) + } + mustQ, ok := boolClause["must"].([]interface{}) + if !ok { + t.Fatalf("expected must clause, got %#v", boolClause["must"]) + } + if len(mustQ) != len(agent_common.BuildPendingManagerCredentialTags()) { + t.Fatalf("expected %d term clauses, got %d", len(agent_common.BuildPendingManagerCredentialTags()), len(mustQ)) + } +} + +func TestBuildCredentialSearchQueryDSLExcludesPendingManagerCredentials(t *testing.T) { + queryDSL := buildCredentialSearchQueryDSL("agent", 10, 5) + query, ok := queryDSL["query"].(util.MapStr) + if !ok { + t.Fatalf("expected query clause, got %#v", queryDSL["query"]) + } + boolQuery, ok := query["bool"].(util.MapStr) + if !ok { + t.Fatalf("expected bool query, got %#v", query["bool"]) + } + if _, ok := boolQuery["must"]; !ok { + t.Fatalf("expected keyword clause to be preserved") + } + mustNotQ, ok := boolQuery["must_not"].([]interface{}) + if !ok || len(mustNotQ) != 1 { + t.Fatalf("expected single must_not clause, got %#v", boolQuery["must_not"]) + } +} diff --git a/modules/security/credential/api/init.go b/modules/security/credential/api/init.go index bc611277..9a120bc8 100644 --- a/modules/security/credential/api/init.go +++ b/modules/security/credential/api/init.go @@ -40,11 +40,11 @@ import ( func Init() { handler := APIHandler{} - api.HandleAPIMethod(api.POST, "/credential", handler.RequirePermission(handler.createCredential, enum.PermissionCredentialWrite)) - api.HandleAPIMethod(api.PUT, "/credential/:id", handler.RequirePermission(handler.updateCredential, enum.PermissionCredentialWrite)) - api.HandleAPIMethod(api.DELETE, "/credential/:id", handler.RequirePermission(handler.deleteCredential, enum.PermissionCredentialWrite)) - api.HandleAPIMethod(api.GET, "/credential/_search", handler.RequirePermission(handler.searchCredential, enum.PermissionCredentialRead)) - api.HandleAPIMethod(api.GET, "/credential/:id", handler.RequirePermission(handler.getCredential, enum.PermissionCredentialRead)) + api.HandleAPIMethod(api.POST, "/credential", handler.RequireSecureTransport(handler.RequireReplayProtection(handler.RequirePermission(handler.createCredential, enum.PermissionCredentialWrite))), api.RequirePermission(api.PermissionKey(enum.PermissionCredentialWrite))) + api.HandleAPIMethod(api.PUT, "/credential/:id", handler.RequireSecureTransport(handler.RequireReplayProtection(handler.RequirePermission(handler.updateCredential, enum.PermissionCredentialWrite))), api.RequirePermission(api.PermissionKey(enum.PermissionCredentialWrite))) + api.HandleAPIMethod(api.DELETE, "/credential/:id", handler.RequirePermission(handler.deleteCredential, enum.PermissionCredentialWrite), api.RequirePermission(api.PermissionKey(enum.PermissionCredentialWrite))) + api.HandleAPIMethod(api.GET, "/credential/_search", handler.RequirePermission(handler.searchCredential, enum.PermissionCredentialRead), api.RequirePermission(api.PermissionKey(enum.PermissionCredentialRead))) + api.HandleAPIMethod(api.GET, "/credential/:id", handler.RequirePermission(handler.getCredential, enum.PermissionCredentialRead), api.RequirePermission(api.PermissionKey(enum.PermissionCredentialRead))) credential.RegisterChangeEvent(func(cred *credential.Credential) { var keys []string diff --git a/modules/security/filter/auth.go b/modules/security/filter/auth.go new file mode 100644 index 00000000..89e20c3c --- /dev/null +++ b/modules/security/filter/auth.go @@ -0,0 +1,88 @@ +package filter + +import ( + log "github.com/cihub/seelog" + consolesecurity "infini.sh/console/core/security" + "infini.sh/framework/core/api" + common "infini.sh/framework/core/api/common" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/global" + frameworksecurity "infini.sh/framework/core/security" + "net/http" +) + +func init() { + api.RegisterUIFilter(&AuthFilter{}) +} + +type AuthFilter struct { + api.Handler +} + +func (f *AuthFilter) GetPriority() int { + return 200 +} + +func (f *AuthFilter) ApplyFilter( + method string, + pattern string, + options *api.HandlerOptions, + next httprouter.Handle, +) httprouter.Handle { + if options == nil || (!options.RequireLogin && !options.OptionLogin) || !common.IsAuthEnable() { + log.Debug(method, ",", pattern, ",skip auth") + return next + } + + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + sessionUser, err := frameworksecurity.ValidateLogin(w, r) + + if global.Env().IsDebug { + log.Debug(method, ",", pattern, ",", sessionUser, ",", err) + } + + if sessionUser != nil && sessionUser.IsValid() { + r = r.WithContext(frameworksecurity.AddUserToContext(r.Context(), prepareConsoleUIUser(sessionUser))) + } + + if !options.OptionLogin { + if err != nil || sessionUser == nil || !sessionUser.IsValid() { + o := api.PrepareErrorJson("invalid login", http.StatusUnauthorized) + f.WriteJSON(w, o, http.StatusUnauthorized) + return + } + } + + next(w, r, ps) + } +} + +func prepareConsoleUIUser(sessionUser *frameworksecurity.UserSessionInfo) *frameworksecurity.UserSessionInfo { + if sessionUser == nil { + return nil + } + + cloned := *sessionUser + cloned.Roles = append([]string(nil), sessionUser.Roles...) + if sessionUser.UserAssignedPermission != nil { + cloned.UserAssignedPermission = frameworksecurity.NewUserAssignedPermission(sessionUser.UserAssignedPermission.GetPermissionKeys(), nil) + } + consolesecurity.EnsureFrameworkDefaultPermissions(&cloned) + + if hasConsoleAdminRole(cloned.Roles) { + cloned.UserAssignedPermission = frameworksecurity.NewUserAssignedPermission(frameworksecurity.GetAllPermissionKeys(), nil) + return &cloned + } + + cloned.UserAssignedPermission = frameworksecurity.GetUserPermissions(&cloned) + return &cloned +} + +func hasConsoleAdminRole(roles []string) bool { + for _, role := range roles { + if role == consolesecurity.RoleAdminName || role == frameworksecurity.RoleAdmin { + return true + } + } + return false +} diff --git a/modules/security/framework_account_bridge.go b/modules/security/framework_account_bridge.go new file mode 100644 index 00000000..a71d1c2e --- /dev/null +++ b/modules/security/framework_account_bridge.go @@ -0,0 +1,263 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import ( + "fmt" + "net/http" + "strings" + + rbac "infini.sh/console/core/security" + "infini.sh/console/modules/security/realm" + "infini.sh/framework/core/orm" + frameworksecurity "infini.sh/framework/core/security" + frameworkrbac "infini.sh/framework/modules/security/native" +) + +type frameworkNativeAccountProvider struct { + adapter rbac.Adapter +} + +var hasNonNativeRealm = realm.HasNonNativeRealm + +func (p frameworkNativeAccountProvider) GetUserByID(id string) (bool, *frameworksecurity.UserAccount, error) { + user, err := p.adapter.User.Get(id) + if err != nil { + return false, nil, err + } + if user.ID == "" || !user.IsEnabled() { + return false, nil, nil + } + return true, toFrameworkUserAccount(&user), nil +} + +func (p frameworkNativeAccountProvider) GetUserByLogin(login string) (bool, *frameworksecurity.UserAccount, error) { + user, err := p.adapter.User.GetBy("name", login) + if err != nil { + return false, nil, err + } + if user == nil || user.ID == "" { + if hasNonNativeRealm() { + return true, &frameworksecurity.UserAccount{ + Name: login, + }, nil + } + return false, nil, nil + } + if !user.IsEnabled() { + return false, nil, nil + } + return true, toFrameworkUserAccount(user), nil +} + +func (p frameworkNativeAccountProvider) CreateUser(name, login, password string, force bool) (*frameworksecurity.UserAccount, error) { + user := &rbac.User{ + Username: login, + Nickname: name, + Email: login, + } + material, err := frameworksecurity.GeneratePasswordMaterial(password) + if err != nil { + return nil, err + } + user.Password = material.Hash + user.PasswordSalt = material.Salt + user.PasswordVerifier = material.Verifier + + id, err := p.adapter.User.Create(user) + if err != nil { + return nil, err + } + user.ID = id + return toFrameworkUserAccount(user), nil +} + +type frameworkRealmPasswordLoginProvider struct{} + +func (frameworkRealmPasswordLoginProvider) AuthenticateByPassword(login, password string) (*frameworksecurity.UserSessionInfo, error) { + ok, user, err := realm.Authenticate(login, password) + if err != nil || !ok || user == nil { + return nil, nil + } + + ok, err = realm.Authorize(user) + if err != nil || !ok { + return nil, err + } + + sessionUser := &frameworksecurity.UserSessionInfo{ + Provider: normalizeFrameworkProvider(user.AuthProvider), + Login: user.Username, + Roles: roleNames(user.Roles), + } + sessionUser.SetUserID(user.ID) + _, privilege := user.GetPermissions() + sessionUser.UserAssignedPermission = frameworksecurity.NewUserAssignedPermission( + rbac.ExpandFrameworkPermissionKeysForPlatformPrivileges(privilege), + nil, + ) + return rbac.EnsureFrameworkDefaultPermissions(sessionUser), nil +} + +func registerFrameworkAccountBridge() { + adapter := rbac.GetAdapter("native") + frameworkrbac.RegisterPasswordChallengeUpgradePersister(func(ctx *orm.Context, user *frameworksecurity.UserAccount) error { + return persistFrameworkChallengeUpgrade(adapter, user) + }) + frameworksecurity.RegisterAuthenticationProvider("console-native-account-bridge", frameworkNativeAccountProvider{adapter: adapter}) + frameworksecurity.RegisterAccountPasswordLoginProvider("console-realm-password-login", frameworkRealmPasswordLoginProvider{}) + frameworksecurity.RegisterHTTPAuthFilterProviderWithPriority("console-bearer-token", func(_ http.ResponseWriter, r *http.Request) (*frameworksecurity.UserClaims, error) { + authorization := strings.TrimSpace(r.Header.Get("Authorization")) + if authorization == "" { + return nil, nil + } + + sessionUser, frameworkErr := frameworksecurity.ValidateAuthorizationHeader(authorization) + if frameworkErr == nil && sessionUser != nil && sessionUser.IsValid() { + if err := validateNativeAccountEnabled(adapter, sessionUser); err != nil { + return nil, err + } + bridgedClaims := frameworksecurity.NewUserClaims() + bridgedClaims.UserSessionInfo = sessionUser + return bridgedClaims, nil + } + + claims, err := rbac.ValidateLogin(authorization) + if err != nil || claims == nil { + return nil, err + } + if err := validateLegacyNativeAccountEnabled(adapter, claims); err != nil { + return nil, err + } + sessionUser = claims.ToSessionInfo() + if sessionUser == nil { + return nil, nil + } + sessionUser.Provider = normalizeFrameworkProvider(sessionUser.Provider) + rbac.EnsureFrameworkDefaultPermissions(sessionUser) + + bridgedClaims := frameworksecurity.NewUserClaims() + bridgedClaims.UserSessionInfo = sessionUser + if claims.RegisteredClaims != nil { + bridgedClaims.RegisteredClaims = claims.RegisteredClaims + } + return bridgedClaims, nil + }, 15) + frameworksecurity.RegisterSessionTokenResponseDecorator("console-platform-privilege", func(token map[string]interface{}, user *frameworksecurity.UserSessionInfo) { + if user == nil { + return + } + token["privilege"] = rbac.CombineUserRoles(user.Roles).Platform + }) +} + +func validateNativeAccountEnabled(adapter rbac.Adapter, sessionUser *frameworksecurity.UserSessionInfo) error { + if sessionUser == nil || !isNativeProvider(sessionUser.Provider) { + return nil + } + userID := strings.TrimSpace(sessionUser.UserID) + if userID == "" { + return fmt.Errorf("user id is empty") + } + user, err := adapter.User.Get(userID) + if err != nil { + return err + } + if !user.IsEnabled() { + return fmt.Errorf("user account [%s] is disabled", sessionUser.Login) + } + return nil +} + +func validateLegacyNativeAccountEnabled(adapter rbac.Adapter, claims *rbac.UserClaims) error { + if claims == nil || claims.ShortUser == nil || !isNativeProvider(claims.ShortUser.Provider) { + return nil + } + userID := strings.TrimSpace(claims.ShortUser.UserId) + if userID == "" { + return fmt.Errorf("user id is empty") + } + user, err := adapter.User.Get(userID) + if err != nil { + return err + } + if !user.IsEnabled() { + rbac.DeleteUserToken(userID) + return fmt.Errorf("user account [%s] is disabled", claims.ShortUser.Username) + } + return nil +} + +func isNativeProvider(provider string) bool { + normalized := strings.ToLower(strings.TrimSpace(provider)) + return normalized == "" || normalized == "native" || normalized == strings.ToLower(frameworksecurity.DefaultNativeAuthBackend) +} + +func persistFrameworkChallengeUpgrade(adapter rbac.Adapter, user *frameworksecurity.UserAccount) error { + if user == nil || user.ID == "" { + return nil + } + legacyUser, err := adapter.User.Get(user.ID) + if err != nil { + return err + } + if legacyUser.ID == "" { + return fmt.Errorf("legacy user [%s] not found", user.ID) + } + legacyUser.PasswordSalt = user.PasswordSalt + legacyUser.PasswordVerifier = user.PasswordVerifier + return adapter.User.Update(&legacyUser) +} + +func toFrameworkUserAccount(user *rbac.User) *frameworksecurity.UserAccount { + if user == nil { + return nil + } + + account := &frameworksecurity.UserAccount{ + Name: user.Username, + Email: user.Email, + Roles: roleNames(user.Roles), + Password: user.Password, + PasswordSalt: user.PasswordSalt, + PasswordVerifier: user.PasswordVerifier, + } + account.ID = user.ID + return account +} + +func roleNames(roles []rbac.UserRole) []string { + out := make([]string, 0, len(roles)) + for _, role := range roles { + out = append(out, role.Name) + } + return out +} + +func normalizeFrameworkProvider(provider string) string { + if strings.EqualFold(strings.TrimSpace(provider), "native") { + return frameworksecurity.DefaultNativeAuthBackend + } + return provider +} diff --git a/modules/security/framework_account_bridge_test.go b/modules/security/framework_account_bridge_test.go new file mode 100644 index 00000000..19609a4f --- /dev/null +++ b/modules/security/framework_account_bridge_test.go @@ -0,0 +1,331 @@ +package security + +import ( + "fmt" + "net" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + rbac "infini.sh/console/core/security" + api2 "infini.sh/framework/core/api" + "infini.sh/framework/core/config" + "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" + "infini.sh/framework/core/orm" + frameworksecurity "infini.sh/framework/core/security" + + "github.com/golang-jwt/jwt/v4" + + _ "infini.sh/console/modules/security/filter" + _ "infini.sh/framework/modules/security/http_filters" + "infini.sh/license" +) + +type bridgeTestMemoryKVStore struct { + values map[string][]byte +} + +func (m *bridgeTestMemoryKVStore) Open() error { return nil } + +func (m *bridgeTestMemoryKVStore) Close() error { return nil } + +func (m *bridgeTestMemoryKVStore) GetValue(bucket string, key []byte) ([]byte, error) { + value, ok := m.values[fmt.Sprintf("%s:%s", bucket, string(key))] + if !ok { + return nil, nil + } + return value, nil +} + +func (m *bridgeTestMemoryKVStore) GetCompressedValue(bucket string, key []byte) ([]byte, error) { + return m.GetValue(bucket, key) +} + +func (m *bridgeTestMemoryKVStore) AddValueCompress(bucket string, key []byte, value []byte) error { + return m.AddValue(bucket, key, value) +} + +func (m *bridgeTestMemoryKVStore) AddValueCompressWithTTL(bucket string, key []byte, value []byte, _ time.Duration) error { + return m.AddValue(bucket, key, value) +} + +func (m *bridgeTestMemoryKVStore) AddValue(bucket string, key []byte, value []byte) error { + m.values[fmt.Sprintf("%s:%s", bucket, string(key))] = value + return nil +} + +func (m *bridgeTestMemoryKVStore) AddValueWithTTL(bucket string, key []byte, value []byte, _ time.Duration) error { + return m.AddValue(bucket, key, value) +} + +func (m *bridgeTestMemoryKVStore) ExistsKey(bucket string, key []byte) (bool, error) { + _, ok := m.values[fmt.Sprintf("%s:%s", bucket, string(key))] + return ok, nil +} + +func (m *bridgeTestMemoryKVStore) DeleteKey(bucket string, key []byte) error { + delete(m.values, fmt.Sprintf("%s:%s", bucket, string(key))) + return nil +} + +var registerBridgeTestKVOnce sync.Once + +func ensureBridgeTestKVStore() { + registerBridgeTestKVOnce.Do(func() { + kv.Register("framework-account-bridge-test", &bridgeTestMemoryKVStore{ + values: map[string][]byte{}, + }) + }) +} + +func newBridgeTestBinding(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on random port: %v", err) + } + defer listener.Close() + + return listener.Addr().String() +} + +func TestFrameworkAccountBridgeAcceptsConsoleBearerToken(t *testing.T) { + registerFrameworkAccountBridge() + + req := httptest.NewRequest("GET", "/account/profile", nil) + req.Header.Set("Authorization", "Bearer "+issueConsoleBridgeTestToken(t, "bridge-user")) + resp := httptest.NewRecorder() + + sessionUser, err := frameworksecurity.ValidateLogin(resp, req) + if err != nil { + t.Fatalf("expected console bearer token to validate through framework bridge, got %v", err) + } + if sessionUser == nil || sessionUser.UserID != "bridge-user" { + t.Fatalf("unexpected bridged session user: %#v", sessionUser) + } + if sessionUser.Provider != frameworksecurity.DefaultNativeAuthBackend { + t.Fatalf("expected bridged provider %q, got %q", frameworksecurity.DefaultNativeAuthBackend, sessionUser.Provider) + } +} + +func TestFrameworkLicenseUIRouteAcceptsConsoleAdminToken(t *testing.T) { + registerFrameworkAccountBridge() + license.Init() + + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newBridgeTestBinding(t) + + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + req := httptest.NewRequest(http.MethodGet, "/_license/info", nil) + req.Header.Set("Authorization", "Bearer "+issueConsoleBridgeTestToken(t, "bridge-admin")) + resp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve framework license ui route: %v", err) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected framework /_license/info ui route to accept console admin token, got %d", resp.Code) + } +} + +func TestFrameworkLicenseUIRouteAcceptsConsoleReadonlyToken(t *testing.T) { + registerFrameworkAccountBridge() + license.Init() + + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newBridgeTestBinding(t) + + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + req := httptest.NewRequest(http.MethodGet, "/_license/info", nil) + req.Header.Set("Authorization", "Bearer "+issueConsoleBridgeTestToken(t, "bridge-readonly", "readonly")) + resp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve framework license ui route: %v", err) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected framework /_license/info ui route to accept console readonly token, got %d", resp.Code) + } +} + +func issueConsoleBridgeTestToken(t *testing.T, userID string, roles ...string) string { + t.Helper() + + ensureBridgeTestKVStore() + + expireAt := time.Now().Add(time.Hour) + if len(roles) == 0 { + roles = []string{"Administrator"} + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, rbac.UserClaims{ + ShortUser: &rbac.ShortUser{ + Provider: "native", + Username: "tester", + UserId: userID, + Roles: roles, + }, + RegisteredClaims: &jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(expireAt), + }, + }) + + tokenString, err := token.SignedString([]byte(rbac.Secret)) + if err != nil { + t.Fatalf("sign token: %v", err) + } + + rbac.SetUserToken(userID, rbac.Token{ + Value: tokenString, + ExpireIn: expireAt.Unix(), + }) + t.Cleanup(func() { + rbac.DeleteUserToken(userID) + }) + + return tokenString +} + +type bridgeTestUserStore struct { + items map[string]rbac.User +} + +func (s *bridgeTestUserStore) Get(id string) (rbac.User, error) { + if user, ok := s.items[id]; ok { + return user, nil + } + return rbac.User{}, nil +} + +func (s *bridgeTestUserStore) GetBy(field string, value interface{}) (*rbac.User, error) { + return nil, nil +} + +func (s *bridgeTestUserStore) Update(user *rbac.User) error { + if user == nil { + return nil + } + s.items[user.ID] = *user + return nil +} + +func (s *bridgeTestUserStore) Create(user *rbac.User) (string, error) { + return "", nil +} + +func (s *bridgeTestUserStore) Delete(id string) error { + return nil +} + +func (s *bridgeTestUserStore) Search(keyword string, from, size int) (orm.Result, error) { + return orm.Result{}, nil +} + +func TestPersistFrameworkChallengeUpgradeUpdatesLegacyUserCredentials(t *testing.T) { + store := &bridgeTestUserStore{ + items: map[string]rbac.User{ + "default_user_admin": { + ORMObjectBase: orm.ORMObjectBase{ID: "default_user_admin"}, + Password: "legacy-hash", + }, + }, + } + adapter := rbac.Adapter{User: store} + user := &frameworksecurity.UserAccount{ + PasswordSalt: "new-salt", + PasswordVerifier: "new-verifier", + } + user.ID = "default_user_admin" + + if err := persistFrameworkChallengeUpgrade(adapter, user); err != nil { + t.Fatalf("expected challenge upgrade persistence to succeed, got %v", err) + } + updated := store.items["default_user_admin"] + if updated.PasswordSalt != "new-salt" { + t.Fatalf("expected password_salt to be updated, got %q", updated.PasswordSalt) + } + if updated.PasswordVerifier != "new-verifier" { + t.Fatalf("expected password_verifier to be updated, got %q", updated.PasswordVerifier) + } +} + +func TestPersistFrameworkChallengeUpgradeSkipsMissingUserID(t *testing.T) { + store := &bridgeTestUserStore{items: map[string]rbac.User{}} + adapter := rbac.Adapter{User: store} + + if err := persistFrameworkChallengeUpgrade(adapter, &frameworksecurity.UserAccount{}); err != nil { + t.Fatalf("expected empty user id to be ignored, got %v", err) + } +} + +func TestFrameworkNativeAccountProviderFallsBackToPlainForExternalRealm(t *testing.T) { + originalHasNonNativeRealm := hasNonNativeRealm + hasNonNativeRealm = func() bool { return true } + t.Cleanup(func() { + hasNonNativeRealm = originalHasNonNativeRealm + }) + + store := &bridgeTestUserStore{items: map[string]rbac.User{}} + provider := frameworkNativeAccountProvider{ + adapter: rbac.Adapter{User: store}, + } + + exists, account, err := provider.GetUserByLogin("ldap-user") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if !exists { + t.Fatal("expected external realm fallback account to exist") + } + if account == nil || account.Name != "ldap-user" { + t.Fatalf("unexpected fallback account: %#v", account) + } + if account.ID != "" { + t.Fatalf("expected fallback account id to be empty, got %q", account.ID) + } +} + +func TestFrameworkNativeAccountProviderNoFallbackWithoutExternalRealm(t *testing.T) { + originalHasNonNativeRealm := hasNonNativeRealm + hasNonNativeRealm = func() bool { return false } + t.Cleanup(func() { + hasNonNativeRealm = originalHasNonNativeRealm + }) + + store := &bridgeTestUserStore{items: map[string]rbac.User{}} + provider := frameworkNativeAccountProvider{ + adapter: rbac.Adapter{User: store}, + } + + exists, account, err := provider.GetUserByLogin("unknown-user") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if exists { + t.Fatal("expected unknown user without external realm fallback to not exist") + } + if account != nil { + t.Fatalf("expected nil account, got %#v", account) + } +} diff --git a/modules/security/license_trial_bridge.go b/modules/security/license_trial_bridge.go new file mode 100644 index 00000000..ddc8cfde --- /dev/null +++ b/modules/security/license_trial_bridge.go @@ -0,0 +1,72 @@ +package security + +import ( + "fmt" + "net/http" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" + "infini.sh/license" +) + +var requestConsoleTrialLicenseRemotely = func(url string, body []byte) (*util.Result, error) { + request := util.NewPostRequest(url, body).AddCommonJSONHeaders() + return util.ExecuteRequest(request) +} + +type consoleLicenseTrialBridge struct { + api.Handler +} + +func registerConsoleLicenseTrialBridge() { + handler := consoleLicenseTrialBridge{} + api.HandleUIMethod(api.POST, "/_license/request_trial", handler.RequestTrialLicense, api.RequireLogin()) +} + +func (handler *consoleLicenseTrialBridge) RequestTrialLicense(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + body, err := handler.GetRawBody(req) + if err != nil { + handler.Error500(w, err.Error()) + return + } + + requestBody := license.TrialRequest{} + if err := util.FromJSONBytes(body, &requestBody); err != nil { + handler.Error500(w, err.Error()) + return + } + + remoteURL := "https://api.infini.cloud/_license/request_trial" + if rawQuery := req.URL.RawQuery; rawQuery != "" { + remoteURL = fmt.Sprintf("%s?%s", remoteURL, rawQuery) + } + + response, err := requestConsoleTrialLicenseRemotely(remoteURL, util.MustToJSONBytes(requestBody)) + if err != nil { + statusCode := http.StatusBadGateway + if response != nil && response.StatusCode > 0 { + statusCode = response.StatusCode + } + handler.WriteError(w, err.Error(), statusCode) + return + } + + result := license.TrialResponse{} + if err := util.FromJSONBytes(response.Body, &result); err != nil { + handler.Error500(w, err.Error()) + return + } + + if result.License != "" { + ok := license.ApplyLicense(result.License) + if ok { + license.PersistLicense(result.License) + } else { + result.License = "" + } + } + + w.WriteHeader(response.StatusCode) + _, _ = w.Write(util.MustToJSONBytes(result)) +} diff --git a/modules/security/license_trial_bridge_test.go b/modules/security/license_trial_bridge_test.go new file mode 100644 index 00000000..04ef0559 --- /dev/null +++ b/modules/security/license_trial_bridge_test.go @@ -0,0 +1,60 @@ +package security + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + api2 "infini.sh/framework/core/api" + "infini.sh/framework/core/config" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" +) + +func TestConsoleTrialLicenseRouteAcceptsConsoleTokenAndForwardsQuery(t *testing.T) { + registerFrameworkAccountBridge() + registerConsoleLicenseTrialBridge() + + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + originalRemote := requestConsoleTrialLicenseRemotely + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + requestConsoleTrialLicenseRemotely = originalRemote + }) + + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + + var requestedURL string + requestConsoleTrialLicenseRemotely = func(url string, body []byte) (*util.Result, error) { + requestedURL = url + if !bytes.Contains(body, []byte(`"product":"console"`)) { + t.Fatalf("expected forwarded body to keep product field, got %s", string(body)) + } + return &util.Result{ + StatusCode: http.StatusOK, + Body: []byte(`{"acknowledged":true}`), + }, nil + } + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newBridgeTestBinding(t) + + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + req := httptest.NewRequest(http.MethodPost, "/_license/request_trial?lang=zh-CN", bytes.NewBufferString(`{"product":"console","email":"user@example.org"}`)) + req.Header.Set("Authorization", "Bearer "+issueConsoleBridgeTestToken(t, "bridge-admin")) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve console trial route: %v", err) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected console trial route to return 200, got %d: %s", resp.Code, resp.Body.String()) + } + if requestedURL != "https://api.infini.cloud/_license/request_trial?lang=zh-CN" { + t.Fatalf("expected lang query to be forwarded, got %s", requestedURL) + } +} diff --git a/modules/security/module.go b/modules/security/module.go index acc3f5a6..04a3dfb3 100644 --- a/modules/security/module.go +++ b/modules/security/module.go @@ -32,12 +32,14 @@ import ( authapi "infini.sh/console/modules/security/api" "infini.sh/console/modules/security/config" credapi "infini.sh/console/modules/security/credential/api" + _ "infini.sh/console/modules/security/filter" "infini.sh/console/modules/security/realm" "infini.sh/console/modules/security/realm/authc/oauth" "infini.sh/framework/core/credential" "infini.sh/framework/core/env" "infini.sh/framework/core/global" "infini.sh/framework/core/orm" + _ "infini.sh/framework/modules/security/access_token" ) type Module struct { @@ -95,6 +97,8 @@ func (module *Module) Start() error { } realm.Init(module.cfg) + registerFrameworkAccountBridge() + registerConsoleLicenseTrialBridge() return nil } diff --git a/modules/security/realm/authc/native/init.go b/modules/security/realm/authc/native/init.go index 496b8cc2..d5a911ac 100644 --- a/modules/security/realm/authc/native/init.go +++ b/modules/security/realm/authc/native/init.go @@ -67,6 +67,9 @@ func (r *NativeRealm) Authenticate(username, password string) (bool, *rbac.User, if user == nil { return false, nil, fmt.Errorf("user account [%s] not found", username) } + if !user.IsEnabled() { + return false, nil, fmt.Errorf("user account [%s] is disabled", username) + } err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(password)) if err == bcrypt.ErrMismatchedHashAndPassword { diff --git a/modules/security/realm/authc/native/load.go b/modules/security/realm/authc/native/load.go index f57336e1..4070e240 100644 --- a/modules/security/realm/authc/native/load.go +++ b/modules/security/realm/authc/native/load.go @@ -31,6 +31,7 @@ import ( _ "embed" "github.com/mitchellh/mapstructure" "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" "path" "strings" @@ -128,6 +129,10 @@ func loadRemoteRolePermission() { } log.Debug("load security permissions,", rbac.RoleMap, rbac.BuiltinRoles) + if !orm.HasHandler() { + log.Warn("skip loading remote roles, ORM handler is not registered") + return + } res, err := handler.Role.Search("", 0, 1000) if err != nil { diff --git a/modules/security/realm/authc/native/load_test.go b/modules/security/realm/authc/native/load_test.go new file mode 100644 index 00000000..df49c5dc --- /dev/null +++ b/modules/security/realm/authc/native/load_test.go @@ -0,0 +1,42 @@ +package native + +import ( + "testing" + + rbac "infini.sh/console/core/security" + "infini.sh/framework/core/orm" +) + +func cloneRoleMap(src map[string]rbac.Role) map[string]rbac.Role { + dst := make(map[string]rbac.Role, len(src)) + for k, v := range src { + dst[k] = v + } + return dst +} + +func TestLoadRemoteRolePermissionSkipsWhenORMUnavailable(t *testing.T) { + if orm.HasHandler() { + t.Skip("test requires ORM handler to be unregistered") + } + + previousRoleMap := cloneRoleMap(rbac.RoleMap) + defer func() { + rbac.RoleMap = previousRoleMap + }() + + loadRemoteRolePermission() + + if len(rbac.RoleMap) != len(rbac.BuiltinRoles) { + t.Fatalf("expected only builtin roles to be loaded, got %d roles", len(rbac.RoleMap)) + } + for name, role := range rbac.BuiltinRoles { + got, ok := rbac.RoleMap[name] + if !ok { + t.Fatalf("expected builtin role %q to be present", name) + } + if got.Name != role.Name { + t.Fatalf("expected builtin role %q, got %q", role.Name, got.Name) + } + } +} diff --git a/modules/security/realm/authc/native/permission.json b/modules/security/realm/authc/native/permission.json index a74e3087..33af9ac6 100644 --- a/modules/security/realm/authc/native/permission.json +++ b/modules/security/realm/authc/native/permission.json @@ -397,9 +397,6 @@ {"name": "indices.field_caps", "methods":["get", "post"], "path": "/:index_name/_field_caps" }, - {"name": "indices.exists_template", "methods":["head"], - "path": "/_template/:template_name" - }, {"name": "indices.field_usage_stats", "methods":["get"], "path": "/:index_name/_field_usage_stats" }, @@ -1022,4 +1019,4 @@ "path": "/_license" } ] -} \ No newline at end of file +} diff --git a/modules/security/realm/authc/native/role.go b/modules/security/realm/authc/native/role.go index cd652fae..7e6e6c65 100644 --- a/modules/security/realm/authc/native/role.go +++ b/modules/security/realm/authc/native/role.go @@ -47,7 +47,7 @@ func (dal *Role) Get(id string) (rbac.Role, error) { role := rbac.Role{} role.ID = id - _, err := orm.Get(&role) + _, err := orm.GetV2(orm.NewContext(), &role) return role, err } @@ -68,18 +68,18 @@ func (dal *Role) GetBy(field string, value interface{}) (rbac.Role, error) { } func (dal *Role) Update(role *rbac.Role) error { - return orm.Save(nil, role) + return orm.Save(orm.NewContext(), role) } func (dal *Role) Create(role *rbac.Role) (string, error) { role.ID = util.GetUUID() - return role.ID, orm.Save(nil, role) + return role.ID, orm.Save(orm.NewContext(), role) } func (dal *Role) Delete(id string) error { role := rbac.Role{} role.ID = id - return orm.Delete(nil, role) + return orm.Delete(orm.NewContext(), &role) } func (dal *Role) Search(keyword string, from, size int) (orm.Result, error) { diff --git a/modules/security/realm/authc/native/user.go b/modules/security/realm/authc/native/user.go index 94047e87..06ffeae8 100644 --- a/modules/security/realm/authc/native/user.go +++ b/modules/security/realm/authc/native/user.go @@ -41,13 +41,13 @@ type User struct { func (dal *User) Get(id string) (rbac.User, error) { user := rbac.User{} user.ID = id - _, err := orm.Get(&user) + _, err := orm.GetV2(orm.NewContext(), &user) return user, err } func (dal *User) GetBy(field string, value interface{}) (*rbac.User, error) { user := &rbac.User{} - err, result := orm.GetBy(field, value, rbac.User{}) + err, result := orm.GetBy(field, value, user) if err != nil { return nil, err } @@ -64,18 +64,18 @@ func (dal *User) GetBy(field string, value interface{}) (*rbac.User, error) { func (dal *User) Update(user *rbac.User) error { - return orm.Update(nil, user) + return orm.Update(orm.NewContext(), user) } func (dal *User) Create(user *rbac.User) (string, error) { user.ID = util.GetUUID() - return user.ID, orm.Save(nil, user) + return user.ID, orm.Save(orm.NewContext(), user) } func (dal *User) Delete(id string) error { user := rbac.User{} user.ID = id - return orm.Delete(nil, user) + return orm.Delete(orm.NewContext(), &user) } func (dal *User) Search(keyword string, from, size int) (orm.Result, error) { diff --git a/modules/security/realm/realm.go b/modules/security/realm/realm.go index cd5da3e9..3f9bd317 100644 --- a/modules/security/realm/realm.go +++ b/modules/security/realm/realm.go @@ -117,3 +117,12 @@ func Authorize(user *rbac.User) (bool, error) { return false, errors.Errorf("failed to authorize user: %v", user.Username) } + +func HasNonNativeRealm() bool { + for _, realm := range realms { + if realm.GetType() != "native" { + return true + } + } + return false +} diff --git a/plugin/api/alerting/alert.go b/plugin/api/alerting/alert.go index c9f3e61e..885bc5f6 100644 --- a/plugin/api/alerting/alert.go +++ b/plugin/api/alerting/alert.go @@ -66,6 +66,18 @@ func (h *AlertAPI) getAlert(w http.ResponseWriter, req *http.Request, ps httprou return } + if source, ok := parseAlertSearchResult(result.Result[0]); ok { + doc := util.MapStr{} + util.FromJSONBytes(util.MustToJSONBytes(source), &doc) + doc["display_state"] = getAlertDisplayState(&source) + h.WriteJSON(w, util.MapStr{ + "found": true, + "_id": id, + "_source": doc, + }, 200) + return + } + h.WriteJSON(w, util.MapStr{ "found": true, "_id": id, @@ -84,15 +96,35 @@ func (h *AlertAPI) searchAlert(w http.ResponseWriter, req *http.Request, ps http priority = h.GetParameterOrDefault(req, "priority", "") sort = h.GetParameterOrDefault(req, "sort", "") ruleID = h.GetParameterOrDefault(req, "rule_id", "") + resourceID = h.GetParameterOrDefault(req, "resource_id", "") min = h.GetParameterOrDefault(req, "min", "") max = h.GetParameterOrDefault(req, "max", "") mustBuilder = &strings.Builder{} sortBuilder = strings.Builder{} ) - mustBuilder.WriteString(fmt.Sprintf(`{"range":{"created":{"gte":"%s", "lte": "%s"}}}`, min, max)) + timeRange := util.MapStr{} + if minValue, ok := normalizeTimeBound(min); ok { + timeRange["gte"] = minValue + } + if maxValue, ok := normalizeTimeBound(max); ok { + timeRange["lte"] = maxValue + } + if len(timeRange) > 0 { + timeFilter := util.MapStr{ + "range": util.MapStr{ + "created": timeRange, + }, + } + mustBuilder.Write(util.MustToJSONBytes(timeFilter)) + } else { + mustBuilder.WriteString(`{"match_all":{}}`) + } if ruleID != "" { mustBuilder.WriteString(fmt.Sprintf(`,{"term":{"rule_id":{"value":"%s"}}}`, ruleID)) } + if resourceID != "" { + mustBuilder.WriteString(fmt.Sprintf(`,{"term":{"resource_id":{"value":"%s"}}}`, resourceID)) + } if sort != "" { sortParts := strings.Split(sort, ",") @@ -137,6 +169,34 @@ func (h *AlertAPI) searchAlert(w http.ResponseWriter, req *http.Request, ps http h.Write(w, res.Raw) } +func getAlertDisplayState(alertItem *alerting.Alert) string { + if alertItem == nil { + return "" + } + if alertItem.State == alerting.AlertStateOK && len(alertItem.RecoverActionResults) > 0 { + return alerting.MessageStateRecovered + } + return alertItem.State +} + +func parseAlertSearchResult(item interface{}) (alerting.Alert, bool) { + switch value := item.(type) { + case alerting.Alert: + return value, true + case *alerting.Alert: + if value == nil { + return alerting.Alert{}, false + } + return *value, true + default: + alertItem := alerting.Alert{} + if err := util.FromJSONBytes(util.MustToJSONBytes(value), &alertItem); err != nil { + return alerting.Alert{}, false + } + return alertItem, true + } +} + func (h *AlertAPI) getAlertStats(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDsl := util.MapStr{ diff --git a/plugin/api/alerting/api.go b/plugin/api/alerting/api.go index 40f18734..4ff7f533 100644 --- a/plugin/api/alerting/api.go +++ b/plugin/api/alerting/api.go @@ -44,12 +44,14 @@ func (alert *AlertAPI) Init() { api.HandleAPIMethod(api.DELETE, "/alerting/rule/:rule_id", alert.RequirePermission(alert.deleteRule, enum.PermissionAlertRuleWrite)) api.HandleAPIMethod(api.DELETE, "/alerting/rule", alert.RequirePermission(alert.batchDeleteRule, enum.PermissionAlertRuleWrite)) api.HandleAPIMethod(api.PUT, "/alerting/rule/:rule_id", alert.RequirePermission(alert.updateRule, enum.PermissionAlertRuleWrite)) + api.HandleAPIMethod(api.POST, "/alerting/rule/:rule_id/_sync_template", alert.RequirePermission(alert.syncRuleTemplate, enum.PermissionAlertRuleWrite)) api.HandleAPIMethod(api.GET, "/alerting/rule/_search", alert.RequirePermission(alert.searchRule, enum.PermissionAlertRuleRead)) api.HandleAPIMethod(api.GET, "/alerting/stats", alert.RequirePermission(alert.getAlertStats, enum.PermissionAlertHistoryRead)) api.HandleAPIMethod(api.POST, "/alerting/rule/info", alert.RequirePermission(alert.fetchAlertInfos, enum.PermissionAlertHistoryRead)) api.HandleAPIMethod(api.POST, "/alerting/rule/preview_metric", alert.RequireLogin(alert.getPreviewMetricData)) api.HandleAPIMethod(api.POST, "/alerting/rule/:rule_id/_enable", alert.RequirePermission(alert.enableRule, enum.PermissionAlertRuleWrite)) api.HandleAPIMethod(api.GET, "/alerting/rule/:rule_id/metric", alert.RequirePermission(alert.getMetricData, enum.PermissionAlertRuleRead)) + api.HandleAPIMethod(api.GET, "/alerting/rule/:rule_id/history_metric", alert.RequirePermission(alert.getHistoryMetricData, enum.PermissionAlertHistoryRead)) api.HandleAPIMethod(api.GET, "/alerting/rule/:rule_id/info", alert.RequirePermission(alert.getRuleDetail, enum.PermissionAlertRuleRead, enum.PermissionAlertMessageRead)) api.HandleAPIMethod(api.POST, "/alerting/rule/_enable", alert.RequirePermission(alert.batchEnableRule, enum.PermissionAlertRuleWrite)) api.HandleAPIMethod(api.POST, "/alerting/rule/_disable", alert.RequirePermission(alert.batchDisableRule, enum.PermissionAlertRuleWrite)) @@ -66,13 +68,13 @@ func (alert *AlertAPI) Init() { api.HandleAPIMethod(api.GET, "/alerting/alert/_search", alert.RequirePermission(alert.searchAlert, enum.PermissionAlertHistoryRead)) api.HandleAPIMethod(api.GET, "/alerting/alert/:alert_id", alert.RequirePermission(alert.getAlert, enum.PermissionAlertHistoryRead)) - api.HandleAPIMethod(api.GET, "/alerting/template/parameters", alert.getTemplateParams) + api.HandleAPIMethod(api.GET, "/alerting/template/parameters", alert.RequirePermission(alert.getTemplateParams, enum.PermissionAlertRuleRead)) api.HandleAPIMethod(api.GET, "/alerting/message/_search", alert.RequirePermission(alert.searchAlertMessage, enum.PermissionAlertMessageRead)) api.HandleAPIMethod(api.POST, "/alerting/message/_ignore", alert.RequirePermission(alert.ignoreAlertMessage, enum.PermissionAlertMessageWrite)) api.HandleAPIMethod(api.GET, "/alerting/message/_stats", alert.RequirePermission(alert.getAlertMessageStats, enum.PermissionAlertMessageRead)) api.HandleAPIMethod(api.GET, "/alerting/message/:message_id", alert.RequirePermission(alert.getAlertMessage, enum.PermissionAlertMessageRead)) - api.HandleAPIMethod(api.GET, "/alerting/message/:message_id/notification", alert.getMessageNotificationInfo) + api.HandleAPIMethod(api.GET, "/alerting/message/:message_id/notification", alert.RequirePermission(alert.getMessageNotificationInfo, enum.PermissionAlertMessageRead)) //just for test //api.HandleAPIMethod(api.GET, "/alerting/rule/test", alert.testRule) diff --git a/plugin/api/alerting/channel.go b/plugin/api/alerting/channel.go index f7b52e00..4555a269 100644 --- a/plugin/api/alerting/channel.go +++ b/plugin/api/alerting/channel.go @@ -45,6 +45,51 @@ import ( "infini.sh/framework/core/util" ) +func buildChannelSort(sort string) []util.MapStr { + appendSort := func(sorters []util.MapStr, seen map[string]struct{}, field, order string) []util.MapStr { + if field == "" { + return sorters + } + if _, ok := seen[field]; ok { + return sorters + } + if order == "" { + order = "asc" + } + seen[field] = struct{}{} + return append(sorters, util.MapStr{ + field: util.MapStr{ + "order": order, + }, + }) + } + + seen := map[string]struct{}{} + sorters := make([]util.MapStr, 0, 4) + sortParts := strings.Split(sort, ":") + sortField := sortParts[0] + sortDirection := "" + if len(sortParts) >= 2 { + sortDirection = sortParts[1] + } + if sortField != "" { + sorters = appendSort(sorters, seen, sortField, sortDirection) + } + + for _, stableSort := range []struct { + field string + order string + }{ + {field: "sub_type", order: "asc"}, + {field: "type", order: "asc"}, + {field: "name", order: "asc"}, + {field: "updated", order: "desc"}, + } { + sorters = appendSort(sorters, seen, stableSort.field, stableSort.order) + } + return sorters +} + func (h *AlertAPI) createChannel(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var obj = &alerting.Channel{} err := h.DecodeJSON(req, obj) @@ -54,7 +99,7 @@ func (h *AlertAPI) createChannel(w http.ResponseWriter, req *http.Request, ps ht return } - err = orm.Create(nil, obj) + err = orm.Create(orm.NewContext(), obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -71,7 +116,7 @@ func (h *AlertAPI) getChannel(w http.ResponseWriter, req *http.Request, ps httpr obj := alerting.Channel{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -93,7 +138,7 @@ func (h *AlertAPI) updateChannel(w http.ResponseWriter, req *http.Request, ps ht obj := alerting.Channel{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -115,7 +160,7 @@ func (h *AlertAPI) updateChannel(w http.ResponseWriter, req *http.Request, ps ht //protect obj.ID = id obj.Created = create - err = orm.Update(nil, &obj) + err = orm.Update(orm.NewContext(), &obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -214,7 +259,7 @@ func (h *AlertAPI) searchChannel(w http.ResponseWriter, req *http.Request, ps ht strFrom = h.GetParameterOrDefault(req, "from", "0") subType = h.GetParameterOrDefault(req, "sub_type", "") typ = h.GetParameterOrDefault(req, "type", "") - sort = h.GetParameterOrDefault(req, "sort", "updated:desc") + sort = h.GetParameterOrDefault(req, "sort", "") ) mustQ := []interface{}{} if keyword != "" { @@ -248,18 +293,6 @@ func (h *AlertAPI) searchChannel(w http.ResponseWriter, req *http.Request, ps ht if from < 0 { from = 0 } - var ( - sortField string - sortDirection string - ) - sortParts := strings.Split(sort, ":") - sortField = sortParts[0] - if len(sortParts) >= 2 { - sortDirection = sortParts[1] - } - if sortDirection == "" { - sortDirection = "asc" - } query := util.MapStr{ "size": size, "from": from, @@ -268,13 +301,7 @@ func (h *AlertAPI) searchChannel(w http.ResponseWriter, req *http.Request, ps ht "must": mustQ, }, }, - "sort": []util.MapStr{ - { - sortField: util.MapStr{ - "order": sortDirection, - }, - }, - }, + "sort": buildChannelSort(sort), } q := orm.Query{ @@ -353,6 +380,12 @@ func (alertAPI *AlertAPI) batchEnableChannel(w http.ResponseWriter, req *http.Re return } if len(channelIDs) > 0 { + err = validateChannelsBeforeEnable(channelIDs) + if err != nil { + log.Error(err) + alertAPI.WriteError(w, err.Error(), http.StatusBadRequest) + return + } err = setChannelEnabled(true, channelIDs) if err != nil { log.Error(err) @@ -400,3 +433,51 @@ func setChannelEnabled(enabled bool, channelIDs []string) error { err := orm.UpdateBy(alerting.Channel{}, util.MustToJSONBytes(q)) return err } + +func validateChannelsBeforeEnable(channelIDs []string) error { + for _, id := range channelIDs { + channel := alerting.Channel{} + channel.ID = id + exists, err := orm.GetV2(orm.NewContext(), &channel) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("channel [%s] not found", id) + } + if err = validateChannelBeforeEnable(&channel); err != nil { + return err + } + } + return nil +} + +func validateChannelBeforeEnable(channel *alerting.Channel) error { + if channel == nil { + return fmt.Errorf("empty channel") + } + + channelType := channel.SubType + if channelType == "" { + channelType = channel.Type + } + if channelType != alerting.ChannelEmail { + return nil + } + + if channel.Email == nil { + return fmt.Errorf("email channel [%s] is incomplete, please configure the smtp server and recipients first", channel.Name) + } + + if channel.Email.ServerID == "" && len(channel.Email.Recipients.To) == 0 { + return fmt.Errorf("email channel [%s] is incomplete, please configure the smtp server and recipients first", channel.Name) + } + if channel.Email.ServerID == "" { + return fmt.Errorf("email channel [%s] is incomplete, please configure the smtp server first", channel.Name) + } + if len(channel.Email.Recipients.To) == 0 { + return fmt.Errorf("email channel [%s] is incomplete, please configure at least one recipient first", channel.Name) + } + + return nil +} diff --git a/plugin/api/alerting/channel_test.go b/plugin/api/alerting/channel_test.go new file mode 100644 index 00000000..cd9f0f24 --- /dev/null +++ b/plugin/api/alerting/channel_test.go @@ -0,0 +1,87 @@ +package alerting + +import ( + "reflect" + "strings" + "testing" + + modelalerting "infini.sh/console/model/alerting" + "infini.sh/framework/core/util" +) + +func TestValidateChannelBeforeEnableRequiresEmailConfig(t *testing.T) { + channel := &modelalerting.Channel{ + Name: "Email Channel", + Type: modelalerting.ChannelEmail, + SubType: modelalerting.ChannelEmail, + } + + err := validateChannelBeforeEnable(channel) + if err == nil { + t.Fatal("expected incomplete email channel to be rejected") + } + if !strings.Contains(err.Error(), "smtp server and recipients") { + t.Fatalf("expected combined config hint, got %v", err) + } +} + +func TestValidateChannelBeforeEnableRequiresRecipients(t *testing.T) { + channel := &modelalerting.Channel{ + Name: "Email Channel", + Type: modelalerting.ChannelEmail, + SubType: modelalerting.ChannelEmail, + Email: &modelalerting.Email{ + ServerID: "smtp-1", + }, + } + + err := validateChannelBeforeEnable(channel) + if err == nil { + t.Fatal("expected missing recipients to be rejected") + } + if !strings.Contains(err.Error(), "at least one recipient") { + t.Fatalf("expected recipient hint, got %v", err) + } +} + +func TestValidateChannelBeforeEnableAcceptsCompleteEmailChannel(t *testing.T) { + channel := &modelalerting.Channel{ + Name: "Email Channel", + Type: modelalerting.ChannelEmail, + SubType: modelalerting.ChannelEmail, + Email: &modelalerting.Email{ + ServerID: "smtp-1", + }, + } + channel.Email.Recipients.To = []string{"ops@example.com"} + + if err := validateChannelBeforeEnable(channel); err != nil { + t.Fatalf("expected complete email channel to pass validation, got %v", err) + } +} + +func TestBuildChannelSortUsesStableChannelOrderByDefault(t *testing.T) { + got := buildChannelSort("") + want := []util.MapStr{ + {"sub_type": util.MapStr{"order": "asc"}}, + {"type": util.MapStr{"order": "asc"}}, + {"name": util.MapStr{"order": "asc"}}, + {"updated": util.MapStr{"order": "desc"}}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("expected stable default sort %#v, got %#v", want, got) + } +} + +func TestBuildChannelSortKeepsRequestedFieldAndStableFallback(t *testing.T) { + got := buildChannelSort("updated:asc") + want := []util.MapStr{ + {"updated": util.MapStr{"order": "asc"}}, + {"sub_type": util.MapStr{"order": "asc"}}, + {"type": util.MapStr{"order": "asc"}}, + {"name": util.MapStr{"order": "asc"}}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("expected requested sort with stable fallback %#v, got %#v", want, got) + } +} diff --git a/plugin/api/alerting/message.go b/plugin/api/alerting/message.go index fe755304..be51f707 100644 --- a/plugin/api/alerting/message.go +++ b/plugin/api/alerting/message.go @@ -277,11 +277,11 @@ func (h *AlertAPI) searchAlertMessage(w http.ResponseWriter, req *http.Request, tags = h.GetParameterOrDefault(req, "tags", "") ) timeRange := util.MapStr{} - if min != "" { - timeRange["gte"] = min + if minValue, ok := normalizeTimeBound(min); ok { + timeRange["gte"] = minValue } - if max != "" { - timeRange["lte"] = max + if maxValue, ok := normalizeTimeBound(max); ok { + timeRange["lte"] = maxValue } if len(timeRange) > 0 { timeFilter := util.MapStr{ @@ -358,16 +358,31 @@ func (h *AlertAPI) searchAlertMessage(w http.ResponseWriter, req *http.Request, return } for _, hit := range esRes.Hits.Hits { - created, _ := parseTime(hit.Source["created"], time.RFC3339) - updated, _ := parseTime(hit.Source["updated"], time.RFC3339) - if !created.IsZero() && !updated.IsZero() { - endTime := time.Now() - if hit.Source["status"] == alerting.MessageStateRecovered { - endTime = updated + alertMessage := &alerting.AlertMessage{} + if err := util.FromJSONBytes(util.MustToJSONBytes(hit.Source), alertMessage); err != nil { + continue + } + incident := resolveAlertMessageIncident(alertMessage) + startAt := getIncidentStartTime(alertMessage, incident) + if !startAt.IsZero() { + hit.Source["trigger_at"] = startAt + } + hit.Source["updated"] = resolveAlertDisplayUpdatedTime(alertMessage, incident, startAt) + endTime := time.Now() + if alertMessage.Status == alerting.MessageStateRecovered { + if recoveredAt := getRecoveredAt(alertMessage); !recoveredAt.IsZero() { + endTime = recoveredAt + } else if !alertMessage.Updated.IsZero() { + endTime = alertMessage.Updated } - hit.Source["duration"] = endTime.Sub(created).Milliseconds() } - + if !startAt.IsZero() { + duration := endTime.Sub(startAt).Milliseconds() + if duration < 0 { + duration = 0 + } + hit.Source["duration"] = duration + } } h.WriteJSON(w, esRes, http.StatusOK) } @@ -381,11 +396,56 @@ func parseTime(t interface{}, layout string) (time.Time, error) { } } +func getRecoveredAt(message *alerting.AlertMessage) time.Time { + if message == nil { + return time.Time{} + } + if !message.RecoveredAt.IsZero() { + return message.RecoveredAt + } + if message.Status == alerting.MessageStateRecovered && !message.Updated.IsZero() { + return message.Updated + } + return time.Time{} +} + +func resolveAlertDisplayUpdatedTime(message *alerting.AlertMessage, incident alertMessageIncident, triggerAt time.Time) time.Time { + if message == nil { + if triggerAt.IsZero() { + return time.Time{} + } + return triggerAt + } + updatedAt := message.Updated + if !incident.LatestAt.IsZero() && (updatedAt.IsZero() || incident.LatestAt.After(updatedAt)) { + updatedAt = incident.LatestAt + } + if message.Status == alerting.MessageStateAlerting { + if !triggerAt.IsZero() && (updatedAt.IsZero() || updatedAt.Before(triggerAt)) { + return triggerAt + } + } + return updatedAt +} + +func getIncidentStartTime(message *alerting.AlertMessage, incident alertMessageIncident) time.Time { + if message != nil && !message.Created.IsZero() { + return message.Created + } + if !incident.TriggerAt.IsZero() { + return incident.TriggerAt + } + if message == nil { + return time.Time{} + } + return message.Created +} + func (h *AlertAPI) getAlertMessage(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { message := &alerting.AlertMessage{ ID: ps.ByName("message_id"), } - exists, err := orm.Get(message) + exists, err := orm.GetV2(orm.NewContext(), message) if !exists || err != nil { log.Error(err) h.WriteJSON(w, util.MapStr{ @@ -397,7 +457,7 @@ func (h *AlertAPI) getAlertMessage(w http.ResponseWriter, req *http.Request, ps rule := &alerting.Rule{ ID: message.RuleID, } - exists, err = orm.Get(rule) + exists, err = orm.GetV2(orm.NewContext(), rule) if !exists || err != nil { log.Error(err) h.WriteError(w, fmt.Sprintf("rule [%s] not found", rule.ID), http.StatusInternalServerError) @@ -422,14 +482,27 @@ func (h *AlertAPI) getAlertMessage(w http.ResponseWriter, req *http.Request, ps } conditions.Items[i].Expression = strings.ReplaceAll(expression, "result", metricExpression) } - var duration time.Duration - if message.Status == alerting.MessageStateRecovered { - duration = message.Updated.Sub(message.Created) - } else { - duration = time.Now().Sub(message.Created) + incident := resolveAlertMessageIncident(message) + resolveAt := getRecoveredAt(message) + + triggerAt := message.Created + if triggerAt.IsZero() { + triggerAt = getIncidentStartTime(message, incident) + } + if resolveAt.IsZero() && !incident.ResolveAt.IsZero() { + resolveAt = incident.ResolveAt + } + endAt := time.Now() + if message.Status == alerting.MessageStateRecovered && !resolveAt.IsZero() { + endAt = resolveAt + } + duration := endAt.Sub(triggerAt) + if duration < 0 { + duration = 0 } detailObj := util.MapStr{ "message_id": message.ID, + "event_id": message.ID, "rule_id": message.RuleID, "rule_name": rule.Name, "rule_enabled": rule.Enabled, @@ -437,9 +510,14 @@ func (h *AlertAPI) getAlertMessage(w http.ResponseWriter, req *http.Request, ps "message": message.Message, "priority": message.Priority, "created": message.Created, - "updated": message.Updated, - "resource_name": rule.Resource.Name, - "resource_id": rule.Resource.ID, + "updated": resolveAlertDisplayUpdatedTime(message, incident, triggerAt), + "recovered_at": message.RecoveredAt, + "trigger_at": triggerAt, + "resolve_at": resolveAt, + "trigger_event_id": incident.TriggerEventID, + "resolve_event_id": incident.ResolveEventID, + "resource_name": firstNonEmptyString(message.ResourceName, rule.Resource.Name), + "resource_id": firstNonEmptyString(message.ResourceID, rule.Resource.ID), "resource_objects": rule.Resource.Objects, "conditions": rule.Conditions, "bucket_conditions": rule.BucketConditions, @@ -455,11 +533,129 @@ func (h *AlertAPI) getAlertMessage(w http.ResponseWriter, req *http.Request, ps h.WriteJSON(w, detailObj, http.StatusOK) } +type alertMessageIncident struct { + TriggerEventID string + ResolveEventID string + TriggerAt time.Time + ResolveAt time.Time + LatestAt time.Time +} + +func resolveAlertMessageIncident(message *alerting.AlertMessage) alertMessageIncident { + if message == nil || message.RuleID == "" || message.Created.IsZero() { + return alertMessageIncident{} + } + + endTime := time.Now() + if resolvedAt := getRecoveredAt(message); message.Status == alerting.MessageStateRecovered && !resolvedAt.IsZero() { + endTime = resolvedAt + } + + must := []util.MapStr{ + { + "term": util.MapStr{ + "rule_id": util.MapStr{ + "value": message.RuleID, + }, + }, + }, + { + "range": util.MapStr{ + "created": util.MapStr{ + "gte": message.Created.Add(-1 * time.Second).UnixMilli(), + "lte": endTime.Add(1 * time.Second).UnixMilli(), + }, + }, + }, + } + if message.ResourceID != "" { + must = append(must, util.MapStr{ + "term": util.MapStr{ + "resource_id": util.MapStr{ + "value": message.ResourceID, + }, + }, + }) + } + + q := orm.Query{ + RawQuery: util.MustToJSONBytes(util.MapStr{ + "size": 100, + "sort": []util.MapStr{ + { + "created": util.MapStr{ + "order": "asc", + }, + }, + }, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": must, + }, + }, + }), + } + + err, result := orm.Search(alerting.Alert{}, &q) + if err != nil || len(result.Result) == 0 { + return alertMessageIncident{} + } + + alerts := make([]alerting.Alert, 0, len(result.Result)) + for _, item := range result.Result { + alertItem, ok := parseAlertSearchResult(item) + if !ok { + continue + } + alerts = append(alerts, alertItem) + } + + return buildAlertMessageIncident(message, alerts) +} + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func buildAlertMessageIncident(message *alerting.AlertMessage, alerts []alerting.Alert) alertMessageIncident { + incident := alertMessageIncident{} + for _, alertItem := range alerts { + if incident.LatestAt.IsZero() || alertItem.Created.After(incident.LatestAt) { + incident.LatestAt = alertItem.Created + } + if alertItem.State == alerting.AlertStateAlerting { + if incident.TriggerEventID == "" || alertItem.Created.Before(incident.TriggerAt) { + incident.TriggerEventID = alertItem.ID + incident.TriggerAt = alertItem.Created + } + } + if message != nil && message.Status == alerting.MessageStateRecovered && getAlertDisplayState(&alertItem) == alerting.MessageStateRecovered { + incident.ResolveEventID = alertItem.ID + incident.ResolveAt = alertItem.Updated + if incident.ResolveAt.IsZero() { + incident.ResolveAt = alertItem.Created + } + } + } + + if incident.TriggerEventID == "" && len(alerts) > 0 { + incident.TriggerEventID = alerts[0].ID + incident.TriggerAt = alerts[0].Created + } + + return incident +} + func (h *AlertAPI) getMessageNotificationInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { message := &alerting.AlertMessage{ ID: ps.ByName("message_id"), } - exists, err := orm.Get(message) + exists, err := orm.GetV2(orm.NewContext(), message) if !exists || err != nil { log.Error(err) h.WriteJSON(w, util.MapStr{ @@ -471,7 +667,7 @@ func (h *AlertAPI) getMessageNotificationInfo(w http.ResponseWriter, req *http.R rule := &alerting.Rule{ ID: message.RuleID, } - exists, err = orm.Get(rule) + exists, err = orm.GetV2(orm.NewContext(), rule) if !exists || err != nil { log.Error(err) h.WriteError(w, fmt.Sprintf("rule [%s] not found", rule.ID), http.StatusInternalServerError) @@ -511,8 +707,8 @@ func getMessageNotificationStats(msg *alerting.AlertMessage) (util.MapStr, error rangeQ := util.MapStr{ "gte": msg.Created.UnixMilli(), } - if msg.Status == alerting.MessageStateRecovered { - rangeQ["lte"] = msg.Updated.UnixMilli() + if resolvedAt := getRecoveredAt(msg); msg.Status == alerting.MessageStateRecovered && !resolvedAt.IsZero() { + rangeQ["lte"] = resolvedAt.UnixMilli() } aggs := util.MapStr{ "grp_normal_channel": util.MapStr{ diff --git a/plugin/api/alerting/message_test.go b/plugin/api/alerting/message_test.go new file mode 100644 index 00000000..48b02a04 --- /dev/null +++ b/plugin/api/alerting/message_test.go @@ -0,0 +1,116 @@ +package alerting + +import ( + "testing" + "time" + + alertmodel "infini.sh/console/model/alerting" +) + +func TestGetAlertDisplayState(t *testing.T) { + alertItem := &alertmodel.Alert{State: alertmodel.AlertStateOK} + if got := getAlertDisplayState(alertItem); got != alertmodel.AlertStateOK { + t.Fatalf("expected ok, got %s", got) + } + + alertItem.RecoverActionResults = []alertmodel.ActionExecutionResult{{ChannelType: "webhook"}} + if got := getAlertDisplayState(alertItem); got != alertmodel.MessageStateRecovered { + t.Fatalf("expected recovered, got %s", got) + } +} + +func TestBuildAlertMessageIncident(t *testing.T) { + start := time.Unix(100, 0) + resolve := start.Add(5 * time.Minute) + message := &alertmodel.AlertMessage{ + Status: alertmodel.MessageStateRecovered, + } + alerts := []alertmodel.Alert{ + {ID: "alert-1", State: alertmodel.AlertStateAlerting, Created: start}, + {ID: "alert-1b", State: alertmodel.AlertStateAlerting, Created: start.Add(2 * time.Minute)}, + {ID: "alert-2", State: alertmodel.AlertStateOK, Created: resolve}, + { + ID: "alert-3", + State: alertmodel.AlertStateOK, + Created: start, + Updated: resolve, + RecoverActionResults: []alertmodel.ActionExecutionResult{{ChannelType: "webhook"}}, + }, + } + + incident := buildAlertMessageIncident(message, alerts) + if incident.TriggerEventID != "alert-1" { + t.Fatalf("expected trigger alert-1, got %s", incident.TriggerEventID) + } + if !incident.TriggerAt.Equal(start) { + t.Fatalf("expected trigger time %v, got %v", start, incident.TriggerAt) + } + if !incident.LatestAt.Equal(resolve) { + t.Fatalf("expected latest incident time %v, got %v", resolve, incident.LatestAt) + } + if incident.ResolveEventID != "alert-3" { + t.Fatalf("expected resolve alert-3, got %s", incident.ResolveEventID) + } + if !incident.ResolveAt.Equal(resolve) { + t.Fatalf("expected resolve time %v, got %v", resolve, incident.ResolveAt) + } +} + +func TestGetRecoveredAtPrefersRecoveredAt(t *testing.T) { + recoveredAt := time.Unix(200, 0) + updatedAt := recoveredAt.Add(-3 * time.Minute) + message := &alertmodel.AlertMessage{ + Status: alertmodel.MessageStateRecovered, + Updated: updatedAt, + RecoveredAt: recoveredAt, + } + + got := getRecoveredAt(message) + if !got.Equal(recoveredAt) { + t.Fatalf("expected recovered_at %v, got %v", recoveredAt, got) + } +} + +func TestResolveAlertDisplayUpdatedTimeForAlerting(t *testing.T) { + triggerAt := time.Unix(300, 0) + updatedAt := triggerAt.Add(-5 * time.Minute) + message := &alertmodel.AlertMessage{ + Status: alertmodel.MessageStateAlerting, + Updated: updatedAt, + } + + got := resolveAlertDisplayUpdatedTime(message, alertMessageIncident{}, triggerAt) + if !got.Equal(triggerAt) { + t.Fatalf("expected updated time to fallback to trigger_at %v, got %v", triggerAt, got) + } +} + +func TestResolveAlertDisplayUpdatedTimeForRecovered(t *testing.T) { + triggerAt := time.Unix(300, 0) + updatedAt := triggerAt.Add(-5 * time.Minute) + message := &alertmodel.AlertMessage{ + Status: alertmodel.MessageStateRecovered, + Updated: updatedAt, + } + + got := resolveAlertDisplayUpdatedTime(message, alertMessageIncident{}, triggerAt) + if !got.Equal(updatedAt) { + t.Fatalf("expected recovered message to keep updated time %v, got %v", updatedAt, got) + } +} + +func TestResolveAlertDisplayUpdatedTimeUsesIncidentLatestAt(t *testing.T) { + triggerAt := time.Unix(300, 0) + message := &alertmodel.AlertMessage{ + Status: alertmodel.MessageStateAlerting, + Updated: time.Unix(290, 0), + } + incident := alertMessageIncident{ + LatestAt: time.Unix(360, 0), + } + + got := resolveAlertDisplayUpdatedTime(message, incident, triggerAt) + if !got.Equal(incident.LatestAt) { + t.Fatalf("expected updated time to use incident latest_at %v, got %v", incident.LatestAt, got) + } +} diff --git a/plugin/api/alerting/rule.go b/plugin/api/alerting/rule.go index ad84ffe2..2e8d4ba3 100644 --- a/plugin/api/alerting/rule.go +++ b/plugin/api/alerting/rule.go @@ -55,6 +55,149 @@ import ( "infini.sh/framework/modules/elastic/common" ) +const ( + defaultRuleNotificationTitle = "🔥 [{{.rule_name}}] Alerting" + defaultRuleNotificationMessage = `- Priority:{{.priority}} +- EventID: {{.event_id}} +- Target: {{.resource_name}}-{{.objects}} +- TriggerAt: {{.trigger_at | datetime}} +{{range .results}} +Group:{{index .group_values 0}}; Value:{{.result_value}}; +{{end}}` + defaultRuleRecoveryTitle = "🌈 [{{.rule_name}}] Resolved" + defaultRuleRecoveryMessage = `- EventID: {{.event_id}} +- Target: {{.resource_name}}-{{.objects}} +- TriggerAt: {{.trigger_at | datetime}} +- ResolveAt: {{.timestamp | datetime}} +- Duration: {{.duration}}` +) + +func ensureNotificationConfig(rule *alerting.Rule) *alerting.NotificationConfig { + if rule.NotificationConfig != nil { + return rule.NotificationConfig + } + if rule.Channels != nil { + rule.NotificationConfig = rule.Channels + return rule.NotificationConfig + } + rule.NotificationConfig = &alerting.NotificationConfig{} + return rule.NotificationConfig +} + +func applyBuiltinRuleTemplates(rule *alerting.Rule) { + notificationConfig := ensureNotificationConfig(rule) + notificationConfig.Title = defaultRuleNotificationTitle + notificationConfig.Message = defaultRuleNotificationMessage + rule.Metrics.Title = defaultRuleNotificationTitle + rule.Metrics.Message = defaultRuleNotificationMessage + + if rule.RecoveryNotificationConfig != nil { + rule.RecoveryNotificationConfig.Title = defaultRuleRecoveryTitle + rule.RecoveryNotificationConfig.Message = defaultRuleRecoveryMessage + } +} + +func normalizeRuleForSave(rule *alerting.Rule) error { + var err error + rule.Metrics.Expression, err = rule.Metrics.GenerateExpression() + if err != nil { + return err + } + + var groups []insight.MetricGroupItem + for _, grp := range rule.Metrics.Groups { + if grp.Field != "" { + groups = append(groups, grp) + } + } + rule.Metrics.Groups = groups + + return nil +} + +func ensureRuleTimestamps(rule *alerting.Rule, now time.Time) bool { + changed := false + if rule.Created.IsZero() { + rule.Created = now + changed = true + } + if rule.Updated.IsZero() || rule.Updated.Before(rule.Created) { + rule.Updated = now + changed = true + } + return changed +} + +func backfillRuleTimestamps(rule *alerting.Rule) { + if rule == nil { + return + } + if !ensureRuleTimestamps(rule, time.Now()) { + return + } + ctx := orm.NewContext() + ctx.Set(orm.CheckExistsBeforeUpdate, false) + ctx.Set(orm.MergePartialFieldsBeforeUpdate, false) + if err := orm.Save(ctx, rule); err != nil { + log.Errorf("failed to backfill rule timestamps for [%s]: %v", rule.ID, err) + } +} + +func persistUpdatedRule(oldRule, rule *alerting.Rule) error { + changeLog, err := util.DiffTwoObject(oldRule, rule) + if err != nil { + log.Error(err) + } + + now := time.Now() + rule.ID = oldRule.ID + rule.Created = oldRule.Created + rule.Updated = now + ensureRuleTimestamps(rule, now) + + if err := normalizeRuleForSave(rule); err != nil { + return err + } + + ctx := &orm.Context{Refresh: orm.WaitForRefresh} + ctx.Set(orm.CheckExistsBeforeUpdate, false) + ctx.Set(orm.MergePartialFieldsBeforeUpdate, false) + if err := orm.Save(ctx, rule); err != nil { + return err + } + saveAlertActivity("alerting_rule_change", "update", util.MapStr{ + "cluster_id": rule.Resource.ID, + "rule_id": rule.ID, + "rule_name": rule.Name, + "cluster_name": rule.Resource.Name, + }, changeLog, oldRule) + + if rule.Enabled { + exists, err := checkResourceExists(rule) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("resource [%s] not found", rule.Resource.ID) + } + task.StopTask(rule.ID) + clearKV(rule.ID) + eng := alerting2.GetEngine(rule.Resource.Type) + ruleTask := task.ScheduleTask{ + ID: rule.ID, + Interval: rule.Schedule.Interval, + Description: rule.Metrics.Expression, + Task: eng.GenerateTask(*rule), + } + task.RegisterScheduleTask(ruleTask) + task.StartTask(ruleTask.ID) + } else { + task.DeleteTask(rule.ID) + } + + return nil +} + func (alertAPI *AlertAPI) createRule(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { rules := []alerting.Rule{} err := alertAPI.DecodeJSON(req, &rules) @@ -109,7 +252,7 @@ func (alertAPI *AlertAPI) createRule(w http.ResponseWriter, req *http.Request, p rule.Creator.Id = user.UserId } - err = orm.Save(nil, &rule) + err = orm.Save(orm.NewContext(), &rule) if err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -147,7 +290,7 @@ func (alertAPI *AlertAPI) getRule(w http.ResponseWriter, req *http.Request, ps h obj := alerting.Rule{} obj.ID = id - _, err := orm.Get(&obj) + _, err := orm.GetV2(orm.NewContext(), &obj) if err != nil { if errors.Is(err, elastic2.ErrNotFound) { alertAPI.WriteJSON(w, util.MapStr{ @@ -160,6 +303,7 @@ func (alertAPI *AlertAPI) getRule(w http.ResponseWriter, req *http.Request, ps h alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) return } + backfillRuleTimestamps(&obj) // adapter version smaller than 1.6.0 if obj.Channels != nil && obj.NotificationConfig == nil { obj.NotificationConfig = obj.Channels @@ -188,7 +332,7 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request obj := alerting.Rule{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { if errors.Is(err, elastic2.ErrNotFound) { alertAPI.WriteJSON(w, util.MapStr{ @@ -201,6 +345,7 @@ func (alertAPI *AlertAPI) getRuleDetail(w http.ResponseWriter, req *http.Request alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) return } + backfillRuleTimestamps(&obj) metricExpression, _ := obj.Metrics.GenerateExpression() conditions := obj.Conditions if obj.BucketConditions != nil { @@ -401,7 +546,7 @@ func (alertAPI *AlertAPI) updateRule(w http.ResponseWriter, req *http.Request, p oldRule := &alerting.Rule{} oldRule.ID = id - exists, err := orm.Get(oldRule) + exists, err := orm.GetV2(orm.NewContext(), oldRule) if !exists || err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -420,68 +565,44 @@ func (alertAPI *AlertAPI) updateRule(w http.ResponseWriter, req *http.Request, p log.Error(err) return } - rule.Metrics.Expression, err = rule.Metrics.GenerateExpression() + rule.ID = id + rule.Created = create + err = persistUpdatedRule(oldRule, rule) if err != nil { alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) return } - changeLog, err := util.DiffTwoObject(oldRule, rule) - if err != nil { - log.Error(err) - } - //protect - rule.ID = id - rule.Created = create - rule.Updated = time.Now() + alertAPI.WriteJSON(w, util.MapStr{ + "_id": rule.ID, + "result": "updated", + }, 200) +} - //filter empty metric group - var groups []insight.MetricGroupItem - for _, grp := range rule.Metrics.Groups { - if grp.Field != "" { - groups = append(groups, grp) - } +func (alertAPI *AlertAPI) syncRuleTemplate(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("rule_id") + oldRule := &alerting.Rule{ID: id} + + exists, err := orm.GetV2(orm.NewContext(), oldRule) + if !exists || err != nil { + log.Error(err) + alertAPI.WriteJSON(w, util.MapStr{ + "_id": id, + "result": "not_found", + }, http.StatusNotFound) + return } - rule.Metrics.Groups = groups - err = orm.Save(nil, rule) + rule := *oldRule + applyBuiltinRuleTemplates(&rule) + + err = persistUpdatedRule(oldRule, &rule) if err != nil { alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) return } - saveAlertActivity("alerting_rule_change", "update", util.MapStr{ - "cluster_id": rule.Resource.ID, - "rule_id": rule.ID, - "rule_name": rule.Name, - "cluster_name": rule.Resource.Name, - }, changeLog, oldRule) - - if rule.Enabled { - exists, err = checkResourceExists(rule) - if err != nil || !exists { - log.Error(err) - alertAPI.WriteJSON(w, util.MapStr{ - "error": err.Error(), - }, http.StatusInternalServerError) - return - } - //update task - task.StopTask(id) - clearKV(rule.ID) - eng := alerting2.GetEngine(rule.Resource.Type) - ruleTask := task.ScheduleTask{ - ID: rule.ID, - Interval: rule.Schedule.Interval, - Description: rule.Metrics.Expression, - Task: eng.GenerateTask(*rule), - } - task.RegisterScheduleTask(ruleTask) - task.StartTask(ruleTask.ID) - } else { - task.DeleteTask(id) - } alertAPI.WriteJSON(w, util.MapStr{ "_id": rule.ID, @@ -501,7 +622,7 @@ func (alertAPI *AlertAPI) deleteRule(w http.ResponseWriter, req *http.Request, p obj := alerting.Rule{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -511,7 +632,7 @@ func (alertAPI *AlertAPI) deleteRule(w http.ResponseWriter, req *http.Request, p return } - err = orm.Delete(nil, &obj) + err = orm.Delete(orm.NewContext(), &obj) if err != nil { alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -736,17 +857,21 @@ func (alertAPI *AlertAPI) getRuleAlertMessageNumbers(ruleIDs []string) (map[stri return ruleAlertNumbers, nil } -func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var ruleIDs = []string{} - alertAPI.DecodeJSON(req, &ruleIDs) +type alertInfoSearchFieldConfig struct { + RuleIDField string + StateField string +} - if len(ruleIDs) == 0 { - alertAPI.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) +var alertInfoSearchFieldConfigs = []alertInfoSearchFieldConfig{ + { + RuleIDField: "rule_id", + StateField: "state", + }, +} + +func searchLatestAlertsByField(esClient elastic.API, ruleIDs []string, fieldConfig alertInfoSearchFieldConfig, sourceFields []string, alertState string) (*elastic.SearchResponse, error) { queryDsl := util.MapStr{ - "_source": []string{"state", "rule_id"}, + "_source": sourceFields, "sort": []util.MapStr{ { "created": util.MapStr{ @@ -755,80 +880,168 @@ func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Reque }, }, "collapse": util.MapStr{ - "field": "rule_id", - }, - "query": util.MapStr{ - "terms": util.MapStr{ - "rule_id": ruleIDs, - }, + "field": fieldConfig.RuleIDField, }, } - searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl)) - if err != nil { - log.Error(err) - alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) - return - } - if len(searchRes.Hits.Hits) == 0 { - alertAPI.WriteJSON(w, util.MapStr{}, http.StatusOK) - return - } - - latestAlertInfos := map[string]util.MapStr{} - for _, hit := range searchRes.Hits.Hits { - if ruleID, ok := hit.Source["rule_id"].(string); ok { - latestAlertInfos[ruleID] = util.MapStr{ - "status": hit.Source["state"], - } - } - } - queryDsl = util.MapStr{ - "_source": []string{"created", "rule_id"}, - "sort": []util.MapStr{ - { - "created": util.MapStr{ - "order": "desc", - }, + if alertState == "" { + queryDsl["query"] = util.MapStr{ + "terms": util.MapStr{ + fieldConfig.RuleIDField: ruleIDs, }, - }, - "collapse": util.MapStr{ - "field": "rule_id", - }, - "query": util.MapStr{ + } + } else { + queryDsl["query"] = util.MapStr{ "bool": util.MapStr{ "must": []util.MapStr{ { "terms": util.MapStr{ - "rule_id": ruleIDs, + fieldConfig.RuleIDField: ruleIDs, }, }, { "term": util.MapStr{ - "state": util.MapStr{ - "value": alerting.AlertStateAlerting, + fieldConfig.StateField: util.MapStr{ + "value": alertState, }, }, }, }, }, - }, + } } - searchRes, err = esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl)) - if err != nil { - log.Error(err) - alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) - return + + return esClient.SearchWithRawQueryDSL(orm.GetWildcardIndexName(alerting.Alert{}), util.MustToJSONBytes(queryDsl)) +} + +func getRemainingRuleIDs(ruleIDs []string, found map[string]struct{}) []string { + remaining := make([]string, 0, len(ruleIDs)) + for _, ruleID := range ruleIDs { + if _, ok := found[ruleID]; ok { + continue + } + remaining = append(remaining, ruleID) } - for _, hit := range searchRes.Hits.Hits { - if ruleID, ok := hit.Source["rule_id"].(string); ok { - if _, ok = latestAlertInfos[ruleID]; ok { - latestAlertInfos[ruleID]["last_notification_time"] = hit.Source["created"] + return remaining +} + +func fetchLatestRuleStatuses(esClient elastic.API, ruleIDs []string) (map[string]util.MapStr, []string) { + results := map[string]util.MapStr{} + found := map[string]struct{}{} + remaining := append([]string(nil), ruleIDs...) + attempts := make([]string, 0, len(alertInfoSearchFieldConfigs)) + + for _, fieldConfig := range alertInfoSearchFieldConfigs { + if len(remaining) == 0 { + break + } + + searchRes, err := searchLatestAlertsByField(esClient, remaining, fieldConfig, []string{"state", "rule_id"}, "") + if err != nil { + attempts = append(attempts, fmt.Sprintf("%s query failed: %v", fieldConfig.RuleIDField, err)) + continue + } + if len(searchRes.Hits.Hits) == 0 { + attempts = append(attempts, fmt.Sprintf("%s query returned no hits", fieldConfig.RuleIDField)) + continue + } + + matched := 0 + for _, hit := range searchRes.Hits.Hits { + ruleID, ok := hit.Source["rule_id"].(string) + if !ok || ruleID == "" { + continue + } + results[ruleID] = util.MapStr{ + "status": hit.Source["state"], + } + found[ruleID] = struct{}{} + matched++ + } + attempts = append(attempts, fmt.Sprintf("%s query matched %d rule(s)", fieldConfig.RuleIDField, matched)) + remaining = getRemainingRuleIDs(ruleIDs, found) + } + + return results, attempts +} + +func fetchLatestRuleNotificationTimes(esClient elastic.API, ruleIDs []string) (map[string]interface{}, []string) { + results := map[string]interface{}{} + found := map[string]struct{}{} + remaining := append([]string(nil), ruleIDs...) + attempts := make([]string, 0, len(alertInfoSearchFieldConfigs)) + + for _, fieldConfig := range alertInfoSearchFieldConfigs { + if len(remaining) == 0 { + break + } + + searchRes, err := searchLatestAlertsByField(esClient, remaining, fieldConfig, []string{"created", "rule_id"}, alerting.AlertStateAlerting) + if err != nil { + attempts = append(attempts, fmt.Sprintf("%s notification query failed: %v", fieldConfig.RuleIDField, err)) + continue + } + if len(searchRes.Hits.Hits) == 0 { + attempts = append(attempts, fmt.Sprintf("%s notification query returned no hits", fieldConfig.RuleIDField)) + continue + } + + matched := 0 + for _, hit := range searchRes.Hits.Hits { + ruleID, ok := hit.Source["rule_id"].(string) + if !ok || ruleID == "" { + continue } + results[ruleID] = hit.Source["created"] + found[ruleID] = struct{}{} + matched++ } + attempts = append(attempts, fmt.Sprintf("%s notification query matched %d rule(s)", fieldConfig.RuleIDField, matched)) + remaining = getRemainingRuleIDs(ruleIDs, found) + } + return results, attempts +} + +func buildRuleStatusUnavailableReason(statusAttempts []string) string { + if len(statusAttempts) == 0 { + return "latest alert status unavailable: no alert history found for this rule yet" } - alertAPI.WriteJSON(w, latestAlertInfos, http.StatusOK) + return fmt.Sprintf( + "latest alert status unavailable: no matching alert history found for this rule yet (attempts: %s)", + strings.Join(statusAttempts, "; "), + ) +} + +func (alertAPI *AlertAPI) fetchAlertInfos(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var ruleIDs = []string{} + alertAPI.DecodeJSON(req, &ruleIDs) + + if len(ruleIDs) == 0 { + alertAPI.WriteJSON(w, util.MapStr{}, http.StatusOK) + return + } + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + latestAlertInfos, statusAttempts := fetchLatestRuleStatuses(esClient, ruleIDs) + lastNotificationTimes, _ := fetchLatestRuleNotificationTimes(esClient, ruleIDs) + + response := make(util.MapStr, len(ruleIDs)) + for _, ruleID := range ruleIDs { + info := util.MapStr{} + if latestInfo, ok := latestAlertInfos[ruleID]; ok { + for k, v := range latestInfo { + info[k] = v + } + } + if lastNotificationTime, ok := lastNotificationTimes[ruleID]; ok { + info["last_notification_time"] = lastNotificationTime + } + if _, ok := info["status"]; !ok { + info["status_error"] = buildRuleStatusUnavailableReason(statusAttempts) + } + response[ruleID] = info + } + alertAPI.WriteJSON(w, response, http.StatusOK) } func (alertAPI *AlertAPI) enableRule(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -843,7 +1056,7 @@ func (alertAPI *AlertAPI) enableRule(w http.ResponseWriter, req *http.Request, p obj := alerting.Rule{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { log.Error(err) alertAPI.WriteJSON(w, util.MapStr{ @@ -858,7 +1071,8 @@ func (alertAPI *AlertAPI) enableRule(w http.ResponseWriter, req *http.Request, p disableRule(&obj) } obj.Enabled = reqObj.Enabled - err = orm.Save(nil, &obj) + ensureRuleTimestamps(&obj, time.Now()) + err = orm.Save(orm.NewContext(), &obj) if err != nil { log.Error(err) alertAPI.WriteError(w, fmt.Sprintf("save rule error:%v", err), http.StatusInternalServerError) @@ -924,7 +1138,7 @@ func checkResourceExists(rule *alerting.Rule) (bool, error) { case "elasticsearch": obj := elastic.ElasticsearchConfig{} obj.ID = rule.Resource.ID - ok, err := orm.Get(&obj) + ok, err := orm.GetV2(orm.NewContext(), &obj) if err != nil { return false, err } @@ -989,11 +1203,160 @@ func (alertAPI *AlertAPI) getPreviewMetricData(w http.ResponseWriter, req *http. }, http.StatusOK) } +// getHistoryMetricData builds a time-series chart from alert-history records instead of source metric +// data. This works for both non-time-series (state snapshot) rules and time-series rules whose +// source data may have been removed by ILM. +func (alertAPI *AlertAPI) getHistoryMetricData(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + ruleID := ps.ByName("rule_id") + rule := &alerting.Rule{ID: ruleID} + exists, err := orm.GetV2(orm.NewContext(), rule) + if !exists || err != nil { + alertAPI.WriteJSON(w, util.MapStr{"_id": ruleID, "found": false}, http.StatusNotFound) + return + } + minStr := alertAPI.Get(req, "min", "") + maxStr := alertAPI.Get(req, "max", "") + _, min, max, err := api.GetMetricRangeAndBucketSize(minStr, maxStr, 60, 15) + if err != nil { + log.Error(err) + alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + metricItem, err := buildMetricFromAlertHistory(rule, min, max) + if err != nil { + log.Error(err) + alertAPI.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + indexName, queryDsl := buildAlertHistoryQuery(rule.ID, min, max) + alertAPI.WriteJSON(w, util.MapStr{ + "metric": metricItem, + "bucket_label": rule.Metrics.BucketLabel, + "request": util.MapStr{ + "index": indexName, + "query": queryDsl, + }, + }, http.StatusOK) +} + +// buildMetricFromAlertHistory queries the alert-history index for a rule and assembles +// a MetricItem whose lines are [timestamp_ms, value] pairs taken from condition_result. +func buildMetricFromAlertHistory(rule *alerting.Rule, minMs, maxMs int64) (*alerting.AlertMetricItem, error) { + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + indexName, queryDsl := buildAlertHistoryQuery(rule.ID, minMs, maxMs) + searchRes, err := esClient.SearchWithRawQueryDSL( + indexName, + util.MustToJSONBytes(queryDsl), + ) + if err != nil { + return nil, err + } + + type dataPoint = [2]interface{} + groupData := map[string][]dataPoint{} + groupOrder := []string{} + groupSeen := map[string]struct{}{} + + for _, hit := range searchRes.Hits.Hits { + // parse timestamp from "created" field + createdStr, _ := hit.Source["created"].(string) + if createdStr == "" { + continue + } + t, err := time.Parse(time.RFC3339Nano, createdStr) + if err != nil { + t, err = time.Parse(time.RFC3339, createdStr) + if err != nil { + continue + } + } + ts := t.UnixNano() / 1e6 + + // condition_result is stored as object (enabled:false), read from _source map + condResultRaw, ok := hit.Source["condition_result"] + if !ok || condResultRaw == nil { + continue + } + condResultBytes, err := util.ToJSONBytes(condResultRaw) + if err != nil { + continue + } + var condResult alerting.ConditionResult + if err = util.FromJSONBytes(condResultBytes, &condResult); err != nil { + continue + } + + for _, item := range condResult.ResultItems { + label := strings.Join(item.GroupValues, "-") + if label == "" { + label, _ = rule.GetOrInitExpression() + } + if _, seen := groupSeen[label]; !seen { + groupSeen[label] = struct{}{} + groupOrder = append(groupOrder, label) + } + val, _ := util.ExtractFloat(item.ResultValue) + groupData[label] = append(groupData[label], dataPoint{ts, val}) + } + } + + formatType := "num" + if rule.Metrics.FormatType != "" { + formatType = rule.Metrics.FormatType + } + metricItem := &alerting.AlertMetricItem{ + MetricItem: common.MetricItem{ + Group: rule.ID, + Key: rule.ID, + Axis: []*common.MetricAxis{ + { + ID: util.GetUUID(), Group: rule.ID, FormatType: formatType, + Position: "left", ShowGridLines: true, TickFormat: "0,0.[00]", Ticks: 5, + }, + }, + }, + } + for _, label := range groupOrder { + metricItem.BucketGroups = append(metricItem.BucketGroups, []string{label}) + metricItem.Lines = append(metricItem.Lines, &common.MetricLine{ + Data: groupData[label], + Metric: common.MetricSummary{ + Label: label, + Group: rule.ID, + TickFormat: "0,0.[00]", + FormatType: formatType, + }, + }) + } + return metricItem, nil +} + +func buildAlertHistoryQuery(ruleID string, minMs, maxMs int64) (string, util.MapStr) { + queryDsl := util.MapStr{ + "size": 1000, + "_source": []string{"created", "condition_result", "state"}, + "sort": []util.MapStr{{"created": util.MapStr{"order": "asc"}}}, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + {"term": util.MapStr{"rule_id": util.MapStr{"value": ruleID}}}, + {"range": util.MapStr{"created": util.MapStr{ + "gte": minMs, + "lte": maxMs, + "format": "epoch_millis", + }}}, + }, + }, + }, + } + return orm.GetWildcardIndexName(alerting.Alert{}), queryDsl +} + func (alertAPI *AlertAPI) getMetricData(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { rule := &alerting.Rule{ ID: ps.ByName("rule_id"), } - exists, err := orm.Get(rule) + exists, err := orm.GetV2(orm.NewContext(), rule) if !exists || err != nil { alertAPI.WriteJSON(w, util.MapStr{ "_id": rule.ID, diff --git a/plugin/api/alerting/time_range.go b/plugin/api/alerting/time_range.go new file mode 100644 index 00000000..aae6e0b7 --- /dev/null +++ b/plugin/api/alerting/time_range.go @@ -0,0 +1,14 @@ +package alerting + +import "strings" + +func normalizeTimeBound(value string) (string, bool) { + v := strings.TrimSpace(value) + if v == "" { + return "", false + } + if strings.EqualFold(v, "auto") { + return "", false + } + return v, true +} diff --git a/plugin/api/data/export.go b/plugin/api/data/export.go index 44dec3e4..22ef9dbd 100644 --- a/plugin/api/data/export.go +++ b/plugin/api/data/export.go @@ -7,6 +7,7 @@ package data import ( "fmt" "net/http" + "time" log "github.com/cihub/seelog" "infini.sh/console/model" @@ -80,19 +81,25 @@ func indexExportData(eds []ExportData, patch bool) error { } buf := util.MustToJSONBytes(row) err := util.FromJSONBytes(buf, obj) - // 当导出无版本号，并且为告警规则，并且导出数据无分类时 - if patch && ed.Type == DataTypeAlertRule { + if ed.Type == DataTypeAlertRule { if rule, ok := obj.(*alerting.Rule); ok { - if len(obj.(*alerting.Rule).Category) == 0 { + if patch && len(rule.Category) == 0 { rule.Category = "Platform" - obj = rule } + now := time.Now() + if rule.Created.IsZero() { + rule.Created = now + } + if rule.Updated.IsZero() || rule.Updated.Before(rule.Created) { + rule.Updated = now + } + obj = rule } } if err != nil { return err } - err = orm.Save(nil, obj) + err = orm.Save(orm.NewContext(), obj) if err != nil { return err } diff --git a/plugin/api/email/api.go b/plugin/api/email/api.go index 47629d97..4b009215 100644 --- a/plugin/api/email/api.go +++ b/plugin/api/email/api.go @@ -45,10 +45,10 @@ type EmailAPI struct { func InitAPI() { email := EmailAPI{} - api.HandleAPIMethod(api.POST, "/email/server/_test", email.RequirePermission(email.testEmailServer, enum.PermissionSmtpServerRead)) + api.HandleAPIMethod(api.POST, "/email/server/_test", email.RequireSecureTransport(email.RequireReplayProtection(email.RequirePermission(email.testEmailServer, enum.PermissionSmtpServerRead)))) api.HandleAPIMethod(api.GET, "/email/server/:email_server_id", email.RequirePermission(email.getEmailServer, enum.PermissionAlertRuleRead)) - api.HandleAPIMethod(api.POST, "/email/server", email.RequirePermission(email.createEmailServer, enum.PermissionSmtpServerWrite)) - api.HandleAPIMethod(api.PUT, "/email/server/:email_server_id", email.RequirePermission(email.updateEmailServer, enum.PermissionSmtpServerWrite)) + api.HandleAPIMethod(api.POST, "/email/server", email.RequireSecureTransport(email.RequireReplayProtection(email.RequirePermission(email.createEmailServer, enum.PermissionSmtpServerWrite)))) + api.HandleAPIMethod(api.PUT, "/email/server/:email_server_id", email.RequireSecureTransport(email.RequireReplayProtection(email.RequirePermission(email.updateEmailServer, enum.PermissionSmtpServerWrite)))) api.HandleAPIMethod(api.DELETE, "/email/server/:email_server_id", email.RequirePermission(email.deleteEmailServer, enum.PermissionSmtpServerWrite)) api.HandleAPIMethod(api.GET, "/email/server/_search", email.RequirePermission(email.searchEmailServer, enum.PermissionSmtpServerRead)) diff --git a/plugin/api/email/common/auth.go b/plugin/api/email/common/auth.go index 619b1021..3a129c3b 100644 --- a/plugin/api/email/common/auth.go +++ b/plugin/api/email/common/auth.go @@ -42,7 +42,7 @@ func GetBasicAuth(srv *model.EmailServer) (basicAuth model2.BasicAuth, err error if srv.CredentialID != "" { cred := credential.Credential{} cred.ID = srv.CredentialID - _, err = orm.Get(&cred) + _, err = orm.GetV2(orm.NewContext(), &cred) if err != nil { return } diff --git a/plugin/api/email/common/pipeline.go b/plugin/api/email/common/pipeline.go index b029f1df..a35ab3da 100644 --- a/plugin/api/email/common/pipeline.go +++ b/plugin/api/email/common/pipeline.go @@ -119,6 +119,7 @@ func GeneratePipelineConfig(servers []model.EmailServer) (string, error) { "refresh_timestamp": time.Now().UnixMilli(), }, "min_tls_version": srv.TLSMinVersion, + "sender": ResolveSender(srv.Sender, srv.Auth.Username), "auth": util.MapStr{ "username": srv.Auth.Username, "password": fmt.Sprintf("$[[keystore.%s]]", key), @@ -147,7 +148,7 @@ func GeneratePipelineConfig(servers []model.EmailServer) (string, error) { }, "processor": []util.MapStr{ { - "smtp": util.MapStr{ + "console_smtp": util.MapStr{ "idle_timeout_in_seconds": 1, "servers": smtpServers, "templates": util.MapStr{ diff --git a/plugin/api/email/common/pipeline_test.go b/plugin/api/email/common/pipeline_test.go new file mode 100644 index 00000000..867497f8 --- /dev/null +++ b/plugin/api/email/common/pipeline_test.go @@ -0,0 +1,58 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package common + +import ( + "strings" + "testing" + + "infini.sh/console/model" + framework_model "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + ucfg "infini.sh/framework/lib/go-ucfg" +) + +func TestGeneratePipelineConfigUsesConsoleSMTP(t *testing.T) { + cfg, err := GeneratePipelineConfig([]model.EmailServer{ + { + ORMObjectBase: orm.ORMObjectBase{ID: "srv-1"}, + Host: "smtp.example.com", + Port: 465, + Sender: "alerts@example.com", + Auth: &framework_model.BasicAuth{ + Username: "user", + Password: ucfg.SecretString("secret"), + }, + }, + }) + if err != nil { + t.Fatalf("GeneratePipelineConfig() error = %v", err) + } + if !strings.Contains(cfg, "console_smtp:") { + t.Fatalf("expected pipeline config to use console_smtp, got %s", cfg) + } + if !strings.Contains(cfg, "sender: alerts@example.com") { + t.Fatalf("expected pipeline config to preserve sender, got %s", cfg) + } +} diff --git a/plugin/api/email/common/sender.go b/plugin/api/email/common/sender.go new file mode 100644 index 00000000..850123d1 --- /dev/null +++ b/plugin/api/email/common/sender.go @@ -0,0 +1,11 @@ +package common + +import "strings" + +func ResolveSender(sender, username string) string { + sender = strings.TrimSpace(sender) + if sender != "" { + return sender + } + return strings.TrimSpace(username) +} diff --git a/plugin/api/email/processor.go b/plugin/api/email/processor.go new file mode 100644 index 00000000..50f202a7 --- /dev/null +++ b/plugin/api/email/processor.go @@ -0,0 +1,402 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package email + +import ( + "fmt" + "io" + + log "github.com/cihub/seelog" + "gopkg.in/gomail.v2" + "infini.sh/console/model" + emailcommon "infini.sh/console/plugin/api/email/common" + "infini.sh/framework/core/config" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/param" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/fasttemplate" +) + +type EmailProcessor struct { + config *EmailProcessorConfig +} + +type EmailServerConfig struct { + Server struct { + Host string `config:"host"` + Port int `config:"port"` + TLS bool `config:"tls"` + } `config:"server"` + Auth struct { + Username string `config:"username"` + Password string `config:"password"` + } `config:"auth"` + MinTLSVersion string `config:"min_tls_version"` + SendFrom string `config:"sender"` + Recipients struct { + To []string `config:"to"` + CC []string `config:"cc"` + BCC []string `config:"bcc"` + } `config:"recipients"` +} + +type EmailTemplate struct { + ContentType string `config:"content_type"` + Subject string `config:"subject"` + Body string `config:"body"` + variableInSubject bool + variableInBody bool + bodyTemplate *fasttemplate.Template + subjectTemplate *fasttemplate.Template +} + +type EmailProcessorConfig struct { + DialTimeoutInSeconds int `config:"dial_timeout_in_seconds"` + MessageField param.ParaKey `config:"message_field"` + VariableStartTag string `config:"variable_start_tag"` + VariableEndTag string `config:"variable_end_tag"` + Variables map[string]interface{} `config:"variables"` + Servers map[string]*EmailServerConfig `config:"servers"` + Templates map[string]*EmailTemplate `config:"templates"` +} + +var loadEmailServerForProcessor = queryEnabledEmailServerConfigByID +var loadEmailServersForProcessor = queryEnabledEmailServerConfigs + +func (processor *EmailProcessor) Name() string { + return "console_smtp" +} + +func init() { + pipeline.RegisterProcessorPlugin("console_smtp", newEmailProcessor) +} + +func newEmailProcessor(c *config.Config) (pipeline.Processor, error) { + cfg := EmailProcessorConfig{ + DialTimeoutInSeconds: 30, + VariableStartTag: "$[[", + VariableEndTag: "]]", + MessageField: "messages", + } + if err := c.Unpack(&cfg); err != nil { + return nil, fmt.Errorf("failed to unpack console_smtp processor config: %s", err) + } + + processor := &EmailProcessor{config: &cfg} + for _, server := range processor.config.Servers { + normalizeEmailServerConfig(server) + } + + for _, tpl := range processor.config.Templates { + if util.ContainStr(tpl.Body, processor.config.VariableStartTag) { + tpl.variableInBody = true + template, err := fasttemplate.NewTemplate(tpl.Body, processor.config.VariableStartTag, processor.config.VariableEndTag) + if err != nil { + return nil, err + } + tpl.bodyTemplate = template + } + if util.ContainStr(tpl.Subject, processor.config.VariableStartTag) { + tpl.variableInSubject = true + template, err := fasttemplate.NewTemplate(tpl.Subject, processor.config.VariableStartTag, processor.config.VariableEndTag) + if err != nil { + return nil, err + } + tpl.subjectTemplate = template + } + } + + return processor, nil +} + +func (processor *EmailProcessor) Process(ctx *pipeline.Context) error { + obj := ctx.Get(processor.config.MessageField) + if obj == nil { + return nil + } + + messages, ok := obj.([]queue.Message) + if !ok || len(messages) == 0 { + return nil + } + + for _, message := range messages { + payload := util.MapStr{} + if err := util.FromJSONBytes(message.Data, &payload); err != nil { + return err + } + + vars := toMapStr(payload["variables"]) + serverID := util.ToString(payload["server_id"]) + if serverID == "" { + serverID = util.ToString(processor.config.Variables["server_id"]) + } + if serverID == "" { + return errors.New("server_id is empty") + } + + resolvedServerID, server, err := processor.resolveServer(serverID) + if err != nil { + return err + } + + templateName := util.ToString(payload["template"]) + if templateName == "" { + templateName = "raw" + } + tpl, ok := processor.config.Templates[templateName] + if !ok || tpl == nil { + return errors.Errorf("template [%v] not found", templateName) + } + + sendTo := append(readStringList(payload["email"]), server.Recipients.To...) + if len(sendTo) == 0 { + sendTo = append(sendTo, readStringList(vars["email"])...) + } + if len(sendTo) == 0 { + log.Errorf("skip email message without recipients, server_id: %v", resolvedServerID) + continue + } + + cc := append([]string{}, server.Recipients.CC...) + cc = append(cc, readStringList(vars["cc"])...) + + mergedVars := util.MapStr{} + mergedVars.Merge(processor.config.Variables) + mergedVars.Merge(vars) + + subject := processor.renderTemplate(tpl.subjectTemplate, tpl.variableInSubject, tpl.Subject, mergedVars) + body := processor.renderTemplate(tpl.bodyTemplate, tpl.variableInBody, tpl.Body, mergedVars) + contentType := tpl.ContentType + if override := util.ToString(vars["content_type"]); override != "" { + contentType = override + } + + log.Debugf("start to send mail to: %v, subject: %v", sendTo, subject) + if err := processor.send(server, sendTo, cc, subject, contentType, body); err != nil { + return err + } + } + + return nil +} + +func normalizeEmailServerConfig(server *EmailServerConfig) { + if server == nil { + return + } + if server.Auth.Username == "" && server.SendFrom != "" { + server.Auth.Username = server.SendFrom + } + if server.SendFrom == "" && server.Auth.Username != "" { + server.SendFrom = server.Auth.Username + } +} + +func buildEmailServerConfig(server model.EmailServer) (*EmailServerConfig, error) { + if err := server.Validate(false); err != nil { + return nil, err + } + auth, err := emailcommon.GetBasicAuth(&server) + if err != nil { + return nil, err + } + cfg := &EmailServerConfig{ + MinTLSVersion: server.TLSMinVersion, + } + cfg.Server.Host = server.Host + cfg.Server.Port = server.Port + cfg.Server.TLS = server.TLS + cfg.SendFrom = emailcommon.ResolveSender(server.Sender, auth.Username) + cfg.Auth.Username = auth.Username + cfg.Auth.Password = auth.Password.Get() + normalizeEmailServerConfig(cfg) + return cfg, nil +} + +func queryEnabledEmailServerConfigByID(serverID string) (*EmailServerConfig, error) { + if serverID == "" { + return nil, nil + } + server := model.EmailServer{} + server.ID = serverID + exists, err := orm.GetV2(orm.NewContext(), &server) + if err != nil { + return nil, err + } + if !exists || !server.Enabled { + return nil, nil + } + return buildEmailServerConfig(server) +} + +func queryEnabledEmailServerConfigs() (map[string]*EmailServerConfig, error) { + q := &orm.Query{Size: 100} + q.Conds = orm.And(orm.Eq("enabled", true)) + err, result := orm.Search(model.EmailServer{}, q) + if err != nil { + return nil, err + } + servers := map[string]*EmailServerConfig{} + for _, row := range result.Result { + server := model.EmailServer{} + buf := util.MustToJSONBytes(row) + util.MustFromJSONBytes(buf, &server) + cfg, err := buildEmailServerConfig(server) + if err != nil { + return nil, err + } + servers[server.ID] = cfg + } + return servers, nil +} + +func (processor *EmailProcessor) resolveServer(serverID string) (string, *EmailServerConfig, error) { + if server, ok := processor.config.Servers[serverID]; ok && server != nil { + return serverID, server, nil + } + server, err := loadEmailServerForProcessor(serverID) + if err != nil { + return "", nil, err + } + if server != nil { + return serverID, server, nil + } + servers, err := loadEmailServersForProcessor() + if err != nil { + return "", nil, err + } + switch len(servers) { + case 0: + return "", nil, errors.Errorf("server_id [%v] not found and no enabled smtp server is available", serverID) + case 1: + for id, fallback := range servers { + return id, fallback, nil + } + default: + return "", nil, errors.Errorf("server_id [%v] not found and multiple enabled smtp servers were found", serverID) + } + return "", nil, errors.Errorf("server_id [%v] not found", serverID) +} + +func (processor *EmailProcessor) renderTemplate(tpl *fasttemplate.Template, enabled bool, raw string, vars util.MapStr) string { + if !enabled || tpl == nil { + return raw + } + + return tpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + value, err := vars.GetValue(tag) + if err != nil { + return -1, err + } + text := util.ToString(value) + if text == "" { + return 0, nil + } + return w.Write([]byte(text)) + }) +} + +func (processor *EmailProcessor) send(server *EmailServerConfig, to []string, cc []string, subject, contentType, body string) error { + if len(to) == 0 { + return errors.New("no recipient found") + } + + message := gomail.NewMessage() + message.SetHeader("From", server.SendFrom) + message.SetHeader("To", to...) + if len(cc) > 0 { + message.SetHeader("Cc", cc...) + } + if len(server.Recipients.BCC) > 0 { + message.SetHeader("Bcc", server.Recipients.BCC...) + } + message.SetHeader("Subject", subject) + message.SetBody(contentType, body) + + minVersion := model.TLSVersion12 + if server.MinTLSVersion != "" { + minVersion = server.MinTLSVersion + } + tlsMinVersion, err := model.GetTLSVersion(minVersion) + if err != nil { + return err + } + + // The standard gopkg.in/gomail.v2 module does not expose NewDialerWithTimeout. + // DialTimeoutInSeconds is therefore no longer mapped 1:1 here and falls back + // to gomail's built-in connect timeout behavior. + d := gomail.NewDialer( + server.Server.Host, + server.Server.Port, + server.Auth.Username, + server.Auth.Password, + ) + d.TLSConfig = newEmailTLSConfig(server.Server.Host, tlsMinVersion) + d.SSL = server.Server.TLS + return d.DialAndSend(message) +} + +func readStringList(value interface{}) []string { + switch v := value.(type) { + case string: + if v == "" { + return nil + } + return []string{v} + case []string: + list := make([]string, 0, len(v)) + for _, item := range v { + if item != "" { + list = append(list, item) + } + } + return list + case []interface{}: + list := make([]string, 0, len(v)) + for _, item := range v { + text := util.ToString(item) + if text != "" { + list = append(list, text) + } + } + return list + default: + return nil + } +} + +func toMapStr(value interface{}) util.MapStr { + switch v := value.(type) { + case util.MapStr: + return v + case map[string]interface{}: + return util.MapStr(v) + default: + return util.MapStr{} + } +} diff --git a/plugin/api/email/processor_test.go b/plugin/api/email/processor_test.go new file mode 100644 index 00000000..41400353 --- /dev/null +++ b/plugin/api/email/processor_test.go @@ -0,0 +1,115 @@ +package email + +import ( + "strings" + "testing" +) + +func TestResolveServerUsesLiveEnabledServer(t *testing.T) { + previousByID := loadEmailServerForProcessor + previousAll := loadEmailServersForProcessor + defer func() { + loadEmailServerForProcessor = previousByID + loadEmailServersForProcessor = previousAll + }() + + loadEmailServerForProcessor = func(serverID string) (*EmailServerConfig, error) { + cfg := &EmailServerConfig{} + cfg.Server.Host = "smtp.example.com" + cfg.Server.Port = 465 + cfg.Server.TLS = true + cfg.Auth.Username = "ops@example.com" + normalizeEmailServerConfig(cfg) + return cfg, nil + } + loadEmailServersForProcessor = func() (map[string]*EmailServerConfig, error) { + t.Fatal("unexpected fallback server query") + return nil, nil + } + + processor := &EmailProcessor{ + config: &EmailProcessorConfig{ + Servers: map[string]*EmailServerConfig{}, + }, + } + resolvedID, server, err := processor.resolveServer("smtp-live") + if err != nil { + t.Fatalf("expected live server lookup to succeed, got %v", err) + } + if resolvedID != "smtp-live" { + t.Fatalf("expected smtp-live, got %s", resolvedID) + } + if server.Server.Host != "smtp.example.com" { + t.Fatalf("expected resolved server host, got %s", server.Server.Host) + } + if server.SendFrom != "ops@example.com" { + t.Fatalf("expected send-from normalized from username, got %s", server.SendFrom) + } +} + +func TestResolveServerFallsBackToOnlyEnabledServer(t *testing.T) { + previousByID := loadEmailServerForProcessor + previousAll := loadEmailServersForProcessor + defer func() { + loadEmailServerForProcessor = previousByID + loadEmailServersForProcessor = previousAll + }() + + loadEmailServerForProcessor = func(serverID string) (*EmailServerConfig, error) { + return nil, nil + } + loadEmailServersForProcessor = func() (map[string]*EmailServerConfig, error) { + cfg := &EmailServerConfig{} + cfg.Server.Host = "smtp.example.com" + cfg.Auth.Username = "ops@example.com" + normalizeEmailServerConfig(cfg) + return map[string]*EmailServerConfig{ + "smtp-fallback": cfg, + }, nil + } + + processor := &EmailProcessor{ + config: &EmailProcessorConfig{ + Servers: map[string]*EmailServerConfig{}, + }, + } + resolvedID, _, err := processor.resolveServer("smtp-stale") + if err != nil { + t.Fatalf("expected single fallback server to be selected, got %v", err) + } + if resolvedID != "smtp-fallback" { + t.Fatalf("expected smtp-fallback, got %s", resolvedID) + } +} + +func TestResolveServerFailsWithMultipleEnabledServers(t *testing.T) { + previousByID := loadEmailServerForProcessor + previousAll := loadEmailServersForProcessor + defer func() { + loadEmailServerForProcessor = previousByID + loadEmailServersForProcessor = previousAll + }() + + loadEmailServerForProcessor = func(serverID string) (*EmailServerConfig, error) { + return nil, nil + } + loadEmailServersForProcessor = func() (map[string]*EmailServerConfig, error) { + return map[string]*EmailServerConfig{ + "smtp-1": {}, + "smtp-2": {}, + }, nil + } + + processor := &EmailProcessor{ + config: &EmailProcessorConfig{ + Servers: map[string]*EmailServerConfig{}, + }, + } + _, _, err := processor.resolveServer("smtp-stale") + if err == nil { + t.Fatal("expected multiple enabled smtp servers to fail") + } + if !strings.Contains(err.Error(), "multiple enabled smtp servers") { + t.Fatalf("expected multi-server hint, got %v", err) + } +} diff --git a/plugin/api/email/server.go b/plugin/api/email/server.go index 02b942cd..145167b5 100644 --- a/plugin/api/email/server.go +++ b/plugin/api/email/server.go @@ -33,6 +33,7 @@ import ( "fmt" "net/http" "strconv" + "strings" "github.com/buger/jsonparser" log "github.com/cihub/seelog" @@ -46,6 +47,22 @@ import ( "infini.sh/framework/core/util" ) +const ( + emailServerTestErrorKeyAuthRequired = "settings.email.server.message.test.error.auth_required" + emailServerTestErrorKeySMTPAuthFailed = "settings.email.server.message.test.error.smtp_auth_failed" + emailServerTestErrorKeySenderMismatch = "settings.email.server.message.test.error.sender_mismatch" + emailServerTestErrorKeyTLSRequired = "settings.email.server.message.test.error.tls_required" + emailServerTestErrorKeySendFailed = "settings.email.server.message.test.error.send_failed" +) + +func newEmailTLSConfig(serverName string, minVersion uint16) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: true, + MinVersion: minVersion, + ServerName: serverName, + } +} + func (h *EmailAPI) createEmailServer(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var obj = &model.EmailServer{} err := h.DecodeJSON(req, obj) @@ -140,7 +157,7 @@ func saveBasicAuthToCredential(srv *model.EmailServer) (string, error) { if err != nil { return "", err } - err = orm.Create(nil, &cred) + err = orm.Create(orm.NewContext(), &cred) if err != nil { return "", err } @@ -153,7 +170,7 @@ func (h *EmailAPI) getEmailServer(w http.ResponseWriter, req *http.Request, ps h obj := model.EmailServer{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -175,7 +192,7 @@ func (h *EmailAPI) updateEmailServer(w http.ResponseWriter, req *http.Request, p obj := model.EmailServer{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -236,7 +253,7 @@ func (h *EmailAPI) deleteEmailServer(w http.ResponseWriter, req *http.Request, p obj := model.EmailServer{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -370,11 +387,12 @@ func (h *EmailAPI) testEmailServer(w http.ResponseWriter, req *http.Request, ps reqBody.Auth = &auth } if reqBody.Auth == nil { - h.WriteError(w, "auth info required", http.StatusInternalServerError) + h.writeEmailServerTestError(w, emailServerTestErrorKeyAuthRequired, "auth info required", http.StatusInternalServerError) return } + sender := common.ResolveSender(reqBody.Sender, reqBody.Auth.Username) message := gomail.NewMessage() - message.SetHeader("From", reqBody.Auth.Username) + message.SetHeader("From", sender) message.SetHeader("To", reqBody.SendTo...) message.SetHeader("Subject", "INFINI platform test email") @@ -384,16 +402,90 @@ func (h *EmailAPI) testEmailServer(w http.ResponseWriter, req *http.Request, ps h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - err = d.DialAndSend(message) if err != nil { - log.Error(err) - h.WriteError(w, err.Error(), http.StatusInternalServerError) + key, reason := classifyEmailServerTestSendError(&reqBody.EmailServer, sender, err) + log.Errorf( + "email server test send failed, host=%s, port=%d, tls=%v, tls_min_version=%s, sender=%s, error_key=%s, reason=%s, err=%v", + reqBody.Host, + reqBody.Port, + reqBody.TLS, + reqBody.TLSMinVersion, + sender, + key, + reason, + err, + ) + h.writeEmailServerTestError(w, key, reason, http.StatusInternalServerError) return } h.WriteAckOKJSON(w) } +func (h *EmailAPI) writeEmailServerTestError(w http.ResponseWriter, key, reason string, statusCode int) { + payload := util.MapStr{ + "status": statusCode, + "error": util.MapStr{ + "reason": reason, + }, + } + if key != "" { + payload["error"].(util.MapStr)["key"] = key + } + h.WriteJSON(w, payload, statusCode) +} + +func classifyEmailServerTestSendError(server *model.EmailServer, sender string, err error) (string, string) { + if err == nil { + return "", "" + } + + rawReason := strings.TrimSpace(err.Error()) + normalizedReason := strings.ToLower(rawReason) + + if isSMTPAuthenticationError(normalizedReason) { + authUsername := "" + if server != nil && server.Auth != nil { + authUsername = strings.TrimSpace(server.Auth.Username) + } + if authUsername != "" && sender != "" && !strings.EqualFold(strings.TrimSpace(sender), authUsername) { + return emailServerTestErrorKeySenderMismatch, "SMTP authentication failed; some providers require the sender address to match the authenticated account or an approved alias" + } + return emailServerTestErrorKeySMTPAuthFailed, "SMTP authentication failed; verify the username, password, or provider authorization code" + } + + if strings.Contains(normalizedReason, "must issue a starttls command first") { + return emailServerTestErrorKeyTLSRequired, "SMTP server requires TLS or STARTTLS before authentication" + } + + if strings.Contains(normalizedReason, "could not send email") { + return emailServerTestErrorKeySendFailed, "SMTP server rejected the test email; verify the sender, recipient, and provider restrictions" + } + + return emailServerTestErrorKeySendFailed, "Unable to send test email; verify the SMTP server address, port, TLS settings, and provider restrictions" +} + +func isSMTPAuthenticationError(reason string) bool { + reason = strings.TrimSpace(strings.ToLower(reason)) + if reason == "" { + return false + } + authIndicators := []string{ + "authentication is required", + "authentication failed", + "auth failed", + "535", + "invalid login", + "invalid credentials", + } + for _, indicator := range authIndicators { + if strings.Contains(reason, indicator) { + return true + } + } + return false +} + // newEmailTestDialer keeps Console on the standard gopkg.in/gomail.v2 module. // The vendored gomail fork exposed NewDialerWithTimeout and used an explicit 3s // timeout here before, but the standard module does not. Test-email requests now @@ -411,7 +503,7 @@ func newEmailTestDialer(server *model.EmailServer) (*gomail.Dialer, error) { } dialer := gomail.NewDialer(server.Host, server.Port, server.Auth.Username, server.Auth.Password.Get()) - dialer.TLSConfig = &tls.Config{InsecureSkipVerify: true, MinVersion: tlsMinVersion} + dialer.TLSConfig = newEmailTLSConfig(server.Host, tlsMinVersion) dialer.SSL = server.TLS return dialer, nil diff --git a/plugin/api/email/server_test.go b/plugin/api/email/server_test.go index c5f71d72..4d7dc80b 100644 --- a/plugin/api/email/server_test.go +++ b/plugin/api/email/server_test.go @@ -2,6 +2,8 @@ package email import ( "crypto/tls" + "errors" + "strings" "testing" consolemodel "infini.sh/console/model" @@ -108,3 +110,72 @@ func TestNewEmailTestDialerRejectsUnsupportedTLSVersion(t *testing.T) { t.Fatal("expected unsupported TLS version to fail") } } + +func TestNewEmailTLSConfigSetsServerName(t *testing.T) { + cfg := newEmailTLSConfig("smtp.example.com", tls.VersionTLS13) + + if cfg.ServerName != "smtp.example.com" { + t.Fatalf("expected server name to be preserved, got %q", cfg.ServerName) + } + if cfg.MinVersion != tls.VersionTLS13 { + t.Fatalf("expected min TLS version %d, got %d", tls.VersionTLS13, cfg.MinVersion) + } + if !cfg.InsecureSkipVerify { + t.Fatal("expected insecure skip verify to remain enabled") + } +} + +func TestClassifyEmailServerTestSendErrorSenderMismatch(t *testing.T) { + server := &consolemodel.EmailServer{ + Sender: "hello@example.com", + Auth: &frameworkmodel.BasicAuth{ + Username: "notify@example.com", + Password: ucfg.SecretString("secret"), + }, + } + + key, reason := classifyEmailServerTestSendError(server, server.Sender, errors.New("gomail: could not send email 1: 550 5.7.1 authentication is required")) + if key != emailServerTestErrorKeySenderMismatch { + t.Fatalf("expected sender mismatch key, got %q", key) + } + if reason == "" { + t.Fatal("expected a human-readable reason") + } +} + +func TestClassifyEmailServerTestSendErrorSMTPAuthFailure(t *testing.T) { + server := &consolemodel.EmailServer{ + Auth: &frameworkmodel.BasicAuth{ + Username: "notify@example.com", + Password: ucfg.SecretString("secret"), + }, + } + + key, reason := classifyEmailServerTestSendError(server, "notify@example.com", errors.New("535 Authentication failed")) + if key != emailServerTestErrorKeySMTPAuthFailed { + t.Fatalf("expected SMTP auth failure key, got %q", key) + } + if reason == "" { + t.Fatal("expected a human-readable reason") + } +} + +func TestClassifyEmailServerTestSendErrorMapsUnknownErrorsToGenericMessage(t *testing.T) { + key, reason := classifyEmailServerTestSendError(nil, "", errors.New("dial tcp: connection refused")) + if key != emailServerTestErrorKeySendFailed { + t.Fatalf("expected generic send failure key, got %q", key) + } + if reason == "" || reason == "dial tcp: connection refused" { + t.Fatalf("expected generic human-readable reason, got %q", reason) + } +} + +func TestClassifyEmailServerTestSendErrorMapsSendRejectionToGenericMessage(t *testing.T) { + key, reason := classifyEmailServerTestSendError(nil, "", errors.New("gomail: could not send email 1: 554 5.5.3 RP:TRC")) + if key != emailServerTestErrorKeySendFailed { + t.Fatalf("expected generic send failure key, got %q", key) + } + if reason == "" || strings.Contains(strings.ToLower(reason), "gomail: could not send email") { + t.Fatalf("expected sanitized human-readable reason, got %q", reason) + } +} diff --git a/plugin/api/index_management/common_command.go b/plugin/api/index_management/common_command.go index 4e453f2e..b66224ce 100644 --- a/plugin/api/index_management/common_command.go +++ b/plugin/api/index_management/common_command.go @@ -50,6 +50,7 @@ func (h *APIHandler) HandleAddCommonCommandAction(w http.ResponseWriter, req *ht } reqParams.Created = time.Now() + reqParams.Creator = h.GetCurrentUser(req) reqParams.ID = util.GetUUID() esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) @@ -132,7 +133,7 @@ func (h *APIHandler) HandleQueryCommonCommandAction(w http.ResponseWriter, req * var ( keyword = h.GetParameterOrDefault(req, "keyword", "") - queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` + queryDSL = `{"query":{"bool":{"must":[%s]}},"sort":[{"created":{"order":"desc","missing":"_last"}}], "size": %d, "from": %d}` strSize = h.GetParameterOrDefault(req, "size", "20") strFrom = h.GetParameterOrDefault(req, "from", "0") filterBuilder = &strings.Builder{} diff --git a/plugin/api/index_management/elasticsearch.go b/plugin/api/index_management/elasticsearch.go index 74864334..86ef328a 100644 --- a/plugin/api/index_management/elasticsearch.go +++ b/plugin/api/index_management/elasticsearch.go @@ -56,12 +56,11 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req // clusterIDs = append(clusterIDs, key) // return true //}) - esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDsl := util.MapStr{ "size": 100, } - clusterFilter, hasAllPrivilege := handler.GetClusterFilter(req, "_id") - if !hasAllPrivilege && clusterFilter == nil { + clusterFilter, hasAllPrivilege := handler.GetClusterFilter(req, "id") + if !hasAllPrivilege && len(clusterFilter) == 0 { handler.WriteJSON(w, util.MapStr{ "nodes_count": 0, "clusters_count": 0, @@ -71,7 +70,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req }, http.StatusOK) return } - if !hasAllPrivilege { + if !hasAllPrivilege && len(clusterFilter) > 0 { queryDsl["query"] = clusterFilter } @@ -85,7 +84,7 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req _ = service.LogAuditLog(auditLog) } - searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.ElasticsearchConfig{}), util.MustToJSONBytes(queryDsl)) + clusterIDs, err := handler.getOverviewClusterIDs(clusterFilter) if err != nil { log.Error(err) handler.WriteJSON(w, util.MapStr{ @@ -93,11 +92,11 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req }, http.StatusInternalServerError) return } - for _, hit := range searchRes.Hits.Hits { - clusterIDs = append(clusterIDs, hit.ID) + if len(clusterIDs) == 0 { + clusterIDs = getConfiguredOverviewClusterIDs(clusterFilter) } - res, err := handler.getLatestClusterMonitorData(clusterIDs) + totalStoreSize, err = handler.getOverviewTotalStoreSize(clusterIDs) if err != nil { log.Error(err) handler.WriteJSON(w, util.MapStr{ @@ -105,23 +104,6 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req }, http.StatusInternalServerError) return } - for _, info := range res.Hits.Hits { - data := util.MapStr(info.Source) - //val, err := data.GetValue("payload.elasticsearch.cluster_stats.nodes.count.total") - //if err != nil { - // log.Warn(err) - //} - //if num, ok := val.(float64); ok { - // totalNode += int(num) - //} - val, err := data.GetValue("payload.elasticsearch.cluster_stats.indices.store.size_in_bytes") - if err != nil { - log.Warn(err) - } - if num, ok := val.(float64); ok { - totalStoreSize += int(num) - } - } hostCount, err := handler.getMetricCount(orm.GetIndexName(host.HostInfo{}), "ip", nil) if err != nil { @@ -156,6 +138,125 @@ func (handler APIHandler) ElasticsearchOverviewAction(w http.ResponseWriter, req handler.WriteJSON(w, resBody, http.StatusOK) } +func buildOverviewClusterQueryDSL(clusterFilter util.MapStr) util.MapStr { + queryDsl := util.MapStr{ + "size": 100, + } + if len(clusterFilter) > 0 { + queryDsl["query"] = clusterFilter + } + return queryDsl +} + +func (handler APIHandler) getOverviewClusterIDs(clusterFilter util.MapStr) ([]interface{}, error) { + esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + queryDsl := buildOverviewClusterQueryDSL(clusterFilter) + + searchRes, err := esClient.SearchWithRawQueryDSL(orm.GetIndexName(elastic.ElasticsearchConfig{}), util.MustToJSONBytes(queryDsl)) + if err != nil { + return nil, err + } + + clusterIDs := make([]interface{}, 0, len(searchRes.Hits.Hits)) + for _, hit := range searchRes.Hits.Hits { + clusterIDs = append(clusterIDs, hit.ID) + } + return clusterIDs, nil +} + +func getConfiguredOverviewClusterIDs(clusterFilter util.MapStr) []interface{} { + allowedClusterIDs, restricted := extractConfiguredOverviewClusterIDs(clusterFilter) + clusterIDs := make([]interface{}, 0) + seen := map[string]struct{}{} + + elastic.WalkConfigs(func(key, value interface{}) bool { + clusterID, ok := key.(string) + if !ok || clusterID == "" { + return true + } + if restricted { + if _, ok := allowedClusterIDs[clusterID]; !ok { + return true + } + } + if _, ok := seen[clusterID]; ok { + return true + } + seen[clusterID] = struct{}{} + clusterIDs = append(clusterIDs, clusterID) + return true + }) + + return clusterIDs +} + +func extractConfiguredOverviewClusterIDs(clusterFilter util.MapStr) (map[string]struct{}, bool) { + if len(clusterFilter) == 0 { + return nil, false + } + + terms, ok := clusterFilter["terms"] + if !ok { + return nil, true + } + + termMap, ok := terms.(util.MapStr) + if !ok { + if genericTerms, ok := terms.(map[string]interface{}); ok { + termMap = util.MapStr(genericTerms) + } else { + return nil, true + } + } + + ids, ok := termMap["id"] + if !ok { + return nil, true + } + + result := map[string]struct{}{} + switch values := ids.(type) { + case []string: + for _, id := range values { + if id != "" { + result[id] = struct{}{} + } + } + case []interface{}: + for _, value := range values { + if id, ok := value.(string); ok && id != "" { + result[id] = struct{}{} + } + } + } + + return result, true +} + +func (handler APIHandler) getOverviewTotalStoreSize(clusterIDs []interface{}) (int, error) { + if len(clusterIDs) == 0 { + return 0, nil + } + + res, err := handler.getLatestClusterMonitorData(clusterIDs) + if err != nil { + return 0, err + } + + totalStoreSize := 0 + for _, info := range res.Hits.Hits { + data := util.MapStr(info.Source) + val, err := data.GetValue("payload.elasticsearch.cluster_stats.indices.store.size_in_bytes") + if err != nil { + log.Warn(err) + } + if num, ok := val.(float64); ok { + totalStoreSize += int(num) + } + } + return totalStoreSize, nil +} + func (handler APIHandler) getLatestClusterMonitorData(clusterIDs []interface{}) (*elastic.SearchResponse, error) { client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) queryDSLTpl := `{ @@ -368,6 +469,7 @@ func (handler APIHandler) ElasticsearchStatusSummaryAction(w http.ResponseWriter "host": util.MapStr{ "online": 0, }, + "total_used_store_in_bytes": 0, }, http.StatusOK) return } @@ -404,6 +506,13 @@ func (handler APIHandler) ElasticsearchStatusSummaryAction(w http.ResponseWriter for _, cid := range clusterIDs { clusterIds = append(clusterIds, cid) } + } else { + clusterIds, err = handler.getOverviewClusterIDs(nil) + if err != nil { + log.Error(err) + handler.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } } hostCount, err := handler.getMetricCount(orm.GetIndexName(elastic.NodeConfig{}), "metadata.host", clusterIds) if err != nil { @@ -411,12 +520,19 @@ func (handler APIHandler) ElasticsearchStatusSummaryAction(w http.ResponseWriter handler.WriteError(w, err.Error(), http.StatusInternalServerError) return } + totalStoreSize, err := handler.getOverviewTotalStoreSize(clusterIds) + if err != nil { + log.Error(err) + handler.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } handler.WriteJSON(w, util.MapStr{ "cluster": clusterGrp, "node": nodeGrp, "host": util.MapStr{ "online": hostCount, }, + "total_used_store_in_bytes": totalStoreSize, }, http.StatusOK) } diff --git a/plugin/api/index_management/elasticsearch_test.go b/plugin/api/index_management/elasticsearch_test.go new file mode 100644 index 00000000..956387d5 --- /dev/null +++ b/plugin/api/index_management/elasticsearch_test.go @@ -0,0 +1,86 @@ +package index_management + +import ( + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +func TestBuildOverviewClusterQueryDSLSkipsEmptyFilter(t *testing.T) { + queryDSL := buildOverviewClusterQueryDSL(nil) + if _, ok := queryDSL["query"]; ok { + t.Fatalf("expected empty filter to skip query clause, got %#v", queryDSL) + } + + queryDSL = buildOverviewClusterQueryDSL(util.MapStr{}) + if _, ok := queryDSL["query"]; ok { + t.Fatalf("expected empty map filter to skip query clause, got %#v", queryDSL) + } +} + +func TestBuildOverviewClusterQueryDSLIncludesFilter(t *testing.T) { + filter := util.MapStr{ + "terms": util.MapStr{ + "id": []string{"cluster-a"}, + }, + } + + queryDSL := buildOverviewClusterQueryDSL(filter) + if got, ok := queryDSL["query"]; !ok || got == nil { + t.Fatalf("expected non-empty filter to be preserved, got %#v", queryDSL) + } +} + +func TestGetConfiguredOverviewClusterIDsUsesLoadedConfigs(t *testing.T) { + for _, id := range []string{"overview-cluster-a", "overview-cluster-b"} { + elastic.UpdateConfig(elastic.ElasticsearchConfig{ORMObjectBase: orm.ORMObjectBase{ID: id}}) + t.Cleanup(func() { + elastic.RemoveInstance(id) + }) + } + + clusterIDs := getConfiguredOverviewClusterIDs(nil) + clusterIDSet := toClusterIDSet(clusterIDs) + for _, id := range []string{"overview-cluster-a", "overview-cluster-b"} { + if _, ok := clusterIDSet[id]; !ok { + t.Fatalf("expected configured cluster %s in fallback IDs, got %#v", id, clusterIDs) + } + } +} + +func TestGetConfiguredOverviewClusterIDsRespectsFilter(t *testing.T) { + for _, id := range []string{"overview-filter-cluster-a", "overview-filter-cluster-b"} { + elastic.UpdateConfig(elastic.ElasticsearchConfig{ORMObjectBase: orm.ORMObjectBase{ID: id}}) + t.Cleanup(func() { + elastic.RemoveInstance(id) + }) + } + + clusterIDs := getConfiguredOverviewClusterIDs(util.MapStr{ + "terms": util.MapStr{ + "id": []string{"overview-filter-cluster-a"}, + }, + }) + clusterIDSet := toClusterIDSet(clusterIDs) + if len(clusterIDSet) != 1 { + t.Fatalf("expected one filtered fallback cluster, got %#v", clusterIDs) + } + if _, ok := clusterIDSet["overview-filter-cluster-a"]; !ok { + t.Fatalf("expected filtered cluster in fallback IDs, got %#v", clusterIDs) + } + if _, ok := clusterIDSet["overview-filter-cluster-b"]; ok { + t.Fatalf("did not expect unfiltered cluster in fallback IDs, got %#v", clusterIDs) + } +} + +func toClusterIDSet(clusterIDs []interface{}) map[string]struct{} { + clusterIDSet := map[string]struct{}{} + for _, clusterID := range clusterIDs { + if id, ok := clusterID.(string); ok { + clusterIDSet[id] = struct{}{} + } + } + return clusterIDSet +} diff --git a/plugin/api/index_management/indices.go b/plugin/api/index_management/indices.go index eaeedaac..b52d73e8 100644 --- a/plugin/api/index_management/indices.go +++ b/plugin/api/index_management/indices.go @@ -137,6 +137,15 @@ func (handler APIHandler) HandleUpdateSettingsAction(w http.ResponseWriter, req handler.WriteJSON(w, resBody, http.StatusInternalServerError) return } + claims, auditLogErr := security.ValidateLoginFromRequest(req) + if auditLogErr == nil && claims != nil && handler.GetHeader(req, "Referer", "") != "" { + auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(claims.Username). + WithLogTypeOperation().WithResourceTypeClusterManagement(). + WithEventName("update index settings").WithEventSourceIP(common.GetClientIP(req)). + WithResourceName(targetClusterID).WithOperationTypeModification(). + WithEventRecord(indexName + ": " + util.MustToJSON(settings)).Build() + _ = service.LogAuditLog(auditLog) + } resBody["result"] = "updated" handler.WriteJSON(w, resBody, http.StatusCreated) } @@ -162,7 +171,7 @@ func (handler APIHandler) HandleCreateIndexAction(w http.ResponseWriter, req *ht targetClusterID := ps.ByName("id") client := elastic.GetClient(targetClusterID) indexName := ps.ByName("index") - claims, auditLogErr := security.ValidateLogin(req.Header.Get("Authorization")) + claims, auditLogErr := security.ValidateLoginFromRequest(req) if auditLogErr == nil && handler.GetHeader(req, "Referer", "") != "" { auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(claims.Username). WithLogTypeOperation().WithResourceTypeClusterManagement(). diff --git a/plugin/api/init.go b/plugin/api/init.go index e525a710..aa2fb957 100644 --- a/plugin/api/init.go +++ b/plugin/api/init.go @@ -36,6 +36,7 @@ import ( "infini.sh/console/plugin/api/layout" "infini.sh/console/plugin/api/notification" "infini.sh/console/plugin/api/platform" + "infini.sh/console/plugin/api/settings" "infini.sh/framework/core/api" ) @@ -52,7 +53,7 @@ func Init(cfg *config.AppConfig) { api.HandleAPIMethod(api.POST, path.Join(esPrefix, "doc/:index"), handler.IndexRequired(handler.HandleAddDocumentAction, "doc.create")) api.HandleAPIMethod(api.PUT, path.Join(esPrefix, "doc/:index/:docId"), handler.IndexRequired(handler.HandleUpdateDocumentAction, "doc.update")) api.HandleAPIMethod(api.DELETE, path.Join(esPrefix, "doc/:index/:docId"), handler.IndexRequired(handler.HandleDeleteDocumentAction, "doc.delete")) - api.HandleAPIMethod(api.GET, path.Join(esPrefix, "doc/_validate"), handler.ValidateDocIDAction) + api.HandleAPIMethod(api.GET, path.Join(esPrefix, "doc/_validate"), handler.RequireClusterPermission(handler.RequirePermission(handler.ValidateDocIDAction, enum.PermissionElasticsearchIndexRead))) api.HandleAPIMethod(api.GET, path.Join(esPrefix, "_cat/indices"), handler.RequireLogin(handler.HandleCatIndicesAction)) api.HandleAPIMethod(api.GET, path.Join(esPrefix, "index/:index/_mappings"), handler.IndexRequired(handler.HandleGetMappingsAction, "indices.get_mapping")) @@ -80,4 +81,6 @@ func Init(cfg *config.AppConfig) { email.InitAPI() data.InitAPI() platform.InitAPI() + settings.InitAPI() + initConsoleSelfAPI() } diff --git a/plugin/api/insight/api.go b/plugin/api/insight/api.go index bd9b8dfb..c0d979f6 100644 --- a/plugin/api/insight/api.go +++ b/plugin/api/insight/api.go @@ -29,6 +29,7 @@ package insight import ( "infini.sh/console/core" + "infini.sh/console/core/security/enum" "infini.sh/framework/core/api" ) @@ -42,18 +43,18 @@ func InitAPI() { api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/data", insight.RequireLogin(insight.HandleGetMetricData)) api.HandleAPIMethod(api.POST, "/elasticsearch/:id/visualization/preview", insight.RequireLogin(insight.HandleGetPreview)) - api.HandleAPIMethod(api.GET, "/insight/visualization/:visualization_id", insight.getVisualization) - api.HandleAPIMethod(api.POST, "/insight/visualization", insight.createVisualization) - api.HandleAPIMethod(api.PUT, "/insight/visualization/:visualization_id", insight.updateVisualization) - api.HandleAPIMethod(api.DELETE, "/insight/visualization/:visualization_id", insight.deleteVisualization) - api.HandleAPIMethod(api.GET, "/insight/visualization/_search", insight.searchVisualization) + api.HandleAPIMethod(api.GET, "/insight/visualization/:visualization_id", insight.RequirePermission(insight.getVisualization, enum.PermissionLayoutRead)) + api.HandleAPIMethod(api.POST, "/insight/visualization", insight.RequirePermission(insight.createVisualization, enum.PermissionLayoutWrite)) + api.HandleAPIMethod(api.PUT, "/insight/visualization/:visualization_id", insight.RequirePermission(insight.updateVisualization, enum.PermissionLayoutWrite)) + api.HandleAPIMethod(api.DELETE, "/insight/visualization/:visualization_id", insight.RequirePermission(insight.deleteVisualization, enum.PermissionLayoutWrite)) + api.HandleAPIMethod(api.GET, "/insight/visualization/_search", insight.RequirePermission(insight.searchVisualization, enum.PermissionLayoutRead)) - api.HandleAPIMethod(api.GET, "/insight/dashboard/:dashboard_id", insight.getDashboard) - api.HandleAPIMethod(api.POST, "/insight/dashboard", insight.createDashboard) - api.HandleAPIMethod(api.PUT, "/insight/dashboard/:dashboard_id", insight.updateDashboard) - api.HandleAPIMethod(api.DELETE, "/insight/dashboard/:dashboard_id", insight.deleteDashboard) - api.HandleAPIMethod(api.GET, "/insight/dashboard/_search", insight.searchDashboard) - api.HandleAPIMethod(api.POST, "/elasticsearch/:id/map_label/_render", insight.renderMapLabelTemplate) - api.HandleAPIMethod(api.GET, "/insight/widget/:widget_id", insight.getWidget) - api.HandleAPIMethod(api.POST, "/insight/widget", insight.RequireLogin(insight.createWidget)) + api.HandleAPIMethod(api.GET, "/insight/dashboard/:dashboard_id", insight.RequirePermission(insight.getDashboard, enum.DashboardReadPermission...)) + api.HandleAPIMethod(api.POST, "/insight/dashboard", insight.RequirePermission(insight.createDashboard, enum.DashboardAllPermission...)) + api.HandleAPIMethod(api.PUT, "/insight/dashboard/:dashboard_id", insight.RequirePermission(insight.updateDashboard, enum.DashboardAllPermission...)) + api.HandleAPIMethod(api.DELETE, "/insight/dashboard/:dashboard_id", insight.RequirePermission(insight.deleteDashboard, enum.DashboardAllPermission...)) + api.HandleAPIMethod(api.GET, "/insight/dashboard/_search", insight.RequirePermission(insight.searchDashboard, enum.DashboardReadPermission...)) + api.HandleAPIMethod(api.POST, "/elasticsearch/:id/map_label/_render", insight.RequireClusterPermission(insight.renderMapLabelTemplate, enum.PermissionElasticsearchClusterRead)) + api.HandleAPIMethod(api.GET, "/insight/widget/:widget_id", insight.RequirePermission(insight.getWidget, enum.PermissionLayoutRead)) + api.HandleAPIMethod(api.POST, "/insight/widget", insight.RequirePermission(insight.createWidget, enum.PermissionLayoutWrite)) } diff --git a/plugin/api/insight/dashboard.go b/plugin/api/insight/dashboard.go index 763fc5ee..3f978029 100644 --- a/plugin/api/insight/dashboard.go +++ b/plugin/api/insight/dashboard.go @@ -45,7 +45,7 @@ func (h *InsightAPI) createDashboard(w http.ResponseWriter, req *http.Request, p log.Error(err) return } - err = orm.Create(nil, obj) + err = orm.Create(orm.NewContext(), obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -65,7 +65,7 @@ func (h *InsightAPI) getDashboard(w http.ResponseWriter, req *http.Request, ps h obj := insight2.Dashboard{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -103,7 +103,7 @@ func (h *InsightAPI) updateDashboard(w http.ResponseWriter, req *http.Request, p obj := insight2.Dashboard{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -125,7 +125,7 @@ func (h *InsightAPI) updateDashboard(w http.ResponseWriter, req *http.Request, p //protect obj.ID = id obj.Created = create - err = orm.Update(nil, &obj) + err = orm.Update(orm.NewContext(), &obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -144,7 +144,7 @@ func (h *InsightAPI) deleteDashboard(w http.ResponseWriter, req *http.Request, p obj := insight2.Dashboard{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -153,7 +153,7 @@ func (h *InsightAPI) deleteDashboard(w http.ResponseWriter, req *http.Request, p return } - err = orm.Delete(nil, &obj) + err = orm.Delete(orm.NewContext(), &obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) diff --git a/plugin/api/insight/metadata.go b/plugin/api/insight/metadata.go index a1a749f4..a1a41b16 100644 --- a/plugin/api/insight/metadata.go +++ b/plugin/api/insight/metadata.go @@ -31,6 +31,7 @@ import ( "bytes" "math" "net/http" + "strconv" "strings" "sync" "text/template" @@ -49,6 +50,53 @@ import ( "infini.sh/framework/core/util" ) +func stringifyFormulaTemplateParam(value interface{}) interface{} { + switch v := value.(type) { + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + case float32: + return strconv.FormatFloat(float64(v), 'f', -1, 32) + case int: + return strconv.Itoa(v) + case int8: + return strconv.FormatInt(int64(v), 10) + case int16: + return strconv.FormatInt(int64(v), 10) + case int32: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + case uint: + return strconv.FormatUint(uint64(v), 10) + case uint8: + return strconv.FormatUint(uint64(v), 10) + case uint16: + return strconv.FormatUint(uint64(v), 10) + case uint32: + return strconv.FormatUint(uint64(v), 10) + case uint64: + return strconv.FormatUint(v, 10) + default: + return value + } +} + +func renderFormulaTemplate(formula string, params map[string]interface{}) (string, error) { + tpl, err := template.New("insight_formula").Parse(formula) + if err != nil { + return "", err + } + normalizedParams := map[string]interface{}{} + for key, value := range params { + normalizedParams[key] = stringifyFormulaTemplateParam(value) + } + msgBuffer := &bytes.Buffer{} + if err := tpl.Execute(msgBuffer, normalizedParams); err != nil { + return "", err + } + return msgBuffer.String(), nil +} + func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { clusterID := ps.MustGetParameter("id") reqBody := struct { @@ -73,7 +121,7 @@ func (h *InsightAPI) HandleGetPreview(w http.ResponseWriter, req *http.Request, view := elastic.View{ ID: reqBody.ViewID, } - exists, err := orm.Get(&view) + exists, err := orm.GetV2(orm.NewContext(), &view) if err != nil || !exists { h.WriteJSON(w, util.MapStr{ "error": err.Error(), @@ -177,7 +225,7 @@ func (h *InsightAPI) HandleGetMetadata(w http.ResponseWriter, req *http.Request, view := elastic.View{ ID: reqBody.ViewID, } - exists, err := orm.Get(&view) + exists, err := orm.GetV2(orm.NewContext(), &view) if err != nil || !exists { h.WriteError(w, err.Error(), http.StatusNotFound) return @@ -217,6 +265,8 @@ func (h *InsightAPI) HandleGetMetricData(w http.ResponseWriter, req *http.Reques return } } + sanitizeAutoRangeInValue(reqBody.Filter) + sanitizeAutoRangeInValue(reqBody.TimeFilter) reqBody.ClusterId = clusterID metricData, err := getMetricData(&reqBody) if err != nil { @@ -297,16 +347,10 @@ func getMetricData(metric *insightpkg.Metric) (interface{}, error) { grpMetricData := &insightpkg.MetricDataItem{} isTimeSeries := false for _, formula = range metric.Formulas { - tpl, err := template.New("insight_formula").Parse(formula) - if err != nil { - return nil, err - } - msgBuffer := &bytes.Buffer{} - err = tpl.Execute(msgBuffer, params) + resolvedFormula, err := renderFormulaTemplate(formula, params) if err != nil { return nil, err } - resolvedFormula := msgBuffer.String() expression, err := govaluate.NewEvaluableExpression(resolvedFormula) if err != nil { return nil, err diff --git a/plugin/api/insight/metadata_test.go b/plugin/api/insight/metadata_test.go new file mode 100644 index 00000000..cd436371 --- /dev/null +++ b/plugin/api/insight/metadata_test.go @@ -0,0 +1,24 @@ +package insight + +import ( + "testing" + + "github.com/Knetic/govaluate" +) + +func TestRenderFormulaTemplateAvoidsScientificNotation(t *testing.T) { + formula, err := renderFormulaTemplate("a/{{.bucket_size_in_second}}", map[string]interface{}{ + "bucket_size_in_second": 3.1536e+06, + }) + if err != nil { + t.Fatalf("unexpected render error: %v", err) + } + + if formula != "a/3153600" { + t.Fatalf("expected non-scientific formula, got %q", formula) + } + + if _, err := govaluate.NewEvaluableExpression(formula); err != nil { + t.Fatalf("expected rendered formula to be parseable, got %v", err) + } +} diff --git a/plugin/api/insight/time_range.go b/plugin/api/insight/time_range.go new file mode 100644 index 00000000..1ca60324 --- /dev/null +++ b/plugin/api/insight/time_range.go @@ -0,0 +1,109 @@ +package insight + +import "strings" + +func sanitizeAutoRangeInValue(val interface{}) bool { + switch typed := val.(type) { + case map[string]interface{}: + return sanitizeAutoRangeInMap(typed) + case []interface{}: + changed := false + for _, item := range typed { + if sanitizeAutoRangeInValue(item) { + changed = true + } + } + return changed + default: + return false + } +} + +func sanitizeAutoRangeInMap(data map[string]interface{}) bool { + if data == nil { + return false + } + changed := false + for key, val := range data { + switch typed := val.(type) { + case map[string]interface{}: + if key == "range" { + if sanitizeAutoRangeNode(typed) { + changed = true + } + if len(typed) == 0 { + delete(data, key) + changed = true + } + continue + } + if sanitizeAutoRangeInMap(typed) { + changed = true + } + if len(typed) == 0 { + delete(data, key) + changed = true + } + case []interface{}: + newList := make([]interface{}, 0, len(typed)) + listChanged := false + for _, item := range typed { + if itemMap, ok := item.(map[string]interface{}); ok { + if sanitizeAutoRangeInMap(itemMap) { + listChanged = true + } + if len(itemMap) == 0 { + listChanged = true + continue + } + newList = append(newList, itemMap) + continue + } + newList = append(newList, item) + } + if listChanged { + data[key] = newList + changed = true + } + } + } + return changed +} + +func sanitizeAutoRangeNode(rangeNode map[string]interface{}) bool { + changed := false + for field, condVal := range rangeNode { + cond, ok := condVal.(map[string]interface{}) + if !ok { + continue + } + for _, boundKey := range []string{"gte", "lte", "gt", "lt", "from", "to"} { + if isAutoRangeValue(cond[boundKey]) { + delete(cond, boundKey) + changed = true + } + } + if !hasRangeBounds(cond) { + delete(cond, "format") + } + if len(cond) == 0 || !hasRangeBounds(cond) { + delete(rangeNode, field) + changed = true + } + } + return changed +} + +func hasRangeBounds(rangeCond map[string]interface{}) bool { + for _, key := range []string{"gte", "lte", "gt", "lt", "from", "to"} { + if _, ok := rangeCond[key]; ok { + return true + } + } + return false +} + +func isAutoRangeValue(value interface{}) bool { + v, ok := value.(string) + return ok && strings.EqualFold(strings.TrimSpace(v), "auto") +} diff --git a/plugin/api/insight/visualization.go b/plugin/api/insight/visualization.go index 2e82aacc..c643e149 100644 --- a/plugin/api/insight/visualization.go +++ b/plugin/api/insight/visualization.go @@ -48,7 +48,7 @@ func (h *InsightAPI) createVisualization(w http.ResponseWriter, req *http.Reques log.Error(err) return } - err = orm.Create(nil, obj) + err = orm.Create(orm.NewContext(), obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -68,7 +68,7 @@ func (h *InsightAPI) getVisualization(w http.ResponseWriter, req *http.Request, obj := insight.Visualization{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -89,7 +89,7 @@ func (h *InsightAPI) updateVisualization(w http.ResponseWriter, req *http.Reques obj := insight.Visualization{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -111,7 +111,7 @@ func (h *InsightAPI) updateVisualization(w http.ResponseWriter, req *http.Reques //protect obj.ID = id obj.Created = create - err = orm.Update(nil, &obj) + err = orm.Update(orm.NewContext(), &obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -130,7 +130,7 @@ func (h *InsightAPI) deleteVisualization(w http.ResponseWriter, req *http.Reques obj := insight.Visualization{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -139,7 +139,7 @@ func (h *InsightAPI) deleteVisualization(w http.ResponseWriter, req *http.Reques return } - err = orm.Delete(nil, &obj) + err = orm.Delete(orm.NewContext(), &obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) diff --git a/plugin/api/insight/widget.go b/plugin/api/insight/widget.go index e5e23c52..36b5bb6f 100644 --- a/plugin/api/insight/widget.go +++ b/plugin/api/insight/widget.go @@ -44,7 +44,7 @@ func (h *InsightAPI) createWidget(w http.ResponseWriter, req *http.Request, ps h log.Error(err) return } - err = orm.Create(nil, obj) + err = orm.Create(orm.NewContext(), obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -61,7 +61,7 @@ func (h *InsightAPI) getWidget(w http.ResponseWriter, req *http.Request, ps http obj := insight.Widget{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, diff --git a/plugin/api/layout/layout.go b/plugin/api/layout/layout.go index 9ccbfa2e..ca0e17c7 100644 --- a/plugin/api/layout/layout.go +++ b/plugin/api/layout/layout.go @@ -78,7 +78,7 @@ func (h *LayoutAPI) getLayout(w http.ResponseWriter, req *http.Request, ps httpr obj := model.Layout{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -100,7 +100,7 @@ func (h *LayoutAPI) updateLayout(w http.ResponseWriter, req *http.Request, ps ht oldLayout := model.Layout{} oldLayout.ID = id - exists, err := orm.Get(&oldLayout) + exists, err := orm.GetV2(orm.NewContext(), &oldLayout) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -141,7 +141,7 @@ func (h *LayoutAPI) deleteLayout(w http.ResponseWriter, req *http.Request, ps ht obj := model.Layout{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, diff --git a/plugin/api/platform/api.go b/plugin/api/platform/api.go index 74f196e0..7a637a73 100644 --- a/plugin/api/platform/api.go +++ b/plugin/api/platform/api.go @@ -67,7 +67,7 @@ func (h *PlatformAPI) searchCollection(w http.ResponseWriter, req *http.Request, return } if api.IsAuthEnable() { - claims, err := security.ValidateLogin(req.Header.Get("Authorization")) + claims, err := security.ValidateLoginFromRequest(req) if err != nil { h.WriteError(w, err.Error(), http.StatusUnauthorized) return diff --git a/plugin/api/self_api.go b/plugin/api/self_api.go new file mode 100644 index 00000000..2e060b15 --- /dev/null +++ b/plugin/api/self_api.go @@ -0,0 +1,190 @@ +package api + +import ( + "crypto/subtle" + "net/http" + "strings" + + log "github.com/cihub/seelog" + consolesecurity "infini.sh/console/core/security" + frameworkapi "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/global" + configcommon "infini.sh/framework/modules/configs/common" +) + +const consoleSelfAccessTokenKeystoreKey = "console_access_token" + +var ( + loadConsoleSelfAccessToken = func() (string, error) { + return configcommon.LoadTokenFromKeystore(consoleSelfAccessTokenKeystoreKey) + } + ensureConsoleSelfAccessToken = func() (string, error) { + return configcommon.EnsureTokenInKeystore(consoleSelfAccessTokenKeystoreKey) + } +) + +var consoleAdditionalProtectedWebAPIRoutes = []frameworkapi.ProtectedAPIRoute{ + {Method: frameworkapi.GET, Path: "/stats/prometheus"}, + {Method: frameworkapi.GET, Path: "/debug/goroutines"}, + {Method: frameworkapi.GET, Path: "/debug/pool/bytes"}, + {Method: frameworkapi.GET, Path: "/_local/files/_list"}, + {Method: frameworkapi.GET, Path: "/_local/files/:file/_list"}, + {Method: frameworkapi.DELETE, Path: "/_local/files/:file"}, +} + +var consoleAccountProxyUIRoutes = []frameworkapi.ProtectedAPIRoute{ + {Method: frameworkapi.POST, Path: "/account/refresh"}, + {Method: frameworkapi.POST, Path: "/account/logout"}, + {Method: frameworkapi.DELETE, Path: "/account/logout"}, + {Method: frameworkapi.GET, Path: "/account/profile"}, + {Method: frameworkapi.PUT, Path: "/account/password"}, +} + +var consolePublicProxyUIRoutes = []frameworkapi.ProtectedAPIRoute{ + {Method: frameworkapi.GET, Path: "/_info"}, + {Method: frameworkapi.GET, Path: "/health"}, + {Method: frameworkapi.GET, Path: "/_license/info"}, + {Method: frameworkapi.POST, Path: "/account/replay_nonce"}, + {Method: frameworkapi.POST, Path: "/account/login/challenge"}, + {Method: frameworkapi.POST, Path: "/account/login"}, + {Method: frameworkapi.POST, Path: "/setup/_validate"}, + {Method: frameworkapi.POST, Path: "/setup/_initialize"}, + {Method: frameworkapi.POST, Path: "/setup/_validate_secret"}, + {Method: frameworkapi.POST, Path: "/setup/_initialize_template"}, + {Method: frameworkapi.GET, Path: "/setting/application"}, + {Method: frameworkapi.GET, Path: "/instance/_get_install_script"}, + {Method: frameworkapi.GET, Path: "/instance/_get_gateway_install_script"}, + {Method: frameworkapi.POST, Path: "/configs/_sync"}, + {Method: frameworkapi.POST, Path: "/configs/_reload"}, +} + +type consoleSelfAPIHandler struct { + frameworkapi.Handler +} + +func initConsoleSelfAPI() { + if _, err := ensureConsoleSelfAccessToken(); err != nil { + log.Errorf("failed to initialize console self access token: %v", err) + } + + handler := consoleSelfAPIHandler{} + registerConsoleProtectedUIRoutes(handler) + registerConsoleAccountProxyUIRoutes(handler) + RefreshConsoleSelfAPIProxyUIRoutes() +} + +func registerConsoleProtectedUIRoutes(handler consoleSelfAPIHandler) { + routes := append([]frameworkapi.ProtectedAPIRoute{}, frameworkapi.DefaultProtectedAPIRoutes...) + routes = append(routes, consoleAdditionalProtectedWebAPIRoutes...) + frameworkapi.RegisterProtectedUIRoutes(routes, handler.requireLoginOrAccessToken(handler.proxyLocalAPI), frameworkapi.AllowOPTIONSS(), frameworkapi.Feature(frameworkapi.FeatureCORS)) +} + +func registerConsoleAccountProxyUIRoutes(handler consoleSelfAPIHandler) { + frameworkapi.RegisterProtectedUIRoutes(consoleAccountProxyUIRoutes, handler.requireLoginOrAccessToken(handler.proxyAccountAPI), frameworkapi.Override(), frameworkapi.AllowOPTIONSS(), frameworkapi.Feature(frameworkapi.FeatureCORS)) +} + +func registerMissingConsoleAPIProxyUIRoutes(handler consoleSelfAPIHandler) { + RefreshConsoleSelfAPIProxyUIRoutes() +} + +func RefreshConsoleSelfAPIProxyUIRoutes() { + handler := consoleSelfAPIHandler{} + frameworkapi.WalkMissingAPIMethodUIRoutes(func(route frameworkapi.MissingAPIMethodUIRoute) { + proxyHandler := httprouter.Handle(handler.proxyLocalAPI) + if shouldProtectConsoleSelfProxyRoute(route.Route, route.Options) { + proxyHandler = handler.requireLoginOrAccessToken(proxyHandler) + } + frameworkapi.HandleUIMethod(route.Route.Method, route.Route.Path, proxyHandler) + }) +} + +func shouldProtectConsoleSelfProxyRoute(route frameworkapi.ProtectedAPIRoute, options *frameworkapi.HandlerOptions) bool { + if !frameworkapi.IsAuthEnable() { + return false + } + if isConsolePublicProxyRoute(route) { + return false + } + if options == nil { + return true + } + if len(options.RequirePermission) > 0 { + return true + } + if options.RequireLogin && !options.OptionLogin { + return true + } + if !options.RequireLogin && !options.OptionLogin { + return false + } + return true +} + +func isConsolePublicProxyRoute(route frameworkapi.ProtectedAPIRoute) bool { + for _, publicRoute := range consolePublicProxyUIRoutes { + if publicRoute.Method == route.Method && publicRoute.Path == route.Path { + return true + } + } + return false +} + +func (h consoleSelfAPIHandler) proxyLocalAPI(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + proxyReq := req.Clone(req.Context()) + applyConsoleLocalAPIAuth(proxyReq) + frameworkapi.ServeRegisteredAPIRequest(w, proxyReq) +} + +func (h consoleSelfAPIHandler) proxyAccountAPI(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + proxyReq := req.Clone(req.Context()) + if validateConsoleSelfAccessToken(extractConsoleSelfAccessToken(req)) { + applyConsoleLocalAPIAuth(proxyReq) + } + frameworkapi.ServeRegisteredAPIRequest(w, proxyReq) +} + +func (h consoleSelfAPIHandler) requireLoginOrAccessToken(next httprouter.Handle) httprouter.Handle { + return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + if validateConsoleSelfAccessToken(extractConsoleSelfAccessToken(req)) { + next(w, req, ps) + return + } + + if frameworkapi.IsAuthEnable() { + if _, err := consolesecurity.ValidateLoginFromRequest(req); err == nil { + next(w, req, ps) + return + } + } + + h.WriteError(w, "unauthorized", http.StatusUnauthorized) + } +} + +func applyConsoleLocalAPIAuth(req *http.Request) { + if req == nil { + return + } + apiCfg := global.Env().SystemConfig.APIConfig + if !apiCfg.Security.Enabled { + return + } + username := strings.TrimSpace(apiCfg.Security.Username) + if username == "" { + return + } + req.SetBasicAuth(username, apiCfg.Security.Password) +} + +func validateConsoleSelfAccessToken(tokenValue string) bool { + expected, err := loadConsoleSelfAccessToken() + if err != nil || expected == "" || tokenValue == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(expected), []byte(strings.TrimSpace(tokenValue))) == 1 +} + +func extractConsoleSelfAccessToken(req *http.Request) string { + return frameworkapi.ExtractBearerOrAPIToken(req) +} diff --git a/plugin/api/self_api_test.go b/plugin/api/self_api_test.go new file mode 100644 index 00000000..adc6ae26 --- /dev/null +++ b/plugin/api/self_api_test.go @@ -0,0 +1,405 @@ +package api + +import ( + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/golang-jwt/jwt/v4" + consolesecurity "infini.sh/console/core/security" + securityapi "infini.sh/console/modules/security/api" + api2 "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/config" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + _ "infini.sh/framework/modules/security/account" + _ "infini.sh/framework/modules/security/http_filters" + simplekv "infini.sh/framework/plugins/simple_kv" +) + +func newTestWebBinding(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on random port: %v", err) + } + defer listener.Close() + + return listener.Addr().String() +} + +func TestExtractConsoleSelfAccessToken(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/stats", nil) + req.Header.Set(model.API_TOKEN, "api-token") + if got := extractConsoleSelfAccessToken(req); got != "api-token" { + t.Fatalf("unexpected api token: %q", got) + } + + req = httptest.NewRequest(http.MethodGet, "/stats", nil) + req.Header.Set("Authorization", "Bearer bearer-token") + if got := extractConsoleSelfAccessToken(req); got != "bearer-token" { + t.Fatalf("unexpected bearer token: %q", got) + } +} + +func TestConsoleSelfAccessTokenValidation(t *testing.T) { + originalLoad := loadConsoleSelfAccessToken + t.Cleanup(func() { + loadConsoleSelfAccessToken = originalLoad + }) + + loadConsoleSelfAccessToken = func() (string, error) { + return "expected-token", nil + } + + if !validateConsoleSelfAccessToken("expected-token") { + t.Fatal("expected self access token to validate") + } + if validateConsoleSelfAccessToken("unexpected-token") { + t.Fatal("expected mismatched self access token to be rejected") + } +} + +func TestConsoleSelfStatsRequireLoginOrAccessToken(t *testing.T) { + originalLoad := loadConsoleSelfAccessToken + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + loadConsoleSelfAccessToken = originalLoad + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + loadConsoleSelfAccessToken = func() (string, error) { + return "expected-token", nil + } + global.Env().SystemConfig.WebAppConfig.Security.Enabled = false + + handler := consoleSelfAPIHandler{} + protected := handler.requireLoginOrAccessToken(func(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodGet, "/stats", nil) + req.Header.Set(model.API_TOKEN, "expected-token") + recorder := httptest.NewRecorder() + protected(recorder, req, nil) + if recorder.Code != http.StatusOK { + t.Fatalf("expected token-authenticated request to succeed, got %d", recorder.Code) + } + + req = httptest.NewRequest(http.MethodGet, "/stats", nil) + recorder = httptest.NewRecorder() + protected(recorder, req, nil) + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("expected missing token to be rejected, got %d", recorder.Code) + } +} + +func TestConsoleSelfTokenAllowsSharedProtectedRoute(t *testing.T) { + originalLoad := loadConsoleSelfAccessToken + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + loadConsoleSelfAccessToken = originalLoad + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + loadConsoleSelfAccessToken = func() (string, error) { + return "expected-token", nil + } + global.Env().SystemConfig.WebAppConfig.Security.Enabled = false + + handler := consoleSelfAPIHandler{} + protected := handler.requireLoginOrAccessToken(func(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + }) + + req := httptest.NewRequest(http.MethodPost, "/pipeline/tasks/_search", nil) + req.Header.Set("Authorization", "Bearer expected-token") + recorder := httptest.NewRecorder() + protected(recorder, req, nil) + if recorder.Code != http.StatusAccepted { + t.Fatalf("expected protected shared route to accept console self token, got %d", recorder.Code) + } +} + +func TestApplyConsoleLocalAPIAuth(t *testing.T) { + originalCfg := global.Env().SystemConfig.APIConfig + t.Cleanup(func() { + global.Env().SystemConfig.APIConfig = originalCfg + }) + + global.Env().SystemConfig.APIConfig.Security.Enabled = true + global.Env().SystemConfig.APIConfig.Security.Username = "local-user" + global.Env().SystemConfig.APIConfig.Security.Password = "local-pass" + + req := httptest.NewRequest(http.MethodGet, "/stats", nil) + applyConsoleLocalAPIAuth(req) + + username, password, ok := req.BasicAuth() + if !ok { + t.Fatal("expected local api auth to be applied") + } + if username != "local-user" || password != "local-pass" { + t.Fatalf("unexpected local api auth credentials: %s/%s", username, password) + } +} + +func TestConsoleAccountProfileUsesConsoleTokenOnUI(t *testing.T) { + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + originalDataDir := global.Env().SystemConfig.PathConfig.Data + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + global.Env().SystemConfig.PathConfig.Data = originalDataDir + }) + + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + global.Env().SystemConfig.PathConfig.Data = t.TempDir() + + kvModule := &simplekv.SimpleKV{} + kvModule.Setup() + if err := kvModule.Start(); err != nil { + t.Fatalf("start simple kv: %v", err) + } + t.Cleanup(func() { + _ = kvModule.Stop() + }) + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newTestWebBinding(t) + + api2.StartWeb(webCfg) + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/account/profile", nil) + req.Header.Set("Authorization", "Bearer "+issueConsoleTestToken(t, "profile-user")) + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve original profile route: %v", err) + } + if resp.Code != http.StatusUnauthorized { + t.Fatalf("expected framework /account/profile ui route to reject console token before override, got %d", resp.Code) + } + api2.StopWeb(webCfg) + + securityapi.Init() + registerConsoleAccountProxyUIRoutes(consoleSelfAPIHandler{}) + + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + resp = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/account/profile", nil) + req.Header.Set("Authorization", "Bearer "+issueConsoleTestToken(t, "profile-user-override")) + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve overridden profile route: %v", err) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected console token to work on overridden /account/profile ui route, got %d", resp.Code) + } + + var profile struct { + ID string `json:"id"` + Roles []string `json:"roles"` + Privilege []string `json:"privilege"` + Permissions []string `json:"permissions"` + } + if err := json.Unmarshal(resp.Body.Bytes(), &profile); err != nil { + t.Fatalf("decode profile response: %v", err) + } + if profile.ID != "profile-user-override" { + t.Fatalf("expected proxied profile to preserve original user id, got %#v", profile) + } + if !containsString(profile.Roles, "Administrator") { + t.Fatalf("expected proxied profile to keep console roles, got %#v", profile) + } + if !containsString(profile.Privilege, "system.security:all") { + t.Fatalf("expected proxied profile to expose console privilege, got %#v", profile) + } + if !containsString(profile.Permissions, "system.security:all") { + t.Fatalf("expected proxied profile permissions to include console authority, got %#v", profile) + } +} + +func TestRefreshConsoleSelfAPIProxyUIRoutesMirrorsLateProtectedAPIRoutes(t *testing.T) { + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + global.Env().SystemConfig.WebAppConfig.Security.Enabled = false + + initConsoleSelfAPI() + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newTestWebBinding(t) + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + api2.HandleAPIMethod(api2.GET, "/late-ui-proxy-route", func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusNoContent) + }) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/late-ui-proxy-route", nil) + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve late ui route before refresh: %v", err) + } + if resp.Code != http.StatusNotFound { + t.Fatalf("expected late API route to be absent before refresh, got %d", resp.Code) + } + + RefreshConsoleSelfAPIProxyUIRoutes() + + resp = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/late-ui-proxy-route", nil) + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve late ui route after refresh: %v", err) + } + if resp.Code != http.StatusNoContent { + t.Fatalf("expected late API route to be mirrored after refresh, got %d", resp.Code) + } +} + +func TestRefreshConsoleSelfAPIProxyUIRoutesKeepsPublicAPIRoutesPublic(t *testing.T) { + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + + initConsoleSelfAPI() + + api2.HandleAPIMethod(api2.GET, "/late-public-ui-proxy-route", func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + }, api2.AllowPublicAccess()) + + RefreshConsoleSelfAPIProxyUIRoutes() + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newTestWebBinding(t) + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/late-public-ui-proxy-route", nil) + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve late public ui route: %v", err) + } + if resp.Code != http.StatusAccepted { + t.Fatalf("expected public API route to stay public on UI mirror, got %d", resp.Code) + } +} + +func TestRefreshConsoleSelfAPIProxyUIRoutesMirrorsLateProtectedAPIRoutesAfterWebStart(t *testing.T) { + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + + global.Env().SystemConfig.WebAppConfig.Security.Enabled = false + + initConsoleSelfAPI() + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newTestWebBinding(t) + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + api2.HandleAPIMethod(api2.GET, "/late-runtime-ui-proxy-route", func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusNoContent) + }) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/late-runtime-ui-proxy-route", nil) + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve runtime ui route before refresh: %v", err) + } + if resp.Code != http.StatusNotFound { + t.Fatalf("expected late API route to be absent before runtime refresh, got %d", resp.Code) + } + + RefreshConsoleSelfAPIProxyUIRoutes() + + resp = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/late-runtime-ui-proxy-route", nil) + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve runtime ui route after refresh: %v", err) + } + if resp.Code != http.StatusNoContent { + t.Fatalf("expected late API route to be mirrored after runtime refresh, got %d", resp.Code) + } +} + +func TestShouldProtectConsoleSelfProxyRouteAllowsKnownPublicRoutes(t *testing.T) { + originalAuthEnabled := global.Env().SystemConfig.WebAppConfig.Security.Enabled + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig.Security.Enabled = originalAuthEnabled + }) + global.Env().SystemConfig.WebAppConfig.Security.Enabled = true + + if shouldProtectConsoleSelfProxyRoute(api2.ProtectedAPIRoute{Method: api2.POST, Path: "/account/login"}, nil) { + t.Fatal("expected account login route to remain public") + } + if shouldProtectConsoleSelfProxyRoute(api2.ProtectedAPIRoute{Method: api2.GET, Path: "/instance/_get_install_script"}, nil) { + t.Fatal("expected install script route to remain public") + } + if shouldProtectConsoleSelfProxyRoute(api2.ProtectedAPIRoute{Method: api2.GET, Path: "/_info"}, nil) { + t.Fatal("expected _info route to remain public") + } + if shouldProtectConsoleSelfProxyRoute(api2.ProtectedAPIRoute{Method: api2.GET, Path: "/health"}, nil) { + t.Fatal("expected health route to remain public") + } + if shouldProtectConsoleSelfProxyRoute(api2.ProtectedAPIRoute{Method: api2.GET, Path: "/_license/info"}, nil) { + t.Fatal("expected license info route to remain public") + } + if shouldProtectConsoleSelfProxyRoute(api2.ProtectedAPIRoute{Method: api2.GET, Path: "/setting/application"}, nil) { + t.Fatal("expected application setting route to remain public") + } + if !shouldProtectConsoleSelfProxyRoute(api2.ProtectedAPIRoute{Method: api2.GET, Path: "/_version"}, nil) { + t.Fatal("expected unannotated non-whitelisted route to be protected") + } +} + +func issueConsoleTestToken(t *testing.T, userID string) string { + t.Helper() + + expireAt := time.Now().Add(time.Hour) + token := jwt.NewWithClaims(jwt.SigningMethodHS256, consolesecurity.UserClaims{ + ShortUser: &consolesecurity.ShortUser{ + Provider: "native", + Username: "tester", + UserId: userID, + Roles: []string{"Administrator"}, + }, + RegisteredClaims: &jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(expireAt), + }, + }) + + tokenString, err := token.SignedString([]byte(consolesecurity.Secret)) + if err != nil { + t.Fatalf("sign token: %v", err) + } + + consolesecurity.SetUserToken(userID, consolesecurity.Token{ + Value: tokenString, + ExpireIn: expireAt.Unix(), + }) + t.Cleanup(func() { + consolesecurity.DeleteUserToken(userID) + }) + + return tokenString +} + +func containsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/plugin/api/settings/api.go b/plugin/api/settings/api.go new file mode 100644 index 00000000..ddd6a358 --- /dev/null +++ b/plugin/api/settings/api.go @@ -0,0 +1,1565 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package settings + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "time" + + log "github.com/cihub/seelog" + "infini.sh/console/core" + "infini.sh/console/core/security/enum" + agentapi "infini.sh/console/modules/agent/api" + managedserver "infini.sh/console/plugin/managed/server" + setupplugin "infini.sh/console/plugin/setup" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/fasttemplate" + configCommon "infini.sh/framework/modules/configs/common" + elasticCommon "infini.sh/framework/modules/elastic/common" +) + +type SettingsAPI struct { + core.Handler +} + +type rawRequester interface { + Request(ctx context.Context, method, url string, body []byte) (*util.Result, error) +} + +type rollupSettingsRequest struct { + Enabled bool `json:"enabled"` +} + +type retentionSettingsRequest struct { + Days int `json:"days"` + MaxSize string `json:"max_size"` +} + +const defaultRetentionDays = 30 +const defaultRetentionMaxSize = "50gb" + +var retentionPolicyPattern = regexp.MustCompile(`-(\d+)days-retention$`) +var retentionSizePattern = regexp.MustCompile(`(?i)^(\d+)([kmgt]b?|b)$`) + +func InitAPI() { + managedserver.SetRefreshManagedLocalTemplatesForInstall(RefreshManagedLocalTemplatesForInstall) + handler := SettingsAPI{} + api.HandleAPIMethod(api.GET, "/setting/system/retention", handler.RequirePermission(handler.getRetentionSetting, enum.PermissionElasticsearchClusterRead)) + api.HandleAPIMethod(api.PUT, "/setting/system/retention", handler.RequireSecureTransport(handler.RequireReplayProtection(handler.RequirePermission(handler.updateRetentionSetting, enum.PermissionElasticsearchClusterWrite)))) + api.HandleAPIMethod(api.GET, "/setting/system/rollup", handler.RequirePermission(handler.getRollupSetting, enum.PermissionElasticsearchClusterRead)) + api.HandleAPIMethod(api.PUT, "/setting/system/rollup", handler.RequireSecureTransport(handler.RequireReplayProtection(handler.RequirePermission(handler.updateRollupSetting, enum.PermissionElasticsearchClusterWrite)))) + api.HandleAPIMethod(api.POST, "/setting/system/local_templates/_refresh", handler.RequireSecureTransport(handler.RequireReplayProtection(handler.RequirePermission(handler.refreshLocalTemplates, enum.PermissionElasticsearchClusterWrite)))) +} + +type managedLocalTemplateSpec struct { + ID string + AppName string + FileName string + Location string + DataFile string +} + +var managedLocalTemplateSpecs = []managedLocalTemplateSpec{ + { + ID: "system_ingest_config_yml", + AppName: "agent", + FileName: "system_ingest_config.yml", + Location: "system_ingest_config.yml", + DataFile: "system_ingest_config.dat", + }, + { + ID: "task_config_tpl", + AppName: "agent", + FileName: "task_config.tpl", + Location: "task_config.tpl", + DataFile: "task_config_tpl.dat", + }, + { + ID: "agent_relay_gateway_config_yml", + AppName: "gateway", + FileName: "relay.yml", + Location: "relay.yml", + DataFile: "gateway_relay.dat", + }, + { + ID: "gateway_migration_yml", + AppName: "gateway", + FileName: "migration.yml", + Location: "migration.yml", + DataFile: "gateway_migration.dat", + }, +} + +func resolveSystemIndexPrefix() string { + ormConfig := elasticCommon.ORMConfig{} + _, err := env.ParseConfig("elastic.orm", &ormConfig) + if err != nil { + log.Warn(err) + } + indexPrefix := strings.TrimSpace(ormConfig.IndexPrefix) + if indexPrefix == "" { + indexPrefix = ".infini_" + } + return indexPrefix +} + +func getMetricsRetentionPolicyID(indexPrefix string, days int) string { + return fmt.Sprintf("ilm_%smetrics-%ddays-retention", indexPrefix, days) +} + +func getRollupRetentionPolicyID(indexPrefix string, days int) string { + return fmt.Sprintf("ilm_%srollup-%ddays-retention", indexPrefix, days) +} + +func getManagedRetentionTemplateNames(indexPrefix string) []string { + return []string{ + indexPrefix + "metrics-rollover", + indexPrefix + "logs-rollover", + indexPrefix + "requests_logging-rollover", + indexPrefix + "async_bulk_results-rollover", + indexPrefix + "alert-history-rollover", + indexPrefix + "activities-rollover", + } +} + +func getManagedRetentionAliases(indexPrefix string) []string { + return []string{ + indexPrefix + "metrics", + indexPrefix + "logs", + indexPrefix + "requests_logging", + indexPrefix + "async_bulk_results", + indexPrefix + "alert-history", + indexPrefix + "activities", + } +} + +func getManagedRetentionIndexPatterns(indexPrefix string) []string { + return []string{ + indexPrefix + "metrics*", + indexPrefix + "logs*", + indexPrefix + "requests_logging*", + indexPrefix + "async_bulk_results*", + indexPrefix + "alert-history*", + indexPrefix + "activities*", + } +} + +func castMap(value interface{}) (map[string]interface{}, bool) { + switch v := value.(type) { + case map[string]interface{}: + return v, true + case util.MapStr: + return v, true + default: + return nil, false + } +} + +func castSlice(value interface{}) ([]interface{}, bool) { + items, ok := value.([]interface{}) + return items, ok +} + +func parseRetentionDays(value interface{}) (int, error) { + if value == nil { + return 0, fmt.Errorf("retention value is empty") + } + rawValue := strings.TrimSpace(fmt.Sprint(value)) + if rawValue == "" || rawValue == "" { + return 0, fmt.Errorf("retention value is empty") + } + rawValue = strings.TrimSuffix(rawValue, "d") + days, err := strconv.Atoi(rawValue) + if err != nil || days <= 0 { + return 0, fmt.Errorf("invalid retention value: %v", value) + } + return days, nil +} + +func parseRetentionDaysFromPolicyID(policyID string) (int, error) { + matches := retentionPolicyPattern.FindStringSubmatch(strings.TrimSpace(policyID)) + if len(matches) != 2 { + return 0, fmt.Errorf("retention days not found in policy id: %s", policyID) + } + return parseRetentionDays(matches[1]) +} + +func normalizeRetentionSize(value interface{}) (string, error) { + rawValue := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(fmt.Sprint(value)), " ", "")) + if rawValue == "" || rawValue == "" { + return "", fmt.Errorf("retention size is empty") + } + if _, err := util.ToBytes(rawValue); err != nil { + return "", err + } + matches := retentionSizePattern.FindStringSubmatch(rawValue) + if len(matches) != 3 { + return "", fmt.Errorf("invalid retention size: %v", value) + } + unit := strings.ToLower(matches[2]) + switch unit { + case "k": + unit = "kb" + case "m": + unit = "mb" + case "g": + unit = "gb" + case "t": + unit = "tb" + } + return matches[1] + unit, nil +} + +func renderSetupDataTemplateContent(content string, replacements map[string]string) (string, error) { + return fasttemplate.ExecuteFuncNetestStringWithErr(content, "$[[", "]]", func(w io.Writer, tag string) (int, error) { + if value, ok := replacements[tag]; ok { + return w.Write([]byte(value)) + } + return w.Write([]byte("$[[" + tag + "]]")) + }) +} + +func resolveSystemClusterEndpointsAndHosts(cfg *elastic.ElasticsearchConfig) (string, []string, []string, error) { + if cfg == nil { + return "", nil, nil, fmt.Errorf("system cluster config not found") + } + schema := strings.TrimSpace(cfg.Schema) + endpoints := []string{} + appendUnique := func(items []string, value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return items + } + for _, item := range items { + if item == value { + return items + } + } + return append(items, value) + } + + if cfg.Endpoint != "" { + endpoints = appendUnique(endpoints, cfg.Endpoint) + } + for _, item := range cfg.Endpoints { + endpoints = appendUnique(endpoints, item) + } + if len(endpoints) == 0 { + if cfg.Host != "" { + if schema == "" { + schema = "http" + } + endpoints = appendUnique(endpoints, fmt.Sprintf("%s://%s", schema, cfg.Host)) + } + for _, item := range cfg.Hosts { + if schema == "" { + schema = "http" + } + endpoints = appendUnique(endpoints, fmt.Sprintf("%s://%s", schema, item)) + } + } + if len(endpoints) == 0 { + return "", nil, nil, fmt.Errorf("system cluster endpoint not found") + } + + hosts := []string{} + for _, endpoint := range endpoints { + parsed, err := url.Parse(endpoint) + if err != nil { + return "", nil, nil, err + } + if schema == "" { + schema = parsed.Scheme + } + host := parsed.Host + if host == "" { + host = endpoint + } + hosts = appendUnique(hosts, host) + } + if schema == "" { + schema = "http" + } + return schema, endpoints, hosts, nil +} + +func loadManagedLocalTemplateContent(dataFile string, replacements map[string]string) (string, error) { + dataFilePath := path.Join(global.Env().GetConfigDir(), "setup", "common", "data", dataFile) + content, err := util.FileGetContent(dataFilePath) + if err != nil { + return "", err + } + if len(content) == 0 { + return "", fmt.Errorf("template file %s is empty", dataFile) + } + return renderSetupDataTemplateContent(string(content), replacements) +} + +func saveManagedLocalTemplate(spec managedLocalTemplateSpec, content string, version int64, updated time.Time) error { + config := agentapi.RemoteConfig{} + config.ID = spec.ID + config.Updated = &updated + config.Metadata = model.Metadata{ + Category: "app_settings", + Name: spec.AppName, + Labels: util.MapStr{ + "instance": "_all", + }, + } + config.Payload = configCommon.ConfigFile{ + Name: spec.FileName, + Location: spec.Location, + Content: content, + Updated: version, + Version: version, + Size: int64(len(content)), + Managed: true, + } + return orm.Save(&orm.Context{Refresh: "wait_for"}, &config) +} + +func refreshManagedLocalTemplates(client elastic.API, cfg *elastic.ElasticsearchConfig) ([]string, error) { + indexPrefix := resolveSystemIndexPrefix() + agentUsername, passwordKey, err := setupplugin.ResolveManagedAgentTemplateCredentials(client, indexPrefix) + if err != nil { + return nil, err + } + schema, endpoints, hosts, err := resolveSystemClusterEndpointsAndHosts(cfg) + if err != nil { + return nil, err + } + replacements := map[string]string{ + "SETUP_AGENT_USERNAME": agentUsername, + "SETUP_AGENT_PASSWORD_KEY": passwordKey, + "SETUP_SCHEME": schema, + "SETUP_HOSTS": string(util.MustToJSONBytes(hosts)), + "SETUP_ENDPOINTS": string(util.MustToJSONBytes(endpoints)), + "SETUP_INDEX_PREFIX": indexPrefix, + "SETUP_RELAY_PARTITION_SIZE": strconv.Itoa(setupplugin.ResolveRelayPartitionSize(client)), + } + now := time.Now() + version := now.Unix() + updatedFiles := make([]string, 0, len(managedLocalTemplateSpecs)) + for _, spec := range managedLocalTemplateSpecs { + content, err := loadManagedLocalTemplateContent(spec.DataFile, replacements) + if err != nil { + return nil, err + } + if err := saveManagedLocalTemplate(spec, content, version, now); err != nil { + return nil, err + } + updatedFiles = append(updatedFiles, spec.Location) + } + return updatedFiles, nil +} + +func rawJSONRequest(requester rawRequester, cfg *elastic.ElasticsearchConfig, method, path string, payload interface{}) (map[string]interface{}, int, error) { + var body []byte + if payload != nil { + body = util.MustToJSONBytes(payload) + } + requestURL := fmt.Sprintf("%s%s", strings.TrimRight(cfg.GetAnyEndpoint(), "/"), path) + resp, err := requester.Request(context.Background(), method, requestURL, body) + if err != nil { + return nil, 0, err + } + if resp.StatusCode == http.StatusNotFound { + return nil, resp.StatusCode, nil + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return nil, resp.StatusCode, fmt.Errorf("%s", resp.Body) + } + if len(resp.Body) == 0 { + return map[string]interface{}{}, resp.StatusCode, nil + } + result := map[string]interface{}{} + if err := util.FromJSONBytes(resp.Body, &result); err != nil { + return nil, resp.StatusCode, err + } + return result, resp.StatusCode, nil +} + +func getLegacyTemplate(requester rawRequester, cfg *elastic.ElasticsearchConfig, templateName string) (map[string]interface{}, int, error) { + response, statusCode, err := rawJSONRequest(requester, cfg, util.Verb_GET, "/_template/"+templateName, nil) + if err != nil || statusCode == http.StatusNotFound { + return nil, statusCode, err + } + template, ok := castMap(response[templateName]) + if !ok { + return nil, statusCode, fmt.Errorf("template %s payload not found", templateName) + } + return template, statusCode, nil +} + +func putLegacyTemplate(requester rawRequester, cfg *elastic.ElasticsearchConfig, templateName string, template map[string]interface{}) error { + _, _, err := rawJSONRequest(requester, cfg, util.Verb_PUT, "/_template/"+templateName, template) + return err +} + +func readTemplateLifecyclePolicyID(template map[string]interface{}) (string, error) { + settings, ok := castMap(template["settings"]) + if !ok { + return "", fmt.Errorf("template settings not found") + } + if policyID, ok := settings["index.lifecycle.name"]; ok { + value := strings.TrimSpace(fmt.Sprint(policyID)) + if value != "" && value != "" { + return value, nil + } + } + if indexSettings, ok := castMap(settings["index"]); ok { + if lifecycle, ok := castMap(indexSettings["lifecycle"]); ok { + value := strings.TrimSpace(fmt.Sprint(lifecycle["name"])) + if value != "" && value != "" { + return value, nil + } + } + } + return "", fmt.Errorf("template lifecycle policy not found") +} + +func setTemplateLifecyclePolicyID(template map[string]interface{}, policyID string) error { + settings, ok := castMap(template["settings"]) + if !ok { + return fmt.Errorf("template settings not found") + } + updated := false + if _, exists := settings["index.lifecycle.name"]; exists { + settings["index.lifecycle.name"] = policyID + updated = true + } + if indexSettings, ok := castMap(settings["index"]); ok { + if lifecycle, ok := castMap(indexSettings["lifecycle"]); ok { + lifecycle["name"] = policyID + updated = true + } + } + if !updated { + return fmt.Errorf("template lifecycle policy not found") + } + return nil +} + +func getILMPolicy(requester rawRequester, cfg *elastic.ElasticsearchConfig, policyID string) (map[string]interface{}, int64, int64, int, error) { + response, statusCode, err := rawJSONRequest( + requester, + cfg, + util.Verb_GET, + "/_ilm/policy/"+url.PathEscape(policyID), + nil, + ) + if err != nil || statusCode == http.StatusNotFound { + return nil, 0, 0, statusCode, err + } + seqNoVal, primaryTermVal := extractILMPolicyConcurrency(response, policyID) + + policy, err := extractILMPolicy(response, policyID) + return policy, seqNoVal, primaryTermVal, statusCode, err +} + +func extractILMPolicyConcurrency(response map[string]interface{}, policyID string) (int64, int64) { + seqNo := int64(-1) + primaryTerm := int64(-1) + + if value, ok := readInt64Value(response["_seq_no"]); ok { + seqNo = value + } + if value, ok := readInt64Value(response["_primary_term"]); ok { + primaryTerm = value + } + if seqNo >= 0 && primaryTerm >= 0 { + return seqNo, primaryTerm + } + + if policyWrapper, ok := castMap(response[policyID]); ok { + if value, ok := readInt64Value(policyWrapper["_seq_no"]); ok { + seqNo = value + } + if value, ok := readInt64Value(policyWrapper["_primary_term"]); ok { + primaryTerm = value + } + } + + return seqNo, primaryTerm +} + +func readInt64Value(value interface{}) (int64, bool) { + switch v := value.(type) { + case json.Number: + parsed, err := v.Int64() + return parsed, err == nil + case float64: + return int64(v), true + case float32: + return int64(v), true + case int: + return int64(v), true + case int64: + return v, true + case int32: + return int64(v), true + case string: + parsed, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64) + return parsed, err == nil + default: + return 0, false + } +} + +func extractILMPolicy(response map[string]interface{}, policyID string) (map[string]interface{}, error) { + if policyWrapper, ok := castMap(response[policyID]); ok { + if policy, ok := castMap(policyWrapper["policy"]); ok { + return policy, nil + } + return nil, fmt.Errorf("ilm policy body not found") + } + + if policy, ok := castMap(response["policy"]); ok { + return policy, nil + } + + if len(response) == 1 { + for _, value := range response { + if policyWrapper, ok := castMap(value); ok { + if policy, ok := castMap(policyWrapper["policy"]); ok { + return policy, nil + } + } + } + } + + return nil, fmt.Errorf("ilm policy payload not found") +} + +func normalizeILMActionForPut(actionName string, rawConfig interface{}) (string, interface{}, bool) { + switch actionName { + case "retry", "timeout": + return "", nil, false + } + + config, ok := castMap(rawConfig) + if !ok { + if rawConfig == nil { + return actionName, util.MapStr{}, true + } + return actionName, rawConfig, true + } + + normalized := util.MapStr{} + for key, value := range config { + normalized[key] = value + } + + switch actionName { + case "rollover": + if value, exists := normalized["min_index_age"]; exists { + if _, hasMaxAge := normalized["max_age"]; !hasMaxAge { + normalized["max_age"] = value + } + delete(normalized, "min_index_age") + } + if value, exists := normalized["min_size"]; exists { + if _, hasMaxSize := normalized["max_size"]; !hasMaxSize { + normalized["max_size"] = value + } + delete(normalized, "min_size") + } + if value, exists := normalized["min_doc_count"]; exists { + if _, hasMaxDocs := normalized["max_docs"]; !hasMaxDocs { + normalized["max_docs"] = value + } + delete(normalized, "min_doc_count") + } + case "index_priority": + actionName = "set_priority" + case "allocation": + actionName = "allocate" + delete(normalized, "wait_for") + case "force_merge": + actionName = "forcemerge" + case "read_only": + actionName = "readonly" + } + + return actionName, normalized, true +} + +func normalizeILMPolicyForPut(policy map[string]interface{}) (map[string]interface{}, error) { + if phases, ok := castMap(policy["phases"]); ok { + return util.MapStr{ + "phases": phases, + }, nil + } + + states, ok := castSlice(policy["states"]) + if !ok { + return nil, fmt.Errorf("ilm phases not found") + } + + phaseMinAge := map[string]interface{}{ + "hot": "0ms", + } + phaseBodies := util.MapStr{} + + for _, stateValue := range states { + state, ok := castMap(stateValue) + if !ok { + continue + } + + stateName := strings.TrimSpace(fmt.Sprint(state["name"])) + if stateName == "" || stateName == "" { + continue + } + + if transitions, ok := castSlice(state["transitions"]); ok { + for _, transitionValue := range transitions { + transition, ok := castMap(transitionValue) + if !ok { + continue + } + targetState := strings.TrimSpace(fmt.Sprint(transition["state_name"])) + if targetState == "" || targetState == "" { + continue + } + if conditions, ok := castMap(transition["conditions"]); ok { + if minIndexAge, exists := conditions["min_index_age"]; exists { + phaseMinAge[targetState] = minIndexAge + } + } + } + } + + phase := util.MapStr{} + if minAge, exists := phaseMinAge[stateName]; exists { + phase["min_age"] = minAge + } + + phaseActions := util.MapStr{} + if actions, ok := castSlice(state["actions"]); ok { + for _, actionValue := range actions { + action, ok := castMap(actionValue) + if !ok { + continue + } + for actionName, actionConfig := range action { + normalizedName, normalizedConfig, shouldInclude := normalizeILMActionForPut(actionName, actionConfig) + if !shouldInclude { + continue + } + phaseActions[normalizedName] = normalizedConfig + } + } + } + if len(phaseActions) > 0 { + phase["actions"] = phaseActions + } + + phaseBodies[stateName] = phase + } + + if len(phaseBodies) == 0 { + return nil, fmt.Errorf("ilm phases not found") + } + + for phaseName, minAge := range phaseMinAge { + if phase, ok := castMap(phaseBodies[phaseName]); ok { + if _, exists := phase["min_age"]; !exists { + phase["min_age"] = minAge + } + } + } + + return util.MapStr{ + "phases": phaseBodies, + }, nil +} + +func putILMPolicy(requester rawRequester, cfg *elastic.ElasticsearchConfig, policyID string, policy map[string]interface{}, seqNo, primaryTerm int64) error { + normalizedPolicy, err := normalizeILMPolicyForPut(policy) + if err != nil { + return err + } + path := "/_ilm/policy/" + url.PathEscape(policyID) + if seqNo >= 0 && primaryTerm >= 0 { + path += fmt.Sprintf("?if_seq_no=%d&if_primary_term=%d", seqNo, primaryTerm) + } + _, _, err = rawJSONRequest( + requester, + cfg, + util.Verb_PUT, + path, + util.MapStr{"policy": normalizedPolicy}, + ) + return err +} + +func deleteILMPolicy(requester rawRequester, cfg *elastic.ElasticsearchConfig, policyID string) error { + _, _, err := rawJSONRequest( + requester, + cfg, + util.Verb_DELETE, + "/_ilm/policy/"+url.PathEscape(policyID), + nil, + ) + return err +} + +func getILMRetentionDays(policy map[string]interface{}) (int, error) { + phases, ok := castMap(policy["phases"]) + if ok { + if deletePhase, ok := castMap(phases["delete"]); ok { + if days, err := parseRetentionDays(deletePhase["min_age"]); err == nil { + return days, nil + } + } + if hotPhase, ok := castMap(phases["hot"]); ok { + if actions, ok := castMap(hotPhase["actions"]); ok { + if rollover, ok := castMap(actions["rollover"]); ok { + return parseRetentionDays(rollover["max_age"]) + } + } + } + } + if _, ok := castSlice(policy["states"]); ok { + return getISMRetentionDays(policy) + } + return 0, fmt.Errorf("ilm phases not found") +} + +func getILMRetentionMaxSize(policy map[string]interface{}) (string, error) { + phases, ok := castMap(policy["phases"]) + if ok { + if hotPhase, ok := castMap(phases["hot"]); ok { + if actions, ok := castMap(hotPhase["actions"]); ok { + if rollover, ok := castMap(actions["rollover"]); ok { + return normalizeRetentionSize(rollover["max_size"]) + } + } + } + } + if _, ok := castSlice(policy["states"]); ok { + return getISMRetentionMaxSize(policy) + } + return "", fmt.Errorf("ilm rollover max_size not found") +} + +func setILMRetentionDays(policy map[string]interface{}, days int) error { + phases, ok := castMap(policy["phases"]) + if ok { + retention := fmt.Sprintf("%dd", days) + updated := false + if hotPhase, ok := castMap(phases["hot"]); ok { + if actions, ok := castMap(hotPhase["actions"]); ok { + if rollover, ok := castMap(actions["rollover"]); ok { + rollover["max_age"] = retention + updated = true + } + } + } + if deletePhase, ok := castMap(phases["delete"]); ok { + deletePhase["min_age"] = retention + updated = true + } + if !updated { + return fmt.Errorf("ilm retention settings not found") + } + return nil + } + if _, ok := castSlice(policy["states"]); ok { + return setISMRetentionDays(policy, days) + } + return fmt.Errorf("ilm phases not found") +} + +func setILMRetentionMaxSize(policy map[string]interface{}, size string) error { + normalizedSize, err := normalizeRetentionSize(size) + if err != nil { + return err + } + phases, ok := castMap(policy["phases"]) + if ok { + if hotPhase, ok := castMap(phases["hot"]); ok { + if actions, ok := castMap(hotPhase["actions"]); ok { + if rollover, ok := castMap(actions["rollover"]); ok { + rollover["max_size"] = normalizedSize + return nil + } + } + } + return fmt.Errorf("ilm rollover settings not found") + } + if _, ok := castSlice(policy["states"]); ok { + return setISMRetentionMaxSize(policy, normalizedSize) + } + return fmt.Errorf("ilm phases not found") +} + +func setRollupILMRetentionDays(policy map[string]interface{}, days int) error { + phases, ok := castMap(policy["phases"]) + if ok { + deletePhase, ok := castMap(phases["delete"]) + if !ok { + return fmt.Errorf("ilm delete phase not found") + } + retention := fmt.Sprintf("%dd", days) + deletePhase["min_age"] = retention + if actions, ok := castMap(deletePhase["actions"]); ok { + if deleteAction, ok := castMap(actions["delete"]); ok { + if _, exists := deleteAction["min_data_age"]; exists { + deleteAction["min_data_age"] = retention + } + } + } + return nil + } + states, ok := castSlice(policy["states"]) + if !ok { + return fmt.Errorf("ilm phases not found") + } + retention := fmt.Sprintf("%dd", days) + updated := false + for _, stateValue := range states { + state, ok := castMap(stateValue) + if !ok { + continue + } + if transitions, ok := castSlice(state["transitions"]); ok { + for _, transitionValue := range transitions { + transition, ok := castMap(transitionValue) + if !ok { + continue + } + if conditions, ok := castMap(transition["conditions"]); ok { + if _, exists := conditions["min_index_age"]; exists { + conditions["min_index_age"] = retention + updated = true + } + } + } + } + if actions, ok := castSlice(state["actions"]); ok { + for _, actionValue := range actions { + action, ok := castMap(actionValue) + if !ok { + continue + } + if deleteAction, ok := castMap(action["delete"]); ok { + if _, exists := deleteAction["min_data_age"]; exists { + deleteAction["min_data_age"] = retention + updated = true + } + } + } + } + } + if !updated { + return fmt.Errorf("ilm retention settings not found") + } + return nil +} + +func getISMRetentionDays(policy map[string]interface{}) (int, error) { + states, ok := castSlice(policy["states"]) + if !ok { + return 0, fmt.Errorf("ism states not found") + } + for _, stateValue := range states { + state, ok := castMap(stateValue) + if !ok { + continue + } + if transitions, ok := castSlice(state["transitions"]); ok { + for _, transitionValue := range transitions { + transition, ok := castMap(transitionValue) + if !ok { + continue + } + if conditions, ok := castMap(transition["conditions"]); ok { + if days, err := parseRetentionDays(conditions["min_index_age"]); err == nil { + return days, nil + } + } + } + } + if actions, ok := castSlice(state["actions"]); ok { + for _, actionValue := range actions { + action, ok := castMap(actionValue) + if !ok { + continue + } + if rollover, ok := castMap(action["rollover"]); ok { + if days, err := parseRetentionDays(rollover["min_index_age"]); err == nil { + return days, nil + } + } + } + } + } + return 0, fmt.Errorf("ism retention days not found") +} + +func getISMRetentionMaxSize(policy map[string]interface{}) (string, error) { + states, ok := castSlice(policy["states"]) + if !ok { + return "", fmt.Errorf("ism states not found") + } + for _, stateValue := range states { + state, ok := castMap(stateValue) + if !ok { + continue + } + if actions, ok := castSlice(state["actions"]); ok { + for _, actionValue := range actions { + action, ok := castMap(actionValue) + if !ok { + continue + } + if rollover, ok := castMap(action["rollover"]); ok { + if size, err := normalizeRetentionSize(rollover["min_size"]); err == nil { + return size, nil + } + } + } + } + } + return "", fmt.Errorf("ism retention max size not found") +} + +func setISMRetentionDays(policy map[string]interface{}, days int) error { + states, ok := castSlice(policy["states"]) + if !ok { + return fmt.Errorf("ism states not found") + } + retention := fmt.Sprintf("%dd", days) + updated := false + for _, stateValue := range states { + state, ok := castMap(stateValue) + if !ok { + continue + } + if actions, ok := castSlice(state["actions"]); ok { + for _, actionValue := range actions { + action, ok := castMap(actionValue) + if !ok { + continue + } + if rollover, ok := castMap(action["rollover"]); ok { + rollover["min_index_age"] = retention + updated = true + } + } + } + if transitions, ok := castSlice(state["transitions"]); ok { + for _, transitionValue := range transitions { + transition, ok := castMap(transitionValue) + if !ok { + continue + } + if conditions, ok := castMap(transition["conditions"]); ok { + if _, exists := conditions["min_index_age"]; exists { + conditions["min_index_age"] = retention + updated = true + } + } + } + } + } + if !updated { + return fmt.Errorf("ism retention settings not found") + } + return nil +} + +func setISMRetentionMaxSize(policy map[string]interface{}, size string) error { + normalizedSize, err := normalizeRetentionSize(size) + if err != nil { + return err + } + states, ok := castSlice(policy["states"]) + if !ok { + return fmt.Errorf("ism states not found") + } + updated := false + for _, stateValue := range states { + state, ok := castMap(stateValue) + if !ok { + continue + } + if actions, ok := castSlice(state["actions"]); ok { + for _, actionValue := range actions { + action, ok := castMap(actionValue) + if !ok { + continue + } + if rollover, ok := castMap(action["rollover"]); ok { + rollover["min_size"] = normalizedSize + updated = true + } + } + } + } + if !updated { + return fmt.Errorf("ism rollover settings not found") + } + return nil +} + +func getRetentionSettings(client elastic.API, cfg *elastic.ElasticsearchConfig) (int, string, error) { + requester, ok := client.(rawRequester) + if !ok { + return 0, "", fmt.Errorf("cluster client does not support raw requests") + } + indexPrefix := resolveSystemIndexPrefix() + if strings.EqualFold(cfg.Distribution, elastic.Opensearch) { + policyID := getMetricsRetentionPolicyID(indexPrefix, defaultRetentionDays) + response, statusCode, err := rawJSONRequest( + requester, + cfg, + util.Verb_GET, + "/_plugins/_ism/policies/"+url.PathEscape(policyID), + nil, + ) + if err != nil { + return 0, "", err + } + if statusCode == http.StatusNotFound { + return defaultRetentionDays, defaultRetentionMaxSize, nil + } + policy, ok := castMap(response["policy"]) + if !ok { + return 0, "", fmt.Errorf("ism policy payload not found") + } + days, err := getISMRetentionDays(policy) + if err != nil { + return 0, "", err + } + maxSize, err := getISMRetentionMaxSize(policy) + if err != nil { + maxSize = defaultRetentionMaxSize + } + return days, maxSize, nil + } + + currentMetricsPolicyID := getMetricsRetentionPolicyID(indexPrefix, defaultRetentionDays) + template, statusCode, err := getLegacyTemplate(requester, cfg, indexPrefix+"metrics-rollover") + if err != nil { + return 0, "", err + } + currentDays := defaultRetentionDays + if statusCode == http.StatusNotFound { + return currentDays, defaultRetentionMaxSize, nil + } + if policyID, err := readTemplateLifecyclePolicyID(template); err == nil { + currentMetricsPolicyID = policyID + if days, parseErr := parseRetentionDaysFromPolicyID(policyID); parseErr == nil { + currentDays = days + } + } + + policy, _, _, statusCode, err := getILMPolicy(requester, cfg, currentMetricsPolicyID) + if err != nil { + return 0, "", err + } + if statusCode == http.StatusNotFound { + return currentDays, defaultRetentionMaxSize, nil + } + days, err := getILMRetentionDays(policy) + if err != nil { + return 0, "", err + } + maxSize, err := getILMRetentionMaxSize(policy) + if err != nil { + maxSize = defaultRetentionMaxSize + } + return days, maxSize, nil +} + +func rolloverManagedRetentionWriteIndices(requester rawRequester, cfg *elastic.ElasticsearchConfig, indexPrefix string) error { + for _, aliasName := range getManagedRetentionAliases(indexPrefix) { + _, statusCode, err := rawJSONRequest( + requester, + cfg, + util.Verb_POST, + "/"+url.PathEscape(aliasName)+"/_rollover", + util.MapStr{ + "conditions": util.MapStr{ + "max_docs": 0, + }, + }, + ) + if statusCode == http.StatusNotFound { + continue + } + if err != nil { + return err + } + } + return nil +} + +func updateRetentionSettings(client elastic.API, cfg *elastic.ElasticsearchConfig, days int, maxSize string) error { + requester, ok := client.(rawRequester) + if !ok { + return fmt.Errorf("cluster client does not support raw requests") + } + normalizedMaxSize, err := normalizeRetentionSize(maxSize) + if err != nil { + return err + } + indexPrefix := resolveSystemIndexPrefix() + if strings.EqualFold(cfg.Distribution, elastic.Opensearch) { + policyID := getMetricsRetentionPolicyID(indexPrefix, defaultRetentionDays) + response, statusCode, err := rawJSONRequest( + requester, + cfg, + util.Verb_GET, + "/_plugins/_ism/policies/"+url.PathEscape(policyID), + nil, + ) + if err != nil { + return err + } + if statusCode == http.StatusNotFound { + return fmt.Errorf("ism policy %s not found", policyID) + } + policy, ok := castMap(response["policy"]) + if !ok { + return fmt.Errorf("ism policy payload not found") + } + if err := setISMRetentionDays(policy, days); err != nil { + return err + } + if err := setISMRetentionMaxSize(policy, normalizedMaxSize); err != nil { + return err + } + _, _, err = rawJSONRequest( + requester, + cfg, + util.Verb_PUT, + "/_plugins/_ism/policies/"+url.PathEscape(policyID), + util.MapStr{"policy": policy}, + ) + return err + } + + currentMetricsPolicyID := getMetricsRetentionPolicyID(indexPrefix, defaultRetentionDays) + if template, statusCode, err := getLegacyTemplate(requester, cfg, indexPrefix+"metrics-rollover"); err != nil { + return err + } else if statusCode != http.StatusNotFound { + if policyID, err := readTemplateLifecyclePolicyID(template); err == nil { + currentMetricsPolicyID = policyID + } + } + newMetricsPolicyID := getMetricsRetentionPolicyID(indexPrefix, days) + metricsPolicy, seqNo, primaryTerm, statusCode, err := getILMPolicy(requester, cfg, currentMetricsPolicyID) + if err != nil { + return err + } + if statusCode == http.StatusNotFound { + return fmt.Errorf("ilm policy %s not found", currentMetricsPolicyID) + } + if err := setILMRetentionDays(metricsPolicy, days); err != nil { + return err + } + if err := setILMRetentionMaxSize(metricsPolicy, normalizedMaxSize); err != nil { + return err + } + if currentMetricsPolicyID != newMetricsPolicyID { + if err := deleteILMPolicy(requester, cfg, newMetricsPolicyID); err != nil { + return err + } + seqNo, primaryTerm = -1, -1 + } + if err := putILMPolicy(requester, cfg, newMetricsPolicyID, metricsPolicy, seqNo, primaryTerm); err != nil { + return err + } + + for _, templateName := range getManagedRetentionTemplateNames(indexPrefix) { + template, statusCode, err := getLegacyTemplate(requester, cfg, templateName) + if err != nil { + return err + } + if statusCode == http.StatusNotFound { + continue + } + if err := setTemplateLifecyclePolicyID(template, newMetricsPolicyID); err != nil { + return err + } + if err := putLegacyTemplate(requester, cfg, templateName, template); err != nil { + return err + } + } + + _, _, err = rawJSONRequest( + requester, + cfg, + util.Verb_PUT, + "/"+strings.Join(getManagedRetentionIndexPatterns(indexPrefix), ",")+"/_settings?allow_no_indices=true&ignore_unavailable=true", + util.MapStr{ + "index": util.MapStr{ + "lifecycle": util.MapStr{ + "name": newMetricsPolicyID, + }, + }, + }, + ) + if err != nil { + return err + } + if err := rolloverManagedRetentionWriteIndices(requester, cfg, indexPrefix); err != nil { + return err + } + + rollupTemplate, statusCode, err := getLegacyTemplate(requester, cfg, "rollup_policy_template") + if err != nil { + return err + } + if statusCode == http.StatusNotFound { + return nil + } + currentRollupPolicyID, err := readTemplateLifecyclePolicyID(rollupTemplate) + if err != nil { + return err + } + newRollupPolicyID := getRollupRetentionPolicyID(indexPrefix, days) + rollupPolicy, rollupSeqNo, rollupPrimaryTerm, statusCode, err := getILMPolicy(requester, cfg, currentRollupPolicyID) + if err != nil { + return err + } + if statusCode == http.StatusNotFound { + return fmt.Errorf("ilm policy %s not found", currentRollupPolicyID) + } + if err := setRollupILMRetentionDays(rollupPolicy, days); err != nil { + return err + } + if currentRollupPolicyID != newRollupPolicyID { + if err := deleteILMPolicy(requester, cfg, newRollupPolicyID); err != nil { + return err + } + rollupSeqNo, rollupPrimaryTerm = -1, -1 + } + if err := putILMPolicy(requester, cfg, newRollupPolicyID, rollupPolicy, rollupSeqNo, rollupPrimaryTerm); err != nil { + return err + } + if err := setTemplateLifecyclePolicyID(rollupTemplate, newRollupPolicyID); err != nil { + return err + } + if err := putLegacyTemplate(requester, cfg, "rollup_policy_template", rollupTemplate); err != nil { + return err + } + _, _, err = rawJSONRequest( + requester, + cfg, + util.Verb_PUT, + "/rollup*/_settings?allow_no_indices=true&ignore_unavailable=true", + util.MapStr{ + "index.lifecycle.name": newRollupPolicyID, + }, + ) + return err +} + +func getRollupEnabled(client elastic.API) (bool, error) { + settings, err := client.GetClusterSettings(nil) + if err != nil { + return false, err + } + rollupEnabled, _ := util.GetMapValueByKeys([]string{"persistent", "rollup", "search", "enabled"}, settings) + switch value := rollupEnabled.(type) { + case string: + return strings.EqualFold(value, "true"), nil + case bool: + return value, nil + default: + return false, nil + } +} + +func getSystemClusterClient() (elastic.API, *elastic.ElasticsearchConfig, error) { + systemClusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + cfg := elastic.GetConfigNoPanic(systemClusterID) + if cfg == nil { + return nil, nil, fmt.Errorf("system cluster config not found") + } + client := elastic.GetClient(systemClusterID) + if client == nil { + return nil, nil, fmt.Errorf("system cluster client not found") + } + return client, cfg, nil +} + +func RefreshManagedLocalTemplatesForInstall() ([]string, error) { + if err := setupplugin.EnsureSystemClusterBasicAuth(); err != nil { + return nil, err + } + client, cfg, err := getSystemClusterClient() + if err != nil { + return nil, err + } + return refreshManagedLocalTemplates(client, cfg) +} + +func (h *SettingsAPI) refreshLocalTemplates(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + updatedFiles, err := RefreshManagedLocalTemplatesForInstall() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, util.MapStr{ + "updated_files": updatedFiles, + }, http.StatusOK) +} + +func (h *SettingsAPI) getRetentionSetting(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + client, cfg, err := getSystemClusterClient() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + days, maxSize, err := getRetentionSettings(client, cfg) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, util.MapStr{ + "days": days, + "max_size": maxSize, + }, http.StatusOK) +} + +func (h *SettingsAPI) updateRetentionSetting(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + payload := retentionSettingsRequest{} + body, err := h.GetRawBody(req) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if err := util.FromJSONBytes(body, &payload); err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if payload.Days <= 0 { + h.WriteError(w, "retention days must be greater than 0", http.StatusBadRequest) + return + } + + client, cfg, err := getSystemClusterClient() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + currentMaxSize := defaultRetentionMaxSize + if payload.MaxSize == "" { + _, currentSize, err := getRetentionSettings(client, cfg) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + currentMaxSize = currentSize + } else { + currentMaxSize = payload.MaxSize + } + normalizedMaxSize, err := normalizeRetentionSize(currentMaxSize) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if err := updateRetentionSettings(client, cfg, payload.Days, normalizedMaxSize); err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, util.MapStr{ + "days": payload.Days, + "max_size": normalizedMaxSize, + }, http.StatusOK) +} + +func updateRollupJobs(requester rawRequester, cfg *elastic.ElasticsearchConfig, action string) error { + response, statusCode, err := rawJSONRequest(requester, cfg, util.Verb_GET, "/_rollup/jobs", nil) + if err != nil { + return err + } + if statusCode == http.StatusNotFound { + return nil + } + jobIDs := getManagedRollupJobIDs(response) + for _, jobID := range jobIDs { + _, statusCode, err := rawJSONRequest( + requester, + cfg, + util.Verb_POST, + fmt.Sprintf("/_rollup/jobs/%s/_%s", url.PathEscape(jobID), action), + nil, + ) + if statusCode == http.StatusNotFound { + continue + } + if err != nil { + return err + } + } + return nil +} + +func getManagedRollupJobIDs(response map[string]interface{}) []string { + jobs, ok := response["jobs"].([]interface{}) + if !ok || len(jobs) == 0 { + return nil + } + jobIDs := make([]string, 0, len(jobs)) + for _, item := range jobs { + job, ok := item.(map[string]interface{}) + if !ok { + continue + } + config, ok := job["config"].(map[string]interface{}) + if !ok { + continue + } + jobID, _ := config["id"].(string) + jobID = strings.TrimSpace(jobID) + if !strings.HasPrefix(jobID, "rollup_") { + continue + } + jobIDs = append(jobIDs, jobID) + } + return jobIDs +} + +func (h *SettingsAPI) getRollupSetting(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + client, _, err := getSystemClusterClient() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + enabled, err := getRollupEnabled(client) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteJSON(w, util.MapStr{ + "enabled": enabled, + }, http.StatusOK) +} + +func (h *SettingsAPI) updateRollupSetting(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + payload := rollupSettingsRequest{} + body, err := h.GetRawBody(req) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if err := util.FromJSONBytes(body, &payload); err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + client, cfg, err := getSystemClusterClient() + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if !payload.Enabled { + requester, ok := client.(rawRequester) + if !ok { + err = fmt.Errorf("cluster client does not support raw requests") + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + err = updateRollupJobs(requester, cfg, "stop") + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + + err = client.UpdateClusterSettings(util.MustToJSONBytes(util.MapStr{ + "persistent": util.MapStr{ + "rollup": util.MapStr{ + "search": util.MapStr{ + "enabled": fmt.Sprintf("%t", payload.Enabled), + }, + }, + }, + })) + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + if payload.Enabled { + requester, ok := client.(rawRequester) + if !ok { + err = fmt.Errorf("cluster client does not support raw requests") + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + err = updateRollupJobs(requester, cfg, "start") + if err != nil { + log.Error(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } + + h.WriteJSON(w, util.MapStr{ + "enabled": payload.Enabled, + }, http.StatusOK) +} diff --git a/plugin/api/settings/api_test.go b/plugin/api/settings/api_test.go new file mode 100644 index 00000000..c63747f3 --- /dev/null +++ b/plugin/api/settings/api_test.go @@ -0,0 +1,759 @@ +package settings + +import ( + "context" + "net/http" + "strings" + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/util" +) + +func TestSetILMRetentionDays(t *testing.T) { + policy := map[string]interface{}{ + "phases": map[string]interface{}{ + "hot": map[string]interface{}{ + "actions": map[string]interface{}{ + "rollover": map[string]interface{}{ + "max_age": "30d", + "max_size": "50gb", + }, + }, + }, + "delete": map[string]interface{}{ + "min_age": "30d", + }, + }, + } + + if err := setILMRetentionDays(policy, 90); err != nil { + t.Fatalf("setILMRetentionDays returned error: %v", err) + } + + days, err := getILMRetentionDays(policy) + if err != nil { + t.Fatalf("getILMRetentionDays returned error: %v", err) + } + if days != 90 { + t.Fatalf("expected 90 retention days, got %d", days) + } +} + +func TestNormalizeRetentionSize(t *testing.T) { + size, err := normalizeRetentionSize("50 GB") + if err != nil { + t.Fatalf("normalizeRetentionSize returned error: %v", err) + } + if size != "50gb" { + t.Fatalf("expected normalized size 50gb, got %s", size) + } +} + +func TestRenderSetupDataTemplateContentSupportsNestedPlaceholders(t *testing.T) { + content := `password: "$[[keystore.$[[SETUP_AGENT_PASSWORD_KEY]]]]"` + "\n" + `hosts: $[[SETUP_HOSTS]]` + rendered, err := renderSetupDataTemplateContent(content, map[string]string{ + "SETUP_AGENT_PASSWORD_KEY": "SYSTEM_CLUSTER_INGEST_PASSWORD", + "SETUP_HOSTS": `["192.168.3.8:9200"]`, + }) + if err != nil { + t.Fatalf("renderSetupDataTemplateContent returned error: %v", err) + } + if !strings.Contains(rendered, `password: "$[[keystore.SYSTEM_CLUSTER_INGEST_PASSWORD]]"`) { + t.Fatalf("expected nested password placeholder to be resolved, got %s", rendered) + } + if !strings.Contains(rendered, `hosts: ["192.168.3.8:9200"]`) { + t.Fatalf("expected hosts array to be rendered, got %s", rendered) + } +} + +func TestResolveSystemClusterEndpointsAndHosts(t *testing.T) { + cfg := &elastic.ElasticsearchConfig{ + Endpoints: []string{"https://192.168.3.8:9200", "https://192.168.3.9:9200"}, + } + schema, endpoints, hosts, err := resolveSystemClusterEndpointsAndHosts(cfg) + if err != nil { + t.Fatalf("resolveSystemClusterEndpointsAndHosts returned error: %v", err) + } + if schema != "https" { + t.Fatalf("expected https schema, got %s", schema) + } + if len(endpoints) != 2 || endpoints[0] != "https://192.168.3.8:9200" || endpoints[1] != "https://192.168.3.9:9200" { + t.Fatalf("unexpected endpoints %#v", endpoints) + } + if len(hosts) != 2 || hosts[0] != "192.168.3.8:9200" || hosts[1] != "192.168.3.9:9200" { + t.Fatalf("unexpected hosts %#v", hosts) + } +} + +func TestSetILMRetentionDaysFromStates(t *testing.T) { + policy := map[string]interface{}{ + "states": []interface{}{ + map[string]interface{}{ + "name": "hot", + "actions": []interface{}{ + map[string]interface{}{ + "rollover": map[string]interface{}{ + "min_index_age": "30d", + }, + }, + }, + "transitions": []interface{}{ + map[string]interface{}{ + "state_name": "delete", + "conditions": map[string]interface{}{ + "min_index_age": "30d", + }, + }, + }, + }, + }, + } + + if err := setILMRetentionDays(policy, 7); err != nil { + t.Fatalf("setILMRetentionDays returned error for states payload: %v", err) + } + + days, err := getILMRetentionDays(policy) + if err != nil { + t.Fatalf("getILMRetentionDays returned error for states payload: %v", err) + } + if days != 7 { + t.Fatalf("expected 7 retention days, got %d", days) + } +} + +func TestSetILMRetentionMaxSize(t *testing.T) { + policy := map[string]interface{}{ + "phases": map[string]interface{}{ + "hot": map[string]interface{}{ + "actions": map[string]interface{}{ + "rollover": map[string]interface{}{ + "max_age": "30d", + "max_size": "50gb", + }, + }, + }, + "delete": map[string]interface{}{ + "min_age": "30d", + }, + }, + } + + if err := setILMRetentionMaxSize(policy, "80 GB"); err != nil { + t.Fatalf("setILMRetentionMaxSize returned error: %v", err) + } + + size, err := getILMRetentionMaxSize(policy) + if err != nil { + t.Fatalf("getILMRetentionMaxSize returned error: %v", err) + } + if size != "80gb" { + t.Fatalf("expected 80gb retention size, got %s", size) + } +} + +func TestNormalizeILMPolicyForPutFromStates(t *testing.T) { + policy := map[string]interface{}{ + "states": []interface{}{ + map[string]interface{}{ + "name": "hot", + "actions": []interface{}{ + map[string]interface{}{ + "retry": map[string]interface{}{ + "count": 3, + "backoff": "exponential", + "delay": "1m", + }, + "rollover": map[string]interface{}{ + "min_index_age": "30d", + "min_size": "50gb", + "min_doc_count": 100000000, + }, + }, + map[string]interface{}{ + "retry": map[string]interface{}{ + "count": 3, + "backoff": "exponential", + "delay": "1m", + }, + "index_priority": map[string]interface{}{ + "priority": 100, + }, + }, + }, + "transitions": []interface{}{ + map[string]interface{}{ + "state_name": "delete", + "conditions": map[string]interface{}{ + "min_index_age": "30d", + }, + }, + }, + }, + map[string]interface{}{ + "name": "delete", + "actions": []interface{}{ + map[string]interface{}{ + "delete": nil, + }, + }, + }, + }, + } + + if err := setILMRetentionDays(policy, 7); err != nil { + t.Fatalf("setILMRetentionDays returned error for states payload: %v", err) + } + + normalized, err := normalizeILMPolicyForPut(policy) + if err != nil { + t.Fatalf("normalizeILMPolicyForPut returned error: %v", err) + } + + phases := normalized["phases"].(map[string]interface{}) + hotPhase := phases["hot"].(map[string]interface{}) + if got := hotPhase["min_age"]; got != "0ms" { + t.Fatalf("expected hot min_age 0ms, got %v", got) + } + hotActions := hotPhase["actions"].(map[string]interface{}) + rollover := hotActions["rollover"].(map[string]interface{}) + if got := rollover["max_age"]; got != "7d" { + t.Fatalf("expected rollover max_age 7d, got %v", got) + } + if got := rollover["max_size"]; got != "50gb" { + t.Fatalf("expected rollover max_size 50gb, got %v", got) + } + if got := rollover["max_docs"]; got != 100000000 { + t.Fatalf("expected rollover max_docs 100000000, got %v", got) + } + if _, exists := rollover["min_index_age"]; exists { + t.Fatalf("expected rollover min_index_age to be removed, got %#v", rollover) + } + if _, exists := rollover["min_size"]; exists { + t.Fatalf("expected rollover min_size to be removed, got %#v", rollover) + } + if _, exists := rollover["min_doc_count"]; exists { + t.Fatalf("expected rollover min_doc_count to be removed, got %#v", rollover) + } + if _, exists := hotActions["retry"]; exists { + t.Fatalf("expected retry metadata to be stripped from ILM actions, got %#v", hotActions) + } + + setPriority := hotActions["set_priority"].(map[string]interface{}) + if got := setPriority["priority"]; got != 100 { + t.Fatalf("expected set_priority priority 100, got %v", got) + } + + deletePhase := phases["delete"].(map[string]interface{}) + if got := deletePhase["min_age"]; got != "7d" { + t.Fatalf("expected delete min_age 7d, got %v", got) + } + deleteActions := deletePhase["actions"].(map[string]interface{}) + if got := deleteActions["delete"]; got == nil { + t.Fatalf("expected delete action payload to be an object, got nil") + } + if deletePayload, ok := deleteActions["delete"].(map[string]interface{}); !ok || len(deletePayload) != 0 { + t.Fatalf("expected empty delete action payload, got %#v", deleteActions["delete"]) + } +} + +func TestSetISMRetentionDays(t *testing.T) { + policy := map[string]interface{}{ + "states": []interface{}{ + map[string]interface{}{ + "name": "hot", + "actions": []interface{}{ + map[string]interface{}{ + "rollover": map[string]interface{}{ + "min_index_age": "30d", + "min_size": "50gb", + }, + }, + }, + "transitions": []interface{}{ + map[string]interface{}{ + "state_name": "delete", + "conditions": map[string]interface{}{ + "min_index_age": "30d", + }, + }, + }, + }, + }, + } + + if err := setISMRetentionDays(policy, 60); err != nil { + t.Fatalf("setISMRetentionDays returned error: %v", err) + } + + days, err := getISMRetentionDays(policy) + if err != nil { + t.Fatalf("getISMRetentionDays returned error: %v", err) + } + if days != 60 { + t.Fatalf("expected 60 retention days, got %d", days) + } +} + +func TestSetISMRetentionMaxSize(t *testing.T) { + policy := map[string]interface{}{ + "states": []interface{}{ + map[string]interface{}{ + "name": "hot", + "actions": []interface{}{ + map[string]interface{}{ + "rollover": map[string]interface{}{ + "min_index_age": "30d", + "min_size": "50gb", + }, + }, + }, + }, + }, + } + + if err := setISMRetentionMaxSize(policy, "120g"); err != nil { + t.Fatalf("setISMRetentionMaxSize returned error: %v", err) + } + + size, err := getISMRetentionMaxSize(policy) + if err != nil { + t.Fatalf("getISMRetentionMaxSize returned error: %v", err) + } + if size != "120gb" { + t.Fatalf("expected 120gb retention size, got %s", size) + } +} + +func TestGetManagedRollupJobIDs(t *testing.T) { + jobIDs := getManagedRollupJobIDs(map[string]interface{}{ + "jobs": []interface{}{ + map[string]interface{}{ + "config": map[string]interface{}{ + "id": "rollup_node_stats", + }, + }, + map[string]interface{}{ + "config": map[string]interface{}{ + "id": "custom_job", + }, + }, + map[string]interface{}{ + "config": map[string]interface{}{ + "id": " rollup_cluster_stats ", + }, + }, + }, + }) + if len(jobIDs) != 2 { + t.Fatalf("expected 2 managed rollup jobs, got %#v", jobIDs) + } + if jobIDs[0] != "rollup_node_stats" || jobIDs[1] != "rollup_cluster_stats" { + t.Fatalf("unexpected managed rollup job ids %#v", jobIDs) + } +} + +func TestUpdateRollupJobsStopsManagedJobsIndividually(t *testing.T) { + requester := &mockRawRequester{ + responses: map[string]*util.Result{ + "GET http://example.com/_rollup/jobs": { + StatusCode: http.StatusOK, + Body: []byte(`{"jobs":[ + {"config":{"id":"rollup_node_stats"}}, + {"config":{"id":"custom_job"}}, + {"config":{"id":"rollup_cluster_health"}} + ]}`), + }, + "POST http://example.com/_rollup/jobs/rollup_node_stats/_stop": { + StatusCode: http.StatusOK, + Body: []byte(`{"stopped":true}`), + }, + "POST http://example.com/_rollup/jobs/rollup_cluster_health/_stop": { + StatusCode: http.StatusOK, + Body: []byte(`{"stopped":true}`), + }, + }, + } + + if err := updateRollupJobs(requester, &elastic.ElasticsearchConfig{Endpoint: "http://example.com"}, "stop"); err != nil { + t.Fatalf("updateRollupJobs returned error: %v", err) + } + + if len(requester.calls) != 3 { + t.Fatalf("expected 3 requests, got %#v", requester.calls) + } + if requester.calls[0] != "GET http://example.com/_rollup/jobs" { + t.Fatalf("expected rollup jobs list request first, got %s", requester.calls[0]) + } + if strings.Contains(strings.Join(requester.calls, "\n"), "custom_job") { + t.Fatalf("expected custom job to be skipped, got %#v", requester.calls) + } +} + +func TestRolloverManagedRetentionWriteIndices(t *testing.T) { + requester := &mockRawRequester{ + responses: map[string]*util.Result{ + "POST http://example.com/.infini_metrics/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_logs/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_requests_logging/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_async_bulk_results/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_alert-history/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_activities/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + }, + } + + if err := rolloverManagedRetentionWriteIndices(requester, &elastic.ElasticsearchConfig{Endpoint: "http://example.com"}, ".infini_"); err != nil { + t.Fatalf("rolloverManagedRetentionWriteIndices returned error: %v", err) + } + + if len(requester.calls) != 6 { + t.Fatalf("expected 6 rollover requests, got %#v", requester.calls) + } +} + +func TestRolloverManagedRetentionWriteIndicesSkipsMissingAlias(t *testing.T) { + requester := &mockRawRequester{ + responses: map[string]*util.Result{ + "POST http://example.com/.infini_metrics/_rollover": { + StatusCode: http.StatusNotFound, + Body: []byte(`{"error":"alias [.infini_metrics] missing"}`), + }, + "POST http://example.com/.infini_logs/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_requests_logging/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_async_bulk_results/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_alert-history/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + "POST http://example.com/.infini_activities/_rollover": { + StatusCode: http.StatusOK, + Body: []byte(`{"rolled_over":true}`), + }, + }, + } + + if err := rolloverManagedRetentionWriteIndices(requester, &elastic.ElasticsearchConfig{Endpoint: "http://example.com"}, ".infini_"); err != nil { + t.Fatalf("rolloverManagedRetentionWriteIndices should ignore 404 aliases, got %v", err) + } +} + +type mockRawRequester struct { + responses map[string]*util.Result + calls []string +} + +func (m *mockRawRequester) Request(_ context.Context, method, requestURL string, _ []byte) (*util.Result, error) { + key := method + " " + requestURL + m.calls = append(m.calls, key) + if response, ok := m.responses[key]; ok { + return response, nil + } + return &util.Result{ + StatusCode: http.StatusNotFound, + Body: []byte(`{"error":"not found"}`), + }, nil +} + +func TestSetRollupILMRetentionDaysFromStates(t *testing.T) { + policy := map[string]interface{}{ + "states": []interface{}{ + map[string]interface{}{ + "name": "hot", + "transitions": []interface{}{ + map[string]interface{}{ + "state_name": "delete", + "conditions": map[string]interface{}{ + "min_index_age": "30d", + }, + }, + }, + }, + map[string]interface{}{ + "name": "delete", + "actions": []interface{}{ + map[string]interface{}{ + "delete": map[string]interface{}{ + "min_data_age": "30d", + }, + }, + }, + }, + }, + } + + if err := setRollupILMRetentionDays(policy, 7); err != nil { + t.Fatalf("setRollupILMRetentionDays returned error for states payload: %v", err) + } + + states := policy["states"].([]interface{}) + hotState := states[0].(map[string]interface{}) + transition := hotState["transitions"].([]interface{})[0].(map[string]interface{}) + if got := transition["conditions"].(map[string]interface{})["min_index_age"]; got != "7d" { + t.Fatalf("expected transition min_index_age 7d, got %v", got) + } + + deleteState := states[1].(map[string]interface{}) + deleteAction := deleteState["actions"].([]interface{})[0].(map[string]interface{})["delete"].(map[string]interface{}) + if got := deleteAction["min_data_age"]; got != "7d" { + t.Fatalf("expected delete min_data_age 7d, got %v", got) + } +} + +func TestNormalizeRollupILMPolicyForPutFromStates(t *testing.T) { + policy := map[string]interface{}{ + "states": []interface{}{ + map[string]interface{}{ + "name": "hot", + "transitions": []interface{}{ + map[string]interface{}{ + "state_name": "delete", + "conditions": map[string]interface{}{ + "min_index_age": "30d", + }, + }, + }, + }, + map[string]interface{}{ + "name": "delete", + "actions": []interface{}{ + map[string]interface{}{ + "delete": map[string]interface{}{ + "timestamp_field": "timestamp.date_histogram", + "min_data_age": "30d", + }, + }, + }, + }, + }, + } + + if err := setRollupILMRetentionDays(policy, 7); err != nil { + t.Fatalf("setRollupILMRetentionDays returned error for states payload: %v", err) + } + + normalized, err := normalizeILMPolicyForPut(policy) + if err != nil { + t.Fatalf("normalizeILMPolicyForPut returned error: %v", err) + } + + phases := normalized["phases"].(map[string]interface{}) + deletePhase := phases["delete"].(map[string]interface{}) + if got := deletePhase["min_age"]; got != "7d" { + t.Fatalf("expected delete min_age 7d, got %v", got) + } + + deleteAction := deletePhase["actions"].(map[string]interface{})["delete"].(map[string]interface{}) + if got := deleteAction["min_data_age"]; got != "7d" { + t.Fatalf("expected delete min_data_age 7d, got %v", got) + } + if got := deleteAction["timestamp_field"]; got != "timestamp.date_histogram" { + t.Fatalf("expected timestamp_field to be preserved, got %v", got) + } +} + +func TestParseRetentionDays(t *testing.T) { + days, err := parseRetentionDays("30d") + if err != nil { + t.Fatalf("parseRetentionDays returned error: %v", err) + } + if days != 30 { + t.Fatalf("expected 30 retention days, got %d", days) + } +} + +func TestParseRetentionDaysFromPolicyID(t *testing.T) { + days, err := parseRetentionDaysFromPolicyID("ilm_.infini_metrics-7days-retention") + if err != nil { + t.Fatalf("parseRetentionDaysFromPolicyID returned error: %v", err) + } + if days != 7 { + t.Fatalf("expected 7 retention days, got %d", days) + } +} + +func TestSetRollupILMRetentionDays(t *testing.T) { + policy := map[string]interface{}{ + "phases": map[string]interface{}{ + "hot": map[string]interface{}{ + "min_age": "0ms", + }, + "delete": map[string]interface{}{ + "min_age": "30d", + "actions": map[string]interface{}{ + "delete": map[string]interface{}{ + "min_data_age": "30d", + }, + }, + }, + }, + } + + if err := setRollupILMRetentionDays(policy, 7); err != nil { + t.Fatalf("setRollupILMRetentionDays returned error: %v", err) + } + + deletePhase := policy["phases"].(map[string]interface{})["delete"].(map[string]interface{}) + if got := deletePhase["min_age"]; got != "7d" { + t.Fatalf("expected delete min_age 7d, got %v", got) + } + deleteAction := deletePhase["actions"].(map[string]interface{})["delete"].(map[string]interface{}) + if got := deleteAction["min_data_age"]; got != "7d" { + t.Fatalf("expected delete min_data_age 7d, got %v", got) + } +} + +func TestExtractILMPolicyByExactPolicyID(t *testing.T) { + response := map[string]interface{}{ + "ilm_.infini_metrics-30days-retention": map[string]interface{}{ + "policy": map[string]interface{}{ + "phases": map[string]interface{}{}, + }, + }, + } + + policy, err := extractILMPolicy(response, "ilm_.infini_metrics-30days-retention") + if err != nil { + t.Fatalf("extractILMPolicy returned error: %v", err) + } + if _, ok := policy["phases"]; !ok { + t.Fatalf("expected extracted policy phases, got %#v", policy) + } +} + +func TestExtractILMPolicyFromSingleEntryFallback(t *testing.T) { + response := map[string]interface{}{ + "ilm_.infini_metrics-7days-retention": map[string]interface{}{ + "policy": map[string]interface{}{ + "phases": map[string]interface{}{}, + }, + }, + } + + policy, err := extractILMPolicy(response, "ilm_.infini_metrics-30days-retention") + if err != nil { + t.Fatalf("extractILMPolicy returned error: %v", err) + } + if _, ok := policy["phases"]; !ok { + t.Fatalf("expected extracted policy phases, got %#v", policy) + } +} + +func TestExtractILMPolicyFromDirectBody(t *testing.T) { + response := map[string]interface{}{ + "policy": map[string]interface{}{ + "phases": map[string]interface{}{}, + }, + } + + policy, err := extractILMPolicy(response, "ilm_.infini_metrics-30days-retention") + if err != nil { + t.Fatalf("extractILMPolicy returned error: %v", err) + } + if _, ok := policy["phases"]; !ok { + t.Fatalf("expected extracted policy phases, got %#v", policy) + } +} + +func TestExtractILMPolicyConcurrencyWithoutMetadata(t *testing.T) { + response := map[string]interface{}{ + "ilm_.infini_metrics-3days-retention": map[string]interface{}{ + "policy": map[string]interface{}{ + "phases": map[string]interface{}{}, + }, + }, + } + + seqNo, primaryTerm := extractILMPolicyConcurrency(response, "ilm_.infini_metrics-3days-retention") + if seqNo != -1 || primaryTerm != -1 { + t.Fatalf("expected missing concurrency metadata to return -1/-1, got %d/%d", seqNo, primaryTerm) + } +} + +func TestExtractILMPolicyConcurrencyFromWrapper(t *testing.T) { + response := map[string]interface{}{ + "ilm_.infini_metrics-3days-retention": map[string]interface{}{ + "_seq_no": float64(7), + "_primary_term": float64(3), + "policy": map[string]interface{}{ + "phases": map[string]interface{}{}, + }, + }, + } + + seqNo, primaryTerm := extractILMPolicyConcurrency(response, "ilm_.infini_metrics-3days-retention") + if seqNo != 7 || primaryTerm != 3 { + t.Fatalf("expected wrapper concurrency metadata 7/3, got %d/%d", seqNo, primaryTerm) + } +} + +func TestPutILMPolicySkipsConcurrencyWhenMetadataMissing(t *testing.T) { + requester := &mockRawRequester{ + responses: map[string]*util.Result{ + "PUT http://example.com/_ilm/policy/ilm_.infini_metrics-3days-retention": { + StatusCode: http.StatusOK, + Body: []byte(`{"acknowledged":true}`), + }, + }, + } + + err := putILMPolicy( + requester, + &elastic.ElasticsearchConfig{Endpoint: "http://example.com"}, + "ilm_.infini_metrics-3days-retention", + map[string]interface{}{ + "phases": map[string]interface{}{ + "hot": map[string]interface{}{ + "actions": map[string]interface{}{ + "rollover": map[string]interface{}{ + "max_age": "3d", + "max_size": "50gb", + }, + }, + }, + "delete": map[string]interface{}{ + "min_age": "3d", + }, + }, + }, + -1, + -1, + ) + if err != nil { + t.Fatalf("putILMPolicy returned error: %v", err) + } + + if len(requester.calls) != 1 { + t.Fatalf("expected one request, got %#v", requester.calls) + } + if requester.calls[0] != "PUT http://example.com/_ilm/policy/ilm_.infini_metrics-3days-retention" { + t.Fatalf("expected put without concurrency query, got %s", requester.calls[0]) + } +} diff --git a/plugin/audit_log/monitoring_interceptor.go b/plugin/audit_log/monitoring_interceptor.go index 6b287c7e..61615c99 100644 --- a/plugin/audit_log/monitoring_interceptor.go +++ b/plugin/audit_log/monitoring_interceptor.go @@ -59,7 +59,7 @@ func (m *MonitoringInterceptor) PreHandle(c context.Context, _ http.ResponseWrit targetClusterID = matches[1] eventName = strings.Replace(matches[2], "/", " ", -1) } - claims, auditLogErr := security.ValidateLogin(request.Header.Get("Authorization")) + claims, auditLogErr := security.ValidateLoginFromRequest(request) if auditLogErr == nil && handler.GetHeader(request, "Referer", "") != "" { auditLog, _ := model.NewAuditLogBuilderWithDefault().WithOperator(claims.Username). WithLogTypeAccess().WithResourceTypeClusterManagement(). diff --git a/plugin/elastic/metadata.go b/plugin/elastic/metadata.go index c3fc56d8..c3c26f9a 100644 --- a/plugin/elastic/metadata.go +++ b/plugin/elastic/metadata.go @@ -40,6 +40,7 @@ import ( "infini.sh/framework/core/pipeline" "infini.sh/framework/core/queue" "infini.sh/framework/core/util" + "time" ) type MetadataProcessor struct { @@ -98,6 +99,8 @@ func (processor *MetadataProcessor) Process(ctx *pipeline.Context) error { switch typ { case "index_health_change": //err = processor.HandleIndexHealthChange(&ev) + case "node_health_change": + err = processor.HandleNodeHealthChange(pop.Data) case "index_state_change": indexState, _, _, err := jsonparser.Get(pop.Data, "payload", "index_state") if err != nil { @@ -156,6 +159,62 @@ func (processor *MetadataProcessor) HandleUnknownNodeStatus(ev []byte) error { return err } +func (processor *MetadataProcessor) HandleNodeHealthChange(ev []byte) error { + clusterID, err := jsonparser.GetString(ev, "metadata", "labels", "cluster_id") + if err != nil { + return err + } + nodeID, err := jsonparser.GetString(ev, "metadata", "labels", "node_id") + if err != nil { + nodeID, err = jsonparser.GetString(ev, "metadata", "labels", "node_uuid") + if err != nil { + return err + } + } + status, err := jsonparser.GetString(ev, "metadata", "labels", "to") + if err != nil { + return err + } + esClient := elastic.GetClient(processor.config.Elasticsearch) + queryDslTpl := `{"script": { + "source": "ctx._source.metadata.labels.status=params.status; ctx._source.timestamp=params.timestamp", + "lang": "painless", + "params": { + "status": %s, + "timestamp": %s + } + }, + "query": { + "bool": { + "must": [ + {"term": { + "metadata.cluster_id": { + "value": %s + } + }}, + {"term": { + "metadata.node_id": { + "value": %s + } + }}, + {"term": { + "metadata.category": { + "value": "elasticsearch" + } + }} + ] + } + }}` + queryDsl := fmt.Sprintf(queryDslTpl, + util.MustToJSON(status), + util.MustToJSON(time.Now().UTC().Format(time.RFC3339Nano)), + util.MustToJSON(clusterID), + util.MustToJSON(nodeID), + ) + _, err = esClient.UpdateByQuery(orm.GetIndexName(elastic.NodeConfig{}), []byte(queryDsl)) + return err +} + func (processor *MetadataProcessor) HandleIndexHealthChange(ev *event.Event) error { // save activity activityInfo := &event.Activity{ diff --git a/plugin/managed/server/auth_validation.go b/plugin/managed/server/auth_validation.go new file mode 100644 index 00000000..4cd07cb0 --- /dev/null +++ b/plugin/managed/server/auth_validation.go @@ -0,0 +1,95 @@ +package server + +import ( + "net/http" + "strings" + + agent_common "infini.sh/console/modules/agent/common" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" + configcommon "infini.sh/framework/modules/configs/common" +) + +const legacyManagedAuthMaxVersion = agent_common.LegacyAgentMaxVersion + +func validateManagedAgentRequestAuth(req *http.Request, instance *model.Instance) error { + return agent_common.ValidateManagerRequestAuth( + req, + instance, + (*model.BasicAuth)(&global.Env().SystemConfig.Configs.ManagerConfig.BasicAuth), + ) +} + +func validateLegacyCompatibleManagedAgentRequestAuth(req *http.Request, instance *model.Instance) error { + version := "" + if instance != nil { + version = instance.Application.Version.VersionNumber + } + return validateLegacyCompatibleManagedAgentRequestAuthForVersion(req, instance, version) +} + +func validateLegacyCompatibleManagedAgentRequestAuthForVersion(req *http.Request, instance *model.Instance, version string) error { + if shouldAllowLegacyManagedRequestWithoutAuthVersion(req, version) { + return nil + } + return validateManagedAgentRequestAuth(req, instance) +} + +func isLegacyManagedRegisterRequest(req *http.Request, client *model.Instance, accessToken *configcommon.RegisterToken) bool { + if client == nil { + return false + } + if !shouldAllowLegacyManagedRequestWithoutAuth(req, client) { + return false + } + return true +} + +func isLegacyManagedBasicAuthRequest(req *http.Request, instance *model.Instance) bool { + if instance == nil { + return false + } + if !isLegacyManagedVersion(instance.Application.Version.VersionNumber) { + return false + } + managerAuth := global.Env().SystemConfig.Configs.ManagerConfig.BasicAuth + if req == nil || agent_common.ExtractManagerToken(req) != "" || strings.TrimSpace(managerAuth.Username) == "" { + return false + } + user, password, ok := req.BasicAuth() + if !ok { + return false + } + return user == managerAuth.Username && password == managerAuth.Password.Get() +} + +func shouldAllowLegacyManagedRequestWithoutAuth(req *http.Request, instance *model.Instance) bool { + if instance == nil { + return false + } + return shouldAllowLegacyManagedRequestWithoutAuthVersion(req, instance.Application.Version.VersionNumber) +} + +func shouldAllowLegacyManagedRequestWithoutAuthVersion(req *http.Request, version string) bool { + return isLegacyManagedVersion(version) +} + +func isLegacyManagedVersion(version string) bool { + version = strings.TrimSpace(version) + if version == "" { + return false + } + parsed, err := util.ParseSemantic(version) + if err != nil { + parsed, err = util.ParseGeneric(version) + if err != nil { + return false + } + } + cmp, err := parsed.Compare(legacyManagedAuthMaxVersion) + if err != nil { + return false + } + return cmp <= 0 +} diff --git a/plugin/managed/server/auth_validation_test.go b/plugin/managed/server/auth_validation_test.go new file mode 100644 index 00000000..0b4f77bf --- /dev/null +++ b/plugin/managed/server/auth_validation_test.go @@ -0,0 +1,133 @@ +package server + +import ( + "net/http/httptest" + "testing" + + agent_common "infini.sh/console/modules/agent/common" + config3 "infini.sh/framework/core/config" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + ucfg "infini.sh/framework/lib/go-ucfg" + configcommon "infini.sh/framework/modules/configs/common" +) + +func TestValidateManagedAgentRequestAuth(t *testing.T) { + oldBasicAuth := global.Env().SystemConfig.Configs.ManagerConfig.BasicAuth + t.Cleanup(func() { + global.Env().SystemConfig.Configs.ManagerConfig.BasicAuth = oldBasicAuth + }) + + global.Env().SystemConfig.Configs.ManagerConfig.BasicAuth = config3.BasicAuth{ + Username: "manager", + Password: ucfg.SecretString("secret"), + } + + t.Run("rejects missing auth", func(t *testing.T) { + req := httptest.NewRequest("POST", "/configs/_sync", nil) + err := validateManagedAgentRequestAuth(req, &model.Instance{}) + if err != agent_common.ErrInvalidManagerBasicAuth { + t.Fatalf("expected missing auth to be rejected, got %v", err) + } + }) + + t.Run("accepts valid basic auth", func(t *testing.T) { + req := httptest.NewRequest("POST", "/configs/_sync", nil) + req.SetBasicAuth("manager", "secret") + if err := validateManagedAgentRequestAuth(req, &model.Instance{}); err != nil { + t.Fatalf("expected basic auth to pass, got %v", err) + } + }) + + t.Run("legacy compatible auth accepts basic auth with credentialed instance", func(t *testing.T) { + req := httptest.NewRequest("POST", "/configs/_sync", nil) + instance := &model.Instance{ + ManagerCredentialID: "cred-1", + Application: env.Application{ + Version: env.Version{VersionNumber: "1.30.4"}, + }, + } + if err := validateLegacyCompatibleManagedAgentRequestAuth(req, instance); err != nil { + t.Fatalf("expected legacy request without manager auth to pass, got %v", err) + } + }) + + t.Run("legacy register detection only depends on legacy version", func(t *testing.T) { + req := httptest.NewRequest("POST", "/instance/_register", nil) + client := &model.Instance{ + Application: env.Application{ + Version: env.Version{VersionNumber: "1.30.4"}, + }, + } + if !isLegacyManagedRegisterRequest(req, client, nil) { + t.Fatal("expected legacy register request to be detected") + } + if !shouldAllowLegacyManagedRequestWithoutAuth(req, client) { + t.Fatal("expected legacy no-auth request to be detected") + } + }) + + t.Run("legacy register request still uses compatibility flow even if access token is present", func(t *testing.T) { + req := httptest.NewRequest("POST", "/instance/_register", nil) + accessToken := &configcommon.RegisterToken{Value: "access-token"} + client := &model.Instance{ + Application: env.Application{ + Version: env.Version{VersionNumber: "1.30.4"}, + }, + } + if !isLegacyManagedRegisterRequest(req, client, accessToken) { + t.Fatal("expected legacy register request to stay on compatibility flow") + } + }) + + t.Run("legacy version still allows no-auth flow even if a manager token header is present", func(t *testing.T) { + req := httptest.NewRequest("POST", "/instance/_register", nil) + req.Header.Set(model.API_TOKEN, "manager-token") + client := &model.Instance{ + Application: env.Application{ + Version: env.Version{VersionNumber: "1.30.4"}, + }, + } + if !shouldAllowLegacyManagedRequestWithoutAuth(req, client) { + t.Fatal("expected legacy version to keep compatibility auth even with token headers present") + } + }) + + t.Run("legacy basic auth helper still recognizes matching basic auth", func(t *testing.T) { + req := httptest.NewRequest("POST", "/configs/_sync", nil) + req.SetBasicAuth("manager", "secret") + instance := &model.Instance{ + Application: env.Application{ + Version: env.Version{VersionNumber: "1.30.4"}, + }, + } + if !isLegacyManagedBasicAuthRequest(req, instance) { + t.Fatal("expected legacy basic auth request to be detected") + } + }) + + t.Run("newer version does not use legacy compatibility flow", func(t *testing.T) { + req := httptest.NewRequest("POST", "/configs/_sync", nil) + instance := &model.Instance{ + Application: env.Application{ + Version: env.Version{VersionNumber: "1.31.1"}, + }, + } + if shouldAllowLegacyManagedRequestWithoutAuth(req, instance) { + t.Fatal("expected newer version to stay on token flow") + } + }) + + t.Run("request legacy version overrides stored newer version", func(t *testing.T) { + req := httptest.NewRequest("POST", "/configs/_sync", nil) + instance := &model.Instance{ + Application: env.Application{ + Version: env.Version{VersionNumber: "1.32.0"}, + }, + } + if err := validateLegacyCompatibleManagedAgentRequestAuthForVersion(req, instance, "1.30.3"); err != nil { + t.Fatalf("expected request legacy version to allow compatibility auth, got %v", err) + } + }) +} diff --git a/plugin/managed/server/config.go b/plugin/managed/server/config.go index e872c124..9bde4f5e 100644 --- a/plugin/managed/server/config.go +++ b/plugin/managed/server/config.go @@ -29,13 +29,17 @@ package server import ( log "github.com/cihub/seelog" + agent_common "infini.sh/console/modules/agent/common" httprouter "infini.sh/framework/core/api/router" config3 "infini.sh/framework/core/config" "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "infini.sh/framework/modules/configs/common" "infini.sh/framework/modules/configs/config" + elastic "infini.sh/framework/modules/elastic" "net/http" "path" "sync" @@ -43,6 +47,49 @@ import ( var configProvidersLock = sync.RWMutex{} var configProviders = []func(instance model.Instance) []*common.ConfigFile{} +var secretProviders = []func(instance model.Instance) *common.Secrets{} + +const managedSecretsHashBucket = "managed_instance_secret_hash" + +type managedConfigLogSummary struct { + Name string `json:"name,omitempty"` + Location string `json:"location,omitempty"` + Updated int64 `json:"updated,omitempty"` + Version int64 `json:"version,omitempty"` + Size int64 `json:"size,omitempty"` + Readonly bool `json:"readonly,omitempty"` + Managed bool `json:"managed"` + Hash string `json:"hash,omitempty"` +} + +func managedConfigEffectiveHash(cfg common.ConfigFile) string { + if cfg.Hash != "" { + return cfg.Hash + } + if cfg.Content == "" { + return "" + } + return util.MD5digest(cfg.Content) +} + +func managedConfigChanged(serverCfg, clientCfg common.ConfigFile) bool { + if serverCfg.Version > clientCfg.Version { + return true + } + if serverCfg.Version < clientCfg.Version { + return false + } + + serverHash := managedConfigEffectiveHash(serverCfg) + clientHash := managedConfigEffectiveHash(clientCfg) + if serverHash != "" && clientHash != "" { + return serverHash != clientHash + } + if serverCfg.Content != "" && clientCfg.Content != "" { + return serverCfg.Content != clientCfg.Content + } + return serverCfg.Updated > clientCfg.Updated +} func RegisterConfigProvider(provider func(instance model.Instance) []*common.ConfigFile) { configProvidersLock.Lock() @@ -50,6 +97,36 @@ func RegisterConfigProvider(provider func(instance model.Instance) []*common.Con configProviders = append(configProviders, provider) } +func RegisterSecretProvider(provider func(instance model.Instance) *common.Secrets) { + configProvidersLock.Lock() + defer configProvidersLock.Unlock() + secretProviders = append(secretProviders, provider) +} + +func summarizeManagedConfigsForLog(cfgs []*common.ConfigFile) []managedConfigLogSummary { + if len(cfgs) == 0 { + return nil + } + + summaries := make([]managedConfigLogSummary, 0, len(cfgs)) + for _, cfg := range cfgs { + if cfg == nil { + continue + } + summaries = append(summaries, managedConfigLogSummary{ + Name: cfg.Name, + Location: cfg.Location, + Updated: cfg.Updated, + Version: cfg.Version, + Size: cfg.Size, + Readonly: cfg.Readonly, + Managed: cfg.Managed, + Hash: cfg.Hash, + }) + } + return summaries +} + func refreshConfigsRepo() { //load config settings from file @@ -120,9 +197,52 @@ func getSecretsForInstance(instance model.Instance) *common.Secrets { } } } + + for _, providerSecrets := range getSecretsFromExternalProviders(instance) { + if providerSecrets == nil { + continue + } + for k, v := range providerSecrets.Keystore { + secrets.Keystore[k] = v + } + } return &secrets } +func getManagedSecretsHash(secrets *common.Secrets) string { + if secrets == nil || len(secrets.Keystore) == 0 { + return "" + } + return util.MD5digestString(util.MustToJSONBytes(secrets)) +} + +func getManagedSecretsHashKey(instance model.Instance) []byte { + return []byte(instance.ID) +} + +func shouldSyncManagedSecrets(instance model.Instance, secrets *common.Secrets) bool { + hash := getManagedSecretsHash(secrets) + if hash == "" { + return false + } + currentHash, err := kv.GetValue(managedSecretsHashBucket, getManagedSecretsHashKey(instance)) + if err != nil { + log.Error(err) + return true + } + return string(currentHash) != hash +} + +func markManagedSecretsSynced(instance model.Instance, secrets *common.Secrets) { + hash := getManagedSecretsHash(secrets) + if hash == "" { + return + } + if err := kv.AddValue(managedSecretsHashBucket, getManagedSecretsHashKey(instance), []byte(hash)); err != nil { + log.Error(err) + } +} + func getConfigsForInstance(instance model.Instance) []*common.ConfigFile { result := []*common.ConfigFile{} @@ -168,6 +288,42 @@ func (h APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, ps htt log.Trace("request:", util.MustToJSON(obj)) } + if common.SupportsManagedAccessToken(obj.Client.Application.Name) { + instance := model.Instance{} + instance.ID = obj.Client.ID + exists, err := orm.GetV2(orm.NewContext(), &instance) + if err == elastic.ErrNotFound { + err = nil + exists = false + } + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + h.WriteError(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + var authErr error + if shouldAllowLegacyManagedRequestWithoutAuthVersion(req, obj.Client.Application.Version.VersionNumber) { + authErr = validateLegacyCompatibleManagedAgentRequestAuthForVersion(req, &instance, obj.Client.Application.Version.VersionNumber) + } else { + authErr = validateManagedAgentRequestAuth(req, &instance) + } + if authErr != nil { + if agent_common.IsManagerAuthFailure(authErr) { + h.WriteError(w, authErr.Error(), http.StatusUnauthorized) + } else { + h.WriteError(w, authErr.Error(), http.StatusInternalServerError) + } + return + } + obj.Client.ManagerCredentialID = instance.ManagerCredentialID + obj.Client.AccessCredentialID = instance.AccessCredentialID + } + + syncManagedInstanceEndpoint(obj.Client) + //TODO, check the client's and the server's hash, if same, skip the sync var res = common.ConfigSyncResponse{} @@ -195,7 +351,7 @@ func (h APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, ps htt } if global.Env().IsDebug { - log.Debugf("get configs for agent(%v): %v", obj.Client.ID, util.MustToJSON(cfgs)) + log.Debugf("get %d configs for agent(%v): %v", len(cfgs), obj.Client.ID, util.MustToJSON(summarizeManagedConfigsForLog(cfgs))) } if cfgs == nil || len(cfgs) == 0 { @@ -238,13 +394,12 @@ func (h APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, ps htt } if global.Env().IsDebug { - log.Debugf("check version for config %v, %v vs %v, %v", k, v.Version, x.Version, x.Managed) + log.Tracef("check version for config %v, %v vs %v, %v", k, v.Version, x.Version, x.Managed) } - //let's diff the version - if v.Version > x.Version { + if managedConfigChanged(v, x) { if global.Env().IsDebug { - log.Trace("get newly version from server, let's sync to client: ", k) + log.Trace("managed config changed, let's sync to client: ", k, ", server version/hash=", v.Version, "/", managedConfigEffectiveHash(v), ", client version/hash=", x.Version, "/", managedConfigEffectiveHash(x)) } res.Configs.UpdatedConfigs[k] = v @@ -285,13 +440,19 @@ func (h APIHandler) syncConfigs(w http.ResponseWriter, req *http.Request, ps htt } } - //only if config changed, we change try to update the client's secrets, //TODO maybe there are coupled - if res.Changed { - secrets := getSecretsForInstance(obj.Client) + secrets := getSecretsForInstance(obj.Client) + secretsChanged := shouldSyncManagedSecrets(obj.Client, secrets) + + // sync secrets when either config changed or the managed secret payload changed. + if res.Changed || secretsChanged { + res.Changed = true res.Secrets = secrets } h.WriteJSON(w, res, 200) + if res.Changed && res.Secrets != nil { + markManagedSecretsSynced(obj.Client, res.Secrets) + } } @@ -307,3 +468,16 @@ func getConfigsFromExternalProviders(client model.Instance) []*common.ConfigFile } return cfgs } + +func getSecretsFromExternalProviders(client model.Instance) []*common.Secrets { + configProvidersLock.Lock() + defer configProvidersLock.Unlock() + var secrets []*common.Secrets + for _, p := range secretProviders { + s := p(client) + if s != nil { + secrets = append(secrets, s) + } + } + return secrets +} diff --git a/plugin/managed/server/config_secrets_test.go b/plugin/managed/server/config_secrets_test.go new file mode 100644 index 00000000..816c9f52 --- /dev/null +++ b/plugin/managed/server/config_secrets_test.go @@ -0,0 +1,132 @@ +package server + +import ( + "fmt" + "testing" + "time" + + "infini.sh/framework/core/kv" + "infini.sh/framework/core/model" + "infini.sh/framework/modules/configs/common" +) + +type managedConfigTestKVStore struct { + values map[string][]byte +} + +func (m *managedConfigTestKVStore) Open() error { return nil } + +func (m *managedConfigTestKVStore) Close() error { return nil } + +func (m *managedConfigTestKVStore) GetValue(bucket string, key []byte) ([]byte, error) { + value, ok := m.values[fmt.Sprintf("%s:%s", bucket, string(key))] + if !ok { + return nil, nil + } + return value, nil +} + +func (m *managedConfigTestKVStore) GetCompressedValue(bucket string, key []byte) ([]byte, error) { + return m.GetValue(bucket, key) +} + +func (m *managedConfigTestKVStore) AddValueCompress(bucket string, key []byte, value []byte) error { + return m.AddValue(bucket, key, value) +} + +func (m *managedConfigTestKVStore) AddValueCompressWithTTL(bucket string, key []byte, value []byte, _ time.Duration) error { + return m.AddValue(bucket, key, value) +} + +func (m *managedConfigTestKVStore) AddValue(bucket string, key []byte, value []byte) error { + m.values[fmt.Sprintf("%s:%s", bucket, string(key))] = value + return nil +} + +func (m *managedConfigTestKVStore) AddValueWithTTL(bucket string, key []byte, value []byte, _ time.Duration) error { + return m.AddValue(bucket, key, value) +} + +func (m *managedConfigTestKVStore) ExistsKey(bucket string, key []byte) (bool, error) { + _, ok := m.values[fmt.Sprintf("%s:%s", bucket, string(key))] + return ok, nil +} + +func (m *managedConfigTestKVStore) DeleteKey(bucket string, key []byte) error { + delete(m.values, fmt.Sprintf("%s:%s", bucket, string(key))) + return nil +} + +func TestShouldSyncManagedSecretsTracksSecretChanges(t *testing.T) { + kv.Register(fmt.Sprintf("managed-config-secrets-%d", time.Now().UnixNano()), &managedConfigTestKVStore{ + values: map[string][]byte{}, + }) + + instance := model.Instance{} + instance.ID = "agent-secret-sync" + + secrets := &common.Secrets{ + Keystore: map[string]common.KeystoreValue{ + "SYSTEM_CLUSTER_INGEST_PASSWORD": { + Type: "plaintext", + Value: "old-password", + }, + }, + } + + if !shouldSyncManagedSecrets(instance, secrets) { + t.Fatal("expected initial secrets sync to be required") + } + + markManagedSecretsSynced(instance, secrets) + + if shouldSyncManagedSecrets(instance, secrets) { + t.Fatal("expected unchanged secrets not to trigger sync") + } + + secrets.Keystore["SYSTEM_CLUSTER_INGEST_PASSWORD"] = common.KeystoreValue{ + Type: "plaintext", + Value: "new-password", + } + + if !shouldSyncManagedSecrets(instance, secrets) { + t.Fatal("expected rotated secrets to trigger sync") + } +} + +func TestShouldSyncManagedSecretsIgnoresEmptySecrets(t *testing.T) { + instance := model.Instance{} + instance.ID = "agent-empty-secrets" + + if shouldSyncManagedSecrets(instance, nil) { + t.Fatal("expected nil secrets not to trigger sync") + } + + if shouldSyncManagedSecrets(instance, &common.Secrets{Keystore: map[string]common.KeystoreValue{}}) { + t.Fatal("expected empty secrets not to trigger sync") + } +} + +func TestManagedConfigChangedByVersion(t *testing.T) { + serverCfg := common.ConfigFile{Version: 2, Content: "a"} + clientCfg := common.ConfigFile{Version: 1, Content: "a"} + if !managedConfigChanged(serverCfg, clientCfg) { + t.Fatal("expected higher version to trigger sync") + } +} + +func TestManagedConfigChangedByContentWhenVersionEqual(t *testing.T) { + serverCfg := common.ConfigFile{Version: 1, Content: "relay-hosts-new"} + clientCfg := common.ConfigFile{Version: 1, Content: "relay-hosts-old"} + if !managedConfigChanged(serverCfg, clientCfg) { + t.Fatal("expected content change with same version to trigger sync") + } +} + +func TestManagedConfigUnchangedWhenVersionAndContentEqual(t *testing.T) { + serverCfg := common.ConfigFile{Version: 1, Content: "same-content"} + clientCfg := common.ConfigFile{Version: 1, Content: "same-content"} + if managedConfigChanged(serverCfg, clientCfg) { + t.Fatal("expected unchanged config not to trigger sync") + } +} diff --git a/plugin/managed/server/instance.go b/plugin/managed/server/instance.go index 0fe130b3..d6ab3a13 100644 --- a/plugin/managed/server/instance.go +++ b/plugin/managed/server/instance.go @@ -28,12 +28,20 @@ package server import ( + "bytes" "context" "fmt" + console_common "infini.sh/console/common" + agent_common "infini.sh/console/modules/agent/common" + frameworkcredential "infini.sh/framework/core/credential" "infini.sh/framework/core/event" "infini.sh/framework/core/global" "infini.sh/framework/core/task" + "io" + "net" "net/http" + "net/http/httptest" + "net/url" "strconv" "strings" "time" @@ -54,14 +62,124 @@ import ( var instanceConfigFiles = map[string][]string{} //map instance->config files TODO lru cache, short life instance should be removed var instanceSecrets = map[string][]common.Secrets{} //map instance->secrets TODO lru cache, short life instance should be removed +var getManagedInstanceByID = func(instance *model.Instance) (bool, error) { + return orm.GetV2(orm.NewContext(), instance) +} + +var saveManagedInstanceRecord = func(instance *model.Instance) error { + return orm.Save(&orm.Context{Refresh: orm.WaitForRefresh}, instance) +} + +var deleteManagedInstanceRecord = func(instance *model.Instance) error { + return orm.Delete(&orm.Context{ + Refresh: orm.WaitForRefresh, + }, instance) +} + +var cleanupDeletedInstanceArtifactsFunc = cleanupDeletedInstanceArtifacts + +func logManagedRegistration(stage string, instance *model.Instance, req *http.Request, detail string) { + if instance == nil { + return + } + + remoteAddr := "" + if req != nil { + remoteAddr = req.RemoteAddr + } + + message := fmt.Sprintf( + "managed agent registration %s: %v[%v], version=%v, endpoint=%v", + stage, + instance.Name, + instance.ID, + instance.Application.Version.VersionNumber, + console_common.MaskLogEndpoint(instance.Endpoint), + ) + if remoteAddr != "" { + message = fmt.Sprintf("%s, remote=%v", message, remoteAddr) + } + if detail != "" { + message = fmt.Sprintf("%s, detail=%v", message, detail) + } + + if stage == "failed" { + log.Warn(message) + return + } + log.Info(message) +} + +func logLegacyManagedRegistration(stage string, instance *model.Instance, req *http.Request, detail string) { + if instance == nil { + return + } + + remoteAddr := "" + if req != nil { + remoteAddr = req.RemoteAddr + } + + message := fmt.Sprintf( + "legacy managed agent registration %s: %v[%v], version=%v, endpoint=%v", + stage, + instance.Name, + instance.ID, + instance.Application.Version.VersionNumber, + console_common.MaskLogEndpoint(instance.Endpoint), + ) + if remoteAddr != "" { + message = fmt.Sprintf("%s, remote=%v", message, remoteAddr) + } + if detail != "" { + message = fmt.Sprintf("%s, detail=%v", message, detail) + } + + log.Warn(message) +} + +func decodeManagedRegisterRequest(req *http.Request) (common.InstanceRegisterRequest, error) { + registerReq := common.InstanceRegisterRequest{} + body, err := io.ReadAll(req.Body) + if err != nil { + return registerReq, err + } + req.Body = io.NopCloser(bytes.NewReader(body)) + + if err := util.FromJSONBytes(body, ®isterReq); err != nil { + return registerReq, err + } + if registerReq.Client.Endpoint != "" || registerReq.Client.ID != "" { + req.Body = io.NopCloser(bytes.NewReader(body)) + return registerReq, nil + } + + legacyInstance := model.Instance{} + if err := util.FromJSONBytes(body, &legacyInstance); err == nil { + if legacyInstance.Endpoint != "" || legacyInstance.ID != "" { + registerReq.Client = legacyInstance + } + } + req.Body = io.NopCloser(bytes.NewReader(body)) + return registerReq, nil +} + func init() { //for public usage, agent can report self to server, usually need to enroll by manager api.HandleAPIMethod(api.POST, common.REGISTER_API, handler.registerInstance) //client register self to config servers + api.HandleUIMethod(api.POST, common.REGISTER_API, handler.registerInstance) + api.HandleAPIMethod(api.POST, instanceTokenExchangeAPI, handler.exchangeInstanceToken) + api.HandleUIMethod(api.POST, instanceTokenExchangeAPI, handler.exchangeInstanceToken) //for public usage, get install script - api.HandleAPIMethod(api.GET, GET_INSTALL_SCRIPT_API, handler.getInstallScript) + api.HandleAPIMethod(api.GET, getInstallScriptAPI, handler.getInstallScript) + api.HandleUIMethod(api.GET, getInstallScriptAPI, handler.getInstallScript) + api.HandleAPIMethod(api.GET, getGatewayInstallScriptAPI, handler.getGatewayInstallScript) + api.HandleUIMethod(api.GET, getGatewayInstallScriptAPI, handler.getGatewayInstallScript) api.HandleAPIMethod(api.POST, "/instance/_generate_install_script", handler.RequireLogin(handler.generateInstallCommand)) + api.HandleAPIMethod(api.POST, "/instance/_generate_gateway_install_script", handler.RequirePermission(handler.generateGatewayInstallCommand, enum.PermissionGatewayInstanceWrite)) + api.HandleAPIMethod(api.POST, "/instance/_prepare_registration", handler.RequirePermission(handler.prepareRegistration, enum.PermissionGatewayInstanceWrite)) api.HandleAPIMethod(api.POST, "/instance", handler.RequirePermission(handler.createInstance, enum.PermissionGatewayInstanceWrite)) api.HandleAPIMethod(api.GET, "/instance/:instance_id", handler.RequirePermission(handler.getInstance, enum.PermissionAgentInstanceRead)) @@ -85,13 +203,12 @@ func init() { } func (h APIHandler) registerInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - - var obj = &model.Instance{} - err := h.DecodeJSON(req, obj) + registerReq, err := decodeManagedRegisterRequest(req) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + obj := ®isterReq.Client if obj.Endpoint == "" { h.WriteError(w, "empty endpoint", http.StatusInternalServerError) return @@ -99,21 +216,186 @@ func (h APIHandler) registerInstance(w http.ResponseWriter, req *http.Request, p oldInst := &model.Instance{} oldInst.ID = obj.ID - exists, err := orm.Get(oldInst) - if exists { - obj.Created = oldInst.Created + exists, err := orm.GetV2(orm.NewContext(), oldInst) + if err == elastic.ErrNotFound { + err = nil + exists = false } - err = orm.Save(nil, obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - log.Infof("register instance: %v[%v], %v", obj.Name, obj.ID, obj.Endpoint) + logManagedRegistration("received", obj, req, fmt.Sprintf("existing=%v", exists)) + + var pendingToConsume *agent_common.PendingRegistrationToken + legacyManagerCredentialMigrated := false + legacyManagedRegister := false + writeRegisterError := func(status int, detail string) { + logManagedRegistration("failed", obj, req, fmt.Sprintf("status=%d, detail=%s", status, detail)) + if legacyManagedRegister { + logLegacyManagedRegistration("failed", obj, req, detail) + } + h.WriteError(w, detail, status) + } + + if common.SupportsManagedAccessToken(obj.Application.Name) { + legacyManagedRegister = isLegacyManagedRegisterRequest(req, obj, registerReq.AccessToken) + if legacyManagedRegister { + logLegacyManagedRegistration("detected", obj, req, "attempting compatibility registration") + } + tokenValue := agent_common.ExtractManagerToken(req) + if exists { + if oldInst.Created != nil { + obj.Created = oldInst.Created + } + obj.ManagerCredentialID = oldInst.ManagerCredentialID + obj.AccessCredentialID = oldInst.AccessCredentialID + obj.BasicAuth = oldInst.BasicAuth + var err error + if legacyManagedRegister { + err = validateLegacyCompatibleManagedAgentRequestAuthForVersion(req, oldInst, obj.Application.Version.VersionNumber) + } else { + err = validateManagedAgentRequestAuthFunc(req, oldInst) + } + if err != nil { + legacyManagerCredentialMigrated, err = migrateLegacyManagedRegisterAuth(req, obj, oldInst, registerReq.AccessToken) + if err != nil { + writeRegisterError(http.StatusInternalServerError, err.Error()) + return + } + if !legacyManagerCredentialMigrated { + if agent_common.IsManagerAuthFailure(err) { + writeRegisterError(http.StatusUnauthorized, err.Error()) + } else { + writeRegisterError(http.StatusInternalServerError, err.Error()) + } + return + } + } + } else { + if legacyManagedRegister { + if err := validateLegacyCompatibleManagedAgentRequestAuth(req, obj); err != nil { + if agent_common.IsManagerAuthFailure(err) { + writeRegisterError(http.StatusUnauthorized, err.Error()) + } else { + writeRegisterError(http.StatusInternalServerError, err.Error()) + } + return + } + } else { + if tokenValue == "" { + writeRegisterError(http.StatusUnauthorized, "missing manager token") + return + } + pending, err := agent_common.FindPendingManagerTokenByValue(tokenValue) + if err != nil { + writeRegisterError(http.StatusInternalServerError, err.Error()) + return + } + if pending == nil { + writeRegisterError(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized)) + return + } + obj.ManagerCredentialID = pending.CredentialID + if err := renamePendingManagerCredential(obj, pending.CredentialID); err != nil { + writeRegisterError(http.StatusInternalServerError, err.Error()) + return + } + if err := upsertInstanceAccessCredentialFunc(obj, registerReq.AccessToken); err != nil { + writeRegisterError(http.StatusInternalServerError, err.Error()) + return + } + pendingToConsume = pending + } + } + } else if exists { + obj.Created = oldInst.Created + } + + if exists && common.SupportsManagedAccessToken(obj.Application.Name) && registerReq.AccessToken != nil && !legacyManagerCredentialMigrated { + if err := upsertInstanceAccessCredentialFunc(obj, registerReq.AccessToken); err != nil { + writeRegisterError(http.StatusInternalServerError, err.Error()) + return + } + } + + err = saveManagedInstanceRecord(obj) + if err != nil { + writeRegisterError(http.StatusInternalServerError, err.Error()) + return + } + if pendingToConsume != nil { + if err := agent_common.MarkPendingRegistrationTokenConsumed(pendingToConsume, obj.ID); err != nil { + writeRegisterError(http.StatusInternalServerError, err.Error()) + return + } + } + + log.Infof("register instance: %v[%v], %v", obj.Name, obj.ID, console_common.MaskLogEndpoint(obj.Endpoint)) + detail := "registered successfully" + if exists { + detail = "updated existing registration successfully" + } + if legacyManagedRegister { + if legacyManagerCredentialMigrated { + detail = "registered successfully with manager credential migration" + } + logLegacyManagedRegistration("succeeded", obj, req, detail) + } else { + logManagedRegistration("succeeded", obj, req, detail) + } h.WriteAckOKJSON(w) } +func migrateLegacyManagedRegisterAuth(req *http.Request, current *model.Instance, existing *model.Instance, accessToken *common.RegisterToken) (bool, error) { + if req == nil || current == nil || existing == nil { + return false, nil + } + managerToken := strings.TrimSpace(agent_common.ExtractManagerToken(req)) + if managerToken == "" || accessToken == nil || strings.TrimSpace(accessToken.Value) == "" { + return false, nil + } + if strings.TrimSpace(existing.ManagerCredentialID) != "" { + if !isLegacyManagedVersion(current.Application.Version.VersionNumber) { + return false, nil + } + pending, err := findPendingManagerTokenByValueFunc(managerToken) + if err != nil { + return false, err + } + if pending == nil { + return false, nil + } + } + if err := upsertInstanceManagerCredentialFunc(current, managerToken); err != nil { + return false, err + } + if err := upsertInstanceAccessCredentialFunc(current, accessToken); err != nil { + return false, err + } + return true, nil +} + +func syncManagedInstanceEndpoint(client model.Instance) { + if client.ID == "" || client.Endpoint == "" { + return + } + + existing := model.Instance{} + existing.ID = client.ID + exists, err := orm.GetV2(orm.NewContext(), &existing) + if err != nil || !exists || existing.Endpoint == client.Endpoint { + return + } + + existing.Endpoint = client.Endpoint + if err := orm.Update(orm.NewContext(), &existing); err != nil { + log.Warnf("failed to update instance endpoint for [%s]: %v", client.ID, err) + } +} + func (h APIHandler) enrollInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { } @@ -125,7 +407,7 @@ func (h *APIHandler) getInstance(w http.ResponseWriter, req *http.Request, ps ht obj := model.Instance{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -147,15 +429,22 @@ func (h *APIHandler) getInstance(w http.ResponseWriter, req *http.Request, ps ht } func (h *APIHandler) createInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - var obj = &model.Instance{} - err := h.DecodeJSON(req, obj) + var reqBody struct { + model.Instance + RegistrationID string `json:"registration_id"` + AccessToken string `json:"access_token"` + } + err := h.DecodeJSON(req, &reqBody) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) return } + obj := &reqBody.Instance + var pendingToConsume *agent_common.PendingRegistrationToken - res, err := h.getInstanceInfo(obj.Endpoint, obj.BasicAuth) + probeAccessToken := effectiveInstanceProbeAccessToken(req, obj.Endpoint, reqBody.AccessToken) + res, err := h.getInstanceInfo(obj.Endpoint, obj.BasicAuth, probeAccessToken) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -171,7 +460,7 @@ func (h *APIHandler) createInstance(w http.ResponseWriter, req *http.Request, ps } obj.Application = res.Application - exists, err := orm.Get(obj) + exists, err := orm.GetV2(orm.NewContext(), obj) if err != nil && err != elastic.ErrNotFound { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -181,12 +470,48 @@ func (h *APIHandler) createInstance(w http.ResponseWriter, req *http.Request, ps h.WriteError(w, "instance already registered", http.StatusInternalServerError) return } - err = orm.Create(nil, obj) + if reqBody.RegistrationID != "" { + pending, err := agent_common.GetPendingRegistrationTokenByID(reqBody.RegistrationID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if pending == nil || pending.Consumed || (pending.ExpiresAt > 0 && time.Now().UnixMilli() > pending.ExpiresAt) { + h.WriteError(w, "registration token is invalid", http.StatusUnauthorized) + return + } + obj.ManagerCredentialID = pending.CredentialID + if err := renamePendingManagerCredential(obj, pending.CredentialID); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + pendingToConsume = pending + } + if strings.TrimSpace(reqBody.AccessToken) != "" { + credentialID, err := agent_common.SaveTokenCredential( + agent_common.BuildAccessCredentialName(obj), + agent_common.BuildAccessCredentialTags(), + reqBody.AccessToken, + ) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + obj.AccessCredentialID = credentialID + obj.BasicAuth = nil + } + err = orm.Create(&orm.Context{Refresh: orm.WaitForRefresh}, obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) return } + if pendingToConsume != nil { + if err := agent_common.MarkPendingRegistrationTokenConsumed(pendingToConsume, obj.ID); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + } h.WriteJSON(w, util.MapStr{ "_id": obj.ID, @@ -201,7 +526,7 @@ func (h *APIHandler) deleteInstance(w http.ResponseWriter, req *http.Request, ps obj := model.Instance{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := getManagedInstanceByID(&obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -210,14 +535,22 @@ func (h *APIHandler) deleteInstance(w http.ResponseWriter, req *http.Request, ps return } - err = orm.Delete(nil, &obj) + err = deleteManagedInstanceRecord(&obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) return } - h.WriteDeletedOKJSON(w, id) + payload := util.MapStr{ + "_id": id, + "result": "deleted", + } + if warnings := cleanupDeletedInstanceArtifactsFunc(&obj); len(warnings) > 0 { + payload["warnings"] = warnings + } + + h.WriteJSON(w, payload, http.StatusOK) } func (h *APIHandler) updateInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -225,7 +558,7 @@ func (h *APIHandler) updateInstance(w http.ResponseWriter, req *http.Request, ps obj := model.Instance{} obj.ID = id - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) if !exists || err != nil { h.WriteJSON(w, util.MapStr{ "_id": id, @@ -247,7 +580,7 @@ func (h *APIHandler) updateInstance(w http.ResponseWriter, req *http.Request, ps //protect obj.ID = id obj.Created = create - err = orm.Update(nil, &obj) + err = orm.Update(orm.NewContext(), &obj) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) log.Error(err) @@ -264,6 +597,7 @@ func (h *APIHandler) searchInstance(w http.ResponseWriter, req *http.Request, ps var ( application = h.GetParameterOrDefault(req, "application", "") + serviceType = strings.TrimSpace(h.GetParameterOrDefault(req, "service_type", "")) keyword = h.GetParameterOrDefault(req, "keyword", "") queryDSL = `{"query":{"bool":{"must":[%s]}}, "size": %d, "from": %d}` strSize = h.GetParameterOrDefault(req, "size", "20") @@ -280,6 +614,12 @@ func (h *APIHandler) searchInstance(w http.ResponseWriter, req *http.Request, ps } mustBuilder.WriteString(fmt.Sprintf(`{"term":{"application.name":"%s"}}`, application)) } + if serviceType != "" { + if mustBuilder.Len() > 0 { + mustBuilder.WriteString(",") + } + mustBuilder.WriteString(fmt.Sprintf(`{"bool":{"should":[{"term":{"labels.service_type":{"value":%q}}},{"term":{"metadata.labels.service_type":{"value":%q}}}],"minimum_should_match":1}}`, serviceType, serviceType)) + } size, _ := strconv.Atoi(strSize) if size <= 0 { @@ -327,54 +667,157 @@ func (h *APIHandler) getInstanceStatus(w http.ResponseWriter, req *http.Request, } q.RawQuery = util.MustToJSONBytes(queryDSL) - err, res := orm.Search(&model.Instance{}, &q) - if err != nil { + instances := []model.Instance{} + if err, _ := orm.SearchWithJSONMapper(&instances, &q); err != nil { log.Error(err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } + result := util.MapStr{} - for _, item := range res.Result { - instance := util.MapStr(item.(map[string]interface{})) - if err != nil { - log.Error(err) + for i := range instances { + instance := instances[i] + var resMap = util.MapStr{} + if !fetchManagedInstanceStats(req, &instance, &resMap) { + result[instance.ID] = util.MapStr{} continue } - endpoint, _ := instance.GetValue("endpoint") + result[instance.ID] = resMap + } + h.WriteJSON(w, result, http.StatusOK) +} - gid, _ := instance.GetValue("id") +func fetchManagedInstanceStats(currentReq *http.Request, instance *model.Instance, stats *util.MapStr) bool { + if instance == nil { + return false + } + if shouldFetchManagedInstanceStatsLocally(instance) && fetchManagedInstanceStatsLocally(stats) { + return true + } - //req := &proxy.Request{ - // Endpoint: endpoint.(string), - // Method: http.MethodGet, - // Path: "/stats", - //} - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - req := &util.Request{ - Method: http.MethodGet, - Path: "/stats", - Context: ctx, - } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() - username, _ := instance.GetValue("basic_auth.username") - if username != nil && username.(string) != "" { - password, _ := instance.GetValue("basic_auth.password") - if password != nil && password.(string) != "" { - req.SetBasicAuth(username.(string), password.(string)) - } + req := &util.Request{ + Method: http.MethodGet, + Path: "/stats", + Context: ctx, + } + if err := agent_common.ApplyInstanceRequestAuth(req, instance); err != nil { + log.Error(err) + return false + } + agent_common.ApplyBearerToken(req, effectiveManagedInstanceAccessToken(currentReq, instance)) + + if _, err := proxyInstanceRequest(instance, req, stats); err != nil { + log.Error(instance.GetEndpoint(), ",", err) + return false + } + return true +} + +func shouldFetchManagedInstanceStatsLocally(instance *model.Instance) bool { + if instance == nil { + return false + } + if isCurrentManagedInstance(instance) { + return true + } + return shouldReuseCurrentRequestAuthForEndpoint(instance.GetEndpoint()) +} + +func isCurrentManagedInstance(instance *model.Instance) bool { + if instance == nil { + return false + } + instanceID := strings.TrimSpace(instance.ID) + if instanceID == "" { + return false + } + return instanceID == strings.TrimSpace(global.Env().SystemConfig.NodeConfig.ID) +} + +func fetchManagedInstanceStatsLocally(stats *util.MapStr) bool { + if stats == nil { + return false + } + res, err := proxyManagedAPIRequestLocally(&util.Request{ + Method: http.MethodGet, + Path: "/stats", + }, stats) + if err != nil { + body := "" + status := 0 + if res != nil { + body = string(res.Body) + status = res.StatusCode } + log.Errorf("local /stats request failed, status: %d, body: %s", status, body) + return false + } + return true +} - var resMap = util.MapStr{} - _, err := ProxyAgentRequest("runtime", endpoint.(string), req, &resMap) - if err != nil { - log.Error(endpoint, ",", err) - result[gid.(string)] = util.MapStr{} - continue +func proxyManagedAPIRequestLocally(req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + if req == nil { + return nil, fmt.Errorf("request is nil") + } + requestPath := strings.TrimSpace(req.Path) + if requestPath == "" { + return nil, fmt.Errorf("request path is empty") + } + + bodyReader := bytes.NewReader(req.Body) + localReq := httptest.NewRequest(req.Method, requestPath, bodyReader) + if req.Context != nil { + localReq = localReq.WithContext(req.Context) + } + for key, value := range req.AllHeaders() { + localReq.Header.Set(key, value) + } + if req.ContentType != "" { + localReq.Header.Set("Content-Type", req.ContentType) + } + applyConsoleLocalAPIAuth(localReq) + + recorder := httptest.NewRecorder() + api.ServeRegisteredAPIRequest(recorder, localReq) + httpResult := recorder.Result() + defer httpResult.Body.Close() + + res := &util.Result{ + StatusCode: recorder.Code, + Body: append([]byte(nil), recorder.Body.Bytes()...), + Headers: map[string][]string{}, + } + for key, values := range httpResult.Header { + res.Headers[strings.ToLower(key)] = append([]string(nil), values...) + } + + if res.StatusCode != http.StatusOK { + return res, fmt.Errorf("request error: %v, %v", nil, string(res.Body)) + } + if responseObjectToUnMarshall != nil && len(res.Body) > 0 { + if err := util.FromJSONBytes(res.Body, responseObjectToUnMarshall); err != nil { + return res, err } - result[gid.(string)] = resMap } - h.WriteJSON(w, result, http.StatusOK) + return res, nil +} + +func applyConsoleLocalAPIAuth(req *http.Request) { + if req == nil { + return + } + apiCfg := global.Env().SystemConfig.APIConfig + if !apiCfg.Security.Enabled { + return + } + username := strings.TrimSpace(apiCfg.Security.Username) + if username == "" { + return + } + req.SetBasicAuth(username, apiCfg.Security.Password) } func (h *APIHandler) clearInstance(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { appName := h.GetParameterOrDefault(req, "app_name", "") @@ -438,7 +881,13 @@ func (h *APIHandler) clearInstanceByAppName(appName string) error { // check whether the instance is still online for _, instanceID := range toRemoveIDs { if inst, ok := instsCache[instanceID]; ok { - _, err = h.getInstanceInfo(inst.Endpoint, inst.BasicAuth) + if _, tokenErr := agent_common.GetTokenCredentialValue(inst.AccessCredentialID); tokenErr != nil { + err = tokenErr + } else { + if inst != nil { + _, err = h.getRuntimeInstanceInfo(inst) + } + } if err == nil { // Skip online instance, do not append to filtered list continue @@ -559,55 +1008,120 @@ func (h *APIHandler) proxy(w http.ResponseWriter, req *http.Request, ps httprout Context: ctx, Body: reqBody, } - if obj.BasicAuth != nil { - req1.SetBasicAuth(obj.BasicAuth.Username, obj.BasicAuth.Password.Get()) + if err := agent_common.ApplyInstanceRequestAuth(req1, obj); err != nil { + panic(err) } - res, err := ProxyAgentRequest("runtime", obj.GetEndpoint(), req1, nil) + res, err := proxyInstanceRequest(obj, req1, nil) if err != nil { panic(err) } + if isSensitiveInfoPath(path) && len(res.Body) > 0 { + res.Body, err = console_common.SanitizeInstanceInfoBytes(res.Body) + if err != nil { + panic(err) + } + } + h.WriteHeader(w, res.StatusCode) h.Write(w, res.Body) } -func (h *APIHandler) getInstanceInfo(endpoint string, basicAuth *model.BasicAuth) (*model.Instance, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - req1 := &util.Request{ - Method: http.MethodGet, - Path: "/_info", - Context: ctx, - } - if basicAuth != nil { - req1.SetBasicAuth(basicAuth.Username, basicAuth.Password.Get()) +func (h *APIHandler) getInstanceInfo(endpoint string, basicAuth *model.BasicAuth, accessToken string) (*model.Instance, error) { + paths := buildInstanceInfoPaths(false, accessToken) + + var lastErr error + for _, infoPath := range paths { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + req1 := &util.Request{ + Method: http.MethodGet, + Path: infoPath, + Context: ctx, + } + if strings.TrimSpace(accessToken) != "" { + agent_common.ApplyBearerToken(req1, accessToken) + } else if basicAuth != nil { + req1.SetBasicAuth(basicAuth.Username, basicAuth.Password.Get()) + } + obj := &model.Instance{} + res, err := ProxyAgentRequest("runtime", endpoint, req1, obj) + cancel() + if err == nil { + return obj, nil + } + if lastErr == nil { + lastErr = err + } + if !shouldFallbackInstanceInfoPath(infoPath, res, err) { + return nil, err + } } - obj := &model.Instance{} - _, err := ProxyAgentRequest("runtime", endpoint, req1, obj) - if err != nil { - return nil, err + + return nil, lastErr +} + +func (h *APIHandler) getRuntimeInstanceInfo(instance *model.Instance) (*model.Instance, error) { + if instance == nil { + return nil, fmt.Errorf("instance is nil") } - return obj, err + paths := buildInstanceInfoPaths(strings.EqualFold(instance.Application.Name, "agent"), "") + var lastErr error + for _, infoPath := range paths { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + req1 := &util.Request{ + Method: http.MethodGet, + Path: infoPath, + Context: ctx, + } + if err := agent_common.ApplyInstanceRequestAuth(req1, instance); err != nil { + cancel() + return nil, err + } + + obj := &model.Instance{} + res, err := proxyInstanceRequest(instance, req1, obj) + cancel() + if err == nil { + return obj, nil + } + if lastErr == nil { + lastErr = err + } + if !shouldFallbackInstanceInfoPath(infoPath, res, err) { + return nil, err + } + } + return nil, lastErr } func (h *APIHandler) tryConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var reqBody = struct { - Endpoint string `json:"endpoint"` - BasicAuth *model.BasicAuth `json:"basic_auth"` + Endpoint string `json:"endpoint"` + BasicAuth *model.BasicAuth `json:"basic_auth"` + AccessToken string `json:"access_token"` }{} err := h.DecodeJSON(req, &reqBody) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - connectRes, err := h.getInstanceInfo(reqBody.Endpoint, reqBody.BasicAuth) + probeAccessToken := effectiveInstanceProbeAccessToken(req, reqBody.Endpoint, reqBody.AccessToken) + connectRes, err := h.getInstanceInfo(reqBody.Endpoint, reqBody.BasicAuth, probeAccessToken) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - h.WriteJSON(w, connectRes, http.StatusOK) + h.WriteJSON(w, console_common.SanitizeInstanceInfoMap(util.MapStr{ + "id": connectRes.ID, + "name": connectRes.Name, + "application": connectRes.Application, + "labels": connectRes.Labels, + "tags": connectRes.Tags, + "description": connectRes.Description, + "status": connectRes.Status, + }), http.StatusOK) } func (h *APIHandler) tryESConnect(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -654,11 +1168,11 @@ func (h *APIHandler) tryESConnect(w http.ResponseWriter, req *http.Request, ps h Context: ctx, Body: body, } - if reqBody.BasicAuth != nil { - req1.SetBasicAuth(reqBody.BasicAuth.Username, reqBody.BasicAuth.Password.Get()) + if err := agent_common.ApplyInstanceRequestAuth(req1, instance); err != nil { + panic(err) } - res, err := ProxyAgentRequest("runtime", instance.GetEndpoint(), req1, nil) + res, err := proxyInstanceRequest(instance, req1, nil) if err != nil { panic(err) } @@ -674,11 +1188,137 @@ func (h *APIHandler) tryESConnect(w http.ResponseWriter, req *http.Request, ps h h.Write(w, res.Body) } +func isSensitiveInfoPath(rawPath string) bool { + if rawPath == "" { + return false + } + parsed, err := url.Parse(rawPath) + if err != nil { + return rawPath == "/_info" || rawPath == "/agent/_info" + } + return parsed.Path == "/_info" || parsed.Path == "/agent/_info" +} + +func buildInstanceInfoPaths(isAgent bool, accessToken string) []string { + if isAgent || strings.TrimSpace(accessToken) != "" { + return []string{"/agent/_info", "/_info"} + } + return []string{"/_info"} +} + +func effectiveInstanceProbeAccessToken(req *http.Request, endpoint, accessToken string) string { + accessToken = strings.TrimSpace(accessToken) + if accessToken != "" { + return accessToken + } + if req == nil || !shouldReuseCurrentRequestAuthForEndpoint(endpoint) { + return "" + } + return agent_common.ExtractBearerToken(req) +} + +func effectiveManagedInstanceAccessToken(req *http.Request, instance *model.Instance) string { + if instance == nil { + return "" + } + if instance.AccessCredentialID != "" || instance.BasicAuth != nil { + return "" + } + if !shouldReuseCurrentRequestAuthForEndpoint(instance.GetEndpoint()) { + return "" + } + return agent_common.ExtractBearerToken(req) +} + +func shouldReuseCurrentRequestAuthForEndpoint(endpoint string) bool { + parsed, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil || parsed == nil { + return false + } + host := strings.TrimSpace(parsed.Hostname()) + if !isLocalManagedEndpointHost(host) { + return false + } + port := endpointPort(parsed) + if port == "" { + return false + } + if global.Env().SystemConfig.WebAppConfig.Enabled && endpointMatchesPort(global.Env().SystemConfig.WebAppConfig.GetEndpoint(), port) { + return true + } + if global.Env().SystemConfig.APIConfig.Enabled && endpointMatchesPort(global.Env().SystemConfig.APIConfig.GetEndpoint(), port) { + return true + } + return false +} + +func isLocalManagedEndpointHost(host string) bool { + host = strings.TrimSpace(host) + if host == "" { + return false + } + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + if ip == nil { + return false + } + if ip.IsLoopback() { + return true + } + for _, localIP := range util.GetLocalIPs() { + if parsed := net.ParseIP(strings.TrimSpace(localIP)); parsed != nil && parsed.Equal(ip) { + return true + } + } + return false +} + +func endpointMatchesPort(endpoint, port string) bool { + parsed, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil || parsed == nil { + return false + } + return endpointPort(parsed) == port +} + +func endpointPort(parsed *url.URL) string { + if parsed == nil { + return "" + } + if port := strings.TrimSpace(parsed.Port()); port != "" { + return port + } + switch strings.ToLower(strings.TrimSpace(parsed.Scheme)) { + case "https": + return "443" + case "http": + return "80" + default: + return "" + } +} + +func shouldFallbackInstanceInfoPath(path string, res *util.Result, err error) bool { + if path != "/agent/_info" || err == nil { + return false + } + if res != nil { + return res.StatusCode == http.StatusNotFound + } + return true +} + // TODO check permission by user func GetRuntimeInstanceByID(instanceID string) (bool, *model.Instance, error) { obj := model.Instance{} obj.ID = instanceID - exists, err := orm.Get(&obj) + exists, err := orm.GetV2(orm.NewContext(), &obj) + if err == elastic.ErrNotFound { + err = nil + exists = false + } if !exists || err != nil { if !exists { err = fmt.Errorf("instance not found") @@ -687,3 +1327,158 @@ func GetRuntimeInstanceByID(instanceID string) (bool, *model.Instance, error) { } return true, &obj, err } + +func renamePendingManagerCredential(instance *model.Instance, credentialID string) error { + if instance == nil || credentialID == "" { + return nil + } + tokenValue, err := agent_common.GetTokenCredentialValue(credentialID) + if err != nil { + return err + } + return agent_common.UpdateTokenCredential( + credentialID, + agent_common.BuildManagerCredentialName(instance), + agent_common.BuildManagerCredentialTags(), + tokenValue, + ) +} + +func upsertInstanceAccessCredential(instance *model.Instance, registerToken *common.RegisterToken) error { + if instance == nil || registerToken == nil || strings.TrimSpace(registerToken.Value) == "" { + return nil + } + if instance.AccessCredentialID != "" { + return agent_common.UpdateTokenCredential( + instance.AccessCredentialID, + agent_common.BuildAccessCredentialName(instance), + agent_common.BuildAccessCredentialTags(), + registerToken.Value, + ) + } + credentialID, err := agent_common.SaveTokenCredential( + agent_common.BuildAccessCredentialName(instance), + agent_common.BuildAccessCredentialTags(), + registerToken.Value, + ) + if err != nil { + return err + } + instance.AccessCredentialID = credentialID + instance.BasicAuth = nil + return nil +} + +var canDeleteCredentialAfterInstanceRemoval = func(credentialID string) (bool, error) { + credentialID = strings.TrimSpace(credentialID) + if credentialID == "" { + return false, nil + } + + q := orm.Query{ + Size: 0, + Conds: orm.And(orm.Eq("credential_id", credentialID)), + } + err, result := orm.Search(elastic2.ElasticsearchConfig{}, &q) + if err != nil { + return false, fmt.Errorf("query elasticsearch config error: %w", err) + } + if result.Total > 0 { + return false, nil + } + + q = orm.Query{ + Size: 0, + Conds: orm.Or( + orm.Eq("manager_credential_id", credentialID), + orm.Eq("access_credential_id", credentialID), + ), + } + err, result = orm.Search(model.Instance{}, &q) + if err != nil { + return false, fmt.Errorf("query instance config error: %w", err) + } + return result.Total == 0, nil +} + +var deleteCredentialByID = func(credentialID string) error { + credentialID = strings.TrimSpace(credentialID) + if credentialID == "" { + return nil + } + + cred := frameworkcredential.Credential{} + cred.ID = credentialID + exists, err := orm.GetV2(orm.NewContext(), &cred) + if err != nil { + return err + } + if !exists { + return nil + } + return orm.Delete(&orm.Context{Refresh: orm.WaitForRefresh}, &cred) +} + +var deletePendingRegistrationTokensByInstanceID = func(instanceID string) error { + instanceID = strings.TrimSpace(instanceID) + if instanceID == "" { + return nil + } + + query := orm.Query{ + Size: 1000, + Conds: orm.And( + orm.Eq("instance_id", instanceID), + ), + } + records := []agent_common.PendingRegistrationToken{} + if err, _ := orm.SearchWithJSONMapper(&records, &query); err != nil { + return err + } + + ctx := &orm.Context{Refresh: orm.WaitForRefresh} + for i := range records { + if err := orm.Delete(ctx, &records[i]); err != nil { + return err + } + } + return nil +} + +func cleanupDeletedInstanceArtifacts(instance *model.Instance) []string { + if instance == nil { + return nil + } + + warnings := []string{} + seenCredentials := map[string]struct{}{} + + for _, credentialID := range []string{instance.ManagerCredentialID, instance.AccessCredentialID} { + credentialID = strings.TrimSpace(credentialID) + if credentialID == "" { + continue + } + if _, exists := seenCredentials[credentialID]; exists { + continue + } + seenCredentials[credentialID] = struct{}{} + + deletable, err := canDeleteCredentialAfterInstanceRemoval(credentialID) + if err != nil { + warnings = append(warnings, fmt.Sprintf("failed to inspect credential [%s]: %v", credentialID, err)) + continue + } + if !deletable { + continue + } + if err := deleteCredentialByID(credentialID); err != nil { + warnings = append(warnings, fmt.Sprintf("failed to delete credential [%s]: %v", credentialID, err)) + } + } + + if err := deletePendingRegistrationTokensByInstanceID(instance.ID); err != nil { + warnings = append(warnings, fmt.Sprintf("failed to delete pending registration token for instance [%s]: %v", instance.ID, err)) + } + + return warnings +} diff --git a/plugin/managed/server/instance_test.go b/plugin/managed/server/instance_test.go new file mode 100644 index 00000000..1100bb50 --- /dev/null +++ b/plugin/managed/server/instance_test.go @@ -0,0 +1,605 @@ +package server + +import ( + "bytes" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/config" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/configs/common" +) + +func TestDecodeManagedRegisterRequestSupportsLegacyRawInstanceBody(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, common.REGISTER_API, bytes.NewBufferString(`{ + "id":"legacy-agent-1", + "name":"LegacyAgent", + "application":{"name":"agent","version":{"number":"1.30.3"}}, + "endpoint":"http://127.0.0.1:2900", + "network":{"ip":["192.168.3.9"],"major_ip":"192.168.3.9"} + }`)) + + registerReq, err := decodeManagedRegisterRequest(req) + if err != nil { + t.Fatalf("decode legacy raw register request: %v", err) + } + if registerReq.Client.ID != "legacy-agent-1" { + t.Fatalf("unexpected client id: %q", registerReq.Client.ID) + } + if registerReq.Client.Endpoint != "http://127.0.0.1:2900" { + t.Fatalf("unexpected client endpoint: %q", registerReq.Client.Endpoint) + } + if registerReq.Client.Application.Name != "agent" { + t.Fatalf("unexpected client application: %q", registerReq.Client.Application.Name) + } +} + +func TestDecodeManagedRegisterRequestSupportsWrappedClientBody(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, common.REGISTER_API, bytes.NewBufferString(`{ + "client":{ + "id":"managed-agent-1", + "name":"ManagedAgent", + "application":{"name":"agent","version":{"number":"1.31.0"}}, + "endpoint":"http://127.0.0.1:2900" + } + }`)) + + registerReq, err := decodeManagedRegisterRequest(req) + if err != nil { + t.Fatalf("decode wrapped register request: %v", err) + } + if registerReq.Client.ID != "managed-agent-1" { + t.Fatalf("unexpected client id: %q", registerReq.Client.ID) + } + if registerReq.Client.Endpoint != "http://127.0.0.1:2900" { + t.Fatalf("unexpected client endpoint: %q", registerReq.Client.Endpoint) + } +} + +func newTestBinding(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on random port: %v", err) + } + defer listener.Close() + + return listener.Addr().String() +} + +func TestShouldFallbackInstanceInfoPath(t *testing.T) { + testCases := []struct { + name string + path string + res *util.Result + err error + expect bool + }{ + { + name: "fallback on agent 404", + path: "/agent/_info", + res: &util.Result{StatusCode: http.StatusNotFound}, + err: assertiveError("request error"), + expect: true, + }, + { + name: "no fallback on agent 401", + path: "/agent/_info", + res: &util.Result{StatusCode: http.StatusUnauthorized}, + err: assertiveError("request error"), + expect: false, + }, + { + name: "fallback on transport error", + path: "/agent/_info", + err: assertiveError("dial tcp"), + expect: true, + }, + { + name: "no fallback on non agent path", + path: "/_info", + res: &util.Result{StatusCode: http.StatusNotFound}, + err: assertiveError("request error"), + expect: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := shouldFallbackInstanceInfoPath(tc.path, tc.res, tc.err); actual != tc.expect { + t.Fatalf("unexpected fallback result: got %v want %v", actual, tc.expect) + } + }) + } +} + +func TestWebsocketProxyRouteServedOnWebWithoutEmbeddingAPI(t *testing.T) { + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newTestBinding(t) + webCfg.EmbeddingAPI = false + + api.StartWeb(webCfg) + defer api.StopWeb(webCfg) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/ws_proxy?path=%2Fws", nil) + if err := api.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve ws proxy ui route: %v", err) + } + if resp.Code != http.StatusBadRequest { + t.Fatalf("expected ws proxy ui route to be registered, got %d", resp.Code) + } +} + +func TestProxyInstanceRequestUsesRegisteredProvider(t *testing.T) { + originalProviders := instanceProxyProviders + instanceProxyProviders = nil + defer func() { + instanceProxyProviders = originalProviders + }() + + called := false + RegisterInstanceProxyProvider(func(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, bool, error) { + called = true + if instance == nil || instance.ID != "agent-1" { + t.Fatalf("unexpected instance: %#v", instance) + } + if req == nil || req.Path != "/stats" { + t.Fatalf("unexpected request: %#v", req) + } + if out, ok := responseObjectToUnMarshall.(*util.MapStr); ok { + (*out)["system"] = util.MapStr{"cpu": 12} + } else { + t.Fatalf("unexpected response object type: %T", responseObjectToUnMarshall) + } + return &util.Result{StatusCode: http.StatusOK}, true, nil + }) + + stats := util.MapStr{} + instance := &model.Instance{} + instance.ID = "agent-1" + + ok := fetchManagedInstanceStats(nil, instance, &stats) + if !ok { + t.Fatal("expected stats fetch to succeed") + } + if !called { + t.Fatal("expected registered proxy provider to be called") + } + if _, exists := stats["system"]; !exists { + t.Fatalf("expected stats to be populated, got %#v", stats) + } +} + +func TestFetchManagedInstanceStatsReturnsFalseOnProxyError(t *testing.T) { + originalProviders := instanceProxyProviders + instanceProxyProviders = nil + defer func() { + instanceProxyProviders = originalProviders + }() + + RegisterInstanceProxyProvider(func(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, bool, error) { + return nil, true, assertiveError("boom") + }) + + instance := &model.Instance{} + instance.ID = "agent-1" + + if fetchManagedInstanceStats(nil, instance, &util.MapStr{}) { + t.Fatal("expected stats fetch to fail") + } +} + +func TestGetRuntimeInstanceInfoUsesAgentInfoPathForAgentInstance(t *testing.T) { + originalProviders := instanceProxyProviders + instanceProxyProviders = nil + defer func() { + instanceProxyProviders = originalProviders + }() + + RegisterInstanceProxyProvider(func(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, bool, error) { + if req == nil || req.Path != "/agent/_info" { + t.Fatalf("unexpected request path: %#v", req) + } + if out, ok := responseObjectToUnMarshall.(*model.Instance); ok { + out.Name = "agent-a" + } else { + t.Fatalf("unexpected response object type: %T", responseObjectToUnMarshall) + } + return &util.Result{StatusCode: http.StatusOK}, true, nil + }) + + instance := &model.Instance{ + Application: env.Application{Name: "agent"}, + } + instance.ID = "agent-1" + + info, err := (&APIHandler{}).getRuntimeInstanceInfo(instance) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if info == nil || info.Name != "agent-a" { + t.Fatalf("unexpected runtime instance info: %#v", info) + } +} + +func TestShouldReuseCurrentRequestAuthForEndpoint(t *testing.T) { + originalEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.WebAppConfig = config.WebAppConfig{Enabled: true} + testEnv.SystemConfig.WebAppConfig.NetworkConfig.Binding = "0.0.0.0:9000" + global.RegisterEnv(testEnv) + defer global.RegisterEnv(originalEnv) + + if !shouldReuseCurrentRequestAuthForEndpoint("https://127.0.0.1:9000") { + t.Fatal("expected loopback console web endpoint to reuse current request auth") + } + if shouldReuseCurrentRequestAuthForEndpoint("https://127.0.0.1:9443") { + t.Fatal("expected different port not to reuse current request auth") + } + if shouldReuseCurrentRequestAuthForEndpoint("https://203.0.113.10:9000") { + t.Fatal("expected remote host not to reuse current request auth") + } +} + +func TestEffectiveInstanceProbeAccessTokenUsesCurrentBearerForLocalConsoleEndpoint(t *testing.T) { + originalEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.WebAppConfig = config.WebAppConfig{Enabled: true} + testEnv.SystemConfig.WebAppConfig.NetworkConfig.Binding = "0.0.0.0:9000" + global.RegisterEnv(testEnv) + defer global.RegisterEnv(originalEnv) + + req := httptest.NewRequest(http.MethodPost, "/instance/try_connect", nil) + req.Header.Set("Authorization", "Bearer current-console-token") + + if got := effectiveInstanceProbeAccessToken(req, "https://127.0.0.1:9000", ""); got != "current-console-token" { + t.Fatalf("expected current bearer token to be reused for local console endpoint, got %q", got) + } + if got := effectiveInstanceProbeAccessToken(req, "https://203.0.113.10:9000", ""); got != "" { + t.Fatalf("expected remote endpoint not to reuse current bearer token, got %q", got) + } + if got := effectiveInstanceProbeAccessToken(req, "https://127.0.0.1:9000", "explicit-token"); got != "explicit-token" { + t.Fatalf("expected explicit access token to win, got %q", got) + } +} + +func TestEffectiveManagedInstanceAccessTokenUsesCurrentBearerForLocalConsoleEndpoint(t *testing.T) { + originalEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.WebAppConfig = config.WebAppConfig{Enabled: true} + testEnv.SystemConfig.WebAppConfig.NetworkConfig.Binding = "0.0.0.0:9000" + global.RegisterEnv(testEnv) + defer global.RegisterEnv(originalEnv) + + req := httptest.NewRequest(http.MethodGet, "/instance/stats", nil) + req.Header.Set("Authorization", "Bearer current-console-token") + + localConsole := &model.Instance{Endpoint: "https://127.0.0.1:9000"} + if got := effectiveManagedInstanceAccessToken(req, localConsole); got != "current-console-token" { + t.Fatalf("expected current bearer token to be reused for local console endpoint, got %q", got) + } + + localConsole.AccessCredentialID = "stored-access-credential" + if got := effectiveManagedInstanceAccessToken(req, localConsole); got != "" { + t.Fatalf("expected stored access credential to win over current bearer token, got %q", got) + } + + remote := &model.Instance{Endpoint: "https://203.0.113.10:9000"} + if got := effectiveManagedInstanceAccessToken(req, remote); got != "" { + t.Fatalf("expected remote endpoint not to reuse current bearer token, got %q", got) + } +} + +func TestCleanupDeletedInstanceArtifactsDeletesOwnedCredentialsAndPendingTokens(t *testing.T) { + oldCanDelete := canDeleteCredentialAfterInstanceRemoval + oldDeleteCredential := deleteCredentialByID + oldDeletePending := deletePendingRegistrationTokensByInstanceID + t.Cleanup(func() { + canDeleteCredentialAfterInstanceRemoval = oldCanDelete + deleteCredentialByID = oldDeleteCredential + deletePendingRegistrationTokensByInstanceID = oldDeletePending + }) + + deletableChecks := []string{} + deletedCredentials := []string{} + deletedPendingInstanceID := "" + + canDeleteCredentialAfterInstanceRemoval = func(credentialID string) (bool, error) { + deletableChecks = append(deletableChecks, credentialID) + return credentialID != "shared-credential", nil + } + deleteCredentialByID = func(credentialID string) error { + deletedCredentials = append(deletedCredentials, credentialID) + return nil + } + deletePendingRegistrationTokensByInstanceID = func(instanceID string) error { + deletedPendingInstanceID = instanceID + return nil + } + + warnings := cleanupDeletedInstanceArtifacts(&model.Instance{ + ORMObjectBase: orm.ORMObjectBase{ID: "probe-1"}, + ManagerCredentialID: "manager-credential", + AccessCredentialID: "shared-credential", + }) + + if len(warnings) != 0 { + t.Fatalf("expected no warnings, got %#v", warnings) + } + if len(deletableChecks) != 2 { + t.Fatalf("expected 2 deletable checks, got %#v", deletableChecks) + } + if len(deletedCredentials) != 1 || deletedCredentials[0] != "manager-credential" { + t.Fatalf("expected only owned credential to be deleted, got %#v", deletedCredentials) + } + if deletedPendingInstanceID != "probe-1" { + t.Fatalf("expected pending tokens for probe-1 to be deleted, got %q", deletedPendingInstanceID) + } +} + +func TestDeleteInstanceReturnsCleanupWarnings(t *testing.T) { + oldGetInstance := getManagedInstanceByID + oldDeleteInstance := deleteManagedInstanceRecord + oldCleanup := cleanupDeletedInstanceArtifactsFunc + t.Cleanup(func() { + getManagedInstanceByID = oldGetInstance + deleteManagedInstanceRecord = oldDeleteInstance + cleanupDeletedInstanceArtifactsFunc = oldCleanup + }) + + getManagedInstanceByID = func(instance *model.Instance) (bool, error) { + instance.ManagerCredentialID = "manager-credential" + return true, nil + } + deleteManagedInstanceRecord = func(instance *model.Instance) error { + if instance == nil || instance.ID != "probe-1" { + t.Fatalf("unexpected instance delete request: %#v", instance) + } + return nil + } + cleanupDeletedInstanceArtifactsFunc = func(instance *model.Instance) []string { + if instance == nil || instance.ID != "probe-1" || instance.ManagerCredentialID != "manager-credential" { + t.Fatalf("unexpected cleanup instance: %#v", instance) + } + return []string{"failed to delete credential [manager-credential]: boom"} + } + + req := httptest.NewRequest(http.MethodDelete, "/instance/probe-1", nil) + rec := httptest.NewRecorder() + + (&APIHandler{}).deleteInstance(rec, req, httprouter.Params{{Key: "instance_id", Value: "probe-1"}}) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + body := rec.Body.String() + if !strings.Contains(body, `"result":"deleted"`) { + t.Fatalf("expected deleted result in body, got %s", body) + } + if !strings.Contains(body, `"warnings":["failed to delete credential [manager-credential]: boom"]`) { + t.Fatalf("expected cleanup warnings in body, got %s", body) + } +} + +func TestShouldFetchManagedInstanceStatsLocallyUsesCurrentInstanceID(t *testing.T) { + originalEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.NodeConfig.ID = "console-self" + global.RegisterEnv(testEnv) + defer global.RegisterEnv(originalEnv) + + instance := &model.Instance{ + Endpoint: "https://203.0.113.10:9443", + } + instance.ID = "console-self" + if !shouldFetchManagedInstanceStatsLocally(instance) { + t.Fatal("expected current instance ID to force local stats fetch") + } +} + +func TestFetchManagedInstanceStatsUsesLocalHandlerForCurrentInstanceID(t *testing.T) { + originalEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.NodeConfig.ID = "console-self" + global.RegisterEnv(testEnv) + defer global.RegisterEnv(originalEnv) + + api.HandleAPIMethod(api.GET, "/stats", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"cluster":{"green":1}}`)) + }) + + originalProviders := instanceProxyProviders + instanceProxyProviders = nil + defer func() { + instanceProxyProviders = originalProviders + }() + RegisterInstanceProxyProvider(func(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, bool, error) { + t.Fatal("expected self stats fetch to use local handler instead of proxy") + return nil, true, nil + }) + + stats := util.MapStr{} + instance := &model.Instance{ + Endpoint: "https://203.0.113.10:9443", + } + instance.ID = "console-self" + if !fetchManagedInstanceStats(nil, instance, &stats) { + t.Fatal("expected stats fetch to succeed") + } + cluster, ok := stats["cluster"].(map[string]interface{}) + if !ok || cluster["green"] == nil { + t.Fatalf("unexpected stats payload: %#v", stats) + } +} + +func TestResolveInstanceWebsocketEndpointPrefersInstanceEndpointForLogViewer(t *testing.T) { + instance := &model.Instance{ + Endpoint: "https://127.0.0.1:2900", + Services: []model.ServiceInfo{ + {Name: "api", Endpoint: "https://127.0.0.1:2900"}, + {Name: "web", Endpoint: "https://127.0.0.1:9000"}, + }, + } + + if got := resolveInstanceWebsocketEndpoint(instance, "/ws", "wss://127.0.0.1:9000"); got != "wss://127.0.0.1:2900" { + t.Fatalf("expected instance websocket endpoint, got %q", got) + } +} + +func TestResolveInstanceWebsocketEndpointFallsBackToInstanceEndpoint(t *testing.T) { + instance := &model.Instance{ + Endpoint: "http://127.0.0.1:2900", + Services: []model.ServiceInfo{ + {Name: "api", Endpoint: "http://127.0.0.1:2900"}, + }, + } + + if got := resolveInstanceWebsocketEndpoint(instance, "/ws", ""); got != "ws://127.0.0.1:2900" { + t.Fatalf("expected instance endpoint websocket fallback, got %q", got) + } +} + +func TestResolveInstanceWebsocketEndpointKeepsExplicitEndpointForNonLogPath(t *testing.T) { + instance := &model.Instance{ + Endpoint: "https://127.0.0.1:2900", + Services: []model.ServiceInfo{ + {Name: "web", Endpoint: "https://127.0.0.1:9000"}, + }, + } + + if got := resolveInstanceWebsocketEndpoint(instance, "/custom", "wss://127.0.0.1:9443/custom"); got != "wss://127.0.0.1:9443/custom" { + t.Fatalf("expected explicit websocket target to be preserved, got %q", got) + } +} + +func TestRewriteWebsocketProxyHeadersRewritesOriginToTargetHost(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/ws_proxy?path=%2Fws", nil) + req.Header.Set("Origin", "https://192.168.3.8:9000") + + target, err := url.Parse("ws://192.168.3.8:8080") + if err != nil { + t.Fatalf("failed to parse target: %v", err) + } + + rewriteWebsocketProxyHeaders(req, target) + + if got := req.Host; got != "192.168.3.8:8080" { + t.Fatalf("expected target host to be applied, got %q", got) + } + if got := req.Header.Get("Origin"); got != "http://192.168.3.8:8080" { + t.Fatalf("expected origin to be rewritten for target websocket host, got %q", got) + } +} + +func TestRewriteWebsocketProxyHeadersKeepsOriginEmptyForNonBrowserClients(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/ws_proxy?path=%2Fws", nil) + + target, err := url.Parse("wss://127.0.0.1:9000") + if err != nil { + t.Fatalf("failed to parse target: %v", err) + } + + rewriteWebsocketProxyHeaders(req, target) + + if got := req.Header.Get("Origin"); got != "" { + t.Fatalf("expected empty origin to remain empty, got %q", got) + } + if got := req.Host; got != "127.0.0.1:9000" { + t.Fatalf("expected target host to be applied, got %q", got) + } +} + +func TestShouldApplyInstanceWebsocketAuthSkipsConsoleLogSocket(t *testing.T) { + instance := &model.Instance{} + instance.Application.Name = "console" + + if shouldApplyInstanceWebsocketAuth(instance, "/ws") { + t.Fatal("expected console websocket log proxy to keep browser session auth") + } +} + +func TestShouldApplyInstanceWebsocketAuthKeepsManagedInstanceAuth(t *testing.T) { + instance := &model.Instance{} + instance.Application.Name = "agent" + + if !shouldApplyInstanceWebsocketAuth(instance, "/ws") { + t.Fatal("expected managed instance websocket proxy to keep instance auth") + } + if !shouldApplyInstanceWebsocketAuth(instance, "/_proxy") { + t.Fatal("expected non-websocket proxy requests to keep instance auth") + } +} + +func TestProxyInstanceRequestUsesLocalHandlerForCurrentInstanceID(t *testing.T) { + originalEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.NodeConfig.ID = "console-self" + testEnv.SystemConfig.APIConfig = config.APIConfig{ + Enabled: true, + Security: config.APISecurityConfig{ + Enabled: true, + Username: "local_api_user", + Password: "local_api_password", + }, + } + global.RegisterEnv(testEnv) + defer global.RegisterEnv(originalEnv) + + api.HandleAPIMethod(api.GET, "/_managed/self_proxy_test", func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + username, password, ok := req.BasicAuth() + if !ok || username != "local_api_user" || password != "local_api_password" { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":{"reason":"unauthorized"},"status":401}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + }) + + originalProviders := instanceProxyProviders + instanceProxyProviders = nil + defer func() { + instanceProxyProviders = originalProviders + }() + RegisterInstanceProxyProvider(func(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, bool, error) { + t.Fatal("expected self proxy to use local handler instead of proxy provider") + return nil, true, nil + }) + + instance := &model.Instance{Endpoint: "https://203.0.113.10:9443"} + instance.ID = "console-self" + resp := util.MapStr{} + + res, err := proxyInstanceRequest(instance, &util.Request{ + Method: http.MethodGet, + Path: "/_managed/self_proxy_test", + }, &resp) + if err != nil { + t.Fatalf("expected proxy to succeed, got %v", err) + } + if res == nil || res.StatusCode != http.StatusOK { + t.Fatalf("unexpected proxy result: %#v", res) + } + if ok, _ := resp["ok"].(bool); !ok { + t.Fatalf("unexpected response payload: %#v", resp) + } +} + +type assertiveError string + +func (e assertiveError) Error() string { + return string(e) +} diff --git a/plugin/managed/server/manager.go b/plugin/managed/server/manager.go index 29ce6d8f..1d3880b5 100644 --- a/plugin/managed/server/manager.go +++ b/plugin/managed/server/manager.go @@ -32,14 +32,17 @@ import ( "fmt" log "github.com/cihub/seelog" "infini.sh/console/core" + agent_common "infini.sh/console/modules/agent/common" "infini.sh/framework/core/api" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" + framework_model "infini.sh/framework/core/model" "infini.sh/framework/core/util" "infini.sh/framework/modules/configs/common" "net" "net/http" "net/url" + "strings" "sync" "time" ) @@ -56,36 +59,140 @@ func init() { api.HandleAPIMethod(api.POST, common.SYNC_API, handler.syncConfigs) //client sync configs from config servers api.HandleAPIMethod(api.POST, "/configs/_reload", handler.refreshConfigsRepo) //client sync configs from config servers - //delegate api to instances - api.HandleAPIFunc("/ws_proxy", func(w http.ResponseWriter, req *http.Request) { - log.Debug(req.RequestURI) - endpoint := req.URL.Query().Get("endpoint") - path := req.URL.Query().Get("path") - var tlsConfig = &tls.Config{ - InsecureSkipVerify: true, - } - target, err := url.Parse(endpoint) - if err != nil { - panic(err) - } - newURL, err := url.Parse(path) - if err != nil { - panic(err) + registerWebsocketProxyRoutes() +} + +func registerWebsocketProxyRoutes() { + api.HandleAPIFunc("/ws_proxy", handleWebsocketProxy) + api.HandleUIFuncMethod(api.GET, "/ws_proxy", handleWebsocketProxy, api.RequireLogin()) +} + +func handleWebsocketProxy(w http.ResponseWriter, req *http.Request) { + log.Debug(req.RequestURI) + endpoint, path, err := prepareWebsocketProxyRequest(req) + if err != nil { + handler.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if endpoint == "" { + handler.WriteError(w, "empty endpoint", http.StatusBadRequest) + return + } + var tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + } + target, err := url.Parse(endpoint) + if err != nil { + panic(err) + } + newURL, err := url.Parse(path) + if err != nil { + panic(err) + } + req.URL.Path = newURL.Path + req.URL.RawPath = newURL.RawPath + req.URL.RawQuery = "" + req.RequestURI = req.URL.RequestURI() + rewriteWebsocketProxyHeaders(req, target) + wsProxy := NewSingleHostReverseProxy(target) + wsProxy.Dial = (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial + wsProxy.TLSClientConfig = tlsConfig + wsProxy.ServeHTTP(w, req) +} + +func rewriteWebsocketProxyHeaders(req *http.Request, target *url.URL) { + if req == nil || target == nil { + return + } + req.Header.Set("HOST", target.Host) + req.Host = target.Host + if strings.TrimSpace(req.Header.Get("Origin")) == "" { + return + } + req.Header.Set("Origin", websocketProxyOrigin(target)) +} + +func websocketProxyOrigin(target *url.URL) string { + if target == nil || strings.TrimSpace(target.Host) == "" { + return "" + } + scheme := "http" + switch strings.ToLower(strings.TrimSpace(target.Scheme)) { + case "wss", "https": + scheme = "https" + case "ws", "http", "": + scheme = "http" + default: + scheme = strings.ToLower(strings.TrimSpace(target.Scheme)) + } + return scheme + "://" + target.Host +} + +func prepareWebsocketProxyRequest(req *http.Request) (string, string, error) { + if req == nil { + return "", "", fmt.Errorf("request is nil") + } + query := req.URL.Query() + endpoint := strings.TrimSpace(query.Get("endpoint")) + path := query.Get("path") + instanceID := strings.TrimSpace(query.Get("instance_id")) + if instanceID == "" { + return endpoint, path, nil + } + + _, instance, err := GetRuntimeInstanceByID(instanceID) + if err != nil { + return "", "", err + } + endpoint = resolveInstanceWebsocketEndpoint(instance, path, endpoint) + if shouldApplyInstanceWebsocketAuth(instance, path) { + if err := agent_common.ApplyInstanceHTTPRequestAuth(req, instance); err != nil { + return "", "", err } - req.URL.Path = newURL.Path - req.URL.RawPath = newURL.RawPath - req.URL.RawQuery = "" - req.RequestURI = req.URL.RequestURI() - req.Header.Set("HOST", target.Host) - req.Host = target.Host - wsProxy := NewSingleHostReverseProxy(target) - wsProxy.Dial = (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial - wsProxy.TLSClientConfig = tlsConfig - wsProxy.ServeHTTP(w, req) - }) + } + return endpoint, path, nil +} + +func shouldApplyInstanceWebsocketAuth(instance *framework_model.Instance, path string) bool { + if instance == nil { + return false + } + if strings.TrimSpace(path) != "/ws" { + return true + } + if strings.EqualFold(strings.TrimSpace(instance.Application.Name), "console") { + return false + } + return true +} + +func normalizeWebsocketEndpoint(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + switch { + case strings.HasPrefix(strings.ToLower(endpoint), "https://"): + return "wss://" + endpoint[len("https://"):] + case strings.HasPrefix(strings.ToLower(endpoint), "http://"): + return "ws://" + endpoint[len("http://"):] + default: + return endpoint + } +} + +func resolveInstanceWebsocketEndpoint(instance *framework_model.Instance, path, fallback string) string { + if strings.TrimSpace(path) == "/ws" && instance != nil && strings.TrimSpace(instance.Endpoint) != "" { + return normalizeWebsocketEndpoint(instance.Endpoint) + } + + if strings.TrimSpace(fallback) != "" { + return normalizeWebsocketEndpoint(fallback) + } + if instance == nil { + return "" + } + return normalizeWebsocketEndpoint(instance.Endpoint) } var mTLSClient *http.Client //TODO get mTLSClient diff --git a/plugin/managed/server/proxy_provider.go b/plugin/managed/server/proxy_provider.go new file mode 100644 index 00000000..e90cb6e3 --- /dev/null +++ b/plugin/managed/server/proxy_provider.go @@ -0,0 +1,52 @@ +package server + +import ( + "fmt" + "sync" + + agent_common "infini.sh/console/modules/agent/common" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" +) + +type InstanceProxyProvider func(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, bool, error) + +var ( + instanceProxyProviders []InstanceProxyProvider + instanceProxyProvidersMu sync.RWMutex +) + +func RegisterInstanceProxyProvider(provider InstanceProxyProvider) { + if provider == nil { + return + } + instanceProxyProvidersMu.Lock() + defer instanceProxyProvidersMu.Unlock() + instanceProxyProviders = append(instanceProxyProviders, provider) +} + +func proxyInstanceRequest(instance *model.Instance, req *util.Request, responseObjectToUnMarshall interface{}) (*util.Result, error) { + if instance != nil && isCurrentManagedInstance(instance) { + res, err := proxyManagedAPIRequestLocally(req, responseObjectToUnMarshall) + if err != nil { + return res, err + } + return res, nil + } + + instanceProxyProvidersMu.RLock() + providers := append([]InstanceProxyProvider(nil), instanceProxyProviders...) + instanceProxyProvidersMu.RUnlock() + + for _, provider := range providers { + res, handled, err := provider(instance, req, responseObjectToUnMarshall) + if handled { + return res, err + } + } + endpoint := agent_common.ResolveInstanceRequestEndpoint(instance, req.Path) + if endpoint == "" { + return nil, fmt.Errorf("instance endpoint is empty") + } + return ProxyAgentRequest("runtime", endpoint, req, responseObjectToUnMarshall) +} diff --git a/plugin/managed/server/script.go b/plugin/managed/server/script.go index 4dbf660f..44faa387 100644 --- a/plugin/managed/server/script.go +++ b/plugin/managed/server/script.go @@ -28,14 +28,27 @@ package server import ( + "crypto/x509" + "encoding/json" + "encoding/pem" "fmt" log "github.com/cihub/seelog" + goversion "github.com/hashicorp/go-version" + console_common "infini.sh/console/common" + consoleconfig "infini.sh/console/config" + consolecore "infini.sh/console/core" "infini.sh/console/core/security" "infini.sh/console/modules/agent/common" httprouter "infini.sh/framework/core/api/router" + frameworkconfig "infini.sh/framework/core/config" + "infini.sh/framework/core/env" "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "infini.sh/framework/lib/fasttemplate" + "io" + "net" "net/url" "os" @@ -48,16 +61,184 @@ import ( type Token struct { CreatedAt time.Time UserID string + Product string } const ExpiredIn = time.Millisecond * 1000 * 60 * 60 -const GET_INSTALL_SCRIPT_API = "/instance/_get_install_script" +const getInstallScriptAPI = "/instance/_get_install_script" +const getGatewayInstallScriptAPI = "/instance/_get_gateway_install_script" +const installScriptTemplate = "install_agent.tpl" +const gatewayInstallScriptTemplate = "install_gateway.tpl" +const legacyInstallScriptTemplate = "install_legacy_agent.tpl" +const legacyInstallScriptVersion = common.LegacyAgentMaxVersion +const installProductAgent = "agent" +const installProductGateway = "gateway" +const agentPackageRelativePath = "/agent/stable" +const gatewayPackageRelativePath = "/gateway/stable" +const defaultAgentDownloadURL = "https://release.infinilabs.com/agent/stable" +const defaultGatewayDownloadURL = "https://release.infinilabs.com/gateway/stable" +const defaultAgentInstallDir = "/infini/agent" +const defaultGatewayInstallDir = "/infini/gateway" +const defaultRelayGatewayInstallDir = "/infini/gateway-relay" +const defaultMigrationGatewayInstallDir = "/infini/gateway-migration" +const defaultRelayGatewayServiceName = "gateway-relay" +const defaultMigrationGatewayServiceName = "gateway-migration" +const defaultManagedGatewayAPIUsername = "managed_gateway" +const gatewayTypeRelay = "relay" +const gatewayTypeMigration = "migration" +var refreshManagedLocalTemplatesForInstall func() ([]string, error) var expiredTokenCache = util.NewCacheWithExpireOnAdd(ExpiredIn, 100) +func SetRefreshManagedLocalTemplatesForInstall(refresh func() ([]string, error)) { + refreshManagedLocalTemplatesForInstall = refresh +} + +type gatewayConfig struct { + Setup *gatewaySetupConfig `config:"setup"` +} + +type gatewaySetupConfig struct { + DownloadURL string `config:"download_url"` + InstallDir string `config:"install_dir"` + Version string `config:"version"` + ConsoleEndpoint string `config:"console_endpoint"` + Port string `config:"port"` +} + +type installCommandRequest struct { + GatewayEndpoints []string `json:"gateway_endpoints"` + ServiceType string `json:"service_type"` + RelayRole string `json:"relay_role"` + EnableReverseChannel bool `json:"enable_reverse_channel"` + NoService bool `json:"no_service"` +} + +func renderAgentReverseChannelEndpoints(req *http.Request, configuredEndpoints []string, enabled bool) string { + if !enabled { + return "[]" + } + if len(configuredEndpoints) == 0 { + return `["${server}"]` + } + return string(util.MustToJSONBytes(resolveAgentReverseChannelEndpoints(req, configuredEndpoints))) +} + +func normalizeGatewayType(gatewayType string) string { + switch strings.ToLower(strings.TrimSpace(gatewayType)) { + case gatewayTypeRelay: + return gatewayTypeRelay + case gatewayTypeMigration: + return gatewayTypeMigration + default: + return gatewayTypeMigration + } +} + +func normalizeRelayRole(role string) string { + switch strings.ToLower(strings.TrimSpace(role)) { + case "primary": + return "primary" + case "secondary": + return "secondary" + default: + return "" + } +} + +func resolveGatewayInstallDir(serviceType, configuredInstallDir string) string { + configuredInstallDir = strings.TrimSpace(configuredInstallDir) + if configuredInstallDir != "" { + return configuredInstallDir + } + + switch normalizeGatewayType(serviceType) { + case gatewayTypeRelay: + return defaultRelayGatewayInstallDir + case gatewayTypeMigration: + return defaultMigrationGatewayInstallDir + default: + return defaultGatewayInstallDir + } +} + +func resolveGatewayServiceName(serviceType string) string { + switch normalizeGatewayType(serviceType) { + case gatewayTypeRelay: + return defaultRelayGatewayServiceName + case gatewayTypeMigration: + return defaultMigrationGatewayServiceName + default: + return defaultMigrationGatewayServiceName + } +} + +func normalizeManagedServerEndpoints(endpoints []string) []string { + if len(endpoints) == 0 { + return nil + } + result := make([]string, 0, len(endpoints)) + seen := map[string]struct{}{} + for _, endpoint := range endpoints { + normalized := strings.TrimRight(strings.TrimSpace(endpoint), "/") + if normalized == "" { + continue + } + if _, exists := seen[normalized]; exists { + continue + } + seen[normalized] = struct{}{} + result = append(result, normalized) + } + return result +} + +func listGatewayManagedEndpoints(serviceType string) ([]string, error) { + queryDSL := util.MapStr{ + "size": 1000, + "query": util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + {"term": util.MapStr{"application.name": "gateway"}}, + }, + }, + }, + } + if serviceType != "" { + queryDSL["query"] = util.MapStr{ + "bool": util.MapStr{ + "must": []util.MapStr{ + {"term": util.MapStr{"application.name": "gateway"}}, + {"term": util.MapStr{"labels.service_type": serviceType}}, + }, + }, + } + } + q := orm.Query{ + RawQuery: util.MustToJSONBytes(queryDSL), + } + instances := []model.Instance{} + if err, _ := orm.SearchWithJSONMapper(&instances, &q); err != nil { + return nil, err + } + endpoints := make([]string, 0, len(instances)) + for _, instance := range instances { + endpoint := strings.TrimSpace(strings.TrimRight(instance.GetEndpoint(), "/")) + if endpoint == "" { + continue + } + endpoints = append(endpoints, endpoint) + } + return normalizeManagedServerEndpoints(endpoints), nil +} + +func resolveAgentRemoteConfigServers(consoleEndpoint string) []string { + return []string{strings.TrimRight(strings.TrimSpace(consoleEndpoint), "/")} +} + func (h *APIHandler) generateInstallCommand(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { - claims, ok := req.Context().Value("user").(*security.UserClaims) - if !ok { + user, err := security.FromUserContext(req.Context()) + if err != nil || user == nil { h.WriteError(w, "user not found", http.StatusInternalServerError) return } @@ -66,116 +247,722 @@ func (h *APIHandler) generateInstallCommand(w http.ResponseWriter, req *http.Req h.WriteError(w, "agent setup config was not found, please configure in the configuration file first", http.StatusInternalServerError) return } + payload := installCommandRequest{} + if req.Body != nil { + defer req.Body.Close() + if err := json.NewDecoder(req.Body).Decode(&payload); err != nil && err != io.EOF { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + } var ( t *Token tokenStr string ) - //TODO: get location from request, validate it - location := "/opt/agent" + location := resolveInstallDir(agCfg.Setup.InstallDir, defaultAgentInstallDir) tokenStr = util.GetUUID() t = &Token{ CreatedAt: time.Now(), - UserID: claims.UserId, + UserID: user.UserId, + Product: installProductAgent, } expiredTokenCache.Put(tokenStr, t) - consoleEndpoint := agCfg.Setup.ConsoleEndpoint - if consoleEndpoint == "" { - consoleEndpoint = getDefaultEndpoint(req) + consoleEndpoint := resolveConsoleEndpoint(req, agCfg.Setup.ConsoleEndpoint) + installVersion := strings.TrimSpace(agCfg.Setup.Version) + endpoint, err := buildInstallScriptURL(consoleEndpoint, tokenStr, installVersion, payload.EnableReverseChannel) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("build agent install script url failed: %v", err) + return + } + downloadURL, err := resolveAgentDownloadURL(consoleEndpoint, agCfg.Setup.DownloadURL) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("resolve agent download url failed: %v", err) + return + } + if strings.TrimSpace(agCfg.Setup.DownloadURL) == "" { + logAutoResolvedDownloadURL("agent", downloadURL, defaultAgentDownloadURL) } - basePath := global.Env().SystemConfig.WebAppConfig.BasePath - if len(basePath) > 0 { - consoleEndpoint = fmt.Sprintf("%s%s", strings.TrimRight(consoleEndpoint, "/"), basePath) + h.WriteJSON(w, util.MapStr{ + "script": buildInstallCommand(endpoint, location, payload.NoService), + "token": tokenStr, + "expired_at": t.CreatedAt.Add(ExpiredIn), + }, http.StatusOK) +} + +func (h *APIHandler) prepareRegistration(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + agCfg := common.GetAgentConfig() + consoleEndpoint := resolveConsoleEndpoint(req, "") + if agCfg != nil && agCfg.Setup != nil { + consoleEndpoint = resolveConsoleEndpoint(req, agCfg.Setup.ConsoleEndpoint) } - endpoint, err := url.JoinPath(consoleEndpoint, GET_INSTALL_SCRIPT_API) + record, tokenValue, err := common.CreatePendingManagerToken(common.AgentPendingTokenSourceUI) if err != nil { - panic(err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return } h.WriteJSON(w, util.MapStr{ - "script": fmt.Sprintf(`curl -ksSL %s?token=%s |sudo bash -s -- -u %s -t %v`, - endpoint, tokenStr, agCfg.Setup.DownloadURL, location), + "id": record.ID, + "endpoint": consoleEndpoint, + "token": tokenValue, + "expired_at": time.UnixMilli(record.ExpiresAt), + }, http.StatusOK) +} + +func (h *APIHandler) generateGatewayInstallCommand(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + user, err := security.FromUserContext(req.Context()) + if err != nil || user == nil { + h.WriteError(w, "user not found", http.StatusInternalServerError) + return + } + payload := installCommandRequest{} + if req.Body != nil { + defer req.Body.Close() + if err := json.NewDecoder(req.Body).Decode(&payload); err != nil && err != io.EOF { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + } + + gwCfg := getGatewayConfig() + if gwCfg == nil || gwCfg.Setup == nil { + h.WriteError(w, "gateway setup config was not found, please configure in the configuration file first", http.StatusInternalServerError) + return + } + + serviceType := normalizeGatewayType(payload.ServiceType) + relayRole := normalizeRelayRole(payload.RelayRole) + if serviceType != gatewayTypeRelay { + relayRole = "" + } + location := resolveGatewayInstallDir(serviceType, gwCfg.Setup.InstallDir) + tokenStr := util.GetUUID() + t := &Token{ + CreatedAt: time.Now(), + UserID: user.UserId, + Product: installProductGateway, + } + expiredTokenCache.Put(tokenStr, t) + + consoleEndpoint := resolveConsoleEndpoint(req, gwCfg.Setup.ConsoleEndpoint) + installVersion := strings.TrimSpace(gwCfg.Setup.Version) + endpoint, err := buildInstallScriptURLForAPI(consoleEndpoint, getGatewayInstallScriptAPI, tokenStr, installVersion) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("build gateway install script url failed: %v", err) + return + } + parsedEndpoint, err := url.Parse(endpoint) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("parse gateway install script url failed: %v", err) + return + } + endpointQuery := parsedEndpoint.Query() + endpointQuery.Set("service_type", serviceType) + if relayRole != "" { + endpointQuery.Set("relay_role", relayRole) + } + parsedEndpoint.RawQuery = endpointQuery.Encode() + endpoint = parsedEndpoint.String() + + downloadURL, err := resolveGatewayDownloadURL(consoleEndpoint, gwCfg.Setup.DownloadURL) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("resolve gateway download url failed: %v", err) + return + } + if strings.TrimSpace(gwCfg.Setup.DownloadURL) == "" { + logAutoResolvedDownloadURL("gateway", downloadURL, defaultGatewayDownloadURL) + } + h.WriteJSON(w, util.MapStr{ + "script": buildGatewayInstallCommand(endpoint, location, payload.NoService), "token": tokenStr, "expired_at": t.CreatedAt.Add(ExpiredIn), }, http.StatusOK) } +func buildInstallCommand(endpoint, location string, noService bool) string { + shell := "sudo bash" + extraArgs := "" + if noService { + shell = "bash" + extraArgs = " --no-service" + } + command := fmt.Sprintf(`curl -ksSL %q |%s -s -- -t %q%s`, + endpoint, shell, location, extraArgs) + return command +} + +func buildGatewayInstallCommand(endpoint, location string, noService bool) string { + shell := "sudo bash" + extraArgs := "" + if noService { + shell = "bash" + extraArgs = " --no-service" + } + command := fmt.Sprintf(`curl -ksSL %q |%s -s -- -d %q%s`, + endpoint, shell, location, extraArgs) + return command +} + +func buildInstallScriptURL(consoleEndpoint, tokenStr, installVersion string, enableReverseChannel bool) (string, error) { + endpoint, err := buildInstallScriptURLForAPI(consoleEndpoint, getInstallScriptAPI, tokenStr, installVersion) + if err != nil { + return "", err + } + parsed, err := url.Parse(endpoint) + if err != nil { + return "", err + } + query := parsed.Query() + if enableReverseChannel { + query.Set("enable_reverse_channel", "true") + } + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} + +func formatBuildVersion(version, buildNumber string) string { + version = strings.TrimSpace(version) + buildNumber = strings.TrimSpace(buildNumber) + if version == "" { + return "" + } + if buildNumber == "" { + return version + } + return fmt.Sprintf("%s-%s", version, buildNumber) +} + +func getDefaultInstallVersion(configuredVersion string) string { + configuredVersion = strings.TrimSpace(configuredVersion) + if configuredVersion != "" { + return configuredVersion + } + return formatBuildVersion(global.Env().GetVersion(), global.Env().GetBuildNumber()) +} + +func buildInstallScriptURLForAPI(consoleEndpoint, apiPath, tokenStr, installVersion string) (string, error) { + parsedURL, err := url.Parse(consoleEndpoint) + if err != nil { + return "", err + } + parsedURL.Path = path.Join(parsedURL.Path, apiPath) + + query := parsedURL.Query() + query.Set("token", tokenStr) + if installVersion != "" { + query.Set("version", installVersion) + } + parsedURL.RawQuery = query.Encode() + return parsedURL.String(), nil +} + +func resolveAgentDownloadURL(consoleEndpoint, downloadURL string) (string, error) { + return resolvePackageDownloadURL(consoleEndpoint, downloadURL, defaultAgentDownloadURL, agentPackageRelativePath) +} + +func resolveGatewayDownloadURL(consoleEndpoint, downloadURL string) (string, error) { + return resolvePackageDownloadURL(consoleEndpoint, downloadURL, defaultGatewayDownloadURL, gatewayPackageRelativePath) +} + +func resolvePackageDownloadURL(consoleEndpoint, downloadURL, defaultDownloadURL, relativePath string) (string, error) { + normalized := strings.TrimRight(strings.TrimSpace(downloadURL), "/") + if normalized != "" { + return normalized, nil + } + + hasSelfHostedPackages, err := consoleconfig.HasSelfHostedPackageFiles(global.Env().SystemConfig.WebAppConfig.UI.LocalPath, strings.TrimPrefix(relativePath, "/")) + if err != nil { + return "", err + } + if hasSelfHostedPackages { + parsedURL, err := url.Parse(consoleEndpoint) + if err != nil { + return "", err + } + parsedURL.Path = path.Join(parsedURL.Path, relativePath) + return parsedURL.String(), nil + } + return defaultDownloadURL, nil +} + +func logAutoResolvedDownloadURL(product, downloadURL, defaultDownloadURL string) { + if downloadURL == defaultDownloadURL { + log.Debugf("%s.setup.download_url is empty, defaulting to public release mirror: %s", product, console_common.MaskLogEndpoint(downloadURL)) + return + } + log.Debugf("%s.setup.download_url is empty, using Console self-hosted package path: %s", product, console_common.MaskLogEndpoint(downloadURL)) +} + +func resolveInstallDir(installDir, defaultInstallDir string) string { + normalized := strings.TrimSpace(installDir) + if normalized != "" { + return normalized + } + return defaultInstallDir +} + +func resolveConsoleEndpoint(req *http.Request, configuredEndpoint string) string { + configuredEndpoint = strings.TrimRight(strings.TrimSpace(configuredEndpoint), "/") + if configuredEndpoint != "" { + return configuredEndpoint + } + + if envEndpoint := strings.TrimRight(strings.TrimSpace(os.Getenv("INFINI_CONSOLE_ENDPOINT")), "/"); envEndpoint != "" { + return envEndpoint + } + + consoleEndpoint := getDefaultEndpoint(req) + basePath := global.Env().SystemConfig.WebAppConfig.BasePath + if len(basePath) > 0 { + consoleEndpoint = fmt.Sprintf("%s%s", strings.TrimRight(consoleEndpoint, "/"), basePath) + } + return consoleEndpoint +} + +func getEndpointHostname(endpoint string) string { + parsed, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil { + return "" + } + return parsed.Hostname() +} + +func resolveConsoleTLSServerName(consoleEndpoint string) string { + hostname := getEndpointHostname(consoleEndpoint) + if hostname != "" && net.ParseIP(hostname) == nil { + return hostname + } + + for _, tlsCfg := range getPreferredConsoleTLSConfigs(consoleEndpoint) { + if serverName := resolveTLSServerNameFromConfig(tlsCfg); serverName != "" { + return serverName + } + } + return hostname +} + +func getPreferredConsoleTLSConfigs(consoleEndpoint string) []*frameworkconfig.TLSConfig { + webCfg := global.Env().SystemConfig.WebAppConfig + apiCfg := global.Env().SystemConfig.APIConfig + + configs := make([]*frameworkconfig.TLSConfig, 0, 2) + if apiCfg.Enabled && endpointMatchesPublishedEndpoint(consoleEndpoint, apiCfg.GetEndpoint()) { + configs = append(configs, &apiCfg.TLSConfig) + } + if webCfg.Enabled && endpointMatchesPublishedEndpoint(consoleEndpoint, webCfg.GetEndpoint()) { + configs = append(configs, &webCfg.TLSConfig) + } + if len(configs) > 0 { + return configs + } + + if webCfg.Enabled { + configs = append(configs, &webCfg.TLSConfig) + } + if apiCfg.Enabled { + configs = append(configs, &apiCfg.TLSConfig) + } + return configs +} + +func endpointMatchesPublishedEndpoint(endpoint, published string) bool { + ep, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil { + return false + } + pub, err := url.Parse(strings.TrimSpace(published)) + if err != nil { + return false + } + + return strings.EqualFold(ep.Scheme, pub.Scheme) && ep.Port() == pub.Port() +} + +func resolveTLSServerNameFromConfig(tlsCfg *frameworkconfig.TLSConfig) string { + if tlsCfg == nil { + return "" + } + if serverName := strings.TrimSpace(tlsCfg.DefaultDomain); serverName != "" { + return serverName + } + return readTLSServerNameFromCertFile(strings.TrimSpace(tlsCfg.TLSCertFile)) +} + +func readTLSServerNameFromCertFile(certFile string) string { + if certFile == "" || !util.FileExists(certFile) { + return "" + } + rawCert, err := os.ReadFile(certFile) + if err != nil { + return "" + } + for len(rawCert) > 0 { + block, rest := pem.Decode(rawCert) + if block == nil { + break + } + rawCert = rest + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + for _, dnsName := range cert.DNSNames { + if dnsName = strings.TrimSpace(dnsName); dnsName != "" { + return dnsName + } + } + commonName := strings.TrimSpace(cert.Subject.CommonName) + if commonName != "" && net.ParseIP(commonName) == nil { + return commonName + } + } + return "" +} + +func resolveAgentReverseChannelEndpoint(req *http.Request, configuredEndpoint string) string { + configuredEndpoint = strings.TrimRight(strings.TrimSpace(configuredEndpoint), "/") + if configuredEndpoint != "" { + return configuredEndpoint + } + + if envEndpoint := strings.TrimRight(strings.TrimSpace(os.Getenv("INFINI_AGENT_REVERSE_CHANNEL_ENDPOINT")), "/"); envEndpoint != "" { + return envEndpoint + } + + if shouldUseAPIEndpointForReverseMTLS() { + return strings.TrimRight(global.Env().SystemConfig.APIConfig.GetEndpoint(), "/") + } + + if canServeAgentReverseOnWeb() { + return resolveConsoleEndpoint(req, "") + } + + if global.Env().SystemConfig.APIConfig.Enabled { + return strings.TrimRight(global.Env().SystemConfig.APIConfig.GetEndpoint(), "/") + } + + return resolveConsoleEndpoint(req, "") +} + +func resolveAgentReverseChannelEndpoints(req *http.Request, configuredEndpoints []string) []string { + endpoints := make([]string, 0, len(configuredEndpoints)) + for _, endpoint := range configuredEndpoints { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + continue + } + endpoints = append(endpoints, endpoint) + } + if len(endpoints) > 0 { + return endpoints + } + return []string{resolveAgentReverseChannelEndpoint(req, "")} +} + +func shouldUseAPIEndpointForReverseMTLS() bool { + apiCfg := global.Env().SystemConfig.APIConfig + return apiCfg.Enabled && + apiCfg.WebsocketConfig.Enabled && + apiCfg.TLSConfig.TLSEnabled && + !apiCfg.TLSConfig.TLSInsecureSkipVerify +} + +func canServeAgentReverseOnWeb() bool { + webCfg := global.Env().SystemConfig.WebAppConfig + if !webCfg.Enabled { + return false + } + if webCfg.EmbeddingAPI && global.Env().SystemConfig.APIConfig.Enabled && global.Env().SystemConfig.APIConfig.WebsocketConfig.Enabled { + return true + } + return webCfg.WebsocketConfig.Enabled +} + +func shouldUseLegacyInstallScriptTemplate(installVersion string) bool { + installVersion = strings.TrimSpace(strings.TrimPrefix(installVersion, "v")) + if installVersion == "" { + return false + } + + requestedVersion, err := goversion.NewVersion(installVersion) + if err != nil { + return false + } + legacyVersion, err := goversion.NewVersion(legacyInstallScriptVersion) + if err != nil { + return false + } + return requestedVersion.LessThan(legacyVersion) || requestedVersion.Equal(legacyVersion) +} + func getDefaultEndpoint(req *http.Request) string { scheme := "http" - if req.TLS != nil { + if consolecore.RequestUsesSecureTransport(req) { scheme = "https" } - return fmt.Sprintf("%s://%s", scheme, req.Host) + return fmt.Sprintf("%s://%s", scheme, getForwardedHost(req)) +} + +func getForwardedHost(req *http.Request) string { + if req == nil { + return "" + } + + if host := strings.TrimSpace(strings.Split(req.Header.Get("X-Forwarded-Host"), ",")[0]); host != "" { + return host + } + + if host := parseForwardedHost(req.Header.Get("Forwarded")); host != "" { + return host + } + + return req.Host +} + +func parseForwardedHost(value string) string { + if value == "" { + return "" + } + + for _, forwardedValue := range strings.Split(value, ",") { + for _, token := range strings.Split(forwardedValue, ";") { + parts := strings.SplitN(strings.TrimSpace(token), "=", 2) + if len(parts) != 2 || !strings.EqualFold(parts[0], "host") { + continue + } + return strings.Trim(parts[1], "\"") + } + } + + return "" } func (h *APIHandler) getInstallScript(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { tokenStr := h.GetParameter(req, "token") - if strings.TrimSpace(tokenStr) == "" { - h.WriteError(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + _, err := validateInstallToken(tokenStr, installProductAgent) + if err != nil { + h.WriteError(w, err.Error(), http.StatusUnauthorized) return } - v := expiredTokenCache.Get(tokenStr) - if v == nil { - h.WriteError(w, "token is invalid", http.StatusUnauthorized) + if _, err := refreshManagedLocalTemplatesForInstall(); err != nil { + log.Errorf("refresh managed local templates failed before generating agent install script: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - t, ok := v.(*Token) - if !ok || t.CreatedAt.Add(ExpiredIn).Before(time.Now()) { - expiredTokenCache.Delete(tokenStr) - h.WriteError(w, "token was expired", http.StatusUnauthorized) + agCfg := common.GetAgentConfig() + if agCfg == nil || agCfg.Setup == nil { + h.WriteError(w, "agent setup config was not found, please configure in the configuration file first", http.StatusInternalServerError) return } - - agCfg := common.GetAgentConfig() - caCert, clientCertPEM, clientKeyPEM, err := common.GenerateServerCert(agCfg.Setup.CACertFile, agCfg.Setup.CAKeyFile) + caCert, clientCertPEM, clientKeyPEM, err := common.GenerateClientCert(agCfg.Setup.CACertFile, agCfg.Setup.CAKeyFile) if err != nil { - log.Error(err) + log.Errorf("generate agent install certs failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - scriptTplPath := path.Join(global.Env().GetConfigDir(), "install_agent.tpl") + installVersion := h.GetParameterOrDefault(req, "version", getDefaultInstallVersion(agCfg.Setup.Version)) + scriptTplFile := installScriptTemplate + if shouldUseLegacyInstallScriptTemplate(installVersion) { + scriptTplFile = legacyInstallScriptTemplate + } + + scriptTplPath := path.Join(global.Env().GetConfigDir(), scriptTplFile) buf, err := os.ReadFile(scriptTplPath) if err != nil { - log.Error(err) + log.Errorf("read agent install script template failed: %v", err) h.WriteError(w, err.Error(), http.StatusInternalServerError) return } tpl := fasttemplate.New(string(buf), "{{", "}}") - downloadURL := agCfg.Setup.DownloadURL - if downloadURL == "" { - downloadURL = "https://release.infinilabs.com/agent/stable/" - } - port := agCfg.Setup.Port if port == "" { port = "8080" } - consoleEndpoint := agCfg.Setup.ConsoleEndpoint - if consoleEndpoint == "" { - consoleEndpoint = getDefaultEndpoint(req) + consoleEndpoint := resolveConsoleEndpoint(req, agCfg.Setup.ConsoleEndpoint) + consoleDomain := resolveConsoleTLSServerName(consoleEndpoint) + remoteConfigServers := resolveAgentRemoteConfigServers(consoleEndpoint) + reverseChannelEnabled := strings.EqualFold(strings.TrimSpace(req.URL.Query().Get("enable_reverse_channel")), "true") + reverseChannelEndpoints := renderAgentReverseChannelEndpoints( + req, + agCfg.Setup.ReverseChannelEndpoints, + reverseChannelEnabled, + ) + downloadURL, err := resolveAgentDownloadURL(consoleEndpoint, agCfg.Setup.DownloadURL) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("resolve agent install script download url failed: %v", err) + return + } + managerTokenRecord, managerTokenValue, err := common.CreatePendingManagerToken(common.AgentPendingTokenSourceVM) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return } _, err = tpl.Execute(w, map[string]interface{}{ - "base_url": agCfg.Setup.DownloadURL, - "console_endpoint": consoleEndpoint, - "client_crt": clientCertPEM, - "client_key": clientKeyPEM, - "ca_crt": caCert, - "port": port, - "token": tokenStr, + "base_url": downloadURL, + "console_endpoint": consoleEndpoint, + "console_domain": consoleDomain, + "remote_config_servers": string(util.MustToJSONBytes(remoteConfigServers)), + "reverse_channel_endpoints": reverseChannelEndpoints, + "embedding_api": "false", + "websocket_enabled": fmt.Sprintf("%t", !reverseChannelEnabled), + "client_crt": clientCertPEM, + "client_key": clientKeyPEM, + "ca_crt": caCert, + "port": port, + "token": tokenStr, + "access_token": managerTokenValue, + "manager_token": managerTokenValue, + "manager_token_key": common.AgentManagerTokenKey(), + "manager_token_id": managerTokenRecord.ID, + "version": installVersion, }) if err != nil { h.WriteError(w, err.Error(), http.StatusInternalServerError) } } + +func (h *APIHandler) getGatewayInstallScript(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + tokenStr := h.GetParameter(req, "token") + _, err := validateInstallToken(tokenStr, installProductGateway) + if err != nil { + h.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + + if _, err := refreshManagedLocalTemplatesForInstall(); err != nil { + log.Errorf("refresh managed local templates failed before generating gateway install script: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + gwCfg := getGatewayConfig() + if gwCfg == nil || gwCfg.Setup == nil { + h.WriteError(w, "gateway setup config was not found, please configure in the configuration file first", http.StatusInternalServerError) + return + } + + agCfg := common.GetAgentConfig() + caCert, clientCertPEM, clientKeyPEM, err := common.GenerateClientCert(agCfg.Setup.CACertFile, agCfg.Setup.CAKeyFile) + if err != nil { + log.Errorf("generate gateway install certs failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + _, relayServerCertPEM, relayServerKeyPEM, err := common.GenerateServerCert(agCfg.Setup.CACertFile, agCfg.Setup.CAKeyFile) + if err != nil { + log.Errorf("generate gateway relay server certs failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + installVersion := h.GetParameterOrDefault(req, "version", getDefaultInstallVersion(gwCfg.Setup.Version)) + scriptTplPath := path.Join(global.Env().GetConfigDir(), gatewayInstallScriptTemplate) + buf, err := os.ReadFile(scriptTplPath) + if err != nil { + log.Errorf("read gateway install script template failed: %v", err) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + tpl := fasttemplate.New(string(buf), "{{", "}}") + consoleEndpoint := resolveConsoleEndpoint(req, gwCfg.Setup.ConsoleEndpoint) + consoleDomain := resolveConsoleTLSServerName(consoleEndpoint) + serviceType := normalizeGatewayType(req.URL.Query().Get("service_type")) + relayRole := normalizeRelayRole(req.URL.Query().Get("relay_role")) + if serviceType != gatewayTypeRelay { + relayRole = "" + } + downloadURL, err := resolveGatewayDownloadURL(consoleEndpoint, gwCfg.Setup.DownloadURL) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Errorf("resolve gateway install script download url failed: %v", err) + return + } + _, managerTokenValue, err := common.CreatePendingManagerToken(common.AgentPendingTokenSourceVM) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + port := strings.TrimSpace(gwCfg.Setup.Port) + if port == "" { + port = "2900" + } + localAPIPassword := util.GetUUID() + _, err = tpl.Execute(w, map[string]interface{}{ + "base_url": downloadURL, + "console_endpoint": consoleEndpoint, + "console_domain": consoleDomain, + "client_crt": clientCertPEM, + "client_key": clientKeyPEM, + "relay_server_crt": relayServerCertPEM, + "relay_server_key": relayServerKeyPEM, + "ca_crt": caCert, + "port": port, + "access_token": managerTokenValue, + "api_security_username": defaultManagedGatewayAPIUsername, + "api_security_password": localAPIPassword, + "service_type": serviceType, + "relay_role": relayRole, + "service_name": resolveGatewayServiceName(serviceType), + "install_dir": resolveGatewayInstallDir(serviceType, ""), + "version": installVersion, + }) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + } +} + +func validateInstallToken(tokenStr, product string) (*Token, error) { + if strings.TrimSpace(tokenStr) == "" { + return nil, fmt.Errorf("%s", http.StatusText(http.StatusUnauthorized)) + } + + v := expiredTokenCache.Get(tokenStr) + if v == nil { + return nil, fmt.Errorf("token is invalid") + } + + t, ok := v.(*Token) + if !ok || t.CreatedAt.Add(ExpiredIn).Before(time.Now()) { + expiredTokenCache.Delete(tokenStr) + return nil, fmt.Errorf("token was expired") + } + if t.Product != product { + return nil, fmt.Errorf("token is invalid") + } + return t, nil +} + +func getGatewayConfig() *gatewayConfig { + cfg := &gatewayConfig{ + Setup: &gatewaySetupConfig{}, + } + _, err := env.ParseConfig("gateway", cfg) + if err != nil { + log.Errorf("gateway config not found: %v", err) + } + return cfg +} diff --git a/plugin/managed/server/script_test.go b/plugin/managed/server/script_test.go new file mode 100644 index 00000000..09dac398 --- /dev/null +++ b/plugin/managed/server/script_test.go @@ -0,0 +1,807 @@ +package server + +import ( + "crypto/tls" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + frameworkconfig "infini.sh/framework/core/config" + "infini.sh/framework/core/global" + "infini.sh/framework/core/util" +) + +func TestGetDefaultEndpoint(t *testing.T) { + tests := []struct { + name string + setup func(req *http.Request) + expected string + }{ + { + name: "tls request uses https", + setup: func(req *http.Request) { + req.TLS = &tls.ConnectionState{} + }, + expected: "https://console.local", + }, + { + name: "forwarded proto and host override request", + setup: func(req *http.Request) { + req.Host = "127.0.0.1:9000" + req.Header.Set("X-Forwarded-Proto", "https") + req.Header.Set("X-Forwarded-Host", "console.example.com") + }, + expected: "https://console.example.com", + }, + { + name: "forwarded header provides proto and host", + setup: func(req *http.Request) { + req.Host = "127.0.0.1:9000" + req.Header.Set("Forwarded", `for=127.0.0.1;proto=https;host=console.example.com:9443`) + }, + expected: "https://console.example.com:9443", + }, + { + name: "plain request uses http host", + setup: func(req *http.Request) { + }, + expected: "http://console.local", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://console.local/instance/_get_install_script", nil) + tt.setup(req) + + if got := getDefaultEndpoint(req); got != tt.expected { + t.Fatalf("expected %q, got %q", tt.expected, got) + } + }) + } +} + +func TestGetInstallScriptFailsWhenManagedTemplatesRefreshFails(t *testing.T) { + oldRefresh := refreshManagedLocalTemplatesForInstall + refreshManagedLocalTemplatesForInstall = func() ([]string, error) { + return nil, fmt.Errorf("refresh failed") + } + t.Cleanup(func() { + refreshManagedLocalTemplatesForInstall = oldRefresh + }) + + token := util.GetUUID() + expiredTokenCache.Put(token, &Token{ + CreatedAt: time.Now(), + Product: installProductAgent, + }) + + req := httptest.NewRequest(http.MethodGet, "http://console.local/instance/_get_install_script?token="+token, nil) + rec := httptest.NewRecorder() + + (&APIHandler{}).getInstallScript(rec, req, nil) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("expected status %d, got %d", http.StatusInternalServerError, rec.Code) + } + if !strings.Contains(rec.Body.String(), "refresh failed") { + t.Fatalf("expected refresh error in body, got %q", rec.Body.String()) + } +} + +func TestResolveConsoleEndpointPriority(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://console.local/instance/_get_install_script", nil) + + t.Run("prefers configured endpoint", func(t *testing.T) { + t.Setenv("INFINI_CONSOLE_ENDPOINT", "http://127.0.0.1:9000") + got := resolveConsoleEndpoint(req, "https://configured.local:9443") + if got != "https://configured.local:9443" { + t.Fatalf("expected configured endpoint, got %q", got) + } + }) + + t.Run("uses env endpoint when configured missing", func(t *testing.T) { + t.Setenv("INFINI_CONSOLE_ENDPOINT", "http://127.0.0.1:9000") + got := resolveConsoleEndpoint(req, "") + if got != "http://127.0.0.1:9000" { + t.Fatalf("expected env endpoint, got %q", got) + } + }) +} + +func TestResolveAgentReverseChannelEndpointPrefersAPIEndpointWhenAPIMTLSIsEnabled(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "https://console.local:9000/instance/_get_install_script", nil) + req.TLS = &tls.ConnectionState{} + oldWebConfig := global.Env().SystemConfig.WebAppConfig + oldAPIConfig := global.Env().SystemConfig.APIConfig + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig = oldWebConfig + global.Env().SystemConfig.APIConfig = oldAPIConfig + }) + + global.Env().SystemConfig.WebAppConfig.Enabled = true + global.Env().SystemConfig.WebAppConfig.EmbeddingAPI = true + global.Env().SystemConfig.WebAppConfig.TLSConfig.TLSEnabled = true + global.Env().SystemConfig.WebAppConfig.NetworkConfig.Publish = "console-web.local:9000" + global.Env().SystemConfig.APIConfig.Enabled = true + global.Env().SystemConfig.APIConfig.WebsocketConfig.Enabled = true + global.Env().SystemConfig.APIConfig.TLSConfig.TLSEnabled = true + global.Env().SystemConfig.APIConfig.TLSConfig.TLSInsecureSkipVerify = false + global.Env().SystemConfig.APIConfig.NetworkConfig.Publish = "console-api.local:2900" + + got := resolveAgentReverseChannelEndpoint(req, "") + if got != "https://console-api.local:2900" { + t.Fatalf("expected api endpoint, got %q", got) + } +} + +func TestResolveAgentReverseChannelEndpointPrefersWebEndpointWhenWebCanServeWSWithoutAPIMTLS(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "https://console.local:9000/instance/_get_install_script", nil) + req.TLS = &tls.ConnectionState{} + oldWebConfig := global.Env().SystemConfig.WebAppConfig + oldAPIConfig := global.Env().SystemConfig.APIConfig + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig = oldWebConfig + global.Env().SystemConfig.APIConfig = oldAPIConfig + }) + + global.Env().SystemConfig.WebAppConfig.Enabled = true + global.Env().SystemConfig.WebAppConfig.EmbeddingAPI = true + global.Env().SystemConfig.WebAppConfig.TLSConfig.TLSEnabled = true + global.Env().SystemConfig.WebAppConfig.NetworkConfig.Publish = "console-web.local:9000" + global.Env().SystemConfig.APIConfig.Enabled = true + global.Env().SystemConfig.APIConfig.WebsocketConfig.Enabled = true + global.Env().SystemConfig.APIConfig.TLSConfig.TLSEnabled = true + global.Env().SystemConfig.APIConfig.TLSConfig.TLSInsecureSkipVerify = true + global.Env().SystemConfig.APIConfig.NetworkConfig.Publish = "console-api.local:2900" + + got := resolveAgentReverseChannelEndpoint(req, "") + if got != "https://console.local:9000" { + t.Fatalf("expected web endpoint, got %q", got) + } +} + +func TestResolveAgentReverseChannelEndpointFallsBackToAPIEndpoint(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://console.local/instance/_get_install_script", nil) + oldWebConfig := global.Env().SystemConfig.WebAppConfig + oldAPIConfig := global.Env().SystemConfig.APIConfig + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig = oldWebConfig + global.Env().SystemConfig.APIConfig = oldAPIConfig + }) + + global.Env().SystemConfig.WebAppConfig.Enabled = true + global.Env().SystemConfig.WebAppConfig.EmbeddingAPI = false + global.Env().SystemConfig.WebAppConfig.WebsocketConfig.Enabled = false + global.Env().SystemConfig.APIConfig.Enabled = true + global.Env().SystemConfig.APIConfig.WebsocketConfig.Enabled = true + global.Env().SystemConfig.APIConfig.TLSConfig.TLSEnabled = true + global.Env().SystemConfig.APIConfig.NetworkConfig.Publish = "console-api.local:2900" + + got := resolveAgentReverseChannelEndpoint(req, "") + if got != "https://console-api.local:2900" { + t.Fatalf("expected api endpoint, got %q", got) + } +} + +func TestResolveAgentReverseChannelEndpointUsesAPIEndpointForReverseMTLSWhenWebCannotServeWS(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "https://console.local/instance/_get_install_script", nil) + req.TLS = &tls.ConnectionState{} + oldWebConfig := global.Env().SystemConfig.WebAppConfig + oldAPIConfig := global.Env().SystemConfig.APIConfig + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig = oldWebConfig + global.Env().SystemConfig.APIConfig = oldAPIConfig + }) + + global.Env().SystemConfig.WebAppConfig.Enabled = true + global.Env().SystemConfig.WebAppConfig.EmbeddingAPI = false + global.Env().SystemConfig.WebAppConfig.WebsocketConfig.Enabled = false + global.Env().SystemConfig.APIConfig.Enabled = true + global.Env().SystemConfig.APIConfig.WebsocketConfig.Enabled = true + global.Env().SystemConfig.APIConfig.TLSConfig.TLSEnabled = true + global.Env().SystemConfig.APIConfig.TLSConfig.TLSInsecureSkipVerify = false + global.Env().SystemConfig.APIConfig.NetworkConfig.Publish = "console-api.local:2900" + + got := resolveAgentReverseChannelEndpoint(req, "") + if got != "https://console-api.local:2900" { + t.Fatalf("expected api endpoint, got %q", got) + } +} + +func TestResolveAgentReverseChannelEndpointsPrefersConfiguredList(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "https://console.local:9000/instance/_get_install_script", nil) + got := resolveAgentReverseChannelEndpoints(req, []string{ + " https://console-api-1.local:9443 ", + "", + "https://console-api-2.local:9443", + }) + + expected := []string{ + "https://console-api-1.local:9443", + "https://console-api-2.local:9443", + } + if strings.Join(got, ",") != strings.Join(expected, ",") { + t.Fatalf("expected %v, got %v", expected, got) + } +} + +func TestBuildGatewayInstallCommand(t *testing.T) { + command := buildGatewayInstallCommand( + "https://console.local/instance/_get_gateway_install_script?token=abc", + "/srv/gateway", + false, + ) + + expected := `curl -ksSL "https://console.local/instance/_get_gateway_install_script?token=abc" |sudo bash -s -- -d "/srv/gateway"` + if command != expected { + t.Fatalf("expected %q, got %q", expected, command) + } +} + +func TestBuildGatewayInstallCommandSkipsSudoForNoServiceMode(t *testing.T) { + command := buildGatewayInstallCommand( + "https://console.local/instance/_get_gateway_install_script?token=abc", + "/srv/gateway", + true, + ) + + expected := `curl -ksSL "https://console.local/instance/_get_gateway_install_script?token=abc" |bash -s -- -d "/srv/gateway" --no-service` + if command != expected { + t.Fatalf("expected %q, got %q", expected, command) + } +} + +func TestResolveInstallDirUsesDefault(t *testing.T) { + got := resolveInstallDir("", defaultAgentInstallDir) + if got != defaultAgentInstallDir { + t.Fatalf("expected %q, got %q", defaultAgentInstallDir, got) + } +} + +func TestResolveInstallDirHonorsConfiguredValue(t *testing.T) { + got := resolveInstallDir("/srv/custom-agent", defaultAgentInstallDir) + if got != "/srv/custom-agent" { + t.Fatalf("expected %q, got %q", "/srv/custom-agent", got) + } +} + +func TestResolveGatewayInstallDirUsesServiceDefaults(t *testing.T) { + if got := resolveGatewayInstallDir(gatewayTypeRelay, ""); got != defaultRelayGatewayInstallDir { + t.Fatalf("expected %q, got %q", defaultRelayGatewayInstallDir, got) + } + if got := resolveGatewayInstallDir(gatewayTypeMigration, ""); got != defaultMigrationGatewayInstallDir { + t.Fatalf("expected %q, got %q", defaultMigrationGatewayInstallDir, got) + } + if got := resolveGatewayInstallDir(gatewayTypeRelay, "/srv/custom-gateway"); got != "/srv/custom-gateway" { + t.Fatalf("expected %q, got %q", "/srv/custom-gateway", got) + } +} + +func TestResolveGatewayServiceNameUsesServiceType(t *testing.T) { + if got := resolveGatewayServiceName(gatewayTypeRelay); got != defaultRelayGatewayServiceName { + t.Fatalf("expected %q, got %q", defaultRelayGatewayServiceName, got) + } + if got := resolveGatewayServiceName(gatewayTypeMigration); got != defaultMigrationGatewayServiceName { + t.Fatalf("expected %q, got %q", defaultMigrationGatewayServiceName, got) + } +} + +func TestNormalizeRelayRole(t *testing.T) { + if got := normalizeRelayRole("primary"); got != "primary" { + t.Fatalf("expected primary, got %q", got) + } + if got := normalizeRelayRole("SECONDARY"); got != "secondary" { + t.Fatalf("expected secondary, got %q", got) + } + if got := normalizeRelayRole("unknown"); got != "" { + t.Fatalf("expected empty for unknown relay role, got %q", got) + } +} + +func TestFormatBuildVersion(t *testing.T) { + if got := formatBuildVersion("1.2.3", "456"); got != "1.2.3-456" { + t.Fatalf("expected combined build version, got %q", got) + } + if got := formatBuildVersion("1.2.3", ""); got != "1.2.3" { + t.Fatalf("expected plain version when build number missing, got %q", got) + } + if got := formatBuildVersion("", "456"); got != "" { + t.Fatalf("expected empty version when version missing, got %q", got) + } +} + +func TestGetDefaultInstallVersionPrefersConfiguredValue(t *testing.T) { + if got := getDefaultInstallVersion("2.0.0-999"); got != "2.0.0-999" { + t.Fatalf("expected configured version to win, got %q", got) + } +} + +func TestGatewayInstallTemplateBootstrapsManagedConfig(t *testing.T) { + templatePath := filepath.Join("..", "..", "..", "config", "install_gateway.tpl") + content, err := os.ReadFile(templatePath) + if err != nil { + t.Fatalf("failed to read gateway install template: %v", err) + } + + rendered := strings.NewReplacer( + "{{base_url}}", "https://mirror.local/gateway/stable", + "{{version}}", "1.2.3-4567", + "{{console_endpoint}}", "https://console.local", + "{{console_domain}}", "console.local", + "{{client_crt}}", "CLIENT_CERT", + "{{client_key}}", "CLIENT_KEY", + "{{relay_server_crt}}", "RELAY_SERVER_CERT", + "{{relay_server_key}}", "RELAY_SERVER_KEY", + "{{ca_crt}}", "CA_CERT", + "{{port}}", "2900", + "{{access_token}}", "BOOTSTRAP_TOKEN", + "{{api_security_username}}", "managed_gateway", + "{{api_security_password}}", "LOCAL_API_PASSWORD", + "{{service_type}}", "relay", + "{{relay_role}}", "primary", + "{{service_name}}", "gateway-relay", + ).Replace(string(content)) + + expectedSnippets := []string{ + `[--no-service]`, + `--no-service Install without system service`, + `echo -e "${ca_crt}" > ${install_dir}/config/ca.crt`, + `echo -e "${relay_server_crt}" > ${install_dir}/config/relay_server.crt`, + `echo -e "${relay_server_key}" > ${install_dir}/config/relay_server.key`, + `cat < ${install_dir}/gateway.yml`, + `access_log_enabled: false`, + `elastic:`, + `skip_init_metadata_on_start: false`, + `metadata_refresh:`, + `health_check:`, + `availability_check:`, + `configs.auto_reload: true`, + `managed: true`, + `panic_on_config_error: false`, + `servers: \$[[env.CONFIG_MANAGER_SERVERS]]`, + `access_token: '\$[[keystore.CONFIGS_MANAGER_ACCESS_TOKEN]]'`, + `SECURITY_ENABLED: true`, + `websocket:`, + `base_path: /ws`, + `security:`, + `username: '\$[[keystore.API_SECURITY_USERNAME]]'`, + `password: '\$[[keystore.API_SECURITY_PASSWORD]]'`, + `cert_file: "config/client.crt"`, + `default_domain: "console.local"`, + `skip_insecure_verify: false`, + `service_type: "${service_type}"`, + `relay_role: "${relay_role}"`, + `access_token="BOOTSTRAP_TOKEN"`, + `keystore add "CONFIGS_MANAGER_ACCESS_TOKEN"`, + `keystore add "configs_manager_bootstrap_token"`, + `keystore add "API_SECURITY_USERNAME"`, + `keystore add "API_SECURITY_PASSWORD"`, + `service_name="gateway-relay"`, + `macos_svc=/Library/LaunchDaemons/${service_name}.plist`, + `linux_svc=/etc/systemd/system/${service_name}.service`, + `SERVICE_NAME="${service_name}" $gateway_svc -service install`, + `SERVICE_NAME="${service_name}" $gateway_svc -service start`, + `cleanup_install_dir_preserving_runtime_data`, + `preserving runtime data in ${install_dir}/data and ${install_dir}/log`, + `--no-service) no_service="true"; shift ;;`, + `if [[ "$no_service" != "true" ]]; then`, + `echo "[gateway] skip service install because --no-service is enabled"`, + `./$(basename "${gateway_svc}") -config gateway.yml`, + `Congratulations, gateway install success!`, + } + + for _, snippet := range expectedSnippets { + if !strings.Contains(rendered, snippet) { + t.Fatalf("expected rendered template to contain %q", snippet) + } + } +} + +func TestAgentInstallTemplateKeepsEmbeddedAPIDisabledWhenReverseChannelEnabled(t *testing.T) { + templatePath := filepath.Join("..", "..", "..", "config", "install_agent.tpl") + content, err := os.ReadFile(templatePath) + if err != nil { + t.Fatalf("failed to read agent install template: %v", err) + } + + rendered := strings.NewReplacer( + "{{base_url}}", "https://mirror.local/agent/stable", + "{{version}}", "1.2.3-4567", + "{{console_endpoint}}", "https://console.local", + "{{reverse_channel_endpoints}}", `["https://console-api.local:2900"]`, + "{{embedding_api}}", "false", + "{{websocket_enabled}}", "false", + "{{client_crt}}", "CLIENT_CERT", + "{{client_key}}", "CLIENT_KEY", + "{{ca_crt}}", "CA_CERT", + "{{port}}", "2900", + ).Replace(string(content)) + + expectedSnippets := []string{ + `cat < ${install_dir}/agent.yml`, + `access_log_enabled: false`, + `embedding_api: false`, + `websocket:`, + `enabled: false`, + `REVERSE_CHANNEL_ENDPOINTS: ["https://console-api.local:2900"]`, + `reverse_channel_endpoints: \$[[env.REVERSE_CHANNEL_ENDPOINTS]]`, + `cert_file: "config/client.crt"`, + `skip_insecure_verify: false`, + } + + for _, snippet := range expectedSnippets { + if !strings.Contains(rendered, snippet) { + t.Fatalf("expected rendered template to contain %q", snippet) + } + } +} + +func TestAgentInstallTemplateKeepsEmbeddedAPIDisabledWithoutReverseChannel(t *testing.T) { + templatePath := filepath.Join("..", "..", "..", "config", "install_agent.tpl") + content, err := os.ReadFile(templatePath) + if err != nil { + t.Fatalf("failed to read agent install template: %v", err) + } + + rendered := strings.NewReplacer( + "{{base_url}}", "https://mirror.local/agent/stable", + "{{version}}", "1.2.3-4567", + "{{console_endpoint}}", "https://console.local", + "{{console_domain}}", "console.local", + "{{reverse_channel_endpoints}}", `[]`, + "{{embedding_api}}", "false", + "{{websocket_enabled}}", "true", + "{{client_crt}}", "CLIENT_CERT", + "{{client_key}}", "CLIENT_KEY", + "{{ca_crt}}", "CA_CERT", + "{{port}}", "2900", + ).Replace(string(content)) + + expectedSnippets := []string{ + `access_log_enabled: false`, + `embedding_api: false`, + `enabled: true`, + `base_path: /ws`, + `REVERSE_CHANNEL_ENDPOINTS: []`, + `default_domain: "console.local"`, + } + + for _, snippet := range expectedSnippets { + if !strings.Contains(rendered, snippet) { + t.Fatalf("expected rendered template to contain %q", snippet) + } + } +} + +func TestGetEndpointHostname(t *testing.T) { + if got := getEndpointHostname("https://demo.infini.cloud:9000/console"); got != "demo.infini.cloud" { + t.Fatalf("expected demo.infini.cloud, got %q", got) + } +} + +func TestResolveConsoleTLSServerNameUsesEndpointHostnameWhenItIsDNS(t *testing.T) { + if got := resolveConsoleTLSServerName("https://console.local:9000"); got != "console.local" { + t.Fatalf("expected console.local, got %q", got) + } +} + +func TestResolveConsoleTLSServerNameFallsBackToTLSDefaultDomainForIPEndpoint(t *testing.T) { + oldWebConfig := global.Env().SystemConfig.WebAppConfig + oldAPIConfig := global.Env().SystemConfig.APIConfig + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig = oldWebConfig + global.Env().SystemConfig.APIConfig = oldAPIConfig + }) + + global.Env().SystemConfig.WebAppConfig.Enabled = true + global.Env().SystemConfig.WebAppConfig.NetworkConfig.Publish = "0.0.0.0:9000" + global.Env().SystemConfig.WebAppConfig.TLSConfig = frameworkconfig.TLSConfig{ + TLSEnabled: true, + DefaultDomain: "console.example.com", + } + global.Env().SystemConfig.APIConfig = frameworkconfig.APIConfig{} + + if got := resolveConsoleTLSServerName("https://192.168.3.8:9000"); got != "console.example.com" { + t.Fatalf("expected console.example.com, got %q", got) + } +} + +func TestResolveConsoleTLSServerNameFallsBackToCertificateDNSNameForIPEndpoint(t *testing.T) { + oldWebConfig := global.Env().SystemConfig.WebAppConfig + oldAPIConfig := global.Env().SystemConfig.APIConfig + t.Cleanup(func() { + global.Env().SystemConfig.WebAppConfig = oldWebConfig + global.Env().SystemConfig.APIConfig = oldAPIConfig + }) + + rootCert, rootKey, rootPEM := util.GetRootCert() + certPEM, _, err := util.GenerateServerCert(rootCert, rootKey, rootPEM, []string{"console.example.com"}) + if err != nil { + t.Fatalf("failed to generate server cert: %v", err) + } + + tempDir := t.TempDir() + certFile := filepath.Join(tempDir, "server.crt") + if err := os.WriteFile(certFile, certPEM, 0600); err != nil { + t.Fatalf("failed to write cert file: %v", err) + } + + global.Env().SystemConfig.WebAppConfig.Enabled = true + global.Env().SystemConfig.WebAppConfig.NetworkConfig.Publish = "0.0.0.0:9000" + global.Env().SystemConfig.WebAppConfig.TLSConfig = frameworkconfig.TLSConfig{ + TLSEnabled: true, + TLSCertFile: certFile, + } + global.Env().SystemConfig.APIConfig = frameworkconfig.APIConfig{} + + if got := resolveConsoleTLSServerName("https://192.168.3.8:9000"); got != "console.example.com" { + t.Fatalf("expected console.example.com, got %q", got) + } +} + +func TestLegacyAgentInstallTemplateSetsConfigTLSServerName(t *testing.T) { + templatePath := filepath.Join("..", "..", "..", "config", "install_legacy_agent.tpl") + content, err := os.ReadFile(templatePath) + if err != nil { + t.Fatalf("failed to read legacy agent install template: %v", err) + } + + rendered := strings.NewReplacer( + "{{base_url}}", "https://mirror.local/agent/stable", + "{{version}}", "1.30.3-4567", + "{{console_endpoint}}", "https://console.local", + "{{console_domain}}", "console.local", + "{{client_crt}}", "CLIENT_CERT", + "{{client_key}}", "CLIENT_KEY", + "{{ca_crt}}", "CA_CERT", + "{{port}}", "2900", + "{{token}}", "TOKEN", + "{{manager_token}}", "MANAGER_TOKEN", + "{{manager_token_key}}", "MANAGER_TOKEN_KEY", + "{{manager_token_id}}", "MANAGER_TOKEN_ID", + ).Replace(string(content)) + + expectedSnippets := []string{ + `servers: # config servers`, + ` default_domain: "console.local"`, + ` skip_insecure_verify: false`, + } + + for _, snippet := range expectedSnippets { + if !strings.Contains(rendered, snippet) { + t.Fatalf("expected rendered template to contain %q", snippet) + } + } +} + +func TestShouldUseLegacyInstallScriptTemplate(t *testing.T) { + tests := []struct { + version string + legacy bool + }{ + {version: "1.30.3", legacy: true}, + {version: "1.30.4", legacy: true}, + {version: "1.31.0", legacy: true}, + {version: "1.31.1", legacy: false}, + } + + for _, tc := range tests { + if got := shouldUseLegacyInstallScriptTemplate(tc.version); got != tc.legacy { + t.Fatalf("expected legacy=%v for version %s, got %v", tc.legacy, tc.version, got) + } + } +} + +func TestAgentInstallTemplateBootstrapsManagerAccessToken(t *testing.T) { + templatePath := filepath.Join("..", "..", "..", "config", "install_agent.tpl") + content, err := os.ReadFile(templatePath) + if err != nil { + t.Fatalf("failed to read agent install template: %v", err) + } + + rendered := strings.NewReplacer( + "{{base_url}}", "https://mirror.local/agent/stable", + "{{version}}", "1.2.3-4567", + "{{console_endpoint}}", "https://console.local", + "{{console_domain}}", "console.local", + "{{client_crt}}", "CLIENT_CERT", + "{{client_key}}", "CLIENT_KEY", + "{{ca_crt}}", "CA_CERT", + "{{port}}", "2900", + "{{token}}", "TOKEN", + "{{access_token}}", "BOOTSTRAP_TOKEN", + "{{manager_token}}", "MANAGER_TOKEN", + "{{manager_token_key}}", "MANAGER_TOKEN_KEY", + "{{manager_token_id}}", "MANAGER_TOKEN_ID", + "{{embedding_api}}", "false", + "{{websocket_enabled}}", "false", + "{{reverse_channel_endpoints}}", "[]", + "{{remote_config_servers}}", `["https://gateway-relay-1:2900","https://gateway-relay-2:2900"]`, + ).Replace(string(content)) + + expectedSnippets := []string{ + `access_token: '\$[[keystore.CONFIGS_MANAGER_ACCESS_TOKEN]]'`, + `access_token="BOOTSTRAP_TOKEN"`, + `keystore add "CONFIGS_MANAGER_ACCESS_TOKEN"`, + `cleanup_install_dir_preserving_runtime_data`, + `preserving runtime data in ${install_dir}/data and ${install_dir}/log`, + `access_log_enabled: false`, + `REMOTE_CONFIG_SERVERS: ${remote_config_servers}`, + `remote_config_servers='["https://gateway-relay-1:2900","https://gateway-relay-2:2900"]'`, + } + for _, snippet := range expectedSnippets { + if !strings.Contains(rendered, snippet) { + t.Fatalf("expected rendered template to contain %q", snippet) + } + } +} + +func TestBuildInstallCommandUsesScriptDefaults(t *testing.T) { + command := buildInstallCommand( + "https://console.local/instance/_get_install_script?token=abc", + "/srv/agent", + false, + ) + + expected := `curl -ksSL "https://console.local/instance/_get_install_script?token=abc" |sudo bash -s -- -t "/srv/agent"` + if command != expected { + t.Fatalf("expected %q, got %q", expected, command) + } +} + +func TestBuildInstallCommandSkipsSudoForNoServiceMode(t *testing.T) { + command := buildInstallCommand( + "https://console.local/instance/_get_install_script?token=abc", + "/srv/agent", + true, + ) + + expected := `curl -ksSL "https://console.local/instance/_get_install_script?token=abc" |bash -s -- -t "/srv/agent" --no-service` + if command != expected { + t.Fatalf("expected %q, got %q", expected, command) + } +} + +func TestBuildInstallScriptURLHonorsReverseChannelOption(t *testing.T) { + withoutReverse, err := buildInstallScriptURL("https://console.local", "abc", "1.2.3", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if strings.Contains(withoutReverse, "enable_reverse_channel=true") { + t.Fatalf("did not expect reverse channel flag in %q", withoutReverse) + } + + withReverse, err := buildInstallScriptURL("https://console.local", "abc", "1.2.3", true) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(withReverse, "enable_reverse_channel=true") { + t.Fatalf("expected reverse channel flag in %q", withReverse) + } + if strings.Contains(withReverse, "gateway_endpoint=") { + t.Fatalf("did not expect relay gateway endpoints in %q", withReverse) + } +} + +func TestRenderAgentReverseChannelEndpoints(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "https://console.local:9000/instance/_get_install_script", nil) + + if got := renderAgentReverseChannelEndpoints(req, nil, false); got != "[]" { + t.Fatalf("expected reverse channel disabled output, got %q", got) + } + + if got := renderAgentReverseChannelEndpoints(req, nil, true); got != `["${server}"]` { + t.Fatalf("expected default server placeholder, got %q", got) + } + + got := renderAgentReverseChannelEndpoints(req, []string{"https://console-api.local:9443"}, true) + if got != `["https://console-api.local:9443"]` { + t.Fatalf("expected configured endpoints, got %q", got) + } +} + +func TestNormalizeGatewayType(t *testing.T) { + if got := normalizeGatewayType("relay"); got != gatewayTypeRelay { + t.Fatalf("expected relay, got %q", got) + } + if got := normalizeGatewayType("migration"); got != gatewayTypeMigration { + t.Fatalf("expected migration, got %q", got) + } + if got := normalizeGatewayType("unknown"); got != gatewayTypeMigration { + t.Fatalf("expected migration fallback, got %q", got) + } +} + +func TestNormalizeManagedServerEndpoints(t *testing.T) { + got := normalizeManagedServerEndpoints([]string{ + " https://gw1.local:2900/ ", + "", + "https://gw1.local:2900", + "https://gw2.local:2900", + }) + expected := []string{"https://gw1.local:2900", "https://gw2.local:2900"} + if strings.Join(got, ",") != strings.Join(expected, ",") { + t.Fatalf("expected %v, got %v", expected, got) + } +} + +func TestResolvePackageDownloadURLUsesOfficialDefault(t *testing.T) { + got, err := resolvePackageDownloadURL("https://console.local/console", "", defaultAgentDownloadURL, agentPackageRelativePath) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + expected := defaultAgentDownloadURL + if got != expected { + t.Fatalf("expected %q, got %q", expected, got) + } +} + +func TestResolvePackageDownloadURLHonorsConfiguredOverride(t *testing.T) { + got, err := resolvePackageDownloadURL("https://console.local", "https://mirror.local/custom/agent", defaultAgentDownloadURL, agentPackageRelativePath) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + expected := "https://mirror.local/custom/agent" + if got != expected { + t.Fatalf("expected %q, got %q", expected, got) + } +} + +func TestResolvePackageDownloadURLUsesConsoleSelfHostedPathWhenPackagesExist(t *testing.T) { + executablePath, err := os.Executable() + if err != nil { + t.Fatalf("failed to get executable path: %v", err) + } + selfHostedDir := filepath.Join(filepath.Dir(executablePath), ".public", "gateway", "stable") + if err := os.MkdirAll(selfHostedDir, 0o755); err != nil { + t.Fatalf("failed to create self-hosted dir: %v", err) + } + testFile := filepath.Join(selfHostedDir, "gateway-1.2.3-linux-amd64.tar.gz") + if err := os.WriteFile(testFile, []byte("test"), 0o644); err != nil { + t.Fatalf("failed to write self-hosted package file: %v", err) + } + t.Cleanup(func() { + _ = os.Remove(testFile) + }) + + got, err := resolvePackageDownloadURL("https://console.local:9000", "", defaultGatewayDownloadURL, gatewayPackageRelativePath) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + expected := "https://console.local:9000/gateway/stable" + if got != expected { + t.Fatalf("expected %q, got %q", expected, got) + } +} + +func TestResolveGatewayDownloadURLUsesOfficialDefault(t *testing.T) { + got, err := resolveGatewayDownloadURL("https://console.local", "") + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if got != defaultGatewayDownloadURL { + t.Fatalf("expected %q, got %q", defaultGatewayDownloadURL, got) + } +} + +func TestValidateInstallTokenRejectsWrongProduct(t *testing.T) { + token := util.GetUUID() + expiredTokenCache.Put(token, &Token{ + CreatedAt: time.Now(), + UserID: "u1", + Product: installProductAgent, + }) + t.Cleanup(func() { + expiredTokenCache.Delete(token) + }) + + if _, err := validateInstallToken(token, installProductGateway); err == nil { + t.Fatal("expected product mismatch to be rejected") + } +} diff --git a/plugin/managed/server/token_exchange.go b/plugin/managed/server/token_exchange.go new file mode 100644 index 00000000..bf07aa66 --- /dev/null +++ b/plugin/managed/server/token_exchange.go @@ -0,0 +1,139 @@ +package server + +import ( + "fmt" + "net/http" + "strings" + + agent_common "infini.sh/console/modules/agent/common" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + configcommon "infini.sh/framework/modules/configs/common" +) + +const instanceTokenExchangeAPI = "/instance/_exchange_token" + +var getRuntimeInstanceByIDFunc = GetRuntimeInstanceByID +var validateManagedAgentRequestAuthFunc = validateManagedAgentRequestAuth +var upsertInstanceManagerCredentialFunc = upsertInstanceManagerCredential +var upsertInstanceAccessCredentialFunc = upsertInstanceAccessCredential +var findPendingManagerTokenByValueFunc = agent_common.FindPendingManagerTokenByValue +var saveManagedInstanceFunc = func(instance *model.Instance) error { + return orm.Save(&orm.Context{Refresh: orm.WaitForRefresh}, instance) +} +var generateManagedAPITokenFunc = func() (string, error) { + return agent_common.GenerateManagedTokenValue() +} + +type tokenExchangeRequest struct { + InstanceID string `json:"instance_id,omitempty"` + AgentAPIToken string `json:"agent_api_token,omitempty"` +} + +type tokenExchangeResponse struct { + ManagerAPIToken string `json:"manager_api_token,omitempty"` +} + +func validateTokenExchangeRequest(reqBody *tokenExchangeRequest) error { + if reqBody == nil { + return fmt.Errorf("empty request") + } + if strings.TrimSpace(reqBody.InstanceID) == "" { + return fmt.Errorf("empty instance id") + } + if strings.TrimSpace(reqBody.AgentAPIToken) == "" { + return fmt.Errorf("empty agent api token") + } + return nil +} + +func upsertInstanceManagerCredential(instance *model.Instance, tokenValue string) error { + if instance == nil { + return fmt.Errorf("instance is nil") + } + tokenValue = strings.TrimSpace(tokenValue) + if tokenValue == "" { + return fmt.Errorf("manager api token is empty") + } + if instance.ManagerCredentialID != "" { + previous, err := agent_common.GetTokenCredentialValue(instance.ManagerCredentialID) + if err != nil { + return err + } + agent_common.RememberPreviousToken(instance.ManagerCredentialID, previous) + return agent_common.UpdateTokenCredential( + instance.ManagerCredentialID, + agent_common.BuildManagerCredentialName(instance), + agent_common.BuildManagerCredentialTags(), + tokenValue, + ) + } + + credentialID, err := agent_common.SaveTokenCredential( + agent_common.BuildManagerCredentialName(instance), + agent_common.BuildManagerCredentialTags(), + tokenValue, + ) + if err != nil { + return err + } + instance.ManagerCredentialID = credentialID + return nil +} + +func (h APIHandler) exchangeInstanceToken(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + reqBody := &tokenExchangeRequest{} + if err := h.DecodeJSON(req, reqBody); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if err := validateTokenExchangeRequest(reqBody); err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + _, instance, err := getRuntimeInstanceByIDFunc(reqBody.InstanceID) + if err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + if instance == nil { + h.WriteError(w, "instance not found", http.StatusBadRequest) + return + } + if !configcommon.SupportsManagedAccessToken(instance.Application.Name) { + h.WriteError(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) + return + } + if err := validateManagedAgentRequestAuthFunc(req, instance); err != nil { + if agent_common.IsManagerAuthFailure(err) { + h.WriteError(w, err.Error(), http.StatusUnauthorized) + } else { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + } + return + } + + managerAPIToken, err := generateManagedAPITokenFunc() + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if err := upsertInstanceManagerCredentialFunc(instance, managerAPIToken); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if err := upsertInstanceAccessCredentialFunc(instance, &configcommon.RegisterToken{Value: reqBody.AgentAPIToken}); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if err := saveManagedInstanceFunc(instance); err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteJSON(w, tokenExchangeResponse{ + ManagerAPIToken: managerAPIToken, + }, http.StatusOK) +} diff --git a/plugin/managed/server/token_exchange_test.go b/plugin/managed/server/token_exchange_test.go new file mode 100644 index 00000000..84d1eb49 --- /dev/null +++ b/plugin/managed/server/token_exchange_test.go @@ -0,0 +1,294 @@ +package server + +import ( + "bytes" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + agent_common "infini.sh/console/modules/agent/common" + "infini.sh/framework/core/env" + "infini.sh/framework/core/model" + configcommon "infini.sh/framework/modules/configs/common" +) + +func TestValidateTokenExchangeRequest(t *testing.T) { + tests := []struct { + name string + req *tokenExchangeRequest + wantErr string + }{ + {name: "nil request", req: nil, wantErr: "empty request"}, + {name: "missing instance", req: &tokenExchangeRequest{AgentAPIToken: "agent-token"}, wantErr: "empty instance id"}, + {name: "missing agent token", req: &tokenExchangeRequest{InstanceID: "agent-1"}, wantErr: "empty agent api token"}, + {name: "valid request", req: &tokenExchangeRequest{InstanceID: "agent-1", AgentAPIToken: "agent-token"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateTokenExchangeRequest(tt.req) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + return + } + if err == nil || err.Error() != tt.wantErr { + t.Fatalf("expected %q, got %v", tt.wantErr, err) + } + }) + } +} + +func TestExchangeInstanceToken(t *testing.T) { + originalGetRuntimeInstanceByIDFunc := getRuntimeInstanceByIDFunc + originalValidateManagedAgentRequestAuthFunc := validateManagedAgentRequestAuthFunc + originalUpsertInstanceManagerCredentialFunc := upsertInstanceManagerCredentialFunc + originalUpsertInstanceAccessCredentialFunc := upsertInstanceAccessCredentialFunc + originalSaveManagedInstanceFunc := saveManagedInstanceFunc + originalGenerateManagedAPITokenFunc := generateManagedAPITokenFunc + t.Cleanup(func() { + getRuntimeInstanceByIDFunc = originalGetRuntimeInstanceByIDFunc + validateManagedAgentRequestAuthFunc = originalValidateManagedAgentRequestAuthFunc + upsertInstanceManagerCredentialFunc = originalUpsertInstanceManagerCredentialFunc + upsertInstanceAccessCredentialFunc = originalUpsertInstanceAccessCredentialFunc + saveManagedInstanceFunc = originalSaveManagedInstanceFunc + generateManagedAPITokenFunc = originalGenerateManagedAPITokenFunc + }) + + instance := &model.Instance{ + Application: env.Application{Name: "agent"}, + } + instance.ID = "agent-1" + + getRuntimeInstanceByIDFunc = func(instanceID string) (bool, *model.Instance, error) { + if instanceID != instance.ID { + t.Fatalf("unexpected instance id: %s", instanceID) + } + return true, instance, nil + } + validateManagedAgentRequestAuthFunc = func(req *http.Request, obj *model.Instance) error { + if obj != instance { + t.Fatalf("unexpected instance: %#v", obj) + } + if token := agent_common.ExtractManagerToken(req); token != "bootstrap-token" { + t.Fatalf("unexpected manager token: %q", token) + } + return nil + } + + managerCredentialUpdated := "" + accessCredentialUpdated := "" + upsertInstanceManagerCredentialFunc = func(obj *model.Instance, tokenValue string) error { + managerCredentialUpdated = tokenValue + obj.ManagerCredentialID = "manager-cred" + return nil + } + upsertInstanceAccessCredentialFunc = func(obj *model.Instance, registerToken *configcommon.RegisterToken) error { + if registerToken == nil { + t.Fatal("expected register token") + } + accessCredentialUpdated = registerToken.Value + obj.AccessCredentialID = "access-cred" + return nil + } + saved := false + saveManagedInstanceFunc = func(obj *model.Instance) error { + saved = true + if obj.ManagerCredentialID != "manager-cred" || obj.AccessCredentialID != "access-cred" { + t.Fatalf("unexpected credential ids on save: %#v", obj) + } + return nil + } + generateManagedAPITokenFunc = func() (string, error) { + return "manager-api-token", nil + } + + req := httptest.NewRequest(http.MethodPost, instanceTokenExchangeAPI, bytes.NewBufferString(`{"instance_id":"agent-1","agent_api_token":"agent-api-token"}`)) + req.Header.Set(model.API_TOKEN, "bootstrap-token") + resp := httptest.NewRecorder() + + APIHandler{}.exchangeInstanceToken(resp, req, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d, body=%s", resp.Code, resp.Body.String()) + } + if !strings.Contains(resp.Body.String(), `"manager_api_token":"manager-api-token"`) { + t.Fatalf("unexpected response body: %s", resp.Body.String()) + } + if managerCredentialUpdated != "manager-api-token" { + t.Fatalf("unexpected manager token persisted: %q", managerCredentialUpdated) + } + if accessCredentialUpdated != "agent-api-token" { + t.Fatalf("unexpected agent access token persisted: %q", accessCredentialUpdated) + } + if !saved { + t.Fatal("expected instance to be saved") + } +} + +func TestExchangeInstanceTokenSupportsGateway(t *testing.T) { + originalGetRuntimeInstanceByIDFunc := getRuntimeInstanceByIDFunc + originalValidateManagedAgentRequestAuthFunc := validateManagedAgentRequestAuthFunc + originalUpsertInstanceManagerCredentialFunc := upsertInstanceManagerCredentialFunc + originalUpsertInstanceAccessCredentialFunc := upsertInstanceAccessCredentialFunc + originalSaveManagedInstanceFunc := saveManagedInstanceFunc + originalGenerateManagedAPITokenFunc := generateManagedAPITokenFunc + t.Cleanup(func() { + getRuntimeInstanceByIDFunc = originalGetRuntimeInstanceByIDFunc + validateManagedAgentRequestAuthFunc = originalValidateManagedAgentRequestAuthFunc + upsertInstanceManagerCredentialFunc = originalUpsertInstanceManagerCredentialFunc + upsertInstanceAccessCredentialFunc = originalUpsertInstanceAccessCredentialFunc + saveManagedInstanceFunc = originalSaveManagedInstanceFunc + generateManagedAPITokenFunc = originalGenerateManagedAPITokenFunc + }) + + instance := &model.Instance{ + Application: env.Application{Name: "gateway"}, + } + instance.ID = "gateway-1" + + getRuntimeInstanceByIDFunc = func(instanceID string) (bool, *model.Instance, error) { + return true, instance, nil + } + validateManagedAgentRequestAuthFunc = func(req *http.Request, obj *model.Instance) error { + return nil + } + upsertInstanceManagerCredentialFunc = func(obj *model.Instance, tokenValue string) error { + obj.ManagerCredentialID = "manager-cred" + return nil + } + upsertInstanceAccessCredentialFunc = func(obj *model.Instance, registerToken *configcommon.RegisterToken) error { + obj.AccessCredentialID = "access-cred" + return nil + } + saveManagedInstanceFunc = func(obj *model.Instance) error { + return nil + } + generateManagedAPITokenFunc = func() (string, error) { + return "manager-api-token", nil + } + + req := httptest.NewRequest(http.MethodPost, instanceTokenExchangeAPI, bytes.NewBufferString(`{"instance_id":"gateway-1","agent_api_token":"gateway-api-token"}`)) + req.Header.Set(model.API_TOKEN, "bootstrap-token") + resp := httptest.NewRecorder() + + APIHandler{}.exchangeInstanceToken(resp, req, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d, body=%s", resp.Code, resp.Body.String()) + } +} + +func TestExchangeInstanceTokenRejectsInvalidManagerAuth(t *testing.T) { + originalGetRuntimeInstanceByIDFunc := getRuntimeInstanceByIDFunc + originalValidateManagedAgentRequestAuthFunc := validateManagedAgentRequestAuthFunc + t.Cleanup(func() { + getRuntimeInstanceByIDFunc = originalGetRuntimeInstanceByIDFunc + validateManagedAgentRequestAuthFunc = originalValidateManagedAgentRequestAuthFunc + }) + + instance := &model.Instance{ + Application: env.Application{Name: "agent"}, + } + instance.ID = "agent-1" + getRuntimeInstanceByIDFunc = func(instanceID string) (bool, *model.Instance, error) { + return true, instance, nil + } + validateManagedAgentRequestAuthFunc = func(req *http.Request, obj *model.Instance) error { + return agent_common.ErrInvalidManagerToken + } + + req := httptest.NewRequest(http.MethodPost, instanceTokenExchangeAPI, bytes.NewBufferString(`{"instance_id":"agent-1","agent_api_token":"agent-api-token"}`)) + resp := httptest.NewRecorder() + + APIHandler{}.exchangeInstanceToken(resp, req, nil) + + if resp.Code != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d, body=%s", resp.Code, resp.Body.String()) + } +} + +func TestExchangeInstanceTokenHandlesLookupError(t *testing.T) { + originalGetRuntimeInstanceByIDFunc := getRuntimeInstanceByIDFunc + t.Cleanup(func() { + getRuntimeInstanceByIDFunc = originalGetRuntimeInstanceByIDFunc + }) + + getRuntimeInstanceByIDFunc = func(instanceID string) (bool, *model.Instance, error) { + return false, nil, errors.New("lookup failed") + } + + req := httptest.NewRequest(http.MethodPost, instanceTokenExchangeAPI, bytes.NewBufferString(`{"instance_id":"agent-1","agent_api_token":"agent-api-token"}`)) + resp := httptest.NewRecorder() + + APIHandler{}.exchangeInstanceToken(resp, req, nil) + + if resp.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d, body=%s", resp.Code, resp.Body.String()) + } +} + +func TestMigrateLegacyManagedRegisterAuthRotatesExistingManagerCredential(t *testing.T) { + originalUpsertInstanceManagerCredentialFunc := upsertInstanceManagerCredentialFunc + originalUpsertInstanceAccessCredentialFunc := upsertInstanceAccessCredentialFunc + originalFindPendingManagerTokenByValueFunc := findPendingManagerTokenByValueFunc + t.Cleanup(func() { + upsertInstanceManagerCredentialFunc = originalUpsertInstanceManagerCredentialFunc + upsertInstanceAccessCredentialFunc = originalUpsertInstanceAccessCredentialFunc + findPendingManagerTokenByValueFunc = originalFindPendingManagerTokenByValueFunc + }) + + current := &model.Instance{ + Application: env.Application{ + Name: "agent", + Version: env.Version{VersionNumber: "1.30.3"}, + }, + } + existing := &model.Instance{ + ManagerCredentialID: "manager-cred", + Application: env.Application{ + Name: "agent", + Version: env.Version{VersionNumber: "1.31.0"}, + }, + } + req := httptest.NewRequest(http.MethodPost, "/instance/_register", nil) + req.Header.Set(model.API_TOKEN, "new-bootstrap-token") + + findPendingManagerTokenByValueFunc = func(tokenValue string) (*agent_common.PendingRegistrationToken, error) { + if tokenValue != "new-bootstrap-token" { + t.Fatalf("unexpected pending manager token lookup: %q", tokenValue) + } + return &agent_common.PendingRegistrationToken{}, nil + } + + managerToken := "" + accessToken := "" + upsertInstanceManagerCredentialFunc = func(instance *model.Instance, tokenValue string) error { + managerToken = tokenValue + return nil + } + upsertInstanceAccessCredentialFunc = func(instance *model.Instance, registerToken *configcommon.RegisterToken) error { + if registerToken == nil { + t.Fatal("expected access token") + } + accessToken = registerToken.Value + return nil + } + + migrated, err := migrateLegacyManagedRegisterAuth(req, current, existing, &configcommon.RegisterToken{Value: "agent-access-token"}) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if !migrated { + t.Fatal("expected legacy registration auth to migrate") + } + if managerToken != "new-bootstrap-token" { + t.Fatalf("expected manager token to rotate, got %q", managerToken) + } + if accessToken != "agent-access-token" { + t.Fatalf("expected access token to update, got %q", accessToken) + } +} diff --git a/plugin/setup/recovery.go b/plugin/setup/recovery.go new file mode 100644 index 00000000..381640ca --- /dev/null +++ b/plugin/setup/recovery.go @@ -0,0 +1,446 @@ +package task + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "path" + "strconv" + "strings" + "syscall" + + "golang.org/x/crypto/ssh/terminal" + "infini.sh/framework/core/credential" + coreelastic "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" + "infini.sh/framework/core/keystore" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/lib/go-ucfg" + elasticcommon "infini.sh/framework/modules/elastic/common" +) + +const systemClusterPassKey = "SYSTEM_CLUSTER_PASS" + +type recoveryCommandDeps struct { + stdin io.Reader + stdout io.Writer + readPassword func() ([]byte, error) + writeSecret func(key string, value []byte) error + systemConfigPath string + currentUsername func() (string, error) + validateAccess func(username, password string) error + syncCredential func(username, password string) error +} + +type systemClusterRecoveryConfig struct { + ClusterID string + Endpoints []string + Username string + Version string + Distribution string + IndexPrefix string +} + +func newRecoveryCommandDeps() recoveryCommandDeps { + systemConfigPath := path.Join(global.Env().GetConfigDir(), "system_config.yml") + return recoveryCommandDeps{ + stdin: os.Stdin, + stdout: os.Stdout, + readPassword: func() ([]byte, error) { + return terminal.ReadPassword(int(syscall.Stdin)) + }, + writeSecret: keystore.SetValue, + systemConfigPath: systemConfigPath, + currentUsername: func() (string, error) { + cfg, err := loadSystemClusterRecoveryConfig(systemConfigPath) + if err != nil { + return "", err + } + return cfg.Username, nil + }, + validateAccess: func(username, password string) error { + cfg, err := loadSystemClusterRecoveryConfig(systemConfigPath) + if err != nil { + return err + } + return validateSystemClusterRecoveryAccess(cfg, username, password) + }, + syncCredential: func(username, password string) error { + cfg, err := loadSystemClusterRecoveryConfig(systemConfigPath) + if err != nil { + return err + } + return syncSystemClusterRecoveryCredential(cfg, username, password) + }, + } +} + +func RunRecoveryCmd(args []string) error { + return runRecoveryCmd(args, newRecoveryCommandDeps()) +} + +func runRecoveryCmd(args []string, deps recoveryCommandDeps) error { + if deps.stdout == nil { + deps.stdout = os.Stdout + } + if deps.stdin == nil { + deps.stdin = os.Stdin + } + if deps.readPassword == nil { + deps.readPassword = func() ([]byte, error) { + return terminal.ReadPassword(int(syscall.Stdin)) + } + } + if deps.writeSecret == nil { + deps.writeSecret = keystore.SetValue + } + if deps.currentUsername == nil { + deps.currentUsername = func() (string, error) { + return "", fmt.Errorf("system cluster username resolver is not configured") + } + } + if deps.validateAccess == nil { + deps.validateAccess = func(username, password string) error { + return nil + } + } + if deps.syncCredential == nil { + deps.syncCredential = func(username, password string) error { + return nil + } + } + + recoveryFS := flag.NewFlagSet("recovery", flag.ContinueOnError) + recoveryFS.SetOutput(deps.stdout) + var ( + password = recoveryFS.String("pass", "", "System cluster password") + username = recoveryFS.String("user", "", "System cluster username") + stdin = recoveryFS.Bool("stdin", false, "Use stdin as the password source") + ) + recoveryFS.Usage = func() { + fmt.Fprintln(deps.stdout, "usage : recovery []") + fmt.Fprintln(deps.stdout, "Update local system cluster credentials so Console can recover from external password rotation.") + fmt.Fprintln(deps.stdout) + fmt.Fprintln(deps.stdout, "Options:") + recoveryFS.PrintDefaults() + fmt.Fprintln(deps.stdout) + fmt.Fprintln(deps.stdout, "Examples:") + fmt.Fprintln(deps.stdout, " ./console recovery -pass new-password") + _, _ = io.WriteString(deps.stdout, " printf '%s' 'new-password' | ./console recovery -stdin\n") + fmt.Fprintln(deps.stdout, " ./console recovery -user admin -pass new-password") + } + + if err := recoveryFS.Parse(args); err != nil { + return err + } + if len(recoveryFS.Args()) > 0 { + return fmt.Errorf("unexpected arguments: %s", strings.Join(recoveryFS.Args(), " ")) + } + + resolvedPassword, err := resolveRecoveryPassword(strings.TrimSpace(*password), *stdin, deps) + if err != nil { + return err + } + if len(resolvedPassword) == 0 { + return fmt.Errorf("system cluster password is required") + } + resolvedUsername := strings.TrimSpace(*username) + if resolvedUsername == "" { + currentUsername, err := deps.currentUsername() + if err != nil { + return err + } + resolvedUsername = strings.TrimSpace(currentUsername) + } + if resolvedUsername == "" { + return fmt.Errorf("system cluster username is required") + } + + if err := deps.validateAccess(resolvedUsername, string(resolvedPassword)); err != nil { + return err + } + + if err := deps.writeSecret(systemClusterPassKey, resolvedPassword); err != nil { + return fmt.Errorf("update system cluster password: %w", err) + } + + trimmedUser := strings.TrimSpace(*username) + if trimmedUser != "" { + if err := updateSystemClusterUsernameFile(deps.systemConfigPath, trimmedUser); err != nil { + return err + } + } + if err := deps.syncCredential(resolvedUsername, string(resolvedPassword)); err != nil { + return err + } + + fmt.Fprintln(deps.stdout, "system cluster recovery updated successfully") + if trimmedUser != "" { + fmt.Fprintf(deps.stdout, "system cluster username updated in %s\n", deps.systemConfigPath) + } + fmt.Fprintln(deps.stdout, "restart console to apply the change") + return nil +} + +func resolveRecoveryPassword(explicitPassword string, useStdin bool, deps recoveryCommandDeps) ([]byte, error) { + if explicitPassword != "" { + return []byte(explicitPassword), nil + } + + if useStdin { + value, err := io.ReadAll(deps.stdin) + if err != nil { + return nil, fmt.Errorf("could not read password from stdin: %w", err) + } + return bytes.TrimRight(value, "\r\n"), nil + } + + fmt.Fprint(deps.stdout, "Enter system cluster password: ") + value, err := deps.readPassword() + fmt.Fprintln(deps.stdout) + if err != nil { + return nil, fmt.Errorf("could not read password from terminal: %w", err) + } + return bytes.TrimSpace(value), nil +} + +func updateSystemClusterUsernameFile(filePath, username string) error { + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("read system config [%s]: %w", filePath, err) + } + + updatedContent, found, err := updateSystemClusterUsernameContent(string(content), username) + if err != nil { + return err + } + if !found { + return fmt.Errorf("CLUSTER_USER not found in system config [%s]", filePath) + } + + if _, err := util.FilePutContent(filePath, updatedContent); err != nil { + return fmt.Errorf("write system config [%s]: %w", filePath, err) + } + return nil +} + +func updateSystemClusterUsernameContent(content, username string) (string, bool, error) { + username = strings.TrimSpace(username) + if username == "" { + return "", false, fmt.Errorf("system cluster username is required") + } + + lines := strings.Split(content, "\n") + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "CLUSTER_USER:") { + continue + } + + idx := strings.Index(line, "CLUSTER_USER:") + if idx < 0 { + continue + } + + lines[i] = line[:idx] + "CLUSTER_USER: " + strconv.Quote(username) + return strings.Join(lines, "\n"), true, nil + } + + return content, false, nil +} + +func loadSystemClusterRecoveryConfig(filePath string) (*systemClusterRecoveryConfig, error) { + content, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("read system config [%s]: %w", filePath, err) + } + + cfg := &systemClusterRecoveryConfig{ + ClusterID: GlobalSystemElasticsearchID, + IndexPrefix: ".infini_", + } + for _, line := range strings.Split(string(content), "\n") { + trimmed := strings.TrimSpace(line) + switch { + case strings.HasPrefix(trimmed, "CLUSTER_ID:"): + cfg.ClusterID = parseTemplateVariableValue(trimmed) + case strings.HasPrefix(trimmed, "CLUSTER_ENDPOINT:"): + raw := parseTemplateVariableValue(trimmed) + + var endpoints []string + if err := json.Unmarshal([]byte(raw), &endpoints); err != nil { + return nil, fmt.Errorf("parse CLUSTER_ENDPOINT failed: %w", err) + } + + cfg.Endpoints = endpoints + case strings.HasPrefix(trimmed, "CLUSTER_USER:"): + cfg.Username = parseTemplateVariableValue(trimmed) + case strings.HasPrefix(trimmed, "CLUSTER_VER:"): + cfg.Version = parseTemplateVariableValue(trimmed) + case strings.HasPrefix(trimmed, "CLUSTER_DISTRIBUTION:"): + cfg.Distribution = parseTemplateVariableValue(trimmed) + case strings.HasPrefix(trimmed, "INDEX_PREFIX:"): + cfg.IndexPrefix = parseTemplateVariableValue(trimmed) + } + } + + if len(cfg.Endpoints) < 1 { + return nil, fmt.Errorf("CLUSTER_ENDPOINT not found in system config [%s]", filePath) + } + return cfg, nil +} + +func parseTemplateVariableValue(line string) string { + idx := strings.Index(line, ":") + if idx < 0 { + return "" + } + value := strings.TrimSpace(line[idx+1:]) + if unquoted, err := strconv.Unquote(value); err == nil { + return strings.TrimSpace(unquoted) + } + return strings.Trim(value, "\"'") +} + +func validateSystemClusterRecoveryAccess(cfg *systemClusterRecoveryConfig, username, password string) error { + client, cleanup, err := newSystemClusterRecoveryClient(cfg, username, password) + if err != nil { + return err + } + defer cleanup() + + if _, err := client.ClusterHealth(context.Background()); err != nil { + return fmt.Errorf("validate system cluster access: %w", err) + } + return nil +} + +func syncSystemClusterRecoveryCredential(cfg *systemClusterRecoveryConfig, username, password string) error { + client, cleanup, err := newSystemClusterRecoveryClient(cfg, username, password) + if err != nil { + return err + } + defer cleanup() + + clusterConfig, err := loadSystemClusterDocument(client, cfg.IndexPrefix, cfg.ClusterID) + if err != nil { + return err + } + + credentialDoc, err := buildUpdatedSystemClusterCredential(client, cfg.IndexPrefix, clusterConfig, username, password) + if err != nil { + return err + } + if credentialDoc != nil { + if _, err := client.Index(cfg.IndexPrefix+"credential", "", credentialDoc.ID, credentialDoc, "wait_for"); err != nil { + return fmt.Errorf("update system cluster credential: %w", err) + } + clusterConfig.CredentialID = credentialDoc.ID + clusterConfig.BasicAuth = nil + } + if _, err := client.Index(cfg.IndexPrefix+"cluster", "", clusterConfig.ID, clusterConfig, "wait_for"); err != nil { + return fmt.Errorf("update system cluster config: %w", err) + } + return nil +} + +func newSystemClusterRecoveryClient(cfg *systemClusterRecoveryConfig, username, password string) (coreelastic.API, func(), error) { + if cfg == nil { + return nil, nil, fmt.Errorf("system cluster config is required") + } + + tempID := cfg.ClusterID + "-recovery" + tempConf := coreelastic.ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: tempID}, + Name: cfg.ClusterID, + Enabled: true, + Endpoints: cfg.Endpoints, + Version: cfg.Version, + Distribution: cfg.Distribution, + BasicAuth: &model.BasicAuth{ + Username: username, + Password: ucfg.SecretString(password), + }, + } + client, err := elasticcommon.InitElasticInstanceWithoutMetadata(tempConf) + if err != nil { + return nil, nil, err + } + return client, func() { + coreelastic.RemoveInstance(tempID) + }, nil +} + +func loadSystemClusterDocument(client coreelastic.API, indexPrefix, clusterID string) (*coreelastic.ElasticsearchConfig, error) { + clusterConfig := &coreelastic.ElasticsearchConfig{} + if err := loadSystemClusterSource(client, indexPrefix+"cluster", clusterID, clusterConfig); err != nil { + return nil, err + } + clusterConfig.ID = clusterID + return clusterConfig, nil +} + +func buildUpdatedSystemClusterCredential(client coreelastic.API, indexPrefix string, clusterConfig *coreelastic.ElasticsearchConfig, username, password string) (*credential.Credential, error) { + if clusterConfig == nil { + return nil, fmt.Errorf("system cluster config is required") + } + + cred := &credential.Credential{} + if strings.TrimSpace(clusterConfig.CredentialID) != "" { + if err := loadSystemClusterSource(client, indexPrefix+"credential", clusterConfig.CredentialID, cred); err != nil { + return nil, err + } + cred.ID = clusterConfig.CredentialID + } else { + cred.ID = util.GetUUID() + cred.Name = fmt.Sprintf("%s (Platform)", clusterConfig.Name) + if cred.Name == " (Platform)" { + cred.Name = "INFINI_SYSTEM (Platform)" + } + } + + cred.Type = credential.BasicAuth + cred.Tags = []string{"ES"} + cred.Invalid = false + cred.Payload = map[string]interface{}{ + credential.BasicAuth: map[string]interface{}{ + "username": username, + "password": password, + }, + } + if err := cred.Encode(); err != nil { + return nil, fmt.Errorf("encode system cluster credential: %w", err) + } + return cred, nil +} + +func loadSystemClusterSource(client coreelastic.API, indexName, documentID string, target interface{}) error { + response, err := client.Get(indexName, "", documentID) + if err != nil { + return err + } + if response == nil || response.RawResult == nil { + return fmt.Errorf("document [%s] not found in index [%s]", documentID, indexName) + } + if response.RawResult.StatusCode == http.StatusNotFound { + return fmt.Errorf("document [%s] not found in index [%s]", documentID, indexName) + } + source, err := response.GetBytesByJsonPath("_source") + if err != nil { + return err + } + if len(source) == 0 { + return fmt.Errorf("document [%s] not found in index [%s]", documentID, indexName) + } + if err := util.FromJSONBytes(source, target); err != nil { + return err + } + return nil +} diff --git a/plugin/setup/recovery_test.go b/plugin/setup/recovery_test.go new file mode 100644 index 00000000..3aa0c7b3 --- /dev/null +++ b/plugin/setup/recovery_test.go @@ -0,0 +1,144 @@ +package task + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestUpdateSystemClusterUsernameContent(t *testing.T) { + original := "configs.template:\n - name: \"system\"\n variable:\n CLUSTER_USER: \"old-admin\"\n CLUSTER_VER: \"8.0.0\"\n" + + updated, found, err := updateSystemClusterUsernameContent(original, "new-admin") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !found { + t.Fatal("expected CLUSTER_USER to be found") + } + if !strings.Contains(updated, `CLUSTER_USER: "new-admin"`) { + t.Fatalf("expected updated username, got:\n%s", updated) + } + if !strings.Contains(updated, `CLUSTER_VER: "8.0.0"`) { + t.Fatalf("expected unrelated content to stay intact, got:\n%s", updated) + } +} + +func TestRunRecoveryCmdUpdatesPasswordAndUser(t *testing.T) { + configDir := t.TempDir() + systemConfigPath := filepath.Join(configDir, "system_config.yml") + if err := os.WriteFile(systemConfigPath, []byte("configs.template:\n - name: \"system\"\n variable:\n CLUSTER_USER: \"old-admin\"\n"), 0644); err != nil { + t.Fatalf("write system config: %v", err) + } + + var ( + storedKey string + storedValue []byte + stdout bytes.Buffer + validatedUser string + validatedPass string + syncedUser string + syncedPass string + currentUserCalls int + ) + err := runRecoveryCmd([]string{"-pass", "new-password", "-user", "new-admin"}, recoveryCommandDeps{ + stdin: strings.NewReader(""), + stdout: &stdout, + readPassword: func() ([]byte, error) { + t.Fatal("unexpected password prompt") + return nil, nil + }, + writeSecret: func(key string, value []byte) error { + storedKey = key + storedValue = append([]byte(nil), value...) + return nil + }, + systemConfigPath: systemConfigPath, + currentUsername: func() (string, error) { + currentUserCalls++ + return "old-admin", nil + }, + validateAccess: func(username, password string) error { + validatedUser = username + validatedPass = password + return nil + }, + syncCredential: func(username, password string) error { + syncedUser = username + syncedPass = password + return nil + }, + }) + if err != nil { + t.Fatalf("run recovery command: %v", err) + } + if currentUserCalls != 0 { + t.Fatalf("expected explicit user to skip current username lookup, got %d calls", currentUserCalls) + } + if validatedUser != "new-admin" || validatedPass != "new-password" { + t.Fatalf("unexpected validated credential: %s / %s", validatedUser, validatedPass) + } + if syncedUser != "new-admin" || syncedPass != "new-password" { + t.Fatalf("unexpected synced credential: %s / %s", syncedUser, syncedPass) + } + + if storedKey != systemClusterPassKey { + t.Fatalf("expected secret key %s, got %s", systemClusterPassKey, storedKey) + } + if string(storedValue) != "new-password" { + t.Fatalf("expected password to be updated, got %q", string(storedValue)) + } + + content, err := os.ReadFile(systemConfigPath) + if err != nil { + t.Fatalf("read system config: %v", err) + } + if !strings.Contains(string(content), `CLUSTER_USER: "new-admin"`) { + t.Fatalf("expected username to be updated, got:\n%s", string(content)) + } + if !strings.Contains(stdout.String(), "restart console to apply the change") { + t.Fatalf("expected restart hint in output, got: %s", stdout.String()) + } +} + +func TestLoadSystemClusterRecoveryConfig(t *testing.T) { + systemConfigPath := filepath.Join(t.TempDir(), "system_config.yml") + content := `configs.template: + - name: "system" + variable: + CLUSTER_ID: infini_default_system_cluster + CLUSTER_ENDPOINT: "https://127.0.0.1:9200" + CLUSTER_USER: "admin" + CLUSTER_VER: "8.18.0" + CLUSTER_DISTRIBUTION: "easysearch" + INDEX_PREFIX: ".demo_" +` + if err := os.WriteFile(systemConfigPath, []byte(content), 0644); err != nil { + t.Fatalf("write system config: %v", err) + } + + cfg, err := loadSystemClusterRecoveryConfig(systemConfigPath) + if err != nil { + t.Fatalf("load system config: %v", err) + } + if cfg.ClusterID != "infini_default_system_cluster" { + t.Fatalf("unexpected cluster id: %s", cfg.ClusterID) + } + if cfg.Endpoint != "https://127.0.0.1:9200" { + t.Fatalf("unexpected endpoint: %s", cfg.Endpoint) + } + if cfg.Username != "admin" { + t.Fatalf("unexpected username: %s", cfg.Username) + } + if cfg.Version != "8.18.0" { + t.Fatalf("unexpected version: %s", cfg.Version) + } + if cfg.Distribution != "easysearch" { + t.Fatalf("unexpected distribution: %s", cfg.Distribution) + } + if cfg.IndexPrefix != ".demo_" { + t.Fatalf("unexpected index prefix: %s", cfg.IndexPrefix) + } +} diff --git a/plugin/setup/setup.go b/plugin/setup/setup.go index cfc60b6d..fdec9c83 100644 --- a/plugin/setup/setup.go +++ b/plugin/setup/setup.go @@ -35,16 +35,21 @@ import ( "net/http" uri2 "net/url" "path" + "regexp" "runtime" + "strconv" "strings" + "sync" + "sync/atomic" "time" + console_common "infini.sh/console/common" + core2 "infini.sh/console/core" "infini.sh/console/core/security" "infini.sh/framework/lib/go-ucfg" elastic2 "infini.sh/framework/modules/elastic" log "github.com/cihub/seelog" - "golang.org/x/crypto/bcrypt" elastic3 "infini.sh/console/modules/elastic/api" security2 "infini.sh/console/modules/security" "infini.sh/framework/core/api" @@ -60,19 +65,19 @@ import ( "infini.sh/framework/core/module" "infini.sh/framework/core/orm" "infini.sh/framework/core/pipeline" + frameworksecurity "infini.sh/framework/core/security" "infini.sh/framework/core/util" "infini.sh/framework/lib/fasthttp" "infini.sh/framework/lib/fasttemplate" keystore2 "infini.sh/framework/lib/keystore" - "infini.sh/framework/modules/elastic/adapter" elastic1 "infini.sh/framework/modules/elastic/common" + frameworkrbac "infini.sh/framework/modules/security/native" "infini.sh/framework/plugins/replay" ) // Easysearch auto create ingest user password -const ingestUser = "infini_ingest" - -var ingestPassword = util.GenerateSecureString(20) +const ingestUser = "infini-ingest" +const systemClusterIngestPasswordKey = "SYSTEM_CLUSTER_INGEST_PASSWORD" type Module struct { api.Handler @@ -87,38 +92,168 @@ func init() { } func (module *Module) Setup() { - - if !global.Env().SetupRequired() { - return - } - - api.HandleAPIMethod(api.POST, "/setup/_validate", module.validate) - api.HandleAPIMethod(api.POST, "/setup/_initialize", module.initialize) - api.HandleAPIMethod(api.POST, "/setup/_validate_secret", module.validateSecret) - api.HandleAPIMethod(api.POST, "/setup/_initialize_template", module.initializeTemplate) + registerPublicSetupRoute := func(method api.Method, path string, handler func(w http.ResponseWriter, req *http.Request, ps httprouter.Params)) { + api.HandleUIMethod(method, path, + handler, + api.AllowPublicAccess()) + } + + registerPublicSetupRoute(api.POST, "/account/replay_nonce", core2.RequireSecureTransport(frameworkrbac.IssueReplayNonce)) + registerPublicSetupRoute(api.POST, "/setup/_validate", core2.RequireSecureTransport(core2.RequireReplayProtection(module.validate))) + registerPublicSetupRoute(api.POST, "/setup/_initialize", core2.RequireSecureTransport(core2.RequireReplayProtection(module.initialize))) + registerPublicSetupRoute(api.POST, "/setup/_validate_secret", core2.RequireSecureTransport(core2.RequireReplayProtection(module.validateSecret))) + registerPublicSetupRoute(api.POST, "/setup/_initialize_template", core2.RequireSecureTransport(core2.RequireReplayProtection(module.initializeTemplate))) + elastic3.RegisterPublicUITestAPI() + + api.HandleAPIMethod(api.POST, "/account/replay_nonce", core2.RequireSecureTransport(frameworkrbac.IssueReplayNonce)) + api.HandleAPIMethod(api.POST, "/setup/_validate", core2.RequireSecureTransport(core2.RequireReplayProtection(module.validate))) + api.HandleAPIMethod(api.POST, "/setup/_initialize", core2.RequireSecureTransport(core2.RequireReplayProtection(module.initialize))) + api.HandleAPIMethod(api.POST, "/setup/_validate_secret", core2.RequireSecureTransport(core2.RequireReplayProtection(module.validateSecret))) + api.HandleAPIMethod(api.POST, "/setup/_initialize_template", core2.RequireSecureTransport(core2.RequireReplayProtection(module.initializeTemplate))) elastic3.InitTestAPI() } var setupFinishedCallback = []func(){} +var setupCallbackOnce sync.Once +var setupInitializeRunning uint32 func RegisterSetupCallback(f func()) { setupFinishedCallback = append(setupFinishedCallback, f) } func InvokeSetupCallback() { - for _, v := range setupFinishedCallback { - v() + setupCallbackOnce.Do(func() { + for _, v := range setupFinishedCallback { + v() + } + }) +} + +func InvokeSetupCallbackAsync() { + go func() { + defer func() { + if r := recover(); r != nil { + log.Errorf("setup callback panic: %v", r) + } + }() + InvokeSetupCallback() + }() +} + +func getOrCreateIngestPassword() (string, error) { + value, err := keystore.GetValue(systemClusterIngestPasswordKey) + if err == nil && len(value) > 0 { + return string(value), nil + } + if err != nil && err != keystore2.ErrKeyDoesntExists { + return "", err + } + + password := util.GenerateSecureString(20) + if err := keystore.SetValue(systemClusterIngestPasswordKey, []byte(password)); err != nil { + return "", err + } + return password, nil +} + +func acquireSetupInitialization() bool { + return atomic.CompareAndSwapUint32(&setupInitializeRunning, 0, 1) +} + +func releaseSetupInitialization() { + atomic.StoreUint32(&setupInitializeRunning, 0) +} + +func EnsureSystemClusterBasicAuth() error { + sysClusterID, ok := lookupSystemClusterID() + if !ok { + return nil + } + conf := elastic.GetConfigNoPanic(sysClusterID) + if conf == nil || (conf.BasicAuth != nil && conf.BasicAuth.Username != "") || conf.CredentialID == "" { + return nil + } + + basicAuth, err := elastic1.GetBasicAuth(conf) + if err != nil { + return err + } + if basicAuth == nil { + return nil + } + + conf.BasicAuth = basicAuth + if meta := elastic.GetMetadata(sysClusterID); meta != nil && meta.Config != nil { + meta.Config.BasicAuth = basicAuth + } + elastic.UpdateConfig(*conf) + return nil +} + +func ResolveManagedAgentTemplateCredentials(client elastic.API, indexPrefix string) (string, string, error) { + if client == nil { + return "", "", fmt.Errorf("system cluster client not found") + } + version := client.GetVersion() + distribution := version.Distribution + if distribution == "" { + distribution = elastic.Elasticsearch + } + if distribution == elastic.Easysearch { + agentPassword, err := getOrCreateIngestPassword() + if err != nil { + return "", "", err + } + if _, err := client.GetPrivileges(); err != nil { + return "", "", err + } + if err := initIngestUser(client, indexPrefix, ingestUser, agentPassword); err != nil { + return "", "", err + } + return ingestUser, systemClusterIngestPasswordKey, nil + } + if err := EnsureSystemClusterBasicAuth(); err != nil { + return "", "", err + } + sysClusterID, ok := lookupSystemClusterID() + if !ok { + return "", "SYSTEM_CLUSTER_PASS", nil } + cfg := elastic.GetConfigNoPanic(sysClusterID) + if cfg != nil && cfg.BasicAuth != nil { + return cfg.BasicAuth.Username, "SYSTEM_CLUSTER_PASS", nil + } + return "", "SYSTEM_CLUSTER_PASS", nil +} + +func lookupSystemClusterID() (string, bool) { + value := global.Lookup(elastic.GlobalSystemElasticsearchID) + sysClusterID, ok := value.(string) + if !ok || sysClusterID == "" { + return "", false + } + return sysClusterID, true } // Start initializes the module and registers a change event for credentials. func (module *Module) Start() error { + if !global.Env().SetupRequired() { + if err := EnsureSystemClusterBasicAuth(); err != nil { + log.Error(err) + } + } credential.RegisterChangeEvent(func(cred *credential.Credential) { if cred == nil { return } - sysClusterID := global.MustLookupString(elastic.GlobalSystemElasticsearchID) + sysClusterID, ok := lookupSystemClusterID() + if !ok { + return + } conf := elastic.GetConfig(sysClusterID) + if conf == nil { + return + } if conf.CredentialID != cred.ID { return } @@ -132,6 +267,11 @@ func (module *Module) Start() error { if err != nil { log.Error(err) } + conf.BasicAuth = &basicAuth + if meta := elastic.GetMetadata(sysClusterID); meta != nil && meta.Config != nil { + meta.Config.BasicAuth = &basicAuth + } + elastic.UpdateConfig(*conf) } }) return nil @@ -154,21 +294,103 @@ type SetupRequest struct { } `json:"cluster"` Skip bool `json:"skip"` + ResetUser bool `json:"reset_user"` BootstrapUsername string `json:"bootstrap_username"` BootstrapPassword string `json:"bootstrap_password"` CredentialSecret string `json:"credential_secret"` InitializeTemplate string `json:"initialize_template"` + Language string `json:"language,omitempty"` + PrimaryShards int `json:"primary_shards"` + AutoExpandReplicas string `json:"auto_expand_replicas"` + EnableRollup bool `json:"enable_rollup"` } var GlobalSystemElasticsearchID = "infini_default_system_cluster" -const VersionTooOld = "elasticsearch_version_too_old" const IndicesExists = "elasticsearch_indices_exists" const TemplateExists = "elasticsearch_template_exists" const VersionNotSupport = "unknown_cluster_version" var cfg1 elastic1.ORMConfig +const defaultSetupAutoExpandReplicas = "0-1" + +var setupAutoExpandReplicasPattern = regexp.MustCompile(`^(false|all|\d+-\d+)$`) + +func (r *SetupRequest) shouldResetBootstrapUser() bool { + if !r.Skip { + return true + } + return r.ResetUser +} + +func validateSetupBootstrap(request *SetupRequest) error { + request.BootstrapUsername = strings.TrimSpace(request.BootstrapUsername) + if !request.shouldResetBootstrapUser() { + request.BootstrapUsername = "" + request.BootstrapPassword = "" + return nil + } + if request.BootstrapUsername == "" { + return fmt.Errorf("bootstrap username is required when resetting administrator") + } + if strings.TrimSpace(request.BootstrapPassword) == "" { + return fmt.Errorf("bootstrap password is required when resetting administrator") + } + if !util.ValidateSecure(request.BootstrapPassword) { + return fmt.Errorf("bootstrap password does not meet security requirements") + } + return nil +} + +func resolveSetupTemplateSettings(client elastic.API, request *SetupRequest) (int, string, error) { + primaryShards := request.PrimaryShards + if primaryShards <= 0 { + health, err := client.ClusterHealth(context.Background()) + if err != nil { + return 0, "", err + } + if health != nil { + if health.NumberOf_data_nodes > 0 { + primaryShards = health.NumberOf_data_nodes + } else if health.NumberOfNodes > 0 { + primaryShards = health.NumberOfNodes + } + } + } + if primaryShards <= 0 { + primaryShards = 1 + } + + autoExpandReplicas := strings.TrimSpace(request.AutoExpandReplicas) + if autoExpandReplicas == "" { + autoExpandReplicas = defaultSetupAutoExpandReplicas + } + if !setupAutoExpandReplicasPattern.MatchString(autoExpandReplicas) { + return 0, "", errors.Errorf("invalid auto expand replicas, expected false, all, or a range like 0-1") + } + + return primaryShards, autoExpandReplicas, nil +} + +func ResolveRelayPartitionSize(client elastic.API) int { + if client == nil { + return 1 + } + + health, err := client.ClusterHealth(context.Background()) + if err != nil || health == nil { + return 1 + } + if health.NumberOf_data_nodes > 0 { + return health.NumberOf_data_nodes + } + if health.NumberOfNodes > 0 { + return health.NumberOfNodes + } + return 1 +} + // validate checks the Elasticsearch cluster configuration and validates the setup. func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { @@ -230,28 +452,12 @@ func (module *Module) validate(w http.ResponseWriter, r *http.Request, ps httpro } //validate version - verInfo, err := adapter.ClusterVersion(elastic.GetMetadata(cfg.ID)) + verInfo, err := console_common.ClusterVersion(elastic.GetMetadata(cfg.ID)) if err != nil { panic(err) } - if verInfo.Version.Distribution == elastic.Elasticsearch { - if verInfo.Version.Number != "" { - ver := &util.Version{} - ver, err = util.ParseSemantic(verInfo.Version.Number) - if err != nil { - panic(err) - } - if ver.Major() == 5 && ver.Minor() < 3 { - errType = VersionTooOld - panic(errors.Errorf("elasticsearch version(%v) should greater than v5.3", verInfo.Version.Number)) - } else if ver.Major() < 5 { - errType = VersionTooOld - panic(errors.Errorf("elasticsearch version(%v) should greater than v5.3", verInfo.Version.Number)) - } - } - } else if verInfo.Version.Distribution != elastic.Easysearch && verInfo.Version.Distribution != elastic.Opensearch { - errType = VersionNotSupport - panic(errors.Errorf("unknown distribution (%v)", verInfo.Version.Distribution)) + if verInfo.Version.Distribution != elastic.Easysearch { + log.Warnf("non-recommended search engine distribution detected: '%v'. For optimal performance and full feature support, it is highly recommended to use Easysearch.", verInfo.Version.Distribution) } cfg1 = elastic1.ORMConfig{} exist, err := env.ParseConfig("elastic.orm", &cfg1) @@ -324,6 +530,7 @@ func (module *Module) initTempClient(request *SetupRequest) (error, elastic.API) Username: request.Cluster.Username, Password: ucfg.SecretString(request.Cluster.Password), }, + Hosts: request.Cluster.Hosts, } if cfg.Endpoint != "" && cfg.Host == "" { @@ -338,7 +545,7 @@ func (module *Module) initTempClient(request *SetupRequest) (error, elastic.API) cfg.ID = GlobalSystemElasticsearchID cfg.Name = "INFINI_SYSTEM (" + util.PickRandomName() + ")" elastic.InitMetadata(&cfg, true) - verInfo, err := adapter.ClusterVersion(elastic.GetMetadata(cfg.ID)) + verInfo, err := console_common.ClusterVersion(elastic.GetMetadata(cfg.ID)) if err != nil { panic(err) } @@ -402,6 +609,11 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http module.WriteError(w, "setup not permitted", http.StatusInternalServerError) return } + if !acquireSetupInitialization() { + module.WriteError(w, "setup is already running", http.StatusConflict) + return + } + defer releaseSetupInitialization() request := &SetupRequest{} err := module.DecodeJSON(r, request) if err != nil { @@ -409,8 +621,8 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http return } - if !util.ValidateSecure(request.BootstrapPassword) { - module.WriteError(w, "invalid bootstrap password", http.StatusInternalServerError) + if err := validateSetupBootstrap(request); err != nil { + module.WriteError(w, err.Error(), http.StatusBadRequest) return } @@ -529,7 +741,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http toSaveCfg := cfg oldCfg := elastic.ElasticsearchConfig{} oldCfg.ID = toSaveCfg.ID - _, _ = orm.Get(&oldCfg) + _, _ = orm.GetV2(orm.NewContext(), &oldCfg) // If the old configuration exists, update it with the new values if oldCfg.Name != "" { toSaveCfg = oldCfg @@ -599,24 +811,46 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http }, } toSaveCfg.Created = &t - err = orm.Save(nil, &toSaveCfg) + err = orm.Save(orm.NewContext(), &toSaveCfg) + if err != nil { + panic(err) + } + previousAgentCredentialID := toSaveCfg.AgentCredentialID + previousNoDefaultAuthForAgent := toSaveCfg.NoDefaultAuthForAgent + err = elastic3.EnsureManagedAgentCredential(&toSaveCfg, oldCfg.Name) if err != nil { panic(err) } + if previousAgentCredentialID != toSaveCfg.AgentCredentialID || previousNoDefaultAuthForAgent != toSaveCfg.NoDefaultAuthForAgent { + err = orm.Save(orm.NewContext(), &toSaveCfg) + if err != nil { + panic(err) + } + } + if meta := elastic.GetMetadata(toSaveCfg.ID); meta != nil && meta.Config != nil { + meta.Config.AgentCredentialID = toSaveCfg.AgentCredentialID + meta.Config.NoDefaultAuthForAgent = toSaveCfg.NoDefaultAuthForAgent + } + elastic.UpdateConfig(toSaveCfg) + if err := EnsureSystemClusterBasicAuth(); err != nil { + panic(err) + } - if request.BootstrapUsername != "" && request.BootstrapPassword != "" { + if request.shouldResetBootstrapUser() { //Save bootstrap user + enabled := true user := security.User{} user.ID = "default_user_" + request.BootstrapUsername user.Username = request.BootstrapUsername user.Nickname = request.BootstrapUsername - var hash []byte - hash, err = bcrypt.GenerateFromPassword([]byte(request.BootstrapPassword), bcrypt.DefaultCost) + material, err := frameworksecurity.GeneratePasswordMaterial(request.BootstrapPassword) if err != nil { panic(err) } - - user.Password = string(hash) + user.Password = material.Hash + user.Enabled = &enabled + user.PasswordSalt = material.Salt + user.PasswordVerifier = material.Verifier role := []security.UserRole{} role = append(role, security.UserRole{ ID: security.RoleAdminName, @@ -625,7 +859,7 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http user.Roles = role now := time.Now() user.Created = &now - err = orm.Save(nil, &user) + err = orm.Save(orm.NewContext(), &user) if err != nil { panic(err) } @@ -638,39 +872,26 @@ func (module *Module) initialize(w http.ResponseWriter, r *http.Request, ps http //save to local file file := path.Join(global.Env().GetConfigDir(), "system_config.yml") _, err = util.FilePutContent(file, fmt.Sprintf("configs.template:\n - name: \"system\"\n path: ./config/system_config.tpl\n variable:\n "+ - "CLUSTER_ID: %v\n CLUSTER_ENDPOINT: \"%v\"\n "+ + "CLUSTER_ID: %v\n CLUSTER_ENDPOINT: [%v]\n "+ "CLUSTER_USER: \"%v\"\n CLUSTER_VER: \"%v\"\n CLUSTER_DISTRIBUTION: \"%v\"\n INDEX_PREFIX: \"%v\"", - GlobalSystemElasticsearchID, cfg.GetAnyEndpoint(), cfg.BasicAuth.Username, cfg.Version, cfg.Distribution, cfg1.IndexPrefix)) + GlobalSystemElasticsearchID, strings.Join(cfg.GetAllEndpoints(), ","), cfg.BasicAuth.Username, cfg.Version, cfg.Distribution, cfg1.IndexPrefix)) if err != nil { panic(err) } - //callback - InvokeSetupCallback() - //place setup lock file setupLock := path.Join(global.Env().GetDataDir(), ".setup_lock") _, err = util.FilePutContent(setupLock, time.Now().String()) if err != nil { panic(err) } - //update credential state - q := util.MapStr{ - "query": util.MapStr{ - "range": util.MapStr{ - "created": util.MapStr{ - "lte": "now-30s", - }, - }, - }, - "script": util.MapStr{ - "source": fmt.Sprintf("ctx._source['invalid'] = %v", secretMismatch), - }, - } - err = orm.UpdateBy(credential.Credential{}, util.MustToJSONBytes(q)) - if err != nil { - log.Error(err) - } + global.Env().CheckSetup() + + // callback can take a while (template init/module starts), do it in background. + InvokeSetupCallbackAsync() + + //update credential state in background to avoid blocking setup response + go updateCredentialState(secretMismatch) success = true } @@ -765,13 +986,32 @@ func createCred(name, username, password string) string { now := time.Now() cred.Created = &now cred.Updated = &now - err = orm.Save(nil, &cred) + err = orm.Save(orm.NewContext(), &cred) if err != nil { panic(err) } return cred.ID } +func updateCredentialState(secretMismatch bool) { + q := util.MapStr{ + "query": util.MapStr{ + "range": util.MapStr{ + "created": util.MapStr{ + "lte": "now-30s", + }, + }, + }, + "script": util.MapStr{ + "source": fmt.Sprintf("ctx._source['invalid'] = %v", secretMismatch), + }, + } + err := orm.UpdateBy(credential.Credential{}, util.MustToJSONBytes(q)) + if err != nil { + log.Error(err) + } +} + // getYamlData reads a YAML file from the setup directory and returns its content as a byte slice. func getYamlData(filename string) []byte { baseDir := path.Join(global.Env().GetConfigDir(), "setup") @@ -820,9 +1060,12 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, } baseDir := path.Join(global.Env().GetConfigDir(), "setup") var ( - dslTplFileName = "noop.tpl" - useCommon = true - rollupEnabled = false + dslTplFileName = "noop.tpl" + useCommon = true + rollupEnabled = false + agentUsername = request.Cluster.Username + agentPassword = request.Cluster.Password + agentPasswordKey = "SYSTEM_CLUSTER_PASS" ) if large, _ := util.VersionCompare(ver.Number, "1.12.1"); large >= 0 { rollupEnabled = true @@ -846,22 +1089,29 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, } case "alerting": dslTplFileName = "alerting.tpl" + if localizedTemplate := resolveAlertingTemplateByLanguage(strings.TrimSpace(request.Language)); localizedTemplate != "" { + dslTplFileName = localizedTemplate + } case "insight": dslTplFileName = "insight.tpl" case "view": dslTplFileName = "view.tpl" case "agent": if ver.Distribution == elastic.Easysearch { - err = keystore.SetValue("SYSTEM_CLUSTER_INGEST_PASSWORD", []byte(ingestPassword)) + agentPassword, err = getOrCreateIngestPassword() if err != nil { panic(err) } + agentUsername = ingestUser + agentPasswordKey = systemClusterIngestPasswordKey client := elastic.GetClient(GlobalSystemElasticsearchID) - if privileges, _ := client.GetPrivileges(); len(privileges) > 0 { - err = initIngestUser(client, cfg1.IndexPrefix, ingestUser, ingestPassword) - if err != nil { - panic(err) - } + _, err = client.GetPrivileges() + if err != nil { + panic(err) + } + err = initIngestUser(client, cfg1.IndexPrefix, agentUsername, agentPassword) + if err != nil { + panic(err) } } dslTplFileName = "agent.tpl" @@ -896,7 +1146,18 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, break } + systemClient := elastic.GetClient(GlobalSystemElasticsearchID) + primaryShards, autoExpandReplicas, err := resolveSetupTemplateSettings(systemClient, request) + if err != nil { + panic(err) + } + relayPartitionSize := ResolveRelayPartitionSize(systemClient) + dslTplFile := path.Join(baseDir, dslTplFileName) + if request.InitializeTemplate == "alerting" && !util.FileExists(dslTplFile) { + dslTplFileName = "alerting.tpl" + dslTplFile = path.Join(baseDir, dslTplFileName) + } if !util.FileExists(dslTplFile) { panic(errors.Errorf("template file %v for setup was missing", dslTplFile)) } @@ -906,6 +1167,7 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, if err != nil { panic(err) } + if len(dsl) == 0 { panic(fmt.Sprintf("got empty template [%s]", dslTplFile)) } @@ -926,8 +1188,10 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, return w.Write(getYamlData("system_ingest_config.dat")) case "SETUP_TASK_CONFIG_TPL": return w.Write(getYamlData("task_config_tpl.dat")) - case "SETUP_AGENT_RELAY_GATEWAY_CONFIG": - return w.Write(getYamlData("agent_relay_gateway_config.dat")) + case "SETUP_GATEWAY_RELAY_CONFIG": + return w.Write(getYamlData("gateway_relay.dat")) + case "SETUP_GATEWAY_MIGRATION_CONFIG": + return w.Write(getYamlData("gateway_migration.dat")) } //ignore unresolved variable return w.Write([]byte("$[[" + tag + "]]")) @@ -940,26 +1204,15 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, case "SETUP_ES_PASSWORD": return w.Write([]byte(request.Cluster.Password)) case "SETUP_AGENT_USERNAME": - if ver.Distribution == elastic.Easysearch { - return w.Write([]byte(ingestUser)) - } else { - return w.Write([]byte(request.Cluster.Username)) - } - case "SETUP_AGENT_PASSWORD": - if ver.Distribution == elastic.Easysearch { - return w.Write([]byte(ingestPassword)) - } else { - return w.Write([]byte(request.Cluster.Password)) - } + return w.Write([]byte(agentUsername)) + case "SETUP_AGENT_PASSWORD_KEY": + return w.Write([]byte(agentPasswordKey)) case "SETUP_SCHEME": return w.Write([]byte(request.Cluster.Schema)) case "SETUP_ENDPOINTS": - endpoints := []string{request.Cluster.Endpoint} + endpoints := make([]string, 0, len(request.Cluster.Hosts)) for _, host := range request.Cluster.Hosts { - endpoint := fmt.Sprintf("%s://%s", request.Cluster.Schema, host) - if !util.StringInArray(endpoints, endpoint) { - endpoints = append(endpoints, endpoint) - } + endpoints = append(endpoints, fmt.Sprintf("%s://%s", request.Cluster.Schema, host)) } endpointBytes := util.MustToJSONBytes(endpoints) endpointBytes = bytes.ReplaceAll(endpointBytes, []byte("\""), []byte("\\\"")) @@ -982,6 +1235,12 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, return w.Write([]byte(request.BootstrapUsername)) case "SETUP_DOC_TYPE": return w.Write([]byte(docType)) + case "SETUP_PRIMARY_SHARDS": + return w.Write([]byte(strconv.Itoa(primaryShards))) + case "SETUP_AUTO_EXPAND_REPLICAS": + return w.Write([]byte(autoExpandReplicas)) + case "SETUP_RELAY_PARTITION_SIZE": + return w.Write([]byte(strconv.Itoa(relayPartitionSize))) } //ignore unresolved variable return w.Write([]byte("$[[" + tag + "]]")) @@ -1017,6 +1276,27 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, }, http.StatusOK) return } + + if request.InitializeTemplate == "rollup" && request.EnableRollup { + err = systemClient.UpdateClusterSettings(util.MustToJSONBytes(util.MapStr{ + "persistent": util.MapStr{ + "rollup": util.MapStr{ + "search": util.MapStr{ + "enabled": "true", + }, + "hours_before": "24", + }, + }, + })) + if err != nil { + module.WriteJSON(w, util.MapStr{ + "success": false, + "log": fmt.Sprintf("initalize rollup settings failed: %v", err), + }, http.StatusOK) + return + } + } + module.WriteJSON(w, util.MapStr{ "success": true, "log": fmt.Sprintf("initalize template [%s] succeed", request.InitializeTemplate), @@ -1026,39 +1306,67 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request, // initIngestUser initializes the ingest user with the required permissions for writing metrics and logs. func initIngestUser(client elastic.API, indexPrefix string, username, password string) error { - roleTpl := `{ - "cluster": [ - "cluster_monitor", - "cluster_composite_ops" - ], - "description": "Provide the minimum permissions for INFINI AGENT to write metrics and logs", - "indices": [{ - "names": [ - "%slogs*", "%smetrics*" - ], - "query": "", - "field_security": [], - "field_mask": [], - "privileges": [ - "create_index","index","manage_aliases","write" - ] - }] - }` - roleBody := fmt.Sprintf(roleTpl, indexPrefix, indexPrefix) - err := client.PutRole(username, []byte(roleBody)) + roleBody := util.MustToJSONBytes(buildIngestRoleBody(indexPrefix)) + err := client.PutRole(username, roleBody) if err != nil { return fmt.Errorf("failed to create ingest role: %w", err) } - userTpl := `{ - "roles": [ - "%s" - ], - "password": "%s"}` - - userBody := fmt.Sprintf(userTpl, username, password) - err = client.PutUser(username, []byte(userBody)) + userBody := util.MustToJSONBytes(util.MapStr{ + "roles": []string{username}, + "password": password, + }) + err = client.PutUser(username, userBody) if err != nil { return fmt.Errorf("failed to create ingest user: %w", err) } return nil } + +func buildIngestRoleBody(indexPrefix string) util.MapStr { + return util.MapStr{ + "cluster": []string{ + "cluster_monitor", + "cluster_composite_ops", + }, + "description": "Provide the minimum permissions for INFINI AGENT to write metrics and logs", + "indices": []util.MapStr{ + { + "names": []string{ + "*", + }, + "query": "", + "field_security": []string{}, + "field_mask": []string{}, + "privileges": []string{ + "manage_aliases", + }, + }, + { + "names": []string{ + fmt.Sprintf("%slogs*", indexPrefix), + fmt.Sprintf("%smetrics*", indexPrefix), + }, + "query": "", + "field_security": []string{}, + "field_mask": []string{}, + "privileges": []string{ + "create_index", + "index", + "manage_aliases", + "write", + }, + }, + }, + } +} + +func resolveAlertingTemplateByLanguage(language string) string { + switch { + case strings.HasPrefix(strings.ToLower(language), "zh"): + return "alerting.zh-CN.tpl" + case strings.HasPrefix(strings.ToLower(language), "en"): + return "alerting.en-US.tpl" + default: + return "" + } +} diff --git a/plugin/setup/setup_test.go b/plugin/setup/setup_test.go new file mode 100644 index 00000000..f44fc506 --- /dev/null +++ b/plugin/setup/setup_test.go @@ -0,0 +1,525 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package task + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + api2 "infini.sh/framework/core/api" + config2 "infini.sh/framework/core/config" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + replaysecurity "infini.sh/framework/core/security/replay" + "infini.sh/framework/core/util" +) + +func TestInvokeSetupCallbackRunsOnlyOnce(t *testing.T) { + previousCallbacks := setupFinishedCallback + previousOnce := setupCallbackOnce + defer func() { + setupFinishedCallback = previousCallbacks + setupCallbackOnce = previousOnce + }() + + setupFinishedCallback = nil + setupCallbackOnce = sync.Once{} + + count := 0 + RegisterSetupCallback(func() { + count++ + }) + RegisterSetupCallback(func() { + count++ + }) + + InvokeSetupCallback() + InvokeSetupCallback() + + if count != 2 { + t.Fatalf("expected callbacks to run once each, got %d invocations", count) + } +} + +func TestAcquireSetupInitialization(t *testing.T) { + releaseSetupInitialization() + defer releaseSetupInitialization() + + if !acquireSetupInitialization() { + t.Fatal("expected first acquire to succeed") + } + if acquireSetupInitialization() { + t.Fatal("expected second acquire to fail while initialization is running") + } + + releaseSetupInitialization() + + if !acquireSetupInitialization() { + t.Fatal("expected acquire to succeed after release") + } +} + +func TestEnsureSystemClusterBasicAuthSkipsWhenSystemClusterUnavailable(t *testing.T) { + previous := global.Lookup(elastic.GlobalSystemElasticsearchID) + defer global.Register(elastic.GlobalSystemElasticsearchID, previous) + + global.Register(elastic.GlobalSystemElasticsearchID, "") + + if err := EnsureSystemClusterBasicAuth(); err != nil { + t.Fatalf("expected nil error when system cluster id is unavailable, got %v", err) + } +} + +func TestSystemIngestTemplateHostsRendersAsYAMLArray(t *testing.T) { + content := string(mustReadSetupDataFile(t, "system_ingest_config.dat")) + content = strings.ReplaceAll(content, "$[[SETUP_AGENT_USERNAME]]", "infini-ingest") + content = strings.ReplaceAll(content, "$[[SETUP_AGENT_PASSWORD_KEY]]", "SYSTEM_CLUSTER_INGEST_PASSWORD") + content = strings.ReplaceAll(content, "$[[SETUP_SCHEME]]", "http") + content = strings.ReplaceAll(content, "$[[SETUP_HOSTS]]", `["192.168.3.8:9200"]`) + content = strings.ReplaceAll(content, "$[[SETUP_INDEX_PREFIX]]", ".infini_") + + if _, err := config2.NewConfigWithYAML([]byte(content), "system_ingest_config.yml"); err != nil { + t.Fatalf("expected rendered system_ingest_config to parse, got %v\n%s", err, content) + } + if !strings.Contains(content, `hosts: ["192.168.3.8:9200"]`) { + t.Fatalf("expected hosts array rendering, got:\n%s", content) + } +} + +func TestTaskConfigTemplateAllowsArrayEndpointSubstitution(t *testing.T) { + content := string(mustReadSetupDataFile(t, "task_config_tpl.dat")) + content = strings.ReplaceAll(content, "$[[CLUSTER_ENDPOINT]]", `http://192.168.3.8:9200`) + + if _, err := config2.NewConfigWithYAML([]byte(content), "task_config.tpl"); err != nil { + t.Fatalf("expected task_config.tpl to parse after endpoint substitution, got %v\n%s", err, content) + } + if !strings.Contains(content, `endpoints: ["http://192.168.3.8:9200"]`) { + t.Fatalf("expected endpoints array rendering, got:\n%s", content) + } +} + +func TestSetupRegistersReplayNonceAPI(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + testEnv.EnableSetup(true) + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + module := &Module{} + module.Setup() + + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/replay_nonce", bytes.NewBufferString(`{"method":"POST","path":"/setup/_validate"}`)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + + api2.ServeRegisteredAPIRequest(resp, req) + + if resp.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d: %s", http.StatusOK, resp.Code, resp.Body.String()) + } + + var body map[string]interface{} + if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + + nonce, ok := body["nonce"].(string) + if !ok || nonce == "" { + t.Fatalf("expected nonce in response, got %#v", body) + } +} + +func TestSetupRegistersReplayNonceUIRoute(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + testEnv.EnableSetup(true) + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + module := &Module{} + module.Setup() + + webCfg := config2.WebAppConfig{} + webCfg.NetworkConfig.Binding = "127.0.0.1:0" + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/replay_nonce", bytes.NewBufferString(`{"method":"POST","path":"/setup/_validate"}`)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve ui request: %v", err) + } + + if resp.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d: %s", http.StatusOK, resp.Code, resp.Body.String()) + } + + var body map[string]interface{} + if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + + nonce, ok := body["nonce"].(string) + if !ok || nonce == "" { + t.Fatalf("expected nonce in response, got %#v", body) + } +} + +func TestSetupRegistersTryConnectUIRoute(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + testEnv.EnableSetup(true) + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + module := &Module{} + module.Setup() + + webCfg := config2.WebAppConfig{} + webCfg.NetworkConfig.Binding = "127.0.0.1:0" + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + nonceReq := httptest.NewRequest(http.MethodPost, "https://console.local/account/replay_nonce", bytes.NewBufferString(`{"method":"POST","path":"/elasticsearch/try_connect"}`)) + nonceReq.Header.Set("Content-Type", "application/json") + nonceResp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(nonceResp, nonceReq); err != nil { + t.Fatalf("serve nonce request: %v", err) + } + + var nonceBody map[string]interface{} + if err := json.Unmarshal(nonceResp.Body.Bytes(), &nonceBody); err != nil { + t.Fatalf("unmarshal nonce response: %v", err) + } + + nonce, ok := nonceBody["nonce"].(string) + if !ok || nonce == "" { + t.Fatalf("expected nonce in response, got %#v", nonceBody) + } + + req := httptest.NewRequest(http.MethodPost, "https://console.local/elasticsearch/try_connect", bytes.NewBufferString(`{"hosts":["127.0.0.1:1"],"schema":"http"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(replaysecurity.HeaderName, nonce) + resp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve try_connect request: %v", err) + } + + if resp.Code == http.StatusNotFound { + t.Fatalf("expected try_connect route to be registered on UI router, got 404") + } +} + +func TestSetupRegistersTryConnectUIRouteWhenSetupNotRequired(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + if err := os.WriteFile(filepath.Join(testEnv.SystemConfig.PathConfig.Data, ".setup_lock"), []byte("1"), 0o644); err != nil { + t.Fatalf("write setup lock: %v", err) + } + testEnv.EnableSetup(true) + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + module := &Module{} + module.Setup() + + webCfg := config2.WebAppConfig{} + webCfg.NetworkConfig.Binding = "127.0.0.1:0" + api2.StartWeb(webCfg) + defer api2.StopWeb(webCfg) + + nonceReq := httptest.NewRequest(http.MethodPost, "https://console.local/account/replay_nonce", bytes.NewBufferString(`{"method":"POST","path":"/elasticsearch/try_connect"}`)) + nonceReq.Header.Set("Content-Type", "application/json") + nonceResp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(nonceResp, nonceReq); err != nil { + t.Fatalf("serve nonce request: %v", err) + } + + var nonceBody map[string]interface{} + if err := json.Unmarshal(nonceResp.Body.Bytes(), &nonceBody); err != nil { + t.Fatalf("unmarshal nonce response: %v", err) + } + + nonce, ok := nonceBody["nonce"].(string) + if !ok || nonce == "" { + t.Fatalf("expected nonce in response, got %#v", nonceBody) + } + + req := httptest.NewRequest(http.MethodPost, "https://console.local/elasticsearch/try_connect", bytes.NewBufferString(`{invalid`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(replaysecurity.HeaderName, nonce) + resp := httptest.NewRecorder() + + if err := api2.ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve try_connect request: %v", err) + } + + if resp.Code == http.StatusNotFound { + t.Fatalf("expected try_connect route to stay registered on UI router when setup is not required, got 404") + } +} + +func TestValidateSetupBootstrapSkipsAdminResetForSkipSetup(t *testing.T) { + req := &SetupRequest{ + Skip: true, + BootstrapUsername: "admin", + BootstrapPassword: "weak", + } + + if err := validateSetupBootstrap(req); err != nil { + t.Fatalf("expected bootstrap validation to be skipped, got %v", err) + } + if req.BootstrapUsername != "" || req.BootstrapPassword != "" { + t.Fatalf("expected bootstrap credentials to be cleared when reset is disabled, got %#v", req) + } +} + +func TestValidateSetupBootstrapRequiresPasswordWhenResettingAdmin(t *testing.T) { + req := &SetupRequest{ + Skip: true, + ResetUser: true, + BootstrapUsername: "admin", + } + + err := validateSetupBootstrap(req) + if err == nil || !strings.Contains(err.Error(), "bootstrap password is required") { + t.Fatalf("expected missing password error, got %v", err) + } +} + +func TestValidateSetupBootstrapRequiresStrongPasswordWhenResettingAdmin(t *testing.T) { + req := &SetupRequest{ + Skip: true, + ResetUser: true, + BootstrapUsername: "admin", + BootstrapPassword: "weakpass", + } + + err := validateSetupBootstrap(req) + if err == nil || !strings.Contains(err.Error(), "does not meet security requirements") { + t.Fatalf("expected password strength error, got %v", err) + } +} + +func TestValidateSetupBootstrapRequiresAdminOnInitialSetup(t *testing.T) { + req := &SetupRequest{ + Skip: false, + } + + err := validateSetupBootstrap(req) + if err == nil || !strings.Contains(err.Error(), "bootstrap username is required") { + t.Fatalf("expected missing username error, got %v", err) + } +} + +func TestGatewayRelayTemplateRendersAsChildConfig(t *testing.T) { + content := renderGatewaySetupData(t, "gateway_relay.dat") + + if _, err := config2.NewConfigWithYAML([]byte(content), "relay.yml"); err != nil { + t.Fatalf("expected rendered relay config to parse, got %v\n%s", err, content) + } + assertNoChildConfigGlobals(t, content) + assertContainsAll(t, content, + `binding: 0.0.0.0:8081`, + `cert_file: "config/relay_server.crt"`, + `key_file: "config/relay_server.key"`, + `metadata_refresh:`, + `remote_configs: false`, + `monitored: true`, + `metadata_cache_enabled: false`, + `password: "$[[keystore.SYSTEM_CLUSTER_INGEST_PASSWORD]]"`, + `queue_name_prefix: gateway_relay_async_bulk`, + `partition_size: 3`, + `elasticsearch: gateway_relay_system`, + `continue_metadata_missing: true`, + `max_connection_per_node: 1000`, + `name: gateway_relay_bulk_request_ingest`, + ) +} + +func TestGatewayMigrationTemplateRendersAsChildConfig(t *testing.T) { + content := renderGatewaySetupData(t, "gateway_migration.dat") + + if _, err := config2.NewConfigWithYAML([]byte(content), "migration.yml"); err != nil { + t.Fatalf("expected rendered migration config to parse, got %v\n%s", err, content) + } + assertNoChildConfigGlobals(t, content) + assertContainsAll(t, content, + `binding: 0.0.0.0:8082`, + `metadata_refresh:`, + `remote_configs: false`, + `monitored: true`, + `password: "$[[keystore.SYSTEM_CLUSTER_INGEST_PASSWORD]]"`, + `name: logging-server`, + `queue_name_prefix: gateway_migration_async_bulk`, + `name: gateway_migration_async_ingest_bulk_requests`, + `name: gateway_migration_request_logging_merge`, + ) +} + +func TestBuildIngestRoleBodyIncludesAliasManagePermission(t *testing.T) { + roleBody := buildIngestRoleBody(".infini_") + raw := util.MustToJSONBytes(roleBody) + + parsed := map[string]interface{}{} + if err := json.Unmarshal(raw, &parsed); err != nil { + t.Fatalf("unmarshal role body: %v", err) + } + + indices, ok := parsed["indices"].([]interface{}) + if !ok || len(indices) < 2 { + t.Fatalf("expected indices permissions, got %#v", parsed["indices"]) + } + first, ok := indices[0].(map[string]interface{}) + if !ok { + t.Fatalf("expected first indices permission object, got %#v", indices[0]) + } + firstNames, ok := first["names"].([]interface{}) + if !ok { + t.Fatalf("expected names array, got %#v", first["names"]) + } + if len(firstNames) != 1 || firstNames[0] != "*" { + t.Fatalf("expected first names to be [*], got %#v", firstNames) + } + privileges, ok := first["privileges"].([]interface{}) + if !ok { + t.Fatalf("expected privileges array, got %#v", first["privileges"]) + } + found := false + for _, p := range privileges { + if p == "manage_aliases" { + found = true + break + } + } + if !found { + t.Fatalf("expected manage_aliases permission, got %#v", privileges) + } + + second, ok := indices[1].(map[string]interface{}) + if !ok { + t.Fatalf("expected second indices permission object, got %#v", indices[1]) + } + secondNames, ok := second["names"].([]interface{}) + if !ok || len(secondNames) != 2 { + t.Fatalf("expected two index names in second permission, got %#v", second["names"]) + } + if secondNames[0] != ".infini_logs*" || secondNames[1] != ".infini_metrics*" { + t.Fatalf("unexpected second names, got %#v", secondNames) + } +} + +func TestAgentSetupTemplateSeedsRelayAndMigrationGatewayConfigs(t *testing.T) { + content := string(mustReadSetupTemplateFile(t, "agent.tpl")) + + assertContainsAll(t, content, + `"location": "relay.yml"`, + `"name": "relay.yml"`, + `"location": "migration.yml"`, + `"name": "migration.yml"`, + `"id": "gateway_migration_yml"`, + ) + if strings.Contains(content, "agent_relay_gateway_config.yml") { + t.Fatalf("expected old relay file name to be removed, got:\n%s", content) + } +} + +func renderGatewaySetupData(t *testing.T, name string) string { + t.Helper() + + content := string(mustReadSetupDataFile(t, name)) + content = strings.ReplaceAll(content, "$[[SETUP_AGENT_USERNAME]]", "infini-ingest") + content = strings.ReplaceAll(content, "$[[SETUP_AGENT_PASSWORD_KEY]]", "SYSTEM_CLUSTER_INGEST_PASSWORD") + content = strings.ReplaceAll(content, "$[[SETUP_ENDPOINTS]]", `["https://192.168.3.185:9201"]`) + content = strings.ReplaceAll(content, "$[[SETUP_INDEX_PREFIX]]", ".infini_") + content = strings.ReplaceAll(content, "$[[SETUP_RELAY_PARTITION_SIZE]]", "3") + return content +} + +func assertNoChildConfigGlobals(t *testing.T, content string) { + t.Helper() + + for _, forbidden := range []string{ + "allow_multi_instance:", + "path.data:", + "path.logs:", + "path.configs:", + "configs.auto_reload:", + "\napi:", + "\nnode:", + } { + if strings.Contains(content, forbidden) { + t.Fatalf("expected child config to omit %q, got:\n%s", forbidden, content) + } + } +} + +func assertContainsAll(t *testing.T, content string, expected ...string) { + t.Helper() + + for _, item := range expected { + if !strings.Contains(content, item) { + t.Fatalf("expected content to contain %q, got:\n%s", item, content) + } + } +} + +func mustReadSetupTemplateFile(t *testing.T, name string) []byte { + t.Helper() + + path := filepath.Join("..", "..", "config", "setup", "common", name) + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", name, err) + } + return content +} + +func mustReadSetupDataFile(t *testing.T, name string) []byte { + t.Helper() + + path := filepath.Join("..", "..", "config", "setup", "common", "data", name) + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", name, err) + } + return content +} diff --git a/service/agent/setup.go b/service/agent/setup.go new file mode 100644 index 00000000..1660e567 --- /dev/null +++ b/service/agent/setup.go @@ -0,0 +1,138 @@ +package agentservice + +import ( + "fmt" + "strings" + "sync" + + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + elasticorm "infini.sh/framework/modules/elastic" +) + +var ( + autoEnrollMu sync.RWMutex + autoEnrollCallback func(clusterIDs []string) +) + +const ( + clusterSettingsCategory = "cluster_settings" + clusterAgentSettings = "agent" +) + +func RegisterAutoEnrollCallback(callback func(clusterIDs []string)) { + autoEnrollMu.Lock() + defer autoEnrollMu.Unlock() + autoEnrollCallback = callback +} + +func TriggerAutoEnroll(clusterIDs []string) { + autoEnrollMu.RLock() + callback := autoEnrollCallback + autoEnrollMu.RUnlock() + if callback == nil { + return + } + callback(clusterIDs) +} + +func NormalizeLogsPaths(paths []string) []string { + seen := map[string]struct{}{} + result := make([]string, 0, len(paths)) + for _, item := range paths { + item = strings.TrimSpace(item) + if item == "" { + continue + } + if _, ok := seen[item]; ok { + continue + } + seen[item] = struct{}{} + result = append(result, item) + } + return result +} + +func NewClusterAgentSettings(clusterID string, logsPaths []string) *model.Setting { + logsPaths = NormalizeLogsPaths(logsPaths) + settings := &model.Setting{ + Metadata: model.Metadata{ + Category: clusterSettingsCategory, + Name: clusterAgentSettings, + Labels: util.MapStr{ + "cluster_id": clusterID, + }, + }, + Payload: util.MapStr{ + "cluster_id": clusterID, + "path_logs": firstString(logsPaths), + "logs_paths": logsPaths, + }, + } + settings.ID = fmt.Sprintf("%s_%s_%s", clusterSettingsCategory, clusterAgentSettings, clusterID) + return settings +} + +func GetClusterLogsPaths(clusterID string) ([]string, error) { + settings := NewClusterAgentSettings(clusterID, nil) + exists, err := orm.GetV2(orm.NewContext(), settings) + if !exists { + if err != nil && err != elasticorm.ErrNotFound { + return nil, err + } + return nil, nil + } + if err != nil { + return nil, err + } + return NormalizeLogsPaths(extractStringSlice(settings.Payload["logs_paths"], util.ToString(settings.Payload["path_logs"]))), nil +} + +func SaveClusterLogsPaths(clusterID string, logsPaths []string) error { + settings := NewClusterAgentSettings(clusterID, logsPaths) + normalized, _ := settings.Payload["logs_paths"].([]string) + if len(normalized) == 0 { + exists, err := orm.GetV2(orm.NewContext(), settings) + if !exists { + if err != nil && err != elasticorm.ErrNotFound { + return err + } + return nil + } + if err != nil { + return err + } + return orm.Delete(&orm.Context{Refresh: orm.WaitForRefresh}, settings) + } + return orm.Save(&orm.Context{Refresh: orm.WaitForRefresh}, settings) +} + +func extractStringSlice(value interface{}, fallback string) []string { + var items []string + switch v := value.(type) { + case nil: + case []string: + items = v + case []interface{}: + items = make([]string, 0, len(v)) + for _, item := range v { + items = append(items, util.ToString(item)) + } + default: + if fallback != "" { + items = []string{fallback} + } + } + if len(items) == 0 && fallback != "" { + items = []string{fallback} + } + return items +} + +func firstString(items []string) string { + if len(items) == 0 { + return "" + } + return items[0] +} diff --git a/service/alerting/action/email.go b/service/alerting/action/email.go index e63d7839..3f2632c5 100644 --- a/service/alerting/action/email.go +++ b/service/alerting/action/email.go @@ -47,6 +47,9 @@ func (act *EmailAction) Execute() ([]byte, error) { if act.Data.ServerID == "" { return nil, fmt.Errorf("parameter server_id must not be empty") } + if len(act.Data.Recipients.To) == 0 { + return nil, fmt.Errorf("parameter recipients.to must not be empty") + } emailMsg := util.MapStr{ "server_id": act.Data.ServerID, "email": act.Data.Recipients.To, diff --git a/service/alerting/common/helper.go b/service/alerting/common/helper.go index 6acff481..daf743d0 100644 --- a/service/alerting/common/helper.go +++ b/service/alerting/common/helper.go @@ -30,10 +30,14 @@ package common import ( "bytes" "fmt" + "infini.sh/console/model" "infini.sh/console/model/alerting" "infini.sh/console/service/alerting/action" "infini.sh/console/service/alerting/funcs" "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "net/url" + "strings" "text/template" ) @@ -58,13 +62,20 @@ func PerformChannel(channel *alerting.Channel, ctx map[string]interface{}) ([]by if err != nil { return nil, err, message } - wh.URL = string(urlBytes) + wh.URL, err = validateRenderedWebhookURL(string(urlBytes)) + if err != nil { + return nil, err, message + } act = &action.WebhookAction{ Data: &wh, Message: string(message), } case alerting.ChannelEmail: - message, err = ResolveMessage(channel.Email.Body, ctx) + if channel.Email == nil { + return nil, fmt.Errorf("empty email channel config"), nil + } + emailCtx := buildEmailTemplateContext(ctx) + message, err = ResolveMessage(channel.Email.Body, emailCtx) if err != nil { return nil, err, message } @@ -72,6 +83,14 @@ func PerformChannel(channel *alerting.Channel, ctx map[string]interface{}) ([]by if err != nil { return nil, err, nil } + err = ensureEmailServerID(channel.Email) + if err != nil { + return nil, err, message + } + err = ensureEmailRecipients(channel.Email) + if err != nil { + return nil, err, message + } act = &action.EmailAction{ Data: channel.Email, Subject: string(subjectBytes), @@ -84,6 +103,43 @@ func PerformChannel(channel *alerting.Channel, ctx map[string]interface{}) ([]by return executeResult, err, message } +func buildEmailTemplateContext(ctx map[string]interface{}) map[string]interface{} { + if ctx == nil { + return nil + } + emailCtx := make(map[string]interface{}, len(ctx)) + for k, v := range ctx { + emailCtx[k] = v + } + for _, key := range []string{"message", "recovery_context"} { + value, ok := emailCtx[key].(string) + if !ok || value == "" { + continue + } + value = strings.ReplaceAll(value, "\r\n", "\n") + emailCtx[key] = strings.ReplaceAll(value, "\n", "
") + } + return emailCtx +} + +func validateRenderedWebhookURL(raw string) (string, error) { + rendered := strings.TrimSpace(raw) + if rendered == "" || rendered == "" { + return "", fmt.Errorf("invalid webhook url: rendered value is empty") + } + parsed, err := url.Parse(rendered) + if err != nil { + return "", fmt.Errorf("invalid webhook url: %w", err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", fmt.Errorf("invalid webhook url: unsupported scheme %q", parsed.Scheme) + } + if parsed.Host == "" { + return "", fmt.Errorf("invalid webhook url: missing host") + } + return rendered, nil +} + func ResolveMessage(messageTemplate string, ctx map[string]interface{}) ([]byte, error) { msg := messageTemplate tmpl, err := template.New("alert-message").Funcs(funcs.GenericFuncMap()).Parse(msg) @@ -102,7 +158,7 @@ func RetrieveChannel(ch *alerting.Channel, raiseChannelEnabledErr bool) (*alerti if ch.ID != "" { refCh := &alerting.Channel{} refCh.ID = ch.ID - _, err := orm.Get(refCh) + _, err := orm.GetV2(orm.NewContext(), refCh) if err != nil { return nil, err } @@ -134,3 +190,66 @@ func RetrieveChannel(ch *alerting.Channel, raiseChannelEnabledErr bool) (*alerti } return ch, nil } + +func ensureEmailServerID(email *alerting.Email) error { + if email == nil { + return fmt.Errorf("empty email channel config") + } + if email.ServerID != "" { + return nil + } + serverID, err := getFallbackEmailServerID() + if err != nil { + return err + } + email.ServerID = serverID + return nil +} + +func ensureEmailRecipients(email *alerting.Email) error { + if email == nil { + return fmt.Errorf("empty email channel config") + } + if len(email.Recipients.To) == 0 { + return fmt.Errorf("email channel recipients are empty, please configure at least one recipient") + } + return nil +} + +func getFallbackEmailServerID() (string, error) { + servers, err := getEnabledEmailServers() + if err != nil { + return "", err + } + return selectFallbackEmailServerID(servers) +} + +func selectFallbackEmailServerID(servers []model.EmailServer) (string, error) { + switch len(servers) { + case 0: + return "", fmt.Errorf("parameter server_id must not be empty and no enabled smtp server is available") + case 1: + return servers[0].ID, nil + default: + return "", fmt.Errorf("parameter server_id must not be empty and multiple enabled smtp servers were found") + } +} + +func getEnabledEmailServers() ([]model.EmailServer, error) { + q := &orm.Query{ + Size: 100, + } + q.Conds = orm.And(orm.Eq("enabled", true)) + err, result := orm.Search(model.EmailServer{}, q) + if err != nil { + return nil, err + } + servers := make([]model.EmailServer, 0, len(result.Result)) + for _, row := range result.Result { + server := model.EmailServer{} + buf := util.MustToJSONBytes(row) + util.MustFromJSONBytes(buf, &server) + servers = append(servers, server) + } + return servers, nil +} diff --git a/service/alerting/common/helper_test.go b/service/alerting/common/helper_test.go new file mode 100644 index 00000000..e4bfdf9d --- /dev/null +++ b/service/alerting/common/helper_test.go @@ -0,0 +1,115 @@ +package common + +import ( + "strings" + "testing" + + "infini.sh/console/model" + modelalerting "infini.sh/console/model/alerting" +) + +func TestSelectFallbackEmailServerID(t *testing.T) { + server := model.EmailServer{} + server.ID = "smtp-1" + + serverID, err := selectFallbackEmailServerID([]model.EmailServer{ + server, + }) + if err != nil { + t.Fatalf("expected single enabled server to be selected, got error: %v", err) + } + if serverID != "smtp-1" { + t.Fatalf("expected smtp-1, got %q", serverID) + } +} + +func TestSelectFallbackEmailServerIDWithoutEnabledServer(t *testing.T) { + _, err := selectFallbackEmailServerID(nil) + if err == nil { + t.Fatal("expected error when no enabled smtp server exists") + } + if !strings.Contains(err.Error(), "no enabled smtp server") { + t.Fatalf("expected missing-server hint, got %v", err) + } +} + +func TestSelectFallbackEmailServerIDWithMultipleEnabledServers(t *testing.T) { + server1 := model.EmailServer{} + server1.ID = "smtp-1" + server2 := model.EmailServer{} + server2.ID = "smtp-2" + + _, err := selectFallbackEmailServerID([]model.EmailServer{ + server1, + server2, + }) + if err == nil { + t.Fatal("expected error when multiple enabled smtp servers exist") + } + if !strings.Contains(err.Error(), "multiple enabled smtp servers") { + t.Fatalf("expected multi-server hint, got %v", err) + } +} + +func TestEnsureEmailRecipients(t *testing.T) { + err := ensureEmailRecipients(nil) + if err == nil { + t.Fatal("expected nil email config to fail") + } + + email := &modelalerting.Email{} + err = ensureEmailRecipients(email) + if err == nil { + t.Fatal("expected missing recipients to fail") + } + if !strings.Contains(err.Error(), "at least one recipient") { + t.Fatalf("expected recipient hint, got %v", err) + } + + email.Recipients.To = []string{"ops@example.com"} + if err = ensureEmailRecipients(email); err != nil { + t.Fatalf("expected configured recipients to pass, got %v", err) + } +} + +func TestValidateRenderedWebhookURL(t *testing.T) { + validURL, err := validateRenderedWebhookURL(" https://open.feishu.cn/open-apis/bot/v2/hook/abc ") + if err != nil { + t.Fatalf("expected valid webhook url, got %v", err) + } + if validURL != "https://open.feishu.cn/open-apis/bot/v2/hook/abc" { + t.Fatalf("expected trimmed webhook url, got %q", validURL) + } + + _, err = validateRenderedWebhookURL("") + if err == nil || !strings.Contains(err.Error(), "rendered value is empty") { + t.Fatalf("expected empty-rendered-value error, got %v", err) + } + + _, err = validateRenderedWebhookURL("open.feishu.cn/hook/abc") + if err == nil || !strings.Contains(err.Error(), "unsupported scheme") { + t.Fatalf("expected unsupported-scheme error, got %v", err) + } +} + +func TestBuildEmailTemplateContextConvertsLineBreaks(t *testing.T) { + input := map[string]interface{}{ + "message": "节点: node-01\n节点: node-02", + "recovery_context": "Node: a\r\nNode: b", + "title": "unchanged", + } + + got := buildEmailTemplateContext(input) + if got["message"] != "节点: node-01
节点: node-02" { + t.Fatalf("expected message line breaks to be converted, got %q", got["message"]) + } + if got["recovery_context"] != "Node: a
Node: b" { + t.Fatalf("expected recovery_context line breaks to be converted, got %q", got["recovery_context"]) + } + if got["title"] != "unchanged" { + t.Fatalf("expected unrelated fields unchanged, got %q", got["title"]) + } + if input["message"] != "节点: node-01\n节点: node-02" { + t.Fatalf("expected input ctx to remain unchanged, got %q", input["message"]) + } +} diff --git a/service/alerting/constants.go b/service/alerting/constants.go index a367a87f..bd16212e 100644 --- a/service/alerting/constants.go +++ b/service/alerting/constants.go @@ -32,23 +32,26 @@ const ( KVLastTermStartTime = "alert_last_term_start_time" KVLastEscalationTime = "alert_last_escalation_time" KVLastMessageState = "alert_last_message_state" + KVLastConditionResults = "alert_last_condition_results" ) const ( - ParamRuleID = "rule_id" //规则 UUID - ParamResourceID = "resource_id" // 资源 UUID - ParamResourceName = "resource_name" // 资源名称 如集群名称 es-v714 - ParamEventID = "event_id" // 检查事件 ID - ParamResults = "results" // - ParamMessage = "message" //检查消息 自定义(模版渲染) - ParamTitle = "title" - ParamThreshold = "threshold" //检查预设值 []string - ParamResultValue = "result_value" //检查结果 {group_tags:["cluster-xxx", "node-xxx"], check_values:[]} - Priority = "priority" //告警等级 - ParamTimestamp = "timestamp" //事件产生时间戳 - ParamGroupValues = "group_values" - ParamIssueTimestamp = "issue_timestamp" - ParamRelationValues = "relation_values" + ParamRuleID = "rule_id" //规则 UUID + ParamResourceID = "resource_id" // 资源 UUID + ParamResourceName = "resource_name" // 资源名称 如集群名称 es-v714 + ParamEventID = "event_id" // 检查事件 ID + ParamResults = "results" // + ParamTotalResults = "total_results" // 未截断的命中总数 + ParamMessage = "message" //检查消息 自定义(模版渲染) + ParamTitle = "title" + ParamThreshold = "threshold" //检查预设值 []string + ParamResultValue = "result_value" //检查结果 {group_tags:["cluster-xxx", "node-xxx"], check_values:[]} + Priority = "priority" //告警等级 + ParamTimestamp = "timestamp" //事件产生时间戳 + ParamGroupValues = "group_values" + ParamIssueTimestamp = "issue_timestamp" + ParamRecoveredResults = "recovered_results" + ParamRelationValues = "relation_values" // rule expression, rule_id, resource_id, resource_name, event_id, condition_name, preset_value,[group_tags, check_values], // check_status ,timestamp, diff --git a/service/alerting/elasticsearch/engine.go b/service/alerting/elasticsearch/engine.go index d1b70d24..3cc15add 100644 --- a/service/alerting/elasticsearch/engine.go +++ b/service/alerting/elasticsearch/engine.go @@ -54,6 +54,144 @@ import ( type Engine struct { } +const alertLogSampleLimit = 3 + +func formatAlertDuration(duration time.Duration) string { + if duration < 0 { + duration = -duration + } + duration = duration.Truncate(time.Second) + if duration <= 0 { + return "0s" + } + + totalSeconds := int64(duration / time.Second) + days := totalSeconds / (24 * 60 * 60) + totalSeconds %= 24 * 60 * 60 + hours := totalSeconds / (60 * 60) + totalSeconds %= 60 * 60 + minutes := totalSeconds / 60 + seconds := totalSeconds % 60 + + var parts []string + if days > 0 { + parts = append(parts, fmt.Sprintf("%dd", days)) + } + if hours > 0 { + parts = append(parts, fmt.Sprintf("%dh", hours)) + } + if minutes > 0 { + parts = append(parts, fmt.Sprintf("%dm", minutes)) + } + if seconds > 0 || len(parts) == 0 { + parts = append(parts, fmt.Sprintf("%ds", seconds)) + } + return strings.Join(parts, "") +} + +func summarizeAlertCondition(cond *alerting.ConditionItem) string { + if cond == nil { + return "" + } + + expression := cond.Expression + if len(expression) > 96 { + expression = expression[:93] + "..." + } + + return fmt.Sprintf( + "period=%d operator=%s values=%v priority=%s type=%s bucket_count=%d expression=%q", + cond.MinimumPeriodMatch, + cond.Operator, + cond.Values, + cond.Priority, + cond.Type, + cond.BucketCount, + expression, + ) +} + +func summarizeAlertConditionResults(items []alerting.ConditionResultItem) string { + if len(items) == 0 { + return "matched=0" + } + + limit := alertLogSampleLimit + if len(items) < limit { + limit = len(items) + } + + samples := make([]string, 0, limit) + for i := 0; i < limit; i++ { + item := items[i] + samples = append(samples, fmt.Sprintf("groups=%v result=%v", item.GroupValues, item.ResultValue)) + } + + summary := fmt.Sprintf("matched=%d sample=[%s]", len(items), strings.Join(samples, "; ")) + if len(items) > limit { + summary += fmt.Sprintf(" ...(and %d more)", len(items)-limit) + } + + return summary +} + +func getConditionResultItemKey(item alerting.ConditionResultItem) string { + return strings.Join(item.GroupValues, "\x1f") +} + +func diffRecoveredConditionResultItems(previous, current []alerting.ConditionResultItem) []alerting.ConditionResultItem { + if len(previous) == 0 { + return nil + } + + currentKeys := make(map[string]struct{}, len(current)) + for _, item := range current { + currentKeys[getConditionResultItemKey(item)] = struct{}{} + } + + recovered := make([]alerting.ConditionResultItem, 0, len(previous)) + for _, item := range previous { + if _, ok := currentKeys[getConditionResultItemKey(item)]; ok { + continue + } + recovered = append(recovered, item) + } + return recovered +} + +func isIncrementalRecoveryEnabled(cfg *alerting.RecoveryNotificationConfig) bool { + return cfg != nil && cfg.Enabled && cfg.EventEnabled && cfg.IncrementalRecoveryEnabled +} + +func getLastConditionResults(ruleID string) ([]alerting.ConditionResultItem, error) { + if ruleID == "" { + return nil, nil + } + + value, err := kv.GetValue(alerting2.KVLastConditionResults, []byte(ruleID)) + if err != nil || len(value) == 0 { + return nil, err + } + + var items []alerting.ConditionResultItem + if err := util.FromJSONBytes(value, &items); err != nil { + return nil, err + } + return items, nil +} + +func saveLastConditionResults(ruleID string, items []alerting.ConditionResultItem) error { + if ruleID == "" { + return nil + } + + value, err := util.ToJSONBytes(items) + if err != nil { + return err + } + return kv.AddValue(alerting2.KVLastConditionResults, []byte(ruleID), value) +} + // GenerateQuery generate a final elasticsearch query dsl object // when RawFilter of rule is not empty, priority use it, otherwise to covert from Filter of rule (todo) // auto generate time filter query and then attach to final query @@ -283,7 +421,40 @@ func getQueryTimeRange(rule *alerting.Rule, filterParam *alerting.FilterParam) ( return timeStart, timeEnd } +func shouldIgnoreTimeFilter(rule *alerting.Rule) bool { + if rule == nil { + return false + } + if rule.Resource.IgnoreTimeFilter { + return true + } + hasNodeIndex := false + for _, object := range rule.Resource.Objects { + if object == ".infini_node" { + hasNodeIndex = true + break + } + } + if !hasNodeIndex { + return false + } + matchPhrase, ok := rule.Resource.RawFilter["match_phrase"].(map[string]interface{}) + if !ok { + return false + } + status, ok := matchPhrase["metadata.labels.status"] + if !ok { + return false + } + return status == "unavailable" +} + func (engine *Engine) generateTimeFilter(rule *alerting.Rule, filterParam *alerting.FilterParam) (map[string]interface{}, error) { + if shouldIgnoreTimeFilter(rule) { + return util.MapStr{ + "match_all": util.MapStr{}, + }, nil + } timeStart, timeEnd := getQueryTimeRange(rule, filterParam) timeQuery := util.MapStr{ "range": util.MapStr{ @@ -536,25 +707,12 @@ func (engine *Engine) CheckCondition(rule *alerting.Rule) (*alerting.ConditionRe LoopData: for i := 0; i < dataLength; i++ { //clear nil value - if targetData.Data[dataKey][i].Value == nil { + if isInvalidMetricValue(targetData.Data[dataKey][i].Value) { continue } - if r, ok := targetData.Data[dataKey][i].Value.(float64); ok { - if math.IsNaN(r) { - continue - } - } - relationValues := map[string]interface{}{} - for _, metric := range rule.Metrics.Items { - md := queryResult.MetricData[idx] - if _, ok := md.Data[metric.Name]; !ok || len(md.Data[metric.Name]) < i { - if global.Env().IsDebug { - log.Debugf("metric data %s not found in query result", metric.Name) - } - // skip this data point - continue LoopData - } - relationValues[metric.Name] = md.Data[metric.Name][i].Value + relationValues, ok := getRelationValues(queryResult, rule.Metrics.Items, idx, i) + if !ok { + continue LoopData } valueExpressionResult, err := valueExpression.Evaluate(relationValues) if err != nil { @@ -575,7 +733,7 @@ func (engine *Engine) CheckCondition(rule *alerting.Rule) (*alerting.ConditionRe triggerCount = 0 } if triggerCount >= cond.MinimumPeriodMatch { - log.Debugf("triggered condition %v, groups: %v\n", cond, targetData.Groups) + log.Tracef("triggered condition: %s, groups=%v", summarizeAlertCondition(&cond), targetData.Groups) // collect group values grpValues := make([]string, 0, len(targetData.Groups)) for _, v := range targetData.Groups { @@ -601,6 +759,42 @@ func (engine *Engine) CheckCondition(rule *alerting.Rule) (*alerting.ConditionRe return conditionResult, nil } +func getRelationValues(queryResult *alerting.QueryResult, metrics []insight.MetricItem, metricIndex int, pointIndex int) (map[string]interface{}, bool) { + if queryResult == nil || metricIndex < 0 || metricIndex >= len(queryResult.MetricData) { + return nil, false + } + + md := queryResult.MetricData[metricIndex] + relationValues := map[string]interface{}{} + for _, metric := range metrics { + values, ok := md.Data[metric.Name] + if !ok || len(values) <= pointIndex { + if global.Env().IsDebug { + log.Debugf("metric data %s not found in query result", metric.Name) + } + return nil, false + } + if isInvalidMetricValue(values[pointIndex].Value) { + if global.Env().IsDebug { + log.Debugf("metric data %s is invalid in query result, point_index=%d", metric.Name, pointIndex) + } + return nil, false + } + relationValues[metric.Name] = values[pointIndex].Value + } + return relationValues, true +} + +func isInvalidMetricValue(value interface{}) bool { + if value == nil { + return true + } + if r, ok := value.(float64); ok { + return math.IsNaN(r) || math.IsInf(r, 0) + } + return false +} + type BucketDiffState struct { ContentChangeState int DocCount int @@ -731,7 +925,7 @@ func (engine *Engine) CheckBucketCondition(rule *alerting.Rule, targetMetricData } if triggerCount >= cond.MinimumPeriodMatch { groupValues := strings.Split(grps, "*") - log.Debugf("triggered condition %v, groups: %v\n", cond, groupValues) + log.Tracef("triggered condition: %s, groups=%v", summarizeAlertCondition(&cond), groupValues) resultItem := alerting.ConditionResultItem{ GroupValues: groupValues, ConditionItem: &cond, @@ -753,8 +947,10 @@ func (engine *Engine) CheckBucketCondition(rule *alerting.Rule, targetMetricData func (engine *Engine) Do(rule *alerting.Rule) error { var ( - alertItem *alerting.Alert - err error + alertItem *alerting.Alert + persistLastConditionItems bool + lastConditionItems []alerting.ConditionResultItem + err error ) defer func() { if err != nil && alertItem == nil { @@ -786,9 +982,14 @@ func (engine *Engine) Do(rule *alerting.Rule) error { } } - err = orm.Save(nil, alertItem) + err = saveAlertItemToES(alertItem) if err != nil { - log.Error(err) + log.Errorf("save alert item failed, rule_id=%s, alert_id=%s, state=%s: %v", rule.ID, alertItem.ID, alertItem.State, err) + } + } + if persistLastConditionItems { + if err := saveLastConditionResults(rule.ID, lastConditionItems); err != nil { + log.Errorf("save last condition results failed, rule_id=%s: %v", rule.ID, err) } } }() @@ -822,7 +1023,16 @@ func (engine *Engine) Do(rule *alerting.Rule) error { if err != nil { return fmt.Errorf("get alert message error: %w", err) } + previousConditionResults, err := getLastConditionResults(rule.ID) + if err != nil { + return fmt.Errorf("get last condition results error: %w", err) + } conditionResults := checkResults.ResultItems + recoveredConditionResults := diffRecoveredConditionResultItems(previousConditionResults, conditionResults) + if !checkResults.QueryResult.Nodata { + persistLastConditionItems = true + lastConditionItems = conditionResults + } var paramsCtx map[string]interface{} if len(conditionResults) == 0 { alertItem.Priority = "" @@ -831,35 +1041,29 @@ func (engine *Engine) Do(rule *alerting.Rule) error { } if alertMessage != nil && alertMessage.Status != alerting.MessageStateRecovered && !checkResults.QueryResult.Nodata { - alertMessage.Status = alerting.MessageStateRecovered - alertMessage.ResourceID = rule.Resource.ID - alertMessage.ResourceName = rule.Resource.Name - err = saveAlertMessage(alertMessage) - if err != nil { - return fmt.Errorf("save alert message error: %w", err) - } - // todo add recover notification to inner system message - // send recover message to channel recoverCfg := rule.RecoveryNotificationConfig if recoverCfg != nil && recoverCfg.EventEnabled && recoverCfg.Enabled { - paramsCtx = newParameterCtx(rule, checkResults, util.MapStr{ - alerting2.ParamEventID: alertMessage.ID, - alerting2.ParamTimestamp: alertItem.Created.Unix(), - "duration": alertItem.Created.Sub(alertMessage.Created).String(), - "trigger_at": alertMessage.Created.Unix(), - }) + paramsCtx = buildRecoveryNotificationParams(rule, checkResults, alertMessage, alertItem) err = attachTitleMessageToCtx(recoverCfg.Title, recoverCfg.Message, paramsCtx) if err != nil { - return err + return fmt.Errorf("resolve recovery notification template for rule [%s] error: %w", rule.ID, err) } - actionResults, _ := performChannels(recoverCfg.Normal, paramsCtx, false) + actionResults, _ := performChannels(rule.ID, "recovery_notification", recoverCfg.Normal, paramsCtx, false) alertItem.RecoverActionResults = actionResults - //clear history notification time - rule.LastNotificationTime = time.Time{} - rule.LastEscalationTime = time.Time{} - _ = kv.DeleteKey(alerting2.KVLastNotificationTime, []byte(rule.ID)) - _ = kv.DeleteKey(alerting2.KVLastEscalationTime, []byte(rule.ID)) } + alertMessage.Status = alerting.MessageStateRecovered + alertMessage.ResourceID = rule.Resource.ID + alertMessage.ResourceName = rule.Resource.Name + alertMessage.RecoveredAt = alertItem.Created + err = saveAlertMessage(alertMessage) + if err != nil { + return fmt.Errorf("save alert message error: %w", err) + } + //clear history notification time + rule.LastNotificationTime = time.Time{} + rule.LastEscalationTime = time.Time{} + _ = kv.DeleteKey(alerting2.KVLastNotificationTime, []byte(rule.ID)) + _ = kv.DeleteKey(alerting2.KVLastEscalationTime, []byte(rule.ID)) } return nil } @@ -874,12 +1078,9 @@ func (engine *Engine) Do(rule *alerting.Rule) error { } } triggerAt := alertItem.Created - if alertMessage != nil { - triggerAt = alertMessage.Created - } paramsCtx = newParameterCtx(rule, checkResults, util.MapStr{ alerting2.ParamTimestamp: alertItem.Created.Unix(), - "duration": alertItem.Created.Sub(triggerAt).String(), + "duration": formatAlertDuration(alertItem.Created.Sub(triggerAt)), "trigger_at": triggerAt.Unix(), }) @@ -902,10 +1103,28 @@ func (engine *Engine) Do(rule *alerting.Rule) error { } else { paramsCtx[alerting2.ParamEventID] = alertMessage.ID } + if alertMessage != nil && alertMessage.Status != alerting.MessageStateRecovered && len(recoveredConditionResults) > 0 { + recoverCfg := rule.RecoveryNotificationConfig + if isIncrementalRecoveryEnabled(recoverCfg) { + recoveryContext, recoveredResults, err := buildRecoveryContext(rule, checkResults.QueryResult, recoveredConditionResults) + if err != nil { + return fmt.Errorf("build partial recovery context for rule [%s] error: %w", rule.ID, err) + } + recoveryParamsCtx := buildRecoveryNotificationParams(rule, checkResults, alertMessage, alertItem) + recoveryParamsCtx[alerting2.ParamRecoveredResults] = recoveredResults + recoveryParamsCtx["recovery_context"] = recoveryContext + err = attachTitleMessageToCtx(recoverCfg.Title, recoverCfg.Message, recoveryParamsCtx) + if err != nil { + return fmt.Errorf("resolve partial recovery template for rule [%s] error: %w", rule.ID, err) + } + actionResults, _ := performChannels(rule.ID, "recovery_notification", recoverCfg.Normal, recoveryParamsCtx, false) + alertItem.RecoverActionResults = actionResults + } + } title, message := rule.GetNotificationTitleAndMessage() err = attachTitleMessageToCtx(title, message, paramsCtx) if err != nil { - return err + return fmt.Errorf("resolve notification template for rule [%s] error: %w", rule.ID, err) } alertItem.Message = paramsCtx[alerting2.ParamMessage].(string) alertItem.Title = paramsCtx[alerting2.ParamTitle].(string) @@ -913,6 +1132,9 @@ func (engine *Engine) Do(rule *alerting.Rule) error { alertMessage = newAlertMessage alertMessage.Title = alertItem.Title alertMessage.Message = alertItem.Message + if alertMessage != nil { + alertMessage.Updated = alertMessage.Created + } err = saveAlertMessage(newAlertMessage) if err != nil { return fmt.Errorf("save alert message error: %w", err) @@ -928,9 +1150,9 @@ func (engine *Engine) Do(rule *alerting.Rule) error { Status: model.NotificationStatusNew, Title: alertItem.Title, Body: alertItem.Message, - Link: "/alerting/message", + Link: fmt.Sprintf("/alerting/message/%s", alertMessage.ID), } - err = orm.Create(nil, notification) + err = orm.Create(orm.NewContext(), notification) if err != nil { return fmt.Errorf("failed to create notification, err: %w", err) } @@ -945,7 +1167,7 @@ func (engine *Engine) Do(rule *alerting.Rule) error { return fmt.Errorf("save alert message error: %w", err) } } - log.Debugf("check condition result of rule %s is %v", conditionResults, rule.ID) + log.Tracef("check condition result of rule %s: %s", rule.ID, summarizeAlertConditionResults(conditionResults)) // if alert message status equals ignored , then skip sending message to channel if alertMessage.Status == alerting.MessageStateIgnored { @@ -982,7 +1204,7 @@ func (engine *Engine) Do(rule *alerting.Rule) error { paramsCtx = newParameterCtx(rule, checkResults, util.MapStr{ alerting2.ParamTimestamp: alertItem.Created.Unix(), "priority": priority, - "duration": alertItem.Created.Sub(alertMessage.Created).String(), + "duration": formatAlertDuration(alertItem.Created.Sub(alertMessage.Created)), "trigger_at": alertMessage.Created.Unix(), }) if alertMessage != nil { @@ -991,7 +1213,7 @@ func (engine *Engine) Do(rule *alerting.Rule) error { } if alertMessage == nil || period > periodDuration { - actionResults, _ := performChannels(notifyCfg.Normal, paramsCtx, false) + actionResults, _ := performChannels(rule.ID, "notification", notifyCfg.Normal, paramsCtx, false) alertItem.ActionExecutionResults = actionResults //change and save last notification time in local kv store when action error count equals zero rule.LastNotificationTime = time.Now() @@ -1021,7 +1243,7 @@ func (engine *Engine) Do(rule *alerting.Rule) error { } } if time.Now().Sub(rule.LastEscalationTime.Local()) > periodDuration { - actionResults, _ := performChannels(notifyCfg.Escalation, paramsCtx, false) + actionResults, _ := performChannels(rule.ID, "escalation", notifyCfg.Escalation, paramsCtx, false) alertItem.EscalationActionResults = actionResults //todo init last escalation time when create task (by last alert item is escalated) rule.LastEscalationTime = time.Now() @@ -1037,23 +1259,114 @@ func (engine *Engine) Do(rule *alerting.Rule) error { } func attachTitleMessageToCtx(title, message string, paramsCtx map[string]interface{}) error { - var ( - tplBytes []byte - err error - ) - tplBytes, err = common.ResolveMessage(message, paramsCtx) + resolvedMessage, err := resolveAlertTemplateText(message, paramsCtx) if err != nil { return fmt.Errorf("resolve message template error: %w", err) } - paramsCtx[alerting2.ParamMessage] = string(tplBytes) - tplBytes, err = common.ResolveMessage(title, paramsCtx) + + resolvedTitle, err := resolveAlertTemplateText(title, paramsCtx) if err != nil { return fmt.Errorf("resolve title template error: %w", err) } - paramsCtx[alerting2.ParamTitle] = string(tplBytes) + resolvedMessage = stripDuplicatedNonLeadingTitleLine(resolvedMessage, resolvedTitle) + paramsCtx[alerting2.ParamMessage] = resolvedMessage + paramsCtx[alerting2.ParamTitle] = resolvedTitle return nil } +func resolveAlertTemplateText(template string, paramsCtx map[string]interface{}) (string, error) { + tplBytes, err := common.ResolveMessage(template, paramsCtx) + if err != nil { + return "", err + } + return normalizeAlertTemplateText(string(tplBytes)), nil +} + +func normalizeAlertTemplateText(text string) string { + text = strings.ReplaceAll(text, "\r\n", "\n") + lines := strings.Split(text, "\n") + result := make([]string, 0, len(lines)) + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + line = collapseDuplicatedLeadingAlertEmoji(line) + result = append(result, splitRepeatedMetricEntryLine(line)...) + } + return strings.Join(result, "\n") +} + +func splitRepeatedMetricEntryLine(line string) []string { + for _, prefix := range []string{"节点:", "Node:"} { + if strings.Count(line, prefix) <= 1 { + continue + } + parts := strings.Split(line, prefix) + entries := make([]string, 0, len(parts)) + for i, part := range parts { + part = strings.TrimSpace(part) + if i == 0 { + if part != "" { + entries = append(entries, part) + } + continue + } + if part == "" { + continue + } + entries = append(entries, prefix+" "+part) + } + if len(entries) > 1 { + return entries + } + } + return []string{line} +} + +func collapseDuplicatedLeadingAlertEmoji(line string) string { + trimmed := strings.TrimLeft(line, " \t") + for _, emoji := range []string{"🌈", "🔥"} { + double := emoji + " " + emoji + if strings.HasPrefix(trimmed, double) { + trimmed = emoji + strings.TrimPrefix(trimmed, emoji+emoji) + trimmed = strings.Replace(trimmed, double, emoji, 1) + } + for strings.HasPrefix(trimmed, emoji+emoji) { + trimmed = emoji + strings.TrimPrefix(trimmed, emoji+emoji) + } + for strings.HasPrefix(trimmed, emoji+"\t"+emoji) { + trimmed = emoji + strings.TrimPrefix(trimmed, emoji+"\t"+emoji) + } + } + return strings.Repeat(" ", len(line)-len(strings.TrimLeft(line, " "))) + trimmed +} + +func stripDuplicatedNonLeadingTitleLine(message, title string) string { + if title == "" || message == "" { + return message + } + normalizedTitle := strings.TrimSpace(title) + if normalizedTitle == "" { + return message + } + lines := strings.Split(message, "\n") + filtered := make([]string, 0, len(lines)) + seenNonEmptyPrefix := false + removed := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if !removed && seenNonEmptyPrefix && trimmed == normalizedTitle { + removed = true + continue + } + if trimmed != "" { + seenNonEmptyPrefix = true + } + filtered = append(filtered, line) + } + return strings.Join(filtered, "\n") +} + func newParameterCtx(rule *alerting.Rule, checkResults *alerting.ConditionResult, extraParams map[string]interface{}) map[string]interface{} { var ( conditionParams []util.MapStr @@ -1098,28 +1411,15 @@ func newParameterCtx(rule *alerting.Rule, checkResults *alerting.ConditionResult } envVariables, err := alerting2.GetEnvVariables() if err != nil { - log.Errorf("get env variables error: %v", err) - } - var ( - min interface{} - max interface{} - ) - if checkResults.QueryResult != nil { - min = checkResults.QueryResult.Min - max = checkResults.QueryResult.Max - if v, ok := min.(int64); ok { - //expand 60s - min = time.UnixMilli(v).Add(-time.Second * 60).UTC().Format("2006-01-02T15:04:05.999Z") - } - if v, ok := max.(int64); ok { - max = time.UnixMilli(v).Add(time.Second * 60).UTC().Format("2006-01-02T15:04:05.999Z") - } + log.Errorf("get env variables error, rule_id=%s: %v", rule.ID, err) } + min, max := resolveTemplateTimeRange(checkResults) paramsCtx := util.MapStr{ alerting2.ParamRuleID: rule.ID, alerting2.ParamResourceID: rule.Resource.ID, alerting2.ParamResourceName: rule.Resource.Name, alerting2.ParamResults: conditionParams, + alerting2.ParamTotalResults: len(checkResults.ResultItems), "objects": rule.Resource.Objects, "first_group_value": firstGroupValue, "first_threshold": firstThreshold, @@ -1131,11 +1431,84 @@ func newParameterCtx(rule *alerting.Rule, checkResults *alerting.ConditionResult } err = util.MergeFields(paramsCtx, extraParams, true) if err != nil { - log.Errorf("merge template params error: %v", err) + log.Errorf("merge template params error, rule_id=%s: %v", rule.ID, err) } return paramsCtx } +func resolveTemplateTimeRange(checkResults *alerting.ConditionResult) (string, string) { + if checkResults != nil && checkResults.QueryResult != nil { + if minTimestamp, ok := parseAlertTimestampMillis(checkResults.QueryResult.Min); ok { + if maxTimestamp, ok := parseAlertTimestampMillis(checkResults.QueryResult.Max); ok { + return formatAlertTimestampMillis(minTimestamp - 60*1000), formatAlertTimestampMillis(maxTimestamp + 60*1000) + } + } + } + + if checkResults != nil && len(checkResults.ResultItems) > 0 { + if issueTimestamp, ok := parseAlertTimestampMillis(checkResults.ResultItems[0].IssueTimestamp); ok { + return formatAlertTimestampMillis(issueTimestamp - 60*1000), formatAlertTimestampMillis(issueTimestamp + 60*1000) + } + } + + now := time.Now().UTC().UnixMilli() + return formatAlertTimestampMillis(now - 5*60*1000), formatAlertTimestampMillis(now + 60*1000) +} + +func parseAlertTimestampMillis(value interface{}) (int64, bool) { + switch v := value.(type) { + case int64: + return v, true + case int: + return int64(v), true + case int32: + return int64(v), true + case float64: + return int64(v), true + case float32: + return int64(v), true + case string: + parsed, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64) + if err != nil { + return 0, false + } + return parsed, true + default: + return 0, false + } +} + +func formatAlertTimestampMillis(timestamp int64) string { + return time.UnixMilli(timestamp).UTC().Format("2006-01-02T15:04:05.999Z") +} + +func buildRecoveryContext(rule *alerting.Rule, queryResult *alerting.QueryResult, recoveredItems []alerting.ConditionResultItem) (string, interface{}, error) { + if len(recoveredItems) == 0 { + return "", nil, nil + } + + recoveredCheckResults := &alerting.ConditionResult{ + ResultItems: recoveredItems, + QueryResult: queryResult, + } + recoveredCtx := newParameterCtx(rule, recoveredCheckResults, nil) + _, messageTemplate := rule.GetNotificationTitleAndMessage() + recoveryContext, err := resolveAlertTemplateText(messageTemplate, recoveredCtx) + if err != nil { + return "", nil, err + } + return recoveryContext, recoveredCtx[alerting2.ParamResults], nil +} + +func buildRecoveryNotificationParams(rule *alerting.Rule, checkResults *alerting.ConditionResult, alertMessage *alerting.AlertMessage, alertItem *alerting.Alert) map[string]interface{} { + return newParameterCtx(rule, checkResults, util.MapStr{ + alerting2.ParamEventID: alertMessage.ID, + alerting2.ParamTimestamp: alertItem.Created.Unix(), + "duration": formatAlertDuration(alertItem.Created.Sub(alertMessage.Created)), + "trigger_at": alertMessage.Created.Unix(), + }) +} + func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.ActionExecutionResult, error) { checkResults, err := engine.CheckCondition(rule) if err != nil { @@ -1155,14 +1528,14 @@ func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.Acti paramsCtx := newParameterCtx(rule, checkResults, util.MapStr{ alerting2.ParamEventID: util.GetUUID(), alerting2.ParamTimestamp: now.Unix(), - "duration": now.Sub(triggerAt).String(), + "duration": formatAlertDuration(now.Sub(triggerAt)), "trigger_at": triggerAt.Unix(), }) if msgType == "escalation" || msgType == "notification" { title, message := rule.GetNotificationTitleAndMessage() err = attachTitleMessageToCtx(title, message, paramsCtx) if err != nil { - return nil, err + return nil, fmt.Errorf("resolve %s template for rule [%s] error: %w", msgType, rule.ID, err) } } else if msgType == "recover_notification" { if rule.RecoveryNotificationConfig == nil { @@ -1170,7 +1543,7 @@ func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.Acti } err = attachTitleMessageToCtx(rule.RecoveryNotificationConfig.Title, rule.RecoveryNotificationConfig.Message, paramsCtx) if err != nil { - return nil, err + return nil, fmt.Errorf("resolve recovery notification template for rule [%s] error: %w", rule.ID, err) } } else { return nil, fmt.Errorf("unkonwn parameter msg type") @@ -1196,14 +1569,14 @@ func (engine *Engine) Test(rule *alerting.Rule, msgType string) ([]alerting.Acti channels = notifyCfg.Normal } if len(channels) > 0 { - actionResults, _ = performChannels(channels, paramsCtx, true) + actionResults, _ = performChannels(rule.ID, msgType, channels, paramsCtx, true) } else { return nil, fmt.Errorf("no useable channel") } return actionResults, nil } -func performChannels(channels []alerting.Channel, ctx map[string]interface{}, raiseChannelEnabledErr bool) ([]alerting.ActionExecutionResult, int) { +func performChannels(ruleID string, phase string, channels []alerting.Channel, ctx map[string]interface{}, raiseChannelEnabledErr bool) ([]alerting.ActionExecutionResult, int) { var errCount int var actionResults []alerting.ActionExecutionResult for _, channel := range channels { @@ -1214,7 +1587,7 @@ func performChannels(channels []alerting.Channel, ctx map[string]interface{}, ra ) _, err := common.RetrieveChannel(&channel, raiseChannelEnabledErr) if err != nil { - log.Error(err) + log.Errorf("retrieve alert channel failed, rule_id=%s, phase=%s, channel_id=%s, channel_name=%s, channel_type=%s: %v", ruleID, phase, channel.ID, channel.Name, channel.Type, err) errCount++ errStr = err.Error() } else { @@ -1223,6 +1596,7 @@ func performChannels(channels []alerting.Channel, ctx map[string]interface{}, ra } resBytes, err, messageBytes = common.PerformChannel(&channel, ctx) if err != nil { + log.Errorf("perform alert channel failed, rule_id=%s, phase=%s, channel_id=%s, channel_name=%s, channel_type=%s: %v", ruleID, phase, channel.ID, channel.Name, channel.Type, err) errCount++ errStr = err.Error() } @@ -1245,14 +1619,14 @@ func (engine *Engine) GenerateTask(rule alerting.Rule) func(ctx context.Context) defer func() { if !global.Env().IsDebug { if err := recover(); err != nil { - log.Error(err) + log.Errorf("alert task panic recovered, rule_id=%s, rule_name=%s: %v", rule.ID, rule.Name, err) debug.PrintStack() } } }() err := engine.Do(&rule) if err != nil { - log.Error(err) + log.Errorf("execute alert rule failed, rule_id=%s, rule_name=%s: %v", rule.ID, rule.Name, err) } } } @@ -1313,7 +1687,18 @@ func getLastAlertMessage(ruleID string, duration time.Duration) (*alerting.Alert func saveAlertMessageToES(message *alerting.AlertMessage) error { message.Updated = time.Now() - return orm.Save(nil, message) + ctx := orm.NewContext().DirectAccess() + ctx.Refresh = orm.WaitForRefresh + ctx.Set(orm.CheckExistsBeforeUpdate, false) + ctx.Set(orm.MergePartialFieldsBeforeUpdate, false) + return orm.Save(ctx, message) +} + +func saveAlertItemToES(alertItem *alerting.Alert) error { + alertItem.Updated = time.Now() + ctx := orm.NewContext().DirectAccess() + ctx.Refresh = orm.WaitForRefresh + return orm.Create(ctx, alertItem) } func saveAlertMessage(message *alerting.AlertMessage) error { diff --git a/service/alerting/elasticsearch/engine_test.go b/service/alerting/elasticsearch/engine_test.go index 75bddc71..667c6031 100644 --- a/service/alerting/elasticsearch/engine_test.go +++ b/service/alerting/elasticsearch/engine_test.go @@ -31,11 +31,13 @@ import ( "fmt" "net/http" "sort" + "strings" "testing" "time" "infini.sh/console/core/insight" "infini.sh/console/model/alerting" + alerting2 "infini.sh/console/service/alerting" "infini.sh/framework/core/elastic" "infini.sh/framework/core/util" "infini.sh/framework/modules/elastic/adapter/elasticsearch" @@ -154,6 +156,309 @@ func TestGenerateAgg(t *testing.T) { fmt.Println(util.MustToJSON(agg)) } +func TestAttachTitleMessageToCtxRemovesBlankLines(t *testing.T) { + paramsCtx := map[string]interface{}{ + "priority": "critical", + "event_id": "evt-1", + "resource_name": "migrator-source", + "objects": []string{"migration-pmc"}, + "trigger_at": "2026-05-20 15:00:00", + "results": []util.MapStr{ + { + "group_values": []string{"migration-pmc"}, + "result_value": "92.82gb", + }, + }, + } + + err := attachTitleMessageToCtx( + "Alert: {{.resource_name}}", + `- Priority:{{.priority}} +- EventID: {{.event_id}} +- Target: {{.resource_name}}-{{.objects}} +- TriggerAt: {{.trigger_at}} + +{{range .results}} +Index: {{index .group_values 0}} of Cluster: {{$.resource_name}}, Max Shard Storage: {{.result_value}} +{{end}}`, + paramsCtx, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := paramsCtx[alerting2.ParamMessage].(string) + if strings.Contains(got, "\n\n") { + t.Fatalf("expected blank lines to be removed, got %q", got) + } + if !strings.Contains(got, "Index: migration-pmc of Cluster: migrator-source, Max Shard Storage: 92.82gb") { + t.Fatalf("expected rendered content, got %q", got) + } +} + +func TestAttachTitleMessageToCtxCollapsesDuplicatedLeadingEmoji(t *testing.T) { + paramsCtx := map[string]interface{}{ + "title": "🌈 [JVM utilization is Too High] Resolved", + } + + err := attachTitleMessageToCtx( + "{{.title}}", + `[ INFINI Platform Alerting ] +🌈 {{.title}}`, + paramsCtx, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := paramsCtx[alerting2.ParamMessage].(string) + if strings.Contains(got, "🌈 🌈") { + t.Fatalf("expected duplicated leading emoji to collapse, got %q", got) + } + if strings.Contains(got, "\n🌈 [JVM utilization is Too High] Resolved") { + t.Fatalf("expected duplicated non-leading title line to be removed, got %q", got) + } + if !strings.Contains(got, "[ INFINI Platform Alerting ]") { + t.Fatalf("expected remaining message content to stay, got %q", got) + } +} + +func TestAttachTitleMessageToCtxStripsDuplicatedNonLeadingTitleLine(t *testing.T) { + paramsCtx := map[string]interface{}{ + "title": "🔥 [Cluster Metrics Collection Anomaly] Alerting", + } + + err := attachTitleMessageToCtx( + "{{.title}}", + `🔥 Incident #d8h972r5aeeotbtl08ug is ongoing +{{.title}} +Priority: warning`, + paramsCtx, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := paramsCtx[alerting2.ParamMessage].(string) + if strings.Contains(got, "\n🔥 [Cluster Metrics Collection Anomaly] Alerting") { + t.Fatalf("expected duplicated title line to be removed from message, got %q", got) + } + if !strings.Contains(got, "🔥 Incident #d8h972r5aeeotbtl08ug is ongoing") { + t.Fatalf("expected incident summary to remain, got %q", got) + } + if !strings.Contains(got, "Priority: warning") { + t.Fatalf("expected remaining message content to remain, got %q", got) + } +} + +func TestAttachTitleMessageToCtxAppendsRecoveryContext(t *testing.T) { + paramsCtx := map[string]interface{}{ + "event_id": "evt-1", + "resource_name": "INFINI_SYSTEM", + "objects": []string{".infini_metrics*"}, + "trigger_at": "2026-06-04 10:45:25", + "timestamp": "2026-06-04 11:13:25", + "duration": "28m", + "recovery_context": "Node: es717 of Cluster: migrator-es717, JVM Usage: 85.9%", + } + + err := attachTitleMessageToCtx( + "🌈 [resolved]", + `EventID: {{.event_id}} +Target: {{.resource_name}}-{{.objects}} +TriggerAt: {{.trigger_at}} +ResolveAt: {{.timestamp}} +Duration: {{.duration}}{{if .recovery_context}} +{{.recovery_context}}{{end}}`, + paramsCtx, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := paramsCtx[alerting2.ParamMessage].(string) + if !strings.Contains(got, "Node: es717 of Cluster: migrator-es717, JVM Usage: 85.9%") { + t.Fatalf("expected recovery context in message, got %q", got) + } +} + +func TestBuildRecoveryNotificationParamsDoesNotIncludeRecoveryContext(t *testing.T) { + rule := &alerting.Rule{ + ID: "rule-1", + Name: "test-rule", + Resource: alerting.Resource{ + ID: "cluster-1", + Name: "cluster-a", + }, + } + checkResults := &alerting.ConditionResult{} + alertMessage := &alerting.AlertMessage{ + ID: "evt-1", + Created: time.Date(2026, 6, 9, 11, 45, 56, 0, time.Local), + } + alertItem := &alerting.Alert{ + Created: time.Date(2026, 6, 9, 11, 47, 56, 0, time.Local), + } + + paramsCtx := buildRecoveryNotificationParams(rule, checkResults, alertMessage, alertItem) + if _, ok := paramsCtx["recovery_context"]; ok { + t.Fatal("expected full recovery params to omit recovery_context") + } + if got := paramsCtx[alerting2.ParamEventID]; got != "evt-1" { + t.Fatalf("expected event id to be preserved, got %v", got) + } + if got := paramsCtx["duration"]; got != "2m" { + t.Fatalf("expected duration to be computed from alert times, got %v", got) + } +} + +func TestNormalizeAlertTemplateTextPreservesMarkdownHardBreakSpacing(t *testing.T) { + input := "EventID: 1 \nTarget: cluster \nTriggerAt: now" + got := normalizeAlertTemplateText(input) + if got != input { + t.Fatalf("expected markdown hard-break spaces to be preserved, got %q", got) + } +} + +func TestNormalizeAlertTemplateTextSplitsRepeatedNodeEntries(t *testing.T) { + input := "节点: es817 所属集群: migrator-es817, JVM 使用率: 65% 节点: es717 所属集群: migrator-es717, JVM 使用率: 56% 节点: node-03 所属集群: es-cluster-3node, JVM 使用率: 42%" + got := normalizeAlertTemplateText(input) + want := "节点: es817 所属集群: migrator-es817, JVM 使用率: 65%\n节点: es717 所属集群: migrator-es717, JVM 使用率: 56%\n节点: node-03 所属集群: es-cluster-3node, JVM 使用率: 42%" + if got != want { + t.Fatalf("expected repeated node entries to be split by lines, got %q", got) + } +} + +func TestFormatAlertDuration(t *testing.T) { + cases := []struct { + name string + input time.Duration + expected string + }{ + {name: "sub-second", input: 500 * time.Millisecond, expected: "0s"}, + {name: "minute-second", input: 5*time.Minute + 59*time.Second + 999*time.Millisecond, expected: "5m59s"}, + {name: "hour-minute", input: time.Hour + 2*time.Minute, expected: "1h2m"}, + {name: "day-hour", input: 24*time.Hour + 3*time.Hour + 4*time.Second, expected: "1d3h4s"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := formatAlertDuration(tc.input); got != tc.expected { + t.Fatalf("expected %q, got %q", tc.expected, got) + } + }) + } +} + +func TestResolveTemplateTimeRangeFromQueryResult(t *testing.T) { + checkResults := &alerting.ConditionResult{ + QueryResult: &alerting.QueryResult{ + Min: int64(1781140000000), + Max: int64(1781140060000), + }, + } + + min, max := resolveTemplateTimeRange(checkResults) + if min != "2026-06-11T01:05:40Z" { + t.Fatalf("expected min to expand query min timestamp, got %q", min) + } + if max != "2026-06-11T01:08:40Z" { + t.Fatalf("expected max to expand query max timestamp, got %q", max) + } +} + +func TestResolveTemplateTimeRangeFallsBackToIssueTimestamp(t *testing.T) { + checkResults := &alerting.ConditionResult{ + ResultItems: []alerting.ConditionResultItem{ + { + IssueTimestamp: "1781140060000", + }, + }, + } + + min, max := resolveTemplateTimeRange(checkResults) + if min != "2026-06-11T01:06:40Z" { + t.Fatalf("expected min to use issue timestamp fallback, got %q", min) + } + if max != "2026-06-11T01:08:40Z" { + t.Fatalf("expected max to use issue timestamp fallback, got %q", max) + } +} + +func TestDiffRecoveredConditionResultItems(t *testing.T) { + makeItem := func(priority string, groups ...string) alerting.ConditionResultItem { + return alerting.ConditionResultItem{ + GroupValues: groups, + ConditionItem: &alerting.ConditionItem{ + Priority: priority, + }, + } + } + + previous := []alerting.ConditionResultItem{ + makeItem("low", "cluster-a", "node-1"), + makeItem("medium", "cluster-a", "node-2"), + makeItem("high", "cluster-a", "node-3"), + } + current := []alerting.ConditionResultItem{ + makeItem("high", "cluster-a", "node-3"), + } + + recovered := diffRecoveredConditionResultItems(previous, current) + if len(recovered) != 2 { + t.Fatalf("expected 2 recovered items, got %d", len(recovered)) + } + if got := strings.Join(recovered[0].GroupValues, ","); got != "cluster-a,node-1" { + t.Fatalf("expected first recovered item to be node-1, got %s", got) + } + if got := strings.Join(recovered[1].GroupValues, ","); got != "cluster-a,node-2" { + t.Fatalf("expected second recovered item to be node-2, got %s", got) + } +} + +func TestDiffRecoveredConditionResultItemsIgnoresPriorityChanges(t *testing.T) { + makeItem := func(priority string, groups ...string) alerting.ConditionResultItem { + return alerting.ConditionResultItem{ + GroupValues: groups, + ConditionItem: &alerting.ConditionItem{ + Priority: priority, + }, + } + } + + previous := []alerting.ConditionResultItem{ + makeItem("high", "cluster-a", "node-1"), + } + current := []alerting.ConditionResultItem{ + makeItem("low", "cluster-a", "node-1"), + } + + recovered := diffRecoveredConditionResultItems(previous, current) + if len(recovered) != 0 { + t.Fatalf("expected no recovered items when only priority changes, got %d", len(recovered)) + } +} + +func TestIsIncrementalRecoveryEnabled(t *testing.T) { + if isIncrementalRecoveryEnabled(nil) { + t.Fatal("expected nil config to disable incremental recovery") + } + + cfg := &alerting.RecoveryNotificationConfig{ + Enabled: true, + EventEnabled: true, + IncrementalRecoveryEnabled: false, + } + if isIncrementalRecoveryEnabled(cfg) { + t.Fatal("expected incremental recovery to stay disabled by default") + } + + cfg.IncrementalRecoveryEnabled = true + if !isIncrementalRecoveryEnabled(cfg) { + t.Fatal("expected incremental recovery to be enabled when all switches are on") + } +} + func TestGeneratePercentilesAggQuery(t *testing.T) { //rule := alerting.Rule{ // ID: util.GetUUID(), @@ -320,3 +625,108 @@ func TestConvertFilterQuery(t *testing.T) { t.Errorf("expect dsl %s but got %s", targetDsl, dsl) } } + +func TestGetRelationValuesSkipsNilMetricValue(t *testing.T) { + queryResult := &alerting.QueryResult{ + MetricData: []insight.MetricData{ + { + Data: map[string][]insight.MetricDataItem{ + "a": {{Timestamp: int64(1), Value: float64(12)}}, + "b": {{Timestamp: int64(1), Value: nil}}, + }, + }, + }, + } + + values, ok := getRelationValues(queryResult, []insight.MetricItem{ + {Name: "a"}, + {Name: "b"}, + }, 0, 0) + + if ok { + t.Fatalf("expected relation values with nil metric to be skipped") + } + if values != nil { + t.Fatalf("expected nil relation values, got %#v", values) + } +} + +func TestGetRelationValuesReturnsValidMetricValues(t *testing.T) { + queryResult := &alerting.QueryResult{ + MetricData: []insight.MetricData{ + { + Data: map[string][]insight.MetricDataItem{ + "a": {{Timestamp: int64(1), Value: float64(12)}}, + "b": {{Timestamp: int64(1), Value: float64(7)}}, + }, + }, + }, + } + + values, ok := getRelationValues(queryResult, []insight.MetricItem{ + {Name: "a"}, + {Name: "b"}, + }, 0, 0) + + if !ok { + t.Fatalf("expected relation values to be available") + } + if values["a"] != float64(12) || values["b"] != float64(7) { + t.Fatalf("unexpected relation values: %#v", values) + } +} + +func TestGenerateTimeFilterIgnoreReturnsMatchAll(t *testing.T) { + eng := &Engine{} + rule := &alerting.Rule{ + Resource: alerting.Resource{ + IgnoreTimeFilter: true, + TimeField: "timestamp", + }, + } + + timeFilter, err := eng.generateTimeFilter(rule, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if timeFilter == nil { + t.Fatal("expected time filter map, got nil") + } + if _, ok := timeFilter["match_all"]; !ok { + t.Fatalf("expected match_all query, got %#v", timeFilter) + } +} + +func TestGenerateTimeFilterReturnsRangeWhenEnabled(t *testing.T) { + eng := &Engine{} + rule := &alerting.Rule{ + Name: "test-rule", + Resource: alerting.Resource{ + TimeField: "timestamp", + }, + Metrics: alerting.Metric{ + Metric: insight.Metric{ + BucketSize: "1m", + }, + }, + } + + timeFilter, err := eng.generateTimeFilter(rule, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + rangeQuery, ok := timeFilter["range"].(util.MapStr) + if !ok { + t.Fatalf("expected range query, got %#v", timeFilter) + } + timeFieldRange, ok := rangeQuery["timestamp"].(util.MapStr) + if !ok { + t.Fatalf("expected timestamp range, got %#v", rangeQuery) + } + if _, ok = timeFieldRange["gte"]; !ok { + t.Fatalf("expected gte in range query, got %#v", timeFieldRange) + } + if _, ok = timeFieldRange["lte"]; !ok { + t.Fatalf("expected lte in range query, got %#v", timeFieldRange) + } +} diff --git a/service/alerting/env.go b/service/alerting/env.go index b81f7e53..36e97ab8 100644 --- a/service/alerting/env.go +++ b/service/alerting/env.go @@ -47,7 +47,7 @@ func GetEnvVariables() (map[string]interface{}, error) { if envVariables != nil && envVariables["INFINI_CONSOLE_ENDPOINT"] == nil { buf, err := kv.GetValue("system", []byte("INFINI_CONSOLE_ENDPOINT")) if err != nil { - log.Error(err) + log.Errorf("get alert env variable from kv failed, bucket=system, key=INFINI_CONSOLE_ENDPOINT: %v", err) } var endpoint string if len(buf) > 0 { diff --git a/service/alerting/funcs/elastic.go b/service/alerting/funcs/elastic.go index 62c3a8b7..4c5e4657 100644 --- a/service/alerting/funcs/elastic.go +++ b/service/alerting/funcs/elastic.go @@ -74,7 +74,7 @@ func lookupMetadata(object string, property string, defaultValue string, id stri buf = util.MustToJSONBytes(cfg) err := util.FromJSONBytes(buf, &cfgM) if err != nil { - log.Error(err) + log.Errorf("lookup metadata decode failed, object=%s, property=%s, id=%s: %v", object, property, id, err) return defaultValue } delete(cfgM, "basic_auth") @@ -83,7 +83,7 @@ func lookupMetadata(object string, property string, defaultValue string, id stri cfg.ID = id err, result := orm.GetBy("metadata.node_id", id, &cfg) if err != nil { - log.Error(err) + log.Errorf("lookup metadata get node failed, object=%s, property=%s, id=%s: %v", object, property, id, err) return defaultValue } if len(result.Result) == 0 { @@ -92,7 +92,7 @@ func lookupMetadata(object string, property string, defaultValue string, id stri buf = util.MustToJSONBytes(result.Result[0]) err = util.FromJSONBytes(buf, &cfgM) if err != nil { - log.Error(err) + log.Errorf("lookup metadata decode failed, object=%s, property=%s, id=%s: %v", object, property, id, err) return defaultValue } case "index": @@ -103,7 +103,7 @@ func lookupMetadata(object string, property string, defaultValue string, id stri q.Conds = orm.And(orm.Eq("metadata.index_id", id)) err, result := orm.Search(cfg, q) if err != nil { - log.Error(err) + log.Errorf("lookup metadata search index failed, object=%s, property=%s, id=%s: %v", object, property, id, err) return defaultValue } if len(result.Result) == 0 { @@ -112,7 +112,7 @@ func lookupMetadata(object string, property string, defaultValue string, id stri buf = util.MustToJSONBytes(result.Result[0]) err = util.FromJSONBytes(buf, &cfgM) if err != nil { - log.Error(err) + log.Errorf("lookup metadata decode failed, object=%s, property=%s, id=%s: %v", object, property, id, err) return defaultValue } } diff --git a/ui.go b/ui.go index c19794ff..025c726e 100644 --- a/ui.go +++ b/ui.go @@ -45,8 +45,19 @@ type UI struct { } func (h UI) InitUI() { + localPath, err := config.ResolveSelfHostedPackageBasePath(h.Config.UI.LocalPath) + if err != nil { + log.Errorf("failed to resolve self-hosted package path [%s]: %v", h.Config.UI.LocalPath, err) + localPath = h.Config.UI.LocalPath + } + + if h.Config.UI.LocalEnabled { + if err := config.EnsureSelfHostedPackageDirs(localPath); err != nil { + log.Errorf("failed to prepare self-hosted package directories under [%s]: %v", localPath, err) + } + } - vfs.RegisterFS(public.StaticFS{StaticFolder: h.Config.UI.LocalPath, TrimLeftPath: h.Config.UI.LocalPath, CheckLocalFirst: h.Config.UI.LocalEnabled, SkipVFS: !h.Config.UI.VFSEnabled}) + vfs.RegisterFS(public.StaticFS{StaticFolder: localPath, TrimLeftPath: localPath, CheckLocalFirst: h.Config.UI.LocalEnabled, SkipVFS: !h.Config.UI.VFSEnabled}) basePath := "/" + strings.Trim(global.Env().SystemConfig.WebAppConfig.BasePath, "/") @@ -57,11 +68,15 @@ func (h UI) InitUI() { // b) Create the final handler that acts as a dispatcher for the sub-path. finalHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Check if the request is for the sub-path's entrypoint. - if r.Method == "GET" && (r.URL.Path == "/" || r.URL.Path == basePath+"/" || r.URL.Path == basePath) { + if r.Method == "GET" && isIndexRequest(r.URL.Path, basePath) { serveDynamicIndex(w, r, basePath) return } + if r.Method == "GET" && shouldDisableCache(r.URL.Path, basePath) { + setNoCacheHeaders(w) + } + // For all other requests, delegate to the static file handler. staticFilesHandler.ServeHTTP(w, r) }) @@ -89,6 +104,25 @@ func (h UI) InitUI() { }) } +func isIndexRequest(path, basePath string) bool { + return path == "/" || + path == "/index.html" || + path == basePath || + path == basePath+"/" || + path == basePath+"/index.html" +} + +func shouldDisableCache(path, basePath string) bool { + return path == "/manifest.json" || + path == basePath+"/manifest.json" +} + +func setNoCacheHeaders(w http.ResponseWriter) { + w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Expires", "0") +} + // Helper function for serving the DYNAMIC index.html with replacements. func serveDynamicIndex(w http.ResponseWriter, r *http.Request, basePath string) { file, err := vfs.VFS().Open("/index.html") @@ -113,9 +147,7 @@ func serveDynamicIndex(w http.ResponseWriter, r *http.Request, basePath string) content = bytes.ReplaceAll(content, []byte(`window.routerBase = "/";`), []byte(`window.routerBase = "`+jsBasePath+`";`)) content = bytes.ReplaceAll(content, []byte(`window.publicPath = "/";`), []byte(`window.publicPath = "`+jsBasePath+`";`)) - w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") - w.Header().Set("Pragma", "no-cache") - w.Header().Set("Expires", "0") + setNoCacheHeaders(w) w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Write(content) } diff --git a/web/.gitignore b/web/.gitignore index a40ed7fa..cc3bad1e 100644 --- a/web/.gitignore +++ b/web/.gitignore @@ -36,3 +36,7 @@ log/ /web/ .github src/common*/ +/public +plugin/enterprise/ +src/components/Licence +locales/*/license.js diff --git a/web/config/.gitignore b/web/config/.gitignore new file mode 100644 index 00000000..ddd6c31e --- /dev/null +++ b/web/config/.gitignore @@ -0,0 +1 @@ +router.enterprise.js \ No newline at end of file diff --git a/web/config/config.js b/web/config/config.js index 22a7be05..c4a2555c 100644 --- a/web/config/config.js +++ b/web/config/config.js @@ -5,25 +5,70 @@ import webpackPlugin from "./plugin.config"; import defaultSettings from "../src/defaultSettings"; import packageJson from "../package.json"; -const ProxyTarget = "http://localhost:9001" +const ProxyTarget = "https://localhost:9000"; + +// 统一代理路径列表 +const proxyPaths = [ + "/elasticsearch/", + "/_search-center/", + "/gateway/", + "/_info", + "/config/", + "/environments/", + "/pipeline/", + "/queue/", + "/task/", + "/tasks/", + "/debug/", + "/alerting/", + "/health", + "/stats", + "/keystore", + "/user", + "/role/", + "/permission/", + "/account/", + "/auth/", + "/notification/", + "/agent/", + "/insight/", + "/host/", + "/_platform/", + "/migration/", + "/comparison/", + "/_license/", + "/setup/", + "/layout", + "/credential", + "/setting/", + "/email/", + "/data/", + "/instance", + "/collection/", +]; + +// 生成代理配置，统一加上 secure: false +const proxy = proxyPaths.reduce(function(acc, path) { + acc[path] = { + target: ProxyTarget, + changeOrigin: true, + secure: false, // 忽略自签名证书 + }; + return acc; +}, {}); export default { - // add for transfer to umi plugins: [ [ "umi-plugin-react", { antd: true, - dva: { - hmr: true, - }, - targets: { - ie: 11, - }, + dva: { hmr: true }, + targets: { ie: 11 }, locale: { - enable: true, // default false - default: "en-US", // default zh-CN - baseNavigator: true, // default true, when it is true, will use `navigator.language` overwrite default + enable: true, + default: "en-US", + baseNavigator: true, }, dynamicImport: { loadingComponent: "./components/PageLoading/index", @@ -39,17 +84,8 @@ export default { : {}), }, ], - // [ - // 'umi-plugin-ga', - // { - // code: 'UA-12123-6', - // judge: () => process.env.APP_TYPE === 'site', - // }, - // ], ], - targets: { - ie: 11, - }, + targets: { ie: 11 }, define: { APP_TYPE: process.env.APP_TYPE || "", ENV: process.env.NODE_ENV, @@ -59,170 +95,16 @@ export default { APP_AUTHOR: packageJson.author, APP_OFFICIAL_WEBSITE: packageJson.official_website || "", }, - // 路由配置 routes: pageRoutes, - // Theme for antd - // https://ant.design/docs/react/customize-theme-cn - theme: { - "primary-color": defaultSettings.primaryColor, - }, - externals: { - // '@antv/data-set': 'DataSet', - }, - proxy: { - "/elasticsearch/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/_search-center/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/static/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/gateway/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/_info": { - target: ProxyTarget, - changeOrigin: true, - }, - "/config/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/environments/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/pipeline/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/queue/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/task/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/tasks/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/debug/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/alerting/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/health": { - target: ProxyTarget, - changeOrigin: true, - }, - "/stats": { - target: ProxyTarget, - changeOrigin: true, - }, - "/keystore": { - target: ProxyTarget, - changeOrigin: true, - }, - "/user": { - target: ProxyTarget, - changeOrigin: true, - }, - "/role/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/permission/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/account/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/notification/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/agent/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/insight/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/host/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/_platform/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/migration/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/comparison/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/_license/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/setup/": { - target: ProxyTarget, - changeOrigin: true, - }, - "/layout": { - target: ProxyTarget, - changeOrigin: true, - }, - "/credential": { - target: ProxyTarget, - changeOrigin: true, - }, - "/setting": { - target: ProxyTarget, - changeOrigin: true, - }, - "/email": { - target: ProxyTarget, - changeOrigin: true, - }, - "/data": { - target: ProxyTarget, - changeOrigin: true, - }, - "/instance": { - target: ProxyTarget, - changeOrigin: true, - }, - "/collection": { - target: ProxyTarget, - changeOrigin: true, - }, - }, + theme: { "primary-color": defaultSettings.primaryColor }, + proxy: proxy, ignoreMomentLocale: true, - lessLoaderOptions: { - javascriptEnabled: true, - }, + ...(process.env.NODE_ENV === "production" ? { devtool: false } : {}), + lessLoaderOptions: { javascriptEnabled: true }, disableRedirectHoist: true, cssLoaderOptions: { modules: true, - getLocalIdent: (context, localIdentName, localName) => { + getLocalIdent: function(context, localIdentName, localName) { if ( context.resourcePath.includes("node_modules") || context.resourcePath.includes("ant.design.pro.less") || @@ -236,33 +118,19 @@ export default { const antdProPath = match[1].replace(".less", ""); const arr = antdProPath .split("/") - .map((a) => a.replace(/([A-Z])/g, "-$1")) - .map((a) => a.toLowerCase()); + .map(function(a) { return a.replace(/([A-Z])/g, "-$1"); }) + .map(function(a) { return a.toLowerCase(); }); return `antd-pro${arr.join("-")}-${localName}`.replace(/--/g, "-"); } return localName; }, }, - - // chainWebpack: webpackPlugin, - cssnano: { - mergeRules: false, - }, - - // extra configuration for egg + cssnano: { mergeRules: false }, runtimePublicPath: true, hash: true, outputPath: "../.public", - manifest: { - fileName: "../.public/manifest.json", - publicPath: "", - }, - + manifest: { fileName: "../.public/manifest.json", publicPath: "" }, copy: ["./src/assets/favicon.ico"], history: "hash", - // exportStatic: { - // // htmlSuffix: true, - // dynamicRoot: true, - // }, sass: {}, }; diff --git a/web/config/router.config.js b/web/config/router.config.js index b65fc989..1f95ef86 100644 --- a/web/config/router.config.js +++ b/web/config/router.config.js @@ -1,3 +1,13 @@ +import fs from 'fs'; +import path from 'path'; + +const pluginRoutes = []; + +const pluginRoutePath = path.resolve('config','router.enterprise.js'); +if (fs.existsSync(pluginRoutePath)) { + pluginRoutes.push(...require(pluginRoutePath).default); +} + export default [ // user { @@ -124,6 +134,54 @@ export default [ authority: ["data.alias:all", "data.alias:read"], exact: false, }, + { + path: "/data/views/create", + name: "view.create", + component: "./DataManagement/IndexPatterns", + hideInMenu: true, + authority: ["data.view:all", "data.view:read"], + exact: false, + }, + { + path: "/data/views/patterns", + component: "./DataManagement/IndexPatterns", + hideInMenu: true, + hideInBreadcrumb: true, + authority: ["data.view:all", "data.view:read"], + exact: false, + }, + { + path: "/data/views/patterns/:id", + name: "view.detail", + component: "./DataManagement/IndexPatterns", + hideInMenu: true, + authority: ["data.view:all", "data.view:read"], + exact: false, + }, + { + path: "/data/views/patterns/:id/:section", + component: "./DataManagement/IndexPatterns", + hideInMenu: true, + hideInBreadcrumb: true, + authority: ["data.view:all", "data.view:read"], + exact: false, + }, + { + path: "/data/views/patterns/:id/:section/:name", + component: "./DataManagement/IndexPatterns", + hideInMenu: true, + hideInBreadcrumb: true, + authority: ["data.view:all", "data.view:read"], + exact: false, + }, + { + path: "/data/views/patterns/:id/:section/:name/:action", + component: "./DataManagement/IndexPatterns", + hideInMenu: true, + hideInBreadcrumb: true, + authority: ["data.view:all", "data.view:read"], + exact: false, + }, { path: "/data/views", name: "view", @@ -155,7 +213,7 @@ export default [ }, ], }, - + ...pluginRoutes, // alerting { path: "/alerting", @@ -277,6 +335,11 @@ export default [ "agent.instance:read", ], routes: [ + { + path: "/resource/runtime", + hideInMenu: true, + hideInBreadcrumb: true, + }, { path: "/resource/runtime/instance/new", name: "runtime.new_instance", @@ -291,6 +354,11 @@ export default [ hideInMenu: true, authority: ["gateway.instance:all"], }, + { + path: "/resource/runtime/instance/:instance_id", + hideInMenu: true, + hideInBreadcrumb: true, + }, { path: "/resource/runtime/instance/:instance_id/task", name: "runtime.task", @@ -381,6 +449,8 @@ export default [ name: "system", icon: "setting", authority: [ + "system.cluster:all", + "system.cluster:read", "system.credential:all", "system.credential:read", "system.security:all", @@ -391,11 +461,27 @@ export default [ "system.smtp_server:read" ], routes: [ + { + path: "/system/settings", + name: "settings", + component: "./System/Settings/index", + authority: [ + "system.cluster:all", + "system.cluster:read", + "system.smtp_server:all", + "system.smtp_server:read", + ], + }, { path: "/system/email_server", - name: "smtp_server", - component: "./System/Email/Server", - authority: ["system.smtp_server:all", "system.smtp_server:read"], + component: "./System/Settings/index", + hideInMenu: true, + authority: [ + "system.cluster:all", + "system.cluster:read", + "system.smtp_server:all", + "system.smtp_server:read", + ], }, { path: "/system/credential", diff --git a/web/docs/test-cases.md b/web/docs/test-cases.md new file mode 100644 index 00000000..d9c66e7d --- /dev/null +++ b/web/docs/test-cases.md @@ -0,0 +1,328 @@ +# Console Web 测试用例基线 + +> 适用范围：当前 `web/` 前端已接入路由与主功能页面。 +> 目标：为后续手工测试、回归测试、自动化测试提供统一覆盖基线。 +> 整理依据：`web/config/router.config.js`、当前菜单与现有页面能力。 + +## 1. 测试目标 + +本用例文档用于覆盖以下风险： + +1. **核心流程不可用**：登录、导航、查询、创建、编辑、删除、详情页跳转失败。 +2. **权限与可见性错误**：无权限菜单可见、按钮显示错误、接口越权。 +3. **配置类功能保存失败**：系统设置、安全设置、凭据、告警配置等保存不生效。 +4. **复杂页面交互异常**：Discover、Alerting、Migration、Comparison、Agent 详情等富交互页面出现状态同步、滚动、表格渲染问题。 +5. **国际化与页面基础质量问题**：缺失文案、breadcrumb 异常、React warning、布局溢出、重复 key、重复路由跳转。 + +## 2. 建议测试分层 + +| 层级 | 目标 | 执行时机 | 建议范围 | +| --- | --- | --- | --- | +| Smoke | 验证主链路可用 | 每次提测 / 每个 PR | 登录、菜单、首页、核心列表页、核心 CRUD | +| Core Regression | 验证业务主流程稳定 | 每日 / 每个迭代 | 用户、角色、告警、Discover、数据工具、系统设置 | +| Full Regression | 验证跨模块与异常路径 | 发版前 | 全模块 + 权限 + 国际化 + 异常场景 | + +## 3. 通用测试维度 + +所有模块都应尽量覆盖以下维度： + +1. **页面进入**：路由可访问、首屏无白屏、无明显报错。 +2. **数据加载**：列表/详情接口成功、空态正确、异常态可提示。 +3. **查询与筛选**：关键字、分页、排序、筛选条件生效。 +4. **新增/编辑/删除**：表单校验、提交成功、提交失败、刷新后结果一致。 +5. **权限控制**：只读用户不可编辑、无权限菜单不可见、按钮不可操作。 +6. **国际化**：中文场景无缺失文案、标题/breadcrumb/按钮/表单提示完整。 +7. **布局稳定性**：无横向顶层滚动、表格内滚动合理、按钮不被挤压。 +8. **状态同步**：URL 参数、筛选条件、详情状态切换后行为一致。 +9. **浏览器交互**：刷新、返回、复制、弹窗关闭、确认框取消。 +10. **异常处理**：接口 4xx/5xx 时提示明确，不出现死循环或重复请求。 + +## 4. 测试环境建议 + +| 维度 | 建议值 | +| --- | --- | +| 浏览器 | Chrome 最新版、Edge 最新版 | +| 语言 | `zh-CN` 必测，`en-US` 抽检 | +| 账号 | 超级管理员、只读账号、受限角色账号 | +| 数据准备 | 至少 1 个平台角色、1 个数据角色、2 个用户、2 个集群、1 个 Agent、1 条告警规则、1 个告警渠道、若干索引/视图/别名 | +| 网络场景 | 正常网络、慢接口、接口 401/403/500 | + +## 5. 模块覆盖矩阵 + +| 模块 | 核心页面/能力 | 最低覆盖要求 | +| --- | --- | --- | +| 用户与登录 | 登录、登出、密码修改、个人设置 | Smoke + Regression | +| 工作台/全局布局 | 首页、菜单、面包屑、语言、全局通知 | Smoke | +| 平台管理 | 概览、监控、活动 | Regression | +| 数据管理 | 索引、别名、视图 | Regression | +| 数据探索 | Discover、分享、导出、空态 | Regression | +| 数据工具 | Migration、Comparison | Regression | +| 告警管理 | 规则、消息、告警详情、渠道 | Regression | +| 开发工具 | Console、常用命令 | Smoke | +| 资源管理 | Runtime、Cluster、Agent | Regression | +| 系统管理 | 设置、凭据、安全、审计 | Regression | +| 账户中心 | 个人信息、密码、通知 | Smoke | + +## 6. 详细测试用例 + +> 优先级说明：P0 = 发版阻断，P1 = 重要回归，P2 = 常规覆盖。 + +### 6.1 认证与全局框架 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| AUTH-001 | P0 | 用户名密码登录成功 | 存在有效账号 | 输入正确账号密码登录 | 登录成功，跳转首页，用户态建立 | +| AUTH-002 | P0 | 错误密码登录失败 | 存在有效账号 | 输入错误密码登录 | 页面提示失败，不进入系统 | +| AUTH-003 | P1 | 未登录访问受保护路由 | 未登录 | 直接打开受保护地址 | 被重定向到登录页 | +| AUTH-004 | P0 | 退出登录 | 已登录 | 点击右上角退出 | 会话失效，返回登录页，无重复 401 循环 | +| AUTH-005 | P1 | 登录页 SSO 成功回调 | 已配置 SSO | 触发 SSO 成功回跳 | 页面进入系统，用户信息正确 | +| AUTH-006 | P1 | 登录页 SSO 失败回调 | 已配置 SSO | 触发失败回跳 | 页面显示失败状态，可重新登录 | +| GLB-001 | P0 | 主框架菜单渲染 | 已登录 | 查看左侧菜单 | 仅展示有权限菜单，无空菜单项 | +| GLB-002 | P1 | 面包屑渲染正确 | 已登录 | 进入新增/编辑详情页 | breadcrumb 无原始路径片段，名称国际化正常 | +| GLB-003 | P1 | 中文国际化完整 | `zh-CN` | 遍历主要页面 | 无 missing message、无英文漏出 | +| GLB-004 | P1 | 英文国际化回退 | `en-US` | 抽检主要页面 | 标题、按钮、表单提示正常 | +| GLB-005 | P1 | 全局错误提示 | 构造接口 500 | 进入任一请求页面 | 页面有错误提示，不死循环 | +| GLB-006 | P1 | 无权限按钮隐藏 | 只读账号 | 访问列表页 | 新建/编辑/删除按钮不可见 | + +### 6.2 工作台 / 首页 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| OVR-001 | P0 | 首页可访问 | 有首页权限 | 打开 `/overview` | 页面正常加载，无白屏 | +| OVR-002 | P1 | 首页卡片数据正常 | 存在监控数据 | 查看各统计卡片 | 数值、图表、趋势正常显示 | +| OVR-003 | P1 | 首页空数据兜底 | 无监控数据 | 打开首页 | 空态提示清晰，无控制台报错 | +| OVR-004 | P1 | 顶部组件布局稳定 | 浏览器宽度缩放 | 缩放窗口 | 无异常横向滚动，不挤压操作区 | + +### 6.3 平台管理 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| CLU-001 | P0 | 平台概览进入 | 已注册集群 | 打开 `/cluster/overview` | 页面加载成功，卡片与图表正常 | +| CLU-002 | P1 | 监控总览进入 | 已注册集群 | 打开 `/cluster/monitor` | 监控列表和图表正常 | +| CLU-003 | P1 | 集群详情跳转 | 列表有数据 | 从监控列表进入某集群详情 | 路由正确，详情数据显示正常 | +| CLU-004 | P1 | 节点详情跳转 | 存在节点数据 | 进入节点监控页面 | 节点图表与指标可用 | +| CLU-005 | P1 | 索引详情跳转 | 存在索引数据 | 进入索引监控页面 | 指标图表与索引信息正常 | +| CLU-006 | P1 | Metric collection mode 展示 | 有 agent / agentless 数据 | 查看列表、详情状态 | 文案国际化正确，无硬编码英文 | +| CLU-007 | P1 | 活动页查询 | 有活动记录 | 打开活动页并筛选 | 列表刷新正确，分页正常 | +| CLU-008 | P2 | 监控空桶场景 | 时间粒度过小 | 切换时间范围/粒度 | 显示建议文案，不报错 | + +### 6.4 数据管理 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| DATA-001 | P0 | 索引列表加载 | 有索引数据 | 打开 `/data/index` | 列表正常加载 | +| DATA-002 | P1 | 索引搜索 | 有多条索引 | 按关键字搜索 | 返回符合关键字的数据 | +| DATA-003 | P1 | 索引分页 | 有超过一页数据 | 切换分页/页大小 | 数据与页码对应正确 | +| DATA-004 | P1 | 别名列表加载 | 有别名 | 打开 `/data/alias` | 列表正常显示 | +| DATA-005 | P1 | 视图列表加载 | 有视图 | 打开 `/data/views` | 列表、空态、操作按钮正常 | +| DATA-006 | P2 | 空数据场景 | 无相关数据 | 分别访问索引/别名/视图页 | 空态友好，无控制台 warning | + +### 6.5 数据探索 Discover + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| DISC-001 | P0 | Discover 页面加载 | 有 Discover 权限 | 打开 `/insight/discover` | 页面正常进入，无白屏 | +| DISC-002 | P0 | 查询条件生效 | 存在可查询数据 | 输入查询条件并执行 | 表格、统计、图表同步刷新 | +| DISC-003 | P1 | 时间范围切换 | 有时间字段 | 切换时间范围 | 数据刷新正确，URL 参数同步 | +| DISC-004 | P1 | URL 恢复状态 | 已设置 query/time | 刷新页面 | 状态恢复，无重复 push warning | +| DISC-005 | P1 | 表格行 key 稳定 | 有重复 `_id` 场景 | 查询结果中翻页/刷新 | 无 duplicate key warning | +| DISC-006 | P1 | 分享链接复制 | 页面有查询状态 | 点击分享 | 复制成功，打开链接可复现当前状态 | +| DISC-007 | P1 | 导出 CSV/Excel | 有结果数据 | 点击导出 | 导出成功，文件内容正确 | +| DISC-008 | P1 | 无索引或无视图空态 | 无数据源 | 打开页面 | 显示国际化空态与创建入口 | +| DISC-009 | P2 | 无结果场景 | 查询无命中 | 执行无结果查询 | 空态正确，无 DOM nesting warning | + +### 6.6 数据工具：Migration + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| MIG-001 | P0 | 迁移列表加载 | 有迁移任务 | 打开 `/data_tools/migration` | 列表正常显示 | +| MIG-002 | P0 | 新建迁移任务 | 有源目标集群 | 创建迁移任务 | 校验通过，创建成功 | +| MIG-003 | P1 | 迁移详情进入 | 存在任务 | 进入详情页 | 分段进度、状态、步骤显示正常 | +| MIG-004 | P1 | Legacy 迁移跳转 | 存在 legacy 链接 | 访问 legacy 路由 | 正确重定向到新路由 | +| MIG-005 | P1 | 迁移表单组件加载 | 打开新建页 | 检查执行节点等字段 | 组件引用正常，无构建缺失 | +| MIG-006 | P2 | 删除迁移任务 | 有可删除任务 | 执行删除 | 删除成功，列表刷新 | + +### 6.7 数据工具：Comparison + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| CMP-001 | P0 | 比对列表加载 | 有比对任务 | 打开 `/data_tools/comparison` | 列表正常显示 | +| CMP-002 | P0 | 新建比对任务 | 有源目标集群 | 创建比对任务 | 提交成功，列表可见 | +| CMP-003 | P1 | 比对详情查看 | 存在任务 | 打开详情页 | 差异结果、状态信息正确 | +| CMP-004 | P1 | Legacy 比对跳转 | 存在 legacy 链接 | 访问 legacy 路由 | 正确重定向 | +| CMP-005 | P2 | 删除比对任务 | 有可删除任务 | 删除任务 | 删除成功，列表同步更新 | + +### 6.8 告警管理 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| ALT-001 | P0 | 告警规则列表加载 | 有规则数据 | 打开 `/alerting/rule` | 列表正常显示 | +| ALT-002 | P0 | 新建规则 | 已配置数据源/渠道 | 创建规则 | 表单校验正确，创建成功 | +| ALT-003 | P0 | 编辑规则 | 已存在规则 | 编辑并保存 | 保存成功，详情更新 | +| ALT-004 | P1 | 启停规则 | 存在启用/禁用入口 | 切换状态 | 状态变更成功 | +| ALT-005 | P1 | 删除规则 | 有可删除规则 | 删除规则 | 删除成功，列表更新 | +| ALT-006 | P1 | 规则详情 URL 状态同步 | 已打开详情页 | 调整时间条件并刷新 | 状态可恢复，无 hash push warning | +| ALT-007 | P1 | 规则详情通知卡片渲染 | 规则有多个渠道 | 查看通知卡片 | 渲染正常，无 key warning | +| ALT-008 | P1 | 消息中心列表 | 有消息记录 | 打开 `/alerting/message` | 列表正常显示 | +| ALT-009 | P1 | 消息详情查看 | 存在消息记录 | 打开消息详情 | 详情页、Tab、时间条件正常 | +| ALT-010 | P1 | 渠道列表/新建/编辑 | 有渠道权限 | 执行列表、新建、编辑 | 全流程正常 | +| ALT-011 | P2 | Priority 缺省值展示 | 存在 priority 为空消息 | 打开消息详情 | 文案国际化正常，无 missing message | + +### 6.9 开发工具 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| DEV-001 | P1 | Console 页面进入 | 有权限 | 打开 `/devtool/console` | 页面加载成功 | +| DEV-002 | P1 | 常用命令页进入 | 有权限 | 打开 `/devtool/command` | 列表正常显示 | +| DEV-003 | P2 | ToolTip 与按钮布局 | 页面已打开 | 鼠标悬停/缩放页面 | 提示文案可见，无布局抖动 | + +### 6.10 资源管理：Runtime / Gateway + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| RT-001 | P0 | Runtime 实例列表 | 有实例 | 打开 `/resource/runtime/instance` | 列表正常显示 | +| RT-002 | P0 | 新建 Runtime 实例 | 有权限 | 新建实例 | 创建成功 | +| RT-003 | P1 | 编辑 Runtime 实例 | 存在实例 | 编辑并保存 | 保存成功 | +| RT-004 | P1 | Runtime 任务页 | 存在实例 | 进入任务页 | 列表/状态正常 | +| RT-005 | P1 | Runtime 队列页 | 存在实例 | 进入队列页 | 队列信息正常 | +| RT-006 | P1 | Runtime 磁盘页 | 存在实例 | 进入磁盘页 | 指标正常 | +| RT-007 | P1 | Runtime 日志页 | 存在实例 | 进入日志页 | 日志展示正常 | +| RT-008 | P1 | Runtime 配置页 | 存在实例 | 进入配置页 | 配置内容正常加载 | + +### 6.11 资源管理：Cluster / Agent + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| RES-001 | P0 | Cluster 列表加载 | 有已注册集群 | 打开 `/resource/cluster` | 列表正常 | +| RES-002 | P0 | 注册集群 | 有权限 | 进入注册流程并提交 | 注册成功 | +| RES-003 | P1 | 编辑集群 | 存在集群 | 编辑并保存 | 保存成功 | +| RES-004 | P0 | Agent 列表加载 | 有 Agent 数据 | 打开 `/resource/agent` | 列表正常 | +| RES-005 | P0 | 注册 Agent | 有权限 | 新建 Agent | 创建成功 | +| RES-006 | P1 | 编辑 Agent | 存在 Agent | 编辑并保存 | 保存成功 | +| RES-007 | P1 | Agent 详情页 Tabs 切换 | Agent 有进程/指标数据 | 展开详情并切换 Tabs | 内容切换正常 | +| RES-008 | P1 | 未知进程列表横向滚动 | Agent 有长进程 ID/命令行 | 点击“未知进程”标签 | 仅表格内部滚动，不挤压顶部操作按钮 | +| RES-009 | P2 | 进程详情表格 key 稳定 | 有多进程数据 | 展开详情、多次刷新 | 无 key warning | + +### 6.12 系统管理：系统设置 / 凭据 / 邮件 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| SYS-001 | P0 | 系统设置页加载 | 有权限 | 打开 `/system/settings` | 页面正常加载 | +| SYS-002 | P0 | 滚动存储大小保存 | 可编辑 retention | 输入数字并保存 | 保存成功，单位默认 GB，刷新后保留 | +| SYS-003 | P1 | retention 旧值回显 | 后端存在 `mb/tb/gb` 等历史值 | 打开设置页 | 前端正确换算并显示数值 | +| SYS-004 | P1 | 邮件服务器配置保存 | 有 SMTP 权限 | 配置并保存 | 保存成功 | +| SYS-005 | P1 | 邮件服务器测试连接 | 已填写配置 | 发送测试 | 返回成功/失败提示 | +| SYS-006 | P0 | 凭据列表加载 | 有凭据 | 打开 `/system/credential` | 列表正常 | +| SYS-007 | P1 | 新建/编辑/删除凭据 | 有权限 | 执行 CRUD | 各操作成功，错误提示明确 | + +### 6.13 系统管理：安全设置 - 用户 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| SEC-U-001 | P0 | 用户列表加载 | 有用户数据 | 打开安全设置用户页 | 列表正常 | +| SEC-U-002 | P0 | 新建用户 | 有可选角色 | 新建用户 | 创建成功，返回初始密码 | +| SEC-U-003 | P1 | 新建用户表单校验 | 打开新建页 | 留空用户名/角色、输入非法邮箱 | 表单阻止提交并显示国际化提示 | +| SEC-U-004 | P0 | 编辑用户 | 存在用户 | 编辑昵称/电话/邮箱/标签并保存 | 保存成功 | +| SEC-U-005 | P0 | 重置用户密码 | 存在用户 | 执行重置密码 | 保存成功，校验规则生效 | +| SEC-U-006 | P0 | 删除用户 | 有可删除用户 | 删除用户 | 删除成功，无 500 错误 | +| SEC-U-007 | P1 | 用户页 breadcrumb 与标题 | 打开新增/编辑/重置页 | 查看面包屑 | 显示国际化标题，无原始 `/user` | + +### 6.14 系统管理：安全设置 - 角色 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| SEC-R-001 | P0 | 角色列表加载 | 有角色数据 | 打开安全设置角色页 | 列表正常 | +| SEC-R-002 | P0 | 新建平台角色 | 有权限 | 新建平台角色并选择功能权限 | 创建成功 | +| SEC-R-003 | P0 | 编辑平台角色 | 存在平台角色 | 编辑并保存 | 保存成功，权限数据正确 | +| SEC-R-004 | P0 | 新建数据角色 | 有集群和权限数据 | 新建数据角色 | 创建成功 | +| SEC-R-005 | P0 | 编辑数据角色 | 存在数据角色 | 编辑并保存 | 保存成功 | +| SEC-R-006 | P0 | 删除角色 | 有可删除角色 | 删除角色 | 删除成功，无 `only non-nil pointer to object is allowed` | +| SEC-R-007 | P1 | 平台角色权限必填校验 | 打开平台角色表单 | 不选择权限直接保存 | 阻止提交并提示 | +| SEC-R-008 | P1 | 数据角色集群权限校验 | 打开数据角色表单 | 缺少集群/集群权限/索引权限 | 阻止提交并提示国际化文案 | +| SEC-R-009 | P1 | 角色页 breadcrumb 与标题 | 打开新增/编辑数据角色页 | 查看面包屑和标题 | 显示国际化标题，无原始 `/role/data` | +| SEC-R-010 | P1 | 角色列表按钮权限控制 | 只读账号 | 进入角色页 | 新建/编辑/删除按钮不可见 | + +### 6.15 系统管理：审计 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| AUD-001 | P1 | 审计日志列表加载 | 有审计数据 | 打开 `/system/audit` | 列表正常 | +| AUD-002 | P1 | 审计日志查询 | 有多条日志 | 按关键字/时间查询 | 返回结果正确 | +| AUD-003 | P2 | 空审计场景 | 无数据 | 打开页面 | 空态正确，无报错 | + +### 6.16 账户中心 + +| 编号 | 优先级 | 场景 | 前置条件 | 步骤 | 预期结果 | +| --- | --- | --- | --- | --- | --- | +| ACC-001 | P1 | 个人信息页加载 | 已登录 | 打开 `/account/settings/base` | 页面正常 | +| ACC-002 | P1 | 修改个人资料 | 已登录 | 编辑资料并保存 | 保存成功 | +| ACC-003 | P0 | 修改个人密码 | 已登录 | 打开 `/account/password` 并修改 | 修改成功，规则校验正确 | +| ACC-004 | P1 | 通知页加载 | 存在通知 | 打开 `/account/notification` | 列表正常 | + +## 7. 专项回归清单 + +以下问题建议进入长期专项回归： + +1. **国际化专项** + - 中文页面无 missing message + - 新增/编辑/详情页 breadcrumb 正确 + - 空态、提示语、按钮、校验文案统一国际化 + +2. **URL 状态同步专项** + - Discover 查询参数 + - Alerting 时间范围参数 + - 列表页筛选、分页、返回后状态恢复 + +3. **表格稳定性专项** + - rowKey 唯一 + - 展开行、Tabs 内嵌表格、长字段换行 + - 表格内部滚动不影响整页布局 + +4. **权限专项** + - 菜单权限 + - 按钮权限 + - 直接访问 URL 的越权处理 + +5. **接口异常专项** + - 401 会话失效 + - 403 无权限 + - 500 服务异常 + - 保存/删除失败后的前端提示与状态回退 + +## 8. 自动化落地建议 + +建议后续按以下顺序补自动化： + +1. **P0 主链路先自动化** + - 登录/退出 + - 用户 CRUD + - 角色 CRUD + - 告警规则 CRUD + - Discover 基础查询 + - 系统设置保存 + +2. **P1 页面稳定性自动化** + - breadcrumb/i18n 快照 + - 列表页搜索/分页 + - Tabs / Drawer / Modal 交互 + - 表格内滚动与长文本场景 + +3. **接口契约与异常场景自动化** + - 401/403/500 + - 空数据响应 + - 字段缺省值 + - 删除/保存失败回滚 + +## 9. 维护建议 + +每次新增功能或修复线上问题时，至少同步更新： + +1. 模块覆盖矩阵 +2. 对应详细测试用例 +3. 专项回归清单 + +建议规则： + +- 新增页面：至少补 1 条进入用例 + 1 条成功流用例 + 1 条失败流用例 +- 新增表单：至少补 1 条校验用例 + 1 条保存成功用例 +- 修复线上 bug：必须补 1 条对应回归用例 diff --git a/web/mock/api.js b/web/mock/api.js index b20bb393..a45f60f4 100644 --- a/web/mock/api.js +++ b/web/mock/api.js @@ -374,22 +374,6 @@ export default { "eol_date": "2023-12-31T10:10:10Z" }, "tagline": "A light-weight but powerful agent." - }, - "basic_auth": {}, - "endpoint": "http://192.168.3.25:2900", - "host": { - "name": "INFINI-4.local", - "os": { - "name": "darwin", - "architecture": "arm64", - "version": "22.6.0" - } - }, - "network": { - "ip": [ - "192.168.3.25" - ], - "major_ip": "192.168.3.25" } }); }, diff --git a/web/mock/rbac/admin.js b/web/mock/rbac/admin.js index 4194be7d..398b96fc 100644 --- a/web/mock/rbac/admin.js +++ b/web/mock/rbac/admin.js @@ -101,18 +101,18 @@ export default { "indices.get_mapping", "indices.upgrade", "indices.validate_query", - "indices.exists_template", + "template.exists", "indices.get_upgrade", "indices.update_aliases", "indices.analyze", "indices.exists", "indices.close", - "indices.delete_template", + "template.delete", "indices.get_field_mapping", "indices.delete_alias", "indices.exists_type", - "indices.get_template", - "indices.put_template", + "template.get", + "template.put", "indices.refresh", "indices.segments", "indices.termvectors", diff --git a/web/package.json b/web/package.json index 16152d19..e87ff6b7 100644 --- a/web/package.json +++ b/web/package.json @@ -118,6 +118,7 @@ "babel-plugin-dva-hmr": "^0.4.1", "babel-plugin-import": "^1.6.3", "babel-plugin-transform-decorators-legacy": "^1.3.4", + "chokidar": "^5.0.0", "cross-env": "^7.0.3", "enzyme": "^3.9.0", "eslint": "^4.18.2", @@ -137,27 +138,48 @@ "sass": "1.69.5", "sass-loader": "8.0.2", "ts-loader": "8.4.0", + "typescript": "4.9.5", "umi": "^2.1.2", "umi-plugin-ga": "^1.1.3", - "umi-plugin-react": "^1.1.1", - "typescript": "4.9.5" + "umi-plugin-react": "^1.1.1" }, "engines": { "node": ">=8.9.0" }, "overrides": { + "@antv/async-hook": "2.2.9", + "@antv/color-util": "2.0.6", + "@antv/dom-util": "2.0.4", + "@antv/g-device-api": "1.6.0", + "@antv/gl-matrix": "2.7.1", + "@antv/g2plot": "2.4.35", + "@antv/l7plot": "0.5.11", + "@antv/matrix-util": "3.0.4", + "@antv/path-util": "2.0.15", + "size-sensor": "1.0.4", "ts-loader": "8.4.0", "typescript": "4.9.5" }, "resolutions": { + "@antv/async-hook": "2.2.9", + "@antv/color-util": "2.0.6", + "@antv/dom-util": "2.0.4", + "@antv/g-device-api": "1.6.0", + "@antv/gl-matrix": "2.7.1", + "@antv/g2plot": "2.4.35", + "@antv/l7plot": "0.5.11", + "@antv/matrix-util": "3.0.4", + "@antv/path-util": "2.0.15", + "size-sensor": "1.0.4", "react-draggable": "4.4.6", "react-resizable": "3.0.5", "@antv/l7-component": "2.23.0" }, "scripts": { - "dev": "cross-env MOCK=none UMI_UI=none NODE_OPTIONS=\"--max_old_space_size=4096\" umi dev", - "mock": "cross-env UMI_UI=none NODE_OPTIONS=\"--max_old_space_size=4096\" umi dev", - "build": "NODE_OPTIONS=\"--max_old_space_size=4096 --trace-deprecation\" umi build && cp -R static/* ../.public/static/", +"preinstall": "node ./scripts/ensure-cnpm.js", +"dev": "cross-env HTTPS=true MOCK=none node ./scripts/run-umi.js dev", +"mock": "node ./scripts/run-umi.js dev", +"build": "node ./scripts/run-umi.js build && cp -R static/* ../.public/static/", "autod": "autod", "docker:dev": "docker-compose -f ./docker/docker-compose.dev.yml up --remove-orphans -d", "docker:stop-dev": "docker-compose -f ./docker/docker-compose.dev.yml down", diff --git a/web/scripts/ensure-cnpm.js b/web/scripts/ensure-cnpm.js new file mode 100644 index 00000000..73955385 --- /dev/null +++ b/web/scripts/ensure-cnpm.js @@ -0,0 +1,18 @@ +"use strict"; + +const execPath = `${process.env.npm_execpath || ""}`.toLowerCase(); +const userAgent = `${process.env.npm_config_user_agent || ""}`.toLowerCase(); + +const isCnpm = + execPath.includes("cnpm") || + execPath.includes("npminstall") || + userAgent.includes("cnpm/") || + userAgent.includes("npminstall/"); + +if (!isCnpm) { + console.error(""); + console.error("This frontend only supports dependency installation via cnpm."); + console.error("Please use: cnpm install"); + console.error(""); + process.exit(1); +} diff --git a/web/scripts/run-umi.js b/web/scripts/run-umi.js new file mode 100644 index 00000000..899afc94 --- /dev/null +++ b/web/scripts/run-umi.js @@ -0,0 +1,250 @@ +"use strict"; + +const { spawnSync } = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const mode = process.argv[2]; + +if (!mode || !["dev", "build"].includes(mode)) { + console.error("usage: node scripts/run-umi.js "); + process.exit(1); +} + +const syncStaticAssets = () => { + const staticDir = path.resolve(__dirname, "../static"); + const publicStaticDir = path.resolve(__dirname, "../public/static"); + + if (!fs.existsSync(staticDir)) { + return; + } + + fs.mkdirSync(path.dirname(publicStaticDir), { recursive: true }); + fs.rmSync(publicStaticDir, { recursive: true, force: true }); + fs.cpSync(staticDir, publicStaticDir, { recursive: true }); +}; + +const walkFiles = (dir, baseDir = dir) => { + if (!fs.existsSync(dir)) { + return []; + } + + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const absolutePath = path.join(dir, entry.name); + if (entry.isDirectory()) { + return walkFiles(absolutePath, baseDir); + } + return [path.relative(baseDir, absolutePath)]; + }); +}; + +const copyFileWithParents = (source, target) => { + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(source, target); +}; + +const removeEmptyParents = (targetPath, stopPath) => { + let current = path.dirname(targetPath); + while (current.startsWith(stopPath) && current !== stopPath) { + if (!fs.existsSync(current) || fs.readdirSync(current).length > 0) { + break; + } + fs.rmdirSync(current); + current = path.dirname(current); + } +}; + +const createPluginSyncSnapshot = (pluginDir, webDir) => { + const licenceRelativePath = path.join("src", "components", "Licence"); + const licenceDir = path.join(webDir, licenceRelativePath); + const snapshotRoot = fs.mkdtempSync(path.join(os.tmpdir(), "console-plugin-sync-")); + const backupRoot = path.join(snapshotRoot, "backup"); + const licenceBackupDir = path.join(snapshotRoot, "licence"); + const backedUpFiles = []; + const createdFiles = []; + const pluginFiles = walkFiles(pluginDir); + + let licenceExisted = false; + if (fs.existsSync(licenceDir)) { + licenceExisted = true; + fs.cpSync(licenceDir, licenceBackupDir, { recursive: true }); + } + + for (const relativePath of pluginFiles) { + if ( + relativePath === licenceRelativePath || + relativePath.startsWith(`${licenceRelativePath}${path.sep}`) + ) { + continue; + } + + const targetPath = path.join(webDir, relativePath); + if (fs.existsSync(targetPath)) { + const backupPath = path.join(backupRoot, relativePath); + copyFileWithParents(targetPath, backupPath); + backedUpFiles.push(relativePath); + } else { + createdFiles.push(relativePath); + } + } + + return { + snapshotRoot, + backupRoot, + backedUpFiles, + createdFiles, + licenceExisted, + licenceBackupDir, + licenceDir, + webDir, + }; +}; + +const restorePluginSyncSnapshot = (snapshot) => { + if (!snapshot) { + return; + } + + const { + snapshotRoot, + backupRoot, + backedUpFiles, + createdFiles, + licenceExisted, + licenceBackupDir, + licenceDir, + webDir, + } = snapshot; + + try { + for (const relativePath of createdFiles) { + const targetPath = path.join(webDir, relativePath); + fs.rmSync(targetPath, { force: true }); + removeEmptyParents(targetPath, webDir); + } + + fs.rmSync(licenceDir, { recursive: true, force: true }); + if (licenceExisted) { + fs.cpSync(licenceBackupDir, licenceDir, { recursive: true }); + } + + for (const relativePath of backedUpFiles) { + const backupPath = path.join(backupRoot, relativePath); + const targetPath = path.join(webDir, relativePath); + copyFileWithParents(backupPath, targetPath); + } + } finally { + fs.rmSync(snapshotRoot, { recursive: true, force: true }); + } +}; + +const syncPluginDirectory = async ({ createSnapshot = false } = {}) => { + const pluginDir = path.resolve(__dirname, "../../plugin/enterprise/web"); + const webDir = path.resolve(__dirname, "../"); + const licenceDir = path.resolve(webDir, "src/components/Licence"); + + if (fs.existsSync(pluginDir)) { + console.log("Plugin directory found. Syncing with the main project..."); + + try { + const snapshot = createSnapshot ? createPluginSyncSnapshot(pluginDir, webDir) : null; + fs.rmSync(licenceDir, { recursive: true, force: true }) + fs.cpSync(pluginDir, webDir, { recursive: true, force: true }); + console.log(`Plugin synced directly to ${webDir}`); + return snapshot; + } catch (err) { + console.error(`Failed to sync plugin to ${webDir}:`, err); + return null; + } + } else { + console.log("No plugin directory found, skipping sync."); + return null; + } +}; + +const watchPluginChanges = async () => { + const chokidar = await import("chokidar"); + + const pluginDir = path.resolve(__dirname, "../../plugin/enterprise/web"); + + if (fs.existsSync(pluginDir)) { + const watcher = chokidar.watch(pluginDir, { persistent: true }); + + watcher.on("change", (filePath) => { + console.log(`File changed: ${filePath}`); + syncPluginDirectory(); + }); + + watcher.on("error", (err) => { + console.error("Error watching plugin directory:", err); + }); + + console.log(`Watching plugin directory for changes: ${pluginDir}`); + } else { + console.log("Plugin directory not found, skipping file watching."); + } +}; + +const run = async () => { + syncStaticAssets(); + + const snapshot = await syncPluginDirectory({ createSnapshot: mode === "build" }); + + if (mode === "dev") { + await watchPluginChanges(); + } + + const defaultOldSpaceSize = + process.env.UMI_MAX_OLD_SPACE_SIZE || (mode === "build" ? "8192" : "4096"); + + const nodeMajorVersion = Number(process.versions.node.split(".")[0] || "0"); + const existingNodeOptions = (process.env.NODE_OPTIONS || "").trim(); + const optionSet = new Set(existingNodeOptions.split(/\s+/).filter(Boolean)); + + if (![...optionSet].some((item) => item.startsWith("--max_old_space_size="))) { + optionSet.add(`--max_old_space_size=${defaultOldSpaceSize}`); + } + + if ( + nodeMajorVersion >= 17 && + !optionSet.has("--openssl-legacy-provider") + ) { + optionSet.add("--openssl-legacy-provider"); + } + + optionSet.add("--trace-deprecation"); + + const env = { + ...process.env, + NODE_OPTIONS: [...optionSet].join(" "), + }; + + if (mode === "dev") { + env.UMI_UI = env.UMI_UI || "none"; + } + + let exitCode = 0; + try { + const result = spawnSync("./node_modules/.bin/umi", [mode], { + stdio: "inherit", + shell: true, + env, + }); + + if (result.error) { + console.error(result.error); + exitCode = 1; + } else { + exitCode = result.status || 0; + } + } finally { + if (mode === "build") { + restorePluginSyncSnapshot(snapshot); + } + } + + process.exit(exitCode); +}; + +run(); diff --git a/web/src/app.js b/web/src/app.js index 79a04614..62c242b1 100644 --- a/web/src/app.js +++ b/web/src/app.js @@ -21,12 +21,28 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -import { getAuthEnabled } from "./utils/authority"; +import { + getAuthEnabled, + getEnterpriseTaskManagerEnabled, + refreshApplicationSettings, +} from "./utils/authority"; +import { startActivityAwareTokenRefresh } from "./utils/auth_session"; import request from "./utils/request"; import { setSetupRequired } from "@/utils/setup"; -import { getHealth } from "@/services/system" import PlatformContainer from "./components/PlatformContainer"; import React from "react"; +import { message, notification } from "antd"; + +message.config({ + maxCount: 3, +}); + +notification.config({ + placement: "topRight", +}); + +const CHUNK_RELOAD_KEY = "console.chunk-reload"; +const CHUNK_RELOAD_WINDOW_MS = 15000; if (!String.prototype.replaceAll) { String.prototype.replaceAll = function(search, replacement) { @@ -35,17 +51,168 @@ if (!String.prototype.replaceAll) { }; } +const isChunkLoadError = (error) => { + const reason = + error?.message || + error?.reason?.message || + error?.reason || + ""; + return /ChunkLoadError|Loading chunk [\d]+ failed|CSS_CHUNK_LOAD_FAILED/i.test( + `${reason}` + ); +}; + +const isInjectedScriptConnectionError = (error) => { + const reason = + error?.message || + error?.reason?.message || + error?.reason || + error || + ""; + const stack = `${error?.stack || error?.reason?.stack || ""}`; + return ( + `${reason}`.includes("Could not establish connection. Receiving end does not exist.") && + (stack.includes("single-file-bootstrap.bundle.js") || stack === "") + ); +}; + +const suppressUnhandledRejection = (event) => { + event.preventDefault?.(); + event.stopImmediatePropagation?.(); + event.stopPropagation?.(); +}; + +const getCanonicalLocation = () => { + const url = new URL(window.location.href); + return `${url.pathname}${url.search}${url.hash}`; +}; + +const shouldReloadForChunkError = () => { + const now = Date.now(); + const currentLocation = getCanonicalLocation(); + + try { + const raw = sessionStorage.getItem(CHUNK_RELOAD_KEY); + const previous = raw ? JSON.parse(raw) : null; + if ( + previous?.location === currentLocation && + now - Number(previous?.timestamp || 0) < CHUNK_RELOAD_WINDOW_MS + ) { + return false; + } + sessionStorage.setItem( + CHUNK_RELOAD_KEY, + JSON.stringify({ + location: currentLocation, + timestamp: now, + }) + ); + } catch (e) {} + + return true; +}; + +const installChunkReloadRecovery = () => { + if (typeof window === "undefined" || window.__consoleChunkReloadRecoveryInstalled) { + return; + } + + window.__consoleChunkReloadRecoveryInstalled = true; + + const handleChunkFailure = (error) => { + if (!isChunkLoadError(error) || !shouldReloadForChunkError()) { + return false; + } + window.location.reload(); + return true; + }; + + window.addEventListener("error", (event) => { + handleChunkFailure(event?.error || new Error(event?.message || "")); + }); + + window.addEventListener("unhandledrejection", (event) => { + if (handleChunkFailure(event?.reason)) { + suppressUnhandledRejection(event); + return; + } + if (isInjectedScriptConnectionError(event?.reason || event)) { + suppressUnhandledRejection(event); + } + }, true); +}; + +const serializeConsoleArgs = (args = []) => { + return args + .map((arg) => { + if (typeof arg === "string") { + return arg; + } + if (arg instanceof Error) { + return arg.stack || arg.message; + } + try { + return JSON.stringify(arg); + } catch (e) { + return String(arg); + } + }) + .join(" "); +}; + +const isUmiDllWarning = (args = []) => { + const message = serializeConsoleArgs(args); + if (!/warning:/i.test(message)) { + return false; + } + const stack = new Error().stack || ""; + return stack.includes("/umi.dll.js"); +}; + +const installConsoleWarningFilter = () => { + if ( + typeof window === "undefined" || + process.env.NODE_ENV === "development" || + window.__consoleWarningFilterInstalled + ) { + return; + } + + window.__consoleWarningFilterInstalled = true; + + ["warn", "error"].forEach((method) => { + const original = console[method]; + if (typeof original !== "function") { + return; + } + console[method] = function(...args) { + if (isUmiDllWarning(args)) { + return; + } + return original.apply(this, args); + }; + }); +}; + +installChunkReloadRecovery(); +installConsoleWarningFilter(); + export async function patchRoutes(routes) { - const healthRes = await getHealth(); - setSetupRequired(`${healthRes?.setup_required}`); + const appSettings = await refreshApplicationSettings(); + setSetupRequired(`${!!appSettings?.setup_required}`); + if (getEnterpriseTaskManagerEnabled() !== "true") { + routes = hideRoutesInMenu(routes, ["data_tools"], ""); + } if (getAuthEnabled() === "false") { routes = filterRoutes(routes, ["system.security"], ""); // routes = disableAuth(routes); } + return routes; } export function render(oldRoutes) { + startActivityAwareTokenRefresh(); oldRoutes(); } @@ -64,7 +231,7 @@ export function render(oldRoutes) { function filterRoutes(routes, names, prefix) { return routes.filter((route) => { if (route.name) { - const pn = `${prefix}.${route.name}`; + const pn = prefix ? `${prefix}.${route.name}` : route.name; if (names.includes(pn)) { return false; } @@ -80,7 +247,27 @@ function filterRoutes(routes, names, prefix) { }); } +function hideRoutesInMenu(routes, names, prefix) { + return (routes || []).map((route) => { + const nextRoute = { ...route }; + if (nextRoute.name) { + const pn = prefix ? `${prefix}.${nextRoute.name}` : nextRoute.name; + if (names.includes(pn)) { + nextRoute.hideInMenu = true; + } + } + if (nextRoute.routes) { + let pn = ""; + if (nextRoute.name) { + pn = prefix ? `${prefix}.${nextRoute.name}` : nextRoute.name; + } + nextRoute.routes = hideRoutesInMenu(nextRoute.routes, names, pn); + } + return nextRoute; + }); +} + export function rootContainer(container) { return React.createElement(PlatformContainer, null, container); -} \ No newline at end of file +} diff --git a/web/src/common/src/DatePicker/Range.jsx b/web/src/common/src/DatePicker/Range.jsx index ccfaeabd..4b51d06f 100644 --- a/web/src/common/src/DatePicker/Range.jsx +++ b/web/src/common/src/DatePicker/Range.jsx @@ -229,6 +229,7 @@ const Range = (props) => { const fullRangeText = useMemo(() => { if (isMinimum || !start || !end) return ""; + if (start === 'auto') return currentLocales[`datepicker.quick_select.${start}`]; if (showPrettyDuration(start, end, commonlyUsedRanges)) { return prettyDuration( start, diff --git a/web/src/common/src/DatePicker/Range.less b/web/src/common/src/DatePicker/Range.less index 5b104a38..f15aa2ed 100644 --- a/web/src/common/src/DatePicker/Range.less +++ b/web/src/common/src/DatePicker/Range.less @@ -23,6 +23,7 @@ overflow: hidden; white-space: nowrap; text-overflow: ellipsis; + letter-spacing: 0; } .down { diff --git a/web/src/common/src/DatePicker/TimeSetting.jsx b/web/src/common/src/DatePicker/TimeSetting.jsx index 630092b0..4cbaf878 100644 --- a/web/src/common/src/DatePicker/TimeSetting.jsx +++ b/web/src/common/src/DatePicker/TimeSetting.jsx @@ -55,7 +55,7 @@ const TimeSetting = props => { unit: timeout.replace(`${value}`, ''), } }, [timeout]) - + return (
{currentLocales[`datepicker.time_setting`]}
@@ -155,7 +155,7 @@ const TimeSetting = props => { ) - } + }
)} @@ -166,4 +166,4 @@ const TimeSetting = props => { ); }; -export default TimeSetting; \ No newline at end of file +export default TimeSetting; diff --git a/web/src/common/src/DatePicker/index.jsx b/web/src/common/src/DatePicker/index.jsx index b4292d08..4df44d36 100644 --- a/web/src/common/src/DatePicker/index.jsx +++ b/web/src/common/src/DatePicker/index.jsx @@ -11,6 +11,12 @@ import { toMilliseconds, fromMilliseconds } from "./utils/utils"; import styles from "./index.less"; const DEFAULT_COMMONLY_USED_RANGES = [ + { + start: "auto", + end: "auto", + label: "Auto", + key: "auto", + }, { start: "now/d", end: "now/d", @@ -96,6 +102,7 @@ const DatePicker = (props) => { recentlyUsedRangesKey, onRefreshChange, onRefresh, + showAutoTimeRange = false, } = props; const [prevQuickSelect, setPrevQuickSelect] = useState(); @@ -237,11 +244,12 @@ const DatePicker = (props) => { timeFields={timeFields} showTimeInterval={showTimeInterval} timeInterval={timeInterval} + timeIntervalDisabled={timeIntervalDisabled} showTimeout={showTimeout} timeout={timeout} autoFitLoading={autoFitLoading} timeZone={timeZone} - commonlyUsedRanges={commonlyUsedRanges} + commonlyUsedRanges={commonlyUsedRanges.filter((item) => item.key !== 'auto' || showAutoTimeRange)} recentlyUsedRanges={recentlyUsedRanges} isMinimum={isMinimum} prevQuickSelect={prevQuickSelect} diff --git a/web/src/common/src/DatePicker/index.less b/web/src/common/src/DatePicker/index.less index 9cd96bcc..bbf070cf 100644 --- a/web/src/common/src/DatePicker/index.less +++ b/web/src/common/src/DatePicker/index.less @@ -42,8 +42,8 @@ align-items: center; margin-left: 4px !important; .play { - min-width: 32px; - max-width: 32px; + min-width: 30px; + max-width: 30px; padding: 0; font-size: 14px; color: #1890ff; diff --git a/web/src/common/src/DatePicker/index.md b/web/src/common/src/DatePicker/index.md index 2868cb8d..b4ec9e70 100644 --- a/web/src/common/src/DatePicker/index.md +++ b/web/src/common/src/DatePicker/index.md @@ -32,4 +32,5 @@ | timeZone | 时区 | string | 'Asia/Shanghai' | 1.0.0 | | onTimeZoneChange | 时区变更的回调 | (timeZone: string) => void | - | 1.0.0 | | commonlyUsedRanges | 快速选择列表 | {start: string, end: string, label: string}[] | [] | 1.0.0 | -| recentlyUsedRangesKey | 时间范围历史字段 | string | - | 1.0.0 | \ No newline at end of file +| recentlyUsedRangesKey | 时间范围历史字段 | string | - | 1.0.0 | +| showAutoTimeRange | 是否显示自动 | boolean | false | 1.0.0 | \ No newline at end of file diff --git a/web/src/common/src/DatePicker/locales/en-US.js b/web/src/common/src/DatePicker/locales/en-US.js index 24a57dfc..b846a03a 100644 --- a/web/src/common/src/DatePicker/locales/en-US.js +++ b/web/src/common/src/DatePicker/locales/en-US.js @@ -1,4 +1,5 @@ export default { + "datepicker.quick_select.auto": "Auto", "datepicker.quick_select.auto_fit": "Auto fit", "datepicker.quick_select.today": "Today", "datepicker.quick_select.this_week": "This week", diff --git a/web/src/common/src/DatePicker/locales/zh-CN.js b/web/src/common/src/DatePicker/locales/zh-CN.js index accc0a49..d292d9fe 100644 --- a/web/src/common/src/DatePicker/locales/zh-CN.js +++ b/web/src/common/src/DatePicker/locales/zh-CN.js @@ -1,4 +1,5 @@ export default { + "datepicker.quick_select.auto": "自动", "datepicker.quick_select.auto_fit": "自动适配", "datepicker.quick_select.today": "今天", "datepicker.quick_select.this_week": "这个星期", diff --git a/web/src/common/src/DropdownList/index.jsx b/web/src/common/src/DropdownList/index.jsx index 7bc44abf..ee8a963d 100644 --- a/web/src/common/src/DropdownList/index.jsx +++ b/web/src/common/src/DropdownList/index.jsx @@ -18,7 +18,7 @@ const DropdownList = (props) => { popoverClassName = "", popoverPlacement = "bottomLeft", children, - width = 300, + width = 400, dropdownWidth, locale = "en-US", allowClear = false, diff --git a/web/src/components/Anchor/index.js b/web/src/components/Anchor/index.js index 57caba70..4e2b917e 100644 --- a/web/src/components/Anchor/index.js +++ b/web/src/components/Anchor/index.js @@ -1,88 +1,155 @@ -import React, { useState, useRef, useEffect } from 'react'; +import React, { useEffect, useRef, useState } from "react"; import { Tooltip, Timeline } from "antd"; import { formatMessage } from "umi/locale"; -import { throttle } from 'lodash'; import "./index.scss"; -function Anchor({ links }) { +const ANCHOR_SCROLL_OFFSET = 120; +const BOTTOM_SELECTION_EPSILON = 4; +const TOP_UPDATE_THRESHOLD = 6; + +function Anchor({ links = [] }) { const [activeID, setActiveID] = useState(links[0]); - const [isFixed, setIsFixed] = useState(false); + const [anchorStyle, setAnchorStyle] = useState({ top: 0 }); + const wrapperRef = useRef(null); const anchorRef = useRef(null); + const frameRef = useRef(null); + const lastTopRef = useRef(0); useEffect(() => { - const handleScroll = () => { + setActiveID(links[0]); + setAnchorStyle({ top: 0 }); + lastTopRef.current = 0; + }, [links]); + + useEffect(() => { + if (!links.length) { + return undefined; + } - setTimeout(() => { - const anchorElements = links.map(link => document.getElementById(link)); - const scrollPosition = window.scrollY || document.documentElement.scrollTop; + const updateAnchorPosition = () => { + frameRef.current = null; + const scrollPosition = window.scrollY || document.documentElement.scrollTop; + let nextActiveID = links[0]; + const lastExistingLink = [...links] + .reverse() + .find((link) => document.getElementById(link)); + const reachedPageBottom = + scrollPosition + window.innerHeight >= + document.documentElement.scrollHeight - BOTTOM_SELECTION_EPSILON; - for (let i = anchorElements.length - 1; i >= 0; i--) { - const anchorElement = anchorElements[i]; + if (reachedPageBottom && lastExistingLink) { + nextActiveID = lastExistingLink; + } else { + for (let i = links.length - 1; i >= 0; i--) { + const anchorElement = document.getElementById(links[i]); if (!anchorElement) { - continue + continue; } - const offsetTop = anchorElement.offsetTop - if (offsetTop <= scrollPosition - 200) { - setActiveID(anchorElement.id); + const offsetTop = + anchorElement.getBoundingClientRect().top + window.pageYOffset; + if (offsetTop <= scrollPosition + ANCHOR_SCROLL_OFFSET) { + nextActiveID = anchorElement.id; break; } } - }, 1000) + } - const offsetLeft = anchorRef.current.getBoundingClientRect().left; - const offsetTop = anchorRef.current.getBoundingClientRect().top; - const children = anchorRef.current.children[0]; + setActiveID((prevActiveID) => + prevActiveID === nextActiveID ? prevActiveID : nextActiveID + ); + + if (!wrapperRef.current || !anchorRef.current) { + return; + } - if (offsetTop <= 0) { - setIsFixed(true); - children.style.top = 0; - children.style.left = offsetLeft + 'px'; + const wrapperRect = wrapperRef.current.getBoundingClientRect(); + const activeElement = document.getElementById(nextActiveID); + const activeRect = activeElement?.getBoundingClientRect(); + const anchorHeight = anchorRef.current.offsetHeight || 0; + const wrapperHeight = wrapperRef.current.offsetHeight || 0; + const viewportPadding = 16; + const maxTopInWrapper = Math.max(wrapperHeight - anchorHeight, 0); + const minVisibleTop = Math.max(0, viewportPadding - wrapperRect.top); + const maxVisibleTop = Math.min( + maxTopInWrapper, + window.innerHeight - viewportPadding - wrapperRect.top - anchorHeight + ); + + let nextTop = activeRect ? activeRect.top - wrapperRect.top : 0; + if (maxVisibleTop >= minVisibleTop) { + nextTop = Math.max(nextTop, minVisibleTop); + nextTop = Math.min(nextTop, maxVisibleTop); } else { - setIsFixed(false); + nextTop = Math.max(nextTop, 0); + nextTop = Math.min(nextTop, maxTopInWrapper); + } + + if (Math.abs(lastTopRef.current - nextTop) >= TOP_UPDATE_THRESHOLD) { + lastTopRef.current = nextTop; + setAnchorStyle({ + top: nextTop, + }); } }; - const throttledScroll = throttle(handleScroll, 1000); + const handleScroll = () => { + if (frameRef.current !== null) { + return; + } + frameRef.current = window.requestAnimationFrame(updateAnchorPosition); + }; - window.addEventListener('scroll', throttledScroll); + updateAnchorPosition(); + window.addEventListener("scroll", handleScroll, { passive: true }); + window.addEventListener("resize", handleScroll); return () => { - window.removeEventListener('scroll', throttledScroll); + window.removeEventListener("scroll", handleScroll); + window.removeEventListener("resize", handleScroll); + if (frameRef.current !== null) { + window.cancelAnimationFrame(frameRef.current); + } }; - }, []); + }, [links]); const handleClick = (targetId) => { const targetElement = document.getElementById(targetId); if (targetElement) { - setActiveID(targetId) + const targetTop = + targetElement.getBoundingClientRect().top + + window.pageYOffset - + ANCHOR_SCROLL_OFFSET; + setActiveID(targetId); window.scrollTo({ - behavior: 'smooth', - top: targetElement.offsetTop + 300, + behavior: "smooth", + top: Math.max(targetTop, 0), }); } }; + if (!links.length) { + return null; + } + return ( -
-
- - {links.map((link, index) => { - const linkElement = document.getElementById(link) - return linkElement ? ( - - handleClick(link)} - > - 1 - - - ) : null - })} +
+
+ + {links.map((link) => ( + + handleClick(link)} + > + 1 + + + ))}
diff --git a/web/src/components/Anchor/index.scss b/web/src/components/Anchor/index.scss index 1dcf37cf..089b4957 100644 --- a/web/src/components/Anchor/index.scss +++ b/web/src/components/Anchor/index.scss @@ -1,38 +1,79 @@ -.p-anchor{ +.anchor-sticky-tabs.ant-tabs { + .ant-tabs-content, + .ant-tabs-content-no-animated, + .ant-tabs-right-content { + min-width: 0; + } + + .ant-tabs-right-content.ant-tabs-card-content, + .ant-tabs-left-content.ant-tabs-card-content { + margin-top: 8px; + } + + .ant-tabs-tabpane { + overflow: visible; + } + + &.ant-tabs-right .ant-tabs-bar, + &.ant-tabs-left .ant-tabs-bar { + padding-inline-start: 0; + margin-inline-start: 0; + border-inline-start: 0; + } + + &.ant-tabs-right .ant-tabs-tab, + &.ant-tabs-left .ant-tabs-tab { + margin-bottom: 8px; + } + +} + +.p-anchor { + position: relative; width: 30px; + flex: 0 0 12px; + margin-left: 12px; + align-self: stretch; box-sizing: border-box; } -.c-anchor{ + +.c-anchor { + position: absolute; + top: 0; + right: 0; width: 30px; box-sizing: border-box; padding-left: 10px; padding-top: 10px; cursor: pointer; + z-index: 2; + transition: top 0.14s ease-out; + will-change: top; + .ant-timeline-item-head { width: 12px; height: 12px; border: 1px solid; } + .ant-timeline-item-tail { left: 6px; } + .ant-timeline-item-head-blue { position: relative; } - .ant-timeline-item-head-blue::before{ - content: ""; + + .ant-timeline-item-head-blue::before { + content: ""; position: absolute; top: 50%; left: 50%; - width: 6px; - height: 6px; + width: 6px; + height: 6px; margin-top: -3px; - margin-left: -3px; + margin-left: -3px; background-color: #1890ff; border-radius: 50%; } } -.fixed-top { - position: fixed; - z-index: 10; -} \ No newline at end of file diff --git a/web/src/components/ClusterSelect/index.jsx b/web/src/components/ClusterSelect/index.jsx index 7e6cd44c..0836e79a 100644 --- a/web/src/components/ClusterSelect/index.jsx +++ b/web/src/components/ClusterSelect/index.jsx @@ -177,6 +177,9 @@ export default connect(({ global, loading }) => ({ }); dispatch({ type: "global/fetchClusterStatus", + payload: { + force: true, + }, }) }} loading={clusterLoading} diff --git a/web/src/components/CollectStatus/index.js b/web/src/components/CollectStatus/index.js index fbd868c9..7c554d4f 100644 --- a/web/src/components/CollectStatus/index.js +++ b/web/src/components/CollectStatus/index.js @@ -1,5 +1,5 @@ import request from "@/utils/request" -import { firstUpperCase, formatToUniversalTime } from "@/utils/utils"; +import { formatToUniversalTime } from "@/utils/utils"; import { Descriptions, Icon, Spin, Tooltip } from "antd"; import moment from "moment"; import { useEffect, useMemo, useRef, useState } from "react"; @@ -39,6 +39,15 @@ const STATUS_ICONS = { ) } +const getMetricCollectionModeLabel = (mode) => { + if (mode === "agent" || mode === "agentless") { + return formatMessage({ + id: `cluster.manage.metric_collection_mode.option.${mode}`, + }); + } + return "Unknown"; +} + export default (props) => { const { fetchUrl, filter={} } = props; @@ -131,9 +140,9 @@ export default (props) => {
{renderIcon()} - {firstUpperCase(data?.metric_collection_mode) || "Unknown"} + {getMetricCollectionModeLabel(data?.metric_collection_mode)}
); -} \ No newline at end of file +} diff --git a/web/src/components/GlobalHeader/DropdownSelect.js b/web/src/components/GlobalHeader/DropdownSelect.js index d3f1ea3d..661953a0 100644 --- a/web/src/components/GlobalHeader/DropdownSelect.js +++ b/web/src/components/GlobalHeader/DropdownSelect.js @@ -188,7 +188,7 @@ class DropdownSelect extends React.Component { > + typeof window !== "undefined" && + /(Mac|iPhone|iPad|iPod)/i.test(window.navigator.platform); + export default class GlobalHeaderRight extends PureComponent { state = { consoleVisible: false, notificationPopupVisible: false }; + bodyOverflow = ""; + bodyPaddingRight = ""; + rootPaddingBottom = ""; + + isConsoleToggleShortcut = (event) => { + const key = (event.key || "").toLowerCase(); + return (event.ctrlKey || event.metaKey) && event.shiftKey && (key === "o" || event.keyCode === 79); + }; + getNoticeData() { const { notices = [] } = this.props; if (notices.length === 0) { @@ -39,42 +52,63 @@ export default class GlobalHeaderRight extends PureComponent { }); return groupBy(newNotices, "type"); } + getScrollbarWidth = () => { + if (typeof window === "undefined") { + return 0; + } + return Math.max( + 0, + window.innerWidth - document.documentElement.clientWidth + ); + }; setConsoleVisible = (visible) => { + const body = document.body; + const root = document.querySelector("#root>div"); this.setState({ consoleVisible: visible, }); - var sl = document.querySelector("#root>div"); - if (sl) { - sl.style.paddingBottom = "0px"; + if (body) { + if (visible) { + this.bodyOverflow = body.style.overflow; + this.bodyPaddingRight = body.style.paddingRight; + body.style.overflow = "hidden"; + body.style.paddingRight = `${this.getScrollbarWidth()}px`; + } else { + body.style.overflow = this.bodyOverflow; + body.style.paddingRight = this.bodyPaddingRight; + } + } + if (root) { + root.style.paddingBottom = "0px"; } }; onKeyDown = (e) => { - const { keyCode } = e; - if (this.keysPressed["17"] && this.keysPressed["16"] && keyCode == 79) { - if (this.state.consoleVisible) document.body.style.overflow = ""; + const hasDevtoolPrivilege = + hasAuthority("devtool.console:all") || + hasAuthority("devtool.console:read"); + if (hasDevtoolPrivilege && this.isConsoleToggleShortcut(e)) { + e.preventDefault(); this.setConsoleVisible(!this.state.consoleVisible); return true; } - this.keysPressed[keyCode] = e.type == "keydown"; return false; }; - onKeyUp = (e) => { - const { keyCode } = e; - delete this.keysPressed[keyCode]; - }; + componentWillUnmount() { + const body = document.body; + if (body) { + body.style.overflow = this.bodyOverflow; + body.style.paddingRight = this.bodyPaddingRight; + } + } constructor(props) { super(props); this.onKeyDown = this.onKeyDown.bind(this); - this.onKeyUp = this.onKeyUp.bind(this); } componentDidMount() { - this.keysPressed = {}; document.addEventListener("keydown", this.onKeyDown, false); - document.addEventListener("keyup", this.onKeyUp, false); } componentWillUnmount() { document.removeEventListener("keydown", this.onKeyDown); - document.removeEventListener("keyup", this.onKeyUp); } render() { @@ -132,6 +166,9 @@ export default class GlobalHeaderRight extends PureComponent { const hasDevtoolPrivilege = hasAuthority("devtool.console:all") || hasAuthority("devtool.console:read"); + const consoleShortcutLabel = isMacPlatform() + ? "Cmd+Shift+O" + : "Ctrl+Shift+O"; return (
@@ -167,16 +204,17 @@ export default class GlobalHeaderRight extends PureComponent { {hasDevtoolPrivilege ? ( - { - const { history, selectedCluster } = this.props; - this.setConsoleVisible(!this.state.consoleVisible); - }} - > - {" "} - - + + { + this.setConsoleVisible(!this.state.consoleVisible); + }} + > + + + ) : null} {APP_OFFICIAL_WEBSITE ? ( )}
diff --git a/web/src/components/GlobalHeader/index.js b/web/src/components/GlobalHeader/index.js index 92c5d267..0f21a5c3 100644 --- a/web/src/components/GlobalHeader/index.js +++ b/web/src/components/GlobalHeader/index.js @@ -60,7 +60,7 @@ export default class GlobalHeader extends PureComponent { {clusterList.length > 0 && _.isObject(this.props.clusterStatus) && clusterVisible && (
{ diff --git a/web/src/components/GlobalHeader/index.less b/web/src/components/GlobalHeader/index.less index 5e737c37..04beb401 100644 --- a/web/src/components/GlobalHeader/index.less +++ b/web/src/components/GlobalHeader/index.less @@ -136,3 +136,7 @@ i.trigger { [tabindex] { outline: none !important; } + +:global(.antd-pro-components-notice-icon-index-noticeButton) { + display: none !important; +} diff --git a/web/src/components/HealthProvider/index.js b/web/src/components/HealthProvider/index.js index 7ee4be04..dbca20c7 100644 --- a/web/src/components/HealthProvider/index.js +++ b/web/src/components/HealthProvider/index.js @@ -12,11 +12,25 @@ export default ({ children, location }) => { const [health, setHealth] = useState() const [visible, setVisible] = useState(false) const intervalRef = useRef(null); + const isMountedRef = useRef(false); + const fetchSeqRef = useRef(0); + const pathname = location?.pathname || ""; + const shouldSuppressHealthModal = + !pathname || + pathname.includes('/guide/initialization') || + pathname.includes('/devtool'); const fetchHealth = async () => { + const fetchSeq = ++fetchSeqRef.current; try { + if (!isMountedRef.current) { + return; + } setModalLoading(true) const res = await getHealth(); + if (!isMountedRef.current || fetchSeq !== fetchSeqRef.current) { + return; + } if(res instanceof Error && res.name === "ERR_CONNECTION_REFUSED"){ setHealth({}) setModalLoading(false) @@ -28,6 +42,9 @@ export default ({ children, location }) => { router.push("/guide/initialization"); } } catch (error) { + if (!isMountedRef.current || fetchSeq !== fetchSeqRef.current) { + return; + } setModalLoading(false) console.log(error); message.error('Check servies health failed!') @@ -35,6 +52,10 @@ export default ({ children, location }) => { } const checkStatus = (health) => { + if (shouldSuppressHealthModal) { + setVisible(false) + return; + } const { status } = health if (['green', 'yellow'].includes(status)) { setVisible(false) @@ -44,29 +65,37 @@ export default ({ children, location }) => { } useEffect(() => { - if (!location?.pathname || location?.pathname.includes('/guide/initialization')) { + fetchSeqRef.current += 1; + if (shouldSuppressHealthModal) { if (intervalRef.current) { clearInterval(intervalRef.current) + intervalRef.current = null; } setHealth() + setVisible(false) return; } - if (!health) { + fetchHealth() + if (intervalRef.current) { + clearInterval(intervalRef.current) + } + intervalRef.current = setInterval(() => { fetchHealth() + }, 5 * 60 * 1000) + return () => { + fetchSeqRef.current += 1; if (intervalRef.current) { clearInterval(intervalRef.current) + intervalRef.current = null; } - intervalRef.current = setInterval(() => { - fetchHealth(true) - }, [5*60*1000]) } - }, [location?.pathname, JSON.stringify(health)]) + }, [shouldSuppressHealthModal, location?.pathname]) useEffect(() => { if (health) { checkStatus(health) } - }, [JSON.stringify(health)]) + }, [health]) useEffect(() => { if (health?.setup_required) { @@ -75,12 +104,16 @@ export default ({ children, location }) => { }, [health?.setup_required]) useEffect(() => { + isMountedRef.current = true; window.setGlobalHealth = setHealth; return () => { + isMountedRef.current = false; + fetchSeqRef.current += 1; if (intervalRef.current) { clearInterval(intervalRef.current) - window.setGlobalHealth = null; + intervalRef.current = null; } + window.setGlobalHealth = null; } }, []) @@ -100,7 +133,7 @@ export default ({ children, location }) => { {children} diff --git a/web/src/components/Icons/Dingding.jsx b/web/src/components/Icons/Dingding.jsx index af08c44f..16d84716 100644 --- a/web/src/components/Icons/Dingding.jsx +++ b/web/src/components/Icons/Dingding.jsx @@ -1,6 +1,6 @@ const Dingding = ({color}) => { return ( - + ) } export default Dingding; diff --git a/web/src/components/Icons/DirectionArrow.jsx b/web/src/components/Icons/DirectionArrow.jsx index 291cc6ab..3bb351d9 100644 --- a/web/src/components/Icons/DirectionArrow.jsx +++ b/web/src/components/Icons/DirectionArrow.jsx @@ -1,3 +1,3 @@ export default ()=>{ - return + return } \ No newline at end of file diff --git a/web/src/components/Icons/ExternalLink.jsx b/web/src/components/Icons/ExternalLink.jsx index 9286284c..79129897 100644 --- a/web/src/components/Icons/ExternalLink.jsx +++ b/web/src/components/Icons/ExternalLink.jsx @@ -1,3 +1,3 @@ export default ()=>{ - return + return } \ No newline at end of file diff --git a/web/src/components/Icons/HornVibration.jsx b/web/src/components/Icons/HornVibration.jsx index 9be24daa..e6cc294d 100644 --- a/web/src/components/Icons/HornVibration.jsx +++ b/web/src/components/Icons/HornVibration.jsx @@ -1,3 +1,3 @@ export default ()=>{ - return + return } \ No newline at end of file diff --git a/web/src/components/Icons/LightningToggle.jsx b/web/src/components/Icons/LightningToggle.jsx index f0f2d009..d7694f63 100644 --- a/web/src/components/Icons/LightningToggle.jsx +++ b/web/src/components/Icons/LightningToggle.jsx @@ -2,7 +2,7 @@ export default () => { return ( { - return + return } \ No newline at end of file diff --git a/web/src/components/Icons/Mute.jsx b/web/src/components/Icons/Mute.jsx index fe01544d..a99f664a 100644 --- a/web/src/components/Icons/Mute.jsx +++ b/web/src/components/Icons/Mute.jsx @@ -1,5 +1,5 @@ export default () => { return ( - + ); }; diff --git a/web/src/components/Icons/NoPermission.jsx b/web/src/components/Icons/NoPermission.jsx index 5c1deda7..3094e5e1 100644 --- a/web/src/components/Icons/NoPermission.jsx +++ b/web/src/components/Icons/NoPermission.jsx @@ -1,6 +1,6 @@ export default () => { return ( - + ); }; \ No newline at end of file diff --git a/web/src/components/Icons/Notification.jsx b/web/src/components/Icons/Notification.jsx index de1b230d..754f3c7b 100644 --- a/web/src/components/Icons/Notification.jsx +++ b/web/src/components/Icons/Notification.jsx @@ -1,6 +1,6 @@ export default () => { return ( - + ); }; diff --git a/web/src/components/Icons/Slack.jsx b/web/src/components/Icons/Slack.jsx index d230a64c..1c59728f 100644 --- a/web/src/components/Icons/Slack.jsx +++ b/web/src/components/Icons/Slack.jsx @@ -1,11 +1,11 @@ export default () => { return ( - + ) } export const SlackWithColor = ({color}) => { return ()=>{ - return + return } } \ No newline at end of file diff --git a/web/src/components/IndexPatternSelect/index.jsx b/web/src/components/IndexPatternSelect/index.jsx index 145727ed..64e52f33 100644 --- a/web/src/components/IndexPatternSelect/index.jsx +++ b/web/src/components/IndexPatternSelect/index.jsx @@ -1,12 +1,22 @@ import DropdownList from "@/common/src/DropdownList"; -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { HealthStatusCircle } from "../infini/health_status_circle"; -import { getLocale } from "umi/locale"; +import { formatMessage, getLocale } from "umi/locale"; import { Icon } from "antd"; import Link from "umi/link"; import styles from "./index.less"; +import request from "@/utils/request"; +import { ESPrefix } from "@/services/common"; + +function formatCount(count) { + if (count == null) return ""; + if (count >= 1000000) return `${(count / 1000000).toFixed(1)}M`; + if (count >= 1000) return `${(count / 1000).toFixed(1)}K`; + return `${count}`; +} export default (props) => { + const t = (id, defaultMessage) => formatMessage({ id, defaultMessage }); const { selectedIndexPattern, onIndexPatternChange, views = [], indices = [] } = props; @@ -14,6 +24,26 @@ export default (props) => { const [filters, setFilters] = useState({ type: ['view', 'index']}) const [groups, setGroups] = useState([]) const [showGroup, setShowGroup] = useState(false) + const [indexCounts, setIndexCounts] = useState({}); + + // Extract clusterID from URL path + useEffect(() => { + const match = window.location.hash.match(/\/elasticsearch\/([^/?]+)/); + if (match && match[1]) { + const clusterID = match[1]; + request(`${ESPrefix}/${clusterID}/_cat/indices`, { method: "GET" }) + .then((res) => { + if (res && !res.error) { + const counts = {}; + Object.keys(res).forEach((key) => { + counts[key] = res[key].docs_count || 0; + }); + setIndexCounts(counts); + } + }) + .catch(() => {}); + } + }, [indices]); const formatData = useMemo(() => { const formatViews = views?.map((item) => ({ @@ -68,20 +98,29 @@ export default (props) => { rowKey="id" data={formatData} renderItem={(item) => ( - <> -
- {item.type === 'view' ? : } -
- {item.name} - +
+ + + {item.type === 'view' ? : } + + {item.name} + + {item.type !== 'view' && indexCounts[item.id] != null && ( + + {formatCount(indexCounts[item.id])} + + )} +
)} renderLabel={(item) => item.name} renderEmptyList={() => { let label = "Create Index"; let link = "/data/index" if (showGroup && groups[0]?.value === 'view') { - label = "Create View"; + label = t("explore.view.btn.create", "Create View"); link = "/data/views/create" + } else { + label = t("explore.index.btn.create", "Create Index"); } const action = ( diff --git a/web/src/components/IndexSelect/index.jsx b/web/src/components/IndexSelect/index.jsx index da868424..1a8f264c 100644 --- a/web/src/components/IndexSelect/index.jsx +++ b/web/src/components/IndexSelect/index.jsx @@ -5,10 +5,14 @@ import { Icon } from "antd"; export default (props) => { - const {indices = [], onChange, mode, placeholder, renderItem, labelField="index", keyField="index", allowClear } = props; + const {indices = [], onChange, value: valueProp, mode, placeholder, renderItem, labelField="index", keyField="index", allowClear } = props; const [sorter, setSorter] = useState([]) - const [value, setValue] = useState([]); + const [value, setValue] = useState(valueProp || []); + + useMemo(() => { + setValue(valueProp || []); + }, [valueProp]); return ( { +export default ({autoInit = false, centerToggle = false, showAdvanced = true}) => { const [tokenLoading, setTokenLoading] = useState(false); + const [enableReverseChannel, setEnableReverseChannel] = useState(false); + const [noService, setNoService] = useState(false); + const [advancedVisible, setAdvancedVisible] = useState(false); const [seletedGateways, setSeletedGateways] = useState([]); const [tokenInfo, setTokenInfo] = useState(); - const fetchTokenInfo = async () => { + const fetchTokenInfo = async ( + reverseChannelEnabled = enableReverseChannel, + noServiceEnabled = noService + ) => { setTokenInfo() setTokenLoading(true) const res = await request('/instance/_generate_install_script', { method: "POST", body: { - gateway_endpoints: seletedGateways + gateway_endpoints: seletedGateways, + enable_reverse_channel: reverseChannelEnabled, + no_service: noServiceEnabled, } }) setTokenInfo(res) @@ -55,6 +62,71 @@ export default ({autoInit = false}) => { id:"agent.install.setup.desc" })}：

+ {showAdvanced ? ( +
+ + {advancedVisible ? ( +
+
+ + {formatMessage({ id: "agent.install.reverse_channel.label" })} + + + + + { + setEnableReverseChannel(checked); + if (autoInit || tokenInfo) { + fetchTokenInfo(checked, noService); + } + }} + /> +
+
+ + {formatMessage({ id: "agent.install.no_sudo.label" })} + + + + + { + setNoService(checked); + if (autoInit || tokenInfo) { + fetchTokenInfo(enableReverseChannel, checked); + } + }} + /> +
+
+ ) : null} +
+ ) : null} +
+
+ {formatMessage({ id: "agent.install.tips.intranet.title" })} +
+
{formatMessage({ id: "agent.install.tips.intranet.desc" })}
+
{tokenInfo.script} @@ -62,40 +134,82 @@ export default ({autoInit = false}) => { type="copy" className={styles.copy} onClick={ - () => message.success(formatMessage({ - id: "agent.install.setup.copy.success" - })) + () => message.open({ + type: "success", + key: "agent-install-copy-success", + content: formatMessage({ + id: "agent.install.setup.copy.success" + }), + }) } />
+ {noService ? ( + +
{formatMessage({ id: "agent.install.no_sudo.tip.desc" })}
+
+ {formatMessage({ id: "agent.install.no_sudo.entrypoint.title" })} +
+ + ENTRYPOINT ["sh", "-c", "cd /path/to/agent && exec ./agent-* -config agent.yml"] + +
+ {formatMessage({ id: "agent.install.no_sudo.cmd.title" })} +
+ + CMD ["sh", "-c", "cd /path/to/agent && exec ./agent-* -config agent.yml"] + +
+ } + /> + ) : null}
{formatMessage({ id:"agent.install.tips.title" })}：
- {/*

- · 支持的自定义变量如下，均为可选参数, 多个环境变量之间以空格分割: +

+ · {formatMessage({ + id:"agent.install.tips.target" + })} -t /opt/agent +

+

+ · {formatMessage({ + id:"agent.install.tips.version" + })} -v 1.30.3-2407

-
-

- BASE_URL: Agent安装包的下载地址，如: https://release.infinilabs.com/agent/stable -

-

- AGENT_VER: Agent版本号，如: 0.4.0-126 -

+

+ · {formatMessage({ + id:"agent.install.tips.download" + })} -u http://192.168.1.1:8080 +

+

+ · {formatMessage({ + id:"agent.install.tips.server" + })} -s http://192.168.1.1:9000 +

+ {noService ? (

- INSTALL_PATH: Agent安装目录, 如: /opt + · {formatMessage({ + id:"agent.install.no_sudo.help_line" + })} --no-service

-
*/} + ) : null}

· {formatMessage({ id:"agent.install.tips.desc" })} {formatMessage({ id:"agent.install.link.manual_install" - })}> + })}

diff --git a/web/src/components/InstallAgent/index.less b/web/src/components/InstallAgent/index.less index 8606c79c..ebc2a423 100644 --- a/web/src/components/InstallAgent/index.less +++ b/web/src/components/InstallAgent/index.less @@ -1,6 +1,74 @@ .installAgent { + .advancedWrap { + margin-bottom: 16px; + } + .advancedWrapCentered { + text-align: left; + } + .advancedToggle { + padding: 0; + height: auto; + color: rgba(0, 0, 0, 0.65); + font-size: 12px; + display: inline-flex; + align-items: center; + gap: 6px; + } + .toggleStack { + display: flex; + flex-direction: row; + flex-wrap: wrap; + align-items: center; + column-gap: 24px; + row-gap: 12px; + margin-top: 8px; + } + .toggleStackCentered { + justify-content: left; + align-items: center; + } + .reverseChannelToggle { + display: flex; + align-items: center; + gap: 8px; + flex: 0 0 auto; + } + .reverseChannelLabel { + display: inline-flex; + align-items: center; + gap: 4px; + } + .reverseChannelInfo { + color: rgba(0, 0, 0, 0.45); + font-size: 12px; + cursor: pointer; + } .gateway { - margin-bottom: 20px; + display: block; + margin: 0 auto 20px; + } + .noServiceAlert { + margin-bottom: 16px; + text-align: left; + } + .noServiceAlertContent { + display: flex; + flex-direction: column; + gap: 8px; + } + .noServiceCodeTitle { + font-weight: 500; + color: #101010; + } + .noServiceCodeBlock { + display: block; + padding: 8px 10px; + border-radius: 4px; + background: #f5f5f5; + color: #434343; + white-space: pre-wrap; + word-break: break-word; + font-family: "SFMono-Regular", Monaco, Menlo, Consolas, "Liberation Mono", "Ubuntu Mono", monospace; } .shell { .installtitle{ @@ -19,7 +87,11 @@ padding: 12px 30px 12px 12px; background: rgb(241, 242, 245); position: relative; - word-break: break-all; + text-align: left; + line-height: 1.6; + white-space: pre-wrap; + word-break: break-word; + font-family: "SFMono-Regular", Monaco, Menlo, Consolas, "Liberation Mono", "Ubuntu Mono", monospace; margin-bottom: 30px; .copy { @@ -35,19 +107,22 @@ .title { font-weight: 400; color: #101010; - font-size: 14px; + font-size: 12px; margin-bottom: 8px; } .content { - font-size: 14px; + font-size: 12px; color: #565656; p { margin-bottom: 6px; } + code { + font-size: 12px; + } .children { margin: 0 8px; } } } } -} \ No newline at end of file +} diff --git a/web/src/components/InstallGateway/index.js b/web/src/components/InstallGateway/index.js new file mode 100644 index 00000000..948a450b --- /dev/null +++ b/web/src/components/InstallGateway/index.js @@ -0,0 +1,317 @@ +import { Alert, Button, Icon, message, Select, Spin, Switch, Tooltip } from "antd"; +import { CopyToClipboard } from "react-copy-to-clipboard"; +import { useEffect, useState } from "react"; +import { formatMessage } from "umi/locale"; + +import request from "@/utils/request"; +import { getDocPathByLang } from "@/utils/utils"; + +const shellContainerStyle = { + borderRadius: 5, + fontSize: 14, + padding: "12px 36px 12px 12px", + background: "rgb(241, 242, 245)", + position: "relative", + textAlign: "left", + lineHeight: 1.6, + whiteSpace: "pre-wrap", + wordBreak: "break-word", + fontFamily: + '"SFMono-Regular", Monaco, Menlo, Consolas, "Liberation Mono", "Ubuntu Mono", monospace', + marginBottom: 24, +}; + +const intranetNoticeStyle = { + background: "#f6ffed", + border: "1px solid #b7eb8f", + borderRadius: 6, + padding: "10px 12px", + marginBottom: 16, + color: "#565656", + fontSize: 12, + lineHeight: 1.8, +}; + +const advancedWrapStyle = { + marginBottom: 16, + textAlign: "left", +}; + +const advancedToggleStyle = { + padding: 0, + height: "auto", + color: "rgba(0, 0, 0, 0.65)", + fontSize: 12, + display: "inline-flex", + alignItems: "center", + gap: 6, +}; + +const toggleRowStyle = { + display: "flex", + flexWrap: "wrap", + alignItems: "center", + columnGap: 24, + rowGap: 12, + marginTop: 8, +}; + +const toggleItemStyle = { + display: "flex", + alignItems: "center", + gap: 8, +}; + +const toggleLabelStyle = { + display: "inline-flex", + alignItems: "center", + gap: 4, +}; + +const infoIconStyle = { + color: "rgba(0, 0, 0, 0.45)", + fontSize: 12, + cursor: "pointer", +}; + +const noServiceAlertStyle = { + marginBottom: 16, + textAlign: "left", +}; + +const noServiceContentStyle = { + display: "flex", + flexDirection: "column", + gap: 8, +}; + +const noServiceCodeTitleStyle = { + fontWeight: 500, + color: "#101010", +}; + +const noServiceCodeStyle = { + display: "block", + padding: "8px 10px", + borderRadius: 4, + background: "#f5f5f5", + color: "#434343", + whiteSpace: "pre-wrap", + wordBreak: "break-word", + fontFamily: + '"SFMono-Regular", Monaco, Menlo, Consolas, "Liberation Mono", "Ubuntu Mono", monospace', +}; + +export default ({ autoInit = false, defaultGatewayType = "migration" }) => { + const { Option } = Select; + const defaultRelayRole = "primary"; + const [loading, setLoading] = useState(false); + const [tokenInfo, setTokenInfo] = useState(); + const [noService, setNoService] = useState(false); + const [gatewayType, setGatewayType] = useState(defaultGatewayType); + const [relayRole, setRelayRole] = useState(defaultRelayRole); + const [advancedVisible, setAdvancedVisible] = useState(false); + + const fetchTokenInfo = async ( + noServiceEnabled = noService, + gatewayTypeValue = gatewayType, + relayRoleValue = relayRole + ) => { + setTokenInfo(undefined); + setLoading(true); + const res = await request("/instance/_generate_gateway_install_script", { + method: "POST", + body: { + no_service: noServiceEnabled, + service_type: gatewayTypeValue, + relay_role: gatewayTypeValue === "relay" ? relayRoleValue : undefined, + }, + }); + setTokenInfo(res); + setLoading(false); + }; + + useEffect(() => { + if (autoInit) { + fetchTokenInfo(false, defaultGatewayType, defaultRelayRole); + } + }, [autoInit, defaultGatewayType]); + + useEffect(() => { + setGatewayType(defaultGatewayType); + setRelayRole(defaultRelayRole); + }, [defaultGatewayType]); + + return ( + + {!autoInit && ( + + )} + {tokenInfo && ( +
+
+ {formatMessage({ id: "gateway.guide.quick_install" })} +
+
+ {formatMessage({ id: "gateway.guide.quick_install.desc" })} +
+
+ + {advancedVisible ? ( +
+
+ + {formatMessage({ id: "gateway.install.type.label" })} + + +
+ {gatewayType === "relay" ? ( +
+ + {formatMessage({ id: "gateway.install.relay_role.label" })} + + +
+ ) : null} +
+ + {formatMessage({ id: "gateway.install.no_sudo.label" })} + + + + + { + setNoService(checked); + if (autoInit || tokenInfo) { + fetchTokenInfo(checked, gatewayType, relayRole); + } + }} + /> +
+
+ ) : null} +
+
+
+ {formatMessage({ id: "gateway.guide.intranet.title" })} +
+
{formatMessage({ id: "gateway.guide.intranet.desc" })}
+
+
+ {tokenInfo.script} + + + message.success(formatMessage({ id: "gateway.guide.shell.copy.success" })) + } + /> + +
+ {noService ? ( + +
{formatMessage({ id: "gateway.install.no_sudo.tip.desc" })}
+
+ {formatMessage({ id: "gateway.install.no_sudo.command.title" })} +
+ + cd /path/to/gateway && ./gateway-* -config gateway.yml + +
+ } + /> + ) : null} +
+
+ {formatMessage({ id: "gateway.guide.tips.title" })} +
+

+ · {formatMessage({ id: "gateway.guide.tips.version" })}{" "} + -v 1.30.3-2407 +

+

+ · {formatMessage({ id: "gateway.guide.tips.directory" })}{" "} + -d /opt/gateway +

+

+ · {formatMessage({ id: "gateway.guide.tips.download_source" })}{" "} + -u http://192.168.1.1:8080 +

+ {noService ? ( +

+ · {formatMessage({ id: "gateway.install.no_sudo.help_line" })}{" "} + --no-service +

+ ) : null} +

+ · {formatMessage({ id: "gateway.guide.tips.content" })}{" "} + + {formatMessage({ id: "gateway.guide.tips.install_manually" })} + +

+
+
+ )} + + ); +}; diff --git a/web/src/components/Licence/index.js b/web/src/components/Licence/index.js index a2f9066d..f11371dd 100644 --- a/web/src/components/Licence/index.js +++ b/web/src/components/Licence/index.js @@ -51,7 +51,7 @@ export default forwardRef((props, ref) => { return ( { const showDeleteConfirm = (record) => { Modal.confirm({ - title: "Are you sure delete this item?", + title: formatMessage({ id: "app.message.confirm.delete" }), content: ( <>
Name: {record.name}
Endpoint: {record.endpoint}
), - okText: "Yes", + okText: formatMessage({ id: "form.button.ok" }), okType: "danger", - cancelText: "No", + cancelText: formatMessage({ id: "form.button.cancel" }), onOk() { onDeleteClick(record.id); }, diff --git a/web/src/components/ListView/components/DatePicker/index.jsx b/web/src/components/ListView/components/DatePicker/index.jsx index cd6cf2d4..79546067 100644 --- a/web/src/components/ListView/components/DatePicker/index.jsx +++ b/web/src/components/ListView/components/DatePicker/index.jsx @@ -4,6 +4,28 @@ import { message } from "antd"; import request from "@/utils/request"; import DatePicker from "@/common/src/DatePicker"; +const normalizeTimeValue = (value, fallback) => { + if (typeof value === "string" || typeof value === "number") { + return `${value}`; + } + if (value && typeof value === "object") { + const keys = ["from", "to", "min", "max", "gte", "lte", "start", "end"]; + for (const key of keys) { + if ( + Object.prototype.hasOwnProperty.call(value, key) && + value[key] !== undefined && + value[key] !== null + ) { + const candidate = value[key]; + if (typeof candidate === "string" || typeof candidate === "number") { + return `${candidate}`; + } + } + } + } + return fallback; +}; + export default (props) => { const { locale = "en-US", @@ -15,6 +37,7 @@ export default (props) => { isRefreshPaused = true, onRefresh, recentlyUsedRangesKey = "listview-recently-used-ranges", + wrapperStyle = {}, } = props; if (timeFields.length == 0) { @@ -23,8 +46,8 @@ export default (props) => { const [range] = useMemo(() => { let range = { - start: timeRange.from || "now-15m", - end: timeRange.to || "now", + start: normalizeTimeValue(timeRange.from, "now-15m"), + end: normalizeTimeValue(timeRange.to, "now"), timeField: timeRange.timeField || "", }; return [range]; @@ -90,8 +113,15 @@ export default (props) => { // setAutoFitLoading(false); // }; + const containerStyle = { + width: "460px", + maxWidth: "55vw", + minWidth: 320, + ...wrapperStyle, + }; + return ( -
+
{ timeZone={currentTimeZone} onTimeZoneChange={setCurrentTimeZone} recentlyUsedRangesKey={recentlyUsedRangesKey} + showAutoTimeRange={true} />
); diff --git a/web/src/components/ListView/components/Search/index.jsx b/web/src/components/ListView/components/Search/index.jsx index 5fff3831..2c332e0f 100644 --- a/web/src/components/ListView/components/Search/index.jsx +++ b/web/src/components/ListView/components/Search/index.jsx @@ -1,20 +1,27 @@ import { Input } from "antd"; +import { formatMessage } from "umi/locale"; const { Search } = Input; export default (props) => { const { value = "", onSearch, placeholder = null } = props; + const handleSearch = (nextValue) => { + onSearch(`${nextValue ?? ""}`.trim()); + }; return ( { onSearch(e.currentTarget.value); }} + onBlur={(e) => { + handleSearch(e.currentTarget.value); + }} /> ); }; diff --git a/web/src/components/ListView/components/Side/SearchFacet.less b/web/src/components/ListView/components/Side/SearchFacet.less index be367094..19356f99 100644 --- a/web/src/components/ListView/components/Side/SearchFacet.less +++ b/web/src/components/ListView/components/Side/SearchFacet.less @@ -11,9 +11,21 @@ .value { display: flex; align-items: center; + gap: 8px; + min-width: 0; :global { label.ant-checkbox-wrapper { - width: 152px; + display: flex; + align-items: center; + flex: 1 1 auto; + min-width: 0; + } + label.ant-checkbox-wrapper > span:first-child { + flex: 0 0 auto; + } + label.ant-checkbox-wrapper > span:last-child { + flex: 1 1 auto; + min-width: 0; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; @@ -21,9 +33,11 @@ } .count { margin-left: auto; + flex: 0 0 auto; color: #888; font-size: 0.85em; text-align: right; + white-space: nowrap; } } } diff --git a/web/src/components/ListView/components/Side/index.jsx b/web/src/components/ListView/components/Side/index.jsx index e20ecddd..13e8ce89 100644 --- a/web/src/components/ListView/components/Side/index.jsx +++ b/web/src/components/ListView/components/Side/index.jsx @@ -8,15 +8,22 @@ export default (props) => { const { aggs, data, filters, onFacetChange, onReset } = props; const facets = useMemo(() => { - const fts = Object.keys(data).map((item) => { - return { - label: item, - field: aggs[item].terms.field, - buckets: data[item]?.buckets || [], - }; - }); + const safeData = data && typeof data === "object" ? data : {}; + const fts = Object.keys(safeData).reduce((items, item) => { + const agg = aggs?.[item]; + const field = agg?.field || agg?.terms?.field; + if (!field) { + return items; + } + items.push({ + label: agg?.label || item, + field, + buckets: safeData[item]?.buckets || [], + }); + return items; + }, []); return fts; - }, [data]); + }, [data, aggs]); return (
diff --git a/web/src/components/ListView/components/TimeLine/index.jsx b/web/src/components/ListView/components/TimeLine/index.jsx index 7a6c660c..7456e630 100644 --- a/web/src/components/ListView/components/TimeLine/index.jsx +++ b/web/src/components/ListView/components/TimeLine/index.jsx @@ -23,21 +23,25 @@ export default (props) => { } }, [data]); + const hasMore = total > dataNew.length; + return (
{ - if (typeof onNext == "function") { + if (!loading && typeof onNext == "function") { onNext(dataNew.length); } }} - hasMore={!loading && total > dataNew.length} + hasMore={hasMore} loader={ -

- Loading... -

+ loading ? ( +
+ Loading... +
+ ) : null } endMessage={null} > @@ -51,6 +55,18 @@ export default (props) => { })} + {!loading && hasMore && dataNew.length > 0 && ( +
{ + if (typeof onNext == "function") { + onNext(dataNew.length); + } + }} + > + Click to load more +
+ )}
); diff --git a/web/src/components/ListView/index.jsx b/web/src/components/ListView/index.jsx index 9a7fe09e..2e23ce30 100644 --- a/web/src/components/ListView/index.jsx +++ b/web/src/components/ListView/index.jsx @@ -10,6 +10,7 @@ import React, { Fragment, } from "react"; import { Card, Table, Button, Input, Icon, Switch, Empty, Spin } from "antd"; +import isEqual from "lodash/isEqual"; import styles from "./index.less"; import { formatMessage, getLocale } from "umi/locale"; import { getTimezone } from "@/utils/utils"; @@ -27,6 +28,122 @@ import DatePicker from "./components/DatePicker"; import TimeLine from "./components/TimeLine"; import useResizeObserver from "@react-hook/resize-observer"; import { WidgetRender } from "@/pages/DataManagement/View/WidgetLoader"; +import { buildContainsQueryString } from "@/lib/elasticsearch/util"; +import moment from "moment"; + +const normalizeTimeRangeValue = (value, fallback = "") => { + if (typeof value === "string" || typeof value === "number") { + return `${value}`; + } + if (value && typeof value === "object") { + const candidates = ["from", "to", "min", "max", "gte", "lte", "start", "end"]; + for (const key of candidates) { + if ( + Object.prototype.hasOwnProperty.call(value, key) && + value[key] !== undefined && + value[key] !== null + ) { + const candidate = value[key]; + if (typeof candidate === "string" || typeof candidate === "number") { + return `${candidate}`; + } + } + } + } + return fallback; +}; + +const normalizeTimeRange = (timeRange = {}, fallback = {}) => { + const merged = { + ...(fallback || {}), + ...(timeRange || {}), + }; + return { + ...merged, + from: normalizeTimeRangeValue(merged.from, fallback?.from || "now-7d"), + to: normalizeTimeRangeValue(merged.to, fallback?.to || "now"), + timeField: + typeof merged.timeField === "string" + ? merged.timeField + : fallback?.timeField || "", + }; +}; + +const normalizeHistogramRange = (timeRange = {}) => { + const from = normalizeTimeRangeValue(timeRange?.from, "now-15m"); + const to = normalizeTimeRangeValue(timeRange?.to, "now"); + if (from === "auto" || to === "auto") { + return { + from: "auto", + to: "auto", + }; + } + return { from, to }; +}; + +const resolveAutoHistogramRange = (timeRange = {}, aggregations = {}) => { + const normalized = normalizeHistogramRange(timeRange); + const isAuto = + normalizeTimeRangeValue(timeRange?.from, "") === "auto" || + normalizeTimeRangeValue(timeRange?.to, "") === "auto"; + const minValue = aggregations?.__listview_min_time?.value; + const maxValue = aggregations?.__listview_max_time?.value; + if ( + Number.isFinite(minValue) && + Number.isFinite(maxValue) && + minValue > 0 && + maxValue > 0 + ) { + const minTs = Number(minValue); + const maxTs = Number(maxValue); + if (maxTs <= minTs) { + const center = maxTs > 0 ? maxTs : Date.now(); + return { + from: moment(center).subtract(15, "minutes").toISOString(), + to: moment(center).toISOString(), + }; + } + return { + from: moment(minTs).toISOString(), + to: moment(maxTs).toISOString(), + }; + } + if (isAuto) { + return { + from: moment().subtract(15, "minutes").toISOString(), + to: moment().toISOString(), + }; + } + if (normalized.from !== "now-15m" || normalized.to !== "now") { + return normalized; + } + return normalized; +}; + +const buildTimestampKeywordFilter = (keyword, timeField) => { + const value = `${keyword ?? ""}`.trim(); + if (!value || !timeField) { + return null; + } + const parsed = moment.tz( + value, + ["YYYY-MM-DD HH:mm:ss", "YYYY-MM-DDTHH:mm:ss"], + true, + getTimezone() + ); + if (!parsed.isValid()) { + return null; + } + return { + range: { + [timeField]: { + gte: parsed.clone().startOf("second").toISOString(), + lte: parsed.clone().endOf("second").toISOString(), + format: "strict_date_optional_time", + }, + }, + }; +}; const Index = forwardRef((props, ref) => { const { @@ -57,6 +174,8 @@ const Index = forwardRef((props, ref) => { onRow = null, showEmptyUI = false, setShowEmptyUI = null, + scroll, + datePickerContainerStyle = {}, } = props; const headerExtra = headerToobarExtra.getExtra @@ -66,16 +185,44 @@ const Index = forwardRef((props, ref) => { ? rowSelectionExtra.getExtra(props) : []; + const getHeaderExtraKey = (item, index) => { + let baseKey; + if (React.isValidElement(item) && item.key != null) { + baseKey = item.key; + } else if (item?.key != null) { + baseKey = item.key; + } else if (item?.id != null) { + baseKey = item.id; + } else if (item?.props?.id != null) { + baseKey = item.props.id; + } + return baseKey != null + ? `header-extra-${String(baseKey)}-${index}` + : `header-extra-${index}`; + }; + const [param, setParam] = useQueryParam("_g", JsonParam); - const [queryParams, setQueryParams] = useState({ - ...defaultQueryParams, - ...param, - from: viewLayout === 'timeline' ? 0 : (defaultQueryParams.from ?? param.from) + const [queryParams, setQueryParams] = useState(() => { + const merged = { + ...defaultQueryParams, + ...(param || {}), + from: + viewLayout === "timeline" + ? 0 + : defaultQueryParams.from ?? param?.from, + }; + return { + ...merged, + timeRange: normalizeTimeRange( + merged?.timeRange, + defaultQueryParams?.timeRange + ), + }; }); const [histogramState, setHistogramState] = useState({ visible: histogramVisible, widget: histogramWidget, - range: defaultQueryParams.timeRange, + range: normalizeHistogramRange(defaultQueryParams.timeRange), }); useEffect(() => { if (histogramEnable) { @@ -137,18 +284,23 @@ const Index = forwardRef((props, ref) => { return [columnsNew, sortOptions, aggOptions, searchFields]; }, [columns]); - const formatAggs = useMemo(() => { + const [formatAggs, aggMetas] = useMemo(() => { let aggsJson = {}; + let metas = {}; let aggsArr = sideEnable ? aggOptions : []; aggsArr.map((item) => { - aggsJson[item.label] = { + aggsJson[item.field] = { terms: { field: item.field, size: item.size, }, }; + metas[item.field] = { + label: item.label, + field: item.field, + }; }); - return aggsJson; + return [aggsJson, metas]; }, [sideEnable, aggOptions]); const formatQueryBody = (queryParams) => { @@ -170,23 +322,49 @@ const Index = forwardRef((props, ref) => { let filter = []; //time range - if (queryParams?.timeRange && queryParams?.timeRange?.timeField) { + if ( + queryParams?.timeRange && + queryParams?.timeRange?.timeField && + queryParams?.timeRange?.from !== "auto" && + queryParams?.timeRange?.to !== "auto" + ) { let range = {}; range[queryParams.timeRange.timeField] = { gte: queryParams.timeRange.from, lte: queryParams.timeRange.to, + format: "strict_date_optional_time", }; filter.push({ range, }); } //query match - if (queryParams?.keyword) { + const keywordFilters = []; + const searchQuery = buildContainsQueryString(queryParams?.keyword); + if (searchQuery) { let query_string = { - query: `*${queryParams.keyword}*`, + query: searchQuery, fields: searchFields, + analyze_wildcard: true, }; - filter.push({ query_string }); + keywordFilters.push({ query_string }); + } + const timestampKeywordFilter = buildTimestampKeywordFilter( + queryParams?.keyword, + queryParams?.timeRange?.timeField + ); + if (timestampKeywordFilter) { + keywordFilters.push(timestampKeywordFilter); + } + if (keywordFilters.length === 1) { + filter.push(keywordFilters[0]); + } else if (keywordFilters.length > 1) { + filter.push({ + bool: { + should: keywordFilters, + minimum_should_match: 1, + }, + }); } //sort by @@ -200,6 +378,14 @@ const Index = forwardRef((props, ref) => { return sortJson; }); } + const aggs = { + ...formatAggs, + }; + const timeField = queryParams?.timeRange?.timeField; + if (timeField) { + aggs.__listview_min_time = { min: { field: timeField } }; + aggs.__listview_max_time = { max: { field: timeField } }; + } return { query: { bool: { @@ -219,7 +405,7 @@ const Index = forwardRef((props, ref) => { }, }, sort: sort, - aggs: formatAggs, + aggs, }; }; @@ -242,6 +428,35 @@ const Index = forwardRef((props, ref) => { return res?.hits?.total?.value || 0; }; const [dataSource, setDataSource] = useState({}); + const tableDataSource = useMemo(() => { + const rows = Array.isArray(dataSource?.data) ? dataSource.data : []; + const rowKeyCounts = new Map(); + + rows.forEach((item, index) => { + const baseKey = + item?.id ?? item?._id ?? item?.key ?? item?._key ?? item?.name ?? null; + const normalizedKey = + baseKey == null || baseKey === "" ? `row-${index}` : String(baseKey); + rowKeyCounts.set(normalizedKey, (rowKeyCounts.get(normalizedKey) || 0) + 1); + }); + + return rows.map((item, index) => { + const baseKey = + item?.id ?? item?._id ?? item?.key ?? item?._key ?? item?.name ?? null; + const normalizedKey = + baseKey == null || baseKey === "" ? `row-${index}` : String(baseKey); + const uniqueKey = + rowKeyCounts.get(normalizedKey) > 1 + ? `${normalizedKey}-${index}` + : normalizedKey; + + return { + ...item, + __listview_row_key: uniqueKey, + }; + }); + }, [dataSource]); + useMemo(async () => { let ds = value; // console.log("dataSource:", value); @@ -361,22 +576,19 @@ const Index = forwardRef((props, ref) => { }; const onTimeRangeChange = ({ start, end, timeField }) => { + const normalizedTimeRange = normalizeTimeRange( + { from: start, to: end, timeField }, + defaultQueryParams?.timeRange + ); setQueryParams((st) => ({ ...st, from: 0, - timeRange: { - from: start, - to: end, - timeField: timeField, - }, + timeRange: normalizedTimeRange, })); if (histogramEnable) { - let range = { - from: start, - to: end, - }; + let range = normalizeHistogramRange(normalizedTimeRange); let series = histogramState.widget?.series?.map((item) => { - item.queries.time_field = timeField; + item.queries.time_field = normalizedTimeRange.timeField; return item; }); let widget = { ...histogramState.widget, series }; @@ -384,9 +596,61 @@ const Index = forwardRef((props, ref) => { } }; + const onHistogramQueriesChange = (nextQueries = {}) => { + const nextRange = nextQueries?.range; + if (!nextRange?.from || !nextRange?.to) { + return; + } + const timeField = + histogramState.widget?.series?.[0]?.queries?.time_field || + queryParams?.timeRange?.timeField || + defaultQueryParams?.timeRange?.timeField || + ""; + onTimeRangeChange({ + start: nextRange.from, + end: nextRange.to, + timeField, + }); + }; + + const histogramQuery = useMemo(() => { + const query = formatQueryBody(queryParams)?.query; + return query ? JSON.stringify(query) : undefined; + }, [JSON.stringify(queryParams)]); + + const histogramRange = useMemo( + () => + resolveAutoHistogramRange( + queryParams?.timeRange || {}, + dataSource?.aggregations || {} + ), + [queryParams?.timeRange, dataSource?.aggregations] + ); + + const histogramAutoRangePending = useMemo(() => { + const fromValue = normalizeTimeRangeValue(queryParams?.timeRange?.from, ""); + const toValue = normalizeTimeRangeValue(queryParams?.timeRange?.to, ""); + const isAuto = fromValue === "auto" || toValue === "auto"; + if (!isAuto) { + return false; + } + const minValue = dataSource?.aggregations?.__listview_min_time?.value; + const maxValue = dataSource?.aggregations?.__listview_max_time?.value; + return !( + Number.isFinite(minValue) && + Number.isFinite(maxValue) && + minValue > 0 && + maxValue > 0 + ); + }, [queryParams?.timeRange, dataSource?.aggregations]); + useEffect(() => { - setParam((st) => ({ ...st, ...queryParams })); - }, [queryParams]); + const nextParam = { ...(param || {}), ...queryParams }; + if (isEqual(param || {}, nextParam)) { + return; + } + setParam(nextParam); + }, [param, queryParams, setParam]); //用 useImperativeHandle 暴露一些外部 ref 能访问的属性 useImperativeHandle(ref, () => ({ @@ -418,7 +682,7 @@ const Index = forwardRef((props, ref) => { {sideEnable && aggOptions.length > 0 ? (
{ onTimeRangeChange={onTimeRangeChange} isRefreshPaused={isRefreshPaused} recentlyUsedRangesKey={collectionName} + wrapperStyle={datePickerContainerStyle} /> ) : null} {headerExtra.map((item, i) => ( - {item} + {item} ))}
@@ -529,11 +794,26 @@ const Index = forwardRef((props, ref) => { className={styles.histogramWrap} style={{ display: histogramState.visible ? "block" : "none" }} > - + {histogramAutoRangePending ? ( +
+ +
+ ) : ( + + )}
) : null} @@ -543,8 +823,9 @@ const Index = forwardRef((props, ref) => { size={"small"} loading={loading} columns={columnsNew} - dataSource={dataSource?.data || []} - rowKey={"id"} + dataSource={tableDataSource} + scroll={tableDataSource.length > 0 ? scroll : undefined} + rowKey={"__listview_row_key"} onChange={onTableChange} pagination={{ size: "small", diff --git a/web/src/components/ListView/index.less b/web/src/components/ListView/index.less index 55cf25cc..d1a1fd6c 100644 --- a/web/src/components/ListView/index.less +++ b/web/src/components/ListView/index.less @@ -39,12 +39,12 @@ cursor: pointer; z-index: 10; position: absolute; - top: 50%; - margin-top: -15px; - width: 12px; - height: 30px; - line-height: 30px; - border-radius: 2px; + top: 24px; + margin-top: 0; + width: 14px; + height: 32px; + line-height: 32px; + border-radius: 0 4px 4px 0; background-color: rgba(234, 244, 255, 1); text-align: center; } @@ -62,6 +62,7 @@ .contentWrap { .expandAndCollapse { right: 0; + border-radius: 4px 0 0 4px; } } } diff --git a/web/src/components/Login/map.js b/web/src/components/Login/map.js index 72e547c2..1709bbb5 100644 --- a/web/src/components/Login/map.js +++ b/web/src/components/Login/map.js @@ -1,5 +1,6 @@ import React from 'react'; import { Icon } from 'antd'; +import { formatMessage } from 'umi/locale'; import styles from './index.less'; export default { @@ -12,7 +13,7 @@ export default { rules: [ { required: true, - message: 'Please enter username!', + message: formatMessage({ id: 'app.login.username.required' }), }, ], }, @@ -26,7 +27,7 @@ export default { rules: [ { required: true, - message: 'Please enter password!', + message: formatMessage({ id: 'app.login.password.required' }), }, ], }, @@ -34,16 +35,16 @@ export default { props: { size: 'large', prefix: , - placeholder: 'mobile number', + placeholder: formatMessage({ id: 'app.login.mobile.placeholder' }), }, rules: [ { required: true, - message: 'Please enter mobile number!', + message: formatMessage({ id: 'app.login.mobile.required' }), }, { pattern: /^1\d{10}$/, - message: 'Wrong mobile number format!', + message: formatMessage({ id: 'app.login.mobile.invalid' }), }, ], }, @@ -51,12 +52,12 @@ export default { props: { size: 'large', prefix: , - placeholder: 'captcha', + placeholder: formatMessage({ id: 'app.login.captcha.placeholder' }), }, rules: [ { required: true, - message: 'Please enter Captcha!', + message: formatMessage({ id: 'app.login.captcha.required' }), }, ], }, diff --git a/web/src/components/Markdown/index.jsx b/web/src/components/Markdown/index.jsx index dad22272..a234be1f 100644 --- a/web/src/components/Markdown/index.jsx +++ b/web/src/components/Markdown/index.jsx @@ -4,6 +4,11 @@ import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; export default ({ source, ref }) => { + const normalizedSource = + typeof source === "string" + ? source.replace(/\r\n/g, "\n").replace(/\n/g, " \n") + : source; + return (
{ className="markdown-body" components={{ a: LinkRenderer }} > - {source} + {normalizedSource}
@@ -37,9 +42,71 @@ export const IFrame = ({ children, styleSelector, ...props }) => { const linkEls = win.parent.document.querySelectorAll(styleSelector); if (linkEls.length) { linkEls.forEach((el) => { - win.document.head.appendChild(el); + win.document.head.appendChild(el.cloneNode(true)); }); } + + const parentWin = win.parent; + const parentDoc = parentWin.document; + const parentEl = contentRef.parentElement || parentDoc.body; + const parentStyles = parentWin.getComputedStyle(parentEl); + const linkProbe = parentDoc.createElement("a"); + linkProbe.href = "#"; + linkProbe.style.position = "absolute"; + linkProbe.style.visibility = "hidden"; + linkProbe.style.pointerEvents = "none"; + parentDoc.body.appendChild(linkProbe); + const linkColor = parentWin.getComputedStyle(linkProbe).color; + linkProbe.remove(); + + const themeStyleId = "markdown-frame-theme"; + const existingThemeStyle = win.document.getElementById(themeStyleId); + if (existingThemeStyle) { + existingThemeStyle.remove(); + } + + const themeStyle = win.document.createElement("style"); + themeStyle.id = themeStyleId; + themeStyle.textContent = ` + html, body { + margin: 0; + padding: 0; + background: transparent; + color: ${parentStyles.color}; + font-size: ${parentStyles.fontSize}; + line-height: ${parentStyles.lineHeight}; + font-family: ${parentStyles.fontFamily}; + } + + .markdown-body { + color: ${parentStyles.color}; + background: transparent; + font-size: ${parentStyles.fontSize}; + line-height: ${parentStyles.lineHeight}; + font-family: ${parentStyles.fontFamily}; + } + + .markdown-body p, + .markdown-body li, + .markdown-body blockquote, + .markdown-body table, + .markdown-body td, + .markdown-body th, + .markdown-body span, + .markdown-body strong, + .markdown-body em { + color: inherit; + } + + .markdown-body a, + .markdown-body a:hover, + .markdown-body a:focus, + .markdown-body a:active, + .markdown-body a:visited { + color: ${linkColor}; + } + `; + win.document.head.appendChild(themeStyle); }, [contentRef, styleSelector]); let docHeight = "auto"; if (contentRef) { diff --git a/web/src/components/NodeSelect/index.jsx b/web/src/components/NodeSelect/index.jsx index 39072a44..f4cb3df6 100644 --- a/web/src/components/NodeSelect/index.jsx +++ b/web/src/components/NodeSelect/index.jsx @@ -5,10 +5,14 @@ import { Icon } from "antd"; export default React.memo((props) => { - const {nodes = [], onChange, mode, placeholder, renderItem, labelField="host", keyField="host", allowClear } = props; + const {nodes = [], onChange, value: valueProp, mode, placeholder, renderItem, labelField="host", keyField="host", allowClear } = props; const [sorter, setSorter] = useState([]) - const [value, setValue] = useState([]); + const [value, setValue] = useState(valueProp || []); + + useMemo(() => { + setValue(valueProp || []); + }, [valueProp]); return ( { tabPosition="right" > {details.map((pane) => ( - + {typeof pane.component == "string" ? ( pane.component ) : ( diff --git a/web/src/components/Overview/Detail/Metrics/MetricIndices.jsx b/web/src/components/Overview/Detail/Metrics/MetricIndices.jsx index d03f7da0..ef5ea24f 100644 --- a/web/src/components/Overview/Detail/Metrics/MetricIndices.jsx +++ b/web/src/components/Overview/Detail/Metrics/MetricIndices.jsx @@ -99,7 +99,7 @@ export default ({ clusterID, clusterName, timeRange, action }) => { const columns = [ { - title: "Name", + title: formatMessage({ id: "overview.column.name" }), dataIndex: "index", render: (text, record) => ( { sorter: (a, b) => sorter.string(a, b, "index"), }, { - title: "Health", + title: formatMessage({ id: "overview.column.health" }), dataIndex: "health", render: (text, record) => , sorter: (a, b) => sorter.string(a, b, "health"), @@ -178,14 +178,14 @@ export default ({ clusterID, clusterName, timeRange, action }) => { >
{ setSearchValue(value); }} onChange={(e) => { setSearchValue(e.currentTarget.value); }} - enterButton + enterButton={formatMessage({ id: "form.button.search" })} />
diff --git a/web/src/components/Overview/Detail/Metrics/MetricNodes.jsx b/web/src/components/Overview/Detail/Metrics/MetricNodes.jsx index 845cdc0b..78f0574c 100644 --- a/web/src/components/Overview/Detail/Metrics/MetricNodes.jsx +++ b/web/src/components/Overview/Detail/Metrics/MetricNodes.jsx @@ -73,7 +73,7 @@ export default ({ clusterID, clusterName, timeRange, action }) => { const columns = [ { - title: "Name", + title: formatMessage({ id: "overview.column.name" }), dataIndex: "name", render: (text, record) => ( { sorter: (a, b) => sorter.string(a, b, "name"), }, { - title: "Status", + title: formatMessage({ id: "overview.column.status" }), dataIndex: "status", render: (text, record) => , sorter: (a, b) => sorter.string(a, b, "status"), @@ -119,14 +119,14 @@ export default ({ clusterID, clusterName, timeRange, action }) => { >
{ setSearchValue(value); }} onChange={(e) => { setSearchValue(e.currentTarget.value); }} - enterButton + enterButton={formatMessage({ id: "form.button.search" })} />
diff --git a/web/src/components/Overview/Detail/Metrics/index.js b/web/src/components/Overview/Detail/Metrics/index.js index 8e886581..9994dc75 100644 --- a/web/src/components/Overview/Detail/Metrics/index.js +++ b/web/src/components/Overview/Detail/Metrics/index.js @@ -155,7 +155,14 @@ export default (props) => {
{overviews.map((item) => ( - + { allowClear={true} style={{ width: filterWidth }} dropdownMatchSelectWidth={false} - placeholder="Filters" + placeholder={formatMessage({ id: "listview.filters.placeholder" })} onChange={onSearchFieldChange} value={searchField} > @@ -218,8 +219,9 @@ export default (props: IProps) => { {...autoCompleteProps} > { onSearchChange(value); setSearchOpen(false); diff --git a/web/src/components/Overview/Monitor/index.jsx b/web/src/components/Overview/Monitor/index.jsx index edfded75..35f2c2cc 100644 --- a/web/src/components/Overview/Monitor/index.jsx +++ b/web/src/components/Overview/Monitor/index.jsx @@ -59,21 +59,66 @@ export const getAllTimeSettingsCache = () => { const getDuration = (from, to) => { if (!from || !to) return; - const bounds = calculateBounds({ - from, - to, - }); - return bounds.max.valueOf() - bounds.min.valueOf() + try { + const bounds = calculateBounds({ + from, + to, + }); + return bounds.max.valueOf() - bounds.min.valueOf() + } catch (e) { + return undefined; + } } +const normalizeTimeValue = (value, fallback = "now-15m", keys = []) => { + const normalizeCandidate = (candidate) => { + const normalized = `${candidate}`.trim(); + if (!normalized || normalized.toLowerCase() === "auto") { + return fallback; + } + return normalized; + }; + if (typeof value === "string" || typeof value === "number") { + if (typeof value === "number" && !Number.isFinite(value)) { + return fallback; + } + return normalizeCandidate(value); + } + if (value && typeof value === "object") { + for (const key of keys) { + if ( + Object.prototype.hasOwnProperty.call(value, key) && + value[key] !== undefined && + value[key] !== null + ) { + const candidate = value[key]; + if (typeof candidate === "string" || typeof candidate === "number") { + return normalizeCandidate(candidate); + } + } + } + } + return fallback; +}; + export const initState = (state = {}) => { const { timeRange, timeInterval, timeout } = state || {} - const from = timeRange?.min || "now-15m" - const to = timeRange?.max || "now" + const from = normalizeTimeValue( + timeRange?.min ?? timeRange?.from, + "now-15m", + ["min", "from", "gte", "start"] + ); + const to = normalizeTimeValue( + timeRange?.max ?? timeRange?.to, + "now", + ["max", "to", "lte", "end"] + ); const duration = getDuration(from, to); - const gtOneHour = moment.duration(duration).asHours() > 1 - const day = moment.duration(duration).asDays(); - const intDay = parseInt(day) + 1; + const durationMs = + Number.isFinite(duration) && duration > 0 ? duration : 15 * 60 * 1000; + const gtOneHour = moment.duration(durationMs).asHours() > 1 + const day = moment.duration(durationMs).asDays(); + const intDay = Math.max(parseInt(day, 10) + 1, 1); return { ...state, timeRange: { @@ -105,10 +150,7 @@ const Monitor = (props) => { const [spinning, setSpinning] = useState(false); const [state, setState] = useState(formatState(initState({ - timeRange: { - min: param?.timeRange?.min || "now-15m", - max: param?.timeRange?.max || "now", - }, + timeRange: param?.timeRange || { min: "now-15m", max: "now" }, timeInterval: formatTimeInterval(param?.timeInterval) || allTimeSettingsCache.timeInterval, timeout: formatTimeout(param?.timeout) || allTimeSettingsCache.timeout || '10s', param: param, @@ -119,20 +161,28 @@ const Monitor = (props) => { const [timeZone, setTimeZone] = useState(() => allTimeSettingsCache.timeZone || getTimezone()); useEffect(() => { - setParam({ ...param, timeRange: state.timeRange, timeInterval: state.timeInterval, timeout: state.timeout }); + const newParam = { + ...param, + timeRange: state.timeRange, + timeInterval: state.timeInterval, + timeout: state.timeout + }; + if (JSON.stringify(newParam) !== JSON.stringify(param)) { + setParam(newParam); + } }, [state.timeRange, state.timeInterval, state.timeout]); const handleTimeChange = ({ start, end, timeInterval, timeout, refresh }) => { - setState(initState({ - ...state, + setState((prevState) => initState({ + ...prevState, param, timeRange: { - min: start, - max: end, + min: start || prevState?.timeRange?.min, + max: end || prevState?.timeRange?.max, }, - timeInterval: timeInterval || state.timeInterval, - timeout: timeout || state.timeout, - refresh + timeInterval: timeInterval || prevState.timeInterval, + timeout: timeout || prevState.timeout, + refresh, })); } @@ -194,7 +244,7 @@ const Monitor = (props) => { <>
-
+
{ setTimeZone(timeZone) }} recentlyUsedRangesKey={'monitor'} + showAutoTimeRange={false} />
-
+
{isSystemCluster(selectedCluster?.id) && getRollupEnabled() === "true" && } + />}
diff --git a/web/src/components/Overview/Monitor/index.less b/web/src/components/Overview/Monitor/index.less index 8d95fe64..a37a2bf2 100644 --- a/web/src/components/Overview/Monitor/index.less +++ b/web/src/components/Overview/Monitor/index.less @@ -1,7 +1,15 @@ .tabs { :global { .ant-tabs .ant-tabs-right-content { - padding-right: 16px !important; + padding-right: 6px !important; } } -} \ No newline at end of file +} + +.statusActions { + display: flex; + align-items: center; + justify-content: flex-end; + flex-wrap: wrap; + gap: 16px; +} diff --git a/web/src/components/Overview/Side/SearchFacet.scss b/web/src/components/Overview/Side/SearchFacet.scss index 39ca32d0..5b29b461 100644 --- a/web/src/components/Overview/Side/SearchFacet.scss +++ b/web/src/components/Overview/Side/SearchFacet.scss @@ -10,10 +10,33 @@ } .search-facet-value { display: flex; + align-items: center; + gap: 8px; + min-width: 0; + :global { + label.ant-checkbox-wrapper { + display: flex; + align-items: center; + flex: 1 1 auto; + min-width: 0; + } + label.ant-checkbox-wrapper > span:first-child { + flex: 0 0 auto; + } + label.ant-checkbox-wrapper > span:last-child { + flex: 1 1 auto; + min-width: 0; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } + } .count { margin-left: auto; + flex: 0 0 auto; color: #888; font-size: 0.85em; + white-space: nowrap; } } } diff --git a/web/src/components/Overview/Side/index.scss b/web/src/components/Overview/Side/index.scss index 68929184..12dff982 100644 --- a/web/src/components/Overview/Side/index.scss +++ b/web/src/components/Overview/Side/index.scss @@ -1,6 +1,15 @@ .search-filter { - margin-top: 56px; - padding: 0 10px; + height: 100%; + .search-filter-title { + font-weight: bold; + margin-bottom: 16px; + display: flex; + justify-content: space-between; + align-items: center; + } + .search-filter-reset { + cursor: pointer; + } .facet-cnt { display: flex; flex-direction: column; diff --git a/web/src/components/Overview/Side/index.tsx b/web/src/components/Overview/Side/index.tsx index 0bfeae90..61a4e1e7 100644 --- a/web/src/components/Overview/Side/index.tsx +++ b/web/src/components/Overview/Side/index.tsx @@ -2,6 +2,8 @@ import Sorter from "@/components/infini/search/sort/sort"; import { getSearchFacets } from "@/lib/elasticsearch/search"; import request from "@/utils/request"; import { useEffect, useState } from "react"; +import { Icon } from "antd"; +import { formatMessage } from "umi/locale"; import SearchFacet from "./SearchFacet"; import './index.scss'; @@ -16,6 +18,7 @@ interface IProps { } filters: {[key: string]: string}; onFacetChange: (facet: any) => void; + onReset?: () => void; } interface IFacet { @@ -32,7 +35,8 @@ export default (props: IProps) => { facetLabels, aggsConfig : { action, params }, filters, - onFacetChange + onFacetChange, + onReset } = props; const [facets, setFacets] = useState([]); @@ -59,7 +63,17 @@ export default (props: IProps) => { return (
-
+
+ {formatMessage({ id: "listview.side.filter" })} + + + +
+
{
) -} \ No newline at end of file +} diff --git a/web/src/components/Overview/index.scss b/web/src/components/Overview/index.scss index 0456e3fb..6ef1fcb6 100644 --- a/web/src/components/Overview/index.scss +++ b/web/src/components/Overview/index.scss @@ -1,17 +1,48 @@ .overview-wrapper { - .content { - display: flex; - gap: 1em; - margin-top: 10px; - .ant-list-item { - border-bottom: none !important; - } - .left { - flex: 1 1 auto; + display: flex; + gap: 10px; + margin-top: 10px; + + .overview-side-wrap { + min-height: calc(100vh - 220px); + padding: 24px 15px; + background-color: #ffffff; + flex: 0 0 220px; + width: 220px; + } + + .overview-content-wrap { + position: relative; + flex: 1 1 auto; + min-width: 0; + } + + .overview-expand-and-collapse { + cursor: pointer; + z-index: 10; + position: absolute; + top: 24px; + left: 0; + width: 14px; + height: 32px; + line-height: 32px; + border-radius: 0 4px 4px 0; + background-color: rgba(234, 244, 255, 1); + text-align: center; + } + + .overview-content-card { + min-height: calc(100vh - 220px); + .content { + .ant-list-item { + border-bottom: none !important; + } .search-result { - min-width: 900px; + width: 100%; + min-width: 0; margin-top: 15px; .table-wrap { + min-width: 0; margin-top: 24px; .ant-progress-inner { border-radius: 0; @@ -23,4 +54,10 @@ } } } + + &.collapse { + .overview-side-wrap { + display: none; + } + } } diff --git a/web/src/components/Overview/index.tsx b/web/src/components/Overview/index.tsx index 7cb54a5f..b70f3314 100644 --- a/web/src/components/Overview/index.tsx +++ b/web/src/components/Overview/index.tsx @@ -9,6 +9,8 @@ import React, { import useFetch from "@/lib/hooks/use_fetch"; import "./index.scss"; import request from "@/utils/request"; +import { Button, Card, Icon } from "antd"; +import { formatMessage } from "umi/locale"; import { JsonParam, useQueryParam } from "use-query-params"; import Header from "./Header"; import List from "./List"; @@ -27,6 +29,7 @@ interface IQueryParams { interface IDetail { key: string; title: string; + titleId?: string; component: any; } @@ -78,7 +81,10 @@ const initialQueryParams = { size: 10, keyword: "", }; - +const tableDefaultPageSize = 20; +const cardDefaultPageSize = initialQueryParams.size; +const getDefaultPageSizeByDisplayType = (displayType: "card" | "table") => + displayType === "table" ? tableDefaultPageSize : cardDefaultPageSize; export default forwardRef((props: IProps, ref: any) => { const [param, setParam] = useQueryParam("_g", JsonParam); const currentTab = param?.tab || "clusters"; @@ -99,11 +105,13 @@ export default forwardRef((props: IProps, ref: any) => { detailTitleConfig, headerConfig = {}, } = props; - const drawRef = useRef(null); + const [sideVisible, setSideVisible] = useState(false); const [searchField, setSearchField] = useState(); const [selectedItem, setSelectedItem] = useState({}); + const [cardInfos, setCardInfos] = useState>({}); + const [cardInfoLoadings, setCardInfoLoadings] = useState>({}); const [dispalyTypeObj, setDispalyTypeObj] = useLocalStorage( "console:overview:displayType", @@ -118,11 +126,8 @@ export default forwardRef((props: IProps, ref: any) => { decode: JSON.parse, } ); - const onDisplayTypeChange = (value: string) => { - let obj = {}; - obj[currentTab] = value; - setDispalyTypeObj({ ...dispalyTypeObj, ...obj }); - }; + const displayType = dispalyTypeObj[currentTab] || "card"; + const defaultPageSize = getDefaultPageSizeByDisplayType(displayType); function reducer( queryParams: IQueryParams, @@ -145,16 +150,51 @@ export default forwardRef((props: IProps, ref: any) => { ...queryParams, size: action.value, }; + case "setPageSizeAndReset": + return { + ...queryParams, + from: 0, + size: action.value, + }; + case "setQuery": + return { + ...queryParams, + ...action.value, + }; default: throw new Error(); } } - const [queryParams, dispatch] = useReducer(reducer, { - from: param?.from || initialQueryParams.from, - size: param?.size || initialQueryParams.size , - keyword: param?.keyword || initialQueryParams.keyword + const [queryParams, dispatch] = useReducer(reducer, { + from: initialQueryParams.from, + size: defaultPageSize, + keyword: param?.keyword ?? initialQueryParams.keyword, }); + const onDisplayTypeChange = (value: string) => { + const nextDisplayType = value === "table" ? "table" : "card"; + setDispalyTypeObj({ ...dispalyTypeObj, [currentTab]: value }); + dispatch({ + type: "setPageSizeAndReset", + value: getDefaultPageSizeByDisplayType(nextDisplayType), + }); + }; + + useEffect(() => { + if (dispalyTypeObj[currentTab] !== displayType) { + setDispalyTypeObj({ ...dispalyTypeObj, [currentTab]: displayType }); + } + }, [currentTab, displayType, dispalyTypeObj, setDispalyTypeObj]); + + useEffect(() => { + dispatch({ + type: "setQuery", + value: { + from: initialQueryParams.from, + size: getDefaultPageSizeByDisplayType(displayType), + }, + }); + }, [currentTab, displayType]); const { run, loading, value } = useFetch( searchAction, { @@ -177,6 +217,7 @@ export default forwardRef((props: IProps, ref: any) => { const result = (value as any)?.hits || {}; const { hits = [] } = result; + const hasSide = sideSorterOptions.length > 0 || aggsParams.length > 0; const initQueryParams = () => { extraQueryFields.forEach((item) => { @@ -204,17 +245,68 @@ export default forwardRef((props: IProps, ref: any) => { dispatch({ type: "pagination", value: 1 }) }; - useEffect(() => { - if (hits.length === 0) { - return; - } + const onSideReset = () => { + const { filters, ...restParams } = param || {}; + setParam(restParams); + dispatch({ type: "pagination", value: 1 }); + }; - // fetchListInfo(); - }, [value]); + const onRefresh = () => { + run(); + }; useEffect(() => { - setParam({ ...param, ...queryParams }); - }, [JSON.stringify(queryParams)]); + if (displayType !== "card" || loading || hits.length === 0 || !infoAction) { + return; + } + const ids = hits + .map((item) => listItemConfig.getId(item)) + .filter((id): id is string => !!id && !cardInfos[id]); + if (ids.length === 0) { + return; + } + let cancelled = false; + const fetchListInfo = async () => { + setCardInfoLoadings((current) => { + const next = { ...current }; + ids.forEach((id) => { + next[id] = true; + }); + return next; + }); + try { + const res = await request( + infoAction, + { + method: "POST", + body: ids, + }, + false, + false + ); + if (!cancelled && res && !res.error) { + setCardInfos((current) => ({ + ...current, + ...res, + })); + } + } finally { + if (!cancelled) { + setCardInfoLoadings((current) => { + const next = { ...current }; + ids.forEach((id) => { + next[id] = false; + }); + return next; + }); + } + } + }; + fetchListInfo(); + return () => { + cancelled = true; + }; + }, [displayType, loading, hits, infoAction, listItemConfig, cardInfos]); useEffect(() => { initQueryParams(); @@ -226,106 +318,141 @@ export default forwardRef((props: IProps, ref: any) => { return ( <> -
-
-
-
{ - setSearchField(value); - setParam({ ...param, search_field: value }); - }} - defaultSearchValue={queryParams?.keyword} - onSearchChange={(value) => { - dispatch({ type: "search", value }) - }} - onFacetChange={onFacetChange} - dispalyType={dispalyTypeObj[currentTab]} - onDisplayTypeChange={onDisplayTypeChange} - autoCompleteConfig={{ - action: searchAction, - highlightFields: searchHighlightFields, - ...searchAutoCompleteConfig, - }} - {...headerConfig} - onCleanSuccess={() => { - dispatch({ type: "pagination", value: 1 }) - }} - /> -
- {dispalyTypeObj[currentTab] == "card" ? ( - - dispatch({ type: "pagination", value: page }) - } - onPageSizeChange={(size) => - dispatch({ type: "pageSizeChange", value: size }) - } - renderItem={(item) => { - const infoField = listItemConfig.getId(item); - return ( - { - setSelectedItem(item); - drawRef.current?.open(); - }} - onChangeFacet={onFacetChange} - infoAction={infoAction} - parentLoading={loading} - /> - ); - }} - /> - ) : ( - ({...item, id: listItemConfig.getId(item)}))} - total={result?.total?.value || 0} - from={queryParams.from} - pageSize={queryParams.size} - loading={loading} - onPageChange={(page) => - dispatch({ type: "pagination", value: page }) - } - onPageSizeChange={(size) => - dispatch({ type: "pageSizeChange", value: size }) - } - onRowClick={(item) => { - setSelectedItem(item); - drawRef.current?.open(); - }} - parentLoading={loading} - /> - )} -
+
+ {hasSide ? ( +
+ {sideVisible ? ( + { + setParam({ + ...param, + sort, + }); + }} + facetLabels={facetLabels} + aggsConfig={{ + action: searchAction, + params: aggsParams, + }} + filters={param?.filters || {}} + onFacetChange={onFacetChange} + onReset={onSideReset} + /> + ) : null}
- { - setParam({ - ...param, - sort, - }); - }} - facetLabels={facetLabels} - aggsConfig={{ - action: searchAction, - params: aggsParams, - }} - filters={param?.filters || {}} - onFacetChange={onFacetChange} - /> + ) : null} +
+ {hasSide ? ( + setSideVisible((visible) => !visible)} + title={ + sideVisible + ? formatMessage({ id: "listview.side.button.collapse" }) + : formatMessage({ id: "listview.side.button.expand" }) + } + > + + + ) : null} + +
+
{ + setSearchField(value); + setParam({ ...param, search_field: value }); + }} + defaultSearchValue={queryParams?.keyword} + onSearchChange={(value) => { + dispatch({ type: "search", value }) + }} + onFacetChange={onFacetChange} + dispalyType={displayType} + onDisplayTypeChange={onDisplayTypeChange} + autoCompleteConfig={{ + action: searchAction, + highlightFields: searchHighlightFields, + ...searchAutoCompleteConfig, + }} + {...headerConfig} + getExtra={(headerProps) => { + const extras = headerConfig.getExtra + ? headerConfig.getExtra(headerProps) + : []; + return [ + , + ...extras, + ]; + }} + onCleanSuccess={() => { + dispatch({ type: "pagination", value: 1 }) + }} + /> +
+ {displayType == "card" ? ( + + dispatch({ type: "pagination", value: page }) + } + onPageSizeChange={(size) => + dispatch({ type: "pageSizeChange", value: size }) + } + renderItem={(item) => { + const infoField = listItemConfig.getId(item); + return ( + { + setSelectedItem(item); + drawRef.current?.open(); + }} + onChangeFacet={onFacetChange} + infoAction={infoAction} + parentLoading={loading} + /> + ); + }} + /> + ) : ( + ({...item, id: listItemConfig.getId(item)}))} + total={result?.total?.value || 0} + from={queryParams.from} + pageSize={queryParams.size} + loading={loading} + onPageChange={(page) => + dispatch({ type: "pagination", value: page }) + } + onPageSizeChange={(size) => + dispatch({ type: "pageSizeChange", value: size }) + } + onRowClick={(item) => { + setSelectedItem(item); + drawRef.current?.open(); + }} + parentLoading={loading} + /> + )} +
+
+
{ } return ( breadcrumb || { - name: url.split("/").pop(), + hideInBreadcrumb: true, } ); }; diff --git a/web/src/components/PageHeader/index.test.js b/web/src/components/PageHeader/index.test.js index d22706e9..1e85ddbb 100644 --- a/web/src/components/PageHeader/index.test.js +++ b/web/src/components/PageHeader/index.test.js @@ -40,4 +40,10 @@ describe('test getBreadcrumb', () => { const urlNameList = urlToList('/userinfo/2144').map(url => getBreadcrumb(routerData, url).name); expect(urlNameList).toEqual(['用户列表', '用户信息']); }); + + it('Hide unmatched path segment', () => { + expect(getBreadcrumb(routerData, '/userinfo/2144/edit')).toEqual({ + hideInBreadcrumb: true, + }); + }); }); diff --git a/web/src/components/PageHeaderWrapper/index.js b/web/src/components/PageHeaderWrapper/index.js index c5d169c2..907cc2c2 100644 --- a/web/src/components/PageHeaderWrapper/index.js +++ b/web/src/components/PageHeaderWrapper/index.js @@ -31,7 +31,7 @@ const PageHeaderWrapper = ({ ); } - return item.name; + return item.name || item.title; }} /> )} diff --git a/web/src/components/RollupStats/index.js b/web/src/components/RollupStats/index.js index 4ade41b7..dd5bc6c7 100644 --- a/web/src/components/RollupStats/index.js +++ b/web/src/components/RollupStats/index.js @@ -108,13 +108,13 @@ export default (props) => { return (
- Rollup Gap + {formatMessage({ id: "cluster.monitor.rollup.gap" })} !loading && fetchData(fetchUrl)} >
@@ -137,9 +137,9 @@ export default (props) => { >
- {renderIcon()} Rollup + {renderIcon()} {formatMessage({ id: "cluster.monitor.tabs.rollup" })}
); -} \ No newline at end of file +} diff --git a/web/src/components/StandardTable/index.js b/web/src/components/StandardTable/index.js index fad3c788..87d2d97c 100644 --- a/web/src/components/StandardTable/index.js +++ b/web/src/components/StandardTable/index.js @@ -91,7 +91,7 @@ class StandardTable extends PureComponent {
+ 已选择 {selectedRowKeys.length} 项   {needTotalList.map(item => ( diff --git a/web/src/components/SuperDatePicker/quick_select_popover/quick_select_popover.tsx b/web/src/components/SuperDatePicker/quick_select_popover/quick_select_popover.tsx index 33ed4aff..a473c757 100644 --- a/web/src/components/SuperDatePicker/quick_select_popover/quick_select_popover.tsx +++ b/web/src/components/SuperDatePicker/quick_select_popover/quick_select_popover.tsx @@ -117,7 +117,7 @@ export class EuiQuickSelectPopover extends Component< const showTimeSelect = !!timeField return ( - + { showTimeSelect && ( <> diff --git a/web/src/components/infini/InputSelect.js b/web/src/components/infini/InputSelect.js index 8d9e1cc4..3e1a708f 100644 --- a/web/src/components/infini/InputSelect.js +++ b/web/src/components/infini/InputSelect.js @@ -13,10 +13,12 @@ class InputSelect extends React.Component{ } onClick = ({ key }) => { + const { normalizeValue } = this.props; + const nextValue = normalizeValue ? normalizeValue(key) : key; this.setState({ - value: key, + value: nextValue, }) - this.triggerChange(key) + this.triggerChange(nextValue) } triggerChange = (val)=>{ let {onChange} = this.props; @@ -26,6 +28,10 @@ class InputSelect extends React.Component{ } handleChange = (ev) => { let val = ev.target.value; + const { normalizeValue } = this.props; + if (normalizeValue) { + val = normalizeValue(val); + } let filterData = this.props.data.slice(); if(val != ""){ filterData = filterData.filter(v=>v.value.includes(val)) diff --git a/web/src/components/infini/SearchInput/index.jsx b/web/src/components/infini/SearchInput/index.jsx new file mode 100644 index 00000000..ef3636a1 --- /dev/null +++ b/web/src/components/infini/SearchInput/index.jsx @@ -0,0 +1,59 @@ +import React from "react"; +import { Input } from "antd"; +import { formatMessage } from "umi/locale"; + +const { Search } = Input; + +const normalizePlaceholder = (placeholder) => { + if (typeof placeholder === "undefined") { + return formatMessage({ id: "listview.search.placeholder" }); + } + + if (typeof placeholder !== "string") { + return placeholder; + } + + const normalizedPlaceholder = placeholder.trim().toLowerCase(); + if ( + normalizedPlaceholder === "type keyword to search" || + normalizedPlaceholder === "search" || + normalizedPlaceholder === "keyword" + ) { + return formatMessage({ id: "listview.search.placeholder" }); + } + + return placeholder; +}; + +const normalizeEnterButton = (enterButton) => { + if (typeof enterButton === "undefined") { + return formatMessage({ id: "form.button.search" }); + } + + if (typeof enterButton !== "string") { + return enterButton; + } + + const normalizedEnterButton = enterButton.trim(); + if ( + normalizedEnterButton.toLowerCase() === "search" || + normalizedEnterButton === "搜索" + ) { + return formatMessage({ id: "form.button.search" }); + } + + return enterButton; +}; + +const SearchInput = ({ placeholder, enterButton, allowClear = true, ...props }) => { + return ( + + ); +}; + +export default SearchInput; diff --git a/web/src/components/infini/TagEditor.jsx b/web/src/components/infini/TagEditor.jsx index 5cebd9ae..6973f2d0 100644 --- a/web/src/components/infini/TagEditor.jsx +++ b/web/src/components/infini/TagEditor.jsx @@ -41,7 +41,7 @@ export default ({ value = [], onChange }) => {
{value.map((tag, index) => ( handleRemove(index)} @@ -60,7 +60,10 @@ export default ({ value = [], onChange }) => { )} {!inputVisible && ( - Add New + {" "} + {formatMessage({ + id: "command.btn.newtag", + })} )}
diff --git a/web/src/components/infini/health_status_circle.tsx b/web/src/components/infini/health_status_circle.tsx index cd5a8c21..8af961e7 100644 --- a/web/src/components/infini/health_status_circle.tsx +++ b/web/src/components/infini/health_status_circle.tsx @@ -5,6 +5,7 @@ export type ClusterHealthStatus = | "green" | "yellow" | "red" + | "unknown" | "available" | "unavailable" | "online" @@ -14,6 +15,7 @@ const statusColorMap: Record = { green: Color.GREEN, yellow: Color.YELLOW, red: Color.RED, + unknown: Color.GREY, available: Color.GREEN, unavailable: Color.UNAVAILABLE, online: Color.GREEN, diff --git a/web/src/components/infini/health_status_rect.tsx b/web/src/components/infini/health_status_rect.tsx index 3c2c5db6..15c36d2b 100644 --- a/web/src/components/infini/health_status_rect.tsx +++ b/web/src/components/infini/health_status_rect.tsx @@ -4,6 +4,7 @@ export type ClusterHealthStatus = | "green" | "yellow" | "red" + | "unknown" | "available" | "unavailable" | "online" @@ -13,6 +14,7 @@ const statusColorMap: Record = { green: Color.GREEN, yellow: Color.YELLOW, red: Color.RED, + unknown: Color.GREY, available: Color.GREEN, unavailable: Color.UNAVAILABLE, online: Color.GREEN, diff --git a/web/src/components/infini/health_status_view.tsx b/web/src/components/infini/health_status_view.tsx index 35d08e08..1208bd3f 100644 --- a/web/src/components/infini/health_status_view.tsx +++ b/web/src/components/infini/health_status_view.tsx @@ -1,11 +1,32 @@ import { HealthStatusCircle } from "@/components/infini/health_status_circle"; +import { formatMessage } from "umi/locale"; + +const getStatusLabel = (status?: string, label?: string) => { + if (label) { + return label; + } + if (!status) { + return "N/A"; + } + + switch (String(status).toLowerCase()) { + case "available": + return formatMessage({ id: "overview.status.available" }); + case "unknown": + return formatMessage({ id: "overview.status.unknown" }); + case "unavailable": + return "unavailable"; + default: + return status; + } +}; export const HealthStatusView = ({ status, label }: props) => { return ( - {label || status || "N/A"} + {getStatusLabel(status, label)} ); diff --git a/web/src/components/infini/search/FilterSearchGroup.jsx b/web/src/components/infini/search/FilterSearchGroup.jsx index 65b7ddf8..b9e773db 100644 --- a/web/src/components/infini/search/FilterSearchGroup.jsx +++ b/web/src/components/infini/search/FilterSearchGroup.jsx @@ -1,23 +1,46 @@ -import { Input, Select } from "antd"; -const { Search } = Input; +import { Input, Select, Button, Icon } from "antd"; +import { formatMessage } from "umi/locale"; + const InputGroup = Input.Group; const Option = Select.Option; const FilterSearchGroup = ({ - enterButton = true, + enterButton = formatMessage({ id: "form.button.search" }), filterWidth = 120, filterFields, + filterValue, + searchValue, onFilterChange, onSearch, onChange, }) => { + const handleClear = () => { + if (typeof onChange == "function") { + onChange(""); + } + if (typeof onSearch == "function") { + onSearch(""); + } + }; + + const handleSearchChange = (e) => { + const value = e.target?.value ?? ""; + if (!value && typeof onFilterChange == "function") { + onFilterChange(undefined); + } + if (typeof onChange == "function") { + onChange(value); + } + }; + return ( - { + + ) : null + } + onPressEnter={() => { if (typeof onSearch == "function") { - onSearch(value); + onSearch(searchValue || ""); } }} - onChange={(e) => { - if (typeof onChange == "function") { - onChange(e.currentTarget.value); + onChange={handleSearchChange} + /> + ); }; diff --git a/web/src/components/monaco-editor/index.jsx b/web/src/components/monaco-editor/index.jsx index 72b55bd7..1443df83 100644 --- a/web/src/components/monaco-editor/index.jsx +++ b/web/src/components/monaco-editor/index.jsx @@ -1,5 +1,6 @@ -import { loader } from "@monaco-editor/react"; +import Editor, { loader, monaco } from "@monaco-editor/react"; loader.config({ paths: { vs: "/static/monaco-editor/min/vs" } }); -export {default as Editor} from "@monaco-editor/react"; \ No newline at end of file +export { Editor, monaco }; +export default Editor; diff --git a/web/src/components/vendor/console/components/CommonCommandModal.tsx b/web/src/components/vendor/console/components/CommonCommandModal.tsx index 08b31542..a3ccdb32 100644 --- a/web/src/components/vendor/console/components/CommonCommandModal.tsx +++ b/web/src/components/vendor/console/components/CommonCommandModal.tsx @@ -1,5 +1,5 @@ // @ts-ignore -import React, { useState, useCallback } from "react"; +import React, { useState, useCallback, useRef } from "react"; import { Modal, Form, Input, Tag, message } from "antd"; import { PlusOutlined } from "@ant-design/icons"; import { formatMessage } from "umi/locale"; @@ -48,7 +48,7 @@ export const TagGenerator = ({ value = [], onChange }: ITagGeneratorProps) => {
{value.map((tag, index) => ( handleRemove(index)} @@ -76,18 +76,27 @@ export const TagGenerator = ({ value = [], onChange }: ITagGeneratorProps) => { interface ICommonCommandModalProps { onClose: () => void; - onConfirm: (params: Record) => void; + onConfirm: (params: Record) => Promise | void; + confirmLoading?: boolean; form: any; } const CommonCommandModal = Form.create()((props: ICommonCommandModalProps) => { const { form } = props; + const submittingRef = useRef(false); const handleConfirm = async () => { + if (submittingRef.current || props.confirmLoading) { + return; + } + submittingRef.current = true; try { const values = await form.validateFields(); - props.onConfirm(values); - } catch (e) {} + await props.onConfirm(values); + } catch (e) { + } finally { + submittingRef.current = false; + } }; return ( @@ -96,9 +105,12 @@ const CommonCommandModal = Form.create()((props: ICommonCommandModalProps) => { visible={true} onCancel={props.onClose} onOk={handleConfirm} + confirmLoading={!!props.confirmLoading} zIndex={1003} cancelText={formatMessage({ id: "form.button.cancel" })} okText={formatMessage({ id: "form.button.save" })} + okButtonProps={{ disabled: !!props.confirmLoading }} + cancelButtonProps={{ disabled: !!props.confirmLoading }} >
diff --git a/web/src/components/vendor/console/components/ConsoleInput.scss b/web/src/components/vendor/console/components/ConsoleInput.scss index f16043f7..3e618bc4 100644 --- a/web/src/components/vendor/console/components/ConsoleInput.scss +++ b/web/src/components/vendor/console/components/ConsoleInput.scss @@ -24,7 +24,22 @@ .conApp__outputContent { height: 100%; flex: 1 1 1px; - font-size: 14px; + font-size: 13px; + font-family: "SFMono-Regular", Monaco, Menlo, Consolas, "Liberation Mono", "Ubuntu Mono", + monospace; +} + +.conApp__editorContent .ace_editor, +.conApp__editorContent .ace_content, +.conApp__editorContent .ace_layer, +.conApp__editorContent textarea, +.conApp__outputContent .ace_editor, +.conApp__outputContent .ace_content, +.conApp__outputContent .ace_layer, +.conApp__outputContent textarea { + font-size: 13px !important; + font-family: "SFMono-Regular", Monaco, Menlo, Consolas, "Liberation Mono", "Ubuntu Mono", + monospace !important; } .conApp__editorActions { @@ -98,4 +113,4 @@ box-shadow: none !important; background-color: transparent !important; } - } \ No newline at end of file + } diff --git a/web/src/components/vendor/console/components/ConsoleInput.tsx b/web/src/components/vendor/console/components/ConsoleInput.tsx index 53ed5913..fc91b814 100644 --- a/web/src/components/vendor/console/components/ConsoleInput.tsx +++ b/web/src/components/vendor/console/components/ConsoleInput.tsx @@ -1,5 +1,5 @@ // @ts-ignore -import React, { useRef, useEffect, CSSProperties, useMemo } from "react"; +import React, { useRef, useEffect, useLayoutEffect, CSSProperties, useMemo } from "react"; import ace from "brace"; import { EuiFlexGroup, @@ -10,7 +10,7 @@ import { } from "@elastic/eui"; import { SenseEditor } from "../entities/sense_editor"; import { LegacyCoreEditor } from "../modules/legacy_core_editor"; -import ConsoleMenu from "./ConsoleMenu"; +import ConsoleMenu, { CONSOLE_MENU_SHORTCUTS } from "./ConsoleMenu"; // import { RequestContextProvider } from '../contexts/request_context'; import { getDocumentation, autoIndent } from "../entities/console_menu_actions"; import "./ConsoleInput.scss"; @@ -19,7 +19,7 @@ import { useSetInputEditor } from "../hooks/use_set_input_editor"; import "@elastic/eui/dist/eui_theme_light.css"; import { instance as registry } from "../contexts/editor_context/editor_registry"; import "antd/dist/antd.css"; -import { retrieveAutoCompleteInfo } from "../modules/mappings/mappings"; +import { clearSubscriptions, retrieveAutoCompleteInfo } from "../modules/mappings/mappings"; import { useSaveCurrentTextObject } from "../hooks/use_save_current_text_object"; import { useEditorReadContext } from "../contexts/editor_context/editor_context"; import { useDataInit } from "../hooks/use_data_init"; @@ -27,6 +27,15 @@ import { useServicesContext } from "../contexts"; import { applyCurrentSettings } from "./apply_editor_settings"; import { subscribeResizeChecker } from "./subscribe_console_resize_checker"; import { formatMessage } from "umi/locale"; +import { hasAuthority } from "@/utils/authority"; +import { + applyConsoleAceFont, + DEFAULT_CONSOLE_FONT_SIZE, +} from "../utils/editor_font"; + +const isMacPlatform = () => + typeof window !== "undefined" && + /(Mac|iPhone|iPad|iPod)/i.test(window.navigator.platform); const abs: CSSProperties = { position: "absolute", @@ -47,6 +56,10 @@ const abs: CSSProperties = { const SendRequestButton = (props: any) => { const sendCurrentRequestToES = useSendCurrentRequestToES(); const saveCurrentTextObject = useSaveCurrentTextObject(); + const sendShortcutLabel = isMacPlatform() ? "Cmd+Enter" : "Ctrl+Enter"; + const tooltipContent = `${formatMessage({ + id: "console.SendRequestButton.ToolTip", + })} (${sendShortcutLabel})`; const { saveCurrentTextObjectRef } = props; useEffect(() => { @@ -54,12 +67,10 @@ const SendRequestButton = (props: any) => { }, [saveCurrentTextObjectRef]); return ( - +
); diff --git a/web/src/components/vendor/discover/public/application/components/sidebar/discover_sidebar.tsx b/web/src/components/vendor/discover/public/application/components/sidebar/discover_sidebar.tsx index 551bce48..69dbac45 100644 --- a/web/src/components/vendor/discover/public/application/components/sidebar/discover_sidebar.tsx +++ b/web/src/components/vendor/discover/public/application/components/sidebar/discover_sidebar.tsx @@ -92,6 +92,7 @@ export interface DiscoverSidebarProps { setIndexPattern: (id: string) => void; isClosed: boolean; indices: string[]; + clusterID?: string; distinctParams: any; onDistinctParamsChange: any; whetherToSample?: boolean; @@ -111,6 +112,7 @@ export function DiscoverSidebar({ setIndexPattern, isClosed, indices, + clusterID, distinctParams, onDistinctParamsChange, onFieldAgg, diff --git a/web/src/components/vendor/index_pattern_management/public/components/breadcrumbs.ts b/web/src/components/vendor/index_pattern_management/public/components/breadcrumbs.ts index 9af2b982..9eceb809 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/breadcrumbs.ts +++ b/web/src/components/vendor/index_pattern_management/public/components/breadcrumbs.ts @@ -18,12 +18,16 @@ */ import { IndexPattern } from "../../../data/public"; +import { formatMessage } from "umi/locale"; export function getListBreadcrumbs() { return [ { - text: "views", - href: `/`, + text: formatMessage({ + id: "explore.viewlist.title", + defaultMessage: "View", + }), + href: `/data/views`, }, ]; } @@ -32,8 +36,11 @@ export function getCreateBreadcrumbs() { return [ ...getListBreadcrumbs(), { - text: "Create view", - href: `/create`, + text: formatMessage({ + id: "explore.createview.title", + defaultMessage: "Create View", + }), + href: `/data/views/create`, }, ]; } @@ -43,7 +50,7 @@ export function getEditBreadcrumbs(indexPattern: IndexPattern) { ...getListBreadcrumbs(), { text: indexPattern.title, - href: `/patterns/${indexPattern.id}`, + href: `/data/views/patterns/${indexPattern.id}`, }, ]; } @@ -64,7 +71,10 @@ export function getCreateFieldBreadcrumbs(indexPattern: IndexPattern) { return [ ...getEditBreadcrumbs(indexPattern), { - text: "Create field", + text: formatMessage({ + id: "explore.view.index_pattern.create_field", + defaultMessage: "Create field", + }), }, ]; } diff --git a/web/src/components/vendor/index_pattern_management/public/components/create_button/create_button.tsx b/web/src/components/vendor/index_pattern_management/public/components/create_button/create_button.tsx index ac672d2d..511f1c8c 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/create_button/create_button.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/create_button/create_button.tsx @@ -106,7 +106,7 @@ export class CreateButton extends Component { {option.text} - {option.isBeta ? {this.renderBetaBadge()} : null} + {option.isBeta ? {this.renderBetaBadge()} : null} {option.description} diff --git a/web/src/components/vendor/index_pattern_management/public/components/create_index_pattern_wizard/components/loading_state/loading_state.tsx b/web/src/components/vendor/index_pattern_management/public/components/create_index_pattern_wizard/components/loading_state/loading_state.tsx index 6ccc8d1f..e5990fe5 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/create_index_pattern_wizard/components/loading_state/loading_state.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/create_index_pattern_wizard/components/loading_state/loading_state.tsx @@ -19,19 +19,11 @@ import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiLoadingSpinner, EuiTitle } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiLoadingSpinner } from '@elastic/eui'; export const LoadingState = () => ( - - -

- Checking for Elasticsearch data -

- - - diff --git a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx index 1fc1bfcd..be9c3806 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -50,19 +50,21 @@ export interface EditIndexPatternProps extends RouteComponentProps { const mappingAPILink = "Mapping API"; -const mappingConflictHeader = "Mapping 冲突"; +const mappingConflictHeader = formatMessage({ + id: "explore.view.index_pattern.mapping_conflict_title", +}); const confirmMessage = "This action resets the popularity counter of each field."; const confirmModalOptionsRefresh = { - confirmButtonText: "Refresh", - title: "Refresh field list?", + confirmButtonText: formatMessage({ id: "form.button.refresh" }), + title: formatMessage({ id: "explore.view.index_pattern.refreshFieldListTitle" }), }; const confirmModalOptionsDelete = { confirmButtonText: "Delete", - title: "Delete view?", + title: formatMessage({ id: "explore.view.index_pattern.delete_confirm" }), }; export const EditIndexPattern = withRouter( @@ -150,7 +152,7 @@ export const EditIndexPattern = withRouter( Promise.resolve( data.indexPatterns.delete(indexPattern.id || id) ).then(function() { - history.push(""); + history.push("/data/views"); }); } } @@ -162,11 +164,36 @@ export const EditIndexPattern = withRouter( // }); }; - const timeFilterHeader = `时间字段: '${indexPattern.timeFieldName}'`; + const timeFilterHeader = formatMessage( + { id: "explore.view.index_pattern.time_field" }, + { field: indexPattern.timeFieldName } + ); + + const mappingConflictLabel = formatMessage( + { id: "explore.view.index_pattern.mapping_conflict_desc" }, + { count: conflictedFields.length } + ); - const mappingConflictLabel = `当前视图匹配的索引有 ${conflictedFields.length} 字段定义了几种类型，如 (string, integer, 等)。您可以继续使用冲突的字段, 但是不能和函数一起使用(系统不知道冲突字段类型)。您可以重新生成索引来解决这个问题`; + const headingAriaLabel = formatMessage({ + id: "explore.view.index_pattern.detail_title", + }); - const headingAriaLabel = "视图详情"; + const breadcrumbList = [ + { + title: formatMessage({ id: "menu.home" }), + href: "/", + }, + { + title: formatMessage({ id: "menu.data" }), + }, + { + title: formatMessage({ id: "menu.data.view" }), + href: "/data/views", + }, + { + title: indexPattern?.viewName || indexPattern?.title, + }, + ]; // chrome.docTitle.change(indexPattern.title); @@ -175,7 +202,7 @@ export const EditIndexPattern = withRouter( ); return ( - +
history.push("/data/views")} /> {/* {showTagsSection && ( diff --git a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern_container.tsx b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern_container.tsx index 888c227a..d8dca256 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern_container.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern_container.tsx @@ -41,7 +41,7 @@ const EditIndexPatternCont: React.FC> = ({ ip.id = props.match.params.id; } if (ip.builtin) { - props.history.push(""); + props.history.push("/data/views"); } setIndexPattern(ip); // setBreadcrumbs(getEditBreadcrumbs(ip)); diff --git a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/index_header/index_header.tsx b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/index_header/index_header.tsx index 299dc91e..495d6120 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/index_header/index_header.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/index_header/index_header.tsx @@ -35,15 +35,20 @@ interface IndexHeaderProps { setDefault?: () => void; refreshFields?: () => void; deleteIndexPatternClick?: () => void; + goBack?: () => void; } const setDefaultAriaLabel = "设置为默认视图"; const setDefaultTooltip = "设置为默认视图"; -const refreshAriaLabel = "重新加载字段列表"; +const refreshAriaLabel = formatMessage({ + id: "explore.view.index_pattern.refreshTooltip", +}); -const refreshTooltip = "刷新字段列表"; +const refreshTooltip = formatMessage({ + id: "explore.view.index_pattern.refreshTooltip", +}); const removeAriaLabel = formatMessage({ id: "explore.view.index_pattern.removeTooltip", @@ -59,6 +64,7 @@ export function IndexHeader({ setDefault, refreshFields, deleteIndexPatternClick, + goBack, }: IndexHeaderProps) { return ( @@ -109,6 +115,14 @@ export function IndexHeader({ )} + {goBack && ( + + + + )} diff --git a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/tabs.tsx b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/tabs.tsx index c57ed6f8..2957167d 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/tabs.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/tabs.tsx @@ -67,6 +67,7 @@ import { useGlobalContext } from "../../../context"; import LayoutList from "@/pages/DataManagement/View/LayoutList" import { ComplexFieldsTable } from "../indexed_fields_table/complex_fields_table"; +import { formatMessage } from "umi/locale"; interface TabsProps extends Pick { indexPattern: IndexPattern; @@ -74,11 +75,17 @@ interface TabsProps extends Pick { saveIndexPattern; //: DataPublicPluginStart['indexPatterns']['updateSavedObject']; } -const searchAriaLabel = "Search fields"; +const searchAriaLabel = formatMessage({ + id: "explore.view.index_pattern.search_fields", +}); -const filterAriaLabel = "Filter field types"; +const filterAriaLabel = formatMessage({ + id: "explore.view.index_pattern.filter_field_types", +}); -const filterPlaceholder = "Search"; +const filterPlaceholder = formatMessage({ + id: "explore.view.index_pattern.search_placeholder", +}); export function Tabs({ indexPattern, @@ -218,7 +225,7 @@ export function Tabs({ history.push(`/patterns/${indexPattern?.id}/complex/create`); }} > - {"Create field"} + {formatMessage({ id: "explore.view.index_pattern.create_field" })} @@ -235,7 +242,7 @@ export function Tabs({ switch (type) { case TAB_INDEXED_FIELDS: return ( - + {getFilterSection(type)} @@ -256,7 +263,7 @@ export function Tabs({ ); case TAB_COMPLEX_FIELDS: return ( - + {getComplexFilterSection()} @@ -275,7 +282,7 @@ export function Tabs({ ); case TAB_SCRIPTED_FIELDS: return ( - + {getFilterSection(type)} @@ -296,7 +303,7 @@ export function Tabs({ ); case TAB_SOURCE_FILTERS: return ( - + {getFilterSection(type)} @@ -347,7 +354,10 @@ export function Tabs({ count = fields.length } return tabs.concat([{ - name: `Complex fields (${count})`, + name: formatMessage( + { id: "explore.view.index_pattern.tab.complex_fields" }, + { count } + ), id: TAB_COMPLEX_FIELDS, content: getContent(TAB_COMPLEX_FIELDS) }]) diff --git a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/utils.ts b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/utils.ts index 73dd3289..22ac8421 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/utils.ts +++ b/web/src/components/vendor/index_pattern_management/public/components/edit_index_pattern/tabs/utils.ts @@ -22,6 +22,7 @@ import { Dictionary, countBy, defaults, uniq } from 'lodash'; import { IndexPatternManagementStart } from '../../../../../../plugins/index_pattern_management/public'; import { TAB_INDEXED_FIELDS, TAB_SCRIPTED_FIELDS, TAB_SOURCE_FILTERS } from '../constants'; import { IndexPattern, IndexPatternField } from '../../../import'; +import { formatMessage } from "umi/locale"; function filterByName(items: IndexPatternField[], filter: string) { const lowercaseFilter = (filter || '').toLowerCase(); @@ -56,13 +57,13 @@ function getTitle(type: string, filteredCount: Dictionary, totalCount: D let title = ''; switch (type) { case 'indexed': - title = 'Fields'; + title = formatMessage({ id: "explore.view.index_pattern.tab.fields" }); break; case 'scripted': - title = 'Scripted fields'; + title = formatMessage({ id: "explore.view.index_pattern.tab.scripted_fields" }); break; case 'sourceFilters': - title = 'Source filters'; + title = formatMessage({ id: "explore.view.index_pattern.tab.source_filters" }); break; } const count = ` (${ @@ -114,9 +115,13 @@ export function getPath(field: IndexPatternField, indexPattern: IndexPattern) { return `/patterns/${indexPattern?.id}/field/${field.name}`; } -const allTypesDropDown = 'All field types'; +const allTypesDropDown = formatMessage({ + id: "explore.view.index_pattern.all_field_types", +}); -const allLangsDropDown = 'All languages'; +const allLangsDropDown = formatMessage({ + id: "explore.view.index_pattern.all_languages", +}); export function convertToEuiSelectOption(options: string[], type: string) { const euiOptions = diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/complex_field_editor.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/complex_field_editor.tsx index 896c943e..665c3e42 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/complex_field_editor.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/complex_field_editor.tsx @@ -64,6 +64,12 @@ import styles from './complex_field_editor.less' import { generate20BitUUID } from '@/utils/utils'; import { Tags } from './field_editor'; +const i18nText = ( + id: string, + defaultMessage: string, + values?: Record +) => formatMessage({ id, defaultMessage }, values); + const getFieldTypeFormatsList = ( field: IndexPatternField['spec'], defaultFieldFormat: FieldFormatInstanceType, @@ -80,7 +86,10 @@ const getFieldTypeFormatsList = ( { id: '', defaultFieldFormat, - title: '- Default -', + title: i18nText( + "explore.view.index_pattern.field_editor.default_option", + "- Default -" + ), }, ...formatsByType, ]; @@ -212,26 +221,36 @@ export class ComplexFieldEditor extends PureComponent   - You already have a field with the name {spec.name}. + {i18nText( + "explore.view.index_pattern.complex_field_editor.duplicate_name", + "You already have a field with the name {name}.", + { name: spec.name } + )} ) : null } isInvalid={isInvalid} error={ isInvalid - ? 'Name is required' + ? i18nText( + "explore.view.index_pattern.complex_field_editor.name_required", + "Name is required" + ) : null } > { this.onFieldChange('name', e.target.value); @@ -248,18 +267,22 @@ export class ComplexFieldEditor extends PureComponent - Format (Default: {defaultFormat}) + {i18nText("explore.view.index_pattern.field_editor.format", "Format")} ( + {i18nText("explore.view.index_pattern.field_editor.default_label", "Default")}: {defaultFormat}) + ) : ( - "Format" + i18nText("explore.view.index_pattern.field_editor.format", "Format") ); return ( - + { - this.hideDeleteModal(); - this.deleteField(); - }} - cancelButtonText='Cancel' - confirmButtonText= 'Delete' - buttonColor="danger" - defaultFocusedButton={EUI_MODAL_CONFIRM_BUTTON} - > -

- You can't recover a deleted field. -
-
- Are you sure you want to do this? -

- + title={i18nText( + "explore.view.index_pattern.complex_field_editor.delete_title", + "Delete field '{name}'", + { name: spec.metric_name || spec.name } + )} + onCancel={this.hideDeleteModal} + onConfirm={() => { + this.hideDeleteModal(); + this.deleteField(); + }} + cancelButtonText={i18nText("form.button.cancel", "Cancel")} + confirmButtonText={i18nText("form.button.delete", "Delete")} + buttonColor="danger" + defaultFocusedButton={EUI_MODAL_CONFIRM_BUTTON} + > +

+ {i18nText( + "explore.view.index_pattern.complex_field_editor.delete_confirm", + "You can't recover a deleted field. Are you sure you want to do this?" + )} +

+ ) : null; }; @@ -348,15 +375,15 @@ export class ComplexFieldEditor extends PureComponent {isCreating ? ( - "Create field" + i18nText("explore.view.index_pattern.create_field", "Create field") ) : ( - "Save field" + i18nText("explore.view.index_pattern.field_editor.save_field", "Save field") )} - Cancel + {i18nText("form.button.cancel", "Cancel")} {!isCreating ? ( @@ -364,7 +391,7 @@ export class ComplexFieldEditor extends PureComponent - Delete + {i18nText("form.button.delete", "Delete")} @@ -476,7 +503,10 @@ export class ComplexFieldEditor extends PureComponent { this.onFieldChange('tags', value) @@ -533,9 +572,13 @@ export class ComplexFieldEditor extends PureComponent

{isCreating ? ( - "Create field" + i18nText("explore.view.index_pattern.create_field", "Create field") ) : ( - `Edit ${spec.metric_name }` + i18nText( + "explore.view.index_pattern.complex_field_editor.edit_title", + "Edit {name}", + { name: spec.metric_name || spec.name } + ) )}

diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/color/color.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/color/color.tsx index 5679d331..2b7a31d4 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/color/color.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/color/color.tsx @@ -210,7 +210,7 @@ export class ColorFormatEditor extends DefaultFormatEditor< ]; return ( - + diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date/date.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date/date.tsx index 46fbaf8d..2634792a 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date/date.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date/date.tsx @@ -47,7 +47,7 @@ export class DateFormatEditor extends DefaultFormatEditor + {defaultPattern}})` diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date_nanos/date_nanos.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date_nanos/date_nanos.tsx index e818b39e..33db4067 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date_nanos/date_nanos.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/date_nanos/date_nanos.tsx @@ -46,7 +46,7 @@ export class DateNanosFormatEditor extends DefaultFormatEditor + {defaultPattern}})` diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/duration/duration.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/duration/duration.tsx index 29cab22b..5bbecf38 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/duration/duration.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/duration/duration.tsx @@ -94,7 +94,7 @@ export class DurationFormatEditor extends DefaultFormatEditor< const { error, samples, hasDecimalError } = this.state; return ( - + + {defaultPattern}})` diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/static_lookup/static_lookup.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/static_lookup/static_lookup.tsx index 17e5ac47..bbbb13c5 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/static_lookup/static_lookup.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/static_lookup/static_lookup.tsx @@ -142,7 +142,7 @@ export class StaticLookupFormatEditor extends DefaultFormatEditor< ]; return ( - + diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/string/string.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/string/string.tsx index f58824f4..aa0e17b8 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/string/string.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/field_format_editor/editors/string/string.tsx @@ -53,7 +53,7 @@ export class StringFormatEditor extends DefaultFormatEditor + + + + { return isVisible ? ( - + ().services.docLinks?.links .scriptedFields; return isVisible ? ( - + { const docLinksScriptedFields = useGlobalContext().docLinks?.links//useKibana().services.docLinks?.links .scriptedFields; return ( - +

diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/scripting_help/test_script.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/scripting_help/test_script.tsx index bd1b3678..8d7ba5d7 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/scripting_help/test_script.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/components/scripting_help/test_script.tsx @@ -161,7 +161,7 @@ export class TestScript extends Component { } return ( - +

First 10 results

 @@ -213,7 +213,7 @@ export class TestScript extends Component { }); return ( - + { render() { return ( - +

Preview results

diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/field_editor.tsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/field_editor.tsx index 9b374966..e91cfd5c 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/field_editor.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/field_editor.tsx @@ -85,6 +85,12 @@ import { formatMessage } from "umi/locale"; import { message } from 'antd'; import { getRollupEnabled } from '@/utils/authority'; +const i18nText = ( + id: string, + defaultMessage: string, + values?: Record +) => formatMessage({ id, defaultMessage }, values); + export const getStatistics = (type) => { if (!type || type === 'string') return ["count", "cardinality"]; return [ @@ -119,7 +125,10 @@ const getFieldTypeFormatsList = ( { id: '', defaultFieldFormat, - title: '- Default -', + title: i18nText( + "explore.view.index_pattern.field_editor.default_option", + "- Default -" + ), }, ...formatsByType, ]; @@ -461,18 +470,22 @@ export class FieldEditor extends PureComponent - Format (Default: {defaultFormat}) + {i18nText("explore.view.index_pattern.field_editor.format", "Format")} ( + {i18nText("explore.view.index_pattern.field_editor.default_label", "Default")}: {defaultFormat}) + ) : ( - "Format" + i18nText("explore.view.index_pattern.field_editor.format", "Format") ); return ( - + + - + Access fields with {`doc['some_field'].value`}. @@ -638,15 +651,15 @@ export class FieldEditor extends PureComponent {isCreating ? ( - "Create field" + i18nText("explore.view.index_pattern.create_field", "Create field") ) : ( - "Save field" + i18nText("explore.view.index_pattern.field_editor.save_field", "Save field") )} - Cancel + {i18nText("form.button.cancel", "Cancel")} {!isCreating && spec.scripted ? ( @@ -654,7 +667,7 @@ export class FieldEditor extends PureComponent - Delete + {i18nText("form.button.delete", "Delete")} @@ -673,7 +686,7 @@ export class FieldEditor extends PureComponent + { this.onMetricSettingsChange('tags', value) @@ -1003,7 +1028,7 @@ export const Tags = ({ value = [], onChange }) => { onClick={showInput} style={{ height: '40px', lineHeight: '40px', fontSize: 14}} > - Add New + {i18nText("explore.view.index_pattern.field_editor.add_new", "Add New")} )} diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency.jsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency.jsx index 35d28f39..8b6b7912 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency.jsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency.jsx @@ -1,6 +1,8 @@ import { EuiComboBox, EuiFormRow } from "@elastic/eui" +import { formatMessage } from "umi/locale"; export default (props) => { + const t = (id, defaultMessage) => formatMessage({ id, defaultMessage }); const { indexPattern, spec, onChange } = props; const keys = Object.keys(spec?.function || {}) const statistic = keys[0] @@ -9,7 +11,7 @@ export default (props) => { return ( <> - + !!item.spec?.name).map((item) => ( @@ -25,7 +27,7 @@ export default (props) => { isClearable={false} /> - + !!item.spec?.name).map((item) => ( diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency_sum_func_value_in_group.jsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency_sum_func_value_in_group.jsx index ec071073..36e1ddf5 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency_sum_func_value_in_group.jsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/latency_sum_func_value_in_group.jsx @@ -1,7 +1,9 @@ import { EuiComboBox, EuiFormRow } from "@elastic/eui" import { getStatistics } from "."; +import { formatMessage } from "umi/locale"; export default (props) => { + const t = (id, defaultMessage) => formatMessage({ id, defaultMessage }); const { indexPattern, spec, onChange } = props; const keys = Object.keys(spec?.function || {}) const statistic = keys[0] @@ -10,7 +12,7 @@ export default (props) => { return ( <> - + !!item.spec?.name).map((item) => ( @@ -31,7 +33,7 @@ export default (props) => { isClearable={false} /> - + !!item.spec?.name).map((item) => ( @@ -47,7 +49,7 @@ export default (props) => { isClearable={false} /> - + !!item.spec?.name).map((item) => ( diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate.jsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate.jsx index 6a3c349e..85a68a2c 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate.jsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate.jsx @@ -1,6 +1,8 @@ import { EuiComboBox, EuiFormRow } from "@elastic/eui" +import { formatMessage } from "umi/locale"; export default (props) => { + const t = (id, defaultMessage) => formatMessage({ id, defaultMessage }); const { indexPattern, spec, onChange } = props; const keys = Object.keys(spec?.function || {}) const statistic = keys[0] @@ -8,7 +10,7 @@ export default (props) => { const { field, group } = func || {} return ( - + !!item.spec?.name).map((item) => ( diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate_sum_func_value_in_group.jsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate_sum_func_value_in_group.jsx index 67781352..a0f468df 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate_sum_func_value_in_group.jsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/rate_sum_func_value_in_group.jsx @@ -1,7 +1,9 @@ import { EuiComboBox, EuiFormRow } from "@elastic/eui" import { getStatistics } from "."; +import { formatMessage } from "umi/locale"; export default (props) => { + const t = (id, defaultMessage) => formatMessage({ id, defaultMessage }); const { indexPattern, spec, onChange } = props; const keys = Object.keys(spec?.function || {}) const statistic = keys[0] @@ -10,7 +12,7 @@ export default (props) => { return ( <> - + !!item.spec?.name).map((item) => ( @@ -30,7 +32,7 @@ export default (props) => { isClearable={false} /> - + !!item.spec?.name).map((item) => ( diff --git a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/sum_func_value_in_group.jsx b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/sum_func_value_in_group.jsx index b287a448..4a35be0f 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/sum_func_value_in_group.jsx +++ b/web/src/components/vendor/index_pattern_management/public/components/field_editor/functions/sum_func_value_in_group.jsx @@ -1,8 +1,10 @@ import { EuiComboBox, EuiFormRow } from "@elastic/eui" import { useState } from "react"; import { getStatistics } from "."; +import { formatMessage } from "umi/locale"; export default (props) => { + const t = (id, defaultMessage) => formatMessage({ id, defaultMessage }); const { indexPattern, spec, onChange } = props; const keys = Object.keys(spec?.function || {}) const statistic = keys[0] @@ -11,7 +13,7 @@ export default (props) => { return ( <> - + !!item.spec?.name).map((item) => ( @@ -31,7 +33,7 @@ export default (props) => { isClearable={false} /> - + !!item.spec?.name).map((item) => ( diff --git a/web/src/components/vendor/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx b/web/src/components/vendor/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx index 732bf7f5..70db4f0d 100644 --- a/web/src/components/vendor/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx +++ b/web/src/components/vendor/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx @@ -43,8 +43,7 @@ import { import { EmptyIndexPatternPrompt } from "./empty_index_pattern_prompt"; import { getIndices } from "../create_index_pattern_wizard/lib"; import { useGlobalContext } from "../../context"; -import { Card, Button, Table, Input, Divider, Popconfirm, message } from "antd"; -import PageHeaderWrapper from "@/components/PageHeaderWrapper"; +import { Card, Button, Table, Input, Divider, Popconfirm, message, Icon } from "antd"; import styles from "@/pages/System/Cluster/step.less"; import { router } from "umi"; import { formatMessage } from "umi/locale"; @@ -53,6 +52,11 @@ import { filterSearchValue, sorter } from "@/utils/utils"; const { Search } = Input; const title = formatMessage({ id: "explore.viewlist.title" }); +const firstColumnIconStyle = { + marginRight: 8, + color: "#999", + fontSize: 12, +}; interface Props extends RouteComponentProps { canSave: boolean; @@ -200,8 +204,10 @@ export const IndexPatternTable = ({ onClick={() => { router.push(`/insight/discover?viewID=${record.id}`); }} + style={{ display: "inline-flex", alignItems: "center" }} > - {text} + + {text} ), sorter: (a: string, b: string) => sorter.string(a, b, "viewName"), @@ -220,6 +226,7 @@ export const IndexPatternTable = ({ }, { title: formatMessage({ id: "table.field.actions" }), + width: 100, render: (text, record) => (
{canSave && !record.builtin ? ( @@ -281,72 +288,70 @@ export const IndexPatternTable = ({ // } return ( - - + +
+
+ { + setSearchValue(value); + }} + onChange={(e) => { + setSearchValue(e.currentTarget.value); + }} + /> +
+
-
- { - setSearchValue(value); - }} - onChange={(e) => { - setSearchValue(e.currentTarget.value); - }} - /> -
- -
{ + onRefreshClick(); }} > - - {createButton} -
+ {formatMessage({ id: "form.button.refresh" })} + + {createButton}
- { - dispatch({ type: "pagination", value: page }); - }, - showSizeChanger: true, - onShowSizeChange: (_, size) => { - dispatch({ type: "pageSizeChange", value: size }); - }, - showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, - }} - columns={columns} - /> - - + +
{ + dispatch({ type: "pagination", value: page }); + }, + showSizeChanger: true, + onShowSizeChange: (_, size) => { + dispatch({ type: "pageSizeChange", value: size }); + }, + showTotal: (total, range) => + `${range[0]}-${range[1]} of ${total} items`, + }} + columns={columns} + /> + ); }; diff --git a/web/src/global.less b/web/src/global.less index deaf97d0..b7a3d974 100644 --- a/web/src/global.less +++ b/web/src/global.less @@ -36,3 +36,5 @@ ul, ol { list-style: none; } + +@layout-header-height: 64px; diff --git a/web/src/layouts/BasicLayout.js b/web/src/layouts/BasicLayout.js index 30348ebe..9fcf8be7 100644 --- a/web/src/layouts/BasicLayout.js +++ b/web/src/layouts/BasicLayout.js @@ -19,8 +19,15 @@ import Header from "./Header"; import Context from "./MenuContext"; import Exception403 from "../pages/Exception/403"; import { GlobalContext } from "./GlobalContext"; -import { getAuthEnabled, getAuthority, isLogin } from "@/utils/authority"; -import { router, history } from "umi"; +import { + APPLICATION_SETTINGS_UPDATED_EVENT, + getAuthEnabled, + getAuthority, + getEnterpriseTaskManagerEnabled, + isLogin, + refreshApplicationSettings, +} from "@/utils/authority"; +import { router } from "umi"; import request from "@/utils/request"; import HealthProvider from "@/components/HealthProvider"; import { getSetupRequired } from "@/utils/setup"; @@ -69,7 +76,7 @@ const { Content } = Layout; function formatter(data, parentAuthority, parentName) { return data .map((item) => { - let locale = "menu"; + let locale; if (parentName && item.name) { locale = `${parentName}.${item.name}`; } else if (item.name) { @@ -97,6 +104,34 @@ function formatter(data, parentAuthority, parentName) { .filter((item) => item); } +function filterMenuDataByName(menuData, names = []) { + return (menuData || []).map((item) => { + const nextItem = { ...item }; + if (names.includes(nextItem.name)) { + nextItem.hideInMenu = true; + } + if (nextItem.children) { + nextItem.children = filterMenuDataByName(nextItem.children, names); + } + return nextItem; + }); +} + +function hasCurrentUserSession(response) { + if (!response || typeof response !== "object" || response.error) { + return false; + } + + const source = response._source || response; + return !!( + source?.user_id || + source?.id || + response?._id || + response?.id || + source?.username + ); +} + const memoizeOneFormatter = memoizeOne(formatter, isEqual); const query = { @@ -133,9 +168,11 @@ class BasicLayout extends React.PureComponent { } state = { + authResolved: getAuthEnabled() !== "true", rendering: true, isMobile: false, menuData: this.getMenuData(), + sessionValid: getAuthEnabled() !== "true", welcomeModal: null, }; @@ -144,13 +181,38 @@ class BasicLayout extends React.PureComponent { this.state.welcomeModal.destroy(); }; + redirectToLogin = () => { + if (router && typeof router.replace === "function") { + router.replace("/user/login"); + return; + } + + const { history } = this.props; + if (history && typeof history.replace === "function") { + history.replace("/user/login"); + return; + } + + window.location.replace("/user/login"); + }; + async componentDidMount() { const { menuData } = this.state; const { dispatch, global } = this.props; + let sessionValid = getAuthEnabled() !== "true"; if (getAuthEnabled() === "true") { - dispatch({ + const response = await dispatch({ type: "user/fetchCurrent", }); + sessionValid = hasCurrentUserSession(response); + this.setState({ + authResolved: true, + sessionValid, + }); + if (!sessionValid) { + this.redirectToLogin(); + return; + } } dispatch({ type: "setting/getSetting", @@ -179,8 +241,31 @@ class BasicLayout extends React.PureComponent { // }); } }); + this.handleDataToolsLicenseRequired = () => { + this.licenceRef?.openToTab?.("license"); + }; + window.addEventListener( + "console:datatools-license-required", + this.handleDataToolsLicenseRequired + ); + this.handleApplicationSettingsUpdated = async () => { + const menuData = this.getMenuData(); + this.setState({ menuData }); + await dispatch({ + type: "global/saveData", + payload: { + menuData, + }, + }); + }; + window.addEventListener( + APPLICATION_SETTINGS_UPDATED_EVENT, + this.handleApplicationSettingsUpdated + ); + await refreshApplicationSettings(); + await this.handleApplicationSettingsUpdated(); let firstLogin = localStorage.getItem("first-login"); - if (firstLogin === "true" && isLogin()) { + if (firstLogin === "true" && isLogin() && sessionValid) { localStorage.setItem("first-login", false); this.state.welcomeModal = Modal.info({ title: ( @@ -214,6 +299,9 @@ class BasicLayout extends React.PureComponent { this.isInited = true; return; } + if (!this.state.authResolved || !this.state.sessionValid) { + return; + } if (isLogin()) { dispatch({ type: "global/fetchClusterList", @@ -240,6 +328,14 @@ class BasicLayout extends React.PureComponent { componentWillUnmount() { cancelAnimationFrame(this.renderRef); unenquireScreen(this.enquireHandler); + window.removeEventListener( + "console:datatools-license-required", + this.handleDataToolsLicenseRequired + ); + window.removeEventListener( + APPLICATION_SETTINGS_UPDATED_EVENT, + this.handleApplicationSettingsUpdated + ); } getContext() { @@ -254,7 +350,11 @@ class BasicLayout extends React.PureComponent { const { route: { routes }, } = this.props; - return memoizeOneFormatter(routes); + let menuData = memoizeOneFormatter(routes); + if (getEnterpriseTaskManagerEnabled() !== "true") { + menuData = filterMenuDataByName(menuData, ["data_tools"]); + } + return menuData; // } @@ -290,8 +390,12 @@ class BasicLayout extends React.PureComponent { if (!currRouterData) { return APP_TITLE; } + const messageId = currRouterData.locale || currRouterData.name; + if (!messageId) { + return APP_TITLE; + } const message = formatMessage({ - id: currRouterData.locale || currRouterData.name, + id: messageId, defaultMessage: currRouterData.name, }); return `${message} - ${APP_TITLE}`; @@ -348,7 +452,16 @@ class BasicLayout extends React.PureComponent { } = this.props; const { isMobile, menuData } = this.state; const isTop = PropsLayout === "topmenu"; + const isDevtoolConsolePage = pathname.startsWith("/devtool/console"); + const hideFooter = isDevtoolConsolePage; const routerConfig = this.matchParamsPath(pathname); + const contentStyle = isDevtoolConsolePage + ? { + ...this.getContentStyle(), + margin: 0, + overflow: "hidden", + } + : this.getContentStyle(); const renderInvalidSecretNotification = () => { const secretMismatch = localStorage.getItem("secret_mismatch"); @@ -357,6 +470,9 @@ class BasicLayout extends React.PureComponent { } return null; }; + if (getAuthEnabled() === "true" && (!this.state.authResolved || !this.state.sessionValid)) { + return null; + } const layout = ( <> {renderInvalidSecretNotification()} @@ -389,7 +505,7 @@ class BasicLayout extends React.PureComponent { {...this.props} /> - + } @@ -398,6 +514,8 @@ class BasicLayout extends React.PureComponent { value={{ ...(this.props.global || {}), dispatch: this.props.dispatch, + authResolved: this.state.authResolved, + sessionValid: this.state.sessionValid, }} > {children} @@ -405,7 +523,7 @@ class BasicLayout extends React.PureComponent { -
+ {!hideFooter ?
: null} diff --git a/web/src/layouts/Footer.js b/web/src/layouts/Footer.js index 28fff455..57436ff7 100644 --- a/web/src/layouts/Footer.js +++ b/web/src/layouts/Footer.js @@ -7,7 +7,7 @@ const FooterView = () => (
©{APP_AUTHOR}, All Rights Reserved.} + copyright={©{APP_AUTHOR}, All Rights Reserved.} />
); diff --git a/web/src/layouts/GuideLayout.js b/web/src/layouts/GuideLayout.js index e5a6ca15..d5e7f51b 100644 --- a/web/src/layouts/GuideLayout.js +++ b/web/src/layouts/GuideLayout.js @@ -8,7 +8,7 @@ import SelectLang from "@/components/SelectLang"; import request from "@/utils/request"; import { router } from "umi"; import { formatMessage } from "umi/locale"; -import { getHealth } from "@/services/system" +import { refreshApplicationSettings } from "@/utils/authority"; const { Header, Footer, Content } = Layout; @@ -17,7 +17,7 @@ export default ({ children }) => { const fetchHealth = async () => { try { - const res = await getHealth(); + const res = await refreshApplicationSettings(); if (!res?.setup_required) { router.push("/"); } diff --git a/web/src/layouts/GuideLayout.less b/web/src/layouts/GuideLayout.less index d408c485..bd6d5960 100644 --- a/web/src/layouts/GuideLayout.less +++ b/web/src/layouts/GuideLayout.less @@ -3,9 +3,10 @@ @pro-header-hover-bg: rgba(0, 0, 0, 0.025); .container { - display: block; + display: flex; flex-direction: column; min-height: 100vh; + overflow: hidden; background-color: #f7f9fc; .header { @@ -50,11 +51,15 @@ } .content { + flex: 1 1 auto; + min-height: 0; + display: flex; + overflow: hidden; } .footer { padding: 0 16px; - margin: 48px 0 24px 0; + margin: 8px 0 12px 0; text-align: center; background-color: #f7f9fc; diff --git a/web/src/layouts/Header.js b/web/src/layouts/Header.js index 69ce157b..a91a2041 100644 --- a/web/src/layouts/Header.js +++ b/web/src/layouts/Header.js @@ -11,6 +11,7 @@ import Authorized from "@/utils/Authorized"; import { getSetupRequired } from "@/utils/setup"; const { Header } = Layout; +const CLUSTER_STATUS_REFRESH_INTERVAL = 60 * 1000; class HeaderView extends PureComponent { state = { @@ -28,15 +29,17 @@ class HeaderView extends PureComponent { componentDidMount() { document.addEventListener("scroll", this.handScroll, { passive: true }); - this.fetchClusterStatus(); + document.addEventListener("visibilitychange", this.handleVisibilityChange); + window.addEventListener("focus", this.handleWindowFocus); + this.fetchClusterStatus({ force: true }); this.handleNoticeVisibleChange(true); } componentWillUnmount() { document.removeEventListener("scroll", this.handScroll); - if (this.fetchClusterStatusTimer) { - clearTimeout(this.fetchClusterStatusTimer); - } + document.removeEventListener("visibilitychange", this.handleVisibilityChange); + window.removeEventListener("focus", this.handleWindowFocus); + this.clearClusterStatusTimer(); } getHeadWidth = () => { @@ -149,24 +152,59 @@ class HeaderView extends PureComponent { }); }; - fetchClusterStatus = async () => { + clearClusterStatusTimer = () => { + if (this.fetchClusterStatusTimer) { + clearTimeout(this.fetchClusterStatusTimer); + this.fetchClusterStatusTimer = null; + } + }; + + scheduleClusterStatusRefresh = () => { + this.clearClusterStatusTimer(); + if (document.visibilityState === "hidden") { + return; + } + this.fetchClusterStatusTimer = setTimeout(() => { + this.fetchClusterStatus(); + }, CLUSTER_STATUS_REFRESH_INTERVAL); + }; + + handleVisibilityChange = () => { + if (document.visibilityState === "hidden") { + this.clearClusterStatusTimer(); + return; + } + this.fetchClusterStatus(); + }; + + handleWindowFocus = () => { + if (document.visibilityState === "hidden") { + return; + } + this.fetchClusterStatus(); + }; + + fetchClusterStatus = async ({ force = false } = {}) => { if ( location.href.indexOf("/guide/initialization") !== -1 || - getSetupRequired() === "true" + getSetupRequired() === "true" || + document.visibilityState === "hidden" ) { + this.clearClusterStatusTimer(); return; } const { dispatch } = this.props; const res = await dispatch({ type: "global/fetchClusterStatus", + payload: { + force, + }, }); - if (this.fetchClusterStatusTimer) { - clearTimeout(this.fetchClusterStatusTimer); - } + this.clearClusterStatusTimer(); if (!res) { return; } - this.fetchClusterStatusTimer = setTimeout(this.fetchClusterStatus, 10000); + this.scheduleClusterStatusRefresh(); }; render() { diff --git a/web/src/layouts/UserLayout.js b/web/src/layouts/UserLayout.js index 9362c59b..63657e6e 100644 --- a/web/src/layouts/UserLayout.js +++ b/web/src/layouts/UserLayout.js @@ -26,7 +26,7 @@ const links = [ ]; const copyright = ( - + Copyright {new Date().getFullYear()} {APP_AUTHOR} ); diff --git a/web/src/lib/elasticsearch/ilm.ts b/web/src/lib/elasticsearch/ilm.ts index 19f18992..a725e295 100644 --- a/web/src/lib/elasticsearch/ilm.ts +++ b/web/src/lib/elasticsearch/ilm.ts @@ -5,40 +5,151 @@ type TransformOptions = { targetDistribution: string, } -export const transform = (config: any, options: TransformOptions) => { - if(options.sourceDistribution === SearchEngines.Opensearch){ - return transformOpensearchToElasticsearch(config); +const pruneKeys = (target: any, keys: string[]) => { + if (!target || typeof target !== "object" || Array.isArray(target)) { + return; } + keys.forEach((key) => { + delete target[key]; + }); +} + +const sanitizeElasticsearchPayload = (config: any) => { + pruneKeys(config, ["version", "modified_date", "modified_date_string", "in_use_by", "_id", "_version", "_seq_no", "_primary_term"]); + return config; +} + +const sanitizeISMPayload = (config: any) => { + if(!config || !config.policy){ + return {}; + } + pruneKeys(config, ["version", "modified_date", "modified_date_string", "in_use_by", "_id", "_version", "_seq_no", "_primary_term", "policy_seq_no", "policy_primary_term"]); + const policy = config.policy; + pruneKeys(policy, ["policy_id", "last_updated_time", "created_time", "_meta"]); + if(!policy["description"]){ + policy["description"] = "tranform with infini console"; + } + if(!policy["default_state"]){ + policy["default_state"] = policy.states?.[0]?.name; + } + if(Array.isArray(policy["ism_template"])){ + policy["ism_template"] = policy["ism_template"] + .filter((item: any) => item && typeof item === "object" && !Array.isArray(item)) + .map((item: any) => { + const nextItem = { ...item }; + pruneKeys(nextItem, ["last_updated_time"]); + nextItem.index_patterns = Array.isArray(nextItem.index_patterns) + ? nextItem.index_patterns.filter((pattern: any) => typeof pattern === "string" && pattern.trim()) + : []; + if(typeof nextItem.priority !== "number"){ + nextItem.priority = 100; + } + return nextItem; + }) + .filter((item: any) => item.index_patterns.length > 0); + } else if(policy["ism_template"] && typeof policy["ism_template"] === "object"){ + policy["ism_template"] = [policy["ism_template"]]; + return sanitizeISMPayload({ policy }); + } else { + delete policy["ism_template"]; + } + if(Array.isArray(policy["ism_template"]) && policy["ism_template"].length === 0){ + delete policy["ism_template"]; + } + return { + policy, + }; +} + +const stripWaitForSnapshotFromPhases = (config: any) => { + const phases = config?.policy?.phases; + if(!phases || typeof phases !== "object"){ + return config; + } + Object.keys(phases).forEach((phaseName) => { + const actions = phases?.[phaseName]?.actions; + if(actions && typeof actions === "object" && !Array.isArray(actions)){ + delete actions.wait_for_snapshot; + } + }); + return config; +} + +const stripDeleteSearchableSnapshotFromPhases = (config: any) => { + const phases = config?.policy?.phases; + if(!phases || typeof phases !== "object"){ + return config; + } + Object.keys(phases).forEach((phaseName) => { + const deleteAction = phases?.[phaseName]?.actions?.delete; + if(deleteAction && typeof deleteAction === "object" && !Array.isArray(deleteAction)){ + delete deleteAction.delete_searchable_snapshot; + } + }); + return config; +} + +const stripWaitForSnapshotFromStates = (config: any) => { + const states = config?.policy?.states; + if(!Array.isArray(states)){ + return config; + } + config.policy.states = states.map((state: any) => { + const actions = Array.isArray(state?.actions) ? state.actions : []; + return { + ...state, + actions: actions.filter((action: any) => { + return !(action && typeof action === "object" && !Array.isArray(action) && action.wait_for_snapshot !== undefined); + }), + }; + }); + return config; +} + +export const transform = (config: any, options: TransformOptions) => { if(options.targetDistribution === SearchEngines.Opensearch){ - return transformElasticsearchToOpensearch(config); + if(returnsInternalILMPolicy(options.sourceDistribution)){ + return stripWaitForSnapshotFromStates(normalizeISMPolicy(config)); + } + return stripWaitForSnapshotFromStates(transformElasticsearchToISM(config)); + } + if(returnsInternalILMPolicy(options.sourceDistribution)){ + return transformISMToElasticsearch(config); + } + if(options.targetDistribution === SearchEngines.Easysearch){ + return stripDeleteSearchableSnapshotFromPhases(stripWaitForSnapshotFromPhases(config)); } - return config } -const transformElasticsearchToOpensearch = (config: any) =>{ +const returnsInternalILMPolicy = (distribution: string) => { + return distribution === SearchEngines.Opensearch || distribution === SearchEngines.Easysearch; +} + +const normalizeISMPolicy = (config: any) => { if(!config || !config.policy){ return {}; } const policy = config.policy; - //rename [{from: "last_updated_time", to: "modified_date"}, {from :"schema_version", to:"version"}].forEach(item=>{ if(config[item.to]){ policy[item.from] = config[item.to]; delete config[item.to]; } }) + return sanitizeISMPayload(config); +} + +const transformElasticsearchToISM = (config: any) =>{ + if(!config || !config.policy){ + return {}; + } + const policy = config.policy; if(policy["phases"]){ policy["states"] = transformPhases(policy["phases"]); delete policy["phases"]; } - policy["description"] = "tranform with infini console"; - policy["default_state"] = policy.states[0]?.name; - policy["ism_template"] = { - "index_patterns": [], - "priority": 100 - } - return config; + return normalizeISMPolicy(config); } const transformPhases = (phases: any)=>{ @@ -52,16 +163,25 @@ const transformPhases = (phases: any)=>{ "transitions": [], } Object.keys(phases[pk].actions).forEach(key => { - if(pk === "delete"){ - delete phases[pk].actions[key]["delete_searchable_snapshot"]; - } //transform action key let tkey = key; if(key === "set_priority"){ tkey = "index_priority"; + } else if(key === "allocate"){ + tkey = "allocation"; + } else if(key === "forcemerge"){ + tkey = "force_merge"; + } else if(key === "readonly"){ + tkey = "read_only"; } //transform action value let tvalue = phases[pk].actions[key]; + if(tvalue && typeof tvalue === "object" && !Array.isArray(tvalue)){ + tvalue = { ...tvalue }; + } + if(pk === "delete" && tvalue && typeof tvalue === "object"){ + delete tvalue["delete_searchable_snapshot"]; + } if(key === "rollover"){ tvalue = transformRollover(tvalue, true) } @@ -96,7 +216,7 @@ const transformPhases = (phases: any)=>{ return states; } -const transformOpensearchToElasticsearch = (config: any)=>{ +const transformISMToElasticsearch = (config: any)=>{ if(!config || !config.policy){ return {}; } @@ -115,7 +235,7 @@ const transformOpensearchToElasticsearch = (config: any)=>{ delete policy[key] } - return config; + return sanitizeElasticsearchPayload(config); } const transformStates = (states: any[])=>{ @@ -123,20 +243,32 @@ const transformStates = (states: any[])=>{ states.forEach((st)=>{ const actions = {}; (st.actions || []).forEach((action: any)=>{ - const ak = Object.keys(action).shift(); - if(!ak) { - return; - } - let tkey = ak; - let tvalue = action[ak]; - if(tkey === "rollover"){ - tvalue = transformRollover(tvalue, false); - } - //transform key - if(tkey === "index_priority"){ - tkey = "set_priority"; - } - actions[tkey] = tvalue + Object.keys(action || {}).forEach((ak)=>{ + if(ak === "retry" || ak === "timeout"){ + return; + } + let tkey = ak; + let tvalue = action[ak]; + if(tvalue && typeof tvalue === "object" && !Array.isArray(tvalue)){ + tvalue = { ...tvalue }; + } + if(tkey === "rollover"){ + tvalue = transformRollover(tvalue, false); + } + if(tkey === "index_priority"){ + tkey = "set_priority"; + } else if(tkey === "allocation"){ + tkey = "allocate"; + if(tvalue && typeof tvalue === "object"){ + delete tvalue.wait_for; + } + } else if(tkey === "force_merge"){ + tkey = "forcemerge"; + } else if(tkey === "read_only"){ + tkey = "readonly"; + } + actions[tkey] = tvalue + }) }) phases[st.name] = { actions @@ -152,6 +284,9 @@ const transformStates = (states: any[])=>{ } const transformRollover = (rolloverCfg: any, reverse: boolean) => { + if(rolloverCfg && typeof rolloverCfg === "object" && !Array.isArray(rolloverCfg) && !reverse){ + delete rolloverCfg.copy_alias; + } [{from: "min_size", to:"max_size"},{from:"min_primary_shard_size", to:"max_primary_shard_size"}, {from: "min_doc_count", to:"max_docs"},{from: "min_index_age", to:"max_age"}].forEach((item)=>{ if(reverse){ @@ -167,4 +302,4 @@ const transformRollover = (rolloverCfg: any, reverse: boolean) => { } }); return rolloverCfg; -} \ No newline at end of file +} diff --git a/web/src/lib/elasticsearch/template.ts b/web/src/lib/elasticsearch/template.ts index 4df0a776..8a9b14fe 100644 --- a/web/src/lib/elasticsearch/template.ts +++ b/web/src/lib/elasticsearch/template.ts @@ -30,11 +30,163 @@ export const transform = (tpl: any, options: TransformOptions) => { delete tpl["template"]; } } - if(options.sourceDistribution != SearchEngines.Opensearch && options.targetDistribution === SearchEngines.Opensearch){ - if(tpl.settings?.index?.lifecycle){ - tpl.settings.index["plugins.index_state_management.rollover_alias"] = tpl.settings.index.lifecycle.rollover_alias; - delete tpl.settings.index["lifecycle"]; + if(options.sourceDistribution !== SearchEngines.Opensearch && options.targetDistribution === SearchEngines.Opensearch){ + tpl = transformElasticsearchTemplateLifecycleToISM(tpl); + } else if(options.sourceDistribution === SearchEngines.Opensearch && options.targetDistribution !== SearchEngines.Opensearch){ + tpl = transformISMTemplateLifecycleToElasticsearch(tpl); + } + return tpl; +} + +const ensureIndexSettings = (tpl: any) => { + tpl.settings = tpl.settings || {}; + tpl.settings.index = tpl.settings.index || {}; + return tpl.settings.index; +} + +const readElasticsearchLifecycleSetting = (settings: any, key: string) => { + if(!settings){ + return; + } + const flatValue = settings[`index.lifecycle.${key}`]; + if(flatValue !== undefined){ + return flatValue; + } + const indexSettings = settings.index || {}; + const nestedValue = indexSettings?.lifecycle?.[key]; + if(nestedValue !== undefined){ + return nestedValue; + } + return indexSettings?.[`lifecycle.${key}`]; +} + +const readISMSetting = (settings: any, key: string) => { + if(!settings){ + return; + } + for(const prefix of [ + "plugins.index_state_management", + "opendistro.index_state_management", + ]){ + const flatValue = settings[`index.${prefix}.${key}`]; + if(flatValue !== undefined){ + return flatValue; } } + const indexSettings = settings.index || {}; + for(const prefix of [ + "plugins.index_state_management", + "opendistro.index_state_management", + ]){ + const prefixedValue = indexSettings?.[`${prefix}.${key}`]; + if(prefixedValue !== undefined){ + return prefixedValue; + } + } + for(const namespace of ["plugins", "opendistro"]){ + const nestedValue = indexSettings?.[namespace]?.index_state_management?.[key]; + if(nestedValue !== undefined){ + return nestedValue; + } + } +} + +const deleteElasticsearchLifecycleSettings = (settings: any) => { + if(!settings){ + return; + } + delete settings["index.lifecycle.name"]; + delete settings["index.lifecycle.rollover_alias"]; + const indexSettings = settings.index || {}; + delete indexSettings["lifecycle.name"]; + delete indexSettings["lifecycle.rollover_alias"]; + if(indexSettings.lifecycle){ + delete indexSettings.lifecycle.name; + delete indexSettings.lifecycle.rollover_alias; + if(Object.keys(indexSettings.lifecycle).length === 0){ + delete indexSettings.lifecycle; + } + } +} + +const deleteISMSettings = (settings: any) => { + if(!settings){ + return; + } + for(const prefix of [ + "plugins.index_state_management", + "opendistro.index_state_management", + ]){ + delete settings[`index.${prefix}.policy_id`]; + delete settings[`index.${prefix}.rollover_alias`]; + } + const indexSettings = settings.index || {}; + for(const prefix of [ + "plugins.index_state_management", + "opendistro.index_state_management", + ]){ + delete indexSettings[`${prefix}.policy_id`]; + delete indexSettings[`${prefix}.rollover_alias`]; + } + for(const namespace of ["plugins", "opendistro"]){ + if(!indexSettings?.[namespace]?.index_state_management){ + continue; + } + delete indexSettings[namespace].index_state_management.policy_id; + delete indexSettings[namespace].index_state_management.rollover_alias; + if(Object.keys(indexSettings[namespace].index_state_management).length === 0){ + delete indexSettings[namespace].index_state_management; + } + if(Object.keys(indexSettings[namespace]).length === 0){ + delete indexSettings[namespace]; + } + } +} + +const transformElasticsearchTemplateLifecycleToISM = (tpl: any) => { + const settings = tpl.settings; + if(!settings){ + return tpl; + } + let policyID = readElasticsearchLifecycleSetting(settings, "name"); + let rolloverAlias = readElasticsearchLifecycleSetting(settings, "rollover_alias"); + if(policyID === undefined && rolloverAlias === undefined){ + policyID = readISMSetting(settings, "policy_id"); + rolloverAlias = readISMSetting(settings, "rollover_alias"); + if(policyID === undefined && rolloverAlias === undefined){ + return tpl; + } + } + deleteElasticsearchLifecycleSettings(settings); + deleteISMSettings(settings); + const indexSettings = ensureIndexSettings(tpl); + if(policyID !== undefined){ + indexSettings["plugins.index_state_management.policy_id"] = policyID; + } + if(rolloverAlias !== undefined){ + indexSettings["plugins.index_state_management.rollover_alias"] = rolloverAlias; + } return tpl; -} \ No newline at end of file +} + +const transformISMTemplateLifecycleToElasticsearch = (tpl: any) => { + const settings = tpl.settings; + if(!settings){ + return tpl; + } + const policyID = readISMSetting(settings, "policy_id"); + const rolloverAlias = readISMSetting(settings, "rollover_alias"); + deleteISMSettings(settings); + if(policyID === undefined && rolloverAlias === undefined){ + return tpl; + } + const indexSettings = ensureIndexSettings(tpl); + indexSettings.lifecycle = indexSettings.lifecycle || {}; + if(policyID !== undefined){ + indexSettings.lifecycle.name = policyID; + } + if(rolloverAlias !== undefined){ + indexSettings.lifecycle.rollover_alias = rolloverAlias; + } + return tpl; +} diff --git a/web/src/lib/elasticsearch/util.js b/web/src/lib/elasticsearch/util.js index 90d12ddf..8c4c6a08 100644 --- a/web/src/lib/elasticsearch/util.js +++ b/web/src/lib/elasticsearch/util.js @@ -52,6 +52,7 @@ export function formatESSearchResult(esResp) { took: took, total: total, data: [], + aggregations: esResp.aggregations, }; } let dataArr = []; @@ -161,12 +162,50 @@ export function extractClusterIDFromURL() { } export function formatTimeRange(timeRange) { - const bounds = calculateBounds({ - from: timeRange.min, - to: timeRange.max, - }); + const rawMin = `${timeRange?.min ?? ""}`.trim().toLowerCase(); + const rawMax = `${timeRange?.max ?? ""}`.trim().toLowerCase(); + if (rawMin === "auto" || rawMax === "auto") { + return { + min: "auto", + max: "auto", + }; + } + let bounds; + try { + bounds = calculateBounds({ + from: timeRange?.min, + to: timeRange?.max, + }); + } catch (e) { + bounds = null; + } + if (!bounds || !Number.isFinite(bounds.min?.valueOf?.()) || !Number.isFinite(bounds.max?.valueOf?.())) { + const min = timeRange?.min; + const max = timeRange?.max; + return { + min: Number.isFinite(min) ? min : Date.now() - 15 * 60 * 1000, + max: Number.isFinite(max) ? max : Date.now(), + }; + } return { min: bounds.min.valueOf(), max: bounds.max.valueOf(), }; } + +export function escapeLuceneQueryTerm(term = "") { + return `${term}`.replace(/([+\-=&|> `*${escapeLuceneQueryTerm(term)}*`) + .join(" AND "); +} diff --git a/web/src/lib/hooks/use_async.js b/web/src/lib/hooks/use_async.js index 5f7894e2..05a9efe7 100644 --- a/web/src/lib/hooks/use_async.js +++ b/web/src/lib/hooks/use_async.js @@ -2,15 +2,18 @@ import * as React from "react"; export default function useAsync(callback, dependencies = [], runInInit = true) { const loadingRef = React.useRef(false); + const pendingRef = React.useRef(false); const [loading, setLoading] = React.useState(true); const [error, setError] = React.useState(); const [value, setValue] = React.useState(); const callbackMemoized = React.useCallback(() => { if (loadingRef.current) { + pendingRef.current = true; return { loading: true, error, value }; } loadingRef.current = true; + pendingRef.current = false; setLoading(true); setError(undefined); // setValue(undefined); @@ -20,6 +23,10 @@ export default function useAsync(callback, dependencies = [], runInInit = true) .finally(() => { loadingRef.current = false; setLoading(false); + if (pendingRef.current) { + pendingRef.current = false; + callbackMemoized(); + } }); }, dependencies); diff --git a/web/src/locales/.gitignore b/web/src/locales/.gitignore new file mode 100644 index 00000000..80b6ceb6 --- /dev/null +++ b/web/src/locales/.gitignore @@ -0,0 +1 @@ +migration.js \ No newline at end of file diff --git a/web/src/locales/en-US.js b/web/src/locales/en-US.js index 4863bb07..74792e47 100644 --- a/web/src/locales/en-US.js +++ b/web/src/locales/en-US.js @@ -18,6 +18,11 @@ import listview from "./en-US/listview"; import audit from "./en-US/audit"; import error from "./en-US/error"; +let migration = {}; +try { + migration = require("./en-US/migration").default ?? {}; +} catch (e) {} + export default { "navBar.lang": "Languages", "layout.user.appslogon": @@ -87,7 +92,7 @@ export default { "form.button.add": "Add", "form.button.edit": "Edit", "form.button.update": "Update", - "form.button.clone": "Clone", + "form.button.clone": "Copy", "form.button.save": "Save", "form.button.submit": "Submit", "form.button.delete": "Delete", @@ -129,8 +134,88 @@ export default { "form.button.clean.deleted.indices.desc": "Are you sure to clean indices that are deleted?", "component.refreshGroup.label.title": "Auto Refresh", "component.refreshGroup.label.every": "Every", + "data_tools.task.keyword": "Search by keyword", + "table.field.id": "ID", "table.field.actions": "Actions", + "table.field.listen_address": "Listen Address", + "table.field.cmdline": "Cmdline", + "table.field.name": "Name", + "table.field.pid": "PID", + "system.security.tab.user": "User", + "system.security.tab.role": "Role", + "system.security.tab.token": "Access Credential", + "system.security.token.table.token": "Access Credential", + "system.security.token.search.placeholder": "Type keyword to search", + "system.security.token.table.name": "Name", + "system.security.token.table.description": "Description", + "system.security.token.table.permissions": "Permissions", + "system.security.token.table.expire": "Expire At", + "system.security.token.never_expire": "Never", + "system.security.token.form.name": "Name", + "system.security.token.form.name.required": "Please input token name!", + "system.security.token.form.description": "Description", + "system.security.token.form.expire": "Expire At", + "system.security.token.form.expire.required": "Please select an expiration time!", + "system.security.token.form.expire.future": "Expiration time must be in the future!", + "system.security.token.form.never_expire": "Never expire", + "system.security.token.drawer.create.title": "Create Access Credential", + "system.security.token.drawer.edit.title": "Edit Access Credential", + "system.security.token.create.result.title": "Access Credential Created", + "system.security.token.create.result.tip": + "Please copy and store this token now. It will only be shown once.", + "system.security.token.copy.tooltip": "Copy", + "system.security.token.copy.success": "Access credential copied to clipboard", + "system.security.token.copy.failed": "Failed to copy access credential", + "system.security.search.placeholder": "Type keyword to search", + "system.security.pagination.total": "{start}-{end} of {total} items", + "system.security.confirm.delete": "Are you sure you want to delete this item?", + "system.role.platform.name.required": "Please input name!", + "system.role.platform.feature_privilege.label": "Feature privileges", + "system.role.platform.feature_privilege.required": + "Please select platform feature privilege!", + "system.role.data.cluster.label": "Cluster", + "system.role.data.cluster.required": "Please select cluster!", + "system.role.data.cluster_privilege.label": "Cluster Privilege", + "system.role.data.cluster_privilege.required": + "Please select cluster privilege!", + "system.role.data.index_privilege.label": "Index Privilege", + "system.role.data.index_privilege.required": + "Please select index privilege!", + "system.role.data.api_privilege.category": "Category", + "system.role.data.api_privilege.privilege": "Privilege", + "system.role.data.index_privilege.index": "Index", + "system.role.data.index_privilege.privilege": "Privilege", + "system.security.user.table.name": "Name", + "system.security.user.table.nickname": "Nickname", + "system.security.user.table.roles": "Roles", + "system.security.user.table.phone": "Phone", + "system.security.user.table.email": "Email", + "system.security.user.table.tags": "Tags", + "system.security.user.table.status": "Status", + "system.security.user.action.reset_password": "Reset Password", + "system.security.user.form.name": "User Name", + "system.security.user.form.nickname": "Nick Name", + "system.security.user.form.phone": "Phone", + "system.security.user.form.email": "Email", + "system.security.user.form.roles": "Role", + "system.security.user.form.tags": "Tags", + "system.security.user.form.name.required": "Please input name!", + "system.security.user.form.email.invalid": "The input is not valid email!", + "system.security.user.form.roles.required": "Please select roles!", + "system.security.user.create.success": "Successfully Created User!", + "system.security.user.create.password": "Password: {password}", + "system.security.user.create.copy_password": "Copy Password", + "system.security.role.table.name": "Name", + "system.security.role.table.type": "Type", + "system.security.role.table.builtin": "Builtin", + "system.security.role.table.description": "Description", + "system.security.role.menu.add_platform": "Add Platform Role", + "system.security.role.menu.add_data": "Add Data Role", + "system.security.role.create.success": "Role created successfully!", + "system.security.role.create.button.view_list": "Back to Role List", + "system.security.role.create.button.continue": "Continue Creating", + "system.security.role.delete.error.assigned_to_users": "Cannot delete role: it is still assigned to users.", "component.globalHeader.search": "Search", "component.globalHeader.search.example1": "Search example 1", @@ -158,6 +243,30 @@ export default { "menu.insight": "DATA INSIGHT", "menu.insight.dashboard": "DASHBOARD", "menu.insight.discover": "DISCOVER", + "insight.config.title": "Insight Config", + "insight.config.tab.search": "Search", + "insight.config.search.track_total_hits": "Track Total Hits", + "insight.config.search.timeout": "Timeout", + "insight.config.search.field_summary": "Field Summary", + "insight.config.search.whether_to_sample": "Whether to Sample", + "insight.config.search.sample_records": "Sample Records", + "insight.config.search.sample_records.all": "All Records", + "insight.config.search.sample_records.manual": "Manual Setting", + "insight.config.search.top_number": "Top Number", + "insight.config.search.time_interval.seconds": "Seconds", + "insight.config.search.time_interval.minutes": "Minutes", + "insight.config.search.time_interval.hours": "Hours", + "insight.config.search.time_interval.days": "Days", + "insight.export.csv": "Export CSV", + "insight.export.excel": "Export Excel", + "insight.export.empty": "No data available to export", + "insight.discover.empty.no_indices_or_views": + "The current cluster has no indices or views", + "insight.discover.empty.create_now": "Create Now", + "insight.share.copy": "Copy share link", + "insight.share.success": "Share link copied", + "insight.share.failed": "Failed to copy share link", + "insight.button.updating": "Updating", //new alert version "menu.alerting.rule": "RULES", @@ -187,6 +296,8 @@ export default { "menu.data.overview": "OVERVIEW", "menu.data.index": "INDEX", "menu.data.view": "VIEW", + "menu.data.view.create": "CREATE", + "menu.data.view.detail": "DETAIL", "menu.data.document": "DOCUMENTS", "menu.data.template": "TEMPLATES", "menu.data.lifecycle": "LIFECYCLES", @@ -226,12 +337,17 @@ export default { "menu.resource.runtime.config": "CONFIG", "menu.resource.cluster": "CLUSTER", "menu.resource.registCluster": "REGISTER CLUSTER", + "menu.resource.editCluster": "EDIT CLUSTER", "menu.sysresourcetem.editCluster": "EDIT CLUSTER", "menu.resource.agent": "AGENTS", "menu.resource.agent.new_instance": "REGISTER AGENT", "menu.resource.agent.edit_instance": "EDIT AGENT", + "menu.data_tools": "DATA TOOLS", + "menu.data_tools.migration": "MIGRATION", + "menu.data_tools.comparison": "COMPARISON", + "menu.search": "SEARCH", "menu.search.overview": "OVERVIEW", "menu.search.template": "TEMPLATES", @@ -245,14 +361,20 @@ export default { "menu.synchronize.pipeline": "PIPELINES", "menu.synchronize.rebuild": "REBUILD", "menu.synchronize.inout": "CONNECT", + "synchronize.rebuild.target_index.label": "Target index name", + "synchronize.rebuild.target_index.required": "Please enter the target index name", + "synchronize.rebuild.target_index.invalid": + "The target index name must be lowercase and use valid Elasticsearch index name characters.", + "synchronize.ingest_pipeline.batch_delay.label": "Pipeline batch delay", + "synchronize.ingest_pipeline.batch_delay.placeholder": "Pipeline batch delay, e.g. 50", "menu.backup": "BACKUP", "menu.backup.overview": "OVERVIEW", "menu.backup.index": "BACKUPS", "menu.backup.lifecycle": "POLICIES", - "menu.system": "SETTINGS", - "menu.system.settings": "SETTINGS", + "menu.system": "SYSTEM", + "menu.system.settings": "SYSTEM SETTINGS", "menu.system.settings.global": "GLOBAL", "menu.system.settings.gateway": "GATEWAY", @@ -389,7 +511,9 @@ export default { "There is no available cluster, click OK to automatically jump to System Settings => Cluster Settings", "app.message.system-tips.no-available-cluster-data-permission": "There is no available cluster, please make sure you have cluster data permission", - "app.message.confirm.delete": "Sure to delete?", + "app.message.confirm.delete": "Are you sure you want to delete this item?", + "app.message.confirm.delete.multiple": "Are you sure you want to delete these {count} items?", + "document.confirm.cancel": "Sure to cancel?", "app.message.warning.table.select-row": "Please select a table row", "app.message.warning.invalid.params": "Invalid parameter", @@ -402,6 +526,13 @@ export default { "app.login.sign-in-with": "Sign in with SSO", "app.login.signup": "Sign up", "app.login.login": "Login", + "app.login.username.required": "Please enter username!", + "app.login.password.required": "Please enter password!", + "app.login.mobile.placeholder": "Mobile number", + "app.login.mobile.required": "Please enter mobile number!", + "app.login.mobile.invalid": "Wrong mobile number format!", + "app.login.captcha.placeholder": "Captcha", + "app.login.captcha.required": "Please enter captcha!", "app.register.register": "Register", "app.register.get-verification-code": "Get code", "app.register.sing-in": "Already have an account?", @@ -624,6 +755,7 @@ export default { ...alias, ...guide, ...license, + ...migration, ...credential, ...overview, ...dashboard, diff --git a/web/src/locales/en-US/agent.js b/web/src/locales/en-US/agent.js index 5be0aa25..1e38a26d 100644 --- a/web/src/locales/en-US/agent.js +++ b/web/src/locales/en-US/agent.js @@ -6,27 +6,87 @@ export default { "agent.instance.associate.labels.select_cluster": "Select Cluster", "agent.instance.associate.tips.associate": "Please select cluster(s) to enroll !", - "agent.instance.associate.set_credential": "Set credential for agent", + "agent.instance.associate.set_credential": "Set probe credential", "agent.instance.associate.set_credential.tips": - "This permission will be used for metrics and log collection. It is recommended to use a user with a reasonable permission range.", + "This credential will be used for metrics and log collection. It is recommended to use a user with a reasonable permission scope.", + "agent.instance.associate.set_logs_paths": "Set probe log collection paths", + "agent.instance.associate.set_logs_paths.tips": + "These log directories are used for the current batch enroll and saved as the default log collection paths for future auto-enrolled nodes.", "agent.instance.associate.tips.connected": "Connection succeeded!", "agent.instance.associate.tips.connected.check": "please set a credential for agent", - "agent.instance.associate.auth.error": "The following clusters need to set credentials for the agent:", + "agent.instance.associate.tips.no_match": + "No node was matched and enrolled. Check the probe credential, node HTTP port, and cluster connectivity, then try again.", + "agent.instance.associate.auth.error": "The following clusters need platform or agent credentials first:", "agent.instance.associate.tips.metric": - "After enroll, the agent will collect metrics for the enrolled cluster", + "Enroll will switch the cluster to Agent mode and auto-enroll newly added nodes afterward", "agent.instance.associate.tips.unregister": "No registration information for this cluster was found in the Console,", "agent.instance.associate.tips.to_register": "go to register", "agent.instance.associate.drawer.title": "Enroll Cluster", "agent.instance.regist": "Agent Registration", + "agent.instance.field.endpoint.placeholder": + "Agent API endpoint e.g. 127.0.0.1:2900", + "agent.instance.field.endpoint.form.required": + "Please input agent API endpoint!", + "agent.instance.registration.copy": "Copy", + "agent.instance.registration.console.title": "INFINI Console Access Credential", + "agent.instance.registration.access.endpoint": "Access Endpoint", + "agent.instance.registration.access.credential": "Access Credential", + "agent.instance.registration.console.endpoint.tip": + "Copy this access endpoint to the Agent managed configuration.", + "agent.instance.registration.console.token.tip": + "Copy and keep this access credential securely.", + "agent.instance.registration.console.load_failed": + "Failed to load INFINI Console access info.", + "agent.instance.registration.menu.register": "Register", + "agent.instance.registration.menu.info": "Credential", + "agent.instance.registration.agent.title": "Agent Access Info", + "agent.instance.registration.agent.token.required": + "Please input access credential!", + "agent.instance.registration.agent.endpoint.tip": + "Enter the Agent access endpoint that Console can reach.", + "agent.instance.registration.agent.token.placeholder": + "Paste the access credential generated on the host", + "agent.instance.registration.agent.token.tip": + "Paste the Agent access credential here so Console can access the Agent later.", + "agent.instance.registration.agent.token.expire.tip": + "This access credential has no fixed expiration and remains valid until it is rotated or replaced.", + "agent.instance.registration.auth.type": "Authentication Method", + "agent.instance.registration.auth.type.access_token": "Access Credential", + "agent.instance.registration.auth.type.basic_auth": "Username / Password", + "agent.form.placeholder.auth.username": "Please input username", + "agent.instance.step.result.button.register_new": "Register Another Agent", + "agent.instance.step.result.button.view_list": "View Agent List", + "agent.instance.column.agent_ip": "Agent IP", + "agent.instance.row_detail.tab.detected_processes": + "Detected Processes ({count})", + "agent.instance.row_detail.tab.unknown_processes": + "Unknown Processes ({count})", + "agent.instance.process.detail.title": "Processes detail", "agent.instance.associate.labels.node_adress": "Node Publish Address", + "agent.instance.associate.labels.logs_paths": "Log Paths", + "agent.instance.associate.labels.logs_paths.placeholder": + "Enter one or more log directories", + "agent.instance.associate.labels.logs_paths.tips": + "The detected path.logs is prefilled. You can append extra directories and Console will deliver them through the same logs_path setting for Agent collection.", "agent.instance.associate.tips.access_failed": "The agent failed to access this node, please update the settings and try again!", + "agent.instance.associate.credential_status.set": "Set", + "agent.instance.associate.credential_status.unset": "Not Set", "agent.install.label.get_cmd": "Get Setup Command", "agent.install.setup.title": "Quick Installation", "agent.install.setup.desc": "Please copy the command below and execute it on the target host, which includes downloading, deploying and starting INFINI Agent", + "agent.install.tips.intranet.title": "Intranet deployment", + "agent.install.tips.intranet.desc": + "The default install directory is /infini/agent. If package files exist under web.ui.path/agent/stable, Console automatically serves them from its own web endpoint; otherwise it falls back to the official release site. Configure agent.setup.download_url only when you need a custom internal mirror.", "agent.install.tips.title": "Tips", + "agent.install.tips.target": "The default install directory is", + "agent.install.tips.version": "To install a specific Agent version, append", + "agent.install.tips.download": + "To use an internal mirror or custom download source, append", + "agent.install.tips.server": + "If the target host cannot access the current Console address directly, append", "agent.install.tips.desc": "The automatic installation of the current version only supports Linux, for non-Linux systems, please select ", "agent.install.link.manual_install": "manual installation", @@ -35,15 +95,34 @@ export default { "The log viewing feature needs to install the INFINI Agent first", "agent.logs.label.log_file": "Log File", "agent.logs.label.goto": "Jump To Line", - "agent.logs.button.view_latest": "View Latest", + "agent.logs.label.latest_lines": "lines", + "agent.logs.button.view_latest": "Follow Latest", "agent.logs.button.goto": "Goto", "agent.install.setup.copy.success": "Copied to clipboard successfully!", + "agent.install.advanced.title": "Advanced configuration", + "agent.install.reverse_channel.label": "Enable reverse channel", + "agent.install.reverse_channel.help": "Disabled by default. Enable only when Console cannot reach the Agent port, such as in containers; when enabled, Agent connects back to Console.", + "agent.install.no_sudo.label": "No sudo access", + "agent.install.no_sudo.help": "Recommended for containers or non-root environments. When enabled, the generated command removes sudo and appends --no-service. After installation, run Agent in the foreground via the container ENTRYPOINT/CMD.", + "agent.install.no_sudo.help_line": "If sudo or system services are unavailable in the target environment, append", + "agent.install.no_sudo.tip.title": "Container / no-sudo mode", + "agent.install.no_sudo.tip.desc": "This mode skips service installation and startup. After installation, run the Agent binary as the container main process and let Docker or Kubernetes handle restart behavior.", + "agent.install.no_sudo.entrypoint.title": "Dockerfile ENTRYPOINT example", + "agent.install.no_sudo.cmd.title": "Dockerfile CMD example", "agent.instance.auto_associate.title": "Auto Enroll", "agent.instance.install.title": "Install Agent", "agent.label.agent_credential": "Agent Credential", "agent.credential.tip": "No credential required", + "agent.instance.button.revoke": "Revoke", + "agent.instance.delete.confirm.title": "Are you sure you want to delete this agent?", + "agent.instance.revoke.confirm.title": "Sure to revoke?", "agent.instance.clear.title": "Clear Offline Instances", "agent.instance.clear.modal.title": "Are you sure you want to clear offline instances?", - "agent.instance.clear.modal.desc": "This operation will delete offline instances that have not reported metrics for 7 days." + "agent.instance.clear.modal.desc": "This operation will delete offline instances that have not reported metrics for 7 days.", + "agent.instance.collection_interval.label": "Collection Interval", + "agent.instance.collection_interval.unit": "s", + "agent.instance.collection_interval.placeholder": "Default 10", + "agent.instance.collection_interval.tip": "How often (in seconds) the Agent collects metrics from this cluster's nodes. Leave empty or set to 0 to use the Agent default (10 s). Changes take effect on the next config sync.", + "agent.instance.collection_interval.save.success": "Collection interval updated" }; diff --git a/web/src/locales/en-US/alert.js b/web/src/locales/en-US/alert.js index 7acf59a7..adcbc7b3 100644 --- a/web/src/locales/en-US/alert.js +++ b/web/src/locales/en-US/alert.js @@ -216,12 +216,21 @@ export default { "alert.rule.table.columnns.category": "Category", "alert.rule.table.columnns.tags": "Tags", "alert.rule.table.columnns.last_notification_time": "Last notification", + "alert.import.submit.success": "Imported successfully", + "alert.import.submit.failed": "Import failed", + "alert.import.select_file": "Please select file", + "alert.import.upload.success": "File uploaded successfully", + "alert.import.upload.failed": "File upload failed", + "alert.import.upload.invalid_json": "Invalid JSON file", // /alerting/rule/edit page 编辑规则页面 "alert.rule.form.title.edit": "Edit Rules", "alert.rule.form.label.select_cluster": "Select cluster", "alert.rule.form.label.select_object": "Select objects", "alert.rule.form.label.filter_condition": "Filter query", "alert.rule.form.label.time_field": "Time field", + "alert.rule.form.label.ignore_time_filter": "Ignore time filter", + "alert.rule.form.help.ignore_time_filter": + "Enable for non-time-series rules to query without a time range.", //Configure alert objects 配置告警对象 "alert.rule.form.title.configure_alert_object": "Configure alert objects", "alert.rule.form.label.alert_metric": "Metrics", @@ -257,6 +266,10 @@ export default { "alert.rule.form.title.configure_alert_channel_recovery": "Notification recovery channels", "alert.rule.form.label.alert_channel": "Notification channels", + "alert.rule.form.label.incremental_recovery_notification": + "Incremental recovery notifications", + "alert.rule.form.help.incremental_recovery_notification": + "When enabled, grouped alerts send recovery notifications as each group recovers.", "alert.rule.form.label.accept_upgrade": "Notification escalation", "alert.rule.form.label.upgrade_notification_waiting_time": "Escalation waiting time", @@ -272,6 +285,7 @@ export default { "Template variables examples:", "alert.rule.form.title.example1": "Example1:", "alert.rule.form.title.example2": "Example2(array traversal):", + "alert.rule.form.title.example": "Example", "alert.rule.form.title.template_function": "Template functions", // alerting/rule/new page 新建规则页面 "alert.rule.form.title.create": "Create Rules", @@ -306,6 +320,16 @@ export default { "alert.message.priority.info": "P4(Info)", "alert.message.priority.ignored": "Ignored", "alert.message.priority.": "Undefined", + "alert.message.priority.undefined": "Undefined", + "alert.message.status.alerting": "Alerting", + "alert.message.status.ignored": "Ignored", + "alert.message.status.recovered": "Recovered", + "alert.message.status.ok": "OK", + "alert.message.status.error": "Error", + "alert.message.status.nodata": "No data", + "alert.message.ignored.time": "Ignored time", + "alert.message.ignored.operator": "Operator", + "alert.message.ignored.reason": "Reason", "alert.message.table.created": "Created time", "alert.message.table.priority": "Priority", @@ -330,11 +354,14 @@ export default { "alert.message.detail.ignored_time": "Ignored time", "alert.message.detail.message": "Event message", "alert.message.detail.alert_metric_status": "Metrics", + "alert.message.detail.no_history_data": "No alert history data available", "alert.message.detail.execution_record": "History", "alert.message.detail.action_message": "Alert message", "alert.message.detail.action_result": "Execution result", "alert.message.detail.action_result_error": "Exection error", "alert.message.detail.alert_info": "Alert Detail", + "alert.message.detail.query_dsl": "Query DSL", + "alert.message.detail.response": "Response", "alert.message.detail.condition.type": "Condition Type", "alert.message.detail.condition": "Condition", "alert.message.detail.bucket_diff_type": "Bucket Diff Type", @@ -356,7 +383,14 @@ export default { "alert.channel.form.advanced.custom": "(Custom)", "alert.channel.form.advanced.load.default": "Load Channel's Default Config", + "alert.channel.form.webhook.url": "Webhook URL", + "alert.channel.form.webhook.url.required": "Please input webhook URL!", + "alert.channel.form.webhook.method": "HTTP Method", + "alert.channel.form.webhook.method.required": "Please select HTTP method!", + "alert.channel.form.webhook.headers": "Headers", "alert.channel.form.webhook.template.title": "Default Body Template", + "alert.channel.form.webhook.body": "Body", + "alert.channel.form.webhook.body.required": "Please input body!", "alert.channel.form.webhook.send.test": "Send Test Message", "alert.channel.form.email.server": "SMTP Server", @@ -375,10 +409,22 @@ export default { "alert.channel.form.email.template.body": "Body", "alert.channel.form.email.template.body.required": "Please input body!", "alert.channel.form.email.send.test": "Send A Test Email", + "alert.rule.form.template.sync": "Sync template", + "alert.rule.form.template.sync.success": "The current rule template has been synced success", + "alert.rule.form.template.sync.failed": "Failed to sync rule template", + "alert.channel.enable.tip.email_incomplete": + "Configure an SMTP server and at least one recipient before enabling this email channel.", + "alert.channel.enable.tip.email_server": + "Configure an SMTP server before enabling this email channel.", + "alert.channel.enable.tip.email_recipients": + "Configure at least one recipient before enabling this email channel.", "alert.channel.empty": "No channels?", "alert.channel.export-import.label": " Channels", "alert.rule.export-import.label": " Rules", + "alert.import.example.title": "Import example", + "alert.import.example.tip.rule": + "Export one rule file first as a template, then edit and import with the same structure.", // 告警任务详情页面 "alert.task.detail.title": "Alert Task Detail", diff --git a/web/src/locales/en-US/cluster.js b/web/src/locales/en-US/cluster.js index f832e285..8e4de550 100644 --- a/web/src/locales/en-US/cluster.js +++ b/web/src/locales/en-US/cluster.js @@ -26,6 +26,7 @@ export default { "cluster.manage.table.column.node_count": "Nodes", "cluster.manage.table.column.endpoint": "Endpoint", "cluster.manage.table.column.monitored": "Monitored", + "cluster.manage.table.column.monitor_toggle": "Monitoring Status", "cluster.manage.table.column.monitor_mode": "Monitor Mode", "cluster.manage.table.column.discovery.enabled": "Discovery", "cluster.manage.table.column.operation": "Actions", @@ -34,18 +35,50 @@ export default { "cluster.manage.table.column.owner": "Owner", "cluster.manage.table.column.area": "Area", "cluster.manage.table.column.location": "Location", + "cluster.manage.field.tls.label": "Enable TLS", + "cluster.manage.field.tags.label": "Tags", "cluster.manage.monitored.on": "ON", "cluster.manage.monitored.off": "OFF", "cluster.manage.metric_collection_mode": "Collect Mode", + "cluster.manage.metric_collection_mode.option.agent": "Agent", + "cluster.manage.metric_collection_mode.option.agentless": "Agentless", "cluster.manage.metric_collection_mode.confirm.title": "Confirm switch metrics collect mode", "cluster.manage.metric_collection_mode.confirm.message": "Are you sure you want to switch to {mode} mode?", "cluster.manage.metric_collection_mode.warning.large_cluster": "The current cluster has {number_of_nodes} or more nodes. It is strongly recommended to use Agent mode for monitoring.", "cluster.manage.metric_collection_mode.confirm.button.ok": "OK", "cluster.manage.metric_collection_mode.confirm.button.cancel": "Cancel", + "cluster.manage.monitoring.enable.action": "Enable monitoring", + "cluster.manage.monitoring.disable.action": "Disable monitoring", + "cluster.manage.monitoring.confirm.enable.title": "Are you sure you want to enable monitoring for this cluster?", + "cluster.manage.monitoring.confirm.disable.title": "Are you sure you want to disable monitoring for this cluster?", + "cluster.manage.monitoring.confirm.cluster": "Cluster: {name}", + "cluster.manage.monitoring.confirm.version": "Version: {version}", + "cluster.manage.monitoring.confirm.endpoint": "Endpoint: {endpoint}", + "cluster.manage.monitoring.notice.unmonitored": "Monitoring is not enabled for this cluster.", + "cluster.manage.monitoring.notice.last_collection_time": "Last data collection time: {timestamp}", + "cluster.manage.monitoring.notice.enable_button": "Go to enable monitoring", + "cluster.manage.monitoring.notice.unavailable_since": "Cluster is not available since: {timestamp}", + "cluster.manage.delete.confirm.title": "Are you sure you want to delete this cluster?", + "cluster.manage.delete.confirm.cluster": "Cluster: {name}", + "cluster.manage.delete.confirm.version": "Version: {version}", + "cluster.manage.delete.confirm.endpoint": "Endpoint: {endpoint}", + "cluster.manage.agent_credential.tip.auto_create": "When agent collection mode is enabled, Console automatically creates the low-privilege infini-agent user for metrics and log collection. You can still override it with a custom agent credential below if needed.", + "cluster.manage.agent_credential.placeholder.auto_create": "Leave empty to auto create or fall back to the platform credential", + "cluster.manage.agent_credential.tip.agentless_skip": "Agent credentials are not needed in Agentless mode. Switch to Agent mode first if you want metrics and logs collected through agents.", + "cluster.manage.agent_logs_paths.label": "Agent log collection paths", + "cluster.manage.agent_logs_paths.placeholder": "Enter one or more log directories", + "cluster.manage.agent_logs_paths.tips": "These paths define the default Agent log collection directories for the cluster and are reused by later batch enroll and auto-enroll flows. Leave empty to keep using each node's detected path.logs.", + "cluster.manage.config_item.enabled": "Enabled", + "cluster.manage.config_item.interval": "Interval", + "cluster.manage.config_item.interval.unit": "s", "cluster.manage.monitor_configs.cluster_health": "Cluster health", "cluster.manage.monitor_configs.cluster_stats": "Cluster stats", + "cluster.manage.monitor_configs.tips.cluster_health": "Collects cluster health summary metrics (green/yellow/red).", + "cluster.manage.monitor_configs.tips.cluster_stats": "Collects cluster-level statistics from _cluster/stats (nodes, shards, storage, etc.).", "cluster.manage.monitor_configs.node_stats": "Node stats", + "cluster.manage.monitor_configs.tips.node_stats": "Collects per-node statistics from _nodes/stats (CPU, memory, disk, thread pools, etc.).", "cluster.manage.monitor_configs.index_stats": "Index stats", + "cluster.manage.monitor_configs.tips.index_stats": "Collects per-index statistics from _stats (documents, size, segments, merges, etc.).", "cluster.manage.monitor_configs.index_health": "Index health", "cluster.manage.monitor_configs.shard_stats": "Shard stats", "cluster.manage.metadata_configs.health_check": "Health check", @@ -54,6 +87,14 @@ export default { "cluster.manage.metadata_configs.metadata_refresh": "Metadata refresh", "cluster.manage.metadata_configs.cluster_settings_check": "Cluster settings check", + "cluster.manage.metadata_configs.tips.health_check": + "Periodically checks cluster health/reachability and updates cluster availability status.", + "cluster.manage.metadata_configs.tips.node_availability_check": + "Performs TCP checks against node addresses. This requires direct TCP access to nodes and cannot work when cluster is reachable only through HTTP proxy.", + "cluster.manage.metadata_configs.tips.metadata_refresh": + "Periodically refreshes metadata such as nodes and index aliases.", + "cluster.manage.metadata_configs.tips.cluster_settings_check": + "Periodically fetches and syncs cluster settings (transient/persistent).", "cluster.manage.monitor_configs.rollup_node_stats": "Node Stats", "cluster.manage.monitor_configs.rollup_index_stats": "Index Stats", "cluster.manage.monitor_configs.rollup_cluster_stats": "Cluster Stats", @@ -88,16 +129,37 @@ export default { "cluster.regist.form.verify.required.cluster_name": "Please input cluster name!", "cluster.regist.form.verify.valid.endpoint": - "Please input a domain name or IP address and port number!", + "Please input a domain name or IP address, with an optional port number!", "cluster.regist.form.verify.required.endpoint": "Please input endpoint!", + "cluster.regist.form.label.probe_path": "Agent Path", + "cluster.regist.form.placeholder.probe_path": "/_cluster/health", + "cluster.regist.form.toggle.probe_path": "Custom Agent Path", + "cluster.regist.form.help.probe_path": + "Optional. Leave empty to use the default / agent path. Only needed for special cases such as WAF restrictions.", + "cluster.regist.form.verify.valid.probe_path": + "Agent path must start with /", "cluster.regist.form.verify.required.credential": "Please select Agent credential!", "cluster.regist.form.verify.required.agent_credential": - "Please select credential!", + "Please select an Agent credential!", "cluster.regist.form.verify.required.auth_username": "Please input auth username!", "cluster.regist.form.verify.required.auth_password": "Please input auth password!", + "cluster.regist.try_connect.failed": + "Cluster connection failed. Check the endpoint, TLS, and authentication settings.", + "cluster.connect.error.health_red": + "The target cluster health is red. Console only allows connecting Easysearch clusters when their health is green. Fix the cluster before connecting again.", + "cluster.connect.error.tls_mismatch": + "The TLS setting does not match the cluster endpoint. Check whether HTTPS should be enabled.", + "cluster.connect.error.auth_required": + "Authentication is required or the credential is invalid. Check the username, password, or saved credential.", + "cluster.connect.error.endpoint_unreachable": + "Unable to connect to the cluster endpoint. Check the address, network accessibility, and TLS setting.", + "cluster.connect.error.non_es_endpoint": + "The endpoint did not return an Elasticsearch-compatible API response. Check that the address and port point to the Elasticsearch API.", + "cluster.connect.error.unexpected_status": + "The cluster returned an unexpected status. Check the address, TLS setting, and authentication settings.", "cluster.regist.form.credential.manual.desc": "*The new authentication information will be added to the credential store after saving", @@ -138,6 +200,9 @@ export default { "cluster.monitor.topn.area": "Area Metric", "cluster.monitor.topn.color": "Color Metric", "cluster.monitor.topn.theme": "Theme", + "cluster.monitor.rollup.gap": "Rollup Gap", + "cluster.monitor.treemap.search_latency_by_index": + "Avg search latency by index", "cluster.monitor.logs.timestamp": "Timestamp", "cluster.monitor.logs.type": "Type", diff --git a/web/src/locales/en-US/command.js b/web/src/locales/en-US/command.js index 5193c91f..f3a5cfcf 100644 --- a/web/src/locales/en-US/command.js +++ b/web/src/locales/en-US/command.js @@ -1,15 +1,20 @@ export default { "command.table.field.name": "Name", "command.table.field.tag": "Tag", + "command.table.field.creator": "Creator", + "command.table.field.created": "Created", "command.table.field.content": "Content", + "command.table.field.summary": "Summary", + "command.table.summary.requests": "{count} requests", "command.manage.edit.title": "Command", "command.btn.newtag": "Add New Tag", "command.message.invalid.tag": "invalid tag", + "command.message.title_exists": "Title already exists", "command.manage.save.title": "Save As Command", "command.manage.title": "COMMANDS", "command.manage.description": "Commonly used commands can help you save frequently used requests and load them quickly through the LOAD command in the development tool.", - "console.menu.copy_as_curl": "Copy As Curl", + "console.menu.copy_as_curl": "Copy As cURL Command", "console.menu.auto_indent": "Auto Indent", "console.menu.save_as_command": "Save As Command", }; diff --git a/web/src/locales/en-US/credential.js b/web/src/locales/en-US/credential.js index 308ea42c..08421811 100644 --- a/web/src/locales/en-US/credential.js +++ b/web/src/locales/en-US/credential.js @@ -15,8 +15,19 @@ export default { "credential.manage.drawer.edit.title": "Credential Detail", "credential.manage.form.type": "Type", + "credential.manage.form.type.required": "Please select type!", "credential.manage.form.name": "Name", + "credential.manage.form.name.required": "Please input name!", "credential.manage.form.username": "Username", + "credential.manage.form.username.required": "Please input username!", "credential.manage.form.password": "Password", + "credential.manage.form.password.required": "Please input password!", + "credential.manage.form.password.placeholder.edit": + "Original password is not displayed", + "credential.manage.form.token": "Token", + "credential.manage.form.token.required": "Please input token!", + "credential.manage.form.token.placeholder": "Please input token!", + "credential.manage.form.token.placeholder.edit": + "Original token is not displayed", "credential.manage.form.tags": "Tags", }; diff --git a/web/src/locales/en-US/error.js b/web/src/locales/en-US/error.js index 88a2cc96..41aa9fed 100644 --- a/web/src/locales/en-US/error.js +++ b/web/src/locales/en-US/error.js @@ -2,4 +2,7 @@ export default { "error.split": ", ", "error.unknown": "unknown error, please try again later or contact the support team!", "error.request_timeout_error": "request timeout, please try again later!", + "error.request.connection_refused": "Failed to connect to the server.", + "error.request.connection_refused.tip": + "Please click the services health above to ensure the system cluster is running properly.", }; diff --git a/web/src/locales/en-US/explore.js b/web/src/locales/en-US/explore.js index 06e1b829..ec588a97 100644 --- a/web/src/locales/en-US/explore.js +++ b/web/src/locales/en-US/explore.js @@ -53,4 +53,66 @@ export default { "explore.indexfield.description": "The current page lists all fields that match the {pattern} index, and the field type is an Elasticsearch data type. To change the type, use the ", "explore.view.index_pattern.removeTooltip": "Delete view", + "explore.view.index_pattern.refreshTooltip": "Refresh field list", + "explore.view.index_pattern.refreshFieldListTitle": "Refresh field list?", + "explore.view.index_pattern.back_to_list": "Back to view list", + "explore.view.index_pattern.detail_title": "View details", + "explore.view.index_pattern.delete_confirm": "Delete view?", + "explore.view.index_pattern.time_field": "Time field: '{field}'", + "explore.view.index_pattern.mapping_conflict_title": "Mapping conflict", + "explore.view.index_pattern.mapping_conflict_desc": + "The indices matched by this view contain {count} field conflicts across multiple types, such as string and integer. You can still inspect conflicted fields, but they cannot be used in functions until the index mappings are reconciled.", + "explore.view.index_pattern.tab.fields": "Fields", + "explore.view.index_pattern.tab.scripted_fields": "Scripted fields", + "explore.view.index_pattern.tab.source_filters": "Source filters", + "explore.view.index_pattern.tab.complex_fields": "Complex fields ({count})", + "explore.view.index_pattern.search_fields": "Search fields", + "explore.view.index_pattern.filter_field_types": "Filter field types", + "explore.view.index_pattern.search_placeholder": "Search", + "explore.view.index_pattern.create_field": "Create field", + "explore.view.index_pattern.all_field_types": "All field types", + "explore.view.index_pattern.all_languages": "All languages", + "explore.view.index_pattern.field_editor.default_option": "- Default -", + "explore.view.index_pattern.field_editor.default_label": "Default", + "explore.view.index_pattern.field_editor.format": "Format", + "explore.view.index_pattern.field_editor.format_help": + "Formatting allows you to control the way that specific values are displayed. It can also cause values to be completely changed and prevent highlighting in Discover from working.", + "explore.view.index_pattern.field_editor.save_field": "Save field", + "explore.view.index_pattern.field_editor.statistics": "Statistics", + "explore.view.index_pattern.field_editor.field": "Field", + "explore.view.index_pattern.field_editor.group_field": "Group Field", + "explore.view.index_pattern.field_editor.dividend_field": "Dividend Field", + "explore.view.index_pattern.field_editor.divisor_field": "Divisor Field", + "explore.view.index_pattern.field_editor.add_new": "Add New", + "explore.view.index_pattern.complex_field_editor.name": "Name", + "explore.view.index_pattern.complex_field_editor.name_required": + "Name is required", + "explore.view.index_pattern.complex_field_editor.new_field_placeholder": + "New field", + "explore.view.index_pattern.complex_field_editor.duplicate_name": + "You already have a field with the name {name}.", + "explore.view.index_pattern.complex_field_editor.metric_name": + "Metric Name", + "explore.view.index_pattern.complex_field_editor.function": "Function", + "explore.view.index_pattern.complex_field_editor.unit": "Unit", + "explore.view.index_pattern.complex_field_editor.tags": "Tags", + "explore.view.index_pattern.complex_field_editor.delete_title": + "Delete field '{name}'", + "explore.view.index_pattern.complex_field_editor.delete_confirm": + "You can't recover a deleted field. Are you sure you want to do this?", + "explore.view.index_pattern.complex_field_editor.edit_title": + "Edit {name}", + "explore.save_queries.title": "Save Queries", + "explore.save_queries.field.title": "Title", + "explore.save_queries.field.tag": "Tag", + "explore.save_queries.field.description": "Description", + "explore.save_queries.button.cancel": "Cancel", + "explore.save_queries.button.save": "Save", + "explore.save_queries.button.update": "Update", + "explore.save_queries.validation.title_required": "Please input title!", + "explore.save_queries.validation.title_exists": "Changed title already exists!", + "explore.load_queries.title": "Load Queries", + "explore.load_queries.search.title": "Please input queries title", + "explore.load_queries.search.tag": "Please select a tag", + "explore.load_queries.updated_at": "Updated at", }; diff --git a/web/src/locales/en-US/gateway.js b/web/src/locales/en-US/gateway.js index c4b589ac..38fdbcf0 100644 --- a/web/src/locales/en-US/gateway.js +++ b/web/src/locales/en-US/gateway.js @@ -4,6 +4,24 @@ export default { "gateway.instance.new.description": "Input the instance endpoint to register the INFINI Gateway step by step.", "gateway.instance.btn.new": "New", + "gateway.instance.install.title": "Install Gateway", + "gateway.install.label.get_cmd": "Get Setup Command", + "gateway.install.advanced.title": "Advanced", + "gateway.install.type.label": "Service type", + "gateway.install.type.migration": "Migration gateway", + "gateway.install.type.relay": "Relay gateway", + "gateway.install.relay_role.label": "Relay role", + "gateway.install.relay_role.primary": "Primary gateway", + "gateway.install.relay_role.secondary": "Secondary gateway", + "gateway.install.no_sudo.label": "No sudo privileges", + "gateway.install.no_sudo.help": + "Recommended for containers or non-root environments. When enabled, the generated command removes sudo and appends --no-service. After installation, run Gateway in the foreground via your container or process manager.", + "gateway.install.no_sudo.help_line": + "If sudo or system services are unavailable in the current environment, append", + "gateway.install.no_sudo.tip.title": "Container / no sudo mode", + "gateway.install.no_sudo.tip.desc": + "This mode does not install or start a system service. After installation, run Gateway in the foreground and let Docker, Kubernetes, or another process manager handle startup and restarts.", + "gateway.install.no_sudo.command.title": "Foreground startup example", "gateway.entry.index.title": "ENTRY", "gateway.entry.index.description": "Entry management helps you to add, view, modify and delete the entry configuration of the INFINI Gateway conveniently and quickly.", @@ -18,10 +36,111 @@ export default { "Modify the instance configuration, and then click the Save button, and it will take effect after the save is successful.", "gateway.instance.field.name.label": "Instance Name", "gateway.instance.field.name.form.required": "Please input instance name!", + "gateway.instance.field.endpoint.label": "Endpoint", "gateway.instance.field.endpoint.form.required": - "Plsease input instance api endpoint!", + "Please input instance API endpoint!", + "gateway.instance.field.endpoint.placeholder": + "Instance API endpoint e.g. 127.0.0.1:2900", + "gateway.instance.field.tls.label": "Enable TLS", "gateway.instance.field.tags.label": "Tags", "gateway.instance.field.description.placeholder": "Instance description", + "gateway.instance.delete.confirm.title": "Are you sure you want to delete this item?", + "gateway.instance.column.application": "Application", + "gateway.instance.column.name": "Name", + "gateway.instance.column.endpoint": "Endpoint", + "gateway.instance.column.status": "Status", + "gateway.instance.column.cpu": "CPU", + "gateway.instance.column.memory": "Memory", + "gateway.instance.column.storage": "Storage", + "gateway.instance.column.uptime": "Uptime", + "gateway.instance.column.tags": "Tags", + "gateway.instance.status.checking": "Checking", + "gateway.instance.status.online": "Online", + "gateway.instance.status.unavailable": "N/A", + "gateway.instance.storage.tooltip": "Free/Total: {free}/{total}", + "gateway.instance.menu.queue": "Queue", + "gateway.instance.menu.task": "Task", + "gateway.instance.menu.logging": "Logging", + "gateway.instance.menu.config": "Config", + "gateway.instance.config.files": "Configs", + "gateway.instance.config.runtime": "Runtime", + "gateway.instance.config.main": "Main", + "gateway.instance.config.location": "Location", + "gateway.instance.config.save.confirm": "Are you sure to save?", + "gateway.queue.tab.fifo": "FIFO", + "gateway.queue.tab.spmc": "SPMC", + "gateway.queue.field.local_storage": "Local Storage", + "gateway.queue.field.depth": "Depth", + "gateway.queue.field.offset": "Offset", + "gateway.queue.field.produce_offset": "Produce Offset", + "gateway.queue.field.consume_offset_earliest": "Consume Offset (Earliest)", + "gateway.queue.field.synchronization_latest_segment": "Synchronization (latest_segment)", + "gateway.queue.field.total_messages": "Total Messages", + "gateway.queue.delete.success": "Deleted {count} queues successfully", + "gateway.queue.delete.error": "Delete queues failed.{detail}", + "gateway.queue.delete_failed": "Delete failed", + "gateway.queue.message.partial_success": "Success: {count}", + "gateway.queue.batch.delete_queues": "Delete Queues", + "gateway.queue.batch.delete_consumers": "Delete Consumers", + "gateway.queue.consumer.title": "Consumers", + "gateway.queue.consumer.field.group": "Group", + "gateway.queue.consumer.field.last_active": "Last Active", + "gateway.queue.consumer.field.source": "Source", + "gateway.queue.consumer.action.reset_offset": "Reset Offset", + "gateway.queue.consumer.delete.success": "Deleted {count} consumers successfully", + "gateway.queue.consumer.delete.error": "Delete consumers failed.{detail}", + "gateway.queue.consumer.reset_offset.title": "Consumer Reset Offset", + "gateway.queue.consumer.reset_offset.new_offset": "New Offset", + "gateway.queue.consumer.reset_offset.offset_required": "Offset is required!", + "gateway.queue.consumer.reset_offset.success": "Reset offset succeeded", + "gateway.queue.message.title": "Message (ID: {id})", + "gateway.queue.message.goto_offset": "Go to offset:", + "gateway.queue.message.goto": "Goto", + "gateway.queue.message.field.message": "Message", + "gateway.queue.message.field.size": "Size", + "gateway.queue.message.load_more": "Load More", + "gateway.instance.logging.tab.realtime": "Realtime Logging", + "gateway.instance.logging.auto_scroll": "Auto Scroll", + "gateway.instance.logging.button.start": "Start", + "gateway.instance.logging.button.stop": "Stop", + "gateway.instance.logging.empty": "Click Start to display realtime logs", + "gateway.instance.logging.copy": "Copy logs", + "gateway.instance.logging.copy.success": "Copied logs to clipboard", + "gateway.instance.logging.endpoint.label": "Endpoint", + "gateway.instance.logging.endpoint.empty": "No instance endpoint is available", + "gateway.instance.logging.placeholder.file_pattern": + "File pattern, e.g. xyz*.go", + "gateway.instance.logging.placeholder.func_pattern": + "Function pattern, e.g. submit*", + "gateway.instance.logging.placeholder.message_pattern": + "Message pattern, e.g. *timeout", + "gateway.instance.logging.connection.connecting": "Connecting", + "gateway.instance.logging.connection.established": "Connected", + "gateway.instance.logging.connection.closing": "Closing", + "gateway.instance.logging.connection.closed": "Disconnected", + "gateway.instance.logging.connection.uninstantiated": "Initializing", + "gateway.task.empty": "No tasks found", + "gateway.task.column.name": "Name", + "gateway.task.column.state": "State", + "gateway.task.column.start_time": "Start Time", + "gateway.task.column.end_time": "End Time", + "gateway.task.confirm.start": "Are you sure you want to start this task?", + "gateway.task.confirm.stop": "Are you sure you want to stop this task?", + "gateway.task.state.starting": "Starting", + "gateway.task.state.started": "Started", + "gateway.task.state.cancelled": "Cancelled", + "gateway.task.state.stopping": "Stopping", + "gateway.task.state.stopped": "Stopped", + "gateway.task.state.failed": "Failed", + "gateway.task.state.finished": "Finished", + "gateway.router.column.name": "Name", + "gateway.router.column.default_flow": "Default Flow", + "gateway.router.column.tracing_flow": "Tracing Flow", + "gateway.router.column.updated": "Last Updated", + "gateway.router.search.placeholder": "Type keyword to search", + "gateway.router.delete.confirm.title": "Are you sure you want to delete this item?", + "gateway.router.btn.new": "New", + "gateway.router.pagination.total": "{start}-{end} of {total} items", "gateway.instance.regist": "Instance Regist", @@ -44,8 +163,17 @@ export default { "gateway.guide.quick_install": "Quick Install", "gateway.guide.quick_install.desc": "Please copy the following command and execute it in the local deployment environment, which includes the download, deployment, and startup of INFINI Gateway:", + "gateway.guide.intranet.title": "Intranet deployment", + "gateway.guide.intranet.desc": + "The default install directory is /infini/gateway. If package files exist under web.ui.path/gateway/stable, Console automatically serves them from its own web endpoint; otherwise it falls back to the official release site. Configure gateway.setup.download_url only when you need a custom internal mirror.", "gateway.guide.shell.copy.success": "Copy succeed!", "gateway.guide.tips.title": "Tips:", + "gateway.guide.tips.version": + "To specify a Gateway version, append", + "gateway.guide.tips.directory": + "To specify an installation directory, append", + "gateway.guide.tips.download_source": + "To use an internal or custom download source, append", "gateway.guide.tips.content": "The current version of automatic installation only supports Linux, For others, please ", "gateway.guide.tips.install_manually": "install manually", diff --git a/web/src/locales/en-US/guide.js b/web/src/locales/en-US/guide.js index 5ccceed9..2aea24df 100644 --- a/web/src/locales/en-US/guide.js +++ b/web/src/locales/en-US/guide.js @@ -2,10 +2,40 @@ export default { "guide.header.title": "Configuration Guide", "guide.initialization.step.configuration": "Configuration", "guide.initialization.step.configuration.desc": - "Connecting to system cluster (elasticsearch required version 5.3 or above).", + "Connecting to system cluster (Easysearch required version 2.3 or above).", "guide.initialization.step.initialization": "Initialization", "guide.initialization.step.initialization.desc": "Initializing basic settings for system indices and templates.", + "guide.initialization.start": "Start Initialization", + "guide.initialization.defaults.message": + "{dataNodes} data nodes detected ({totalNodes} total nodes). Primary shards default to {primaryShards} based on node count.", + "guide.initialization.primary_shards": "Primary Shards", + "guide.initialization.primary_shards.help": + "Defaults to the detected data-node count. In most cases it should match the number of nodes storing the data.", + "guide.initialization.primary_shards.invalid": + "Please enter a primary shard count greater than 0.", + "guide.initialization.auto_expand_replicas": "Auto Expand Replicas", + "guide.initialization.auto_expand_replicas.help": + "Defaults to 0-1. You can enter false, all, or a range such as 0-1.", + "guide.initialization.auto_expand_replicas.invalid": + "Invalid auto expand replicas value. Use false, all, or a range such as 0-1.", + "guide.initialization.rollup": "Initialize Rollup Templates", + "guide.initialization.rollup.help": + "Available only on Easysearch 1.12.1 and above. Turn it off to skip rollup template and job initialization.", + "guide.initialization.task.template_ilm": "Initialize templates and ILM", + "guide.initialization.task.rollup": "Initialize rollup templates", + "guide.initialization.task.insight": "Initialize dashboards and visualizations", + "guide.initialization.task.alerting": "Initialize built-in alerting rules and channels", + "guide.initialization.task.agent": "Initialize agent setup templates", + "guide.initialization.task.view": "Initialize data view templates", + "guide.initialization.task.start": "Starting: {task}", + "guide.initialization.task.success": "Completed: {task}", + "guide.initialization.task.failed": "Failed: {task} {reason}", + "guide.initialization.task.status.pending": "Pending", + "guide.initialization.task.status.running": "Running", + "guide.initialization.task.status.success": "Completed", + "guide.initialization.task.status.failed": "Failed", + "guide.initialization.task.status.skipped": "Skipped", "guide.initialization.step.settings": "Settings", "guide.initialization.step.settings.desc": "Set the default user for login and credential secret.", @@ -13,23 +43,31 @@ export default { "guide.initialization.step.finish.desc": `Configuration completed, Start the journey of ${APP_DOMAIN} Console.`, "guide.cluster.host": "Host", "guide.cluster.host.required": "Please input host!", - "guide.cluster.host.validate": "Please input IP address and port number!", + "guide.cluster.host.validate": + "Please input a domain name or IP address, with an optional port number!", "guide.cluster.auth": "Auth", "guide.cluster.test.connection": "Test Connection", "guide.cluster.test.connection.error.version": - "Elasticsearch required version 5.3 or above.", + "Easysearch required version 2.3 or above.", "guide.cluster.test.connection.failed": "Cluster connection failed.", + "guide.cluster.test.connection.localhost": + "The cluster address cannot use localhost. Please enter a remote address that the gateway can access.", "guide.cluster.validate.elasticsearch_version_too_old": - "Elasticsearch version is too old.", + "Easysearch version is too old.", "guide.cluster.validate.elasticsearch_indices_exists": "Some related indices are already exists in the target cluster.", "guide.cluster.validate.elasticsearch_template_exists": "Some related templates are already exists in the target cluster.", "guide.cluster.validate.default": - "Some related data are already exists in the target cluster.", + "Existing data was detected in the target cluster.", + "guide.cluster.validate.localhost_address": + "Localhost addresses cannot be used to register a cluster.", + "guide.cluster.validate.localhost.sub": + "Change the cluster address to a remote address that the gateway can access, then try again.", "guide.cluster.validate.sub": - "Perform the following requests in other terminal tools can delete the existing data, but you may lost data.", - "guide.cluster.validate.sub.strong": "[DO IT AT YOUR OWN RISK!]", + "If you want to continue, run the following requests in another terminal tool to remove the existing data, but data may be lost.", + "guide.cluster.validate.sub.strong": "[Please proceed with caution!]", + "guide.cluster.validate.refresh": "Recheck", "guide.cluster.skip": "Skip", "guide.cluster.skip.desc": "You can also skip this step and reuse the existing data.", @@ -47,11 +85,11 @@ export default { "guide.confirm.password.validate": "The two passwords that you entered do not match!", "guide.password.strength.invalid": "Password does not meet all security requirements.", "guide.password.rules.title": "Password must contain", - "guide.password.rule.length": "At least 8 characters long", + "guide.password.rule.length": "At least 10 characters long", "guide.password.rule.uppercase": "At least one uppercase letter (A-Z)", "guide.password.rule.lowercase": "At least one lowercase letter (a-z)", "guide.password.rule.digit": "At least one number (0-9)", - "guide.password.rule.special": "At least one special character", + "guide.password.rule.special": "At least one special character (!@#%^&*_+-=?)", "guide.credential_secret": "Secret Key", "guide.credential_secret.required": "Please input credential secret key!", "guide.credential_secret.tips": @@ -70,9 +108,22 @@ export default { "guide.completed": "Initialization completed!", "guide.enter.console": `Enter ${APP_DOMAIN} Console`, + "guide.initialization.finish.pending": "Finishing initialization...", + "guide.initialization.finish.pending.desc": "You are on the final step now. You can download the configuration first and enter Console after initialization completes.", + "guide.initialization.finish.pending.button": "Initializing", + "guide.initialization.finish.failed": "Initialization not completed", + "guide.initialization.finish.failed.desc": "Review the error details below, adjust the settings if needed, and try again.", + "guide.initialization.finish.error.invalid_bootstrap_password": + "The administrator password is invalid. Review the password requirements and try again.", + "guide.initialization.finish.error.bootstrap_password_strength": + "The administrator password does not meet the security requirements. Update it and try again.", + "guide.initialization.finish.error.bootstrap_password_required": + "Administrator password is required when resetting the administrator account.", + "guide.initialization.finish.error.bootstrap_username_required": + "Administrator username is required when resetting the administrator account.", "health.modal.title": "Services are limited", - "health.modal.desc": `Please check the status of the ${APP_DOMAIN} Console related services to ensure that the ${APP_DOMAIN} Console can work correctly.`, + "health.modal.desc": `Please click the services health above to ensure the system cluster is running properly.`, "health.modal.services.title": "Services Health", "guide.startup.modal.title": `Welcome to ${APP_DOMAIN} Console`, diff --git a/web/src/locales/en-US/indices.js b/web/src/locales/en-US/indices.js index 400bf91f..baa815ac 100644 --- a/web/src/locales/en-US/indices.js +++ b/web/src/locales/en-US/indices.js @@ -16,7 +16,25 @@ export default { "indices.show_unavailable_index": "Show unavailable", "indices.show_unavailable_node": "Show unavailable", "indices.button.filters": "Filters", - "indices.field.name.placeholder": "Please input index name", - "indices.field.name.required_message": - "Please input a name of at least five characters!", + "indices.tab.mappings": "Mappings", + "indices.tab.edit_settings": "Edit settings", + "indices.hint.edit_json": "Edit, then save your JSON", +"indices.field.name.placeholder": "Please input index name", +"indices.field.name.required_message": + "Please input a name of at least five characters!", +"indices.field.name.lowercase_message": + "Index name must be lowercase and can only contain letters, numbers, dots, hyphens, and underscores", +"indices.delete.modal.title.single": "Delete index", +"indices.delete.modal.title.batch": "Delete {count} indices", +"indices.delete.modal.cluster": + "You are about to delete these indices in cluster {cluster}:", +"indices.delete.modal.special_index": "Special index", +"indices.delete.modal.special_warning.title": + "Deleting a special index can break Console!", +"indices.delete.modal.special_warning.description": + "Special indices are critical for internal operations. If you delete a special index, you can't recover it. Make sure you have appropriate backups.", +"indices.delete.modal.special_warning.confirm": + "I understand the consequences of deleting a special index", +"indices.delete.modal.description": + "You can't recover a deleted index. Make sure you have appropriate backups.", }; diff --git a/web/src/locales/en-US/listview.js b/web/src/locales/en-US/listview.js index c4ef5e79..2acce2b7 100644 --- a/web/src/locales/en-US/listview.js +++ b/web/src/locales/en-US/listview.js @@ -1,4 +1,5 @@ export default { + "listview.filters.placeholder": "Filters", "listview.search.placeholder": "Search ...", "listview.search.response.tip": "Found {total} records ({took} millisecond)", "listview.sort": "Sort", diff --git a/web/src/locales/en-US/overview.js b/web/src/locales/en-US/overview.js index ba0e3786..140406fd 100644 --- a/web/src/locales/en-US/overview.js +++ b/web/src/locales/en-US/overview.js @@ -7,15 +7,78 @@ export default { "overview.title.message": "Notification", "overview.title.cluster": "Clusters", "overview.title.node": "Nodes", + "overview.title.index": "Indices", "overview.title.host": "Hosts", "overview.title.disk": "Disk Usage", "overview.title.quick": "Quick Access", "overview.title.product_activities": "Product News", "overview.title.cluster_activities": "Platform Activities", + "overview.info.cluster": "Cluster Info", + "overview.info.node": "Node Info", + "overview.info.index": "Index Info", + "overview.detail.metrics": "Metrics", + "overview.detail.infos": "Infos", + "overview.column.name": "Name", + "overview.column.version": "Version", + "overview.column.health": "Health", + "overview.column.status": "Status", + "overview.column.nodes": "Nodes", + "overview.column.indices": "Indices", + "overview.column.shards": "Shards", + "overview.column.docs": "Docs", + "overview.column.store": "Store", + "overview.column.store_size": "Store Size", + "overview.column.replicas": "Replicas", + "overview.column.document_count": "Document Count", + "overview.column.data": "Data", + "overview.column.disk_usage": "Disk Usage", + "overview.column.jvm_heap": "JVM Heap", + "overview.column.transport_address": "Transport Address", + "overview.column.host_name": "Host Name", + "overview.column.agent_status": "Agent Status", + "overview.column.cpu_usage": "CPU Usage", + "overview.column.load_average": "Load Average", + "overview.column.disk_free_space": "Disk Free Space", + "overview.column.disk_used_space": "Disk Used Space", + "overview.column.uptime": "Uptime", + "overview.column.timestamp": "Timestamp", + "overview.column.index": "Index", + "overview.column.shard": "Shard", + "overview.column.prirep": "Prirep", + "overview.column.state": "State", + "overview.column.ip": "IP", + "overview.column.node": "Node", + "overview.column.cluster": "Cluster", + "overview.column.pid": "PID", + "overview.column.homepath": "Home Path", + "overview.column.endpoint": "Endpoint", + "overview.column.search_rate": "Search Rate", + "overview.column.indexing_rate": "Indexing Rate", + "overview.column.indexing_bytes": "Indexing Bytes", + "overview.column.primary_indexing_rate": "Pri Indexing Rate", + "overview.column.primary_indexing_bytes": "Pri Indexing Bytes", + "overview.statistic.primary": "Primary", + "overview.statistic.total_shards": "Total shards", + "overview.statistic.updated": "Updated", + "overview.statistic.type": "Type", + "overview.statistic.master_node": "Master Node", + "overview.statistic.not_master_node": "Not Master Node", + "overview.status.unavailable": "not available", + "overview.status.available": "available", + "overview.status.unknown": "unknown", + "overview.status.closed": "closed", + "overview.status.deleted": "deleted", + "overview.status.index_since": "Index is {status} since: {timestamp}", + "overview.status.node_since": "Node is not available since: {timestamp}", "overview.message.alert": "Alert", "overview.message.notice": "Notice", "overview.message.todo": "Todo", + "platform.notification.table.title": "Title", + "platform.notification.table.created": "Created", + "platform.notification.table.status": "Status", + "platform.notification.status.new": "New", + "platform.notification.status.read": "Read", "overview.quick.alert": "Setup Alerting Rules", "overview.quick.dev_tools": "Access Dev Tools", @@ -23,4 +86,6 @@ export default { "overview.quick.security": "Configure Security", "overview.quick.discover": "Discover Your Data", "overview.quick.monitor": "Cluster Metrics Ops", + "overview.quick.migration": "Data Migration", + "overview.quick.comparison": "Data Comparison", }; diff --git a/web/src/locales/en-US/settings.js b/web/src/locales/en-US/settings.js index e8486fe2..7cdfcad0 100644 --- a/web/src/locales/en-US/settings.js +++ b/web/src/locales/en-US/settings.js @@ -1,6 +1,69 @@ export default { "settings.email.server.empty.label1": "You can add email servers here", "settings.email.server.empty.label2": - "The alart center can send a notification to the recipient through the designated mail server", + "The alert center can send notifications to recipients through the designated mail server.", "settings.email.server.empty.button.new": "Add email server", + "settings.email.server.form.name": "Name", + "settings.email.server.form.host": "Host", + "settings.email.server.form.port": "Port", + "settings.email.server.form.tls_min_version": "TLS Min Version", + "settings.email.server.form.tls": "TLS", + "settings.email.server.form.sender": "Sender", + "settings.email.server.form.enabled": "Enabled", + "settings.email.server.form.recipient": "Recipient", + "settings.email.server.form.recipient.placeholder": "Please input recipient", + "settings.email.server.form.test.button": "Send a Test Email", + "settings.email.server.form.validation.name": "Please input name!", + "settings.email.server.form.validation.host": + "Please input SMTP server host!", + "settings.email.server.form.validation.port": + "Please input SMTP server port!", + "settings.email.server.form.validation.recipient": + "Recipient email is invalid", + "settings.email.server.form.temp_name": "New Config Name", + "settings.email.server.message.test.success": "Sent successfully", + "settings.email.server.message.test.error.auth_required": + "SMTP credentials are required. Please check the username and password.", + "settings.email.server.message.test.error.smtp_auth_failed": + "SMTP authentication failed. Check the username, password, or provider-specific authorization code.", + "settings.email.server.message.test.error.sender_mismatch": + "SMTP authentication failed. Some providers require the sender address to match the authenticated account or an approved alias.", + "settings.email.server.message.test.error.tls_required": + "The SMTP server requires TLS or STARTTLS before authentication. Check the TLS setting and port.", + "settings.email.server.message.test.error.send_failed": + "Failed to send the test email. Check the sender, recipient, SMTP settings, and provider restrictions.", + "settings.system.tab.general": "General", + "settings.system.tab.email": "Email Server", + "settings.system.retention.title": "Data Retention", + "settings.system.retention.description": + "Update how many days system-managed data is retained before ILM deletes it.", + "settings.system.retention.help": + "The default retention is 30 days and the default rollover size is 50 GB. Saving this setting updates the system ILM retention policy for managed system indices.", + "settings.system.retention.unit": "days", + "settings.system.retention.size.label": "Rollover size", + "settings.system.retention.size.unit": "GB", + "settings.system.retention.save": "Save", + "settings.system.retention.update.success": + "Data retention updated successfully", + "settings.system.retention.validation.days": + "Please enter a valid retention days value", + "settings.system.retention.validation.max_size": + "Please enter a valid rollover size in GB, such as 50", + "settings.system.rollup.title": "Rollup", + "settings.system.rollup.description": + "Enable or stop the system cluster rollup jobs from Console system settings.", + "settings.system.rollup.enabled": "On", + "settings.system.rollup.disabled": "Off", + "settings.system.rollup.help": + "Turning Rollup off will stop rollup jobs and disable rollup search in cluster settings.", + "settings.system.rollup.update.success": "Rollup setting updated successfully", + "settings.system.advanced.title": "Advanced settings", + "settings.system.local_templates.title": "Local template refresh", + "settings.system.local_templates.description": + "Refresh the built-in local configuration templates in the system cluster after a binary upgrade.", + "settings.system.local_templates.refresh": "Refresh templates", + "settings.system.local_templates.help": + "This overwrites system_ingest_config.yml, task_config.tpl, relay.yml, and migration.yml in the system cluster and triggers instances to sync them again.", + "settings.system.local_templates.update.success": + "Local templates updated successfully", }; diff --git a/web/src/locales/zh-CN.js b/web/src/locales/zh-CN.js index 80441dfb..eec8ec7f 100644 --- a/web/src/locales/zh-CN.js +++ b/web/src/locales/zh-CN.js @@ -18,6 +18,11 @@ import listview from "./zh-CN/listview"; import audit from "./zh-CN/audit"; import error from "./zh-CN/error"; +let migration = {}; +try { + migration = require("./zh-CN/migration").default ?? {}; +} catch (e) {} + export default { "navBar.lang": "语言", "layout.user.appslogon": "专业的开源搜索与实时数据分析企业级管控平台", @@ -92,7 +97,7 @@ export default { "form.button.add": "添加", "form.button.edit": "编辑", "form.button.update": "更新", - "form.button.clone": "克隆", + "form.button.clone": "复制", "form.button.save": "保存", "form.button.submit": "提交", "form.button.delete": "删除", @@ -134,8 +139,85 @@ export default { "form.button.clean.deleted.indices.desc": "确定清除已删除的索引吗？", "component.refreshGroup.label.title": "自动刷新", "component.refreshGroup.label.every": "每隔", + "data_tools.task.keyword": "按关键词搜索", + "table.field.id": "编号", "table.field.actions": "操作", + "table.field.listen_address": "监听地址", + "table.field.cmdline": "命令行", + "table.field.name": "名称", + "table.field.pid": "进程ID", + "system.security.tab.user": "用户", + "system.security.tab.role": "角色", + "system.security.tab.token": "访问凭据", + "system.security.token.table.token": "访问凭据", + "system.security.token.search.placeholder": "输入关键字搜索", + "system.security.token.table.name": "名称", + "system.security.token.table.description": "描述", + "system.security.token.table.permissions": "权限", + "system.security.token.table.expire": "过期时间", + "system.security.token.never_expire": "永不过期", + "system.security.token.form.name": "名称", + "system.security.token.form.name.required": "请输入 Token 名称！", + "system.security.token.form.description": "描述", + "system.security.token.form.expire": "有效期至", + "system.security.token.form.expire.required": "请选择有效期！", + "system.security.token.form.expire.future": "有效期必须晚于当前时间！", + "system.security.token.form.never_expire": "永不过期", + "system.security.token.drawer.create.title": "创建访问凭据", + "system.security.token.drawer.edit.title": "编辑访问凭据", + "system.security.token.create.result.title": "访问凭据已创建", + "system.security.token.create.result.tip": + "请立即复制并妥善保存该 Token。该值只会展示一次。", + "system.security.token.copy.tooltip": "复制", + "system.security.token.copy.success": "访问凭据已复制到剪贴板", + "system.security.token.copy.failed": "访问凭据复制失败", + "system.security.search.placeholder": "输入关键字搜索", + "system.security.pagination.total": "{start}-{end} / 共 {total} 条", + "system.security.confirm.delete": "确认删除？", + "system.role.platform.name.required": "请输入名称！", + "system.role.platform.feature_privilege.label": "功能权限", + "system.role.platform.feature_privilege.required": "请选择平台功能权限！", + "system.role.data.cluster.label": "集群", + "system.role.data.cluster.required": "请选择集群！", + "system.role.data.cluster_privilege.label": "集群权限", + "system.role.data.cluster_privilege.required": "请选择集群权限！", + "system.role.data.index_privilege.label": "索引权限", + "system.role.data.index_privilege.required": "请选择索引权限！", + "system.role.data.api_privilege.category": "分类", + "system.role.data.api_privilege.privilege": "权限", + "system.role.data.index_privilege.index": "索引", + "system.role.data.index_privilege.privilege": "权限", + "system.security.user.table.name": "名称", + "system.security.user.table.nickname": "昵称", + "system.security.user.table.roles": "角色", + "system.security.user.table.phone": "电话", + "system.security.user.table.email": "邮箱", + "system.security.user.table.tags": "标签", + "system.security.user.table.status": "状态", + "system.security.user.action.reset_password": "重置密码", + "system.security.user.form.name": "用户名", + "system.security.user.form.nickname": "昵称", + "system.security.user.form.phone": "电话", + "system.security.user.form.email": "邮箱", + "system.security.user.form.roles": "角色", + "system.security.user.form.tags": "标签", + "system.security.user.form.name.required": "请输入名称！", + "system.security.user.form.email.invalid": "邮箱格式不正确！", + "system.security.user.form.roles.required": "请选择角色！", + "system.security.user.create.success": "用户创建成功！", + "system.security.user.create.password": "密码：{password}", + "system.security.user.create.copy_password": "复制密码", + "system.security.role.table.name": "名称", + "system.security.role.table.type": "类型", + "system.security.role.table.builtin": "内置", + "system.security.role.table.description": "描述", + "system.security.role.menu.add_platform": "添加平台角色", + "system.security.role.menu.add_data": "添加数据角色", + "system.security.role.create.success": "角色创建成功！", + "system.security.role.create.button.view_list": "返回角色列表", + "system.security.role.create.button.continue": "继续创建", + "system.security.role.delete.error.assigned_to_users": "无法删除角色：该角色仍被用户使用。", "component.globalHeader.search": "站内搜索", "component.globalHeader.search.example1": "搜索提示一", @@ -159,6 +241,29 @@ export default { "menu.insight": "数据分析", "menu.insight.dashboard": "数据看板", "menu.insight.discover": "数据探索", + "insight.config.title": "数据探索配置", + "insight.config.tab.search": "搜索", + "insight.config.search.track_total_hits": "跟踪总命中数", + "insight.config.search.timeout": "超时时间", + "insight.config.search.field_summary": "字段摘要", + "insight.config.search.whether_to_sample": "是否采样", + "insight.config.search.sample_records": "采样记录", + "insight.config.search.sample_records.all": "全部记录", + "insight.config.search.sample_records.manual": "手动设置", + "insight.config.search.top_number": "Top 数量", + "insight.config.search.time_interval.seconds": "秒", + "insight.config.search.time_interval.minutes": "分钟", + "insight.config.search.time_interval.hours": "小时", + "insight.config.search.time_interval.days": "天", + "insight.export.csv": "导出 CSV", + "insight.export.excel": "导出 Excel", + "insight.export.empty": "当前没有可导出的数据", + "insight.discover.empty.no_indices_or_views": "当前集群没有索引或视图", + "insight.discover.empty.create_now": "立即创建", + "insight.share.copy": "复制分享链接", + "insight.share.success": "分享链接已复制", + "insight.share.failed": "复制分享链接失败", + "insight.button.updating": "更新中", "menu.alerting": "告警管理", "menu.alerting.overview": "概览", @@ -192,6 +297,8 @@ export default { "menu.data.overview": "平台概览", "menu.data.index": "索引管理", "menu.data.view": "视图管理", + "menu.data.view.create": "创建", + "menu.data.view.detail": "详情", "menu.data.document": "文档管理", "menu.data.template": "模版管理", "menu.data.lifecycle": "周期管理", @@ -237,6 +344,10 @@ export default { "menu.resource.agent.new_instance": "探针注册", "menu.resource.agent.edit_instance": "编辑探针", + "menu.data_tools": "数据工具", + "menu.data_tools.migration": "数据迁移", + "menu.data_tools.comparison": "数据比对", + "menu.search": "搜索管理", "menu.search.overview": "概览", "menu.search.template": "搜索模板", @@ -250,6 +361,12 @@ export default { "menu.synchronize.pipeline": "数据加工", "menu.synchronize.rebuild": "数据重建", "menu.synchronize.inout": "导入导出", + "synchronize.rebuild.target_index.label": "目标索引名", + "synchronize.rebuild.target_index.required": "请输入目标索引名称", + "synchronize.rebuild.target_index.invalid": + "目标索引名必须使用小写字母并且符合 Elasticsearch 索引命名要求。", + "synchronize.ingest_pipeline.batch_delay.label": "管道批延迟", + "synchronize.ingest_pipeline.batch_delay.placeholder": "管道批延迟，例如 50", "menu.backup": "备份管理", "menu.backup.overview": "概览", @@ -387,6 +504,8 @@ export default { "app.message.system-tips.no-available-cluster-data-permission": "当前没有可用集群，请确保您有集群数据权限", "app.message.confirm.delete": "确定要删除？", + "app.message.confirm.delete.multiple": "确定要删除这 {count} 项吗？", + "document.confirm.cancel": "确定要取消？", "app.message.warning.invalid.params": "无效的参数", "app.message.warning.table.select-row": "请选择表格行", @@ -399,6 +518,13 @@ export default { "app.login.sign-in-with": "使用单点登录方式", "app.login.signup": "注册账户", "app.login.login": "登录", + "app.login.username.required": "请输入用户名！", + "app.login.password.required": "请输入密码！", + "app.login.mobile.placeholder": "手机号", + "app.login.mobile.required": "请输入手机号！", + "app.login.mobile.invalid": "手机号格式错误！", + "app.login.captcha.placeholder": "验证码", + "app.login.captcha.required": "请输入验证码！", "app.register.register": "注册", "app.register.get-verification-code": "获取验证码", "app.register.sing-in": "使用已有账户登录", @@ -615,6 +741,7 @@ export default { ...alias, ...guide, ...license, + ...migration, ...credential, ...overview, ...dashboard, diff --git a/web/src/locales/zh-CN/agent.js b/web/src/locales/zh-CN/agent.js index a675daa1..4141a09f 100644 --- a/web/src/locales/zh-CN/agent.js +++ b/web/src/locales/zh-CN/agent.js @@ -5,26 +5,81 @@ export default { "agent.instance.associate.labels.cluster_version": "版本", "agent.instance.associate.labels.select_cluster": "关联到集群", "agent.instance.associate.tips.associate": "请选择要关联的集群！", - "agent.instance.associate.set_credential": "为代理设置凭据", - "agent.instance.associate.set_credential.tips": "此权限将用于度量和日志收集。建议使用具有合理权限范围的用户。", + "agent.instance.associate.set_credential": "为探针设置凭据", + "agent.instance.associate.set_credential.tips": "此凭据将用于指标和日志收集，建议使用权限范围合理的用户。", + "agent.instance.associate.set_logs_paths": "为探针设置日志采集路径", + "agent.instance.associate.set_logs_paths.tips": + "这些日志目录会用于本次批量关联，并保存为后续自动补关联节点的默认日志采集路径。", "agent.instance.associate.tips.connected": "连接成功！", "agent.instance.associate.tips.connected.check": "请设置凭据", - "agent.instance.associate.auth.error": "以下集群需要为 Agent 设置凭据：", + "agent.instance.associate.tips.no_match": + "未能匹配并关联到节点，请检查探针凭据、节点 HTTP 端口和集群连通性后重试。", + "agent.instance.associate.auth.error": "以下集群需要先设置平台凭据或 Agent 凭据：", "agent.instance.associate.tips.metric": - "关联后 Agent 会对关联的集群进行指标采集操作", + "关联后会自动切换为 Agent 采集模式，后续新增节点也会自动补充关联", "agent.instance.associate.tips.unregister": "没有在 Console 中找到该集群的注册信息，", "agent.instance.associate.tips.to_register": "前往注册", "agent.instance.associate.drawer.title": "关联集群", "agent.instance.regist": "探针注册", + "agent.instance.field.endpoint.placeholder": + "探针地址，例如：127.0.0.1:2900", + "agent.instance.field.endpoint.form.required": "请输入探针 API 地址！", + "agent.instance.registration.copy": "复制", + "agent.instance.registration.console.title": "INFINI Console 访问凭据", + "agent.instance.registration.access.endpoint": "访问地址", + "agent.instance.registration.access.credential": "访问凭据", + "agent.instance.registration.console.endpoint.tip": + "复制到探针托管配置。", + "agent.instance.registration.console.token.tip": + "复制并妥善保存此凭据。", + "agent.instance.registration.console.load_failed": "获取 INFINI Console 访问信息失败。", + "agent.instance.registration.menu.register": "注册", + "agent.instance.registration.menu.info": "凭据", + "agent.instance.registration.agent.title": "探针访问信息", + "agent.instance.registration.agent.token.required": "请输入访问凭据！", + "agent.instance.registration.agent.endpoint.tip": + "请输入可访问的探针地址。", + "agent.instance.registration.agent.token.placeholder": + "请粘贴目标主机生成的探针访问凭据", + "agent.instance.registration.agent.token.tip": + "请粘贴探针访问凭据。", + "agent.instance.registration.agent.token.expire.tip": + "凭据长期有效，轮换或替换后失效。", + "agent.instance.registration.auth.type": "认证方式", + "agent.instance.registration.auth.type.access_token": "访问凭据", + "agent.instance.registration.auth.type.basic_auth": "用户名 / 密码", + "agent.form.placeholder.auth.username": "请输入用户名", + "agent.instance.step.result.button.register_new": "继续注册新探针", + "agent.instance.step.result.button.view_list": "查看探针列表", + "agent.instance.column.agent_ip": "探针 IP", + "agent.instance.row_detail.tab.detected_processes": "已识别进程（{count}）", + "agent.instance.row_detail.tab.unknown_processes": "未知进程（{count}）", + "agent.instance.process.detail.title": "进程详情", "agent.instance.associate.labels.node_adress": "节点地址", + "agent.instance.associate.labels.logs_paths": "日志目录", + "agent.instance.associate.labels.logs_paths.placeholder": + "请输入一个或多个日志目录", + "agent.instance.associate.labels.logs_paths.tips": + "默认已带出节点的 path.logs，可继续追加其他目录；这些目录会通过同一个 logs_path 下发给 Agent 采集。", "agent.instance.associate.tips.access_failed": "探针未能成功访问该节点，请修改设置后再试！", + "agent.instance.associate.credential_status.set": "已设置", + "agent.instance.associate.credential_status.unset": "未设置", "agent.install.label.get_cmd": "获取安装命令", "agent.install.setup.title": "快速安装", "agent.install.setup.desc": "请复制下方命令并在目标主机上执行，其包含 INFINI Agent 的下载、部署及启动", + "agent.install.tips.intranet.title": "内网部署说明", + "agent.install.tips.intranet.desc": + "默认安装目录为 /infini/agent，通常无需额外配置；如需使用内网镜像，再配置自定义下载地址即可。", "agent.install.tips.title": "提示", + "agent.install.tips.target": "默认安装目录为", + "agent.install.tips.version": "如需指定 Agent 版本，可追加", + "agent.install.tips.download": + "如需使用内网或自定义下载源，可追加", + "agent.install.tips.server": + "如目标主机无法直接访问当前 Console 地址，可追加", "agent.install.tips.desc": "当前版本自动安装仅支持 Linux ，非 Linux 系统请选择", "agent.install.link.manual_install": "手动安装", @@ -32,15 +87,34 @@ export default { "agent.install.logs.tips": "日志查看功能需先安装探针(INFINI Agent)", "agent.logs.label.log_file": "日志文件", "agent.logs.label.goto": "跳转至行", - "agent.logs.button.view_latest": "查看最新", + "agent.logs.label.latest_lines": "行", + "agent.logs.button.view_latest": "跟随最新", "agent.logs.button.goto": "确定", "agent.install.setup.copy.success": "已成功复制到剪贴板！", + "agent.install.advanced.title": "高级配置", + "agent.install.reverse_channel.label": "启用反向通道", + "agent.install.reverse_channel.help": "默认关闭。仅在 Agent 端口不可达时开启（如容器环境）；开启后 Agent 会回连 Console。", + "agent.install.no_sudo.label": "无 sudo 权限", + "agent.install.no_sudo.help": "适用于容器或非 root 环境。开启后生成的命令会去掉 sudo，并自动追加 --no-service，安装完成后请将 Agent 作为容器 ENTRYPOINT/CMD 前台运行。", + "agent.install.no_sudo.help_line": "如当前环境没有 sudo 或不支持系统服务，可追加", + "agent.install.no_sudo.tip.title": "容器/无 sudo 模式说明", + "agent.install.no_sudo.tip.desc": "此模式不会安装或启动系统服务。安装完成后，请把 Agent 二进制作为容器主进程运行，由 Docker 或 Kubernetes 负责拉起与重启。", + "agent.install.no_sudo.entrypoint.title": "Dockerfile ENTRYPOINT 示例", + "agent.install.no_sudo.cmd.title": "Dockerfile CMD 示例", "agent.instance.auto_associate.title": "自动关联集群", - "agent.instance.install.title": "安装 Agent", + "agent.instance.install.title": "安装探针", - "agent.label.agent_credential": "代理凭据", + "agent.label.agent_credential": "探针凭据", "agent.credential.tip": "不需要凭据", + "agent.instance.button.revoke": "撤销", + "agent.instance.delete.confirm.title": "确定要删除这个探针吗？", + "agent.instance.revoke.confirm.title": "确定要撤销吗？", "agent.instance.clear.title": "清理离线实例", "agent.instance.clear.modal.title": "您确定要清理离线实例？", - "agent.instance.clear.modal.desc": "该操作将会删除离线并且 7 天没有上报指标的实例" + "agent.instance.clear.modal.desc": "该操作将会删除离线并且 7 天没有上报指标的实例", + "agent.instance.collection_interval.label": "采集间隔", + "agent.instance.collection_interval.unit": "秒", + "agent.instance.collection_interval.placeholder": "默认 10", + "agent.instance.collection_interval.tip": "节点指标采集间隔（秒）。留空或设为 0 使用默认值（10 秒）。修改后将在下次配置同步时生效。", + "agent.instance.collection_interval.save.success": "采集间隔已更新" }; diff --git a/web/src/locales/zh-CN/alert.js b/web/src/locales/zh-CN/alert.js index 5b68d38b..1323ea22 100644 --- a/web/src/locales/zh-CN/alert.js +++ b/web/src/locales/zh-CN/alert.js @@ -204,12 +204,21 @@ export default { "alert.rule.table.columnns.category": "分类", "alert.rule.table.columnns.tags": "标签", "alert.rule.table.columnns.last_notification_time": "最近告警", + "alert.import.submit.success": "导入成功", + "alert.import.submit.failed": "导入失败", + "alert.import.select_file": "请选择文件", + "alert.import.upload.success": "文件上传成功", + "alert.import.upload.failed": "文件上传失败", + "alert.import.upload.invalid_json": "无效的 JSON 文件", // /alerting/rule/edit page 编辑规则页面 "alert.rule.form.title.edit": "编辑规则", "alert.rule.form.label.select_cluster": "选择集群", "alert.rule.form.label.select_object": "选择告警对象", "alert.rule.form.label.filter_condition": "筛选条件", "alert.rule.form.label.time_field": "时间字段", + "alert.rule.form.label.ignore_time_filter": "忽略时间过滤", + "alert.rule.form.help.ignore_time_filter": + "针对非时序规则开启后，将不再按时间范围过滤。", //Configure alert objects 配置告警对象 "alert.rule.form.title.configure_alert_object": "配置告警对象", "alert.rule.form.label.alert_metric": "告警指标", @@ -244,6 +253,9 @@ export default { "alert.rule.form.title.configure_alert_channel": "告警通知", "alert.rule.form.title.configure_alert_channel_recovery": "告警恢复通知", "alert.rule.form.label.alert_channel": "告警渠道", + "alert.rule.form.label.incremental_recovery_notification": "分步恢复通知", + "alert.rule.form.help.incremental_recovery_notification": + "开启后，分组告警在部分分组恢复时会立即发送恢复通知。", "alert.rule.form.label.accept_upgrade": "接收升级", "alert.rule.form.label.upgrade_notification_waiting_time": "升级通知等待时间", "alert.rule.form.label.silent_period": "沉默周期", @@ -257,6 +269,7 @@ export default { "alert.rule.form.title.template_variables_examples": "模板变量示例：", "alert.rule.form.title.example1": "示例1:", "alert.rule.form.title.example2": "示例2(数组遍历):", + "alert.rule.form.title.example": "示例", "alert.rule.form.title.template_function": "模板函数", // alerting/rule/new page 新建规则页面 "alert.rule.form.title.create": "新建规则", @@ -290,6 +303,16 @@ export default { "alert.message.priority.info": "P4(Info)", "alert.message.priority.ignored": "Ignored", "alert.message.priority.": "Undefined", + "alert.message.priority.undefined": "未定义", + "alert.message.status.alerting": "告警中", + "alert.message.status.ignored": "已忽略", + "alert.message.status.recovered": "已恢复", + "alert.message.status.ok": "正常", + "alert.message.status.error": "错误", + "alert.message.status.nodata": "无数据", + "alert.message.ignored.time": "忽略时间", + "alert.message.ignored.operator": "操作人", + "alert.message.ignored.reason": "原因", "alert.message.table.created": "触发时间", "alert.message.table.priority": "告警级别", @@ -313,11 +336,14 @@ export default { "alert.message.detail.ignored_time": "忽略时间", "alert.message.detail.message": "事件内容", "alert.message.detail.alert_metric_status": "告警指标状态", + "alert.message.detail.no_history_data": "暂无告警历史数据", "alert.message.detail.execution_record": "执行记录", "alert.message.detail.action_message": "通知内容", "alert.message.detail.action_result": "执行结果", "alert.message.detail.action_result_error": "规则执行错误", "alert.message.detail.alert_info": "告警详情", + "alert.message.detail.query_dsl": "查询 DSL", + "alert.message.detail.response": "响应", "alert.message.detail.condition.type": "触发条件类型", "alert.message.detail.condition": "触发条件", "alert.message.detail.bucket_diff_type": "分桶对比类型", @@ -339,7 +365,14 @@ export default { "alert.channel.form.advanced.custom": "(自定义)", "alert.channel.form.advanced.load.default": "加载渠道默认配置", + "alert.channel.form.webhook.url": "Webhook 地址", + "alert.channel.form.webhook.url.required": "请输入 Webhook 地址！", + "alert.channel.form.webhook.method": "请求方法", + "alert.channel.form.webhook.method.required": "请选择请求方法！", + "alert.channel.form.webhook.headers": "请求头", "alert.channel.form.webhook.template.title": "默认内容模板", + "alert.channel.form.webhook.body": "请求体", + "alert.channel.form.webhook.body.required": "请输入请求体！", "alert.channel.form.webhook.send.test": "发送测试消息", "alert.channel.form.email.server": "邮件服务器", @@ -357,10 +390,21 @@ export default { "alert.channel.form.email.template.body": "正文", "alert.channel.form.email.template.body.required": "请输入正文！", "alert.channel.form.email.send.test": "发送测试邮件", + "alert.rule.form.template.sync": "同步模板", + "alert.rule.form.template.sync.success": "当前规则模板同步成功", + "alert.rule.form.template.sync.failed": "同步规则模板失败", + "alert.channel.enable.tip.email_incomplete": + "启用邮件渠道前，请先配置邮件服务器和至少一个收件人。", + "alert.channel.enable.tip.email_server": + "启用邮件渠道前，请先配置邮件服务器。", + "alert.channel.enable.tip.email_recipients": + "启用邮件渠道前，请先配置至少一个收件人。", "alert.channel.empty": "没有告警渠道？", "alert.channel.export-import.label": "告警渠道", "alert.rule.export-import.label": "告警规则", + "alert.import.example.title": "导入示例", + "alert.import.example.tip.rule": "建议先导出一份规则文件作为模板，再按相同结构编辑后导入。", // 告警任务详情页面 "alert.task.detail.title": "告警任务详情", diff --git a/web/src/locales/zh-CN/cluster.js b/web/src/locales/zh-CN/cluster.js index 2a3099c1..273e099b 100644 --- a/web/src/locales/zh-CN/cluster.js +++ b/web/src/locales/zh-CN/cluster.js @@ -26,6 +26,7 @@ export default { "cluster.manage.label.provider.digital-ocean": "Digital Ocean", "cluster.manage.label.region": "集群位置", "cluster.manage.table.column.monitored": "监控启用", + "cluster.manage.table.column.monitor_toggle": "监控状态", "cluster.manage.table.column.monitor_mode": "监控模式", "cluster.manage.table.column.discovery.enabled": "节点发现", "cluster.manage.table.column.operation": "操作", @@ -34,24 +35,60 @@ export default { "cluster.manage.table.column.owner": "管理人员", "cluster.manage.table.column.area": "服务器可用区", "cluster.manage.table.column.location": "位置", + "cluster.manage.field.tls.label": "开启 TLS", + "cluster.manage.field.tags.label": "标签", "cluster.manage.monitored.on": "启用", "cluster.manage.monitored.off": "关闭", "cluster.manage.metric_collection_mode": "采集模式", + "cluster.manage.metric_collection_mode.option.agent": "探针", + "cluster.manage.metric_collection_mode.option.agentless": "非探针", "cluster.manage.metric_collection_mode.confirm.title": "确认切换监控方式", "cluster.manage.metric_collection_mode.confirm.message": "您确认要切换为 {mode} 模式吗？", "cluster.manage.metric_collection_mode.warning.large_cluster": "当前集群节点数大于等于 {number_of_nodes} 个，强烈建议使用 Agent 模式进行监控。", "cluster.manage.metric_collection_mode.confirm.button.ok": "确定", "cluster.manage.metric_collection_mode.confirm.button.cancel": "取消", + "cluster.manage.monitoring.enable.action": "启用监控", + "cluster.manage.monitoring.disable.action": "禁用监控", + "cluster.manage.monitoring.confirm.enable.title": "确认启用这个集群的监控吗？", + "cluster.manage.monitoring.confirm.disable.title": "确认禁用这个集群的监控吗？", + "cluster.manage.monitoring.confirm.cluster": "集群：{name}", + "cluster.manage.monitoring.confirm.version": "版本：{version}", + "cluster.manage.monitoring.confirm.endpoint": "地址：{endpoint}", + "cluster.manage.monitoring.notice.unmonitored": "当前集群尚未启用监控。", + "cluster.manage.monitoring.notice.last_collection_time": "最近一次数据采集时间：{timestamp}", + "cluster.manage.monitoring.notice.enable_button": "前往启用监控", + "cluster.manage.monitoring.notice.unavailable_since": "集群自 {timestamp} 起不可用", + "cluster.manage.delete.confirm.title": "确认删除这个集群吗？", + "cluster.manage.delete.confirm.cluster": "集群：{name}", + "cluster.manage.delete.confirm.version": "版本：{version}", + "cluster.manage.delete.confirm.endpoint": "地址：{endpoint}", + "cluster.manage.agent_credential.tip.auto_create": "切换为探针采集模式后，Console 会自动创建低权限的 infini-agent 用户，用于指标和日志采集；如有需要，也可以在下方改用自定义探针凭据。", + "cluster.manage.agent_credential.placeholder.auto_create": "留空将自动创建，失败时回退使用平台凭据", + "cluster.manage.agent_credential.tip.agentless_skip": "当前为非探针采集模式，无需配置探针凭据；如需通过 Agent 采集指标和日志，请先切换为探针模式。", + "cluster.manage.agent_logs_paths.label": "探针日志采集路径", + "cluster.manage.agent_logs_paths.placeholder": "请输入一个或多个日志目录", + "cluster.manage.agent_logs_paths.tips": "这里配置的是该集群默认的 Agent 日志采集目录，会用于后续批量关联和自动补关联的新节点；留空则继续使用节点自身探测到的 path.logs。", + "cluster.manage.config_item.enabled": "启用", + "cluster.manage.config_item.interval": "间隔", + "cluster.manage.config_item.interval.unit": "秒", "cluster.manage.monitor_configs.cluster_health": "集群健康状态指标", "cluster.manage.monitor_configs.cluster_stats": "集群指标", + "cluster.manage.monitor_configs.tips.cluster_health": "采集集群健康状态（green/yellow/red）等概览指标。", + "cluster.manage.monitor_configs.tips.cluster_stats": "采集 _cluster/stats 的集群级统计信息（节点数、分片、存储等）。", "cluster.manage.monitor_configs.node_stats": "节点指标", + "cluster.manage.monitor_configs.tips.node_stats": "采集 _nodes/stats 的各节点统计信息（CPU、内存、磁盘、线程池等）。", "cluster.manage.monitor_configs.index_stats": "索引指标", + "cluster.manage.monitor_configs.tips.index_stats": "采集 _stats 的各索引统计信息（文档数、大小、分段、合并等）。", "cluster.manage.monitor_configs.index_health": "索引健康状态指标", "cluster.manage.monitor_configs.shard_stats": "分片指标", "cluster.manage.metadata_configs.health_check": "健康检查", "cluster.manage.metadata_configs.node_availability_check": "节点可用性检查", "cluster.manage.metadata_configs.metadata_refresh": "元数据同步", "cluster.manage.metadata_configs.cluster_settings_check": "集群设置检查", + "cluster.manage.metadata_configs.tips.health_check": "定时检查集群健康与可达性，用于更新集群可用状态。", + "cluster.manage.metadata_configs.tips.node_availability_check": "基于 TCP 对节点地址做可用性探测，需要直接访问节点；当集群仅能通过 HTTP 代理访问时将无法工作。", + "cluster.manage.metadata_configs.tips.metadata_refresh": "定时刷新节点、索引别名等元数据信息。", + "cluster.manage.metadata_configs.tips.cluster_settings_check": "定时获取并同步集群设置（如 transient/persistent settings）。", "cluster.manage.monitor_configs.rollup_node_stats": "节点指标", "cluster.manage.monitor_configs.rollup_index_stats": "索引指标", "cluster.manage.monitor_configs.rollup_cluster_stats": "集群指标", @@ -68,7 +105,7 @@ export default { "cluster.regist.step.complete.title": "完成", "cluster.regist.step.connect.label.auth": "身份验证", "cluster.regist.step.connect.label.credential": "选择用户凭据", - "cluster.regist.step.connect.label.agent_credential": "选择 Agent 凭据", + "cluster.regist.step.connect.label.agent_credential": "选择探针凭据", "cluster.regist.step.connect.credential.manual": "手动输入", "cluster.regist.step.connect.label.username": "用户名", "cluster.regist.step.connect.label.password": "密码", @@ -82,12 +119,32 @@ export default { "cluster.regist.step.complete.tls.yes": "是", "cluster.regist.step.complete.tls.no": "否", "cluster.regist.form.verify.required.cluster_name": "请输入集群名称！", - "cluster.regist.form.verify.valid.endpoint": "请输入域名或 IP 地址和端口号！", + "cluster.regist.form.verify.valid.endpoint": "请输入域名或 IP 地址，可选端口号！", "cluster.regist.form.verify.required.endpoint": "请输入 endpoint 地址！", + "cluster.regist.form.label.probe_path": "检测路径", + "cluster.regist.form.placeholder.probe_path": "/_cluster/health", + "cluster.regist.form.toggle.probe_path": "自定义检测路径", + "cluster.regist.form.help.probe_path": + "可选。留空时默认使用 / 进行探测，仅在 WAF 等特殊场景下需要填写。", + "cluster.regist.form.verify.valid.probe_path": + "检测路径必须以 / 开头！", "cluster.regist.form.verify.required.credential": "请选择用户凭据！", - "cluster.regist.form.verify.required.agent_credential": "请选择 Agent 凭据！", + "cluster.regist.form.verify.required.agent_credential": "请选择探针凭据！", "cluster.regist.form.verify.required.auth_username": "请输入授权用户名！", "cluster.regist.form.verify.required.auth_password": "请输入授权密码！", + "cluster.regist.try_connect.failed": "集群连接失败，请检查地址、TLS 与认证信息。", + "cluster.connect.error.health_red": + "目标集群当前健康状态为 red，仅允许连接健康状态为 green 的集群，请先修复集群后再连接。", + "cluster.connect.error.tls_mismatch": + "TLS 设置与集群地址不匹配，请检查是否应该启用 HTTPS。", + "cluster.connect.error.auth_required": + "集群需要认证，或当前凭据无效，请检查用户名、密码或凭据配置。", + "cluster.connect.error.endpoint_unreachable": + "无法连接到集群地址，请检查地址、网络连通性以及 TLS 设置。", + "cluster.connect.error.non_es_endpoint": + "目标地址返回的不是 Elasticsearch 兼容 API 响应，请检查地址和端口是否填写正确。", + "cluster.connect.error.unexpected_status": + "集群返回了异常状态码，请检查地址、TLS 设置和认证信息。", "cluster.regist.form.credential.manual.desc": "*新的身份验证信息将在保存后添加到凭据库", @@ -128,6 +185,8 @@ export default { "cluster.monitor.topn.area": "面积指标", "cluster.monitor.topn.color": "颜色指标", "cluster.monitor.topn.theme": "主题", + "cluster.monitor.rollup.gap": "汇聚延迟", + "cluster.monitor.treemap.search_latency_by_index": "按索引统计平均查询延迟", "cluster.monitor.logs.timestamp": "时间戳", "cluster.monitor.logs.type": "类型", @@ -139,9 +198,9 @@ export default { "cluster.monitor.logs.empty.agentless": "没有数据，请安装 Agent 并更改集群采集模式为 Agent 。", "cluster.monitor.tabs.overview": "概览", - "cluster.monitor.tabs.rollup": "Rollup", + "cluster.monitor.tabs.rollup": "汇聚", "cluster.monitor.tabs.advanced": "高级", - "cluster.monitor.tabs.topn": "TopN", + "cluster.monitor.tabs.topn": "视图", "cluster.monitor.tabs.logs": "日志", "cluster.monitor.tabs.nodes": "节点", "cluster.monitor.tabs.indices": "索引", @@ -363,15 +422,15 @@ export default { "In Flight Requests Breaker", "cluster.metrics.node.axis.model_inference_breaker.title": "Model Inference Breaker", - "cluster.metrics.axis.rollup_cluster_health.title": "Rollup Cluster Health", - "cluster.metrics.axis.rollup_index_health.title": "Rollup Index Health", - "cluster.metrics.axis.rollup_cluster_stats.title": "Rollup Cluster Stats", - "cluster.metrics.axis.rollup_index_stats.title": "Rollup Index Stats", - "cluster.metrics.axis.rollup_node_stats.title": "Rollup Node Stats", + "cluster.metrics.axis.rollup_cluster_health.title": "汇聚集群健康状态", + "cluster.metrics.axis.rollup_index_health.title": "汇聚索引健康状态", + "cluster.metrics.axis.rollup_cluster_stats.title": "汇聚集群指标", + "cluster.metrics.axis.rollup_index_stats.title": "汇聚索引指标", + "cluster.metrics.axis.rollup_node_stats.title": "汇聚节点指标", "cluster.metrics.axis.rollup_shard_stats_metrics.title": - "Rollup Shard Stats Metrics", + "汇聚分片指标", "cluster.metrics.axis.rollup_shard_stats_state.title": - "Rollup Shard Stats State", + "汇聚分片状态", //overview "overview.card.cluster.total_count": "集群总数", "overview.card.host.total_count": "主机总数", diff --git a/web/src/locales/zh-CN/command.js b/web/src/locales/zh-CN/command.js index 764ac138..27a64985 100644 --- a/web/src/locales/zh-CN/command.js +++ b/web/src/locales/zh-CN/command.js @@ -1,15 +1,20 @@ export default { "command.table.field.name": "名称", "command.table.field.tag": "标签", + "command.table.field.creator": "创建人", + "command.table.field.created": "创建时间", "command.table.field.content": "内容", + "command.table.field.summary": "摘要", + "command.table.summary.requests": "{count} 个请求", "command.manage.edit.title": "常用命令", "command.btn.newtag": "新建标签", "command.message.invalid.tag": "无效的标签内容", + "command.message.title_exists": "标题已存在", "command.manage.save.title": "保存为常用命令", "command.manage.title": "常用命令管理", "command.manage.description": "常用命令可以帮助您保存常用的请求，并且在开发工具里面通过 LOAD 命令快速地加载。", - "console.menu.copy_as_curl": "复制为Curl命令", + "console.menu.copy_as_curl": "复制为 cURL 命令", "console.menu.auto_indent": "自动缩进", "console.menu.save_as_command": "保存为常用命令", }; diff --git a/web/src/locales/zh-CN/credential.js b/web/src/locales/zh-CN/credential.js index 137b64de..2475a829 100644 --- a/web/src/locales/zh-CN/credential.js +++ b/web/src/locales/zh-CN/credential.js @@ -15,8 +15,17 @@ export default { "credential.manage.drawer.edit.title": "凭据详情", "credential.manage.form.type": "凭据类型", + "credential.manage.form.type.required": "请选择凭据类型！", "credential.manage.form.name": "凭据名称", + "credential.manage.form.name.required": "请输入凭据名称！", "credential.manage.form.username": "用户名", + "credential.manage.form.username.required": "请输入用户名！", "credential.manage.form.password": "密码", + "credential.manage.form.password.required": "请输入密码！", + "credential.manage.form.password.placeholder.edit": "原始密码不会显示", + "credential.manage.form.token": "Token", + "credential.manage.form.token.required": "请输入 Token！", + "credential.manage.form.token.placeholder": "请输入 Token！", + "credential.manage.form.token.placeholder.edit": "原始 Token 不会显示", "credential.manage.form.tags": "标签", }; diff --git a/web/src/locales/zh-CN/error.js b/web/src/locales/zh-CN/error.js index 0cc8cc81..5adcbd9d 100644 --- a/web/src/locales/zh-CN/error.js +++ b/web/src/locales/zh-CN/error.js @@ -2,4 +2,7 @@ export default { "error.split": "，", "error.unknown": "未知错误，请稍后重试或者联系支持团队！", "error.request_timeout_error": "请求超时，请稍后重试！", + "error.request.connection_refused": "连接服务器失败。", + "error.request.connection_refused.tip": + "请点击上方的服务受限，以确保系统集群正常运行。", } diff --git a/web/src/locales/zh-CN/explore.js b/web/src/locales/zh-CN/explore.js index 31392a4a..77ad4dc8 100644 --- a/web/src/locales/zh-CN/explore.js +++ b/web/src/locales/zh-CN/explore.js @@ -51,4 +51,66 @@ export default { "explore.indexfield.description": "当前页面列出匹配 {pattern} 索引的所有字段，字段类型为搜索引擎内的类型。 若需要更改类型，请使用", "explore.view.index_pattern.removeTooltip": "删除视图", + "explore.view.index_pattern.refreshTooltip": "刷新字段列表", + "explore.view.index_pattern.refreshFieldListTitle": "刷新字段列表？", + "explore.view.index_pattern.back_to_list": "返回视图列表", + "explore.view.index_pattern.detail_title": "视图详情", + "explore.view.index_pattern.delete_confirm": "删除视图？", + "explore.view.index_pattern.time_field": "时间字段：'{field}'", + "explore.view.index_pattern.mapping_conflict_title": "Mapping 冲突", + "explore.view.index_pattern.mapping_conflict_desc": + "当前视图匹配的索引中有 {count} 个字段存在多种类型定义，例如 string、integer 等。您仍可查看这些冲突字段，但无法把它们用于函数计算；如需消除冲突，请重新整理索引映射。", + "explore.view.index_pattern.tab.fields": "字段", + "explore.view.index_pattern.tab.scripted_fields": "脚本字段", + "explore.view.index_pattern.tab.source_filters": "源过滤器", + "explore.view.index_pattern.tab.complex_fields": "复杂字段 ({count})", + "explore.view.index_pattern.search_fields": "搜索字段", + "explore.view.index_pattern.filter_field_types": "筛选字段类型", + "explore.view.index_pattern.search_placeholder": "搜索", + "explore.view.index_pattern.create_field": "创建字段", + "explore.view.index_pattern.all_field_types": "全部字段类型", + "explore.view.index_pattern.all_languages": "全部语言", + "explore.view.index_pattern.field_editor.default_option": "- 默认 -", + "explore.view.index_pattern.field_editor.default_label": "默认", + "explore.view.index_pattern.field_editor.format": "格式", + "explore.view.index_pattern.field_editor.format_help": + "格式化允许您控制特定值的显示方式。它也可能完全改变值的显示结果，并导致 Discover 中的高亮失效。", + "explore.view.index_pattern.field_editor.save_field": "保存字段", + "explore.view.index_pattern.field_editor.statistics": "统计项", + "explore.view.index_pattern.field_editor.field": "字段", + "explore.view.index_pattern.field_editor.group_field": "分组字段", + "explore.view.index_pattern.field_editor.dividend_field": "被除数字段", + "explore.view.index_pattern.field_editor.divisor_field": "除数字段", + "explore.view.index_pattern.field_editor.add_new": "新增", + "explore.view.index_pattern.complex_field_editor.name": "名称", + "explore.view.index_pattern.complex_field_editor.name_required": + "名称不能为空", + "explore.view.index_pattern.complex_field_editor.new_field_placeholder": + "新字段", + "explore.view.index_pattern.complex_field_editor.duplicate_name": + "已存在名为 {name} 的字段。", + "explore.view.index_pattern.complex_field_editor.metric_name": + "指标名称", + "explore.view.index_pattern.complex_field_editor.function": "函数", + "explore.view.index_pattern.complex_field_editor.unit": "单位", + "explore.view.index_pattern.complex_field_editor.tags": "标签", + "explore.view.index_pattern.complex_field_editor.delete_title": + "删除字段“{name}”", + "explore.view.index_pattern.complex_field_editor.delete_confirm": + "删除后的字段无法恢复，确认要继续吗？", + "explore.view.index_pattern.complex_field_editor.edit_title": + "编辑 {name}", + "explore.save_queries.title": "保存查询", + "explore.save_queries.field.title": "标题", + "explore.save_queries.field.tag": "标签", + "explore.save_queries.field.description": "描述", + "explore.save_queries.button.cancel": "取消", + "explore.save_queries.button.save": "保存", + "explore.save_queries.button.update": "更新", + "explore.save_queries.validation.title_required": "请输入标题！", + "explore.save_queries.validation.title_exists": "修改后的标题已存在！", + "explore.load_queries.title": "加载查询", + "explore.load_queries.search.title": "请输入查询标题", + "explore.load_queries.search.tag": "请选择标签", + "explore.load_queries.updated_at": "更新时间", }; diff --git a/web/src/locales/zh-CN/gateway.js b/web/src/locales/zh-CN/gateway.js index 9db5da3e..11e51f54 100644 --- a/web/src/locales/zh-CN/gateway.js +++ b/web/src/locales/zh-CN/gateway.js @@ -3,6 +3,23 @@ export default { "gateway.instance.new.title": "实例注册", "gateway.instance.new.description": "输入网关地址分步注册极限网关。", "gateway.instance.btn.new": "新建", + "gateway.instance.install.title": "安装网关", + "gateway.install.label.get_cmd": "获取安装命令", + "gateway.install.advanced.title": "高级配置", + "gateway.install.type.label": "服务类型", + "gateway.install.type.migration": "迁移网关", + "gateway.install.type.relay": "转发网关", + "gateway.install.relay_role.label": "中继角色", + "gateway.install.relay_role.primary": "主网关", + "gateway.install.relay_role.secondary": "备网关", + "gateway.install.no_sudo.label": "无 sudo 权限", + "gateway.install.no_sudo.help": + "适用于容器或非 root 环境。开启后生成的命令会去掉 sudo，并自动追加 --no-service，安装完成后请将 Gateway 作为容器主进程或以前台方式运行。", + "gateway.install.no_sudo.help_line": "如当前环境没有 sudo 或不支持系统服务，可追加", + "gateway.install.no_sudo.tip.title": "容器/无 sudo 模式说明", + "gateway.install.no_sudo.tip.desc": + "此模式不会安装或启动系统服务。安装完成后，请以前台方式运行 Gateway，并由 Docker、Kubernetes 或其他进程管理器负责拉起与重启。", + "gateway.install.no_sudo.command.title": "前台启动示例", "gateway.entry.index.title": "入口管理", "gateway.entry.index.description": "入口管理帮助您方便快捷地添加、查看、修改以及删除极限网关的入口配置。", @@ -17,9 +34,110 @@ export default { "修改实例配置，然后点击保存按钮，保存成功之后生效。", "gateway.instance.field.name.label": "实例名称", "gateway.instance.field.name.form.required": "请输入实例名称！", - "gateway.instance.field.endpoint.form.required": "请输入实例 api endpoint！", + "gateway.instance.field.endpoint.label": "地址", + "gateway.instance.field.endpoint.form.required": "请输入实例 API 地址！", + "gateway.instance.field.endpoint.placeholder": + "实例 API 地址，例如：127.0.0.1:2900", + "gateway.instance.field.tls.label": "开启 TLS", "gateway.instance.field.tags.label": "标签", "gateway.instance.field.description.placeholder": "实例描述", + "gateway.instance.delete.confirm.title": "确定要删除这条记录吗？", + "gateway.instance.column.application": "应用", + "gateway.instance.column.name": "名称", + "gateway.instance.column.endpoint": "地址", + "gateway.instance.column.status": "状态", + "gateway.instance.column.cpu": "CPU", + "gateway.instance.column.memory": "内存", + "gateway.instance.column.storage": "存储", + "gateway.instance.column.uptime": "运行时长", + "gateway.instance.column.tags": "标签", + "gateway.instance.status.checking": "检测中", + "gateway.instance.status.online": "在线", + "gateway.instance.status.unavailable": "不可用", + "gateway.instance.storage.tooltip": "空闲/总量：{free}/{total}", + "gateway.instance.menu.queue": "队列", + "gateway.instance.menu.task": "任务", + "gateway.instance.menu.logging": "日志", + "gateway.instance.menu.config": "配置", + "gateway.instance.config.files": "配置文件", + "gateway.instance.config.runtime": "运行时", + "gateway.instance.config.main": "主配置", + "gateway.instance.config.location": "位置", + "gateway.instance.config.save.confirm": "确定要保存吗？", + "gateway.queue.tab.fifo": "FIFO", + "gateway.queue.tab.spmc": "SPMC", + "gateway.queue.field.local_storage": "本地存储", + "gateway.queue.field.depth": "深度", + "gateway.queue.field.offset": "偏移量", + "gateway.queue.field.produce_offset": "生产偏移量", + "gateway.queue.field.consume_offset_earliest": "消费偏移量（最早）", + "gateway.queue.field.synchronization_latest_segment": "同步（latest_segment）", + "gateway.queue.field.total_messages": "消息总数", + "gateway.queue.delete.success": "成功删除 {count} 个队列", + "gateway.queue.delete.error": "删除队列失败。{detail}", + "gateway.queue.delete_failed": "删除失败", + "gateway.queue.message.partial_success": "成功：{count}", + "gateway.queue.batch.delete_queues": "删除队列", + "gateway.queue.batch.delete_consumers": "删除消费者", + "gateway.queue.consumer.title": "消费者", + "gateway.queue.consumer.field.group": "分组", + "gateway.queue.consumer.field.last_active": "最后活跃时间", + "gateway.queue.consumer.field.source": "来源", + "gateway.queue.consumer.action.reset_offset": "重置偏移量", + "gateway.queue.consumer.delete.success": "成功删除 {count} 个消费者", + "gateway.queue.consumer.delete.error": "删除消费者失败。{detail}", + "gateway.queue.consumer.reset_offset.title": "重置消费者偏移量", + "gateway.queue.consumer.reset_offset.new_offset": "新偏移量", + "gateway.queue.consumer.reset_offset.offset_required": "请输入偏移量！", + "gateway.queue.consumer.reset_offset.success": "偏移量重置成功", + "gateway.queue.message.title": "消息（ID：{id}）", + "gateway.queue.message.goto_offset": "跳转到偏移量：", + "gateway.queue.message.goto": "跳转", + "gateway.queue.message.field.message": "消息", + "gateway.queue.message.field.size": "大小", + "gateway.queue.message.load_more": "加载更多", + "gateway.instance.logging.tab.realtime": "实时日志", + "gateway.instance.logging.auto_scroll": "自动滚动", + "gateway.instance.logging.button.start": "开始", + "gateway.instance.logging.button.stop": "停止", + "gateway.instance.logging.empty": "点击开始后显示实时日志", + "gateway.instance.logging.copy": "复制日志", + "gateway.instance.logging.copy.success": "日志已复制到剪贴板", + "gateway.instance.logging.endpoint.label": "连接地址", + "gateway.instance.logging.endpoint.empty": "当前实例缺少可用地址", + "gateway.instance.logging.placeholder.file_pattern": + "文件匹配，例如：xyz*.go", + "gateway.instance.logging.placeholder.func_pattern": + "函数匹配，例如：submit*", + "gateway.instance.logging.placeholder.message_pattern": + "消息匹配，例如：*timeout", + "gateway.instance.logging.connection.connecting": "连接中", + "gateway.instance.logging.connection.established": "已连接", + "gateway.instance.logging.connection.closing": "关闭中", + "gateway.instance.logging.connection.closed": "已断开", + "gateway.instance.logging.connection.uninstantiated": "初始化中", + "gateway.task.empty": "暂无任务", + "gateway.task.column.name": "名称", + "gateway.task.column.state": "状态", + "gateway.task.column.start_time": "开始时间", + "gateway.task.column.end_time": "结束时间", + "gateway.task.confirm.start": "确定要启动这个任务吗？", + "gateway.task.confirm.stop": "确定要停止这个任务吗？", + "gateway.task.state.starting": "启动中", + "gateway.task.state.started": "已启动", + "gateway.task.state.cancelled": "已取消", + "gateway.task.state.stopping": "停止中", + "gateway.task.state.stopped": "已停止", + "gateway.task.state.failed": "失败", + "gateway.task.state.finished": "已完成", + "gateway.router.column.name": "名称", + "gateway.router.column.default_flow": "默认流程", + "gateway.router.column.tracing_flow": "追踪流程", + "gateway.router.column.updated": "最后更新时间", + "gateway.router.search.placeholder": "输入关键字搜索", + "gateway.router.delete.confirm.title": "确定要删除吗？", + "gateway.router.btn.new": "新建", + "gateway.router.pagination.total": "{start}-{end} / 共 {total} 项", "gateway.instance.regist": "实例注册", @@ -41,8 +159,14 @@ export default { "gateway.guide.quick_install": "快速安装", "gateway.guide.quick_install.desc": "请复制下方命令并在本地部署环境执行，其包含 INFINI Gateway 的下载、部署及启动：", + "gateway.guide.intranet.title": "内网部署说明", + "gateway.guide.intranet.desc": + "默认安装目录为 /infini/gateway，通常无需额外配置；如需使用内网镜像，再配置自定义下载地址即可。", "gateway.guide.shell.copy.success": "复制成功！", "gateway.guide.tips.title": "提示：", + "gateway.guide.tips.version": "如需指定 Gateway 版本，可追加", + "gateway.guide.tips.directory": "如需指定安装目录，可追加", + "gateway.guide.tips.download_source": "如需使用内网或自定义下载源，可追加", "gateway.guide.tips.content": "当前版本自动安装仅支持 Linux， 非 Linux 系统请", "gateway.guide.tips.install_manually": "手动安装", diff --git a/web/src/locales/zh-CN/guide.js b/web/src/locales/zh-CN/guide.js index 45c8b136..af30539a 100644 --- a/web/src/locales/zh-CN/guide.js +++ b/web/src/locales/zh-CN/guide.js @@ -2,10 +2,39 @@ export default { "guide.header.title": "配置向导", "guide.initialization.step.configuration": "配置", "guide.initialization.step.configuration.desc": - "连接系统集群（Elasticsearch 要求 5.3 或更高版本）。", + "连接系统集群（Easysearch 要求 2.3 或更高版本）。", "guide.initialization.step.initialization": "初始化", "guide.initialization.step.initialization.desc": "初始化系统索引和模板的基本设置。", + "guide.initialization.start": "开始初始化", + "guide.initialization.defaults.message": + "已检测到 {dataNodes} 个可用数据节点（总节点数 {totalNodes}），主分片默认按节点数推荐为 {primaryShards}。", + "guide.initialization.primary_shards": "主分片数", + "guide.initialization.primary_shards.help": + "默认按可用数据节点数计算，通常保持与承载数据的节点数一致即可。", + "guide.initialization.primary_shards.invalid": "请输入大于 0 的主分片数。", + "guide.initialization.auto_expand_replicas": "自动副本", + "guide.initialization.auto_expand_replicas.help": + "默认使用 0-1，可填写 false、all 或形如 0-1 的范围值。", + "guide.initialization.auto_expand_replicas.invalid": + "自动副本格式无效，请输入 false、all 或形如 0-1 的范围值。", + "guide.initialization.rollup": "初始化 Rollup 模板", + "guide.initialization.rollup.help": + "仅 Easysearch 1.12.1 及以上版本支持。关闭后将跳过 Rollup 模板和作业初始化。", + "guide.initialization.task.template_ilm": "初始化模板和 ILM", + "guide.initialization.task.rollup": "初始化 Rollup 模板", + "guide.initialization.task.insight": "初始化仪表盘和可视化模板", + "guide.initialization.task.alerting": "初始化内置告警规则和通道", + "guide.initialization.task.agent": "初始化 Agent 安装模板", + "guide.initialization.task.view": "初始化数据视图模板", + "guide.initialization.task.start": "开始执行：{task}", + "guide.initialization.task.success": "执行成功：{task}", + "guide.initialization.task.failed": "执行失败：{task} {reason}", + "guide.initialization.task.status.pending": "待执行", + "guide.initialization.task.status.running": "执行中", + "guide.initialization.task.status.success": "已完成", + "guide.initialization.task.status.failed": "失败", + "guide.initialization.task.status.skipped": "已跳过", "guide.initialization.step.settings": "设置", "guide.initialization.step.settings.desc": "设置登录的默认用户以及凭据密钥。", @@ -13,22 +42,28 @@ export default { "guide.initialization.step.finish.desc": `配置完成，开启 ${APP_DOMAIN} Console 之旅。`, "guide.cluster.host": "集群地址", "guide.cluster.host.required": "请输入集群地址！", - "guide.cluster.host.validate": "请输入IP地址和端口号！", + "guide.cluster.host.validate": "请输入域名或 IP 地址，可选端口号！", "guide.cluster.auth": "身份验证", "guide.cluster.test.connection": "连接测试", "guide.cluster.test.connection.error.version": - "Elasticsearch 要求 5.3 或更高版本。", + "Easysearch 要求 2.3 或更高版本。", "guide.cluster.test.connection.failed": "连接集群失败。", + "guide.cluster.test.connection.localhost": + "集群地址不能使用本地地址，请填写网关可访问的远程地址。", "guide.cluster.validate.elasticsearch_version_too_old": - "Elasticsearch 版本太旧。", + "Easysearch 版本太旧。", "guide.cluster.validate.elasticsearch_indices_exists": "目标群集中已存在一些相关索引。", "guide.cluster.validate.elasticsearch_template_exists": "目标群集中已存在一些相关模板。", - "guide.cluster.validate.default": "目标群集中已存在一些相关数据。", + "guide.cluster.validate.default": "目标群集中检测到已有数据。", + "guide.cluster.validate.localhost_address": "无法使用本地地址注册集群。", + "guide.cluster.validate.localhost.sub": + "请将集群地址修改为网关可访问的远程地址后重新检测。", "guide.cluster.validate.sub": - "在其他终端工具中执行以下请求可以删除现有数据，但可能会丢失数据。", - "guide.cluster.validate.sub.strong": "[风险自负！]", + "如果确认要继续，可以在其他终端工具中执行以下请求清理现有数据，但可能会丢失数据。", + "guide.cluster.validate.sub.strong": "[请谨慎操作！]", + "guide.cluster.validate.refresh": "重新检测", "guide.cluster.skip": "跳过", "guide.cluster.skip.desc": "您也可以跳过此步骤并重用现有数据。", "guide.user.title": "初始化管理员账户", @@ -45,11 +80,11 @@ export default { "guide.confirm.password.validate": "您输入的两次密码不一致！", "guide.password.strength.invalid": "密码未满足所有安全要求。", "guide.password.rules.title": "密码必须符合以下规则", - "guide.password.rule.length": "长度至少为 8 个字符", + "guide.password.rule.length": "长度至少为 10 个字符", "guide.password.rule.uppercase": "至少一个大写字母 (A-Z)", "guide.password.rule.lowercase": "至少一个小写字母 (a-z)", "guide.password.rule.digit": "至少一个数字 (0-9)", - "guide.password.rule.special": "至少一个特殊字符", + "guide.password.rule.special": "至少一个特殊字符（!@#%^&*_+-=?）", "guide.credential_secret": "凭据密钥", "guide.credential_secret.required": "请输入凭据密钥！", "guide.credential_secret.tips": @@ -67,10 +102,23 @@ export default { "guide.completed": "初始化完成！", "guide.enter.console": `进入 ${APP_DOMAIN} Console`, + "guide.initialization.finish.pending": "正在完成初始化...", + "guide.initialization.finish.pending.desc": "已进入最后一步，您可以先下载配置，初始化完成后再进入 Console。", + "guide.initialization.finish.pending.button": "初始化中", + "guide.initialization.finish.failed": "初始化未完成", + "guide.initialization.finish.failed.desc": "请根据下方错误详情检查配置后重试。", + "guide.initialization.finish.error.invalid_bootstrap_password": + "管理员密码无效，请检查是否满足安全要求后重试。", + "guide.initialization.finish.error.bootstrap_password_strength": + "管理员密码未满足安全要求，请修改后重试。", + "guide.initialization.finish.error.bootstrap_password_required": + "已启用重置管理员账户，请填写管理员密码。", + "guide.initialization.finish.error.bootstrap_username_required": + "已启用重置管理员账户，请填写管理员用户名。", "health.modal.title": "服务受限", "health.modal.desc": - "请检查 Console 相关服务状态，以确保 Console 能正常运行。", + "请点击上方的服务受限，以确保系统集群正常运行。", "health.modal.services.title": "服务状态", "guide.startup.modal.title": `欢迎使用 ${APP_DOMAIN} Console`, diff --git a/web/src/locales/zh-CN/indices.js b/web/src/locales/zh-CN/indices.js index 43274447..4a9086be 100644 --- a/web/src/locales/zh-CN/indices.js +++ b/web/src/locales/zh-CN/indices.js @@ -16,6 +16,19 @@ export default { "indices.show_unavailable_index": "显示不可用索引", "indices.show_unavailable_node": "显示不可用节点", "indices.button.filters": "设置过滤", - "indices.field.name.placeholder": "请输入名称", - "indices.field.name.required_message": "请输入至少五个字符的名称！", + "indices.tab.mappings": "映射", + "indices.tab.edit_settings": "编辑设置", + "indices.hint.edit_json": "编辑后保存您的 JSON", +"indices.field.name.placeholder": "请输入名称", +"indices.field.name.required_message": "请输入至少五个字符的名称！", +"indices.field.name.lowercase_message": "索引名称必须使用小写字母，仅允许字母、数字、点、连字符和下划线", +"indices.delete.modal.title.single": "删除索引", +"indices.delete.modal.title.batch": "删除 {count} 个索引", +"indices.delete.modal.cluster": "即将在集群 {cluster} 中删除以下索引：", +"indices.delete.modal.special_index": "特殊索引", +"indices.delete.modal.special_warning.title": "删除特殊索引可能导致 Console 无法正常工作！", +"indices.delete.modal.special_warning.description": + "特殊索引用于 Console 的内部运行。一旦删除将无法恢复，请确认已经做好备份。", +"indices.delete.modal.special_warning.confirm": "我已了解删除特殊索引的后果", +"indices.delete.modal.description": "已删除的索引无法恢复，请确认已经做好备份。", }; diff --git a/web/src/locales/zh-CN/listview.js b/web/src/locales/zh-CN/listview.js index 3459c924..6ef9b7fc 100644 --- a/web/src/locales/zh-CN/listview.js +++ b/web/src/locales/zh-CN/listview.js @@ -1,4 +1,5 @@ export default { + "listview.filters.placeholder": "筛选", "listview.search.placeholder": "搜索", "listview.search.response.tip": "找到 {total} 条结果 （用时 {took} 毫秒）", "listview.sort": "排序", diff --git a/web/src/locales/zh-CN/overview.js b/web/src/locales/zh-CN/overview.js index 0f4419ca..00ca0bb3 100644 --- a/web/src/locales/zh-CN/overview.js +++ b/web/src/locales/zh-CN/overview.js @@ -6,15 +6,78 @@ export default { "overview.title.message": "消息", "overview.title.cluster": "集群", "overview.title.node": "节点", + "overview.title.index": "索引", "overview.title.host": "主机", "overview.title.disk": "已用存储", "overview.title.quick": "快捷入口", "overview.title.product_activities": "产品动态", "overview.title.cluster_activities": "集群动态", + "overview.info.cluster": "集群信息", + "overview.info.node": "节点信息", + "overview.info.index": "索引信息", + "overview.detail.metrics": "指标", + "overview.detail.infos": "信息", + "overview.column.name": "名称", + "overview.column.version": "版本", + "overview.column.health": "健康", + "overview.column.status": "状态", + "overview.column.nodes": "节点", + "overview.column.indices": "索引", + "overview.column.shards": "分片", + "overview.column.docs": "文档", + "overview.column.store": "存储", + "overview.column.store_size": "存储大小", + "overview.column.replicas": "副本", + "overview.column.document_count": "文档数", + "overview.column.data": "数据", + "overview.column.disk_usage": "磁盘使用率", + "overview.column.jvm_heap": "JVM 堆内存", + "overview.column.transport_address": "传输地址", + "overview.column.host_name": "主机名称", + "overview.column.agent_status": "探针状态", + "overview.column.cpu_usage": "CPU 使用率", + "overview.column.load_average": "负载均值", + "overview.column.disk_free_space": "磁盘剩余空间", + "overview.column.disk_used_space": "磁盘已用空间", + "overview.column.uptime": "在线时长", + "overview.column.timestamp": "时间戳", + "overview.column.index": "索引", + "overview.column.shard": "分片", + "overview.column.prirep": "主/副本", + "overview.column.state": "状态", + "overview.column.ip": "IP", + "overview.column.node": "节点", + "overview.column.cluster": "集群", + "overview.column.pid": "进程ID", + "overview.column.homepath": "安装路径", + "overview.column.endpoint": "端点", + "overview.column.search_rate": "查询速率", + "overview.column.indexing_rate": "索引速率", + "overview.column.indexing_bytes": "索引字节速率", + "overview.column.primary_indexing_rate": "主分片索引速率", + "overview.column.primary_indexing_bytes": "主分片索引字节速率", + "overview.statistic.primary": "主分片", + "overview.statistic.total_shards": "总分片数", + "overview.statistic.updated": "更新时间", + "overview.statistic.type": "类型", + "overview.statistic.master_node": "主节点", + "overview.statistic.not_master_node": "非主节点", + "overview.status.unavailable": "不可用", + "overview.status.available": "可用", + "overview.status.unknown": "未知", + "overview.status.closed": "已关闭", + "overview.status.deleted": "已删除", + "overview.status.index_since": "索引自 {timestamp} 起状态为 {status}", + "overview.status.node_since": "节点自 {timestamp} 起不可用", "overview.message.alert": "告警", "overview.message.notice": "通知", "overview.message.todo": "待办", + "platform.notification.table.title": "标题", + "platform.notification.table.created": "创建时间", + "platform.notification.table.status": "状态", + "platform.notification.status.new": "新通知", + "platform.notification.status.read": "已读", "overview.quick.alert": "告警管理", "overview.quick.dev_tools": "开发工具", @@ -22,5 +85,7 @@ export default { "overview.quick.security": "安全管理", "overview.quick.discover": "数据探索", "overview.quick.monitor": "监控指标", + "overview.quick.migration": "数据迁移", + "overview.quick.comparison": "数据比对", }; diff --git a/web/src/locales/zh-CN/settings.js b/web/src/locales/zh-CN/settings.js index b05b08ea..709b9897 100644 --- a/web/src/locales/zh-CN/settings.js +++ b/web/src/locales/zh-CN/settings.js @@ -1,6 +1,62 @@ export default { "settings.email.server.empty.label1": "您可以在此添加邮件服务器", "settings.email.server.empty.label2": - "告警中心可通指定的邮件服务器向收件人发送通知", + "告警中心可通过指定的邮件服务器向收件人发送通知。", "settings.email.server.empty.button.new": "添加邮件服务器", + "settings.email.server.form.name": "名称", + "settings.email.server.form.host": "主机", + "settings.email.server.form.port": "端口", + "settings.email.server.form.tls_min_version": "TLS 最低版本", + "settings.email.server.form.tls": "TLS", + "settings.email.server.form.sender": "发件人", + "settings.email.server.form.enabled": "启用", + "settings.email.server.form.recipient": "收件人", + "settings.email.server.form.recipient.placeholder": "请输入收件人", + "settings.email.server.form.test.button": "发送测试邮件", + "settings.email.server.form.validation.name": "请输入名称！", + "settings.email.server.form.validation.host": "请输入 SMTP 服务器主机！", + "settings.email.server.form.validation.port": "请输入 SMTP 服务器端口！", + "settings.email.server.form.validation.recipient": "收件人邮箱格式不正确", + "settings.email.server.form.temp_name": "新建配置名称", + "settings.email.server.message.test.success": "发送成功", + "settings.email.server.message.test.error.auth_required": "SMTP 认证信息不能为空，请检查用户名和密码配置。", + "settings.email.server.message.test.error.smtp_auth_failed": + "SMTP 认证失败，请检查用户名、密码或邮箱服务商要求的授权码。", + "settings.email.server.message.test.error.sender_mismatch": + "SMTP 认证失败，部分邮箱服务商要求发件人地址与认证账号一致，或必须是该账号已授权的别名。", + "settings.email.server.message.test.error.tls_required": + "SMTP 服务器要求先启用 TLS/STARTTLS，请检查 TLS 配置和端口是否正确。", + "settings.email.server.message.test.error.send_failed": + "测试邮件发送失败，请检查发件人、收件人、SMTP 配置以及邮箱服务商限制。", + "settings.system.tab.general": "通用设置", + "settings.system.tab.email": "邮件服务器", + "settings.system.retention.title": "数据保留天数", + "settings.system.retention.description": + "设置系统托管数据在被 ILM 删除前保留多少天。", + "settings.system.retention.help": + "默认保留 30 天，默认滚动存储大小为 50 GB。保存后会更新系统托管索引对应的 ILM 保留策略。", + "settings.system.retention.unit": "天", + "settings.system.retention.size.label": "滚动存储大小", + "settings.system.retention.size.unit": "GB", + "settings.system.retention.save": "保存", + "settings.system.retention.update.success": "数据保留天数已更新", + "settings.system.retention.validation.days": "请输入有效的保留天数", + "settings.system.retention.validation.max_size": + "请输入有效的滚动存储大小，单位为 GB，例如 50", + "settings.system.rollup.title": "数据汇聚", + "settings.system.rollup.description": + "可在系统设置中启用或停止系统集群的数据汇聚任务。", + "settings.system.rollup.enabled": "开启", + "settings.system.rollup.disabled": "关闭", + "settings.system.rollup.help": + "关闭数据汇聚会逐个停止现有的数据汇聚任务，并同步关闭集群设置中的 rollup search。", + "settings.system.rollup.update.success": "数据汇聚设置已更新", + "settings.system.advanced.title": "高级设置", + "settings.system.local_templates.title": "本地模板更新", + "settings.system.local_templates.description": + "刷新系统集群中的内置本地配置模板，用于在二进制升级后同步 Agent 与 Gateway 的托管模板内容。", + "settings.system.local_templates.refresh": "更新模板", + "settings.system.local_templates.help": + "会覆盖系统集群中的 system_ingest_config.yml、task_config.tpl、relay.yml 和 migration.yml，并触发实例重新同步。", + "settings.system.local_templates.update.success": "本地模板已更新", }; diff --git a/web/src/models/global.js b/web/src/models/global.js index a7933ad7..10a7ccb2 100644 --- a/web/src/models/global.js +++ b/web/src/models/global.js @@ -14,12 +14,67 @@ import router from "umi/router"; import _ from "lodash"; import { getAuthEnabled, hasAuthority } from "@/utils/authority"; import { formatMessage } from "umi/locale"; +import { getPreferredCluster } from "@/utils/setup"; // import ReactGA from "react-ga"; // ReactGA.initialize("G-L0XH1C4CVP"); const MENU_COLLAPSED_KEY = "search-center:menu:collapsed"; const COUSOLE_VERSION_KEY = "console:version"; +const CLUSTER_STATUS_CACHE_TTL = 60 * 1000; +const CONSOLE_WELCOME_BANNER_STYLE = + "color:#1677ff;font-size:14px;font-weight:700;font-family:Inter,-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;"; + +const mergeUniqueClusters = (existing = [], incoming = []) => { + const seenIDs = new Set(); + const seenUUIDs = new Set(); + const result = []; + + const append = (item) => { + if (!item?.id) { + return; + } + const uuid = `${item.cluster_uuid || ""}`.trim(); + if (seenIDs.has(item.id)) { + return; + } + if (uuid && seenUUIDs.has(uuid)) { + return; + } + seenIDs.add(item.id); + if (uuid) { + seenUUIDs.add(uuid); + } + result.push(item); + }; + + // Prefer fresh data from latest fetch, then backfill old entries that don't conflict. + incoming.forEach(append); + existing.forEach(append); + return result; +}; + +const formatBuildDate = (value) => { + if (!value) { + return value; + } + + const buildDate = new Date(value); + if (Number.isNaN(buildDate.getTime())) { + return value; + } + + return buildDate.toLocaleString(undefined, { + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + timeZoneName: "short", + }); +}; export default { namespace: "global", @@ -30,6 +85,8 @@ export default { notices: [], clusterVisible: true, clusterList: [], + clusterStatus: {}, + clusterStatusFetchedAt: 0, selectedCluster: {}, selectedClusterID: null, search: { @@ -86,97 +143,105 @@ export default { }); }, *fetchClusterList({ payload }, { call, put, select, take }) { - let res = yield call(searchClusterConfig, payload); - if (res.error) { - message.error(res.error); - return false; - } - res = formatESSearchResult(res); - let { clusterList, search, selectedClusterID } = yield select( - (state) => state.global - ); - let data = res.data - .filter((item) => item.enabled) - .map((item) => { - return { + try { + let res = yield call(searchClusterConfig, payload); + + if (!res) { + message.error("No response from cluster service"); + return false; + } + + if (res.error) { + message.error(String("Error: " + (res.error.reason || res.error.message || res.error))); + return false; + } + + res = formatESSearchResult(res); + + let { clusterList, search, selectedClusterID } = yield select( + (state) => state.global + ); + + let data = res.data + .filter((item) => item.enabled) + .map((item) => ({ ...item, distribution: item.distribution || "elasticsearch", cluster_uuid: item.cluster_uuid || "", - }; - }); + })); - if (clusterList.length === 0 && !payload.name) { - if (data.length === 0 && location.href.indexOf("user/login") === -1) { - if (getAuthEnabled() && !hasAuthority("system.cluster:all")) { + if (clusterList.length === 0 && !payload.name) { + if (data.length === 0 && location.href.indexOf("user/login") === -1) { + if (getAuthEnabled() && !hasAuthority("system.cluster:all")) { + Modal.info({ + title: formatMessage({ id: "app.message.system-tips" }), + content: formatMessage({ + id: + "app.message.system-tips.no-available-cluster-data-permission", + }), + okText: formatMessage({ id: "form.button.ok" }), + }); + return; + } Modal.info({ title: formatMessage({ id: "app.message.system-tips" }), content: formatMessage({ id: - "app.message.system-tips.no-available-cluster-data-permission", + "app.message.system-tips.no-available-cluster-redirect-setting", }), okText: formatMessage({ id: "form.button.ok" }), - onOk() {}, + onOk() { + router.push("/resource/cluster"); + }, }); - return; } - Modal.info({ - title: formatMessage({ id: "app.message.system-tips" }), - content: formatMessage({ - id: - "app.message.system-tips.no-available-cluster-redirect-setting", - }), - okText: formatMessage({ id: "form.button.ok" }), - onOk() { - router.push("/resource/cluster"); - }, - }); } - } - if (!selectedClusterID) { - const targetID = extractClusterIDFromURL(); - let idx = data.findIndex((item) => { - return item.id == targetID; - }); - // idx = idx > -1 ? idx : 0; - if (idx == -1) { - let cstatus = yield put({ - type: "fetchClusterStatus", - }); - yield take("fetchClusterStatus/@@end"); - let { clusterStatus } = yield select((state) => state.global); - idx = data.findIndex((item) => { - return clusterStatus[item.id]?.available; + + if (!selectedClusterID) { + const targetID = extractClusterIDFromURL(); + let nextSelectedCluster = getPreferredCluster(data, { + targetClusterID: targetID, }); - if (idx == -1) { - idx = 0; + + if (!nextSelectedCluster || (targetID && nextSelectedCluster.id !== targetID)) { + yield put({ type: "fetchClusterStatus" }); + yield take("fetchClusterStatus/@@end"); + let { clusterStatus } = yield select((state) => state.global); + const availableCluster = data.find( + (item) => clusterStatus[item.id]?.available + ); + nextSelectedCluster = + nextSelectedCluster || availableCluster || data[0]; } + + yield put({ + type: "saveData", + payload: { + selectedCluster: nextSelectedCluster, + selectedClusterID: nextSelectedCluster?.id, + }, + }); } + + let newClusterList = + search.name !== payload.name + ? data + : mergeUniqueClusters(clusterList, data); + yield put({ type: "saveData", payload: { - selectedCluster: data[idx], - selectedClusterID: (data[idx] || {}).id, + clusterList: newClusterList, + clusterTotal: res.total, + search: { ...search, cluster: payload }, }, }); + + return data; + } catch (err) { + message.error(err.message || "Unknown error occurred while fetching clusters"); + return false; } - let newClusterList = []; - if (search.name != payload.name) { - newClusterList = data; - } else { - newClusterList = clusterList.concat(data); - } - yield put({ - type: "saveData", - payload: { - clusterList: newClusterList, - clusterTotal: res.total, - search: { - ...search, - cluster: payload, - }, - }, - }); - return data; }, *reloadClusterList({ payload }, { call, put, select }) { yield put({ @@ -203,6 +268,17 @@ export default { if (pathname.startsWith("/exception")) { return; } + + const dataToolsNewMatch = pathname.match( + /^\/data_tools\/(migration|comparison)\/new(?:\/elasticsearch\/[^/]+\/?)?$/ + ); + if (dataToolsNewMatch) { + const normalizedPath = `/data_tools/${dataToolsNewMatch[1]}/new`; + if (pathname !== normalizedPath) { + history.replace(normalizedPath + (search || "")); + } + return; + } const global = yield select((state) => state.global); if (pathname && global.selectedClusterID) { @@ -234,21 +310,38 @@ export default { if (location.href.indexOf("#/user/login") > -1) { return false; } - let res = yield call(getClusterStatus, payload); + const options = payload || {}; + const { force = false, maxAge = CLUSTER_STATUS_CACHE_TTL } = options; + const { clusterStatus, clusterStatusFetchedAt } = yield select( + (state) => state.global + ); + + if ( + !force && + clusterStatusFetchedAt > 0 && + Date.now() - clusterStatusFetchedAt < maxAge + ) { + return clusterStatus; + } + + let res = yield call(getClusterStatus); if (!res) { return false; } - const { clusterStatus } = yield select((state) => state.global); if (res.error) { console.log(res.error); return false; } + const nextPayload = { + clusterStatusFetchedAt: Date.now(), + }; if (!_.isEqual(res, clusterStatus)) { + nextPayload.clusterStatus = res; + } + if (Object.keys(nextPayload).length > 0) { yield put({ type: "saveData", - payload: { - clusterStatus: res, - }, + payload: nextPayload, }); } return res; @@ -271,11 +364,13 @@ export default { } //please do not delete - console.log(`Welcome to ${APP_TITLE}!`); - console.log("version.number:", data?.application?.version?.number); - console.log("version.build_number:", data?.application?.version?.build_number); - console.log("version.build_hash:", data?.application?.version?.build_hash); - console.log("version.build_date:", data?.application?.version?.build_date); + console.log(`%cWelcome to ${APP_TITLE}!`, CONSOLE_WELCOME_BANNER_STYLE); + console.log("version:", data?.application?.version?.number); + console.log("build_number:", data?.application?.version?.build_number); + console.log("build_hash:", data?.application?.version?.build_hash); + console.log("build_date:", + formatBuildDate(data?.application?.version?.build_date) + ); } else { console.log("fetch console info failed, ", data); return false; @@ -416,6 +511,8 @@ export default { "/guide", "/resource", "/platform/notification", + "/data_tools/migration", + "/data_tools/comparison", ]; if (clusterHiddenPath.some((p) => pathname.startsWith(p))) { clusterVisible = false; diff --git a/web/src/models/login.js b/web/src/models/login.js index d9adbec7..24e52241 100644 --- a/web/src/models/login.js +++ b/web/src/models/login.js @@ -1,29 +1,140 @@ import { routerRedux } from "dva/router"; import { stringify } from "qs"; -import { fakeAccountLogin, getFakeCaptcha } from "@/services/api"; -import { setAuthority } from "@/utils/authority"; +import { + fakeAccountLogin, + fakeAccountLogout, + getAccountLoginChallenge, + getFakeCaptcha, +} from "@/services/api"; +import { + getAuthEnabled, + setAuthority, + syncAuthorityFromResponse, +} from "@/utils/authority"; import { getPageQuery } from "@/utils/utils"; import { reloadAuthorized } from "@/utils/Authorized"; import * as CurrentUser from "@/utils/CurrentUser"; -import { getAuthEnabled } from "@/utils/authority"; +import { buildPasswordProof } from "@/utils/password"; +import { + clearStoredLoginResponse, + storeLoginResponse, +} from "@/utils/auth_session"; +import { formatMessage } from "umi/locale"; + +let logoutInProgress = false; + +const invalidCredentialReasons = [ + "invalid login or password", + "invalid username or password", + "authentication_exception", + "user not found", +]; + +const getLoginErrorMessage = (response) => { + const reason = response?.error?.reason || response?.message || ""; + const normalizedReason = String(reason || "").toLowerCase(); + + if (normalizedReason === "invalid bootstrap password") { + return formatMessage({ + id: "guide.initialization.finish.error.invalid_bootstrap_password", + }); + } + + if ( + [401, 403].includes(response?.httpStatus) || + invalidCredentialReasons.some((item) => normalizedReason.includes(item)) + ) { + return formatMessage({ id: "app.login.message-invalid-credentials" }); + } + + if (response?.errorObject?.key) { + return formatMessage({ + id: response.errorObject.key, + defaultMessage: + reason || formatMessage({ id: "app.login.message-invalid-credentials" }), + }); + } + + return reason || formatMessage({ id: "app.login.message-invalid-credentials" }); +}; + +const normalizeLoginResponse = (response, type) => { + if (response?.status === "ok") { + return response; + } + + return { + ...(response && typeof response === "object" ? response : {}), + status: "error", + type, + message: getLoginErrorMessage(response), + }; +}; export default { namespace: "login", state: { status: undefined, + message: "", }, effects: { *login({ payload }, { call, put }) { - const response = yield call(fakeAccountLogin, payload); + const username = payload.username || payload.userName; + const passwordPayload = { + userName: username, + password: payload.password, + type: payload.type, + }; + let loginPayload = null; + let response; + + try { + const challenge = yield call(getAccountLoginChallenge, { username }); + + if (challenge?.status === "ok" && challenge?.method === "challenge") { + const proof = yield call(buildPasswordProof, { + password: payload.password, + username, + challengeId: challenge.challenge_id, + nonce: challenge.nonce, + salt: challenge.salt, + iterations: challenge.iterations, + }); + loginPayload = { + userName: username, + type: payload.type, + challenge_id: challenge.challenge_id, + proof, + }; + } else if (challenge?.status === "ok" && challenge?.method === "plain") { + // Keep plaintext fallback only when the backend explicitly asks for it. + loginPayload = passwordPayload; + } else { + throw new Error( + challenge?.error?.reason || challenge?.message || "unsupported login challenge response" + ); + } + + response = yield call(fakeAccountLogin, loginPayload); + } catch (error) { + response = { + status: "error", + error: { + reason: error?.message || "", + }, + }; + } + + response = normalizeLoginResponse(response, payload.type); yield put({ type: "changeLoginStatus", payload: response, }); // Login successfully if (response.status === "ok") { - setAuthority(response.privilege); + syncAuthorityFromResponse(response); reloadAuthorized(); if (getAuthEnabled()) { yield put({ @@ -33,7 +144,7 @@ export default { }, }); } - localStorage.setItem("login-response", JSON.stringify(response)); + storeLoginResponse(response); const urlParams = new URL(window.location.href); const params = getPageQuery(); let { redirect } = params; @@ -57,41 +168,55 @@ export default { yield call(getFakeCaptcha, payload); }, - *logout(_, { put }) { - yield put({ - type: "changeLoginStatus", - payload: { - status: false, - currentAuthority: "guest", - }, - }); - localStorage.removeItem("login-response"); - reloadAuthorized(); - //clear selected cluster state - yield put({ - type: "global/saveData", - payload: { - selectedClusterID: null, - }, - }); - yield put( - routerRedux.push({ - pathname: "/user/login", - search: stringify({ - redirect: window.location.href, - }), - }) - ); + *logout({ payload = {} }, { call, put }) { + if (logoutInProgress) { + return; + } + logoutInProgress = true; + try { + if (payload.skipServerLogout !== true) { + yield call(fakeAccountLogout); + } + yield put({ + type: "changeLoginStatus", + payload: { + status: false, + currentAuthority: "guest", + }, + }); + clearStoredLoginResponse(); + reloadAuthorized(); + //clear selected cluster state + yield put({ + type: "global/saveData", + payload: { + selectedClusterID: null, + }, + }); + yield put( + routerRedux.push({ + pathname: "/user/login", + search: stringify({ + redirect: window.location.href, + }), + }) + ); + } finally { + logoutInProgress = false; + } }, }, reducers: { changeLoginStatus(state, { payload }) { - setAuthority(payload.currentAuthority); + if (typeof payload?.currentAuthority !== "undefined") { + setAuthority(payload.currentAuthority); + } return { ...state, status: payload.status, type: payload.type, + message: payload.message || "", }; }, }, diff --git a/web/src/models/user.js b/web/src/models/user.js index a3c716d7..b2a50b65 100644 --- a/web/src/models/user.js +++ b/web/src/models/user.js @@ -1,5 +1,20 @@ import { query as queryUsers, queryCurrent } from "@/services/user"; -import { setCurrentUser, getCurrentUser } from "@/utils/CurrentUser"; +import { setCurrentUser } from "@/utils/CurrentUser"; +import { reloadAuthorized } from "@/utils/Authorized"; +import { syncAuthorityFromResponse } from "@/utils/authority"; + +function normalizeCurrentUser(payload) { + const source = payload?._source || payload; + if (!source || typeof source !== "object") { + return {}; + } + + return { + ...source, + user_id: source.user_id || source.id || payload?._id || payload?.id, + nick_name: source.nick_name || source.name, + }; +} export default { namespace: "user", @@ -22,10 +37,15 @@ export default { }, *fetchCurrent(_, { call, put }) { const response = yield call(queryCurrent); + if (response && !response.error) { + syncAuthorityFromResponse(response); + reloadAuthorized(); + } yield put({ type: "saveCurrentUser", payload: response, }); + return response; }, }, @@ -37,16 +57,15 @@ export default { }; }, saveCurrentUser(state, action) { + const currentUser = normalizeCurrentUser(action.payload); //update localStorage - if (action.payload && action.payload._source) { - setCurrentUser(action.payload._source); - } + setCurrentUser(currentUser); return { ...state, currentUser: { ...state.currentUser, - ...(action.payload._source || {}), + ...currentUser, }, }; }, diff --git a/web/src/pages/.gitignore b/web/src/pages/.gitignore new file mode 100644 index 00000000..182a047d --- /dev/null +++ b/web/src/pages/.gitignore @@ -0,0 +1 @@ +DataTools \ No newline at end of file diff --git a/web/src/pages/Account/Settings/BaseView.js b/web/src/pages/Account/Settings/BaseView.js index f191d331..04f0b7d4 100644 --- a/web/src/pages/Account/Settings/BaseView.js +++ b/web/src/pages/Account/Settings/BaseView.js @@ -12,7 +12,7 @@ const { Option } = Select; // 头像组件 方便以后独立，增加裁剪之类的功能 const AvatarView = ({ avatar }) => ( - +
+ + + + {formatMessage({ id: "app.settings.security.password-description" })} ：{passwordStrength.strong} @@ -94,7 +94,7 @@ class SecurityView extends Component { render() { return ( - + { @@ -33,14 +34,31 @@ export const ExtraStep = Form.create({ name: "instance_step_edit" })( return ( <> - + {initialValue?.endpoint} - + {initialValue?.version.number} - - {initialValue?.status} + + diff --git a/web/src/pages/Agent/Instance/Step/initial_step.jsx b/web/src/pages/Agent/Instance/Step/initial_step.jsx index 1a4be329..872e861f 100644 --- a/web/src/pages/Agent/Instance/Step/initial_step.jsx +++ b/web/src/pages/Agent/Instance/Step/initial_step.jsx @@ -1,35 +1,149 @@ -import { Form, Input, Switch, Icon } from "antd"; +import { + Form, + Input, + Switch, + Icon, + Divider, + Spin, + Tooltip, + Radio, +} from "antd"; +import React from "react"; import { formatMessage } from "umi/locale"; -import { isTLS, removeHttpSchema } from "@/utils/utils"; +import request from "@/utils/request"; +import { + isTLS, + isValidEndpointHost, + normalizeEndpointHost, + removeHttpSchema, +} from "@/utils/utils"; + +const AUTH_TYPE_ACCESS_TOKEN = "access_token"; +const AUTH_TYPE_BASIC_AUTH = "basic_auth"; + +const renderLabel = (labelId, tip) => { + const label = formatMessage({ + id: labelId, + }); + if (!tip) { + return label; + } + return ( + + {label} + + + + + ); +}; @Form.create() export class InitialStep extends React.Component { constructor(props) { super(props); + const hasAccessToken = !!props.initialValue?.access_token; + const hasBasicAuth = !!props.initialValue?.basic_auth?.username; this.state = { - needAuth: props.initialValue?.basic_auth !== undefined, isPageTLS: isTLS(props.initialValue?.endpoint), + preparingRegistration: false, + needAuth: hasAccessToken || hasBasicAuth, + authType: hasBasicAuth ? AUTH_TYPE_BASIC_AUTH : AUTH_TYPE_ACCESS_TOKEN, }; } - handleAuthChange = (val) => { + + componentDidMount() { + this.syncRegistrationInfo(this.props.initialValue); + if (!this.props.initialValue?.registration_id) { + this.prepareRegistration(); + } + } + + syncRegistrationInfo = (values = {}) => { + if (typeof this.props.onRegistrationInfoChange === "function") { + this.props.onRegistrationInfoChange({ + console_endpoint: values?.console_endpoint, + manager_token: values?.manager_token, + registration_expired_at: values?.registration_expired_at, + }); + } + }; + + prepareRegistration = async () => { this.setState({ - needAuth: val, + preparingRegistration: true, + }); + const res = await request("/instance/_prepare_registration", { + method: "POST", + }); + this.setState({ + preparingRegistration: false, + }); + if (res?.error) { + return; + } + this.props.form.setFieldsValue({ + registration_id: res.id, + registration_expired_at: res.expired_at, + console_endpoint: res.endpoint, + manager_token: res.token, + }); + this.syncRegistrationInfo({ + console_endpoint: res.endpoint, + manager_token: res.token, + registration_expired_at: res.expired_at, }); }; + handleEndpointChange = (event) => { const val = event.target.value; this.setState({ - isPageTLS: isTLS(val) - }) + isPageTLS: isTLS(val), + }); }; + isPageTLSChange = (val) => { this.setState({ isPageTLS: val, }); }; + + handleAuthChange = (val) => { + this.setState({ + needAuth: val, + }); + if (!val) { + this.props.form.setFieldsValue({ + access_token: "", + "basic_auth.username": undefined, + "basic_auth.password": undefined, + }); + } + }; + + handleAuthTypeChange = (event) => { + const nextType = event.target.value; + this.setState({ + authType: nextType, + }); + if (nextType === AUTH_TYPE_ACCESS_TOKEN) { + this.props.form.setFieldsValue({ + "basic_auth.username": undefined, + "basic_auth.password": undefined, + }); + return; + } + this.props.form.setFieldsValue({ + access_token: "", + }); + }; + render() { const { - form: { getFieldDecorator }, + form: { getFieldDecorator, getFieldValue }, initialValue, } = this.props; const formItemLayout = { @@ -42,98 +156,223 @@ export class InitialStep extends React.Component { sm: { span: 16 }, }, }; + const agentAccessTokenTip = [ + formatMessage({ + id: "agent.instance.registration.agent.token.tip", + }), + formatMessage({ + id: "agent.instance.registration.agent.token.expire.tip", + }), + ].join(" "); return ( - - - {getFieldDecorator("endpoint", { - initialValue: removeHttpSchema(initialValue?.endpoint || ""), - normalize: (value) => { - return removeHttpSchema(value || "").trim() - }, - validateTrigger: ["onChange", "onBlur"], - rules: [ - { - required: true, - message: formatMessage({ - id: "gateway.instance.field.endpoint.form.required", - }), - }, - { - type: "string", - pattern: /^[\w\.\-_~%]+(\:\d+)?\s*$/, //(https?:\/\/)? - message: formatMessage({ - id: "cluster.regist.form.verify.valid.endpoint", - }), + + + {getFieldDecorator("registration_id", { + initialValue: initialValue?.registration_id, + })()} + {getFieldDecorator("registration_expired_at", { + initialValue: initialValue?.registration_expired_at, + })()} + {getFieldDecorator("console_endpoint", { + initialValue: initialValue?.console_endpoint, + })()} + {getFieldDecorator("manager_token", { + initialValue: initialValue?.manager_token, + })()} + + + {formatMessage({ + id: "agent.instance.registration.agent.title", + })} + + + + {getFieldDecorator("endpoint", { + initialValue: removeHttpSchema(initialValue?.endpoint || ""), + normalize: (value) => { + return normalizeEndpointHost(value); }, - ], - })()} - - - {getFieldDecorator("isTLS", { - initialValue: isTLS(initialValue?.endpoint), - })( - } - unCheckedChildren={} - checked={this.state.isPageTLS} - onChange={this.isPageTLSChange} - /> - )} - - - } - unCheckedChildren={} - /> - - {this.state.needAuth === true ? ( -
- - {getFieldDecorator("basic_auth.username", { - initialValue: initialValue?.basic_auth?.username || "", - rules: [ - { - required: true, - message: "Please input auth username!", - }, - ], - })()} - - - {getFieldDecorator("basic_auth.password", { - initialValue: initialValue?.basic_auth?.password || "", - rules: [ - { - required: true, - message: "Please input auth password!", + validateTrigger: ["onChange", "onBlur"], + rules: [ + { + required: true, + message: formatMessage({ + id: "agent.instance.field.endpoint.form.required", + }), + }, + { + validator: (rule, value, callback) => { + if (!value || isValidEndpointHost(value)) { + callback(); + return; + } + callback(formatMessage({ + id: "cluster.regist.form.verify.valid.endpoint", + })); }, - ], - })( - + }, + ], + })( + + )} + + + + {getFieldDecorator("isTLS", { + initialValue: isTLS(initialValue?.endpoint), + })( + } + unCheckedChildren={} + checked={this.state.isPageTLS} + onChange={this.isPageTLSChange} + /> + )} + + + + {getFieldDecorator("isAuth", { + initialValue: this.state.needAuth, + valuePropName: "checked", + })( + } + unCheckedChildren={} + /> + )} + + + {this.state.needAuth ? ( + <> + + {getFieldDecorator("auth_type", { + initialValue: this.state.authType, + })( + + + {formatMessage({ + id: "agent.instance.registration.auth.type.access_token", + })} + + + {formatMessage({ + id: "agent.instance.registration.auth.type.basic_auth", + })} + + + )} + + + {this.state.authType === AUTH_TYPE_ACCESS_TOKEN ? ( + + {getFieldDecorator("access_token", { + initialValue: initialValue?.access_token || "", + rules: [ + { + required: true, + message: formatMessage({ + id: "agent.instance.registration.agent.token.required", + }), + }, + ], + })( + + )} + + ) : ( + <> + + {getFieldDecorator("basic_auth.username", { + initialValue: initialValue?.basic_auth?.username || "", + rules: [ + { + required: true, + message: formatMessage({ + id: "cluster.regist.form.verify.required.auth_username", + }), + }, + ], + })( + + )} + + + {getFieldDecorator("basic_auth.password", { + initialValue: initialValue?.basic_auth?.password || "", + rules: [ + { + required: true, + message: formatMessage({ + id: "cluster.regist.form.verify.required.auth_password", + }), + }, + ], + })( + + )} + + )} - -
- ) : null} - + + ) : null} + + ); } } diff --git a/web/src/pages/Agent/Instance/Step/result_step.jsx b/web/src/pages/Agent/Instance/Step/result_step.jsx index a69223b6..d1fbe611 100644 --- a/web/src/pages/Agent/Instance/Step/result_step.jsx +++ b/web/src/pages/Agent/Instance/Step/result_step.jsx @@ -36,7 +36,10 @@ export const ResultStep = (props) => {
- Endpoint ： + {formatMessage({ + id: "gateway.instance.field.endpoint.label", + })} + ： {removeHttpSchema(instanceConfig?.endpoint)} @@ -44,7 +47,10 @@ export const ResultStep = (props) => { - TLS： + {formatMessage({ + id: "gateway.instance.field.tls.label", + })} + ： {formatMessage({ @@ -57,12 +63,16 @@ export const ResultStep = (props) => { ); const actions = ( - + ); diff --git a/web/src/pages/Agent/Instance/components/AgentCredential.jsx b/web/src/pages/Agent/Instance/components/AgentCredential.jsx index 71ab3b1a..de248c25 100644 --- a/web/src/pages/Agent/Instance/components/AgentCredential.jsx +++ b/web/src/pages/Agent/Instance/components/AgentCredential.jsx @@ -1,5 +1,5 @@ import { Alert, Button, Form, message } from "antd"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { formatMessage } from "umi/locale"; import request from "@/utils/request"; @@ -25,25 +25,32 @@ export default Form.create()((props) => { const needAuth = !!(record.credential_id || record.basic_auth?.username); + useEffect(() => { + setIsManual(!record.agent_credential_id && !!record.agent_basic_auth?.username); + }, [record.agent_credential_id, record.agent_basic_auth?.username]); + const onConfirm = async () => { form.validateFields(async (errors, values) => { if (errors) return; setSaveLoading(true); const { credential_id, basic_auth, metric_collection_mode } = record; + const isManualCredential = values.agent_credential_id === MANUAL_VALUE; + const manualAuth = isManualCredential + ? { + username: values.agent_username, + password: values.agent_password, + } + : undefined; const res = await request(`${ESPrefix}/${record.id}`, { method: "PUT", body: { credential_id, basic_auth, metric_collection_mode, - agent_credential_id: - values.agent_credential_id !== MANUAL_VALUE - ? values.agent_credential_id - : undefined, - agent_basic_auth: { - username: values.agent_username, - password: values.agent_password, - }, + agent_credential_id: isManualCredential + ? undefined + : values.agent_credential_id, + agent_basic_auth: manualAuth, }, }); if (res?.result === "updated") { @@ -52,20 +59,42 @@ export default Form.create()((props) => { id: "app.message.update.success", }) ); - const res = await request(`/elasticsearch/${record.id}`); - if (res?.found) { - onAgentCredentialSave(res._source); - if (res._source?.agent_credential_id) { + const latestRecordResponse = await request(`/elasticsearch/${record.id}`); + if (latestRecordResponse?.found) { + const latestSource = latestRecordResponse._source || {}; + const nextRecord = { + ...record, + ...latestSource, + id: record.id, + agent_credential_id: isManualCredential + ? undefined + : latestSource.agent_credential_id || values.agent_credential_id, + agent_basic_auth: isManualCredential + ? { + username: + latestSource.agent_basic_auth?.username || + manualAuth?.username, + // API read may not return password; keep freshly saved password for test connect. + password: + latestSource.agent_basic_auth?.password || + manualAuth?.password, + } + : latestSource.agent_basic_auth, + }; + onAgentCredentialSave(nextRecord); + if (nextRecord?.agent_credential_id) { setIsManual(false); } form.setFieldsValue({ - agent_credential_id: res._source?.agent_credential_id - ? res._source?.agent_credential_id - : res._source?.agent_basic_auth?.username + agent_credential_id: nextRecord?.agent_credential_id + ? nextRecord?.agent_credential_id + : nextRecord?.agent_basic_auth?.username ? MANUAL_VALUE : undefined, - agent_username: res._source.agent_basic_auth?.username, - agent_password: res._source.agent_basic_auth?.password, + agent_username: nextRecord.agent_basic_auth?.username, + agent_password: isManualCredential + ? manualAuth?.password + : nextRecord.agent_basic_auth?.password, }); } } else { diff --git a/web/src/pages/Agent/Instance/components/AgentCredentialForm.jsx b/web/src/pages/Agent/Instance/components/AgentCredentialForm.jsx index 5d5041b1..5cfa68a8 100644 --- a/web/src/pages/Agent/Instance/components/AgentCredentialForm.jsx +++ b/web/src/pages/Agent/Instance/components/AgentCredentialForm.jsx @@ -1,9 +1,10 @@ import React, { useEffect, useMemo, useState } from "react"; -import { Button, Divider, Form, Input, Select, Row, Col } from "antd"; +import { Button, Form, Input, Select, Tooltip } from "antd"; import { formatMessage } from "umi/locale"; import useFetch from "@/lib/hooks/use_fetch"; import { formatESSearchResult } from "@/lib/elasticsearch/util"; +import { hasAuthority } from "@/utils/authority"; export const MANUAL_VALUE = "manual"; @@ -19,8 +20,11 @@ export default (props) => { isManual, setIsManual, } = props; + const canReadCredential = + hasAuthority("system.credential:all") || + hasAuthority("system.credential:read"); - const { loading, error, value, run } = useFetch( + const { loading, value, run } = useFetch( "/credential/_search", { queryParams: { @@ -39,9 +43,66 @@ export default (props) => { } }; - const { data, total } = useMemo(() => { + const { data } = useMemo(() => { return formatESSearchResult(value); }, [value]); + const credentialGroupStyle = { + display: "flex", + alignItems: "center", + minWidth: 0, + }; + const credentialSelectWrapStyle = { + flex: 1, + minWidth: 0, + }; + const refreshButtonSize = 32; + const refreshButtonStyle = { + width: refreshButtonSize, + minWidth: refreshButtonSize, + height: refreshButtonSize, + padding: 0, + marginLeft: -1, + borderTopLeftRadius: 0, + borderBottomLeftRadius: 0, + zIndex: 1, + display: "flex", + alignItems: "center", + justifyContent: "center", + }; + const refreshButtonWrapStyle = { + display: "flex", + alignItems: "center", + flex: `0 0 ${refreshButtonSize}px`, + }; + + const credentialOptions = useMemo(() => { + const options = data.map((item) => ({ + id: item.id, + name: item.name, + })); + if ( + initialValue?.agent_credential_id && + !options.find((item) => item.id === initialValue.agent_credential_id) + ) { + options.unshift({ + id: initialValue.agent_credential_id, + name: initialValue.agent_credential_id, + }); + } + return options; + }, [data, initialValue?.agent_credential_id]); + + useEffect(() => { + if (canReadCredential) { + run(); + } + }, [canReadCredential, run]); + + useEffect(() => { + if (canReadCredential && initialValue?.agent_credential_id) { + run(); + } + }, [canReadCredential, initialValue?.agent_credential_id, run]); if (!needAuth) { return null; @@ -69,16 +130,35 @@ export default (props) => { }, ], })( - +
+
+ +
+
+ + +
+
)} {isManual && ( diff --git a/web/src/pages/Agent/Instance/components/Associate.jsx b/web/src/pages/Agent/Instance/components/Associate.jsx index d93cb494..cc38d7da 100644 --- a/web/src/pages/Agent/Instance/components/Associate.jsx +++ b/web/src/pages/Agent/Instance/components/Associate.jsx @@ -1,11 +1,36 @@ import { useGlobal } from "@/layouts/GlobalContext"; import request from "@/utils/request"; -import { Form, Input, Switch, Icon, Button, Select } from "antd"; +import { Form, Input, Switch, Icon, Button, Select, Alert } from "antd"; import { useMemo, useRef, useState } from "react"; import { Link, router } from "umi"; import { formatMessage } from "umi/locale"; import CredentialForm from "../../../System/Cluster/CredentialForm"; +const normalizeLogsPaths = (items = []) => + Array.from( + new Set( + items + .reduce((result, item) => { + if (Array.isArray(item)) { + return result.concat(item); + } + result.push(item); + return result; + }, []) + .map((item) => `${item || ""}`.trim()) + .filter(Boolean) + ) + ); + +const getDetectedLogsPaths = (data = {}) => + normalizeLogsPaths([ + data?.logs_paths, + data?.path_logs, + data?.node_info?.logs_paths, + data?.node_info?.path_logs, + data?.node_info?.settings?.path?.logs, + ]); + export const Associate = Form.create({ name: "associate_form" })((props) => { const formItemLayout = { labelCol: { @@ -150,7 +175,11 @@ export const Associate = Form.create({ name: "associate_form" })((props) => { ], })()} - + {getFieldDecorator("isTLS", { initialValue: record?.schema === "https", valuePropName: "checked", @@ -183,8 +212,10 @@ export const Associate = Form.create({ name: "associate_form" })((props) => { isManual={state.isManual} /> - @@ -196,21 +227,30 @@ export const Associate = Form.create({ name: "associate_form" })((props) => { const Result = ({ data, onComplete, loading = false }) => { const { clusterList = [] } = useGlobal(); - const clusters = useMemo( - () => { - if (!data.cluster_info.cluster_uuid) { - return []; - } - return clusterList.filter((item) => { - return item.cluster_uuid == data.cluster_info.cluster_uuid; + const detectedLogsPaths = useMemo(() => getDetectedLogsPaths(data), [data]); + const detectedLogsPath = detectedLogsPaths.join(", "); + const clusters = useMemo(() => { + const clusterUUID = `${data?.cluster_info?.cluster_uuid || ""}`.trim(); + if (!clusterUUID) { + return []; + } + const clusterName = `${data?.cluster_info?.cluster_name || ""}`.trim(); + const toTs = (item) => new Date(item?.updated || item?.created || 0).getTime() || 0; + return (clusterList || []) + .filter((item) => `${item?.cluster_uuid || ""}`.trim() === clusterUUID) + .sort((a, b) => { + const aName = `${a?.name || a?.raw_name || ""}`.trim(); + const bName = `${b?.name || b?.raw_name || ""}`.trim(); + const aNameMatched = clusterName && aName === clusterName ? 1 : 0; + const bNameMatched = clusterName && bName === clusterName ? 1 : 0; + if (aNameMatched !== bNameMatched) { + return bNameMatched - aNameMatched; + } + return toTs(b) - toTs(a); }); - }, - clusterList, - data.cluster_info.cluster_uuid - ); - const selectedID = clusters[0]?.id || ""; + }, [clusterList, data?.cluster_info?.cluster_uuid, data?.cluster_info?.cluster_name]); + const selectedID = useMemo(() => clusters[0]?.id || "", [clusters]); const clusterRef = useRef(); - const onAssociateClick = () => { if (typeof onComplete === "function") { const clusterID = clusterRef.current.rcSelect.state?.value[0] || ""; @@ -222,7 +262,6 @@ const Result = ({ data, onComplete, loading = false }) => { publish_address: data?.node_info?.http?.publish_address, node_name: data?.node_info?.name, path_home: data?.node_info?.settings?.path?.home, - path_logs: data?.node_info?.settings?.path?.logs, // credential_id:values.credential_id, }); } @@ -265,7 +304,7 @@ const Result = ({ data, onComplete, loading = false }) => {
Node UUID：{data?.id}
Publish Address：{data?.node_info?.http?.publish_address}
Path Home：{data?.node_info?.settings?.path?.home}
-
Path Log：{data?.node_info?.settings?.path?.logs}
+
Path Log：{detectedLogsPath}
@@ -317,15 +356,17 @@ const Result = ({ data, onComplete, loading = false }) => { ) : ( - - {formatMessage({ + + /> )}
+ + + ); + } + return ( +
+ + {currentInterval > 0 + ? `${currentInterval} ${formatMessage({ id: "agent.instance.collection_interval.unit" })}` + : `10 ${formatMessage({ id: "agent.instance.collection_interval.unit" })}`} + + {hasAuthority("agent.instance:all") && ( + onIntervalEdit(clusterID, currentInterval || null)} + /> + )} +
+ ); + }, + }, + { + title: formatMessage({ id: "table.field.actions" }), + width: 140, render: (text, record) => ( -
+
{/* onDeleteClick(record.id, agentID)} @@ -219,7 +367,9 @@ export const AgentRowDetail = ({ agentID, t }) => { hasAuthority("agent.instance:all") && ( <> onRevoke({ cluster_id: record.cluster_id, @@ -228,15 +378,17 @@ export const AgentRowDetail = ({ agentID, t }) => { }) } > - ) } - { onDetailClick(record.id, record.cluster_id); }} @@ -244,10 +396,12 @@ export const AgentRowDetail = ({ agentID, t }) => { {formatMessage({ id: "agent.instance.table.operation.detail", })} - + ) : ( - { setState((st) => { return { @@ -257,17 +411,17 @@ export const AgentRowDetail = ({ agentID, t }) => { }; }); }} - > - {formatMessage({ - id: "agent.instance.table.operation.associate", - })} - + > + {formatMessage({ + id: "agent.instance.table.operation.associate", + })} + )}
), }, ], - [agentID] + [agentID, btnLoading, t, intervalEdit, onIntervalEdit, onIntervalSave, onIntervalCancel] ); const onRefreshClick = async () => { setQueryParams((st) => { @@ -304,26 +458,52 @@ export const AgentRowDetail = ({ agentID, t }) => { }); }; - const onUnknownProcessEnroll = async (clusterIDs) => { - if (!Array.isArray(clusterIDs) || clusterIDs.length === 0) { + const onUnknownProcessEnroll = async (clusters) => { + const clusterItems = Array.isArray(clusters) + ? clusters + .map((item) => + typeof item === "string" + ? { cluster_id: item } + : item + ) + .filter((item) => item?.cluster_id) + : []; + const clusterIDs = clusterItems.map((item) => item.cluster_id); + if (clusterIDs.length === 0) { message.warn( formatMessage({ id: "agent.instance.associate.tips.associate" }) ); return; } + const previousNodeCount = nodes.length; + const previousUnknownCount = unknownProcess.length; setBtnLoading(true); const res = await request(`/instance/${agentID}/node/_discovery`, { method: "POST", body: { cluster_id: clusterIDs, + clusters: clusterItems, }, }); setBtnLoading(false); if (res && !res.error) { - message.success(formatMessage({ id: "app.message.operate.success" })); + const nextNodeCount = Object.keys(res.nodes || {}).length; + const nextUnknownCount = Array.isArray(res.unknown_process) + ? res.unknown_process.length + : previousUnknownCount; + const hasBoundNode = + nextNodeCount > previousNodeCount || + nextUnknownCount < previousUnknownCount; if (res.nodes) { setDataSource({ ...res, t: Date.now() }); } + if (!hasBoundNode) { + message.warning( + formatMessage({ id: "agent.instance.associate.tips.no_match" }) + ); + return; + } + message.success(formatMessage({ id: "app.message.operate.success" })); } else { console.log("onUnknownProcessEnroll error:", res); return; @@ -358,16 +538,18 @@ export const AgentRowDetail = ({ agentID, t }) => { }; return ( -
+
{ setState({ ...state, processesTab: tabKey }); }} tabBarExtraContent={ -
+
{hasAuthority("agent.instance:all") && state.processesTab === "unknown" ? (
- `${range[0]}-${range[1]} of ${total} items`, - }} - columns={columns} - /> +
+
+ formatMessage( + { id: "system.security.pagination.total" }, + { start: range[0], end: range[1], total } + ), + }} + columns={columns.filter((item) => item.key !== "collection_interval")} + /> + diff --git a/web/src/pages/Agent/Instance/components/RowDetail.less b/web/src/pages/Agent/Instance/components/RowDetail.less new file mode 100644 index 00000000..5c941720 --- /dev/null +++ b/web/src/pages/Agent/Instance/components/RowDetail.less @@ -0,0 +1,107 @@ +.detail { + width: 0; + max-width: 100%; + min-width: 0; + min-width: 100%; + overflow: hidden; +} + +.detailTabs { + min-width: 0; +} + +.detailTabs :global(.ant-tabs-bar) { + margin-bottom: 12px; +} + +.detailTabs :global(.ant-tabs-nav-container), +.detailTabs :global(.ant-tabs-content), +.detailTabs :global(.ant-tabs-tabpane) { + min-width: 0; +} + +.tableWrap { + width: 0; + max-width: 100%; + min-width: 0; + min-width: 100%; + overflow: hidden; +} + +.cmdlineWrap { + display: block; + width: 100%; + white-space: normal; + word-break: break-all; +} + +.cellWrap { + display: flex; + align-items: center; + width: 100%; + min-width: 0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + + a, span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-width: 0; + } +} + +.cellIcon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 16px; + height: 16px; + min-width: 16px; + margin-right: 4px; + flex-shrink: 0; + + :global(.anticon) { + display: inline-flex; + align-items: center; + justify-content: center; + width: 16px; + height: 16px; + line-height: 1; + font-size: 16px; + vertical-align: top; + } +} + +.cellContent { + flex: 1; + display: block; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + + > a, + > span { + display: block; + } +} + +.actionWrap { + display: flex; + align-items: flex-start; + justify-content: flex-start; + width: 100%; + white-space: nowrap; +} + +.actionWrap :global(.ant-divider-vertical) { + align-self: flex-start; + margin: 0 8px; +} + +.actionWrap :global(.ant-btn-link) { + height: auto; + padding: 0; +} diff --git a/web/src/pages/Agent/Instance/components/SetAgentCredential.jsx b/web/src/pages/Agent/Instance/components/SetAgentCredential.jsx index fa896851..6b3b0638 100644 --- a/web/src/pages/Agent/Instance/components/SetAgentCredential.jsx +++ b/web/src/pages/Agent/Instance/components/SetAgentCredential.jsx @@ -16,11 +16,40 @@ export default connect()((props) => { const [status, setStatus] = useState({}); const [testLoading, setTestLoading] = useState(false); + const getTryConnectErrorMessage = (error) => { + if (!error) { + return ""; + } + if (typeof error === "string") { + return error; + } + if (typeof error?.reason === "string") { + return error.reason; + } + try { + return JSON.stringify(error); + } catch (e) { + return `${error}`; + } + }; + + const extractHostFromEndpoint = (value) => { + const endpoint = `${value || ""}`.trim(); + if (!endpoint) { + return ""; + } + const normalized = endpoint.replace(/^https?:\/\//i, ""); + return normalized.split("/")[0] || ""; + }; + const onAgentCredentialSave = async (values) => { const newSelectedCluster = cloneDeep(selectedCluster); const index = newSelectedCluster.findIndex((item) => item.id === values.id); if (index !== -1) { - newSelectedCluster[index] = values; + newSelectedCluster[index] = { + ...newSelectedCluster[index], + ...values, + }; setSelectedCluster(newSelectedCluster); } dispatch({ @@ -32,12 +61,16 @@ export default connect()((props) => { }); dispatch({ type: "global/fetchClusterStatus", + payload: { + force: true, + }, }) }; const expandedRowRender = (record) => { return ( onAgentCredentialSave(values)} /> @@ -46,40 +79,61 @@ export default connect()((props) => { const tryConnect = async (values) => { setTestLoading(true); - const body = { - basic_auth: { - username: values.agent_basic_auth?.username, - password: values.agent_basic_auth?.password, - }, - host: values.host, - credential_id: - values.agent_credential_id !== MANUAL_VALUE - ? values.agent_credential_id - : undefined, - schema: values.schema || "http", - }; - if ( - values.credential_id && - !body.credential_id && - (!body.basic_auth.username || !body.basic_auth.password) - ) { - message.warning(formatMessage({ id: "agent.instance.associate.tips.connected.check" })); + try { + const hosts = (values?.hosts || []) + .map((item) => `${item || ""}`.trim()) + .filter(Boolean); + const endpointHost = extractHostFromEndpoint(values?.endpoint); + const endpoints = (values?.endpoints || []) + .map((item) => extractHostFromEndpoint(item)) + .filter(Boolean); + const host = `${values?.host || ""}`.trim() || endpointHost || endpoints[0] || hosts[0]; + const username = values.agent_basic_auth?.username; + const password = values.agent_basic_auth?.password; + const body = { + host, + hosts: hosts.length > 0 ? hosts : host ? [host] : undefined, + schema: values.schema || "http", + credential_id: + values.agent_credential_id && values.agent_credential_id !== MANUAL_VALUE + ? values.agent_credential_id + : undefined, + }; + if (username || password) { + body.basic_auth = { + username, + password, + }; + } + if (!body.host && (!body.hosts || body.hosts.length === 0)) { + message.warning( + formatMessage({ id: "cluster.regist.form.verify.required.endpoint" }) + ); + return; + } + if ( + values.credential_id && + !body.credential_id && + (!username || !password) + ) { + message.warning(formatMessage({ id: "agent.instance.associate.tips.connected.check" })); + return; + } + const res = await request(`${ESPrefix}/try_connect`, { + method: "POST", + body, + showErrorInner: true, + }, false, false); + setStatus((prev) => ({ + ...prev, + [values.id]: { + status: res?.status, + error: getTryConnectErrorMessage(res?.error), + }, + })); + } finally { setTestLoading(false); - return; } - const res = await request(`${ESPrefix}/try_connect`, { - method: "POST", - body, - showErrorInner: true, - }, false, false); - setStatus({ - ...status, - [values.id]: { - status: res?.status, - error: res?.error, - }, - }); - setTestLoading(false); }; return ( @@ -131,7 +185,13 @@ export default connect()((props) => { dataIndex: "agent_credential_id", key: "agent_credential_id", render: (text, record) => { - return record.agent_credential_id ? "Set" : "No set"; + return record.agent_credential_id || record.agent_basic_auth?.username + ? formatMessage({ + id: "agent.instance.associate.credential_status.set", + }) + : formatMessage({ + id: "agent.instance.associate.credential_status.unset", + }); }, }, { @@ -141,8 +201,11 @@ export default connect()((props) => { render: (text, record) => { if (!status[record.id]) return "-"; if (status[record.id].error) { + const errorMessage = getTryConnectErrorMessage( + status[record.id].error + ); return ( - + {formatMessage({ id: "alert.rule.table.columnns.status.failed"})} ); diff --git a/web/src/pages/Agent/Instance/components/UnknownAssociate.jsx b/web/src/pages/Agent/Instance/components/UnknownAssociate.jsx index cf512a85..533148ae 100644 --- a/web/src/pages/Agent/Instance/components/UnknownAssociate.jsx +++ b/web/src/pages/Agent/Instance/components/UnknownAssociate.jsx @@ -1,28 +1,38 @@ import { useGlobal } from "@/layouts/GlobalContext"; import request from "@/utils/request"; -import { Form, Input, Switch, Icon, Button, Alert } from "antd"; -import { useEffect, useMemo, useRef, useState } from "react"; -import { Link, router } from "umi"; +import { Button, Alert } from "antd"; +import { useEffect, useState } from "react"; import { formatMessage } from "umi/locale"; import ClusterSelect from "@/components/ClusterSelect"; import SetAgentCredential from "./SetAgentCredential"; export default ({ onBatchEnroll, loading }) => { - const { clusterList = [], clusterStatus } = useGlobal(); + const { clusterList = [] } = useGlobal(); const [selectedCluster, setSelectedCluster] = useState([]); const [auths, setAuths] = useState([]); + const needCredentialSetup = (item) => { + const needPlatformAuth = !!(item?.credential_id || item?.basic_auth?.username); + if (!needPlatformAuth) { + return false; + } + return !item?.agent_credential_id && !item?.agent_basic_auth?.username; + }; const onBatchEnrollClick = () => { if (selectedCluster.length === 0) return; const newAuths = [...auths] selectedCluster.forEach((item) => { - if (item.credential_id && !item.agent_credential_id) { + if (needCredentialSetup(item)) { newAuths.push(item) } }) setAuths(newAuths) if (newAuths.length === 0 && typeof onBatchEnroll === "function") { - onBatchEnroll(selectedCluster.map((item) => item.id)); + onBatchEnroll( + selectedCluster.map((item) => ({ + cluster_id: item.id, + })) + ); } }; @@ -46,7 +56,6 @@ export default ({ onBatchEnroll, loading }) => { dropdownWidth={400} selectedCluster={selectedCluster} onChange={(item) => { - console.log("onChange item:", item); setSelectedCluster(item); }} /> @@ -73,15 +82,16 @@ export default ({ onBatchEnroll, loading }) => { ) }
-
- - {formatMessage({ - id: "agent.instance.associate.tips.metric", - })} - -
+
{ columns={columns} /> { diff --git a/web/src/pages/Agent/Instance/form.jsx b/web/src/pages/Agent/Instance/form.jsx index fb29076d..26096eb4 100644 --- a/web/src/pages/Agent/Instance/form.jsx +++ b/web/src/pages/Agent/Instance/form.jsx @@ -17,7 +17,12 @@ import useFetch from "@/lib/hooks/use_fetch"; import request from "@/utils/request"; import { formatMessage } from "umi/locale"; import TagEditor from "@/components/infini/TagEditor"; -import { isTLS, removeHttpSchema } from "@/utils/utils"; +import { + isTLS, + isValidEndpointHost, + normalizeEndpointHost, + removeHttpSchema, +} from "@/utils/utils"; const formItemLayout = { labelCol: { @@ -99,31 +104,50 @@ const InstanceForm = React.forwardRef((props) => { ], })()} - + {getFieldDecorator("endpoint", { initialValue: removeHttpSchema(editValue?.endpoint), normalize: (value) => { - return removeHttpSchema(value || "").trim() + return normalizeEndpointHost(value); }, validateTrigger: ["onChange", "onBlur"], rules: [ { required: true, message: formatMessage({ - id: "gateway.instance.field.endpoint.form.required", + id: "agent.instance.field.endpoint.form.required", }), }, { - type: "string", - pattern: /^[\w\.\-_~%]+(\:\d+)?$/, //(https?:\/\/)? - message: formatMessage({ - id: "cluster.regist.form.verify.valid.endpoint", - }), + validator: (rule, value, callback) => { + if (!value || isValidEndpointHost(value)) { + callback(); + return; + } + callback(formatMessage({ + id: "cluster.regist.form.verify.valid.endpoint", + })); + }, }, ], - })()} + })( + + )} - + {getFieldDecorator("isTLS", { initialValue: isTLS(editValue?.endpoint), })( @@ -158,7 +182,14 @@ const InstanceForm = React.forwardRef((props) => { {getFieldDecorator("basic_auth.username", { initialValue: editValue?.basic_auth?.username, rules: [], - })()} + })( + + )} { {getFieldDecorator("basic_auth.password", { initialValue: editValue?.basic_auth?.password, rules: [], - })()} + })( + + )} ) : ( @@ -202,7 +239,7 @@ const InstanceForm = React.forwardRef((props) => { diff --git a/web/src/pages/Agent/Instance/index.jsx b/web/src/pages/Agent/Instance/index.jsx index d5744fe7..95c0ef14 100644 --- a/web/src/pages/Agent/Instance/index.jsx +++ b/web/src/pages/Agent/Instance/index.jsx @@ -3,7 +3,6 @@ import { Card, Table, Popconfirm, - Divider, Form, Row, Col, @@ -11,11 +10,15 @@ import { Input, message, Drawer, + Dropdown, + Menu, Radio, Tag, Select, Checkbox, Modal, + Icon, + Tooltip, } from "antd"; import { formatMessage } from "umi/locale"; import useFetch from "@/lib/hooks/use_fetch"; @@ -43,10 +46,93 @@ import AutoEnroll from "./components/AutoEnroll"; import { sorter } from "@/utils/utils"; import { HealthStatusView } from "@/components/infini/health_status_view"; import { isNumber } from "lodash"; +import SearchInput from "@/components/infini/SearchInput"; +import { CopyToClipboard } from "react-copy-to-clipboard"; -const { Search } = Input; +const menuItemContentStyle = { + display: "inline-flex", + alignItems: "center", + gap: 8, +}; const AgentList = (props) => { + const renderCopyButton = (text, style = {}) => { + const button = ( + + + + ); + + if (!text) { + return button; + } + + return ( + { + message.open({ + type: "success", + key: "agent-registration-copy-success", + content: formatMessage({ + id: "agent.install.setup.copy.success", + }), + }); + }} + > + {button} + + ); + }; + + const renderReadonlyBlock = (text) => ( +
+
+ {text || "-"} +
+ {renderCopyButton(text)} +
+ ); + + const renderWrapCell = (text) => ( +
+ {text} +
+ ); + const [queryParams, setQueryParams] = React.useState({ size: 20, }); @@ -61,7 +147,14 @@ const AgentList = (props) => { ); const [isLoading, setIsLoading] = React.useState(loading); const [btnLoading, setBtnLoading] = React.useState(false); + const [registrationAccessLoading, setRegistrationAccessLoading] = React.useState(false); const [delInstId, setDelInstId] = React.useState(""); + const [consoleAccessModal, setConsoleAccessModal] = React.useState({ + visible: false, + consoleEndpoint: "", + managerToken: "", + registrationExpiredAt: "", + }); const onDeleteClick = useCallback( async (instanceID) => { const deleteRes = await request(`/instance/${instanceID}`, { @@ -90,26 +183,46 @@ const AgentList = (props) => { }); }; const [instanceStatus, setInstanceStatus] = React.useState({}); + const [instanceStatusLoading, setInstanceStatusLoading] = React.useState(false); + const [expandedRowKeys, setExpandedRowKeys] = React.useState([]); + + const isAgentStatusResolved = useCallback( + (instanceID) => Object.prototype.hasOwnProperty.call(instanceStatus, instanceID), + [instanceStatus] + ); + + const isAgentOnline = useCallback( + (instanceID) => !!instanceStatus[instanceID]?.system, + [instanceStatus] + ); const columns = useMemo( () => [ { - title: "Name", + title: formatMessage({ id: "gateway.instance.column.name" }), + width: 180, dataIndex: "name", + render: (text) => renderWrapCell(text), sorter: (a, b) => sorter.string(a, b, "name"), }, { - title: "Endpoint", + title: formatMessage({ id: "gateway.instance.column.endpoint" }), + width: 220, dataIndex: "endpoint", + render: (text) => renderWrapCell(text), sorter: (a, b) => sorter.string(a, b, "endpoint"), }, { - title: "Status", + title: formatMessage({ id: "gateway.instance.column.status" }), width: 120, dataIndex: "status", render: (text, record) => { const status = instanceStatus[record.id]?.system ? "online" : "N/A"; - return ; + const label = + text === "online" || text === "Online" + ? formatMessage({ id: "gateway.instance.status.online" }) + : text; + return ; }, sorter: (a, b) => { const status1 = instanceStatus[a.id]?.system ? 1 : 0; @@ -118,7 +231,7 @@ const AgentList = (props) => { }, }, { - title: "CPU", + title: formatMessage({ id: "gateway.instance.column.cpu" }), width: 100, render: (text, record) => { return instanceStatus[record.id]?.system?.cpu || @@ -140,7 +253,7 @@ const AgentList = (props) => { }, }, { - title: "Memory", + title: formatMessage({ id: "gateway.instance.column.memory" }), width: 130, render: (text, record) => { if (!instanceStatus[record.id]?.system) { @@ -162,7 +275,7 @@ const AgentList = (props) => { }, }, { - title: "Uptime", + title: formatMessage({ id: "gateway.instance.column.uptime" }), width: 130, render: (text, record) => { if (!instanceStatus[record.id]?.system) { @@ -209,41 +322,52 @@ const AgentList = (props) => { // return text; // }, // }, - // { - // title: "Version", - // dataIndex: "version.number", - // }, - // { - // title: "Last Updated", - // dataIndex: "updated", - // render: (text) => { - // return moment(text).format("YYYY-MM-DD HH:mm:ss"); - // }, - // }, { - title: formatMessage({ id: "table.field.actions" }), + title: formatMessage({ id: "overview.column.version" }), width: 120, + render: (text, record) => record?.application?.version?.number || null, + }, + { + title: formatMessage({ id: "table.field.actions" }), + width: 100, render: (text, record) => (
{hasAuthority("agent.instance:all") ? ( - <> - {/* Task Assignment*/} - {/* onTaskSettingsClick(record)}>Task Settings - */} - - {formatMessage({ id: "form.button.edit" })} - - - onDeleteClick(record.id)} + + + + + + {formatMessage({ id: "form.button.edit" })} + + + + + onDeleteClick(record.id)} + > + + + {formatMessage({ id: "form.button.delete" })} + + + + + } + trigger={["click"]} + > + e.preventDefault()} > - {formatMessage({ id: "form.button.delete" })} - - + + + ) : null}
), @@ -267,10 +391,14 @@ const AgentList = (props) => { useEffect(() => { const fetchStatus = async () => { if (!instances || instances.length == 0) { + setInstanceStatus({}); + setInstanceStatusLoading(false); return; } + setInstanceStatus({}); + setInstanceStatusLoading(true); const instanceIDs = instances.map((inst) => inst.id); - const statusRes = await request(`/instance/stats`, { + const statusRes = await request(`/agent/instance/stats`, { method: "POST", body: instanceIDs, }); @@ -278,9 +406,59 @@ const AgentList = (props) => { if (statusRes && !statusRes.error) { setInstanceStatus(statusRes); } + setInstanceStatusLoading(false); }; fetchStatus(); - }, [value]); + }, [instances]); + + useEffect(() => { + setExpandedRowKeys((keys) => + keys.filter( + (key) => + instances.some((instance) => instance.id === key) && isAgentOnline(key) + ) + ); + }, [instances, isAgentOnline]); + + const onExpand = useCallback( + (expanded, record) => { + if (!expanded) { + setExpandedRowKeys((keys) => keys.filter((key) => key !== record.id)); + return; + } + if (!isAgentStatusResolved(record.id) || !isAgentOnline(record.id)) { + return; + } + setExpandedRowKeys((keys) => + keys.includes(record.id) ? keys : [...keys, record.id] + ); + }, + [isAgentOnline, isAgentStatusResolved] + ); + + const renderExpandIcon = useCallback( + ({ expanded, onExpand, record }) => { + const resolved = isAgentStatusResolved(record.id); + if (!resolved) { + return instanceStatusLoading ? ( + + ) : ( + + ); + } + if (!isAgentOnline(record.id)) { + return ; + } + return ( + onExpand(record, event)} + /> + ); + }, + [instanceStatusLoading, isAgentOnline, isAgentStatusResolved] + ); const handleTableChange = (pagination, filters, sorter, extra) => { const { pageSize, current } = pagination; @@ -345,9 +523,16 @@ const AgentList = (props) => { if (delInstId == record.id) { return null; } - return ; + if (!isAgentOnline(record.id)) { + return null; + } + return ( +
+ +
+ ); }, - [queryParams.t, delInstId] + [queryParams.t, delInstId, isAgentOnline] ); const onAutoEnroll = async (clusterIDs) => { @@ -389,7 +574,7 @@ const AgentList = (props) => { }, }); if(statusRes && statusRes.acknowledged){ - message.success("submit successfully"); + message.success(formatMessage({ id: "app.message.operate.success" })); } setClearLoading(false); } @@ -401,31 +586,86 @@ const AgentList = (props) => {
{formatMessage({ id: "agent.instance.clear.modal.desc" })}
), - okText: "Yes", + okText: formatMessage({ id: "form.button.ok" }), okType: "danger", - cancelText: "No", + cancelText: formatMessage({ id: "form.button.cancel" }), onOk() { onClearClick(); }, }); }, []); + const showConsoleAccessInfo = useCallback(async () => { + setRegistrationAccessLoading(true); + const res = await request("/instance/_prepare_registration", { + method: "POST", + }); + setRegistrationAccessLoading(false); + if (res?.error) { + message.error( + formatMessage({ id: "agent.instance.registration.console.load_failed" }) + ); + return; + } + setConsoleAccessModal({ + visible: true, + consoleEndpoint: res?.endpoint || "", + managerToken: res?.token || "", + registrationExpiredAt: res?.expired_at || "", + }); + }, []); + + const registrationActionMenu = ( + { + if (key === "register") { + router.push(`/resource/agent/new`); + return; + } + if (key === "console-info") { + showConsoleAccessInfo(); + } + }} + > + + + + {formatMessage({ id: "agent.instance.registration.menu.register" })} + + + + + + {formatMessage({ id: "agent.instance.registration.menu.info" })} + + + + ); + + const consoleTokenTip = formatMessage({ + id: "agent.instance.registration.console.token.tip", + }); + return (
-
- + { onSearchClick(value); }} @@ -438,30 +678,36 @@ const AgentList = (props) => { style={{ display: "flex", alignItems: "center", + flexWrap: "wrap", + justifyContent: "flex-end", gap: 10, }} > { hasAuthority("agent.instance:all") && ( <> - + {hasAuthority("agent.instance:all") ? ( - + + + ) : null}
@@ -496,7 +745,9 @@ const AgentList = (props) => { loading={isLoading} bordered dataSource={instances} + scroll={instances.length > 0 ? { x: "max-content" } : undefined} rowKey={"id"} + tableLayout="fixed" pagination={{ size: "small", pageSize: queryParams.size, @@ -508,7 +759,9 @@ const AgentList = (props) => { columns={columns} onChange={handleTableChange} expandedRowRender={expandedRowRender} - scroll={{x: 'max-content' }} + expandedRowKeys={expandedRowKeys} + onExpand={onExpand} + expandIcon={renderExpandIcon} /> { > + + setConsoleAccessModal((state) => ({ + ...state, + visible: false, + })) + } + destroyOnClose + > +
+
+ {formatMessage({ + id: "agent.instance.registration.access.endpoint", + })} +
+ {renderReadonlyBlock(consoleAccessModal.consoleEndpoint)} +
+ {formatMessage({ + id: "agent.instance.registration.console.endpoint.tip", + })} +
+
+
+
+ {formatMessage({ + id: "agent.instance.registration.access.credential", + })} +
+ {renderReadonlyBlock(consoleAccessModal.managerToken)} +
{consoleTokenTip}
+
+ ); @@ -728,11 +1016,11 @@ const DiscoverAgent = ({ addSuccessCb }) => { }, []); const columns = [ { - title: "Agent IP", + title: formatMessage({ id: "agent.instance.column.agent_ip" }), dataIndex: "remote_ip", }, { - title: "Version", + title: formatMessage({ id: "overview.column.version" }), dataIndex: "version", }, { @@ -742,7 +1030,9 @@ const DiscoverAgent = ({ addSuccessCb }) => { {hasAuthority("agent.instance:all") ? ( <> onDeleteClick(record.id)} > {formatMessage({ id: "form.button.delete" })} @@ -795,6 +1085,7 @@ const DiscoverAgent = ({ addSuccessCb }) => { columns={columns} rowKey="id" dataSource={instances} + scroll={instances.length > 0 ? { x: "max-content" } : undefined} />
- + {formatMessage({ + id: "agent.instance.regist", + })} +
+ + + +
diff --git a/web/src/pages/Alerting/Alert/Detail.jsx b/web/src/pages/Alerting/Alert/Detail.jsx index b45ab7a4..fb28ced6 100644 --- a/web/src/pages/Alerting/Alert/Detail.jsx +++ b/web/src/pages/Alerting/Alert/Detail.jsx @@ -22,7 +22,6 @@ import { filterSearchValue, sorter, formatUtcTimeToLocal, - firstUpperCase, } from "@/utils/utils"; import { HealthStatusView } from "@/components/infini/health_status_view"; import { PriorityColor, RuleStautsColor } from "../utils/constants"; @@ -33,12 +32,32 @@ import ClusterName from "@/pages/System/Cluster/components/ClusterName"; import Markdown from "@/components/Markdown"; import { ExpressionView } from "./ExpressionView"; import { useHistory } from "react-router-dom"; +import { stripDuplicatedAlertTitle } from "../utils/message"; const Detail = (props) => { const [param, setParam] = useQueryParam("_g", JsonParam); const eventID = props.match.params?.event_id; const [alertDetail, setAlertDetail] = useState({}); + const [ruleDetail, setRuleDetail] = useState({}); + const displayState = + alertDetail?.display_state || alertDetail?.state || ruleDetail?.state; + const alertPriority = + alertDetail?.priority || + alertDetail?.condition_result?.result_items?.[0]?.condition_item?.priority || + ruleDetail?.bucket_conditions?.items?.[0]?.priority || + ruleDetail?.conditions?.items?.[0]?.priority; + const expressionItems = + alertDetail?.condition?.items?.length > 0 + ? alertDetail?.condition?.items + : ruleDetail?.bucket_conditions?.items?.length > 0 + ? ruleDetail?.bucket_conditions?.items + : ruleDetail?.conditions?.items; + const alertMessage = stripDuplicatedAlertTitle( + alertDetail?.message, + alertDetail?.title + ); + const fetchAlertDetail = (id) => { const fetchData = async () => { const res = await request(`/alerting/alert/${id}`, { @@ -51,6 +70,20 @@ const Detail = (props) => { fetchData(); }; + const fetchRuleDetail = (id) => { + const fetchData = async () => { + const res = await request(`/alerting/rule/${id}/info`, { + method: "GET", + }); + if (res && !res.error) { + setRuleDetail(res); + } else { + setRuleDetail({}); + } + }; + fetchData(); + }; + const onAckClick = useCallback(async () => { const res = await request(`alerting/alert/_acknowledge`, { method: "POST", @@ -69,6 +102,14 @@ const Detail = (props) => { fetchAlertDetail(eventID); }, [eventID]); + useEffect(() => { + if (!alertDetail?.rule_id) { + setRuleDetail({}); + return; + } + fetchRuleDetail(alertDetail.rule_id); + }, [alertDetail?.rule_id]); + const AlertMessageView = ({ content }) => { const [visible, setVisible] = useState(false); const onVisibleClick = () => { @@ -153,8 +194,15 @@ const Detail = (props) => { })} content={ } /> @@ -179,10 +227,12 @@ const Detail = (props) => { + alertPriority && + displayState != "ok" && + displayState != "recovered" ? ( + {formatMessage({ - id: `alert.message.priority.${alertDetail?.priority}`, + id: `alert.message.priority.${alertPriority}`, })} ) : ( @@ -224,7 +274,7 @@ const Detail = (props) => { title={formatMessage({ id: "alert.rule.table.columnns.expression", })} - content={} + content={} /> @@ -249,7 +299,7 @@ const Detail = (props) => { })} content={ - + } /> @@ -260,7 +310,9 @@ const Detail = (props) => {
{ { const [editValue, setEditValue] = useState(props.value || {}); const { getFieldDecorator } = props.form; const history = useHistory(); + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "alerting", locale: "menu.alerting" }, + { title: "channel", locale: "menu.alerting.channel", href: "/alerting/channel" }, + { + title: "channel-form", + locale: + props.title === formatMessage({ id: "alert.channel.form.title.edit" }) + ? "menu.alerting.edit_channel" + : "menu.alerting.new_channel", + }, + ]; const [currentType, setCurrentType] = useState(); @@ -117,7 +129,7 @@ const ChannelForm = (props) => { }, [editValue?.sub_type, editValue?.type]) return ( - + { - {formatMessage({id: "alert.channel.form.email.server.new"})} > + {formatMessage({id: "alert.channel.form.email.server.new"})} > { <> { !isAdvanced && ( - + {getFieldDecorator([valueProps, 'webhook', 'url'].filter((item) => !!item).join('.'), { initialValue: editValue?.url, rules: [ { required: true, - message: "Please input URL!", + message: formatMessage({ + id: "alert.channel.form.webhook.url.required", + }), }, ], })()} ) } - + {getFieldDecorator([valueProps, 'webhook', 'method'].filter((item) => !!item).join('.'), { initialValue: editValue?.method || "POST", rules: [ { required: true, - message: "Please select method!", + message: formatMessage({ + id: "alert.channel.form.webhook.method.required", + }), }, ], })( @@ -74,7 +82,9 @@ export default (props) => { )} - + { ) } - }> + } + > {getFieldDecorator([valueProps, 'webhook', 'body'].filter((item) => !!item).join('.'), { initialValue: editValue?.body, rules: [ { required: true, - message: "Please input body!", + message: formatMessage({ + id: "alert.channel.form.webhook.body.required", + }), }, ], })( diff --git a/web/src/pages/Alerting/Channel/Index.jsx b/web/src/pages/Alerting/Channel/Index.jsx index dcb855d8..bfb35720 100644 --- a/web/src/pages/Alerting/Channel/Index.jsx +++ b/web/src/pages/Alerting/Channel/Index.jsx @@ -15,6 +15,7 @@ import { Icon, Dropdown, Menu, + Tooltip, } from "antd"; import { formatMessage } from "umi/locale"; import useFetch from "@/lib/hooks/use_fetch"; @@ -43,8 +44,8 @@ import FormEmail from "./FormEmail"; import Import from "../components/Import"; import Export from "../components/Export"; import DiscordWithColor from "@/components/Icons/DiscordWithColor"; +import SearchInput from "@/components/infini/SearchInput"; -const { Search } = Input; const { Option } = Select; export const CHANNELS = [ @@ -175,13 +176,34 @@ const Index = (props) => { } else { console.log("operate failed,", res); message.error( - formatMessage({ - id: "app.message.operate.failed", - }) + res?.message || + res?.error || + formatMessage({ + id: "app.message.operate.failed", + }) ); } }, []); + const getChannelEnableHint = useCallback((record) => { + const channelType = record?.sub_type || record?.type; + if (channelType !== "email") { + return ""; + } + const hasServer = !!record?.email?.server_id; + const hasRecipients = (record?.email?.recipients?.to || []).length > 0; + if (!hasServer && !hasRecipients) { + return formatMessage({ id: "alert.channel.enable.tip.email_incomplete" }); + } + if (!hasServer) { + return formatMessage({ id: "alert.channel.enable.tip.email_server" }); + } + if (!hasRecipients) { + return formatMessage({ id: "alert.channel.enable.tip.email_recipients" }); + } + return ""; + }, []); + const defaultSelectedRows = { rowKeys: [], rows: [], @@ -228,16 +250,25 @@ const Index = (props) => { { title: formatMessage({ id: "alert.channel.table.columns.enable" }), dataIndex: "enabled", - render: (value, record) => ( - { - onEnableClick([record.id], checked ? "enable" : "disable"); - }} - /> - ), + render: (value, record) => { + const enableHint = getChannelEnableHint(record); + return ( + + { + if (checked && enableHint) { + message.warning(enableHint); + return; + } + onEnableClick([record.id], checked ? "enable" : "disable"); + }} + /> + + ); + }, }, { title: formatMessage({ id: "alert.message.detail.updated" }), @@ -257,7 +288,7 @@ const Index = (props) => { onDeleteClick([record.id])} > {formatMessage({ id: "form.button.delete" })} @@ -363,11 +394,11 @@ const Index = (props) => { width: 600, }} > -
- { dispatch({ type: "search", value: value }); @@ -468,7 +499,7 @@ const Index = (props) => { {formatMessage({ id: "form.button.new" })} - @@ -494,7 +525,10 @@ const Index = (props) => { dispatch({ type: "pageSizeChange", value: size }); }, showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, + formatMessage( + { id: "system.security.pagination.total" }, + { start: range[0], end: range[1], total } + ), }} columns={columns} rowSelection={rowSelection} diff --git a/web/src/pages/Alerting/Message/Index.jsx b/web/src/pages/Alerting/Message/Index.jsx index 5f03f972..5936b6b1 100644 --- a/web/src/pages/Alerting/Message/Index.jsx +++ b/web/src/pages/Alerting/Message/Index.jsx @@ -59,6 +59,54 @@ import { getLocale } from "umi/locale"; const { Search } = Input; const { Option } = Select; +const isValidAlertTime = (value) => { + if (!value) return false; + const parsed = moment(value); + return parsed.isValid() && parsed.year() > 1; +}; + +const getAlertDisplayStartTime = (record = {}) => + isValidAlertTime(record?.trigger_at) + ? record.trigger_at + : record?.created; + +const calcSafeDuration = (msgItem) => { + const triggerAt = msgItem?.trigger_at; + const resolveAt = msgItem?.updated; + + const start = moment(triggerAt); + const end = resolveAt ? moment(resolveAt) : moment(); + + if (!start.isValid() || !end.isValid()) return "-"; + + const diffMs = end.diff(start); + + if (diffMs < 0) return "-"; + + return moment.duration(diffMs).humanize(); +}; + +const normalizeQueryTimeValue = (value, fallback = "auto", keys = []) => { + if (typeof value === "string" || typeof value === "number") { + return `${value}`; + } + if (value && typeof value === "object") { + for (const key of keys) { + if ( + Object.prototype.hasOwnProperty.call(value, key) && + value[key] !== undefined && + value[key] !== null + ) { + const candidate = value[key]; + if (typeof candidate === "string" || typeof candidate === "number") { + return `${candidate}`; + } + } + } + } + return fallback; +}; + const Index = (props) => { const [param, setParam] = useQueryParam("_g", JsonParam); const [searchValue, setSearchValue] = React.useState(""); @@ -74,13 +122,24 @@ const Index = (props) => { const [refresh, setRefresh] = useState({ isRefreshPaused: false }); const [timeZone, setTimeZone] = useState(() => getTimezone()); + const initialStartTime = normalizeQueryTimeValue( + param?.start_time, + "auto", + ["from", "min", "gte", "start"] + ); + const initialEndTime = normalizeQueryTimeValue( + param?.end_time, + "auto", + ["to", "max", "lte", "end"] + ); + const initialQueryParams = { from: 0, size: 10, // status: "alerting", - start_time: "now-7d", - end_time: "now", - ...param, + ...(param || {}), + start_time: initialStartTime, + end_time: initialEndTime, }; const alertReducer = (queryParams, action) => { @@ -90,6 +149,12 @@ const Index = (props) => { ...queryParams, priority: action.value, }; + case "priorityAndStatus": + return { + ...queryParams, + status: action.status, + priority: action.priority, + }; case "status": return { ...queryParams, @@ -278,19 +343,24 @@ const Index = (props) => { title: formatMessage({ id: "alert.message.table.created" }), dataIndex: "created", width: 180, - render: (text, record) => ( - {formatUtcTimeToLocal(text)} - ), + render: (text, record) => { + const displayStartTime = getAlertDisplayStartTime(record); + return ( + + {formatUtcTimeToLocal(displayStartTime)} + + ); + }, }, { title: formatMessage({ id: "alert.message.table.duration" }), dataIndex: "duration", - render: (text, record) => moment.duration(text).humanize(), + render: (text, record) => calcSafeDuration(record), }, { title: formatMessage({ id: "alert.message.table.status" }), dataIndex: "status", - width: 80, + width: 140, render: (text, record) => { return ; // @@ -356,19 +426,45 @@ const Index = (props) => { }); }; + const onWidgetQueriesChange = (queries = {}) => { + if (!queries?.range?.from || !queries?.range?.to) { + return; + } + onTimeChange({ + start: queries.range.from, + end: queries.range.to, + }); + }; + const fetchMessages = (queryParams) => { setLoading(true); - let params = queryParams; - if (queryParams?.start_time && queryParams.end_time) { + const normalizedStartTime = normalizeQueryTimeValue( + queryParams?.start_time, + "auto", + ["from", "min", "gte", "start"] + ); + const normalizedEndTime = normalizeQueryTimeValue( + queryParams?.end_time, + "auto", + ["to", "max", "lte", "end"] + ); + let params = { + ...queryParams, + start_time: normalizedStartTime, + end_time: normalizedEndTime, + }; + if (normalizedStartTime && normalizedEndTime) { const bounds = calculateBounds({ - from: queryParams?.start_time, - to: queryParams.end_time, + from: normalizedStartTime || "auto", + to: normalizedEndTime || "auto", }); - params = { - ...queryParams, - min: bounds.min.valueOf(), - max: bounds.max.valueOf(), - }; + if (bounds?.min && bounds?.max) { + params = { + ...params, + min: bounds.min.valueOf(), + max: bounds.max.valueOf(), + }; + } } const fetchData = async () => { @@ -404,9 +500,8 @@ const Index = (props) => { }; useEffect(() => { - setParam({ ...param, ...queryParams }); + setParam(prev => ({ ...prev, ...queryParams })); fetchMessages(queryParams); - fetchMessageStats(queryParams); }, [queryParams]); @@ -451,7 +546,19 @@ const Index = (props) => { >
{title}
- + { + if (window.location.pathname === `/alerting/message/${id}`) { + e.preventDefault(); + } + }} + > @@ -517,26 +624,64 @@ const Index = (props) => { }; }, [dataSource.aggregations]); + const widgetRange = useMemo(() => { + const startTime = normalizeQueryTimeValue( + queryParams?.start_time, + "auto", + ["from", "min", "gte", "start"] + ); + const endTime = normalizeQueryTimeValue( + queryParams?.end_time, + "auto", + ["to", "max", "lte", "end"] + ); + if ( + startTime && + endTime && + startTime !== "auto" && + endTime !== "auto" + ) { + return { + from: startTime, + to: endTime, + }; + } + if (minUpdated && maxUpdated) { + const minMoment = moment(minUpdated); + const maxMoment = moment(maxUpdated); + if (minMoment.isValid() && maxMoment.isValid()) { + if (maxMoment.valueOf() <= minMoment.valueOf()) { + return { + from: maxMoment.clone().subtract(15, "minutes").toISOString(), + to: maxMoment.toISOString(), + }; + } + return { + from: minUpdated, + to: maxUpdated, + }; + } + } + return { + from: "auto", + to: "auto", + }; + }, [queryParams?.start_time, queryParams?.end_time, minUpdated, maxUpdated]); + const filterPriorityAndStatus = (params) => { - dispatch({ - type: "timeChange", - value: { - start_time: "", - end_time: "", - }, - }); if (params.type === "priority") { dispatch({ - type: "status", - value: "alerting", + type: "priorityAndStatus", + status: "alerting", + priority: params.value, }); } else { dispatch({ - type: "priority", - value: undefined, + type: "priorityAndStatus", + status: "alerting", + priority: undefined, }); } - dispatch(params); }; return ( @@ -561,10 +706,9 @@ const Index = (props) => { >
@@ -658,11 +802,12 @@ const Index = (props) => { -
+
{ > {hasAuthority("alerting.message:all") ? ( - @@ -703,12 +848,10 @@ const Index = (props) => { >
{ +const resolveAlertTime = (value) => { + if (!value) return ""; + const parsed = moment(value); + return parsed.isValid() && parsed.year() > 1 ? value : ""; +}; - const { rule_id, created, updated, expression } = msgItem; - const [rule, setRule] = useState() - const [loading, setLoading] = useState() +const safeParseJSON = (value) => { + if (!value) return null; + if (typeof value === "object") return value; + if (typeof value !== "string") return null; + try { + return JSON.parse(value); + } catch (e) { + return null; + } +}; - const fetchRule = async (id) => { - if (!id) { - setRule() - return; +const buildCopyRequest = (msgItem, ruleID, min, max) => { + const queryDSL = safeParseJSON(msgItem?.condition_result?.query_result?.query); + const objects = + msgItem?.resource?.objects || + msgItem?.resource_objects || + msgItem?.objects || + []; + const index = Array.isArray(objects) && objects.length > 0 ? objects.join(",") : ""; + if (queryDSL) { + if (index) { + return `GET ${index}/_search\n${JSON.stringify(queryDSL, null, 2)}`; } - setLoading(true) - const res = await request(`/alerting/rule/${id}`) - setRule(res?._source || undefined) - setLoading(false) + return JSON.stringify(queryDSL, null, 2); } - useEffect(() => { - fetchRule(rule_id) - }, [rule_id]) + const resourceID = msgItem?.resource_id || msgItem?.resource?.resource_id || ""; + const state = msgItem?.state || "alerting"; + const filters = [ + { term: { rule_id: { value: ruleID } } }, + { term: { resource_id: { value: resourceID } } }, + { term: { state: { value: state } } }, + ].filter((item) => { + const field = Object.keys(item.term || {})[0]; + return field ? item.term[field].value !== "" : false; + }); + + return `GET .infini_alert-history/_search +${JSON.stringify( + { + aggs: { + filter_agg: { + aggs: { + time_buckets: { + aggs: { + priority_buckets: { + aggs: { + a: { + value_count: { + field: "id", + }, + }, + }, + terms: { + field: "priority", + order: [{ _count: "desc" }], + size: 10, + }, + }, + }, + auto_date_histogram: { + field: "created", + buckets: 120, + }, + }, + }, + filter: { + bool: { + filter: filters, + must: [ + { + range: { + created: { + gte: Number(min), + lte: Number(max), + }, + }, + }, + ], + must_not: [], + should: [], + }, + }, + }, + }, + size: 0, + }, + null, + 2 + )}`; +}; - const widget = useMemo(() => { - if (!rule) return; - const { resource = {} } = rule - return buildWidgetByRule(rule, { - "cluster_id": resource.resource_id, - "indices": resource.objects, - "time_field": resource.time_field, - "raw_filter": resource.raw_filter, - }, created, updated) - }, [JSON.stringify(rule), updated, created]) +export default ({ msgItem, range, onRangeChange }) => { + const { rule_id, expression } = msgItem; + const created = resolveAlertTime(msgItem?.trigger_at) || msgItem?.created; + const updated = resolveAlertTime(msgItem?.resolve_at) || msgItem?.updated; - const highlightRange = useMemo(() => { - if (!created || created === updated) return undefined; - return { - from: moment(created).valueOf(), - to: moment(updated).valueOf() + const [rule, setRule] = useState(); + const [loading, setLoading] = useState(false); + const [metricData, setMetricData] = useState({}); + const [latestRequest, setLatestRequest] = useState(""); + + const fetchRule = async (id) => { + if (!id) { setRule(); return; } + const res = await request(`/alerting/rule/${id}`); + setRule(res?._source || undefined); + }; + + const fetchHistoryMetric = async (id, min, max) => { + if (!id) return; + setLoading(true); + try { + const res = await request(`/alerting/rule/${id}/history_metric`, { + method: "GET", + queryParams: { min, max }, + }); + setLatestRequest(buildCopyRequest(msgItem, id, min, max)); + if (res && !res.error) { + setMetricData(res.metric || {}); + } + } finally { + setLoading(false); } - }, [created, updated]) + }; + + useEffect(() => { + fetchRule(rule_id); + }, [rule_id]); + + useEffect(() => { + if (!rule_id) return; + // expand time range slightly beyond the alert window so context is visible + const from = created ? moment(created).subtract(5, "minutes") : moment().subtract(1, "hour"); + const to = updated ? moment(updated).add(5, "minutes") : moment(); + fetchHistoryMetric(rule_id, from.valueOf(), to.valueOf()); + }, [rule_id, created, updated]); + + const lineAnnotations = useMemo(() => { + if (!rule?.conditions?.items) return []; + return rule.conditions.items.map((item) => { + const sortValues = [...(item.values || [])].sort((a, b) => a - b); + return { + dataValues: sortValues.map((dv, dk) => ({ + dataValue: parseInt(dv), + details: ( + + Priority: {item.priority} + {sortValues.length > 1 ? `-${dk}` : ""} + {item.expression ? <>
Expression: {item.expression} : null} + + ), + })), + lineColor: PriorityColor[item.priority], + }; + }); + }, [rule]); + + const highlightCoords = useMemo(() => { + if (!created) return { x0: 0, x1: 0, y0: 0, y1: 0 }; + const x0 = moment(created).valueOf(); + const x1 = updated ? moment(updated).valueOf() : moment().valueOf(); + return { x0, x1, y0: 0, y1: Number.MAX_SAFE_INTEGER }; + }, [created, updated]); + + const hasMetricHistory = (metricData?.lines || []).length > 0; + const cardTitle = hasMetricHistory + ? formatMessage({ id: "alert.message.detail.alert_metric_status" }) + : formatMessage({ id: "alert.message.detail.title.alert_history" }); return ( - - {formatMessage({id:"alert.message.detail.alert_metric_status"})} - - - - - } bodyStyle={{ height: 250, padding: 1 }} loading={loading}> - { - rule ? ( - - ) : + + {cardTitle} + {hasMetricHistory ? ( + + + + ) : null} + } + bodyStyle={{ height: 250, padding: 1 }} + loading={loading} + > + {hasMetricHistory ? ( +
+ + formatter.full_dates(value), + }} + theme={{ + lineSeriesStyle: { + line: { visible: false, strokeWidth: 0 }, + point: { visible: true, radius: 2, strokeWidth: 0 }, + }, + }} + /> + + {lineAnnotations.map((item, i) => ( + } + style={{ + line: { dash: [5, 5], stroke: item.lineColor || "black", opacity: 0.8, strokeWidth: 1 }, + }} + /> + ))} + + {metricData.axis?.map((item) => ( + + ))} + {metricData.lines?.map((item, i) => ( + + ))} + + {latestRequest ? ( +
+ + message.success(formatMessage({ id: "cluster.metrics.request.copy.success" })) + } + > + + + + +
+ ) : null} +
+ ) : ( +
+ +
+ )} - ) -} \ No newline at end of file + ); +}; diff --git a/web/src/pages/Alerting/Message/components/EventDetail.jsx b/web/src/pages/Alerting/Message/components/EventDetail.jsx index 8ec4fca6..d7215fd5 100644 --- a/web/src/pages/Alerting/Message/components/EventDetail.jsx +++ b/web/src/pages/Alerting/Message/components/EventDetail.jsx @@ -12,15 +12,37 @@ import EventMessageCard from "./EventMessageCard"; import EventDetailCard from "./EventDetailCard"; import NotificationCard from "./NotificationCard"; import AlertChartCard from "./AlertChartCard"; -import { useHistory } from "react-router-dom"; +import { useHistory, useLocation } from "react-router-dom"; import WidgetLoader from "@/pages/DataManagement/View/WidgetLoader"; import DatePicker from "@/common/src/DatePicker"; import { getLocale } from "umi/locale"; import { getTimezone } from "@/utils/utils"; import { JsonParam, QueryParamProvider, useQueryParam } from "use-query-params"; +import isEqual from "lodash/isEqual"; const { Title } = Typography; +const hasResolvedAtValue = (value) => { + if (!value) { + return false; + } + const resolvedAt = moment(value); + return resolvedAt.isValid() && resolvedAt.year() > 1; +}; + +const getAlertStartTime = (messageDetail = {}) => + hasResolvedAtValue(messageDetail?.trigger_at) ? messageDetail.trigger_at : messageDetail?.created; + +const getAlertEndTime = (messageDetail = {}) => { + if (hasResolvedAtValue(messageDetail?.resolve_at)) { + return messageDetail.resolve_at; + } + if (messageDetail?.status == "recovered") { + return messageDetail?.updated; + } + return ""; +}; + const MessageDetail = (props) => { const messageID = props?.messageID; @@ -49,22 +71,33 @@ const MessageDetail = (props) => { const [refresh, setRefresh] = useState({ isRefreshPaused: true }); const [timeZone, setTimeZone] = useState(() => getTimezone()); + const syncedTimeRange = useMemo( + () => ({ + min: timeRange.min, + max: timeRange.max, + }), + [timeRange.max, timeRange.min] + ); - useMemo(() => { - setParam({ ...param, timeRange: timeRange }); - }, [timeRange]); + useEffect(() => { + if (isEqual(param?.timeRange, syncedTimeRange)) { + return; + } + setParam({ ...param, timeRange: syncedTimeRange }); + }, [param, setParam, syncedTimeRange]); const updateTimeRange = (messageDetail) => { - let startTimestamp = moment(messageDetail.created).valueOf(); + let startTimestamp = moment(getAlertStartTime(messageDetail)).valueOf(); let endTimestamp = moment().valueOf(); + const resolvedAt = getAlertEndTime(messageDetail); - if (messageDetail?.status == "recovered") { - endTimestamp = moment(messageDetail.updated).valueOf(); + if (hasResolvedAtValue(resolvedAt)) { + endTimestamp = moment(resolvedAt).valueOf(); } setTimeRange({ ...timeRange, - min: moment(startTimestamp).format(), - max: moment(endTimestamp).format("YYYY-MM-DDTHH:mm:ss.SSS"), + min: moment(startTimestamp).toISOString(), + max: moment(endTimestamp).toISOString(), }); } @@ -82,19 +115,21 @@ const MessageDetail = (props) => { } }, []); const history = useHistory(); + const location = useLocation(); + const backTo = location?.state?.from || "/alerting/message"; return ( { - history.goBack(); + history.push(backTo); }} > {formatMessage({ id: "form.button.goback" })} }>
- +
@@ -129,9 +164,16 @@ const MessageDetail = (props) => { + from: timeRange.min || "auto", + to: timeRange.max || "auto" + }} + onRangeChange={({ from, to }) => { + handleTimeChange({ + start: from, + end: to, + }); + }} + />
{ {messageDetail?.rule_id ? { + if (!queries?.range?.from || !queries?.range?.to) { + return; + } + handleTimeChange({ + start: queries.range.from, + end: queries.range.to, + }); }} - queryParams={{state:"alerting", rule_id: messageDetail?.rule_id}} /> : null}
{messageDetail.message_id && - - + + } diff --git a/web/src/pages/Alerting/Message/components/EventDetailCard.jsx b/web/src/pages/Alerting/Message/components/EventDetailCard.jsx index 58aa7239..0c39337d 100644 --- a/web/src/pages/Alerting/Message/components/EventDetailCard.jsx +++ b/web/src/pages/Alerting/Message/components/EventDetailCard.jsx @@ -5,9 +5,35 @@ import { PriorityColor } from "../../utils/constants"; import { formatMessage } from "umi/locale"; import EventMessageStatus from "./EventMessageStatus"; +const calcSafeDuration = (msgItem) => { + const triggerAt = msgItem?.trigger_at; + const resolveAt = msgItem?.updated; + + const start = moment(triggerAt); + const end = resolveAt ? moment(resolveAt) : moment(); + + if (!start.isValid() || !end.isValid()) return "-"; + + const diffMs = end.diff(start); + + if (diffMs < 0) return "-"; + + return moment.duration(diffMs).humanize(); +}; + +const isValidAlertTime = (value) => { + if (!value) { + return false; + } + const parsed = moment(value); + return parsed.isValid() && parsed.year() > 1; +}; + export default ({msgItem})=>{ const labelSpan = 6; const vSpan = 18; + const triggerAt = isValidAlertTime(msgItem?.trigger_at) ? msgItem.trigger_at : msgItem?.created; + const resolveAt = isValidAlertTime(msgItem?.resolve_at) ? msgItem.resolve_at : msgItem?.updated; const isBucketDiff = !!(msgItem && msgItem.bucket_conditions) @@ -39,15 +65,15 @@ export default ({msgItem})=>{ {formatMessage({ id: "alert.message.table.created" })} - {formatUtcTimeToLocal(msgItem?.created)} + {formatUtcTimeToLocal(triggerAt)} {msgItem.status === "recovered" ? {formatMessage({ id: "alert.message.detail.recover_time" })} - {formatUtcTimeToLocal(msgItem?.updated)} + {formatUtcTimeToLocal(resolveAt)}:null} {formatMessage({ id: "alert.message.table.duration" })} - {moment.duration(msgItem?.duration).humanize()} + {calcSafeDuration(msgItem)} {formatMessage({ id: "alert.message.detail.condition.type" })} @@ -67,7 +93,7 @@ export default ({msgItem})=>{ {formatMessage({ id: "alert.message.detail.updated" })} - {formatUtcTimeToLocal(msgItem?.updated)} + {formatUtcTimeToLocal(resolveAt)} diff --git a/web/src/pages/Alerting/Message/components/EventMessageCard.jsx b/web/src/pages/Alerting/Message/components/EventMessageCard.jsx index 82579177..b00ae633 100644 --- a/web/src/pages/Alerting/Message/components/EventMessageCard.jsx +++ b/web/src/pages/Alerting/Message/components/EventMessageCard.jsx @@ -2,12 +2,14 @@ import Markdown from "@/components/Markdown"; import { Card } from "antd"; import { useState, useCallback, useRef, useEffect } from "react"; import { formatMessage } from "umi/locale"; +import { stripDuplicatedAlertTitle } from "../../utils/message"; -export default ({ message }) => { +export default ({ message, title }) => { const [state, setState] = useState({ hasMore: false, style: { maxHeight: 110, overflowY: "hidden" }, }); + const content = stripDuplicatedAlertTitle(message, title); const itemRef = useCallback((node) => { setTimeout(() => { if (node && node.scrollHeight > node.offsetHeight) { @@ -26,7 +28,7 @@ export default ({ message }) => { title={formatMessage({ id: "alert.rule.form.label.event_message" })} >
- +
{state.hasMore ? (
diff --git a/web/src/pages/Alerting/Message/components/EventMessageStatus.jsx b/web/src/pages/Alerting/Message/components/EventMessageStatus.jsx index cebc0770..6b2fe92c 100644 --- a/web/src/pages/Alerting/Message/components/EventMessageStatus.jsx +++ b/web/src/pages/Alerting/Message/components/EventMessageStatus.jsx @@ -1,22 +1,43 @@ import { Tag, Tooltip, Icon } from "antd"; import { MessageStautsColor } from "../../utils/constants"; -import { formatUtcTimeToLocal, firstUpperCase } from "@/utils/utils"; +import { formatUtcTimeToLocal } from "@/utils/utils"; +import { formatMessage } from "umi/locale"; export default ({ record }) => { const text = record.status; + const statusLabel = formatMessage({ + id: `alert.message.status.${text}`, + defaultMessage: text, + }); const title = (
- Ignored time: {formatUtcTimeToLocal(record.ignored_time)} + + {formatMessage({ id: "alert.message.ignored.time" })}:{" "} + {formatUtcTimeToLocal(record.ignored_time)} +
- Operator: {record.ignored_user} + + {formatMessage({ id: "alert.message.ignored.operator" })}:{" "} + {record.ignored_user} +
-

Message: {record.ignored_reason}

+

+ {formatMessage({ id: "alert.message.ignored.reason" })}:{" "} + {record.ignored_reason} +

); return ( -
+
- {firstUpperCase(text)} + {statusLabel} {text === "ignored" ? ( diff --git a/web/src/pages/Alerting/Message/components/ExpressionCard.jsx b/web/src/pages/Alerting/Message/components/ExpressionCard.jsx index c5f00f1d..0f8219f1 100644 --- a/web/src/pages/Alerting/Message/components/ExpressionCard.jsx +++ b/web/src/pages/Alerting/Message/components/ExpressionCard.jsx @@ -3,12 +3,19 @@ import { hasAuthority } from "@/utils/authority"; import { Card, Divider, Icon } from "antd"; import { Link } from "umi"; import { formatMessage } from "umi/locale"; +import { useLocation } from "react-router-dom"; export default ({expression, ruleID})=>{ + const location = useLocation(); + const ruleDetailLink = { + pathname: `/alerting/rule/${ruleID}`, + search: `?back_to=${encodeURIComponent(`${location.pathname}${location.search}`)}`, + }; + return ( - {formatMessage({ id: "form.button.view" })} + {formatMessage({ id: "form.button.view" })} {hasAuthority("alerting.rule:all")? <> diff --git a/web/src/pages/Alerting/Message/components/MessageCard.jsx b/web/src/pages/Alerting/Message/components/MessageCard.jsx index cda03564..ae2bbe88 100644 --- a/web/src/pages/Alerting/Message/components/MessageCard.jsx +++ b/web/src/pages/Alerting/Message/components/MessageCard.jsx @@ -9,6 +9,7 @@ import ClusterName from "@/pages/System/Cluster/components/ClusterName"; import { useGlobalClusters } from "@/layouts/GlobalContext"; import Markdown from "@/components/Markdown"; import { Link } from "umi"; +import { useLocation } from "react-router-dom"; const DescriptionItem = ({ title, content }) => (
{ return null; } const data = props?.data || {}; + const location = useLocation(); + const ruleDetailLink = { + pathname: `/alerting/rule/${data.rule_id}`, + search: `?back_to=${encodeURIComponent(`${location.pathname}${location.search}`)}`, + }; const clusterM = useGlobalClusters(); @@ -50,7 +56,7 @@ const MessageCard = (props) => {
{data?.rule_name}} + content={{data?.rule_name}} /> diff --git a/web/src/pages/Alerting/Message/components/MessageDetail.jsx b/web/src/pages/Alerting/Message/components/MessageDetail.jsx index 3fc11c11..7bdcf5c3 100644 --- a/web/src/pages/Alerting/Message/components/MessageDetail.jsx +++ b/web/src/pages/Alerting/Message/components/MessageDetail.jsx @@ -15,6 +15,27 @@ import AlertChartCard from "./AlertChartCard"; const { Title } = Typography; +const hasResolvedAtValue = (value) => { + if (!value) { + return false; + } + const resolvedAt = moment(value); + return resolvedAt.isValid() && resolvedAt.year() > 1; +}; + +const getAlertStartTime = (messageDetail = {}) => + hasResolvedAtValue(messageDetail?.trigger_at) ? messageDetail.trigger_at : messageDetail?.created; + +const getAlertEndTime = (messageDetail = {}) => { + if (hasResolvedAtValue(messageDetail?.resolve_at)) { + return messageDetail.resolve_at; + } + if (messageDetail?.status == "recovered") { + return messageDetail?.updated; + } + return ""; +}; + const MessageDetail = (props) => { const messageID = props?.messageID; @@ -40,22 +61,34 @@ const MessageDetail = (props) => { }); const updateTimeRange = (messageDetail) => { - let startTimestamp = moment(messageDetail.created).valueOf(); + let startTimestamp = moment(getAlertStartTime(messageDetail)).valueOf(); let endTimestamp = moment().valueOf(); + const resolvedAt = getAlertEndTime(messageDetail); - if (messageDetail?.status == "recovered") { - endTimestamp = moment(messageDetail.updated).valueOf(); + if (hasResolvedAtValue(resolvedAt)) { + endTimestamp = moment(resolvedAt).valueOf(); } - const duration = moment(messageDetail.updated).valueOf() - moment(messageDetail.created).valueOf() + const duration = Math.max(endTimestamp - startTimestamp, 0); setTimeRange({ ...timeRange, - min: moment(startTimestamp).subtract(duration, 'ms').format("YYYY-MM-DDTHH:mm:ss.SSS"), - max: moment(endTimestamp).add(duration, 'ms').format("YYYY-MM-DDTHH:mm:ss.SSS"), + min: moment(startTimestamp).subtract(duration, 'ms').toISOString(), + max: moment(endTimestamp).add(duration, 'ms').toISOString(), }); } + const handleChartRangeChange = ({ from, to }) => { + if (!from || !to) { + return; + } + setTimeRange((previous) => ({ + ...previous, + min: from, + max: to, + })); + }; + useEffect(() => { if (messageID) { fetchMessageDetail(messageID); @@ -76,7 +109,7 @@ const MessageDetail = (props) => { {} + }} onRangeChange={handleChartRangeChange} />} diff --git a/web/src/pages/Alerting/Message/components/NotificationCard.jsx b/web/src/pages/Alerting/Message/components/NotificationCard.jsx index 57c6c9b9..1ece5024 100644 --- a/web/src/pages/Alerting/Message/components/NotificationCard.jsx +++ b/web/src/pages/Alerting/Message/components/NotificationCard.jsx @@ -50,8 +50,8 @@ export default ({msgItem})=>{ {formatMessage({id:"alert.rule.lable.alerting_channels"})}
Sent on: {formatUtcTimeToLocal(lastTime)}
- {normalStats.map((item)=>{ - return
+ {normalStats.map((item, index)=>{ + return
{item.channel_name} @@ -65,8 +65,8 @@ export default ({msgItem})=>{ {value?.alerting?.escalation_throttle_period ? {formatMessage({id:"alert.rule.label.wait_time"})}: {value?.alerting?.escalation_throttle_period}: null} {willSend ?
It will be sent at：{formatUtcTimeToLocal(willSend)}
: null}
- {escalationStats.map((item)=>{ - return
+ {escalationStats.map((item, index)=>{ + return
{item.channel_name} @@ -90,8 +90,8 @@ export default ({msgItem})=>{ {formatMessage({id:"alert.rule.lable.recovery_channels"})} {recoveryTime ?
Sent on: {formatUtcTimeToLocal(recoveryTime)}
: null}
- {recoverStats.map((item)=>{ - return
+ {recoverStats.map((item, index)=>{ + return
{item.channel_name}
@@ -122,4 +122,4 @@ const getChannelIcon = (typ) => { return Discord; } return Slack; -} \ No newline at end of file +} diff --git a/web/src/pages/Alerting/Message/components/RuleRecordChart.jsx b/web/src/pages/Alerting/Message/components/RuleRecordChart.jsx index cee147b9..9871447e 100644 --- a/web/src/pages/Alerting/Message/components/RuleRecordChart.jsx +++ b/web/src/pages/Alerting/Message/components/RuleRecordChart.jsx @@ -27,6 +27,14 @@ import { MonitorDatePicker } from "@/components/infini/MonitorDatePicker"; import { calculateBounds } from "@/components/vendor/data/common/query/timefilter"; import metricsStyles from "@/pages/Cluster/Metrics.scss"; +const resolveAlertTime = (value) => { + if (!value) { + return ""; + } + const parsed = moment(value); + return parsed.isValid() && parsed.year() > 1 ? value : ""; +}; + const RuleRecordChart = ({ data: messageDetail }) => { if (!messageDetail?.message_id) { return null; @@ -43,21 +51,23 @@ const RuleRecordChart = ({ data: messageDetail }) => { }); const [timeRange, setTimeRange] = React.useState({ - min: "now-1d", - max: "now", + min: "auto", + max: "auto", timeFormatter: formatter.dates(1), }); const [metricData, setMetricData] = useState({}); const [lineAnnotations] = useMemo(() => { - let startTimestamp = moment(messageDetail.created).valueOf(); + const startTime = resolveAlertTime(messageDetail.trigger_at) || messageDetail.created; + const endTime = resolveAlertTime(messageDetail.resolve_at) || messageDetail.updated; + let startTimestamp = moment(startTime).valueOf(); let endTimestamp = moment().valueOf(); let from = startTimestamp - parseInt(messageDetail.duration / 4); let to = endTimestamp; - if (messageDetail?.status == "recovered") { - endTimestamp = moment(messageDetail.updated).valueOf(); + if (messageDetail?.status == "recovered" && endTime) { + endTimestamp = moment(endTime).valueOf(); to = endTimestamp + parseInt(messageDetail.duration / 4); } setTimeRange({ @@ -112,7 +122,7 @@ const RuleRecordChart = ({ data: messageDetail }) => { }); const fetchData = async () => { - let url = `/alerting/rule/${ruleID}/metric`; + let url = `/alerting/rule/${ruleID}/history_metric`; const res = await request(url, { method: "GET", queryParams: { diff --git a/web/src/pages/Alerting/Message/components/RuleRecords.jsx b/web/src/pages/Alerting/Message/components/RuleRecords.jsx index c11767b0..94012259 100644 --- a/web/src/pages/Alerting/Message/components/RuleRecords.jsx +++ b/web/src/pages/Alerting/Message/components/RuleRecords.jsx @@ -4,7 +4,7 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { formatESSearchResult } from "@/lib/elasticsearch/util"; import Link from "umi/link"; import request from "@/utils/request"; -import { formatUtcTimeToLocal, firstUpperCase } from "@/utils/utils"; +import { formatUtcTimeToLocal } from "@/utils/utils"; import { HealthStatusView } from "@/components/infini/health_status_view"; import { MessageStautsColor, @@ -18,7 +18,16 @@ import { PriorityIconText } from "../../components/Statistic"; import WidgetLoader from "@/pages/DataManagement/View/WidgetLoader"; const Option = Select.Option; -const RuleRecords = ({ ruleID, timeRange, showAertMetric = false, refresh }) => { +const RuleRecords = ({ + ruleID, + resourceID, + resolveEventID, + messageStatus, + timeRange, + showAertMetric = false, + refresh, + onTimeRangeChange, +}) => { if (!ruleID || !timeRange.min) { return null; } @@ -26,13 +35,14 @@ const RuleRecords = ({ ruleID, timeRange, showAertMetric = false, refresh }) => const [loading, setLoading] = React.useState(true); const bounds = calculateBounds({ - from: timeRange.min || "now-1d", - to: timeRange.max || "now", + from: timeRange.min || "auto", + to: timeRange.max || "auto", }); const initialQueryParams = { from: 0, size: 10, rule_id: ruleID, + resource_id: resourceID, min: bounds.min.valueOf(), max: bounds.max.valueOf(), }; @@ -70,6 +80,16 @@ const RuleRecords = ({ ruleID, timeRange, showAertMetric = false, refresh }) => const onRefreshClick = () => { dispatch({ type: "refresh" }); }; + + const onWidgetQueriesChange = (queries = {}) => { + if (!queries?.range?.from || !queries?.range?.to || typeof onTimeRangeChange !== "function") { + return; + } + onTimeRangeChange({ + start: queries.range.from, + end: queries.range.to, + }); + }; const [queryParams, dispatch] = React.useReducer( alertReducer, initialQueryParams @@ -87,18 +107,26 @@ const RuleRecords = ({ ruleID, timeRange, showAertMetric = false, refresh }) => title: formatMessage({ id: "alert.message.table.execution_status" }), dataIndex: "state", render: (text, record) => { + const displayState = + messageStatus === "recovered" && record.id === resolveEventID + ? "recovered" + : record.display_state || text; + const displayLabel = formatMessage({ + id: `alert.message.status.${displayState}`, + defaultMessage: displayState, + }); return (
- {firstUpperCase(text)} + {displayLabel} - {text != "ok" ? ( + {displayState != "ok" && displayState != "recovered" ? ( ) : null}
@@ -183,7 +211,7 @@ const RuleRecords = ({ ruleID, timeRange, showAertMetric = false, refresh }) => dispatch({ type: "state", value: value }); }} > - {Object.keys(RuleStautsColor).map((item) => { + {Object.keys(RuleStautsColor).filter((item) => item !== "recovered").map((item) => { return (
) : null} diff --git a/web/src/pages/Alerting/Rule/Edit.jsx b/web/src/pages/Alerting/Rule/Edit.jsx index a03b4fcc..15445e1c 100644 --- a/web/src/pages/Alerting/Rule/Edit.jsx +++ b/web/src/pages/Alerting/Rule/Edit.jsx @@ -13,18 +13,20 @@ export default Form.create({ name: "rule_form_edit" })((props) => { const { loading, error, value } = useFetch( `/alerting/rule/${ruleID}`, null, - [] + [ruleID] ); const onSaveClick = useCallback( async (values) => { setSubmitLoading(true) - if (value._source.alert_objects) { - delete value._source.alert_objects; + const sourceValue = { ...(value?._source || {}) }; + if (sourceValue.alert_objects) { + delete sourceValue.alert_objects; } const newVal = { - ...value._source, + ...sourceValue, ...values[0], + updated: new Date().toISOString(), }; const saveRes = await request(`/alerting/rule/${ruleID}`, { diff --git a/web/src/pages/Alerting/Rule/Form.jsx b/web/src/pages/Alerting/Rule/Form.jsx index 525f2d37..3cae60fd 100644 --- a/web/src/pages/Alerting/Rule/Form.jsx +++ b/web/src/pages/Alerting/Rule/Form.jsx @@ -67,13 +67,16 @@ const tailFormItemLayout = { }; const RuleForm = (props) => { - const { submitLoading } = props; + const { submitLoading, clusterList = [] } = props; const editValue = props.value || {}; - const { getFieldDecorator } = props.form; + const { getFieldDecorator, setFieldsValue } = props.form; const history = useHistory(); const [recoveryEnabled, setRecoveryEnabled] = useState( editValue?.recovery_notification_config?.event_enabled || false ); + const [ignoreTimeFilter, setIgnoreTimeFilter] = useState( + Boolean(editValue?.resource?.ignore_time_filter) + ); const [objectFields, setObjectFields] = useState({}); const [objectFieldsQueryParams, setObjectFieldsQueryParams] = useState({ @@ -272,74 +275,71 @@ const RuleForm = (props) => { // }, 200); }, [props.form]); - const handleSubmit = useCallback( - (parmas) => { - props.form.validateFields((err, values) => { - if (err) { - if (parmas.is_test) { - message.error("please check rule config!"); - } - return false; + const handleSubmit = (parmas) => { + props.form.validateFields((err, values) => { + if (err) { + if (parmas.is_test) { + message.error("please check rule config!"); } + return false; + } - let newValues = cloneDeep(values); - - switch (parmas?.category) { - case "notification": - newValues.notification_config[ - "normal" - ] = newValues.notification_config["normal"] - .filter((item, i) => i == parmas.channel_index) - .map((item) => ({ - ...item, - ...(parmas?.channel || {}), - enabled: true, - })); - newValues.notification_config["escalation"] = []; - newValues.recovery_notification_config["normal"] = []; - break; - case "escalation": - newValues.notification_config[ - "escalation" - ] = newValues.notification_config["escalation"] - .filter((item, i) => i == parmas.channel_index) - .map((item) => ({ - ...item, - ...(parmas?.channel || {}), - enabled: true, - })); - newValues.notification_config["normal"] = []; - newValues.recovery_notification_config["normal"] = []; - break; - case "recover_notification": - newValues.recovery_notification_config[ - "normal" - ] = newValues.recovery_notification_config["normal"] - .filter((item, i) => i == parmas.channel_index) - .map((item) => ({ - ...item, - ...(parmas?.channel || {}), - enabled: true, - })); - newValues.notification_config["normal"] = []; - newValues.notification_config["escalation"] = []; - break; - } + let newValues = cloneDeep(values); - const alert_objects = formatAlertObjects(newValues); + switch (parmas?.category) { + case "notification": + newValues.notification_config[ + "normal" + ] = newValues.notification_config["normal"] + .filter((item, i) => i == parmas.channel_index) + .map((item) => ({ + ...item, + ...(parmas?.channel || {}), + enabled: true, + })); + newValues.notification_config["escalation"] = []; + newValues.recovery_notification_config["normal"] = []; + break; + case "escalation": + newValues.notification_config[ + "escalation" + ] = newValues.notification_config["escalation"] + .filter((item, i) => i == parmas.channel_index) + .map((item) => ({ + ...item, + ...(parmas?.channel || {}), + enabled: true, + })); + newValues.notification_config["normal"] = []; + newValues.recovery_notification_config["normal"] = []; + break; + case "recover_notification": + newValues.recovery_notification_config[ + "normal" + ] = newValues.recovery_notification_config["normal"] + .filter((item, i) => i == parmas.channel_index) + .map((item) => ({ + ...item, + ...(parmas?.channel || {}), + enabled: true, + })); + newValues.notification_config["normal"] = []; + newValues.notification_config["escalation"] = []; + break; + } - if (parmas?.is_test) { - onSendTestClick(alert_objects[0], parmas?.category); - return; - } + const alert_objects = formatAlertObjects(newValues); - if (typeof props.onSaveClick == "function") { - props.onSaveClick(alert_objects); - } - }); - }, - [props.form] - ); + if (parmas?.is_test) { + onSendTestClick(alert_objects[0], parmas?.category); + return; + } + + if (typeof props.onSaveClick == "function") { + props.onSaveClick(alert_objects); + } + }); + }; const [testState, setTestState] = useState({ loading: false, result: "" }); const onSendTestClick = useCallback(async (values, type) => { @@ -383,14 +383,24 @@ const RuleForm = (props) => { editValue?.resource?.objects || [] ); const selectedClusterDefault = editValue.id - ? { + ? clusterList.find( + (item) => item.id === editValue?.resource?.resource_id + ) || { id: editValue.resource.resource_id, name: editValue.resource.resource_name, + distribution: editValue?.resource?.distribution, } : props.selectedCluster; - const [selectedCluster, setSelectedCluster] = useState( - selectedClusterDefault - ); + const [selectedCluster, setSelectedCluster] = useState(selectedClusterDefault); + useEffect(() => { + if (!editValue.id) return; + const selectedClusterFromList = clusterList.find( + (item) => item.id === editValue?.resource?.resource_id + ); + if (selectedClusterFromList) { + setSelectedCluster(selectedClusterFromList); + } + }, [clusterList, editValue.id, editValue?.resource?.resource_id]); if (!editValue.id) { useMemo(() => { setSelectedCluster(props.selectedCluster); @@ -478,7 +488,7 @@ const RuleForm = (props) => { > {props.clusterList.length > 0 ? ( { @@ -529,7 +539,9 @@ const RuleForm = (props) => { })} extra={ { } trigger="click" > - Example + + {formatMessage({ + id: "alert.rule.form.title.example", + })} + } > @@ -602,6 +618,30 @@ const RuleForm = (props) => {
+ + {getFieldDecorator("resource.ignore_time_filter", { + valuePropName: "checked", + initialValue: ignoreTimeFilter, + })( + { + setIgnoreTimeFilter(checked); + if (checked) { + setFieldsValue({ + "resource.time_field": undefined, + }); + } + }} + /> + )} + { initialValue: editValue?.resource?.time_field, rules: [ { - required: true, + required: !ignoreTimeFilter, message: "Please select time field!", }, ], })(
{ setImportVisible(false)} diff --git a/web/src/pages/Alerting/Rule/components/NotificationCard.jsx b/web/src/pages/Alerting/Rule/components/NotificationCard.jsx index 98374450..2c6bcf94 100644 --- a/web/src/pages/Alerting/Rule/components/NotificationCard.jsx +++ b/web/src/pages/Alerting/Rule/components/NotificationCard.jsx @@ -32,9 +32,9 @@ export default ({notificationConfig, recoverNotificationConfig})=>{ }> {formatMessage({id:"alert.rule.lable.alerting_channels"})}
- {(notificationConfig?.normal || []).map((item)=>{ + {(notificationConfig?.normal || []).map((item, index)=>{ const color = item.enabled === true ? '': "rgb(187, 187, 187)"; - return
+ return
{item.name}
@@ -45,9 +45,9 @@ export default ({notificationConfig, recoverNotificationConfig})=>{ {formatMessage({id:"alert.rule.lable.escalation_channels"})} {notificationConfig?.escalation_throttle_period ? {formatMessage({id:"alert.rule.label.wait_time"})}: {notificationConfig?.escalation_throttle_period}: null}
- {(notificationConfig?.escalation || []).map((item)=>{ + {(notificationConfig?.escalation || []).map((item, index)=>{ const color = item.enabled === true ? '': "rgb(187, 187, 187)"; - return
+ return
{item.name}
@@ -69,9 +69,9 @@ export default ({notificationConfig, recoverNotificationConfig})=>{ }> {formatMessage({id:"alert.rule.lable.recovery_channels"})}
- {(recoverNotificationConfig?.normal || []).map((item)=>{ + {(recoverNotificationConfig?.normal || []).map((item, index)=>{ const color = item.enabled === true ? '': "rgb(187, 187, 187)"; - return
+ return
{item.name}
@@ -104,4 +104,4 @@ const getChannelIcon = (typ, color) => { return Discord; } return Slack; -} \ No newline at end of file +} diff --git a/web/src/pages/Alerting/Rule/components/RuleDetail.jsx b/web/src/pages/Alerting/Rule/components/RuleDetail.jsx index 72599f76..55f02f2a 100644 --- a/web/src/pages/Alerting/Rule/components/RuleDetail.jsx +++ b/web/src/pages/Alerting/Rule/components/RuleDetail.jsx @@ -11,9 +11,10 @@ import { Tabs, } from "antd"; import Link from "umi/link"; +import router from "umi/router"; import { formatMessage } from "umi/locale"; import { useCallback, useEffect, useMemo, useState, useRef } from "react"; -import { useHistory } from "react-router-dom"; +import { useLocation } from "react-router-dom"; import request from "@/utils/request"; import { formatter, getFormatter } from "@/utils/format"; import { calculateBounds } from "@/components/vendor/data/common/query/timefilter"; @@ -30,7 +31,6 @@ import ExternalLink from "@/components/Icons/ExternalLink"; import NotificationCard from "./NotificationCard"; import Sum from "@/components/Icons/Sum"; import WidgetLoader, { - WidgetRender, } from "@/pages/DataManagement/View/WidgetLoader"; import MessageRecord from "./MessageRecord"; import { hasAuthority } from "@/utils/authority"; @@ -38,9 +38,21 @@ import DatePicker from "@/common/src/DatePicker"; import { getLocale } from "umi/locale"; import { getTimezone } from "@/utils/utils"; import moment from "moment"; +import isEqual from "lodash/isEqual"; const { Title } = Typography; +const formatRuleDetailTime = (value, fallback) => { + const parsed = moment(value); + if (parsed.isValid() && parsed.year() > 1) { + return formatUtcTimeToLocal(value); + } + if (fallback) { + return formatUtcTimeToLocal(fallback); + } + return "-"; +}; + export const buildWidgetByRule = (rule, queries, created, updated) => { if (!rule) return; @@ -60,6 +72,7 @@ export const buildWidgetByRule = (rule, queries, created, updated) => { pattern: "0.00%", }, }; + let query; try { // handle empty raw_filter @@ -82,6 +95,10 @@ export const buildWidgetByRule = (rule, queries, created, updated) => { } } + const ignoreTimeFilter = Boolean( + queries?.ignore_time_filter || queries?.ignoreTimeFilter + ); + const config = { bucket_size: bucketSize, format: formatMapping[format_type], @@ -102,7 +119,7 @@ export const buildWidgetByRule = (rule, queries, created, updated) => { queries: { cluster_id: queries.cluster_id, indices: queries.indices, - time_field: queries.time_field, + time_field: ignoreTimeFilter ? undefined : queries.time_field, dsl: query, }, type: "line", @@ -120,13 +137,17 @@ const RuleDetail = (props) => { if (!ruleID) { return null; } + const location = useLocation(); const [param, setParam] = useQueryParam("_g", JsonParam); - const history = useHistory(); + const backTo = useMemo(() => { + const searchParams = new URLSearchParams(location.search); + return searchParams.get("back_to"); + }, [location.search]); const [state, setState] = React.useState({ spinning: false, timeRange: { - min: param?.timeRange?.min || "now-7d", - max: param?.timeRange?.max || "now", + min: param?.timeRange?.min || "auto", + max: param?.timeRange?.max || "auto", timeFormatter: formatter.dates(1), }, }); @@ -135,10 +156,20 @@ const RuleDetail = (props) => { const [refresh, setRefresh] = useState({ isRefreshPaused: true }); const [timeZone, setTimeZone] = useState(() => getTimezone()); + const syncedTimeRange = useMemo( + () => ({ + min: state.timeRange.min, + max: state.timeRange.max, + }), + [state.timeRange.max, state.timeRange.min] + ); - useMemo(() => { - setParam({ ...param, timeRange: state.timeRange }); - }, [state.timeRange]); + useEffect(() => { + if (isEqual(param?.timeRange, syncedTimeRange)) { + return; + } + setParam({ ...param, timeRange: syncedTimeRange }); + }, [param, setParam, syncedTimeRange]); const handleTimeChange = ({ start, end, refresh }) => { setState({ @@ -153,6 +184,16 @@ const RuleDetail = (props) => { }); }; + const handleWidgetQueriesChange = (queries = {}) => { + if (!queries?.range?.from || !queries?.range?.to) { + return; + } + handleTimeChange({ + start: queries.range.from, + end: queries.range.to, + }); + }; + const [ruleDetail, setRuleDetail] = useState({}); const fetcDetail = (id) => { const fetchData = async () => { @@ -197,16 +238,6 @@ const RuleDetail = (props) => { }, 2000); }, []); - const widget = useMemo(() => { - if (!ruleDetail || !ruleDetail.rule_name) return; - return buildWidgetByRule(ruleDetail, { - cluster_id: ruleDetail.resource_id, - indices: ruleDetail.resource_objects, - time_field: ruleDetail.resource_time_field, - raw_filter: ruleDetail.resource_raw_filter, - }, ruleDetail?.created, ruleDetail?.updated); - }, [ruleDetail]); - return (
{ style={{ marginLeft: 20 }} type="primary" onClick={() => { - history.goBack(); + router.push(backTo || "/alerting/rule"); }} > {formatMessage({ id: "form.button.goback" })} @@ -241,8 +272,8 @@ const RuleDetail = (props) => { {formatMessage( { id: "alert.rule.detail.title.changed_desc" }, { - updated: formatUtcTimeToLocal(ruleDetail?.updated), - created: formatUtcTimeToLocal(ruleDetail?.created), + updated: formatRuleDetailTime(ruleDetail?.updated, ruleDetail?.created), + created: formatRuleDetailTime(ruleDetail?.created), user: ruleDetail?.creator?.name, } )} @@ -265,7 +296,12 @@ const RuleDetail = (props) => {
{ruleDetail?.alerting_message ? ( { bodyStyle={{ height: 250, padding: 1 }} > {ruleDetail.rule_name ? ( - ) : ( @@ -389,15 +423,16 @@ const RuleDetail = (props) => { })} bodyStyle={{ height: 250, padding: 1 }} > - +
@@ -416,7 +451,12 @@ const RuleDetail = (props) => { key="alerts" tab={formatMessage({ id: "alert.rule.detail.title.alert_event" })} > - + { timeRange={state.timeRange} showAertMetric={true} refresh={state.refresh} + onTimeRangeChange={handleTimeChange} /> diff --git a/web/src/pages/Alerting/Rule/components/RuleMetricChart.jsx b/web/src/pages/Alerting/Rule/components/RuleMetricChart.jsx index 3ef3fa53..75962be8 100644 --- a/web/src/pages/Alerting/Rule/components/RuleMetricChart.jsx +++ b/web/src/pages/Alerting/Rule/components/RuleMetricChart.jsx @@ -1,4 +1,4 @@ -import { Table, Button, Divider, Tag, Icon } from "antd"; +import { Table, Button, Divider, Tag, Icon, Empty, Tooltip, message } from "antd"; import { Axis, Chart, @@ -27,12 +27,22 @@ import { MonitorDatePicker } from "@/components/infini/MonitorDatePicker"; import { calculateBounds } from "@/components/vendor/data/common/query/timefilter"; import metricsStyles from "@/pages/Cluster/Metrics.scss"; import _ from "lodash"; +import { CopyToClipboard } from "react-copy-to-clipboard"; + +const buildCopyRequestText = (requestPayload) => { + const index = requestPayload?.index; + const query = requestPayload?.query; + if (!index || !query) { + return ""; + } + return `GET ${index}/_search\n${JSON.stringify(query, null, 2)}`; +}; const RuleMetricChart = ({ conditions, values }) => { const [timeRange, setTimeRange] = React.useState( timeRange || { - min: "now-15m", - max: "now", + min: "auto", + max: "auto", timeFormatter: formatter.dates(1), } ); @@ -46,6 +56,13 @@ const RuleMetricChart = ({ conditions, values }) => { }); const [metricData, setMetricData] = useState({}); + const [latestRequest, setLatestRequest] = useState(""); + const hasMetricData = useMemo(() => { + if (!Array.isArray(metricData?.lines)) { + return false; + } + return metricData.lines.some((line) => Array.isArray(line?.data) && line.data.length > 0); + }, [metricData]); const [lineAnnotations] = useMemo(() => { //LineAnnotation @@ -90,6 +107,7 @@ const RuleMetricChart = ({ conditions, values }) => { }); if (res && !res.error) { setMetricData(res.metric); + setLatestRequest(buildCopyRequestText(res.request)); } console.log("preview_metric res:", res); }; @@ -120,8 +138,34 @@ const RuleMetricChart = ({ conditions, values }) => { let disableHeaderFormat = false; let headerUnit = ""; + if (!hasMetricData) { + return ( +
+ +
+ ); + } + return ( -
+
{ ); })} + {latestRequest ? ( +
+ + message.success(formatMessage({ id: "cluster.metrics.request.copy.success" })) + } + > + + + + +
+ ) : null}
); }; diff --git a/web/src/pages/Alerting/Rule/components/RuleRecordChart.jsx b/web/src/pages/Alerting/Rule/components/RuleRecordChart.jsx index c4c65d3b..b26a736a 100644 --- a/web/src/pages/Alerting/Rule/components/RuleRecordChart.jsx +++ b/web/src/pages/Alerting/Rule/components/RuleRecordChart.jsx @@ -1,4 +1,4 @@ -import { Table, Button, Divider, Tag, Icon } from "antd"; +import { Table, Button, Divider, Tag, Icon, Tooltip, message, Empty } from "antd"; import { Axis, Chart, @@ -26,6 +26,16 @@ import { PriorityColor, RuleStautsColor } from "../../utils/constants"; import { MonitorDatePicker } from "@/components/infini/MonitorDatePicker"; import { calculateBounds } from "@/components/vendor/data/common/query/timefilter"; import metricsStyles from "@/pages/Cluster/Metrics.scss"; +import { CopyToClipboard } from "react-copy-to-clipboard"; + +const buildCopyRequestText = (requestPayload) => { + const index = requestPayload?.index; + const query = requestPayload?.query; + if (!index || !query) { + return ""; + } + return `GET ${index}/_search\n${JSON.stringify(query, null, 2)}`; +}; const RuleRecordChart = ({ ruleID, timeRange, conditions, clusterID }) => { if (!ruleID) { @@ -41,6 +51,13 @@ const RuleRecordChart = ({ ruleID, timeRange, conditions, clusterID }) => { }); const [metricData, setMetricData] = useState({}); + const [latestRequest, setLatestRequest] = useState(""); + const hasMetricData = useMemo(() => { + if (!Array.isArray(metricData?.lines)) { + return false; + } + return metricData.lines.some((line) => Array.isArray(line?.data) && line.data.length > 0); + }, [metricData]); const [lineAnnotations] = useMemo(() => { //LineAnnotation @@ -74,7 +91,7 @@ const RuleRecordChart = ({ ruleID, timeRange, conditions, clusterID }) => { }); const fetchData = async () => { - let url = `/alerting/rule/${ruleID}/metric`; + let url = `/alerting/rule/${ruleID}/history_metric`; const res = await request(url, { method: "GET", queryParams: { @@ -84,6 +101,7 @@ const RuleRecordChart = ({ ruleID, timeRange, conditions, clusterID }) => { }); if (res && !res.error) { setMetricData(res.metric); + setLatestRequest(buildCopyRequestText(res.request)); if(res.bucket_label && res.bucket_label.enabled === true){ fetchBucketLabels(res.metric, res.bucket_label.template); } @@ -122,8 +140,26 @@ const RuleRecordChart = ({ ruleID, timeRange, conditions, clusterID }) => { } }; + if (!hasMetricData) { + return ( +
+ +
+ ); + } + return ( -
+
{ ); })} + {latestRequest ? ( +
+ + message.success(formatMessage({ id: "cluster.metrics.request.copy.success" })) + } + > + + + + +
+ ) : null}
); }; diff --git a/web/src/pages/Alerting/components/ExportAndImport/index.jsx b/web/src/pages/Alerting/components/ExportAndImport/index.jsx index 9c480f21..38d3fdd6 100644 --- a/web/src/pages/Alerting/components/ExportAndImport/index.jsx +++ b/web/src/pages/Alerting/components/ExportAndImport/index.jsx @@ -121,6 +121,17 @@ export default Form.create()((props) => { }); }; + const clearUploadState = () => { + setUploadState((prev) => ({ + ...(prev || {}), + fileList: [], + data: undefined, + })); + form.setFieldsValue({ + upload: [], + }); + }; + const renderExportBody = () => { const associatedTypes = types .filter((item) => !item.isMain) @@ -157,11 +168,13 @@ export default Form.create()((props) => { if (Array.isArray(e)) { return e; } - setUploadState({ - ...(uploadState || {}), - fileList: [e.file], - }); - return e && e.fileList; + const fileList = e?.fileList ? e.fileList.slice(-1) : []; + setUploadState((prev) => ({ + ...(prev || {}), + fileList, + ...(fileList.length === 0 ? { data: undefined } : {}), + })); + return fileList; }; const renderImportBody = () => { @@ -170,7 +183,16 @@ export default Form.create()((props) => { fileList: uploadState?.fileList || [], multiple: false, name: "file", + onRemove() { + clearUploadState(); + return true; + }, onChange(info) { + const fileList = info?.fileList ? info.fileList.slice(-1) : []; + if (info.file.status === "removed" || fileList.length === 0) { + clearUploadState(); + return; + } if (info.file.status !== "uploading") { //cat json content let reader = new FileReader(); @@ -178,20 +200,21 @@ export default Form.create()((props) => { const jsonStr = e.target.result; try { const jsonObj = JSON.parse(jsonStr); - setUploadState({ - ...(uploadState || {}), + setUploadState((prev) => ({ + ...(prev || {}), + fileList, data: jsonObj, - }); + })); } catch { - message.error(`${info.file.name} is an invalid json file!`); + message.error(formatMessage({ id: "alert.import.upload.invalid_json" })); } }; reader.readAsText(info.file.originFileObj); } if (info.file.status === "done") { - message.success(`${info.file.name} file uploaded successfully`); + message.success(formatMessage({ id: "alert.import.upload.success" })); } else if (info.file.status === "error") { - message.error(`${info.file.name} file upload failed.`); + message.error(formatMessage({ id: "alert.import.upload.failed" })); } }, }; @@ -212,7 +235,7 @@ export default Form.create()((props) => { rules: [ { required: true, - message: "Please select file", + message: formatMessage({ id: "alert.import.select_file" }), }, ], })( @@ -226,7 +249,7 @@ export default Form.create()((props) => { {data && ( { bodyStyle={{ padding: 0, height: "calc(100vh - 110px)", - overflow: "auto", + overflow: "hidden", }} destroyOnClose > -
-
- {body} - -
-
- - +
+
+
+ {body} + +
+
+ + +
diff --git a/web/src/pages/Alerting/components/ExportAndImport/index.less b/web/src/pages/Alerting/components/ExportAndImport/index.less index 1c17b946..99214fd6 100644 --- a/web/src/pages/Alerting/components/ExportAndImport/index.less +++ b/web/src/pages/Alerting/components/ExportAndImport/index.less @@ -1,7 +1,5 @@ .actions { - position: absolute; - right: 0; - bottom: 0; + flex: none; width: 100%; border-top: 1px solid #e9e9e9; padding: 11px 16px; diff --git a/web/src/pages/Alerting/components/ExportAndImportDrawer.jsx b/web/src/pages/Alerting/components/ExportAndImportDrawer.jsx index 57b1472b..9eb34ac3 100644 --- a/web/src/pages/Alerting/components/ExportAndImportDrawer.jsx +++ b/web/src/pages/Alerting/components/ExportAndImportDrawer.jsx @@ -21,29 +21,38 @@ export default (props) => { bodyStyle={{ padding: 0, height: "calc(100vh - 110px)", - overflow: "auto", + overflow: "hidden", }} destroyOnClose > -
{children}
- - +
{children}
+
+ + +
); diff --git a/web/src/pages/Alerting/components/Import.jsx b/web/src/pages/Alerting/components/Import.jsx index 6b1de7e7..c1158c34 100644 --- a/web/src/pages/Alerting/components/Import.jsx +++ b/web/src/pages/Alerting/components/Import.jsx @@ -6,6 +6,7 @@ import { Icon, Menu, Select, + Tooltip, Upload, message, } from "antd"; @@ -31,6 +32,7 @@ export default Form.create()((props) => { visible = false, title, types = [], + exampleType, form, onSuccess, onClose, @@ -53,11 +55,11 @@ export default Form.create()((props) => { body: uploadState.data, }); if (res?.acknowledged) { - message.success("Imported succeed!"); + message.success(formatMessage({ id: "alert.import.submit.success" })); if (onClose) onClose(); if (onSuccess) onSuccess(); } else { - message.error("Imported failed!"); + message.error(formatMessage({ id: "alert.import.submit.failed" })); } setLoading(false); }; @@ -71,24 +73,73 @@ export default Form.create()((props) => { }); }; + const clearUploadState = () => { + setUploadState((prev) => ({ + ...(prev || {}), + fileList: [], + data: undefined, + })); + form.setFieldsValue({ + upload: [], + }); + }; + const normFile = (e) => { if (Array.isArray(e)) { return e; } - setUploadState({ - ...(uploadState || {}), - fileList: [e.file], - }); - return e && e.fileList; + const fileList = e?.fileList ? e.fileList.slice(-1) : []; + setUploadState((prev) => ({ + ...(prev || {}), + fileList, + ...(fileList.length === 0 ? { data: undefined } : {}), + })); + return fileList; }; const renderImportBody = () => { + const alertRuleExample = `{ + "metadatas": [ + { + "type": "AlertRule", + "items": [ + { + "id": "rule_cpu_high", + "name": "CPU High Alert", + "enabled": true, + "resource_id": "your_cluster_id", + "priority": "critical" + } + ] + }, + { + "type": "AlertChannel", + "items": [ + { + "id": "channel_email_default", + "name": "Default Email Channel" + } + ] + } + ] +}`; + const showRuleExample = false; + //const showRuleExample = exampleType === "AlertRule"; const uploadProps = { accept: "application/json", fileList: uploadState?.fileList || [], multiple: false, name: "file", + onRemove() { + clearUploadState(); + return true; + }, onChange(info) { + const fileList = info?.fileList ? info.fileList.slice(-1) : []; + if (info.file.status === "removed" || fileList.length === 0) { + clearUploadState(); + return; + } if (info.file.status !== "uploading") { //cat json content let reader = new FileReader(); @@ -96,20 +147,21 @@ export default Form.create()((props) => { const jsonStr = e.target.result; try { const jsonObj = JSON.parse(jsonStr); - setUploadState({ - ...(uploadState || {}), + setUploadState((prev) => ({ + ...(prev || {}), + fileList, data: jsonObj, - }); + })); } catch { - message.error(`${info.file.name} is an invalid json file!`); + message.error(formatMessage({ id: "alert.import.upload.invalid_json" })); } }; reader.readAsText(info.file.originFileObj); } if (info.file.status === "done") { - message.success(`${info.file.name} file uploaded successfully`); + message.success(formatMessage({ id: "alert.import.upload.success" })); } else if (info.file.status === "error") { - message.error(`${info.file.name} file upload failed.`); + message.error(formatMessage({ id: "alert.import.upload.failed" })); } }, }; @@ -123,14 +175,42 @@ export default Form.create()((props) => { } catch {} return ( - <> +
+ {showRuleExample ? ( + <> +
+ {formatMessage({ id: "alert.import.example.title" })} + + + +
+ +
+ + ) : null} {getFieldDecorator("upload", { getValueFromEvent: normFile, rules: [ { required: true, - message: "Please select file", + message: formatMessage({ id: "alert.import.select_file" }), }, ], })( @@ -144,7 +224,7 @@ export default Form.create()((props) => { {data && ( { }} /> )} - +
); }; diff --git a/web/src/pages/Alerting/components/Statistic.jsx b/web/src/pages/Alerting/components/Statistic.jsx index f0ef392a..b9361871 100644 --- a/web/src/pages/Alerting/components/Statistic.jsx +++ b/web/src/pages/Alerting/components/Statistic.jsx @@ -6,7 +6,7 @@ import Notification from "@/components/Icons/Notification"; import Mute from "@/components/Icons/Mute"; export default ({ stats = {}, dispatch }) => { - const keys = ["critical", "high", "medium", "low", "info", "ignored"]; + const keys = ["critical", "high", "medium", "low", "info"]; let total = 0; for (let key of keys) { total += stats[key] || 0; diff --git a/web/src/pages/Alerting/utils/constants.js b/web/src/pages/Alerting/utils/constants.js index 8cdc27ba..64b2f8ba 100644 --- a/web/src/pages/Alerting/utils/constants.js +++ b/web/src/pages/Alerting/utils/constants.js @@ -24,6 +24,7 @@ export const RuleStautsColor = { alerting: "red", error: "error", ok: "green", + recovered: "green", nodata: "gray", }; diff --git a/web/src/pages/Alerting/utils/message.js b/web/src/pages/Alerting/utils/message.js new file mode 100644 index 00000000..a1f39d09 --- /dev/null +++ b/web/src/pages/Alerting/utils/message.js @@ -0,0 +1,30 @@ +export const stripDuplicatedAlertTitle = (message, title) => { + if ( + typeof message !== "string" || + !message || + typeof title !== "string" || + !title + ) { + return message; + } + + const normalizedTitle = title.trim(); + if (!normalizedTitle) { + return message; + } + + const lines = message.replace(/\r\n/g, "\n").split("\n"); + let removed = false; + const filtered = lines.filter((line) => { + if (removed) { + return true; + } + if (line.trim() !== normalizedTitle) { + return true; + } + removed = true; + return false; + }); + + return filtered.join("\n"); +}; diff --git a/web/src/pages/Backup/BakAndRestore.js b/web/src/pages/Backup/BakAndRestore.js index 15a22751..575359b6 100644 --- a/web/src/pages/Backup/BakAndRestore.js +++ b/web/src/pages/Backup/BakAndRestore.js @@ -42,7 +42,7 @@ class BakAndRestore extends Component { title: "仓库名", dataIndex: "id", render: (text, record) => ( - + this.handleSnapshotClick(record)}>{record.id} ), @@ -54,7 +54,7 @@ class BakAndRestore extends Component { { title: "操作", render: (text, record) => ( - + {/* this.handleDownload(record)}>下载 */} ( - + this.handleSnapshotClick(record)}>{record.id} ), @@ -102,7 +102,7 @@ class BakAndRestore extends Component { { title: "操作", render: (text, record) => ( - + this.handleDownload(record)}>下载 +
{this.snapshotTable()}
 diff --git a/web/src/pages/Backup/BakCycle.js b/web/src/pages/Backup/BakCycle.js index ff30013c..e65d9892 100644 --- a/web/src/pages/Backup/BakCycle.js +++ b/web/src/pages/Backup/BakCycle.js @@ -249,7 +249,7 @@ class BakCycle extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>设置 { @@ -415,7 +415,7 @@ class BakCycle extends PureComponent { } = this.props; return ( - +
diff --git a/web/src/pages/Cluster/Activities.jsx b/web/src/pages/Cluster/Activities.jsx index 0004dddf..59e1f1b3 100644 --- a/web/src/pages/Cluster/Activities.jsx +++ b/web/src/pages/Cluster/Activities.jsx @@ -9,6 +9,7 @@ import request from "@/utils/request"; import ListView from "@/components/ListView"; import GenerateDesc from "@/pages/Overview/components/Activities/GenerateDesc"; import { getSystemClusterID } from "@/utils/setup"; +import moment from "moment"; const opers = { delete: "deleted", @@ -22,45 +23,102 @@ const iconType = { create: "plus-square", }; +const isZeroTimeValue = (value) => { + if (typeof value !== "string") { + return false; + } + const normalized = value.trim().toLowerCase(); + return ( + normalized === "0001-01-01t00:00:00z" || + normalized === "0001-01-01t00:00:00.000z" + ); +}; + const serializeDiffValue = (v) => { + if (v === null || v === undefined || v === "") { + return "-"; + } + if (isZeroTimeValue(v)) { + return "-"; + } if (typeof v == "string") { + const parsed = moment(v, moment.ISO_8601, true); + if (parsed.isValid()) { + return formatUtcTimeToLocal(parsed.toISOString()); + } return v; } return JSON.stringify(v); }; -const generateDiff = (diff) => { +const buildTemplateContext = (record) => { + const labels = record?.metadata?.labels || {}; + return { + ...labels, + timestamp: record?.timestamp, + trigger_at: labels.trigger_at || record?.timestamp, + }; +}; + +const resolveTemplateString = (value, record) => { + if (typeof value !== "string" || !value.includes("{{")) { + return value; + } + const context = buildTemplateContext(record); + return value.replace( + /\{\{\s*\.([a-zA-Z0-9_]+)(\s*\|\s*datetime)?\s*\}\}/g, + (match, key, datetimeFlag) => { + const raw = context?.[key]; + if (raw === null || raw === undefined || raw === "") { + return match; + } + if (datetimeFlag) { + const parsed = moment(raw, moment.ISO_8601, true); + if (parsed.isValid()) { + return formatUtcTimeToLocal(parsed.toISOString()); + } + } + return `${raw}`; + } + ); +}; + +const generateDiff = (diff, record) => { return (diff || []).map((changeLog, i) => { - const fieldPath = changeLog.path.join("."); + const fieldPath = Array.isArray(changeLog?.path) + ? changeLog.path.join(".") + : "-"; + const fromValue = serializeDiffValue(resolveTemplateString(changeLog.from, record)); + const toValue = serializeDiffValue(resolveTemplateString(changeLog.to, record)); switch (changeLog.type) { case "create": return (
- {fieldPath}: {serializeDiffValue(changeLog.to)} + {fieldPath}: {toValue}
); case "delete": return (
- {fieldPath}: {serializeDiffValue(changeLog.from)} + {fieldPath}: {fromValue}
); case "update": return (
- {fieldPath}: {serializeDiffValue(changeLog.from)}{" "} + {fieldPath}: {fromValue}{" "} - {serializeDiffValue(changeLog.to)} + {toValue}
); } }); }; -const DiffItem = ({ diff }) => { +const DiffItem = ({ diff, record }) => { const [state, setState] = useState({ hasMore: false, style: { maxHeight: 63, overflowY: "hidden" }, @@ -79,7 +137,7 @@ const DiffItem = ({ diff }) => { return (
- {generateDiff(diff || {})} + {generateDiff(diff || {}, record)}
{state.hasMore ? (
@@ -194,7 +252,7 @@ export default (props) => {
{record.changelog && record.changelog.length > 0 ? ( - + ) : null} ); @@ -269,15 +327,16 @@ export default (props) => { defaultQueryParams={{ from: 0, size: 20, - timeRange: { from: "now-7d", to: "now", timeField: timeField }, + timeRange: { from: "auto", to: "auto", timeField: timeField }, sort: [[timeField, "desc"]], }} dateTimeEnable={true} isRefreshPaused={true} sortEnable={true} sideEnable={true} - sideVisible={true} + sideVisible={false} sidePlacement="left" + datePickerContainerStyle={{ width: 320, maxWidth: "45vw", minWidth: 270 }} histogramEnable={histogramState.enable} histogramVisible={histogramState.visible} histogramWidget={histogramState.widget} diff --git a/web/src/pages/Cluster/Metrics.js b/web/src/pages/Cluster/Metrics.js index bc87dfe7..10772505 100644 --- a/web/src/pages/Cluster/Metrics.js +++ b/web/src/pages/Cluster/Metrics.js @@ -143,8 +143,8 @@ class ClusterMonitor extends PureComponent { clusterID: null, activeTab: props.param?.tab || "cluster", timeRange: { - min: "now-1h", //moment().subtract(1, 'h').toISOString(), - max: "now", //moment().toISOString() + min: "auto", + max: "auto", timeFormatter: formatter.dates(1), }, }; @@ -168,24 +168,36 @@ class ClusterMonitor extends PureComponent { fetchDataCount++; const { dispatch } = this.props; const { timeRange } = this.state; - const bounds = calculateBounds({ - from: timeRange.min, - to: timeRange.max, - }); - dispatch({ - type: "clusterMonitor/fetchClusterMetrics", - payload: { - timeRange: { - min: bounds.min.valueOf(), - max: bounds.max.valueOf(), + const useAutoRange = + `${timeRange?.min || ""}`.toLowerCase() === "auto" || + `${timeRange?.max || ""}`.toLowerCase() === "auto"; + const resolvedRange = useAutoRange + ? { min: "auto", max: "auto" } + : (() => { + const bounds = calculateBounds({ + from: timeRange.min, + to: timeRange.max, + }); + return { + min: bounds.min.valueOf(), + max: bounds.max.valueOf(), + }; + })(); + Promise.resolve( + dispatch({ + type: "clusterMonitor/fetchClusterMetrics", + payload: { + timeRange: resolvedRange, + cluster_id: this.state.clusterID, }, - cluster_id: this.state.clusterID, - }, - }).then((res) => { - this.setState({ - spinning: false, + }) + ) + .catch(() => null) + .finally(() => { + this.setState({ + spinning: false, + }); }); - }); }; componentDidUpdate(prevProps, prevState, snapshot) { @@ -284,14 +296,17 @@ class ClusterMonitor extends PureComponent { } handleTimeChange = ({ start, end }) => { - const bounds = calculateBounds({ - from: start, - to: end, - }); - const day = moment - .duration(bounds.max.valueOf() - bounds.min.valueOf()) - .asDays(); - const intDay = parseInt(day) + 1; + let intDay = 1; + if (`${start || ""}`.toLowerCase() !== "auto" && `${end || ""}`.toLowerCase() !== "auto") { + const bounds = calculateBounds({ + from: start, + to: end, + }); + const day = moment + .duration(bounds.max.valueOf() - bounds.min.valueOf()) + .asDays(); + intDay = parseInt(day) + 1; + } this.setState( { timeRange: { @@ -369,7 +384,12 @@ class ClusterMonitor extends PureComponent { {!clusterAvailable ? (
- Cluster is not availabe since: {clusterStats?.timestamp} + {formatMessage( + { + id: "cluster.manage.monitoring.notice.unavailable_since", + }, + { timestamp: clusterStats?.timestamp || "N/A" } + )}
) : !clusterMonitored && @@ -378,15 +398,24 @@ class ClusterMonitor extends PureComponent { .isAfter(clusterStats?.timestamp) ? (
- Cluster is not monitored.{" "} + {formatMessage({ + id: "cluster.manage.monitoring.notice.unmonitored", + })}{" "}
- Last data collection time: {clusterStats?.timestamp} + {formatMessage( + { + id: "cluster.manage.monitoring.notice.last_collection_time", + }, + { timestamp: clusterStats?.timestamp || "N/A" } + )}
) : null} diff --git a/web/src/pages/Cluster/NewOverview.js b/web/src/pages/Cluster/NewOverview.js index 74ebde6a..ab55201e 100644 --- a/web/src/pages/Cluster/NewOverview.js +++ b/web/src/pages/Cluster/NewOverview.js @@ -17,10 +17,14 @@ import { default as Hosts } from "./components/host/host_table"; const { TabPane } = Tabs; const panes = [ - { title: "Clusters", component: Clusters, key: "clusters" }, - { title: "Nodes", component: Nodes, key: "nodes" }, - { title: "Indices", component: Indices, key: "indices" }, - // { title: "Hosts", component: Hosts, key: "hosts" }, + { + titleId: "overview.title.cluster", + component: Clusters, + key: "clusters", + }, + { titleId: "overview.title.node", component: Nodes, key: "nodes" }, + { titleId: "overview.title.index", component: Indices, key: "indices" }, + // { titleId: "overview.title.host", component: Hosts, key: "hosts" }, ]; const NewOverview = (props) => { @@ -128,7 +132,12 @@ const NewOverview = (props) => { activeKey={param?.tab || "clusters"} > {panes.map((pane) => ( - + {typeof pane.component == "string" ? ( pane.component ) : ( diff --git a/web/src/pages/Cluster/Settings/Repository.js b/web/src/pages/Cluster/Settings/Repository.js index c4d71d7c..6f4c6102 100644 --- a/web/src/pages/Cluster/Settings/Repository.js +++ b/web/src/pages/Cluster/Settings/Repository.js @@ -24,7 +24,7 @@ class Repository extends Component { { title: '仓库名', dataIndex: 'id', - render: (text, record) => ( + render: (text, record) => ( this.handleRepoClick(record)}>{record.id} ) @@ -36,7 +36,7 @@ class Repository extends Component { { title: '操作', render: (text, record) => ( - + {/* this.handleDownload(record)}>下载 */} { @@ -82,7 +82,7 @@ class Repository extends Component { render() { return ( - + { - const bounds = calculateBounds({ - from: timeRange.min, - to: timeRange.max, - }); - let params = { - min: bounds.min.valueOf(), - max: bounds.max.valueOf(), - }; + const params = formatTimeRange(timeRange); if (overview) { params.overview = overview; } @@ -57,8 +50,55 @@ export default ({ const metrics = React.useMemo(() => { const { metrics = {} } = value || {}; - return Object.values(metrics) - .sort((a, b) => a.order - b.order) + const sorted = Object.values(metrics).sort((a, b) => a.order - b.order); + + // Fill gaps for auto_date_histogram (which only returns non-empty buckets) + sorted.forEach((metric) => { + if (!metric.lines) return; + metric.lines.forEach((line) => { + if (!line.data || line.data.length < 2) return; + + if (line.type === "Bar") { + // Bar chart data: [{x, y, g}, ...] — fill gaps with gray "empty" bars + const timestamps = [...new Set(line.data.map((d) => d.x))].sort( + (a, b) => a - b + ); + if (timestamps.length < 2) return; + // Infer interval from most common gap + const gaps = []; + for (let i = 1; i < timestamps.length; i++) { + gaps.push(timestamps[i] - timestamps[i - 1]); + } + gaps.sort((a, b) => a - b); + const intervalMs = gaps[Math.floor(gaps.length / 2)]; // median + if (!intervalMs || intervalMs <= 0) return; + + const existingSet = new Set(timestamps.map(String)); + const filledData = [...line.data]; + const startTs = timestamps[0]; + const endTs = timestamps[timestamps.length - 1]; + const maxSlots = 200; + let count = 0; + for ( + let ts = startTs; + ts <= endTs && count < maxSlots; + ts += intervalMs + ) { + count++; + if (!existingSet.has(String(ts))) { + filledData.push({ x: ts, y: 100, g: "empty" }); + } + } + filledData.sort((a, b) => a.x - b.x || a.g.localeCompare(b.g)); + line.data = filledData; + } else { + // Line chart data: [[timestamp, value], ...] — no gap-fill needed, + // @elastic/charts LineSeries connects dots naturally + } + }); + }); + + return sorted; }, [value]); const chartRefs = React.useRef(); @@ -222,11 +262,11 @@ export default ({ data={item.data} color={({ specId, yAccessor, splitAccessors }) => { const g = splitAccessors.get("g"); - if ( - yAccessor === "y" && - ["red", "yellow", "green"].includes(g) - ) { - return g; + if (yAccessor === "y") { + if (g === "empty") return "#D3DAE6"; + if (["red", "yellow", "green"].includes(g)) { + return g; + } } return null; }} diff --git a/web/src/pages/Cluster/components/clusters.js b/web/src/pages/Cluster/components/clusters.js index dac48b84..2cc707c9 100644 --- a/web/src/pages/Cluster/components/clusters.js +++ b/web/src/pages/Cluster/components/clusters.js @@ -31,7 +31,7 @@ import { JsonParam, useQueryParam } from "use-query-params"; import { HealthStatusCircle } from "@/components/infini/health_status_circle"; import Sorter from "@/components/infini/search/sort/sort"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; const InputGroup = Input.Group; const Option = Select.Option; const filterWidth = 120; @@ -113,25 +113,17 @@ const Clusters = (props) => { [queryParams, param?.filters, param?.sort] ); - const fetchFilterAggs = async () => { - const res = await request(`${ESPrefix}/cluster/_search`, { - method: "POST", - body: { - size: 0, - aggs: aggsParams, - }, - }); - if (res?.aggregations) { - const fts = getSearchFacets(res, Object.keys(facetLabels)); - if (fts.length > 0) { - setFacets(fts); - } - } - }; - const hits = value?.hits?.hits || []; const [infos, setInfos] = useState({}); const [facets, setFacets] = useState([]); + React.useEffect(() => { + if (!value?.aggregations) { + return; + } + const fts = getSearchFacets(value, Object.keys(facetLabels)); + setFacets(fts || []); + }, [value?.aggregations]); + React.useEffect(() => { if (hits?.length == 0) { return; @@ -160,8 +152,6 @@ const Clusters = (props) => { if (!param?.filters) { setParam({ ...param, filters: initialParams.filters }); } - - fetchFilterAggs(); }, []); const hitsTotal = value?.hits?.total?.value || 0; @@ -295,7 +285,7 @@ const Clusters = (props) => { open={searchOpen} onDropdownVisibleChange={setSearchOpen} > - { diff --git a/web/src/pages/Cluster/components/detail/metric_nodes.jsx b/web/src/pages/Cluster/components/detail/metric_nodes.jsx index b7dd53ea..68a3cc7c 100644 --- a/web/src/pages/Cluster/components/detail/metric_nodes.jsx +++ b/web/src/pages/Cluster/components/detail/metric_nodes.jsx @@ -79,6 +79,8 @@ export const MetricNodes = ({ { title: "Name", dataIndex: "name", + fixed: "left", + width: 180, render: (text, record) => ( , sorter: (a, b) => sorter.string(a, b, "status"), }, @@ -143,6 +146,7 @@ export const MetricNodes = ({ size={"small"} bordered dataSource={hits} + scroll={{ x: 300 }} rowKey={(record, index) => index} columns={columns} pagination={{ diff --git a/web/src/pages/Cluster/components/detail/metric_topn.jsx b/web/src/pages/Cluster/components/detail/metric_topn.jsx index e20cd1ef..d7a73877 100644 --- a/web/src/pages/Cluster/components/detail/metric_topn.jsx +++ b/web/src/pages/Cluster/components/detail/metric_topn.jsx @@ -2,6 +2,7 @@ import React, { useState } from "react"; import { ESPrefix } from "@/services/common"; import useFetch from "@/lib/hooks/use_fetch"; import Treemap from "@/components/infini/charts/Treemap"; +import { getLocalizedTreemapTitle } from "@/utils/treemap_title"; export const MetricTopN = (props) => { const clusterID = props.data?._id || null; @@ -18,7 +19,7 @@ export const MetricTopN = (props) => { return (
-
{treemapResult?._source?.name}
+
{getLocalizedTreemapTitle(treemapResult?._source?.name)}
{treemapData.children ? ( { const id = props.data?._id || null; @@ -21,9 +22,9 @@ export const Metrics = (props)=>{ return (
-
{treemapResult?._source?.name}
+
{getLocalizedTreemapTitle(treemapResult?._source?.name)}
) -} \ No newline at end of file +} diff --git a/web/src/pages/Cluster/components/host/hosts.jsx b/web/src/pages/Cluster/components/host/hosts.jsx index f110fa8e..a802e743 100644 --- a/web/src/pages/Cluster/components/host/hosts.jsx +++ b/web/src/pages/Cluster/components/host/hosts.jsx @@ -8,7 +8,7 @@ import HostDetail from "./host_detail"; import { TagList } from "../tag"; import HostCard from "./host_card"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; const Hosts = (props) => { const [collapse, setCollapse] = React.useState(false); @@ -64,7 +64,7 @@ const Hosts = (props) => {
- dispatch({ type: "search", value: value })} diff --git a/web/src/pages/Cluster/components/index/indices.jsx b/web/src/pages/Cluster/components/index/indices.jsx index 755d07a6..c2a6c4bb 100644 --- a/web/src/pages/Cluster/components/index/indices.jsx +++ b/web/src/pages/Cluster/components/index/indices.jsx @@ -32,7 +32,7 @@ import SearchSelectFacet from "../search_select_facet"; import { FilteredTags } from "../filtered_tags"; import Sorter from "@/components/infini/search/sort/sort"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; const InputGroup = Input.Group; const Option = Select.Option; const filterWidth = 120; @@ -97,26 +97,6 @@ const Indices = (props) => { setParam({ ...param, ...queryParams }); }, [queryParams]); - const fetchFilterAggs = async () => { - const res = await request(`${ESPrefix}/index/_search`, { - method: "POST", - body: { - size: 0, - aggs: aggsParams, - }, - }); - if (res?.aggregations) { - const fts = getSearchFacets(res, Object.keys(facetLabels)); - const clusterFts = getSearchFacets(res, ["metadata.cluster_name"]); - if (fts.length > 0) { - setFacets({ - chb: fts, - clusterFts, - }); - } - } - }; - const { loading, error, value } = useFetch( `${ESPrefix}/index/_search`, { @@ -141,6 +121,17 @@ const Indices = (props) => { const hits = value?.hits?.hits || []; const hitsTotal = value?.hits?.total?.value || 0; + React.useEffect(() => { + if (!value?.aggregations) { + return; + } + const fts = getSearchFacets(value, Object.keys(facetLabels)); + const clusterFts = getSearchFacets(value, ["metadata.cluster_name"]); + setFacets({ + chb: fts || [], + clusterFts: clusterFts || [], + }); + }, [value?.aggregations]); const [itemDetail, setItemDetail] = React.useState({}); const handleItemDetail = (item) => { @@ -173,8 +164,6 @@ const Indices = (props) => { if (!param?.filters) { setParam({ ...param, filters: initialParams.filters }); } - - fetchFilterAggs(); }, []); function renderOption(item) { @@ -292,7 +281,7 @@ const Indices = (props) => { open={searchOpen} onDropdownVisibleChange={setSearchOpen} > - { diff --git a/web/src/pages/Cluster/components/index/overview/shards.jsx b/web/src/pages/Cluster/components/index/overview/shards.jsx index 4fba6678..16f6e76c 100644 --- a/web/src/pages/Cluster/components/index/overview/shards.jsx +++ b/web/src/pages/Cluster/components/index/overview/shards.jsx @@ -10,7 +10,13 @@ import { formatter } from "@/utils/format"; import { filterSearchValue, sorter, formatUtcTimeToLocal } from "@/utils/utils"; import { formatTimeRange } from "@/lib/elasticsearch/util"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; + +const searchButton = ( + +); const Shards = ({ clusterID, @@ -161,15 +167,15 @@ const Shards = ({ }} >
- { - setSearchValue(value); + setSearchValue(value.trim()); }} onChange={(e) => { - setSearchValue(e.target.value); + setSearchValue(e.target.value.trim()); }} />
@@ -181,11 +187,11 @@ const Shards = ({ }} >
diff --git a/web/src/pages/Cluster/components/index_metric.jsx b/web/src/pages/Cluster/components/index_metric.jsx index e104ca3f..768a0e6b 100644 --- a/web/src/pages/Cluster/components/index_metric.jsx +++ b/web/src/pages/Cluster/components/index_metric.jsx @@ -17,11 +17,11 @@ import styles from "../Metrics.scss"; import { Spin, Radio, Select, Skeleton } from "antd"; import { formatter, getFormatter, getNumFormatter } from "@/utils/format"; import "./node_metric.scss"; -import { calculateBounds } from "@/components/vendor/data/common/query/timefilter"; import moment from "moment"; import { formatMessage } from "umi/locale"; import MetricContainer from "./metric_container"; import _ from "lodash"; +import { formatTimeRange } from "@/lib/elasticsearch/util"; const gorupOrder = [ "operations", @@ -64,14 +64,7 @@ export default ({ }); }; const queryParams = React.useMemo(() => { - const bounds = calculateBounds({ - from: timeRange.min, - to: timeRange.max, - }); - let newParams = { - min: bounds.min.valueOf(), - max: bounds.max.valueOf(), - }; + const newParams = formatTimeRange(timeRange); if (param.top) { newParams.top = param.top; } @@ -92,8 +85,40 @@ export default ({ const grpMetrics = _.groupBy(value?.metrics, "group"); let metrics = {}; Object.keys(grpMetrics).forEach((k) => { - metrics[k] = (grpMetrics[k] || []) - .sort((a, b) => a.order - b.order) + const items = (grpMetrics[k] || []).sort((a, b) => a.order - b.order); + // Fill gaps for auto_date_histogram bar charts + items.forEach((metric) => { + if (!metric.lines) return; + metric.lines.forEach((line) => { + if (!line.data || line.data.length < 2 || line.type !== "Bar") return; + const timestamps = [...new Set(line.data.map((d) => d.x))].sort( + (a, b) => a - b + ); + if (timestamps.length < 2) return; + const gaps = []; + for (let i = 1; i < timestamps.length; i++) { + gaps.push(timestamps[i] - timestamps[i - 1]); + } + gaps.sort((a, b) => a - b); + const intervalMs = gaps[Math.floor(gaps.length / 2)]; + if (!intervalMs || intervalMs <= 0) return; + const existingSet = new Set(timestamps.map(String)); + const filledData = [...line.data]; + const startTs = timestamps[0]; + const endTs = timestamps[timestamps.length - 1]; + const maxSlots = 200; + let count = 0; + for (let ts = startTs; ts <= endTs && count < maxSlots; ts += intervalMs) { + count++; + if (!existingSet.has(String(ts))) { + filledData.push({ x: ts, y: 100, g: "empty" }); + } + } + filledData.sort((a, b) => a.x - b.x || a.g.localeCompare(b.g)); + line.data = filledData; + }); + }); + metrics[k] = items; }); return metrics; }, [value]); @@ -310,11 +335,11 @@ export default ({ splitAccessors, }) => { const g = splitAccessors.get("g"); - if ( - yAccessor === "y" && - ["red", "yellow", "green"].includes(g) - ) { - return g; + if (yAccessor === "y") { + if (g === "empty") return "#D3DAE6"; + if (["red", "yellow", "green"].includes(g)) { + return g; + } } return null; }} diff --git a/web/src/pages/Cluster/components/node/nodes.jsx b/web/src/pages/Cluster/components/node/nodes.jsx index fe1e3946..7ffccdb6 100644 --- a/web/src/pages/Cluster/components/node/nodes.jsx +++ b/web/src/pages/Cluster/components/node/nodes.jsx @@ -33,7 +33,7 @@ import { FilteredTags } from "../filtered_tags"; import { HealthStatusCircle } from "@/components/infini/health_status_circle"; import Sorter from "@/components/infini/search/sort/sort"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; const InputGroup = Input.Group; const Option = Select.Option; const filterWidth = 120; @@ -139,29 +139,6 @@ const Nodes = (props) => { setParam({ ...param, ...queryParams }); }, [queryParams]); - const fetchFilterAggs = async () => { - const res = await request(`${ESPrefix}/node/_search`, { - method: "POST", - body: { - size: 0, - aggs: aggsParams, - }, - }); - if (res?.aggregations) { - const fts = getSearchFacets(res, Object.keys(facetLabels)); - const clusterFts = getSearchFacets(res, ["metadata.cluster_name"]); - if (fts.length > 0 || clusterFts.length > 0) { - dispatch({ - type: "setFacets", - value: { - facets: fts, - clusterFacets: clusterFts, - }, - }); - } - } - }; - const { loading, error, value } = useFetch( `${ESPrefix}/node/_search`, { @@ -190,6 +167,21 @@ const Nodes = (props) => { const hits = value?.hits?.hits || []; const hitsTotal = value?.hits?.total?.value || 0; + React.useEffect(() => { + if (!value?.aggregations) { + return; + } + const fts = getSearchFacets(value, Object.keys(facetLabels)); + const clusterFts = getSearchFacets(value, ["metadata.cluster_name"]); + dispatch({ + type: "setFacets", + value: { + facets: fts || [], + clusterFacets: clusterFts || [], + }, + }); + }, [value?.aggregations]); + // const [infos, setInfos] = useState({}); // const [facets, setFacets] = useState([]); React.useEffect(() => { @@ -220,7 +212,6 @@ const Nodes = (props) => { if (!param?.filters) { setParam({ ...param, filters: initialParams.filters }); } - fetchFilterAggs(); }, []); const [itemDetail, setItemDetail] = React.useState({}); @@ -350,7 +341,7 @@ const Nodes = (props) => { open={searchOpen} onDropdownVisibleChange={setSearchOpen} > - { diff --git a/web/src/pages/Cluster/components/node/overview/shards.jsx b/web/src/pages/Cluster/components/node/overview/shards.jsx index 75677db8..ad49d696 100644 --- a/web/src/pages/Cluster/components/node/overview/shards.jsx +++ b/web/src/pages/Cluster/components/node/overview/shards.jsx @@ -10,7 +10,13 @@ import { formatter } from "@/utils/format"; import { filterSearchValue, sorter, formatUtcTimeToLocal } from "@/utils/utils"; import { formatTimeRange } from "@/lib/elasticsearch/util"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; + +const searchButton = ( + +); const Shards = ({ clusterID, clusterName, nodeID, timeRange, setSpinning }) => { if (!clusterID || !nodeID) { @@ -148,15 +154,15 @@ const Shards = ({ clusterID, clusterName, nodeID, timeRange, setSpinning }) => { }} >
- { - setSearchValue(value); + setSearchValue(value.trim()); }} onChange={(e) => { - setSearchValue(e.target.value); + setSearchValue(e.target.value.trim()); }} />
@@ -183,11 +189,11 @@ const Shards = ({ clusterID, clusterName, nodeID, timeRange, setSpinning }) => { />
diff --git a/web/src/pages/Cluster/components/node/overview/statistic_bar.jsx b/web/src/pages/Cluster/components/node/overview/statistic_bar.jsx index 4a3407de..e37d04ed 100644 --- a/web/src/pages/Cluster/components/node/overview/statistic_bar.jsx +++ b/web/src/pages/Cluster/components/node/overview/statistic_bar.jsx @@ -6,6 +6,7 @@ import { formatUtcTimeToLocal } from "@/utils/utils"; import moment from "moment"; import { HealthStatusCircle } from "@/components/infini/health_status_circle"; import OverviewStatistic from "../../overview_statistic"; +import { formatMessage } from "umi/locale"; const vstyle = { fontSize: 12, @@ -44,7 +45,7 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { { key: "Status", value: nodeValue?.status || "N/A", - title: "Status", + title: formatMessage({ id: "overview.column.status" }), vstyle: { ...vstyle, display: "flex", @@ -57,39 +58,41 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { value: nodeValue?.jvm?.uptime ? moment.duration(nodeValue?.jvm?.uptime).humanize() : "N/A", - title: "Uptime", + title: formatMessage({ id: "overview.column.uptime" }), }, { key: "Type", - value: nodeValue?.is_master_node ? "Master Node" : "Not Master Node", - title: "Type", + value: nodeValue?.is_master_node + ? formatMessage({ id: "overview.statistic.master_node" }) + : formatMessage({ id: "overview.statistic.not_master_node" }), + title: formatMessage({ id: "overview.statistic.type" }), }, { key: "Transport Address", value: nodeValue?.transport_address || "N/A", - title: "Transport Address", + title: formatMessage({ id: "overview.column.transport_address" }), }, { key: "Indices", value: nodeValue?.shard_info?.indices_count, - title: "Indices", + title: formatMessage({ id: "overview.column.indices" }), }, { key: "Shards", value: (nodeValue?.shard_info?.shard_count || 0) + (nodeValue?.shard_info?.replicas_count || 0), - title: "Shards", + title: formatMessage({ id: "overview.column.shards" }), }, { key: "Documents", value: formatter.number(nodeValue?.indices?.docs?.count || 0), - title: "Documents", + title: formatMessage({ id: "indices.field.docs_count" }), }, { key: "Data", value: formatter.bytes(nodeValue?.indices?.store?.size_in_bytes || 0), - title: "Data", + title: formatMessage({ id: "overview.column.data" }), }, { key: "JVM Heap", @@ -104,7 +107,7 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { : 0 ).toFixed(2) + "%)", - title: "JVM Heap", + title: formatMessage({ id: "overview.column.jvm_heap" }), }, { key: "Free Disk Space", @@ -117,7 +120,7 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { : 0 ).toFixed(2) + "%)", - title: "Free Disk Space", + title: formatMessage({ id: "overview.column.disk_free_space" }), }, ]; } @@ -126,10 +129,14 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { {!isAvailable ? (
- Node is not availabe since:{" "} - {nodeValue?.timestamp - ? formatUtcTimeToLocal(nodeValue?.timestamp) - : "N/A"} + {formatMessage( + { id: "overview.status.node_since" }, + { + timestamp: nodeValue?.timestamp + ? formatUtcTimeToLocal(nodeValue?.timestamp) + : "N/A", + } + )}
) : null} diff --git a/web/src/pages/Cluster/components/node_metric.jsx b/web/src/pages/Cluster/components/node_metric.jsx index 63e93b8c..8022acd6 100644 --- a/web/src/pages/Cluster/components/node_metric.jsx +++ b/web/src/pages/Cluster/components/node_metric.jsx @@ -37,6 +37,37 @@ const gorupOrder = [ "cache", ]; +const normalizeNodeSelection = (value, nodes = []) => { + if (Array.isArray(value)) { + return value; + } + if (!value) { + return []; + } + if (typeof value === "string") { + return nodes.includes(value) ? [value] : [value]; + } + if (value?.host) { + return [value.host]; + } + return []; +}; + +const extractNodeNames = (value) => { + if (Array.isArray(value)) { + return value + .map((item) => (typeof item === "string" ? item : item?.host)) + .filter(Boolean); + } + if (typeof value === "string") { + return [value]; + } + if (value?.host) { + return [value.host]; + } + return []; +}; + export default ({ clusterID, timezone, @@ -86,8 +117,9 @@ export default ({ if (param.top) { newParams.top = param.top; } - if (param.node_name) { - newParams.node_name = param.node_name; + const selectedNodeNames = extractNodeNames(param.node_name); + if (selectedNodeNames.length > 0) { + newParams.node_name = selectedNodeNames; } return newParams; }, [param, timeRange]); @@ -131,6 +163,10 @@ export default ({ } return (nodes || []).map((item) => item?.ip + ":" + item?.port); }, [nodes]); + const selectedNodeNames = React.useMemo( + () => normalizeNodeSelection(param.node_name, nodeNames), + [param.node_name, nodeNames] + ); const pointerUpdate = (event) => { chartRefs.current.forEach((ref) => { @@ -189,7 +225,7 @@ export default ({ style={{ width: 200 }} onChange={nodeValueChange} placeholder="Select node" - value={param.node_name} + value={selectedNodeNames} showSearch={true} > {nodeNames.map((name) => ( diff --git a/web/src/pages/Cluster/components/overview/advanced.jsx b/web/src/pages/Cluster/components/overview/advanced.jsx index 6d18d4dd..7bc32519 100644 --- a/web/src/pages/Cluster/components/overview/advanced.jsx +++ b/web/src/pages/Cluster/components/overview/advanced.jsx @@ -36,6 +36,7 @@ const Advanced = ({ />
{ let val = value ? [value] : []; setSearchFilterFields(val); diff --git a/web/src/pages/Cluster/components/overview/nodes.jsx b/web/src/pages/Cluster/components/overview/nodes.jsx index a04223bc..546ee6ae 100644 --- a/web/src/pages/Cluster/components/overview/nodes.jsx +++ b/web/src/pages/Cluster/components/overview/nodes.jsx @@ -96,6 +96,8 @@ const Nodes = ({ { title: "Name", dataIndex: "name", + width: 220, + fixed: "left", render: (text, record) => (
@@ -231,6 +233,8 @@ const Nodes = ({ { let val = value ? [value] : []; setSearchFilterFields(val); @@ -275,6 +279,7 @@ const Nodes = ({ dataSource={hits} rowKey={"id"} columns={columns} + scroll={{ x: 1200 }} pagination={{ size: "small", pageSize: queryParams.size, diff --git a/web/src/pages/Cluster/components/overview/statistic_bar.jsx b/web/src/pages/Cluster/components/overview/statistic_bar.jsx index e46f3c05..2188b5ca 100644 --- a/web/src/pages/Cluster/components/overview/statistic_bar.jsx +++ b/web/src/pages/Cluster/components/overview/statistic_bar.jsx @@ -123,10 +123,16 @@ const StatisticBar = ({ {!clusterAvailable ? (
- Cluster is not availabe since:{" "} - {value?.summary?.timestamp - ? formatUtcTimeToLocal(value?.summary?.timestamp) - : "N/A"} + {formatMessage( + { + id: "cluster.manage.monitoring.notice.unavailable_since", + }, + { + timestamp: value?.summary?.timestamp + ? formatUtcTimeToLocal(value?.summary?.timestamp) + : "N/A", + } + )}
) : !clusterMonitored && @@ -135,16 +141,28 @@ const StatisticBar = ({ .isAfter(value?.summary?.timestamp) ? (
- Cluster is not monitored.{" "} + {formatMessage({ + id: "cluster.manage.monitoring.notice.unmonitored", + })}{" "}
- Last data collection time:{" "} - {value?.summary?.timestamp - ? formatUtcTimeToLocal(value?.summary?.timestamp) - : "N/A"} + {formatMessage( + { + id: "cluster.manage.monitoring.notice.last_collection_time", + }, + { + timestamp: value?.summary?.timestamp + ? formatUtcTimeToLocal(value?.summary?.timestamp) + : "N/A", + } + )}
) : null} diff --git a/web/src/pages/Cluster/components/queue_metric.jsx b/web/src/pages/Cluster/components/queue_metric.jsx index 9b561b1a..f2c98ba9 100644 --- a/web/src/pages/Cluster/components/queue_metric.jsx +++ b/web/src/pages/Cluster/components/queue_metric.jsx @@ -16,11 +16,11 @@ import styles from "../Metrics.scss"; import { Spin, Radio, Select, Skeleton, Row, Col } from "antd"; import { formatter, getFormatter, getNumFormatter } from "@/utils/format"; import "./node_metric.scss"; -import { calculateBounds } from "@/components/vendor/data/common/query/timefilter"; import moment from "moment"; import { formatMessage } from "umi/locale"; import MetricContainer from "./metric_container"; import _ from "lodash"; +import { formatTimeRange } from "@/lib/elasticsearch/util"; const gorupOrder = [ "thread_pool_write", @@ -33,6 +33,37 @@ const gorupOrder = [ "thread_pool_force_merge", ]; +const normalizeNodeSelection = (value, nodes = []) => { + if (Array.isArray(value)) { + return value; + } + if (!value) { + return []; + } + if (typeof value === "string") { + return nodes.includes(value) ? [value] : [value]; + } + if (value?.host) { + return [value.host]; + } + return []; +}; + +const extractNodeNames = (value) => { + if (Array.isArray(value)) { + return value + .map((item) => (typeof item === "string" ? item : item?.host)) + .filter(Boolean); + } + if (typeof value === "string") { + return [value]; + } + if (value?.host) { + return [value.host]; + } + return []; +}; + export default ({ clusterID, timezone, @@ -67,19 +98,13 @@ export default ({ [param] ); const queryParams = React.useMemo(() => { - const bounds = calculateBounds({ - from: timeRange.min, - to: timeRange.max, - }); - let newParams = { - min: bounds.min.valueOf(), - max: bounds.max.valueOf(), - }; + const newParams = formatTimeRange(timeRange); if (param.top) { newParams.top = param.top; } - if (param.node_name) { - newParams.node_name = param.node_name; + const selectedNodeNames = extractNodeNames(param.node_name); + if (selectedNodeNames.length > 0) { + newParams.node_name = selectedNodeNames; } return newParams; }, [param, timeRange]); @@ -123,6 +148,10 @@ export default ({ } return (nodes || []).map((item) => item?.ip + ":" + item?.port); }, [nodes]); + const selectedNodeNames = React.useMemo( + () => normalizeNodeSelection(param.node_name, nodeNames), + [param.node_name, nodeNames] + ); const pointerUpdate = (event) => { chartRefs.current.forEach((ref) => { @@ -180,7 +209,7 @@ export default ({ style={{ width: 200 }} onChange={nodeValueChange} placeholder="Select node" - value={param.node_name} + value={selectedNodeNames} showSearch={true} > {nodeNames.map((name) => ( diff --git a/web/src/pages/Cluster/components/search_facet.scss b/web/src/pages/Cluster/components/search_facet.scss index 39ca32d0..5b29b461 100644 --- a/web/src/pages/Cluster/components/search_facet.scss +++ b/web/src/pages/Cluster/components/search_facet.scss @@ -10,10 +10,33 @@ } .search-facet-value { display: flex; + align-items: center; + gap: 8px; + min-width: 0; + :global { + label.ant-checkbox-wrapper { + display: flex; + align-items: center; + flex: 1 1 auto; + min-width: 0; + } + label.ant-checkbox-wrapper > span:first-child { + flex: 0 0 auto; + } + label.ant-checkbox-wrapper > span:last-child { + flex: 1 1 auto; + min-width: 0; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } + } .count { margin-left: auto; + flex: 0 0 auto; color: #888; font-size: 0.85em; + white-space: nowrap; } } } diff --git a/web/src/pages/DataManagement/Alias.js b/web/src/pages/DataManagement/Alias.js index de0f7176..183a9fb4 100644 --- a/web/src/pages/DataManagement/Alias.js +++ b/web/src/pages/DataManagement/Alias.js @@ -14,6 +14,7 @@ import { AutoComplete, Switch, Popconfirm, + Icon, } from "antd"; import PageHeaderWrapper from "@/components/PageHeaderWrapper"; import "@/assets/headercontent.scss"; @@ -21,9 +22,15 @@ import { formatMessage } from "umi/locale"; import { hasAuthority } from "@/utils/authority"; import { isMatch, sorter } from "@/utils/utils"; import { Link } from "umi"; +import SearchInput from "@/components/infini/SearchInput"; const FormItem = Form.Item; const { TextArea } = Input; +const firstColumnIconStyle = { + marginRight: 8, + color: "#999", + fontSize: 12, +}; const UpdateForm = Form.create()((props) => { const { @@ -145,7 +152,8 @@ class AliasManage extends PureComponent { selectedRows: [], formValues: {}, updateFormValues: {}, - pageSize: 10, + pageSize: 20, + editMode: "UPDATE", }; columns = [ @@ -153,25 +161,34 @@ class AliasManage extends PureComponent { title: formatMessage({ id: "alias.table.field.name" }), dataIndex: "alias", sorter: (a, b) => sorter.string(a, b, "alias"), + render: (text) => ( +
+ + {text} +
+ ), }, { title: formatMessage({ id: "alias.table.field.write_index" }), + with: 150, dataIndex: "write_index", sorter: (a, b) => sorter.string(a, b, "write_index"), render: (text, record) => { + if (!record.write_index) return "-"; return {text}; }, }, { title: formatMessage({ id: "table.field.actions" }), + width: 100, render: (text, record) => { return ( - + {/* this.handleUpdateModalVisible(true, record)}>别名设置*/} {/**/} {hasAuthority("data.alias:all") ? ( this.handleDeleteAliasClick(record)} > {" "} @@ -278,10 +295,8 @@ class AliasManage extends PureComponent { let newState = { updateModalVisible: !!flag, updateFormValues: values, + editMode: values.alias ? "UPDATE" : "NEW", }; - if (!values.alias) { - newState.editMode = "NEW"; - } this.setState(newState); }; @@ -301,10 +316,10 @@ class AliasManage extends PureComponent { this.handleModalVisible(); }; - handleUpdate = (fields) => { + handleUpdate = async (fields) => { let upVals = {}; for (let k in fields) { - if (fields[k]) { + if (fields[k] !== undefined && fields[k] !== null && fields[k] !== "") { if (k === "filter") { upVals[k] = JSON.parse(fields[k]); } else { @@ -313,15 +328,25 @@ class AliasManage extends PureComponent { } } const { dispatch } = this.props; - dispatch({ + const res = await dispatch({ type: "alias/update", payload: { actionBody: upVals, clusterID: this.props.selectedClusterID, }, }); - - message.success("updated successfully"); + if (!res?.acknowledged) { + message.error(formatMessage({ id: "app.message.update.failed" })); + return; + } + message.success( + formatMessage({ + id: + this.state.editMode === "NEW" + ? "app.message.create.success" + : "app.message.update.success", + }) + ); this.handleUpdateModalVisible(); }; @@ -385,7 +410,7 @@ class AliasManage extends PureComponent { }} >
- { + return text || text === 0 ? text : "-"; + }, }, { title: formatMessage({ id: "alias.table.field.search_routing" }), dataIndex: "search_routing", + render: (text) => { + return text || text === 0 ? text : "-"; + }, }, { title: formatMessage({ id: "alias.table.field.filter" }), dataIndex: "filter", render: (text) => { - return text ? JSON.stringify(text) : ""; + return text ? JSON.stringify(text) : "-"; }, }, { @@ -525,7 +556,7 @@ class AliasIndexTable extends React.Component { { this.props.handleDeleteClick({ ...record, @@ -551,7 +582,7 @@ class AliasIndexTable extends React.Component { size={"small"} pagination={{ size: "small", - pageSize: 10, + pageSize: 20, showSizeChanger: true, showTotal: (total, range) => `${range[0]}-${range[1]} of ${total} items`, diff --git a/web/src/pages/DataManagement/Discover.jsx b/web/src/pages/DataManagement/Discover.jsx index 868ed364..d72fa414 100644 --- a/web/src/pages/DataManagement/Discover.jsx +++ b/web/src/pages/DataManagement/Discover.jsx @@ -70,6 +70,7 @@ import { BooleanParam } from "use-query-params"; import { Link, Route } from "umi"; +import { formatMessage } from "umi/locale"; import { ESPrefix } from "@/services/common"; import TraceChart from "./trace_chart"; import TraceSearch from "./SearchFlow/TraceSearch"; @@ -87,6 +88,148 @@ import { getTimezone } from "@/utils/utils"; import { hasAuthority } from "@/utils/authority"; const SidebarMemoized = React.memo(DiscoverSidebar); +const SHARE_STATE_VERSION = 1; +const MANUAL_REFRESH_DEBOUNCE_MS = 800; + +// Parse the interval string returned by auto_date_histogram (e.g. "1h", "30m", "1d") +function parseAutoInterval(intervalStr, buckets) { + const match = intervalStr.match(/^(\d+)([smhdwMqy])$/); + if (!match) { + return buckets.getInterval(true); + } + const value = parseInt(match[1], 10); + const unitMap = { s: 'second', m: 'minute', h: 'hour', d: 'day', w: 'week', M: 'month', q: 'quarter', y: 'year' }; + const unit = match[2]; + return { + esValue: value, + esUnit: unit, + description: `${value} ${unitMap[unit] || unit}${value > 1 ? 's' : ''}`, + }; +} + +const encodeShareState = (value) => { + try { + return btoa(unescape(encodeURIComponent(JSON.stringify(value)))) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/g, ""); + } catch (e) { + return ""; + } +}; + +const decodeShareState = (value) => { + if (!value) { + return null; + } + try { + const normalized = value.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normalized + "=".repeat((4 - normalized.length % 4) % 4); + return JSON.parse(decodeURIComponent(escape(atob(padded)))); + } catch (e) { + return null; + } +}; + +const updateHashQueryParams = (href, patch = {}) => { + try { + const url = new URL(href); + const hashValue = (url.hash || "#").slice(1); + const [hashPath, hashQuery = ""] = hashValue.split("?"); + const params = new URLSearchParams(hashQuery); + + Object.entries(patch).forEach(([key, value]) => { + params.delete(key); + if ( + value === undefined || + value === null || + value === "" || + (Array.isArray(value) && value.length === 0) + ) { + return; + } + if (Array.isArray(value)) { + value.forEach((item) => { + if (item !== undefined && item !== null && item !== "") { + params.append(key, String(item)); + } + }); + return; + } + params.set(key, String(value)); + }); + + const nextQuery = params.toString(); + url.hash = nextQuery ? `${hashPath}?${nextQuery}` : hashPath; + return url.toString(); + } catch (e) { + return href; + } +}; + +const areArrayValuesEqual = (left = [], right = []) => { + if (left === right) { + return true; + } + if (!Array.isArray(left) || !Array.isArray(right) || left.length !== right.length) { + return false; + } + return left.every((item, index) => item === right[index]); +}; + +const hasFieldInIndexPattern = (indexPattern, fieldName) => { + if (!indexPattern || !fieldName) { + return false; + } + if (indexPattern.fields?.getByName) { + return !!indexPattern.fields.getByName(fieldName); + } + if (Array.isArray(indexPattern.fields)) { + return indexPattern.fields.some( + (field) => field?.name === fieldName || field?.displayName === fieldName + ); + } + return false; +}; + +const getDateFieldsFromIndexPattern = (indexPattern) => { + if (!indexPattern?.fields) { + return []; + } + if (indexPattern.fields?.getByType) { + return indexPattern.fields + .getByType("date") + .map((field) => field?.displayName || field?.name) + .filter(Boolean); + } + if (Array.isArray(indexPattern.fields)) { + return indexPattern.fields + .filter((field) => field?.spec?.type === "date" || field?.type === "date") + .map((field) => field?.displayName || field?.name) + .filter(Boolean); + } + return []; +}; + +const resolveValidTimeFieldName = (indexPattern, preferredTimeField) => { + const candidates = [preferredTimeField, indexPattern?.timeFieldName].filter(Boolean); + const validCandidate = candidates.find((fieldName) => + hasFieldInIndexPattern(indexPattern, fieldName) + ); + if (validCandidate) { + return validCandidate; + } + const dateFields = getDateFieldsFromIndexPattern(indexPattern); + return dateFields.length === 1 ? dateFields[0] : ""; +}; + +const buildSortByTimeField = (indexPattern, preferredTimeField) => { + const timeFieldName = resolveValidTimeFieldName(indexPattern, preferredTimeField); + return { + timeFieldName, + sort: timeFieldName ? [[timeFieldName, "desc"]] : [], + }; +}; const { filterManager, @@ -125,6 +268,11 @@ const Discover = (props) => { const [insightLoading, setInsightLoading] = useState(false); const [showResultCount, setShowResultCount] = useState(true); const [selectedQueries, setSelectedQueries] = useState(); + const appliedShareRef = useRef(""); + const manualRefreshRef = useRef({ + lastTriggeredAt: 0, + timer: null, + }); const [columnsParam, setColumnsParam] = useQueryParam("columns", ArrayParam); @@ -141,11 +289,12 @@ const Discover = (props) => { "sq", StringParam ); + const [shareParam] = useQueryParam("share", StringParam); const [trackTotalHits, setTrackTotalHits] = useQueryParam( "tth", BooleanParam ); - const [timeout, setTimeout] = useState(localStorage.getItem('search_time_out') || '60s'); + const [searchTimeout, setSearchTimeout] = useState(localStorage.getItem('search_time_out') || '60s'); const [whetherToSample, setWhetherToSample] = useQueryParam( "wts", BooleanParam @@ -219,21 +368,15 @@ const Discover = (props) => { typ, props.selectedCluster?.id ); + const { timeFieldName, sort } = buildSortByTimeField(IP); + IP.timeFieldName = timeFieldName; subscriptions.unsubscribe(); props.changeIndexPattern(IP); - if (IP.timeFieldName) { - setState({ - ...state, - columns: ["_source"], - sort: [[IP.timeFieldName, 'desc']], - }); - } else { - setState({ - ...state, - columns: ["_source"], - sort: [], - }); - } + setState({ + ...state, + columns: ["_source"], + sort, + }); if (filters && filters.length > 0) { if (isReset) { filterManager.setFilters(filters); @@ -254,15 +397,15 @@ const Discover = (props) => { props.selectedCluster?.id ); subscriptions.unsubscribe(); - IP.timeFieldName = timeField; + const { timeFieldName, sort } = buildSortByTimeField(IP, timeField); + IP.timeFieldName = timeFieldName; props.changeIndexPattern(IP); - const newSort = [[timeField, 'desc']] - setState({ - ...state, + setState((st) => ({ + ...st, columns: ["_source"], - sort: newSort, - }); - updateQuery({ sort: newSort }); + sort, + })); + updateQuery({ indexPattern: IP, sort }); }; //const indexPatterns = [{"id":"1ccce5c0-bb9a-11eb-957b-939add21a246","type":"index-pattern","namespaces":["default"],"updated_at":"2021-05-23T07:40:14.747Z","version":"WzkxOTEsNDhd","attributes":{"title":"test-custom*","timeFieldName":"created_at","fields":"[{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"address\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"address.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"address\"}}},{\"count\":0,\"conflictDescriptions\":{\"text\":[\"test-custom1\"],\"long\":[\"test-custom\",\"test-custom8\",\"test-custom9\"]},\"name\":\"age\",\"type\":\"conflict\",\"esTypes\":[\"text\",\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"age.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"age\"}}},{\"count\":0,\"name\":\"created_at\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"email\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"email.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"email\"}}},{\"count\":0,\"name\":\"hobbies\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"hobbies.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"hobbies\"}}},{\"count\":0,\"name\":\"id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"id\"}}},{\"count\":0,\"name\":\"name\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"name.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"name\"}}}]"},"references":[],"migrationVersion":{"index-pattern":"7.6.0"}}]; @@ -282,6 +425,7 @@ const Discover = (props) => { const scrollableDesktop = useRef(null); const [queryFrom, setQueryFrom] = React.useState(0); + const searchAfterRef = useRef(null); const columns = state.columns; const traceParams = useRef({}); @@ -332,6 +476,7 @@ const Discover = (props) => { } if (!_payload?.isScrollLoad) { setQueryFrom(0); + searchAfterRef.current = null; } const params = getSearchParams( @@ -341,7 +486,9 @@ const Discover = (props) => { _payload?.aggs || aggs, distinctParams || {}, _payload?.isScrollLoad ? queryFrom : 0, - trackTotalHits + trackTotalHits, + 20, + _payload?.isScrollLoad ? searchAfterRef.current : null, ); const filters = cloneDeep(params?.body?.query?.bool?.filter || []) @@ -372,12 +519,20 @@ const Discover = (props) => { }; } res.hits.hits = res.hits.hits || []; + // Track search_after from last hit's sort for next page + if (res.hits.hits.length > 0) { + const lastHit = res.hits.hits[res.hits.hits.length - 1]; + searchAfterRef.current = lastHit.sort || null; + } setSearchRes(res); const { query } = queryStringManager.getQuery(); if (query != queryParam) { setQueryParam(query); } - setTimeParam([timefilter._time?.from, timefilter._time?.to]); + const nextTimeParam = [timefilter._time?.from, timefilter._time?.to]; + if (!areArrayValuesEqual(timeParam || [], nextTimeParam)) { + setTimeParam(nextTimeParam); + } // let filters = filterManager.getFilters(); // const tfilter = timefilter.createFilter(indexPattern); @@ -402,6 +557,43 @@ const Discover = (props) => { ] ); + const onManualRefresh = useCallback( + (payload) => { + const now = Date.now(); + const elapsed = now - manualRefreshRef.current.lastTriggeredAt; + const triggerRefresh = () => { + manualRefreshRef.current.lastTriggeredAt = Date.now(); + manualRefreshRef.current.timer = null; + updateQuery(payload); + }; + if (elapsed >= MANUAL_REFRESH_DEBOUNCE_MS) { + if (manualRefreshRef.current.timer) { + clearTimeout(manualRefreshRef.current.timer); + manualRefreshRef.current.timer = null; + } + triggerRefresh(); + return; + } + if (manualRefreshRef.current.timer) { + clearTimeout(manualRefreshRef.current.timer); + } + manualRefreshRef.current.timer = setTimeout( + triggerRefresh, + MANUAL_REFRESH_DEBOUNCE_MS - elapsed + ); + }, + [updateQuery] + ); + + useEffect(() => { + return () => { + if (manualRefreshRef.current.timer) { + clearTimeout(manualRefreshRef.current.timer); + manualRefreshRef.current.timer = null; + } + }; + }, []); + const onQueriesSelect = async (record) => { queryStringManager.setQuery(record.query); timefilter.setTime(record.time_filter); @@ -410,21 +602,20 @@ const Discover = (props) => { "index", props.selectedCluster?.id ); - IP.timeFieldName = record.time_field; + const { timeFieldName, sort } = buildSortByTimeField(IP, record.time_field); + IP.timeFieldName = timeFieldName; subscriptions.unsubscribe(); props.changeIndexPattern(IP); const newState = { ...state, columns: record.filter?.columns || ["_source"], - } - if (record.time_field) { - newState.sort = [[record.time_field, 'desc']] - } + sort, + }; setState(newState); if (record.filter?.filters?.length > 0) { filterManager.setFilters(record.filter?.filters); } - updateQuery(); + updateQuery({ indexPattern: IP, sort }); setSelectedQueries(record); if (selectedQueriesId !== record.id) { setSelectedQueriesId(record.id); @@ -478,7 +669,11 @@ const Discover = (props) => { return { histogramData: null, timeChartProps: null }; } const buckets = getTimeBuckets(state.interval); - const interval = buckets.getInterval(true); + const aggregations = searchRes.aggregations; + const autoInterval = aggregations["counts"].interval; + const interval = autoInterval + ? parseAutoInterval(autoInterval, buckets) + : buckets.getInterval(true); const chartTable = { columns: [ { @@ -491,11 +686,38 @@ const Discover = (props) => { ], rows: [], }; - let aggregations = searchRes.aggregations; - aggregations["counts"].buckets.forEach((bk) => { - chartTable.rows.push(bk); - }); + const responseBuckets = aggregations["counts"].buckets || []; + const bounds = buckets.getBounds(); + const intervalMs = moment.duration(interval.esValue, interval.esUnit).asMilliseconds(); + + const startMs = bounds.min ? bounds.min.valueOf() : 0; + const endMs = bounds.max ? bounds.max.valueOf() : 0; + const maxBuckets = 200; + const canFillGaps = intervalMs > 0 && startMs > 0 && endMs > startMs + && responseBuckets.length > 0 + && ((endMs - startMs) / intervalMs) <= maxBuckets; + + if (canFillGaps) { + // Use the first bucket key as the aligned start, then fill gaps + const firstKey = responseBuckets[0].key; + // Align start to interval boundary based on first bucket key + const alignedStart = firstKey - Math.ceil((firstKey - startMs) / intervalMs) * intervalMs; + const bucketMap = new Map(); + responseBuckets.forEach((bk) => { + bucketMap.set(bk.key, bk.doc_count); + }); + for (let t = alignedStart; t < endMs; t += intervalMs) { + chartTable.rows.push({ + key: t, + doc_count: bucketMap.has(t) ? bucketMap.get(t) : 0, + }); + } + } else { + responseBuckets.forEach((bk) => { + chartTable.rows.push(bk); + }); + } //console.log(interval, moment.duration('1', 'd')) const dimensions = { @@ -580,25 +802,146 @@ const Discover = (props) => { // }); // }; useEffect(() => { - setColumnsParam(state.columns); - }, [state.columns]); + if (!areArrayValuesEqual(columnsParam || [], state.columns || [])) { + setColumnsParam(state.columns); + } + }, [columnsParam, state.columns]); + + const getShareUrl = useCallback(() => { + const currentQuery = queryStringManager.getQuery()?.query || ""; + const currentTime = timefilter.getTime() || {}; + const shareState = encodeShareState({ + version: SHARE_STATE_VERSION, + columns: state.columns || [], + filters: filterManager.getFilters() || [], + histogramVisible, + mode, + sort: state.sort || [], + timeField: indexPattern.timeFieldName || "", + }); + + return updateHashQueryParams(window.location.href, { + columns: state.columns, + index: indexPattern.type === "index" ? indexPattern.id : undefined, + query: currentQuery, + share: shareState || undefined, + sq: undefined, + time: [currentTime.from, currentTime.to], + viewID: indexPattern.type === "index" ? undefined : indexPattern.id, + }); + }, [ + histogramVisible, + indexPattern.id, + indexPattern.timeFieldName, + indexPattern.type, + mode, + state.columns, + state.sort, + ]); + + useEffect(() => { + if ( + !shareParam || + appliedShareRef.current === shareParam || + !indexPattern?.id || + !props.selectedCluster?.id + ) { + return; + } + + const sharedState = decodeShareState(shareParam); + appliedShareRef.current = shareParam; + if (!sharedState || sharedState.version !== SHARE_STATE_VERSION) { + return; + } + + let cancelled = false; + + const applySharedState = async () => { + let nextIndexPattern = indexPattern; + if ( + sharedState.timeField && + sharedState.timeField !== indexPattern.timeFieldName + ) { + nextIndexPattern = await services.indexPatternService.get( + indexPattern.id, + indexPattern.type, + props.selectedCluster?.id + ); + if (cancelled) { + return; + } + subscriptions.unsubscribe(); + nextIndexPattern.timeFieldName = resolveValidTimeFieldName( + nextIndexPattern, + sharedState.timeField + ); + props.changeIndexPattern(nextIndexPattern); + } + + if (Array.isArray(sharedState.filters)) { + if (sharedState.filters.length > 0) { + filterManager.setFilters(sharedState.filters); + } else { + filterManager.removeAll(); + } + } + + const fallbackSort = buildSortByTimeField( + nextIndexPattern, + sharedState.timeField + ).sort; + const nextSort = + Array.isArray(sharedState.sort) && sharedState.sort.length > 0 + ? sharedState.sort.filter( + (sortItem) => + Array.isArray(sortItem) && + sortItem.length > 0 && + hasFieldInIndexPattern(nextIndexPattern, sortItem[0]) + ) + : fallbackSort; + + setState((st) => ({ + ...st, + columns: + Array.isArray(sharedState.columns) && sharedState.columns.length > 0 + ? sharedState.columns + : st.columns, + sort: nextSort.length > 0 ? nextSort : fallbackSort, + })); + const effectiveSort = nextSort.length > 0 ? nextSort : fallbackSort; + + if (sharedState.mode) { + setMode(sharedState.mode); + } + if (typeof sharedState.histogramVisible === "boolean") { + setHistogramVisible(sharedState.histogramVisible); + } + + updateQuery({ + indexPattern: nextIndexPattern, + sort: effectiveSort, + }); + }; + + applySharedState(); + + return () => { + cancelled = true; + }; + }, [shareParam, indexPattern?.id, props.selectedCluster?.id]); useEffect(() => { if (indexPattern) { - if (indexPattern.timeFieldName) { - const newSort = [[indexPattern.timeFieldName, 'desc']] - setState({ - ...state, - sort: newSort - }); - updateQuery({ sort: newSort }); - } else { - setState({ - ...state, - sort: [] - }); - updateQuery({ sort: [] }); + const { timeFieldName, sort } = buildSortByTimeField(indexPattern); + if (indexPattern.timeFieldName !== timeFieldName) { + indexPattern.timeFieldName = timeFieldName; } + setState({ + ...state, + sort, + }); + updateQuery({ sort }); } }, [indexPattern]); @@ -698,7 +1041,9 @@ const Discover = (props) => { } } const took = searchRes.took || 1; - const hits = searchRes.hits.total?.value || searchRes.hits.total; + const hitsTotal = searchRes.hits.total; + const hits = hitsTotal?.value ?? hitsTotal ?? 0; + const hitsRelation = hitsTotal?.relation || "eq"; const resetQuery = () => {}; const showDatePicker = indexPattern.timeFieldName != ""; @@ -1055,7 +1400,7 @@ const Discover = (props) => { queryString: queryStringManager, timefilter, storage, - onQuerySubmit: updateQuery, + onQuerySubmit: onManualRefresh, services, dateRangeFrom: timeParam && timeParam[0] || "now-15m", // change by hardy dateRangeTo: timeParam && timeParam[1] || "now", @@ -1094,6 +1439,7 @@ const Discover = (props) => { { // layout, // onChange: setLayout, // }} + getShareUrl={getShareUrl} isEmpty={resultState === "none" && queryFrom === 0} onQueriesSelect={onQueriesSelect} onQueriesRemove={(id) => { @@ -1126,12 +1473,13 @@ const Discover = (props) => { searchInfo={{ took, total: hits, + totalRelation: hitsRelation, ...timeChartProps, }} selectedQueriesId={selectedQueriesId} searchConfig={{ trackTotalHits, - timeout: timeout, + timeout: searchTimeout, whetherToSample, sampleSize, topNumber, @@ -1143,7 +1491,7 @@ const Discover = (props) => { setTrackTotalHits(value); break; case 'time_out': - setTimeout(value) + setSearchTimeout(value) localStorage.setItem('search_time_out', value) break; case 'whether_to_sample': @@ -1246,6 +1594,7 @@ const Discover = (props) => { //unmappedFieldsConfig={unmappedFieldsConfig} //useNewFieldsApi={useNewFieldsApi} indices={props.indices} + clusterID={props.selectedCluster?.id} distinctParams={distinctParams} onDistinctParamsChange={onDistinctParamsChange} total={hits} @@ -1411,13 +1760,7 @@ const DiscoverUI = (props) => { }, [props.selectedCluster]); const getTimeFields = (IP) => { - const timeFields = []; - IP.fields.forEach((field) => { - if (field.spec.type == "date") { - timeFields.push(field.displayName); - } - }); - return timeFields; + return getDateFieldsFromIndexPattern(IP); }; useEffect(() => { if (queryParam) { @@ -1498,13 +1841,7 @@ const DiscoverUI = (props) => { } } const timeFields = getTimeFields(defaultIP); - if ( - timeFields && - timeFields.length == 1 && - defaultIP.timeFieldName == "" - ) { - defaultIP.timeFieldName = timeFields[0]; - } + defaultIP.timeFieldName = resolveValidTimeFieldName(defaultIP); setState({ indexPatternList: ils, indexPattern: defaultIP, @@ -1540,13 +1877,7 @@ const DiscoverUI = (props) => { const changeIndexPattern = React.useCallback( (indexPattern) => { const timeFields = getTimeFields(indexPattern); - if ( - timeFields && - timeFields.length == 1 && - indexPattern.timeFieldName == "" - ) { - indexPattern.timeFieldName = timeFields[0]; - } + indexPattern.timeFieldName = resolveValidTimeFieldName(indexPattern); setState({ ...state, indexPattern, @@ -1587,14 +1918,16 @@ const DiscoverUI = (props) => { - The current cluster has no indices or views + {formatMessage({ id: "insight.discover.empty.no_indices_or_views" })} } image={Empty.PRESENTED_IMAGE_SIMPLE} > {hasAuthority("data.index:all") && ( - - + + )} diff --git a/web/src/pages/DataManagement/Document.js b/web/src/pages/DataManagement/Document.js index 885832d3..07d00c4b 100644 --- a/web/src/pages/DataManagement/Document.js +++ b/web/src/pages/DataManagement/Document.js @@ -6,7 +6,7 @@ import { Col, Form, Row, Select, Input, Card, Icon, Table, InputNumber, Popconfirm, Divider, Button, Tooltip, Modal, DatePicker, message, Cascader, List,Radio } from 'antd'; -import Editor, {monaco} from '@monaco-editor/react'; +import Editor, { monaco } from '@/components/monaco-editor'; import moment from 'moment'; import {createDependencyProposals} from './autocomplete'; import InputSelect from '@/components/infini/InputSelect'; @@ -264,7 +264,10 @@ class JSONTable extends React.Component{ this.handleEditClick(item)} />, - this.handleDeleteClick(item)}> + this.handleDeleteClick(item)} + > , ]}> @@ -361,7 +364,10 @@ class EditableCell extends React.Component { )} - this.cancel(record.id)}> + this.cancel(record.id)} + > {formatMessage({id:'form.button.cancel'})} @@ -370,9 +376,12 @@ class EditableCell extends React.Component { {formatMessage({id:'form.button.edit'})} - this.delete(record)}> - {formatMessage({id:'form.button.delete'})} - + this.delete(record)} + > + {formatMessage({id:'form.button.delete'})} +
); }, @@ -890,4 +899,4 @@ class Doucment extends React.Component { } -export default Doucment; \ No newline at end of file +export default Doucment; diff --git a/web/src/pages/DataManagement/Index.js b/web/src/pages/DataManagement/Index.js index d68ffc5e..7ade0d11 100644 --- a/web/src/pages/DataManagement/Index.js +++ b/web/src/pages/DataManagement/Index.js @@ -37,11 +37,23 @@ import IconText from "@/components/infini/IconText"; import AutoTextEllipsis from "@/components/AutoTextEllipsis"; import commonStyles from "@/common.less" -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; const FormItem = Form.Item; const { TextArea } = Input; const { TabPane } = Tabs; +const placeholder = `{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1 + }, + "mappings": { + "properties": { + "field1": { "type": "text" }, + "field2": { "type": "keyword" } + } + } +}`; class JSONWrapper extends PureComponent { state = { @@ -82,19 +94,28 @@ class JSONWrapper extends PureComponent { } @Form.create() class CreateForm extends React.Component { + state = { + editorValue: "" + }; okHandle = () => { const { handleAdd, form } = this.props; - const me = this; form.validateFields((err, fieldsValue) => { if (err) return; - fieldsValue["config"] = me.editor.getValue(); + + fieldsValue["config"] = this.state.editorValue || "{}"; handleAdd(fieldsValue); form.resetFields(); + + if (this.editor) this.editor.setValue(""); + this.setState({ editorValue: "" }); }); }; onEditorDidMount = (editor) => { this.editor = editor; + editor.onDidChangeModelContent(() => { + this.setState({ editorValue: editor.getValue() }); + }); }; render() { @@ -122,13 +143,19 @@ class CreateForm extends React.Component { }), min: 5, }, - ], + { + pattern: /^[a-z0-9._-]*$/, + message: formatMessage({ + id: "indices.field.name.lowercase_message", + }), + }, + ], })( - + )} this.setState({ editorValue: value })} options={{ minimap: { enabled: false, @@ -174,7 +203,7 @@ class Index extends PureComponent { editingIndex: {}, indexActiveKey: "overview", showSystemIndices: false, - pageSize: 10, + pageSize: 20, deleteIndexItems: [], deleteIndexVisible: false, deleteIndexConfirm: false, @@ -234,7 +263,7 @@ class Index extends PureComponent { title: formatMessage({ id: "table.field.actions" }), width: 116, render: (text, record) => ( - + { this.setState({ @@ -247,8 +276,8 @@ class Index extends PureComponent { {hasAuthority("data.index:all") ? [ - , - this.showDeleteConfirm([record.index])}> + , + this.showDeleteConfirm([record.index])}> {formatMessage({ id: "form.button.delete" })} , ] @@ -266,6 +295,9 @@ class Index extends PureComponent { componentDidUpdate(oldProps, newState, snapshot) { if (oldProps.clusterID != this.props.clusterID) { this.fetchData(); + this.setState({ + selectedRowKeys: [], + }); } } @@ -304,12 +336,13 @@ class Index extends PureComponent { }).then(function(value) { if (value) { that.fetchData(); - message.success("deleted"); + message.success(formatMessage({ id: "app.message.delete.success" })); that.setState({ deleteIndexVisible: false, + selectedRowKeys: [], }); } else { - message.error("delete failed"); + message.error(formatMessage({ id: "app.message.delete.failed" })); } }); }; @@ -507,10 +540,12 @@ class Index extends PureComponent { }} >
- { this.handleSearch(value); }} @@ -536,6 +571,16 @@ class Index extends PureComponent { defaultChecked={this.state.showSystemIndices} />
+ {hasAuthority("data.index:all") ? ( + + ) : null} ) : null} - {hasAuthority("data.index:all") ? ( - - ) : null}
{ this.setState({ @@ -653,7 +689,10 @@ class Index extends PureComponent { - +
{hasAuthority("data.index:all") ? ( - +
- Edit, then save your JSON + {formatMessage({ id: "indices.hint.edit_json" })}
diff --git a/web/src/pages/DataManagement/IndexLifeCycle.js b/web/src/pages/DataManagement/IndexLifeCycle.js index ea6a2738..34c16362 100644 --- a/web/src/pages/DataManagement/IndexLifeCycle.js +++ b/web/src/pages/DataManagement/IndexLifeCycle.js @@ -138,7 +138,7 @@ class IndexLifeCycle extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>设置 { @@ -334,7 +334,7 @@ class IndexLifeCycle extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
{this.renderForm()}
diff --git a/web/src/pages/DataManagement/IndexPatterns.jsx b/web/src/pages/DataManagement/IndexPatterns.jsx index bd16c017..f17114d1 100644 --- a/web/src/pages/DataManagement/IndexPatterns.jsx +++ b/web/src/pages/DataManagement/IndexPatterns.jsx @@ -22,15 +22,41 @@ import { Card, Empty } from "antd"; import { CreateEditComplexFieldContainer } from "@/components/vendor/index_pattern_management/public/components/edit_index_pattern/create_edit_complex_field"; const IndexPatterns = (props) => { - if (!props.selectedCluster?.id) { - return ; - } const history = useMemo(() => { return new ScopedHistory(props.history, "/data/views"); }, [props.history]); + const breadcrumbList = useMemo(() => { + const pathname = props.location?.pathname || ""; + const items = [ + { + title: formatMessage({ id: "menu.home" }), + href: "/", + }, + { + title: formatMessage({ id: "menu.data" }), + }, + { + title: formatMessage({ id: "menu.data.view" }), + href: "/data/views", + }, + ]; + + if (pathname.startsWith("/data/views/create")) { + items.push({ + title: formatMessage({ id: "menu.data.view.create" }), + }); + } else if (pathname.startsWith("/data/views/patterns/")) { + items.push({ + title: formatMessage({ id: "menu.data.view.detail" }), + }); + } + + return items; + }, [props.location?.pathname]); + const createComponentKey = useMemo(() => { - const { http, uiSettings } = useGlobalContext(); + const { http } = useGlobalContext(); http.getServerBasePath = () => { return `${ESPrefix}/` + props.selectedCluster?.id; }; @@ -38,6 +64,10 @@ const IndexPatterns = (props) => { }, [props.selectedCluster]); useEffect(() => { + if (!props.selectedCluster?.id) { + return; + } + const { http, uiSettings } = useGlobalContext(); const initFetch = async () => { const defaultIndex = await http.fetch( @@ -48,11 +78,21 @@ const IndexPatterns = (props) => { initFetch(); }, [props.selectedCluster]); + if (!props.selectedCluster?.id) { + return ( + + + + + + ); + } + return ( - + @@ -76,10 +116,12 @@ const IndexPatterns = (props) => { - + + + diff --git a/web/src/pages/DataManagement/IndexSummary.js b/web/src/pages/DataManagement/IndexSummary.js index 4e3bad76..b9021b36 100644 --- a/web/src/pages/DataManagement/IndexSummary.js +++ b/web/src/pages/DataManagement/IndexSummary.js @@ -172,7 +172,7 @@ class IndexSummary extends Component { render() { let data = JSON.parse(datasource); return ( - +
diff --git a/web/src/pages/DataManagement/IndexTemplate.js b/web/src/pages/DataManagement/IndexTemplate.js index 8bbf8ccc..0f0f907a 100644 --- a/web/src/pages/DataManagement/IndexTemplate.js +++ b/web/src/pages/DataManagement/IndexTemplate.js @@ -195,7 +195,7 @@ class IndexTemplate extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>设置 { @@ -391,7 +391,7 @@ class IndexTemplate extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
{this.renderForm()}
diff --git a/web/src/pages/DataManagement/Insight/InsightBar/index.less b/web/src/pages/DataManagement/Insight/InsightBar/index.less index f45ae1a6..d3583e1a 100644 --- a/web/src/pages/DataManagement/Insight/InsightBar/index.less +++ b/web/src/pages/DataManagement/Insight/InsightBar/index.less @@ -2,15 +2,25 @@ position: absolute; right: 0px; top: 52px; + display: inline-flex; + align-items: center; + + .exportActions { + display: flex; + align-items: center; + } :global { .anticon { color: #006BB4; - margin-top: 4px; padding: 4px; - font-size: 16px; + font-size: 16px; + line-height: 1; + display: inline-flex; + align-items: center; + justify-content: center; margin-left: 4px; cursor: pointer; } } -} \ No newline at end of file +} diff --git a/web/src/pages/DataManagement/Insight/InsightBar/index.tsx b/web/src/pages/DataManagement/Insight/InsightBar/index.tsx index 16c541b2..3a58da24 100644 --- a/web/src/pages/DataManagement/Insight/InsightBar/index.tsx +++ b/web/src/pages/DataManagement/Insight/InsightBar/index.tsx @@ -9,6 +9,7 @@ import { Icon, message } from "antd"; import SearchInfo from "../SearchInfo"; import Histogram from "../Histogram"; import ViewLayout from "../ViewLayout"; +import { formatMessage } from "umi/locale"; export interface IQueries { clusterId: string; @@ -33,6 +34,7 @@ export interface IRecord { export interface IProps { queries: IQueries; + exportHits?: any[]; loading: boolean; isEmpty: boolean; mode: string; @@ -52,11 +54,63 @@ export interface IProps { showLayoutListIcon: boolean; viewLayout: any; onViewLayoutChange: (layout: any) => void; + getShareUrl?: () => string; } +const normalizeExportValue = (value: any) => { + if (value === null || value === undefined) { + return ""; + } + if (typeof value === "object") { + return JSON.stringify(value); + } + return String(value); +}; + +const escapeCSVValue = (value: any) => { + const text = normalizeExportValue(value); + return `"${text.replace(/"/g, '""')}"`; +}; + +const escapeHTML = (value: any) => { + return normalizeExportValue(value) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +}; + +const sanitizeFileName = (value: string) => { + return (value || "insight") + .replace(/[\\/:*?"<>|]+/g, "_") + .replace(/\s+/g, "_"); +}; + +const copyText = async (text: string) => { + if (!text) { + return false; + } + if (navigator?.clipboard?.writeText) { + await navigator.clipboard.writeText(text); + return true; + } + + const input = document.createElement("textarea"); + input.value = text; + input.setAttribute("readonly", "readonly"); + input.style.position = "fixed"; + input.style.top = "-9999px"; + document.body.appendChild(input); + input.select(); + const copied = document.execCommand("copy"); + document.body.removeChild(input); + return copied; +}; + export default forwardRef((props: IProps, ref: any) => { const { queries, + exportHits = [], loading: searchLoading, isEmpty, mode, @@ -73,7 +127,8 @@ export default forwardRef((props: IProps, ref: any) => { showLayoutListIcon, viewLayout, onViewLayoutChange, - histogramProps = {} + histogramProps = {}, + getShareUrl, } = props; const { @@ -92,6 +147,105 @@ export default forwardRef((props: IProps, ref: any) => { const [tags, setTags] = useState([]); const [loading, setLoading] = useState(false); + const getFlattenedHit = (hit: any) => { + if (indexPattern?.flattenHit) { + return indexPattern.flattenHit(hit, true) || {}; + } + return hit?._source || {}; + }; + + const getExportColumns = () => { + const visibleColumns = (columns || []).filter((item) => item); + if (visibleColumns.length > 0 && !visibleColumns.includes("_source")) { + return visibleColumns; + } + const fieldSet = new Set(); + exportHits.forEach((hit) => { + Object.keys(getFlattenedHit(hit)).forEach((field) => { + fieldSet.add(field); + }); + }); + return Array.from(fieldSet); + }; + + const downloadFile = (content: string, type: string, extension: string) => { + const baseName = sanitizeFileName(indexPattern?.title || "insight"); + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const blob = new Blob(["\ufeff", content], { type }); + const url = window.URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = `${baseName}-${timestamp}.${extension}`; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + window.URL.revokeObjectURL(url); + }; + + const handleExport = (type: "csv" | "excel") => { + if (!exportHits || exportHits.length === 0) { + message.warning(formatMessage({ id: "insight.export.empty" })); + return; + } + + const exportColumns = getExportColumns(); + if (exportColumns.length === 0) { + message.warning(formatMessage({ id: "insight.export.empty" })); + return; + } + + const rows = exportHits.map((hit) => { + const flattened = getFlattenedHit(hit); + return exportColumns.map((field) => flattened[field]); + }); + + if (type === "csv") { + const content = [ + exportColumns.map(escapeCSVValue).join(","), + ...rows.map((row) => row.map(escapeCSVValue).join(",")), + ].join("\n"); + downloadFile(content, "text/csv;charset=utf-8;", "csv"); + return; + } + + const content = ` + + + + + +
+ + ${exportColumns.map((field) => ``).join("")} + + + ${rows + .map( + (row) => + `${row.map((cell) => ``).join("")}` + ) + .join("")} + +
${escapeHTML(field)}
${escapeHTML(cell)}
+ + + `; + downloadFile(content, "application/vnd.ms-excel;charset=utf-8;", "xls"); + }; + + const handleShare = async () => { + try { + const shareUrl = getShareUrl?.(); + const copied = await copyText(shareUrl || ""); + if (!copied) { + throw new Error("copy failed"); + } + message.success(formatMessage({ id: "insight.share.success" })); + } catch (error) { + message.error(formatMessage({ id: "insight.share.failed" })); + } + }; + const onSelect = (item: IRecord) => { setRecord(item) onQueriesSelect(item) @@ -212,6 +366,23 @@ export default forwardRef((props: IProps, ref: any) => { onClick={() => handleModeChange("table")} /> */} { !isEmpty && mode !== "table" && } + + + handleExport("csv")} + /> + handleExport("excel")} + /> + { @@ -44,14 +45,14 @@ export default (props: IProps) => { className={styles.form} colon={false} > - + onChange(checked, 'track_total_hits')} /> - +
{
- Field Summary - + {formatMessage({ id: "insight.config.search.field_summary" })} + onChange(checked, 'whether_to_sample')} /> - + {sampleRecords==='manual'? { onChange={(value) => onChange(value, 'sample_size')} />:null} - + { ) -} \ No newline at end of file +} diff --git a/web/src/pages/DataManagement/Insight/InsightConfig/index.tsx b/web/src/pages/DataManagement/Insight/InsightConfig/index.tsx index cfc6bee1..4af2f2b6 100644 --- a/web/src/pages/DataManagement/Insight/InsightConfig/index.tsx +++ b/web/src/pages/DataManagement/Insight/InsightConfig/index.tsx @@ -1,5 +1,6 @@ import { Drawer, Icon, Tabs } from "antd"; import { useState } from "react"; +import { formatMessage } from "umi/locale"; import SearchConfig from "./SearchConfig"; import LayoutConfig from "./LayoutConfig"; @@ -25,17 +26,17 @@ export default (props: IProps) => { const { searchConfig, onSearchConfigChange, layoutConfig } = props; const [visible, setVisible] = useState(false); - const [activeKey, setActiveKey] = useState("Search"); + const [activeKey, setActiveKey] = useState("search"); return ( <> setVisible(true)} /> setVisible(false)} visible={visible} @@ -49,7 +50,7 @@ export default (props: IProps) => { onChange={setActiveKey} // tabBarExtraContent={ setVisible(false)}/>} > - + {/* @@ -59,4 +60,4 @@ export default (props: IProps) => { ) -} \ No newline at end of file +} diff --git a/web/src/pages/DataManagement/Insight/LoadQueries/index.tsx b/web/src/pages/DataManagement/Insight/LoadQueries/index.tsx index 1ee476d8..b8ddda5c 100644 --- a/web/src/pages/DataManagement/Insight/LoadQueries/index.tsx +++ b/web/src/pages/DataManagement/Insight/LoadQueries/index.tsx @@ -1,5 +1,6 @@ import { Drawer, Icon, Input, List, Popconfirm, Select, Tag } from "antd"; import { useMemo, useState } from "react"; +import { formatMessage } from "umi/locale"; import { IRecord } from "../InsightBar"; import styles from "./index.less"; @@ -32,9 +33,13 @@ export default (props: IProps) => { return ( <> - setVisible(true)}/> + setVisible(true)} + /> setVisible(false)} visible={visible} @@ -43,7 +48,7 @@ export default (props: IProps) => {
{ const {value} = e.target; setTimeout(() => { @@ -53,7 +58,7 @@ export default (props: IProps) => { style={{ width: 300, marginRight: 8 }} /> )} - + {getFieldDecorator('description', { initialValue: record?.description, })()} - + diff --git a/web/src/pages/DataManagement/Insight/SaveQueries/index.tsx b/web/src/pages/DataManagement/Insight/SaveQueries/index.tsx index f1973416..016e14ec 100644 --- a/web/src/pages/DataManagement/Insight/SaveQueries/index.tsx +++ b/web/src/pages/DataManagement/Insight/SaveQueries/index.tsx @@ -2,6 +2,7 @@ import { Drawer, Icon } from 'antd'; import { useMemo, useState } from 'react'; import { IRecord } from '../InsightBar'; import WrappedSaveQueriesForm from './SaveQueriesForm'; +import { formatMessage } from 'umi/locale'; interface IProps { tags: string[], @@ -31,11 +32,11 @@ export default (props: IProps) => { setVisible(true)} /> { setVisible(false) diff --git a/web/src/pages/DataManagement/Insight/SearchInfo/Info.tsx b/web/src/pages/DataManagement/Insight/SearchInfo/Info.tsx index e527ab52..921eece5 100644 --- a/web/src/pages/DataManagement/Insight/SearchInfo/Info.tsx +++ b/web/src/pages/DataManagement/Insight/SearchInfo/Info.tsx @@ -31,6 +31,7 @@ export interface IProps { */ stateInterval: string; total: number; + totalRelation?: string; took?: number; } @@ -40,6 +41,7 @@ export default ({ timeRange, stateInterval, total, + totalRelation, took, }: IProps) => { const [interval, setInterval] = useState(stateInterval); @@ -69,7 +71,7 @@ export default ({ >
- Found {total}{" "} + Found {totalRelation === "gte" ? `${total.toLocaleString()}+` : total.toLocaleString()}{" "} records {took && ( ({took} milliscond) diff --git a/web/src/pages/DataManagement/Insight/SearchInfo/index.tsx b/web/src/pages/DataManagement/Insight/SearchInfo/index.tsx index 1e7b783a..872efe5d 100644 --- a/web/src/pages/DataManagement/Insight/SearchInfo/index.tsx +++ b/web/src/pages/DataManagement/Insight/SearchInfo/index.tsx @@ -4,31 +4,50 @@ import Info, { IProps } from "./Info"; import styles from './index.scss'; export default (props: IProps & { loading: boolean }) => { - - const { loading, total } = props + const { loading, total } = props; const [showResultCount, setShowResultCount] = useState(true); - const timerRef = useRef(null) - const autoHiddenRef = useRef(true) + const timerRef = useRef | null>(null); + const autoHiddenRef = useRef(true); + const isMountedRef = useRef(true); // 防止卸载后 setState + // 处理自动隐藏逻辑 useEffect(() => { + if (!showResultCount) return; + if (timerRef.current) { - clearTimeout(timerRef.current) + clearTimeout(timerRef.current); } - if (showResultCount) { - timerRef.current = setTimeout(() => { - if (autoHiddenRef.current) { - setShowResultCount(false) - } - }, 3000); - } - }, [showResultCount]) + timerRef.current = setTimeout(() => { + if (autoHiddenRef.current && isMountedRef.current) { + setShowResultCount(false); + } + }, 3000); + + // 清理函数 + return () => { + if (timerRef.current) { + clearTimeout(timerRef.current); + } + }; + }, [showResultCount]); + + // 更新 autoHiddenRef useEffect(() => { - if (loading) { - autoHiddenRef.current = true - } - }, [loading]) + autoHiddenRef.current = loading ? true : autoHiddenRef.current; + }, [loading]); + + // 组件卸载时标记 + useEffect(() => { + isMountedRef.current = true; + return () => { + isMountedRef.current = false; + if (timerRef.current) { + clearTimeout(timerRef.current); + } + }; + }, []); if (typeof total !== 'number' || total <= 0) return null; @@ -38,20 +57,21 @@ export default (props: IProps & { loading: boolean }) => { placement="left" title={null} overlayClassName={styles.searchInfo} - content={( - + } + > + { + autoHiddenRef.current = showResultCount ? true : false; + setShowResultCount(!showResultCount); + }} /> - )}> - { - if (showResultCount) { - autoHiddenRef.current = true - } else { - autoHiddenRef.current = false - } - setShowResultCount(!showResultCount) - }}/> - ) + ); } \ No newline at end of file diff --git a/web/src/pages/DataManagement/SearchFlow/Form.jsx b/web/src/pages/DataManagement/SearchFlow/Form.jsx index 52c4f6dd..a0d8ebff 100644 --- a/web/src/pages/DataManagement/SearchFlow/Form.jsx +++ b/web/src/pages/DataManagement/SearchFlow/Form.jsx @@ -22,6 +22,7 @@ import { useGlobal } from "@/layouts/GlobalContext"; export default Form.create({ name: "trace_template_form" })( ({ form, history, match }) => { const { selectedCluster } = useGlobal(); + const clusterId = selectedCluster?.id; const mode = React.useMemo(() => { const { template_id } = match.params; if (template_id) { @@ -34,8 +35,11 @@ export default Form.create({ name: "trace_template_form" })( const { template_id } = match.params; if (template_id) { const fetchData = async () => { + if (!clusterId) { + return false; + } const res = await request( - `${ESPrefix}/${selectedCluster.id}/trace_template/${template_id}`, + `${ESPrefix}/${clusterId}/trace_template/${template_id}`, {} ); if (!res || res.error) { @@ -45,7 +49,7 @@ export default Form.create({ name: "trace_template_form" })( }; fetchData(); } - }, [selectedCluster]); + }, [clusterId]); const handleSubmit = React.useCallback(() => { form.validateFields(async (errors, values) => { @@ -53,8 +57,11 @@ export default Form.create({ name: "trace_template_form" })( return; } let res = null; + if (!clusterId) { + return false; + } if (mode == "NEW") { - res = request(`${ESPrefix}/${selectedCluster.id}/trace_template`, { + res = request(`${ESPrefix}/${clusterId}/trace_template`, { method: "POST", body: values, }); @@ -62,7 +69,7 @@ export default Form.create({ name: "trace_template_form" })( const { template_id } = match.params; res = request( - `${ESPrefix}/${selectedCluster.id}/trace_template/${template_id}`, + `${ESPrefix}/${clusterId}/trace_template/${template_id}`, { method: "PUT", body: { @@ -79,7 +86,7 @@ export default Form.create({ name: "trace_template_form" })( if (mode == "NEW") form.resetFields(); message.success("save succeed"); }); - }, [selectedCluster, mode, editValue]); + }, [clusterId, mode, editValue]); const { getFieldDecorator } = form; const formItemLayout = { @@ -104,7 +111,7 @@ export default Form.create({ name: "trace_template_form" })( }, }, }; - if (!selectedCluster.id) { + if (!clusterId) { return null; } return ( diff --git a/web/src/pages/DataManagement/SearchFlow/SearchFlow.jsx b/web/src/pages/DataManagement/SearchFlow/SearchFlow.jsx index ba93b56e..991b6a8a 100644 --- a/web/src/pages/DataManagement/SearchFlow/SearchFlow.jsx +++ b/web/src/pages/DataManagement/SearchFlow/SearchFlow.jsx @@ -24,10 +24,14 @@ import request from "@/utils/request"; export default ({}) => { const { selectedCluster } = useGlobal(); + const clusterId = selectedCluster?.id; const handleDeleteClick = useCallback( async (id) => { + if (!clusterId) { + return false; + } const res = await request( - `${ESPrefix}/${selectedCluster.id}/trace_template/${id}`, + `${ESPrefix}/${clusterId}/trace_template/${id}`, { method: "DELETE", } @@ -36,12 +40,12 @@ export default ({}) => { return false; } message.success("delete succeed"); - setQueryParams({ - ...queryParams, + setQueryParams((st) => ({ + ...st, t: new Date().valueOf(), - }); + })); }, - [selectedCluster] + [clusterId] ); const columns = [ { @@ -70,12 +74,12 @@ export default ({}) => { ), }, @@ -83,11 +87,12 @@ export default ({}) => { const [queryParams, setQueryParams] = React.useState({}); const { loading, error, value } = useFetch( - `${ESPrefix}/${selectedCluster.id}/trace_template`, + `${ESPrefix}/${clusterId}/trace_template`, { queryParams: queryParams, }, - [selectedCluster, queryParams] + [clusterId, queryParams], + !!clusterId ); const { data: templates, total } = React.useMemo(() => { if (!value) { @@ -119,7 +124,7 @@ export default ({}) => { />
-
diff --git a/web/src/pages/DataManagement/SearchFlow/TraceSearch.jsx b/web/src/pages/DataManagement/SearchFlow/TraceSearch.jsx index 15beff75..706ba645 100644 --- a/web/src/pages/DataManagement/SearchFlow/TraceSearch.jsx +++ b/web/src/pages/DataManagement/SearchFlow/TraceSearch.jsx @@ -1,20 +1,23 @@ import React, { useRef } from "react"; -import { Input, Select, Button } from "antd"; +import { Input, Select } from "antd"; import useFetch from "@/lib/hooks/use_fetch"; import { ESPrefix } from "@/services/common"; import { useGlobal } from "@/layouts/GlobalContext"; import { formatESSearchResult } from "@/lib/elasticsearch/util"; import { on } from "@svgdotjs/svg.js"; +import SearchInput from "@/components/infini/SearchInput"; export default ({ onTraceIDSearch }) => { const { selectedCluster } = useGlobal(); + const clusterId = selectedCluster?.id; const [queryParams, setQueryParams] = React.useState({ size: 1000 }); const { loading, error, value } = useFetch( - `${ESPrefix}/${selectedCluster.id}/trace_template`, + `${ESPrefix}/${clusterId}/trace_template`, { queryParams: queryParams, }, - [selectedCluster, queryParams] + [clusterId, queryParams], + !!clusterId ); const [selectedTemplate, setSelectedTemplate] = React.useState(); const { data: templates, total } = React.useMemo(() => { @@ -59,15 +62,11 @@ export default ({ onTraceIDSearch }) => { ))} - - 搜索 - - } + disabled={!clusterId} onSearch={onTraceSearch} /> diff --git a/web/src/pages/DataManagement/SearchFlow/newsteps/Parameters.jsx b/web/src/pages/DataManagement/SearchFlow/newsteps/Parameters.jsx index e8198937..e00ebe83 100644 --- a/web/src/pages/DataManagement/SearchFlow/newsteps/Parameters.jsx +++ b/web/src/pages/DataManagement/SearchFlow/newsteps/Parameters.jsx @@ -21,16 +21,16 @@ export default ({ onChange, parameters }) => { }, { title: formatMessage({ id: "table.field.actions" }), - render: (text, record) => ( -
- handleDeleteClick(record.key)} - > - Delete - -
- ), + render: (text, record) => ( +
+ handleDeleteClick(record.key)} + > + {formatMessage({ id: "form.button.delete" })} + +
+ ), }, ]; const handleDeleteClick = (key) => { diff --git a/web/src/pages/DataManagement/View/LayoutList.jsx b/web/src/pages/DataManagement/View/LayoutList.jsx index ea035be7..f1b67630 100644 --- a/web/src/pages/DataManagement/View/LayoutList.jsx +++ b/web/src/pages/DataManagement/View/LayoutList.jsx @@ -8,6 +8,7 @@ import { formatMessage } from "umi/locale"; import request from "@/utils/request"; import styles from './LayoutList.less'; import { withRouter } from "react-router-dom"; +import SearchInput from "@/components/infini/SearchInput"; export default withRouter((props) => { const { layout, indexPattern, clusterId, isView, onRowSelect } = props; @@ -141,7 +142,7 @@ export default withRouter((props) => { return ( isView ? ( - + { selectedRow?.id !== record.id && ( { @@ -153,16 +154,16 @@ export default withRouter((props) => { { defaultAction } ) : ( - + Edit onRemove(record.id)} > - Delete + {formatMessage({ id: "form.button.delete" })} { defaultAction } @@ -194,7 +195,7 @@ export default withRouter((props) => { }} >
- { />
) -}) \ No newline at end of file +}) diff --git a/web/src/pages/DataManagement/View/Widget/WidgetBody/Chart.jsx b/web/src/pages/DataManagement/View/Widget/WidgetBody/Chart.jsx index ade30b7c..f4f0d238 100644 --- a/web/src/pages/DataManagement/View/Widget/WidgetBody/Chart.jsx +++ b/web/src/pages/DataManagement/View/Widget/WidgetBody/Chart.jsx @@ -55,7 +55,9 @@ export default (props) => { fetchParamsCache, handleContextMenu, isFullScreen, - onResultChange + onResultChange, + lockInteractions, + autoApplyRangeFilter, } = props; const { series = [] } = record; @@ -67,7 +69,11 @@ export default (props) => { const chartRef = useRef(null) - const isLockRef = useRef(isEdit || isFullScreen) + const getIsInteractionLocked = () => + isEdit || + (typeof lockInteractions === "boolean" ? lockInteractions : isFullScreen); + + const isLockRef = useRef(getIsInteractionLocked()) const fetchData = async (params, zoom, refresh) => { if (['iframe'].includes(type) || !params || !params.currentQueries) return; @@ -113,7 +119,22 @@ export default (props) => { newRange = getZoomRange(zoom, range, isTimeSeries) rangeFilter = buildFilterRange(filter, time_field, newRange) } - const bucketSize = isTimeSeries ? getBucketSize(bucketSizeCache || bucket_size, newRange) : undefined + const rawBucketSize = bucketSizeCache || bucket_size; + const bucketSize = isTimeSeries + ? (rawBucketSize === 'auto' ? 'auto' : getBucketSize(rawBucketSize, newRange)) + : undefined + // Dynamic bucket count for auto_date_histogram: 80 for >=7d, 120 otherwise + let autoBuckets; + if (bucketSize === 'auto' && newRange) { + try { + const bounds = calculateBounds(newRange, { forceNow: getForceNow() }); + if (bounds && bounds.min && bounds.max) { + const rangeMs = bounds.max.valueOf() - bounds.min.valueOf(); + const sevenDaysMs = 7 * 24 * 60 * 60 * 1000; + autoBuckets = rangeMs >= sevenDaysMs ? 80 : 120; + } + } catch (e) { /* ignore */ } + } if (fetchParamsCache) { fetchParamsCache.current.bucketSize = bucketSize } @@ -216,6 +237,7 @@ export default (props) => { sort: item.sort || [], formula: item.formula || 'a', bucket_size: bucketSize, + ...(autoBuckets ? { buckets: autoBuckets } : {}), } }) const promises = bodys.map((item) => getWidgetData(item)) @@ -355,8 +377,8 @@ export default (props) => { } useEffect(() => { - isLockRef.current = isEdit || isFullScreen - }, [isEdit, isFullScreen]) + isLockRef.current = getIsInteractionLocked() + }, [isEdit, isFullScreen, lockInteractions]) const onChartElementClick = (data, callback) => { if (isLockRef.current) return; @@ -397,7 +419,10 @@ export default (props) => { const { id, title, bucket_size, series = [] } = record; const { type } = series[0] || {} const { range, bucketSizeCache } = currentParams.currentQueries; - const bucketSize = currentParams?.isTimeSeries ? getBucketSize(bucketSizeCache || bucket_size, range) : undefined + const rawBucketSizeForRender = bucketSizeCache || bucket_size; + const bucketSize = currentParams?.isTimeSeries + ? (rawBucketSizeForRender === 'auto' ? 'auto' : getBucketSize(rawBucketSizeForRender, range)) + : undefined const widget = WIDGETS.find((w) => w.type === type); let isGroup = false; if (result.data?.[0]?.group) { @@ -430,10 +455,11 @@ export default (props) => { isTimeSeries={currentParams?.isTimeSeries} highlightRange={highlightRange} currentQueries={currentParams?.currentQueries || {}} - isLock={isEdit || isFullScreen} + isLock={getIsInteractionLocked()} isEdit={isEdit} isFullScreen={isFullScreen} handleContextMenu={handleContextMenu} + autoApplyRangeFilter={autoApplyRangeFilter} onChartElementClick={onChartElementClick} /> ) @@ -521,7 +547,7 @@ export default (props) => { const isSingleMetric = series.length === 1; let newData = Array.isArray(data) ? data : []; - if (Number.isInteger(size)) { + if (Number.isInteger(size) && !currentParams?.isTimeSeries) { newData = newData.map((item) => { item = Array.isArray(item) ? item : []; if(order == "desc"){ @@ -555,6 +581,15 @@ export default (props) => { name: isSingleMetric ? item.group : `${item.group}-${item.name}`, })) } + newData = [...newData].sort((a, b) => { + const timestampDiff = Number(a?.timestamp || 0) - Number(b?.timestamp || 0); + if (timestampDiff !== 0) { + return timestampDiff; + } + const nameA = `${a?.name || ""}`; + const nameB = `${b?.name || ""}`; + return nameA.localeCompare(nameB); + }); } else { if (isGroup) { newData = newData.map((item) => ({ @@ -570,11 +605,11 @@ export default (props) => { } } - if (order === "desc") { + if (!currentParams?.isTimeSeries && order === "desc") { newData = newData.sort((a, b) => b.value - a.value) } - if (order === "asc") { + if (!currentParams?.isTimeSeries && order === "asc") { newData = newData.sort((a, b) => a.value - b.value) } diff --git a/web/src/pages/DataManagement/View/Widget/WidgetConfig/WidgetConfigDrawer.jsx b/web/src/pages/DataManagement/View/Widget/WidgetConfig/WidgetConfigDrawer.jsx index 85c2e602..b268f0d8 100644 --- a/web/src/pages/DataManagement/View/Widget/WidgetConfig/WidgetConfigDrawer.jsx +++ b/web/src/pages/DataManagement/View/Widget/WidgetConfig/WidgetConfigDrawer.jsx @@ -42,7 +42,7 @@ export default (props) => { padding: 0, height: "calc(100vh - 55px)", }} - wrapClassName={styles.widgetConfig} + className={styles.widgetConfig} onClose={() => onVisibleChange(false)} destroyOnClose > diff --git a/web/src/pages/DataManagement/View/Widget/WidgetEmpty/index.jsx b/web/src/pages/DataManagement/View/Widget/WidgetEmpty/index.jsx index fb130a4a..dd9aa42d 100644 --- a/web/src/pages/DataManagement/View/Widget/WidgetEmpty/index.jsx +++ b/web/src/pages/DataManagement/View/Widget/WidgetEmpty/index.jsx @@ -1,4 +1,5 @@ import { Icon, Popconfirm } from 'antd'; +import { formatMessage } from "umi/locale"; import styles from './index.less'; export default (props) => { @@ -9,7 +10,7 @@ export default (props) => { return (
handleRemove(record)} >
x
@@ -23,4 +24,4 @@ export default (props) => { />
) -} \ No newline at end of file +} diff --git a/web/src/pages/DataManagement/View/Widget/index.jsx b/web/src/pages/DataManagement/View/Widget/index.jsx index 63d0175b..662bbe3f 100644 --- a/web/src/pages/DataManagement/View/Widget/index.jsx +++ b/web/src/pages/DataManagement/View/Widget/index.jsx @@ -43,6 +43,8 @@ export default (props) => { hideHeader, displayOptions={}, onResultChange, + lockInteractions, + autoApplyRangeFilter, } = props; const [cacheRecord, setCacheRecord] = useState(record) @@ -100,6 +102,130 @@ export default (props) => { setCacheRecord(record) } + const parseBucketSize = (bucketSize) => { + const matched = `${bucketSize || ""}`.trim().match(/^(\d+)(ms|s|m|h|d|w|M|y)$/); + if (!matched) { + return null; + } + const value = parseInt(matched[1], 10); + if (!Number.isInteger(value) || value <= 0) { + return null; + } + return { + value, + unit: matched[2], + }; + } + + const getMomentAddUnit = (unit) => { + const unitMap = { + ms: "milliseconds", + s: "seconds", + m: "minutes", + h: "hours", + d: "days", + w: "weeks", + M: "months", + y: "years", + }; + return unitMap[unit]; + } + + const floorMomentToBucket = (value, bucketSize) => { + const parsedBucket = parseBucketSize(bucketSize); + const currentMoment = moment(value); + if (!parsedBucket || !currentMoment.isValid()) { + return null; + } + const { value: bucketValue, unit } = parsedBucket; + switch (unit) { + case "ms": + return currentMoment.clone(); + case "s": + return currentMoment.clone().milliseconds(0).seconds( + Math.floor(currentMoment.seconds() / bucketValue) * bucketValue + ); + case "m": + return currentMoment + .clone() + .startOf("hour") + .add(Math.floor(currentMoment.minutes() / bucketValue) * bucketValue, "minutes"); + case "h": { + // Use UTC to match ES bucket alignment (no time_zone in backend queries) + const utcH = moment.utc(value); + return moment( + utcH.clone().startOf("day") + .add(Math.floor(utcH.hours() / bucketValue) * bucketValue, "hours") + .valueOf() + ); + } + case "d": { + const utcD = moment.utc(value); + return moment( + utcD.clone().startOf("month") + .add(Math.floor((utcD.date() - 1) / bucketValue) * bucketValue, "days") + .valueOf() + ); + } + case "w": { + const baseMoment = moment.utc(0).startOf("week"); + const diff = moment.utc(value).clone().startOf("week").diff(baseMoment, "weeks"); + return moment(baseMoment.clone().add(Math.floor(diff / bucketValue) * bucketValue, "weeks").valueOf()); + } + case "M": { + const utcM = moment.utc(value); + return moment( + utcM.clone().startOf("year") + .add(Math.floor(utcM.month() / bucketValue) * bucketValue, "months") + .valueOf() + ); + } + case "y": { + const baseMoment = moment.utc(0).startOf("year"); + const diff = moment.utc(value).clone().startOf("year").diff(baseMoment, "years"); + return moment(baseMoment.clone().add(Math.floor(diff / bucketValue) * bucketValue, "years").valueOf()); + } + default: + return null; + } + } + + const ceilMomentToBucket = (value, bucketSize) => { + const parsedBucket = parseBucketSize(bucketSize); + const currentMoment = moment(value); + const flooredMoment = floorMomentToBucket(value, bucketSize); + if (!parsedBucket || !currentMoment.isValid() || !flooredMoment) { + return null; + } + if (currentMoment.valueOf() === flooredMoment.valueOf()) { + return flooredMoment.clone(); + } + return flooredMoment.clone().add(parsedBucket.value, getMomentAddUnit(parsedBucket.unit)); + } + + const alignRangeToBucket = (range, bucketSize) => { + const parsedBucket = parseBucketSize(bucketSize); + if (!parsedBucket || !range?.from || !range?.to) { + return null; + } + const startMoment = floorMomentToBucket(range.from, bucketSize); + let endMoment = ceilMomentToBucket(range.to, bucketSize); + const rawEndMoment = moment(range.to); + if (!startMoment || !endMoment) { + return null; + } + if ( + endMoment.valueOf() === startMoment.valueOf() || + (rawEndMoment.isValid() && endMoment.valueOf() === rawEndMoment.valueOf()) + ) { + endMoment = endMoment.clone().add(parsedBucket.value, getMomentAddUnit(parsedBucket.unit)); + } + return { + from: startMoment.toISOString(), + to: endMoment.clone().subtract(1, "millisecond").toISOString(), + }; + } + const handleZoom = (newZoom) => { if (newZoom === 0) { setZoom(newZoom) @@ -144,7 +270,10 @@ export default (props) => { if (!params || !params.range) return; const { range } = params; let newRange = {}; - if (range.from === range.to) { + const alignedRange = alignRangeToBucket(range, fetchParamsCacheRef.current?.bucketSize); + if (alignedRange) { + newRange = alignedRange; + } else if (range.from === range.to) { if (fetchParamsCacheRef.current?.bucketSize) { let value = parseInt(fetchParamsCacheRef.current?.bucketSize) let unit = (fetchParamsCacheRef.current?.bucketSize).replace(/\d+/gi,"") @@ -253,6 +382,8 @@ export default (props) => { queriesBarParams={queriesBarParams} handleContextMenu={handleContextMenu} isFullScreen={isFullScreen} + lockInteractions={lockInteractions} + autoApplyRangeFilter={autoApplyRangeFilter} onResultChange={onResultChange} /> diff --git a/web/src/pages/DataManagement/View/Widget/widgets/area/Visualization.jsx b/web/src/pages/DataManagement/View/Widget/widgets/area/Visualization.jsx index 634a9c87..d4666780 100644 --- a/web/src/pages/DataManagement/View/Widget/widgets/area/Visualization.jsx +++ b/web/src/pages/DataManagement/View/Widget/widgets/area/Visualization.jsx @@ -11,7 +11,7 @@ import moment from "moment"; export default (props) => { - const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, brushMenu, currentQueries = {}, handleContextMenu } = props; + const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, brushMenu, currentQueries = {}, handleContextMenu, autoApplyRangeFilter } = props; const { id, is_percent, drilling = {}, legend } = record; @@ -26,13 +26,18 @@ export default (props) => { brushMenuRef.current.close() }, onEnd: (params, position) => { - brushMenuRef.current.open({ + const nextParams = { ...currentQueries, range: { ...(currentQueries.range || {}), ...(params.range || {}) } - }, position) + }; + if (autoApplyRangeFilter) { + handleContextMenu(nextParams, TYPE_RANGE_FILTER); + return; + } + brushMenuRef.current.open(nextParams, position) } }) } diff --git a/web/src/pages/DataManagement/View/Widget/widgets/column/Visualization.jsx b/web/src/pages/DataManagement/View/Widget/widgets/column/Visualization.jsx index 62b8cbd6..595db6a8 100644 --- a/web/src/pages/DataManagement/View/Widget/widgets/column/Visualization.jsx +++ b/web/src/pages/DataManagement/View/Widget/widgets/column/Visualization.jsx @@ -12,7 +12,7 @@ import { Icon } from "antd"; export default (props) => { - const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, currentQueries = {}, handleContextMenu, onChartElementClick } = props; + const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, currentQueries = {}, handleContextMenu, onChartElementClick, autoApplyRangeFilter } = props; const { id, is_stack, is_percent, drilling = {}, series, legend } = record; @@ -39,13 +39,18 @@ export default (props) => { brushMenuRef.current.close() }, onEnd: (params, position) => { - brushMenuRef.current?.open({ + const nextParams = { ...currentQueries, range: { ...(currentQueries.range || {}), ...(params.range || {}) } - }, position) + }; + if (autoApplyRangeFilter) { + handleContextMenu(nextParams, TYPE_RANGE_FILTER); + return; + } + brushMenuRef.current?.open(nextParams, position) }, }) } diff --git a/web/src/pages/DataManagement/View/Widget/widgets/date-histogram/Visualization.jsx b/web/src/pages/DataManagement/View/Widget/widgets/date-histogram/Visualization.jsx index 99649992..1da9a504 100644 --- a/web/src/pages/DataManagement/View/Widget/widgets/date-histogram/Visualization.jsx +++ b/web/src/pages/DataManagement/View/Widget/widgets/date-histogram/Visualization.jsx @@ -12,7 +12,7 @@ import { Icon } from "antd"; export default (props) => { - const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, currentQueries = {}, handleContextMenu, onChartElementClick } = props; + const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, currentQueries = {}, handleContextMenu, onChartElementClick, autoApplyRangeFilter } = props; const { id, is_stack, is_percent, drilling = {}, series, legend, colors } = record; @@ -39,13 +39,18 @@ export default (props) => { brushMenuRef.current.close() }, onEnd: (params, position) => { - brushMenuRef.current?.open({ + const nextParams = { ...currentQueries, range: { ...(currentQueries.range || {}), ...(params.range || {}) } - }, position) + }; + if (autoApplyRangeFilter) { + handleContextMenu(nextParams, TYPE_RANGE_FILTER); + return; + } + brushMenuRef.current?.open(nextParams, position) }, }) } diff --git a/web/src/pages/DataManagement/View/Widget/widgets/index.jsx b/web/src/pages/DataManagement/View/Widget/widgets/index.jsx index 323c5e46..0309a3f2 100644 --- a/web/src/pages/DataManagement/View/Widget/widgets/index.jsx +++ b/web/src/pages/DataManagement/View/Widget/widgets/index.jsx @@ -105,6 +105,19 @@ export const getXOptions = (record, result, params) => { const { isGroup, bucketSize = '', isTimeSeries } = params + // Infer time format from data range when bucketSize is "auto" + const inferAutoTimeFormat = () => { + if (!data || data.length < 2) return 'HH:mm'; + const timestamps = data.map((d) => d.timestamp).filter(Boolean).sort((a, b) => a - b); + if (timestamps.length < 2) return 'HH:mm'; + const rangeMs = timestamps[timestamps.length - 1] - timestamps[0]; + const sevenDays = 7 * 24 * 60 * 60 * 1000; + const oneDay = 24 * 60 * 60 * 1000; + if (rangeMs >= sevenDays) return 'YYYY-MM-DD'; + if (rangeMs >= oneDay) return 'MM-DD HH:mm'; + return 'HH:mm'; + }; + const options = { xField: isTimeSeries ? 'timestamp' : 'group', xAxis: { @@ -125,7 +138,9 @@ export const getXOptions = (record, result, params) => { 'h': 'HH:mm', 'd': 'YYYY-MM-DD' } - return formatTime(value, timeFormatters[bucketSize.replace(/\d+/g, '')]); + const unit = bucketSize.replace(/\d+/g, ''); + const fmt = timeFormatters[unit] || inferAutoTimeFormat(); + return formatTime(value, fmt); } }, }, @@ -175,6 +190,8 @@ export const getTooltipOption = (record, bucketSize = '', showTitle = true) => { 'h': 'YYYY-MM-DD HH:mm', 'd': 'YYYY-MM-DD' } + const unit = bucketSize.replace(/\d+/g, ''); + const tooltipFmt = timeFormatters[unit] || 'YYYY-MM-DD HH:mm'; const validItems = items.filter(item => item.value !== undefined); const sortedItems = [...validItems].sort((a, b) => b.value - a.value); return ( @@ -182,7 +199,7 @@ export const getTooltipOption = (record, bucketSize = '', showTitle = true) => { { showTitle && (
- { Number.isInteger(Number(title)) ? formatTime(title, timeFormatters[bucketSize.replace(/\d+/g, '')]) : title} + { Number.isInteger(Number(title)) ? formatTime(title, tooltipFmt) : title}
) } diff --git a/web/src/pages/DataManagement/View/Widget/widgets/line/Visualization.jsx b/web/src/pages/DataManagement/View/Widget/widgets/line/Visualization.jsx index def3241e..2f071b51 100644 --- a/web/src/pages/DataManagement/View/Widget/widgets/line/Visualization.jsx +++ b/web/src/pages/DataManagement/View/Widget/widgets/line/Visualization.jsx @@ -11,7 +11,7 @@ import moment from "moment"; export default (props) => { - const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, currentQueries = {}, handleContextMenu } = props; + const { record, result, options, isGroup, isLock, onReady, bucketSize, isTimeSeries, highlightRange, currentQueries = {}, handleContextMenu, autoApplyRangeFilter } = props; const { id, drilling = {}, legend } = record; @@ -26,13 +26,18 @@ export default (props) => { brushMenuRef.current.close() }, onEnd: (params, position) => { - brushMenuRef.current.open({ + const nextParams = { ...currentQueries, range: { ...(currentQueries.range || {}), ...(params.range || {}) } - }, position) + }; + if (autoApplyRangeFilter) { + handleContextMenu(nextParams, TYPE_RANGE_FILTER); + return; + } + brushMenuRef.current.open(nextParams, position) } }) } diff --git a/web/src/pages/DataManagement/View/WidgetLoader.jsx b/web/src/pages/DataManagement/View/WidgetLoader.jsx index dc8b31d5..919ede4f 100644 --- a/web/src/pages/DataManagement/View/WidgetLoader.jsx +++ b/web/src/pages/DataManagement/View/WidgetLoader.jsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from "react" +import { useEffect, useMemo, useRef, useState } from "react" import Widget from "./Widget" import styles from "./WidgetLoader.less" import { Empty, Icon, message, Spin, Tooltip } from "antd" @@ -11,10 +11,21 @@ import { CopyToClipboard } from "react-copy-to-clipboard"; import { formatMessage } from "umi/locale"; export const WidgetRender = (props) => { - - const { widget, range, query, queryParams = {}, highlightRange = {}, refresh, showCopy = true } = props; + + const { + widget, + range, + query, + queryParams = {}, + highlightRange = {}, + refresh, + showCopy = true, + onGlobalQueriesChange = () => {}, + onHighlightRangeChange = () => {}, + } = props; const [globalRangeCache, setGlobalRangeCache] = useState() const [requests, setRequests] = useState([]) + const fallbackWidgetIdRef = useRef(`widget-${Math.random().toString(36).slice(2)}`) const filters = useMemo(() => { const newFilters = [] @@ -33,17 +44,68 @@ export const WidgetRender = (props) => { return mergeFilters([], newFilters) }, [JSON.stringify(queryParams)]) + const normalizeRangeValue = (value, fallback) => { + if (typeof value === "string" || typeof value === "number") { + return `${value}`; + } + if (value && typeof value === "object") { + for (const key of ["from", "to", "min", "max", "gte", "lte", "start", "end"]) { + if ( + Object.prototype.hasOwnProperty.call(value, key) && + value[key] !== undefined && + value[key] !== null + ) { + const candidate = value[key]; + if (typeof candidate === "string" || typeof candidate === "number") { + return `${candidate}`; + } + } + } + } + return fallback; + }; + const formatTimeRange = useMemo(() => { - if (!range.from || !range.to) return { from: 'now-15m', to: 'now'} - const bounds = calculateBounds(range); + const from = normalizeRangeValue(range?.from, "now-15m"); + const to = normalizeRangeValue(range?.to, "now"); + if (from === "auto" || to === "auto") { + return { from: "auto", to: "auto" }; + } + if (!from || !to) { + return { from: "now-15m", to: "now" }; + } + try { + const bounds = calculateBounds({ from, to }); + return { + from: moment(bounds.min.valueOf()).tz(getTimezone()).utc().format(), + to: moment(bounds.max.valueOf()).tz(getTimezone()).utc().format(), + }; + } catch (e) { + return { from: "now-15m", to: "now" }; + } + }, [JSON.stringify(range), refresh]); + + const hasRenderableWidget = useMemo(() => ( + !!widget && + Array.isArray(widget?.series) && + widget.series.length > 0 + ), [widget]); + + const renderWidget = useMemo(() => { + if (!hasRenderableWidget) { + return widget; + } + if (widget.id) { + return widget; + } return { - from: moment(bounds.min.valueOf()).tz(getTimezone()).utc().format(), - to: moment(bounds.max.valueOf()).tz(getTimezone()).utc().format(), + ...widget, + id: fallbackWidgetIdRef.current, }; - }, [JSON.stringify(range), refresh]); + }, [widget, hasRenderableWidget]); return ( - widget ? ( + hasRenderableWidget ? (
{ showCopy && requests.length > 0 && ( @@ -57,7 +119,7 @@ export const WidgetRender = (props) => { ) } { hideHeader: true, hideBorder: true, }} - globalRangeCache={globalRangeCache} - onGlobalRangeCacheChange={setGlobalRangeCache} - onGlobalQueriesChange={() => {}} - onClone={() => {}} - onRemove={() => {}} - onSave={() => {}} - onFullElement={() => {}} - clusterList={[]} - highlightRange={highlightRange} - onHighlightRangeChange={() => {}} - onResultChange={(res) => { - setRequests(Array.isArray(res) ? res.filter((item) => !!item.request).map((item) => item.request) : []) - }} - refresh={refresh} + lockInteractions={false} + autoApplyRangeFilter={true} + globalRangeCache={globalRangeCache} + onGlobalRangeCacheChange={setGlobalRangeCache} + onGlobalQueriesChange={onGlobalQueriesChange} + onClone={() => {}} + onRemove={() => {}} + onSave={() => {}} + onFullElement={() => {}} + clusterList={[]} + highlightRange={highlightRange} + onHighlightRangeChange={onHighlightRangeChange} + onResultChange={(res) => { + setRequests(Array.isArray(res) ? res.filter((item) => !!item.request).map((item) => item.request) : []) + }} + refresh={refresh} />
) : @@ -123,4 +187,4 @@ export default (props) => {
) -} \ No newline at end of file +} diff --git a/web/src/pages/DataManagement/View/utils/utils.js b/web/src/pages/DataManagement/View/utils/utils.js index 383b8aef..28900f90 100644 --- a/web/src/pages/DataManagement/View/utils/utils.js +++ b/web/src/pages/DataManagement/View/utils/utils.js @@ -6,12 +6,14 @@ export const dslToQueryFilters = (dsl) => { let dslFilter = {} if (dsl) { try { - const dslObject = JSON.parse(dsl) || {} - if (dslObject.bool) { - dslFilter = dslObject.bool + const parsedDsl = typeof dsl === "string" ? JSON.parse(dsl) : dsl + const dslObject = parsedDsl || {} + const queryObject = dslObject.query || dslObject + if (queryObject.bool) { + dslFilter = queryObject.bool } else { dslFilter = { - filter: [dslObject] + filter: [queryObject] } } } catch (error) { diff --git a/web/src/pages/DataManagement/components/DeleteIndexModal.jsx b/web/src/pages/DataManagement/components/DeleteIndexModal.jsx index 2252e199..49fdf15f 100644 --- a/web/src/pages/DataManagement/components/DeleteIndexModal.jsx +++ b/web/src/pages/DataManagement/components/DeleteIndexModal.jsx @@ -1,11 +1,19 @@ -import { Modal, Checkbox, Tag, Badge, Alert, Icon, Tooltip } from "antd"; +import { Modal, Checkbox, Tag, Badge, Alert, Icon } from "antd"; import { useCallback, useState, forwardRef, useMemo } from "react"; +import { FormattedMessage } from 'react-intl'; import useFetch from "@/lib/hooks/use_fetch"; import request from "@/utils/request"; import { router } from "umi"; import { formatMessage } from "umi/locale"; export default (props) => { + const selectedCluster = props.selectedCluster || {}; + const selectedClusterId = selectedCluster?.id || ""; + const selectedClusterName = + (typeof selectedCluster?.name === "string" && selectedCluster?.name) || + (typeof selectedCluster?.label === "string" && selectedCluster?.label) || + (typeof selectedCluster?.title === "string" && selectedCluster?.title) || + selectedClusterId; const [len, hasSpecialIndex] = useMemo(() => { let items = props.items; let len = items.length; @@ -23,12 +31,27 @@ export default (props) => { 1 ? `Delete ${len} indices` : "Delete index"} + title={formatMessage( + { + id: + len > 1 + ? "indices.delete.modal.title.batch" + : "indices.delete.modal.title.single", + }, + { count: len } + )} onCancel={props.onCancel} onOk={props.onOk} okButtonProps={{ disabled: !props.deleteIndexConfirm }} > -

You are about to delete these indices in cluster {props.selectedCluster.name}:

+

+ {selectedClusterName || selectedClusterId || "-"} + }} + /> +

    {props.items.map((item) => { return ( @@ -36,7 +59,8 @@ export default (props) => { {" "} {item.startsWith(".") ? ( - Special index + {" "} + {formatMessage({ id: "indices.delete.modal.special_index" })} ) : null} @@ -47,8 +71,12 @@ export default (props) => { {hasSpecialIndex ? (

    @@ -61,14 +89,18 @@ export default (props) => { } }} > - I understand the consequences of deleting a special index + {formatMessage({ + id: "indices.delete.modal.special_warning.confirm", + })}

    ) : ( -

    - You can't recover a deleted index. Make sure you have appropriate - backups. +

    + + ⚠️ + + {formatMessage({ id: "indices.delete.modal.description" })}

    )} diff --git a/web/src/pages/DataManagement/context.js b/web/src/pages/DataManagement/context.js index 2d81faef..3c1b30c1 100644 --- a/web/src/pages/DataManagement/context.js +++ b/web/src/pages/DataManagement/context.js @@ -29,8 +29,11 @@ import { import { FieldFormatsRegistry } from "../../components/vendor/data/common/field_formats"; import { baseFormattersPublic } from "../../components/vendor/data/public/field_formats"; import { deserializeFieldFormat } from "../../components/vendor/data/public/field_formats/utils/deserialize"; +import { getHighlightRequest } from "../../components/vendor/data/common/field_formats"; +import { UI_SETTINGS } from "../../components/vendor/data/common/constants"; import { ESPrefix } from "@/services/common"; import request from "@/utils/request"; +import { getTimezone } from "@/utils/utils"; const timeBucketConfig = { "histogram:maxBars": 100, @@ -99,6 +102,7 @@ fieldFormats.deserialize = deserializeFieldFormat.bind(fieldFormats); const indexPatternsApiClient = new IndexPatternsApiClient(http); const uiconfigs = { ["metaFields"]: ["_source", "_id", "_type", "_index"], //'_score' + [UI_SETTINGS.DOC_HIGHLIGHT]: true, defaultIndex: "", }; const uiSettings = { @@ -252,15 +256,29 @@ export const getContext = () => { }; }; -const getEsQuery = (indexPattern) => { - const timeFilter = timefilter.createFilter(indexPattern); - return buildEsQuery( - indexPattern, - queryStringManager.getQuery(), - [...filterManager.getFilters(), ...(timeFilter ? [timeFilter] : [])] - // getEsQueryConfig(getUiSettings()) - ); -}; + const getEsQuery = (indexPattern) => { + const timeFilter = timefilter.createFilter(indexPattern); + + const rawQuery = queryStringManager.getQuery(); + try { + return buildEsQuery( + indexPattern, + [ + { + ...rawQuery, + query: rawQuery?.query, + } + ], + [...filterManager.getFilters(), ...(timeFilter ? [timeFilter] : [])] + ); + } catch (e) { + console.warn("KQL parse failed, fallback to match_all", e); + + return { + query: { match_all: {} } + }; + } + }; const getSearchParams = ( indexPattern, @@ -271,6 +289,7 @@ const getSearchParams = ( queryFrom, trackTotalHits, size = 20, + searchAfter, ) => { // const timeExp = calculateAutoTimeExpression(timefilter.getTime()); const timeExp = getTimeBuckets(internal).getInterval(true).expression; @@ -293,14 +312,17 @@ const getSearchParams = ( timeExp.includes("y") || timeExp.includes("M"); + // Reduce buckets for large time ranges to improve performance + const bounds = timefilter.getBounds(); + const rangeMs = bounds.max - bounds.min; + const sevenDaysMs = 7 * 24 * 60 * 60 * 1000; + const bucketCount = rangeMs >= sevenDaysMs ? 80 : 120; const aggs = { counts: { - date_histogram: { - //calendar_interval: - [isCalendarInterval ? "calendar_interval" : "fixed_interval"]: timeExp, + auto_date_histogram: { field: indexPattern.timeFieldName, - min_doc_count: 1, - time_zone: "Asia/Shanghai", + buckets: bucketCount, + time_zone: getTimezone(), }, }, }; @@ -308,21 +330,28 @@ const getSearchParams = ( index: indexPattern.index || indexPattern.title, body: { query: getEsQuery(indexPattern), - from: queryFrom || 0, size: size, - - highlight: { - pre_tags: ["@highlighted-field@"], - post_tags: ["@/highlighted-field@"], - fields: {"*": {}} - }, sort: esSort, // }, }; - if (indexPattern.timeFieldName) { + const highlightRequest = getHighlightRequest( + esRequest.body.query, + uiSettings.get(UI_SETTINGS.DOC_HIGHLIGHT) + ); + if (highlightRequest) { + esRequest.body.highlight = highlightRequest; + } + // Use search_after for pagination, fall back to from for first page + if (searchAfter) { + esRequest.body["search_after"] = searchAfter; + } else { + esRequest.body["from"] = queryFrom || 0; + } + // Only include aggs on the first page + if (!searchAfter && indexPattern.timeFieldName) { esRequest.body["aggs"] = aggs; } - if (inputAggs) { + if (!searchAfter && inputAggs) { esRequest.body["aggs"] = inputAggs; } if (distinctParams?.field && distinctParams?.enabled) { @@ -330,6 +359,9 @@ const getSearchParams = ( } if (trackTotalHits) { esRequest.body["track_total_hits"] = trackTotalHits; + } else { + // Default to 10000 to avoid expensive full count on large indices + esRequest.body["track_total_hits"] = 10000; } return esRequest; diff --git a/web/src/pages/DataManagement/models/alias.js b/web/src/pages/DataManagement/models/alias.js index ef3cebdf..bb828642 100644 --- a/web/src/pages/DataManagement/models/alias.js +++ b/web/src/pages/DataManagement/models/alias.js @@ -1,6 +1,17 @@ import { getAliasList, doAlias } from "@/services/alias"; import { getIndices } from "@/services/indices"; +const normalizeAliasDetail = (item = {}) => { + const indexes = Array.isArray(item.indexes) ? item.indexes : []; + const explicitWriteIndex = indexes.find((index) => index?.is_write_index)?.index; + + return { + ...item, + indexes, + write_index: item.write_index || explicitWriteIndex || "", + }; +}; + export default { namespace: "alias", @@ -10,7 +21,7 @@ export default { const res = yield call(getAliasList, payload); let aliasList = []; for (let k in res) { - aliasList.push(res[k]); + aliasList.push(normalizeAliasDetail(res[k])); } yield put({ type: "saveData", diff --git a/web/src/pages/DataManagement/models/index.js b/web/src/pages/DataManagement/models/index.js index e5780b31..4a09c9e5 100644 --- a/web/src/pages/DataManagement/models/index.js +++ b/web/src/pages/DataManagement/models/index.js @@ -7,6 +7,7 @@ import { createIndex, } from "@/services/indices"; import { message } from "antd"; +import { formatMessage } from "umi/locale"; export default { namespace: "index", @@ -59,7 +60,7 @@ export default { return false; } if (resp.result == "updated") { - message.success("save successfully"); + message.success(formatMessage({ id: "app.message.save.success" })); } let { settings } = yield select((state) => state.index); settings[payload.index] = payload.settings; diff --git a/web/src/pages/DevTool/Console.tsx b/web/src/pages/DevTool/Console.tsx index a9d51488..eb596ac6 100644 --- a/web/src/pages/DevTool/Console.tsx +++ b/web/src/pages/DevTool/Console.tsx @@ -22,6 +22,7 @@ import { ResizeBar } from "@/components/infini/resize_bar"; import maximizeSvg from "@/assets/window-maximize.svg"; import restoreSvg from "@/assets/window-restore.svg"; import ClusterSelect from "@/components/ClusterSelect"; +import { getPreferredCluster } from "@/utils/setup"; const MaximizeIcon = (props = {}) => { return ; @@ -143,14 +144,16 @@ function calcHeightToPX(height: string) { } export const ConsoleUI = ({ - selectedCluster, - clusterList, - clusterStatus, + selectedCluster = {}, + clusterList = [], + clusterStatus = {}, minimize = false, onMinimizeClick, resizeable = false, height = "50vh", mode = "global", + reservePageSpace = mode === "global", + disableHoverScrollLock = false, }: any) => { const clusterMap = useMemo(() => { let cm = {}; @@ -158,27 +161,29 @@ export const ConsoleUI = ({ return cm; } (clusterList || []).map((cluster: any) => { - cluster.status = clusterStatus[cluster.id]?.health?.status; + const nextCluster = { ...cluster }; + nextCluster.status = clusterStatus[cluster.id]?.health?.status; if (!clusterStatus[cluster.id]?.available) { - cluster.status = "unavailable"; + nextCluster.status = "unavailable"; } - cm[cluster.id] = cluster; + cm[cluster.id] = nextCluster; }); return cm; }, [clusterList, clusterStatus]); const initialDefaultState = () => { - let defaultCluster = selectedCluster; - if (!defaultCluster.id) { - defaultCluster = clusterList[0]; - } - const defaultActiveKey = `${defaultCluster.id || - ""}:${new Date().valueOf()}`; - const defaultState = defaultCluster + const defaultCluster = getPreferredCluster(clusterList, { + selectedClusterID: selectedCluster?.id, + }); + const defaultClusterID = defaultCluster?.id || ""; + const defaultActiveKey = defaultClusterID + ? `${defaultClusterID}:${new Date().valueOf()}` + : ""; + const defaultState = defaultClusterID ? { panes: [ { key: defaultActiveKey, - cluster_id: defaultCluster.id, + cluster_id: defaultClusterID, title: defaultCluster.name, }, ], @@ -339,8 +344,8 @@ export const ConsoleUI = ({ ), }; - setClusterID(tabState.activeKey?.split(":")[0]); - const panes = tabState.panes.filter((pane: any) => { + setClusterID(tabState.activeKey?.split(":")[0] || ""); + const panes = (tabState.panes || []).filter((pane: any) => { pane.closable = true; return typeof clusterMap[pane.cluster_id] != "undefined"; }); @@ -376,10 +381,16 @@ export const ConsoleUI = ({ }, [isFullscreen]); const disableWindowScroll = () => { + if (mode !== "global") { + return; + } document.body.style.overflow = "hidden"; }; const enableWindowScroll = () => { + if (mode !== "global") { + return; + } document.body.style.overflow = ""; }; const onTabNodeMoved = (newOrder: string[]) => { @@ -391,10 +402,13 @@ export const ConsoleUI = ({ }); }; useEffect(() => { - if (mode != "global") { + var sl = document.querySelector("#root>div"); + if (!reservePageSpace) { + if (sl) { + sl.style.paddingBottom = "0px"; + } return; } - var sl = document.querySelector("#root>div"); if (sl) { if (typeof editorHeight == "number") sl.style.paddingBottom = editorHeight + "px"; @@ -402,7 +416,12 @@ export const ConsoleUI = ({ sl.style.paddingBottom = editorHeight; } } - }, [editorHeight]); + return () => { + if (sl) { + sl.style.paddingBottom = "0px"; + } + }; + }, [editorHeight, reservePageSpace]); return (
    @@ -476,5 +495,5 @@ export default connect(({ global, loading }) => ({ selectedCluster: global.selectedCluster, clusterList: global.clusterList, clusterStatus: global.clusterStatus, - height: window.innerHeight - 75 + "px", + height: window.innerHeight - 64 + "px", }))(ConsoleUI); diff --git a/web/src/pages/DevTool/Index.jsx b/web/src/pages/DevTool/Index.jsx index 1e61d3b9..7985f515 100644 --- a/web/src/pages/DevTool/Index.jsx +++ b/web/src/pages/DevTool/Index.jsx @@ -16,19 +16,25 @@ import { ConsoleUI } from "@/pages/DevTool/Console"; const Index = (props) => { return ( - - {}} - clusterStatus={props.clusterStatus} - resizeable={false} - height={props.height} - mode="page" - /> - +
    + + {}} + clusterStatus={props.clusterStatus} + resizeable={false} + height={props.height} + mode="page" + /> + +
    ); }; @@ -36,5 +42,5 @@ export default connect(({ global }) => ({ selectedCluster: global.selectedCluster, clusterList: global.clusterList, clusterStatus: global.clusterStatus, - height: window.innerHeight - 80 + "px", + height: window.innerHeight - 64 + "px", }))(Index); diff --git a/web/src/pages/Endpoints/Execute.js b/web/src/pages/Endpoints/Execute.js index 74d19dc3..c042c98e 100644 --- a/web/src/pages/Endpoints/Execute.js +++ b/web/src/pages/Endpoints/Execute.js @@ -39,7 +39,7 @@ const desc2 = ( ); const extra = ( - +
    + diff --git a/web/src/pages/Forms/StepForm/Step1.js b/web/src/pages/Forms/StepForm/Step1.js index a62e2941..e85bcea4 100644 --- a/web/src/pages/Forms/StepForm/Step1.js +++ b/web/src/pages/Forms/StepForm/Step1.js @@ -35,7 +35,7 @@ class Step1 extends React.PureComponent { }); }; return ( - +
    {getFieldDecorator('payAccount', { diff --git a/web/src/pages/Forms/StepForm/Step3.js b/web/src/pages/Forms/StepForm/Step3.js index 74e1a6b8..7ed79776 100644 --- a/web/src/pages/Forms/StepForm/Step3.js +++ b/web/src/pages/Forms/StepForm/Step3.js @@ -51,7 +51,7 @@ class Step3 extends React.PureComponent {
    ); const actions = ( - + diff --git a/web/src/pages/Forms/StepForm/index.js b/web/src/pages/Forms/StepForm/index.js index 98902a0a..b43aebe8 100644 --- a/web/src/pages/Forms/StepForm/index.js +++ b/web/src/pages/Forms/StepForm/index.js @@ -31,7 +31,7 @@ export default class StepForm extends PureComponent { content="将一个冗长或用户不熟悉的表单任务分成多个步骤，指导用户完成。" > - + diff --git a/web/src/pages/Forms/TableForm.js b/web/src/pages/Forms/TableForm.js index db18446b..a649d93d 100644 --- a/web/src/pages/Forms/TableForm.js +++ b/web/src/pages/Forms/TableForm.js @@ -237,7 +237,7 @@ class TableForm extends PureComponent { const { loading, data } = this.state; return ( - + /\.ya?ml$/i.test(name); + const Index = (props) => { const [param, setParam] = useQueryParam("_g", JsonParam); const [loading, setLoading] = useState(false); @@ -24,13 +26,34 @@ const Index = (props) => { const editorRef = useRef(null); const instanceID = props.match.params.instance_id; + const visibleConfigs = useMemo(() => { + return Object.keys(config.configs || {}).reduce((result, key) => { + if (isVisibleManagedConfig(key)) { + result[key] = config.configs[key]; + } + return result; + }, {}); + }, [config.configs]); + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "resource", locale: "menu.resource" }, + { + title: "runtime_instance", + locale: "menu.resource.runtime.instance", + href: "/resource/runtime/instance", + }, + { + title: "runtime_config", + locale: "menu.resource.runtime.config", + }, + ]; const onRefresh = () => { loadConfig(); }; const onViewClick = (key) => { - let obj = config[key] ?? config.configs[key]; + let obj = config[key] ?? visibleConfigs[key]; setCurrentConfig(obj); if (obj) { setParam({ ...param, key: key }); @@ -120,10 +143,12 @@ const Index = (props) => { useEffect(() => { //加载默认配置文件 let defaultKey = param?.key || "runtime"; - if (config[defaultKey] || config.configs[defaultKey]) { + if (config[defaultKey] || visibleConfigs[defaultKey]) { onViewClick(defaultKey); + } else if (config.runtime) { + onViewClick("runtime"); } - }, [config.runtime, config.main, config.configs]); + }, [config.runtime, config.main, visibleConfigs]); const RenderView = ({ data }) => { let splits = data?.name?.split("."); @@ -138,18 +163,30 @@ const Index = (props) => { title={data?.name} extra={ hasAuthority("gateway.instance:all") && - config.configs?.[data?.name] ? ( + visibleConfigs?.[data?.name] ? ( { onUpdateClick(data?.name); }} - okText="Yes" - cancelText="No" + okText={formatMessage({ + id: "form.button.ok", + defaultMessage: "OK", + })} + cancelText={formatMessage({ + id: "form.button.cancel", + defaultMessage: "Cancel", + })} > ) : null @@ -157,7 +194,13 @@ const Index = (props) => { > {/*
    updated:{config?.updated}
    */} {data?.location ? ( -
    Location:{data?.location}
    +
    + {formatMessage({ + id: "gateway.instance.config.location", + defaultMessage: "Location", + })} + :{data?.location} +
    ) : null}
    @@ -181,14 +224,23 @@ const Index = (props) => { }; return ( - +
    + } @@ -202,7 +254,10 @@ const Index = (props) => { onViewClick("runtime"); }} > - Runtime + {formatMessage({ + id: "gateway.instance.config.runtime", + defaultMessage: "Runtime", + })}
    { onViewClick("main"); }} > - Main + {formatMessage({ + id: "gateway.instance.config.main", + defaultMessage: "Main", + })}
    - {Object.keys(config.configs).map((item) => { + {Object.keys(visibleConfigs).map((item) => { return (
    { const [queryParams, setQueryParams] = React.useState({}); @@ -111,7 +111,7 @@ const EntryList = (props) => { onDeleteClick(record.id)} > {formatMessage({ id: "form.button.delete" })} @@ -169,7 +169,7 @@ const EntryList = (props) => { }} >
    - { const [queryParams, setQueryParams] = React.useState({}); @@ -84,7 +84,7 @@ const FlowList = (props) => { { onDeleteClick(record.id); }} @@ -146,7 +146,7 @@ const FlowList = (props) => { }} >
    - { const { match } = props; @@ -15,13 +11,42 @@ const Logging = (props = {}) => { null, [] ); + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "resource", locale: "menu.resource" }, + { + title: "runtime_instance", + locale: "menu.resource.runtime.instance", + href: "/resource/runtime/instance", + }, + { + title: "runtime_logging", + locale: "menu.resource.runtime.logging", + }, + ]; return ( - +
    - - - {(value && value.found) ? : null } + props.history.go(-1)} + style={{ marginLeft: 10 }} + > + {formatMessage({ id: "form.button.goback" })} + + } + > + + {value && value.found ? ( + + ) : null}
    @@ -30,4 +55,4 @@ const Logging = (props = {}) => { ); }; -export default Logging; \ No newline at end of file +export default Logging; diff --git a/web/src/pages/Gateway/Instance/Logging/viewer.jsx b/web/src/pages/Gateway/Instance/Logging/viewer.jsx index 27b0a9f6..2b118618 100644 --- a/web/src/pages/Gateway/Instance/Logging/viewer.jsx +++ b/web/src/pages/Gateway/Instance/Logging/viewer.jsx @@ -1,58 +1,110 @@ import React, { useState, - useCallback, useEffect, useMemo, useRef, } from "react"; import { Button, - Card, + Empty, Input, Icon, - Tabs, Select, Switch, + Tooltip, + message, } from "antd"; +import { CopyToClipboard } from "react-copy-to-clipboard"; import useWebSocket, { ReadyState } from "react-use-websocket"; import request from "@/utils/request"; +import { formatMessage } from "umi/locale"; +import "./viewer.less"; -const WebsocketLogViewer = ({instance={}}) => { - let {endpoint = ""} = instance; - if(endpoint === ""){ - console.error("empty endpoint"); - return; +const normalizeWebsocketEndpoint = (endpoint = "") => { + const value = `${endpoint || ""}`.trim(); + if (!value) { + return ""; } - const wsSchema = location.protocol.replace(":", "") === "https" ? "wss": "ws"; - const url = new URL(endpoint); - if(url.protocol === "https:"){ - endpoint = endpoint.replace("https://", "wss://") - }else{ - endpoint = endpoint.replace("http://", "ws://") + if (value.startsWith("https://")) { + return `wss://${value.slice("https://".length)}`; } - const [socketUrl, setSocketUrl] = useState(`${wsSchema}://${location.host}/ws_proxy?endpoint=${endpoint}&path=/ws`); //(`ws://${url.host}/ws`); + if (value.startsWith("http://")) { + return `ws://${value.slice("http://".length)}`; + } + return value; +}; + +const getRealtimeLogEndpoint = (instance = {}) => { + const services = Array.isArray(instance.services) ? instance.services : []; + const webService = services.find((service = {}) => { + return ( + `${service.name || ""}`.trim().toLowerCase() === "web" && + `${service.endpoint || ""}`.trim() !== "" + ); + }); + return normalizeWebsocketEndpoint(instance.endpoint || webService?.endpoint || ""); +}; + +const WebsocketLogViewer = ({ instance = {} }) => { + const endpoint = getRealtimeLogEndpoint(instance); + if (endpoint === "") { + return ( + + ); + } + let url; + try { + url = new URL(endpoint); + } catch (err) { + return ( + + ); + } + const wsSchema = + location.protocol.replace(":", "") === "https" ? "wss" : "ws"; + const socketUrl = `${wsSchema}://${location.host}/ws_proxy?instance_id=${encodeURIComponent( + instance.id + )}&endpoint=${encodeURIComponent(endpoint)}&path=${encodeURIComponent("/ws")}`; const [pubMessages, setPubMessages] = useState([]); const [loggingConfig, setLoggingConfig] = useState({}); const messageEnd = useRef(); + const logPanelRef = useRef(); const didUnmount = useRef(false); - const { sendMessage, lastMessage, readyState, getWebSocket } = useWebSocket(socketUrl, { - shouldReconnect: (closeEvent) => { + const { readyState } = useWebSocket(socketUrl, { + shouldReconnect: () => { return didUnmount.current === false; }, reconnectAttempts: 30, reconnectInterval: 10000, onMessage: (ev) => { - const rawMsg = ev.data; + const rawMsg = typeof ev.data === "string" ? ev.data : ""; + if (!rawMsg) { + return; + } const [msgType] = rawMsg.split(" ", 1); const msg = rawMsg.substr(msgType.length + 1); if (msgType !== "PUBLIC") { if (msgType == "CONFIG") { + const trimmedMsg = msg.trim(); + if (!trimmedMsg || !/^[\[{]/.test(trimmedMsg)) { + return; + } let configObj = {}; try { - configObj = JSON.parse(msg); + configObj = JSON.parse(trimmedMsg); } catch (err) { - console.error(err); + return; } setLoggingConfig(configObj); } @@ -61,15 +113,12 @@ const WebsocketLogViewer = ({instance={}}) => { setPubMessages((msgs) => { let newMsgs = msgs.concat(msg); - if(newMsgs.length > 10000) { - newMsgs = newMsgs.slice(newMsgs.length - 10000) + if (newMsgs.length > 10000) { + newMsgs = newMsgs.slice(newMsgs.length - 10000); } - return newMsgs + return newMsgs; }); }, - onOpen: (ev) => { - console.log("connected"); - }, }); useEffect(() => { return () => { @@ -77,14 +126,6 @@ const WebsocketLogViewer = ({instance={}}) => { }; }, []); - const connectionStatus = { - [ReadyState.CONNECTING]: "Connecting", - [ReadyState.OPEN]: "Established", - [ReadyState.CLOSING]: "Closing", - [ReadyState.CLOSED]: "Closed", - [ReadyState.UNINSTANTIATED]: "Uninstantiated", - }[readyState]; - const updateRealtimeConfig = async (realtime) => { const newLoggingConfig = { ...loggingConfig, @@ -115,15 +156,55 @@ const WebsocketLogViewer = ({instance={}}) => { const [autoScrollToBottom, setAutoScrollToBottom] = useState(true); useEffect(() => { if (autoScrollToBottom === true) { - messageEnd.current?.scrollIntoView(); + const panel = logPanelRef.current; + if (panel) { + panel.scrollTop = panel.scrollHeight; + } else { + messageEnd.current?.scrollIntoView({ block: "end" }); + } } }, [pubMessages.length]); + const connectionMeta = useMemo( + () => ({ + [ReadyState.CONNECTING]: { + icon: "sync", + className: "realtime-log-viewer__status realtime-log-viewer__status--connecting", + text: formatMessage({ id: "gateway.instance.logging.connection.connecting" }), + }, + [ReadyState.OPEN]: { + icon: "check-circle", + theme: "filled", + className: "realtime-log-viewer__status realtime-log-viewer__status--open", + text: formatMessage({ id: "gateway.instance.logging.connection.established" }), + }, + [ReadyState.CLOSING]: { + icon: "disconnect", + className: "realtime-log-viewer__status realtime-log-viewer__status--closing", + text: formatMessage({ id: "gateway.instance.logging.connection.closing" }), + }, + [ReadyState.CLOSED]: { + icon: "close-circle", + theme: "filled", + className: "realtime-log-viewer__status realtime-log-viewer__status--closed", + text: formatMessage({ id: "gateway.instance.logging.connection.closed" }), + }, + [ReadyState.UNINSTANTIATED]: { + icon: "pause-circle", + className: "realtime-log-viewer__status realtime-log-viewer__status--idle", + text: formatMessage({ id: "gateway.instance.logging.connection.uninstantiated" }), + }, + }), + [] + ); + const copyText = useMemo(() => pubMessages.join("\n"), [pubMessages]); + return ( -
    -
    -
    +
    +
    +
    { const value = ev.target.value; onInputChange("func_pattern", value); }} - style={{ width: 185, marginRight: 5 }} + className="realtime-log-viewer__input" key="funcPattern" - placeholder="FuncPattern, eg: submit*" + placeholder={formatMessage({ + id: "gateway.instance.logging.placeholder.func_pattern", + })} /> { onChange={(ev) => { onInputChange("message_pattern", ev.target.value); }} - style={{ width: 220, marginRight: 5 }} + className="realtime-log-viewer__input realtime-log-viewer__input--wide" key="msgPattern" - placeholder="MessagePattern, eg: *timeout" + placeholder={formatMessage({ + id: "gateway.instance.logging.placeholder.message_pattern", + })} />
    -
    Auto Scroll
    -
    - +
    +
    + + + {formatMessage({ id: "gateway.instance.logging.auto_scroll" })} + + +
    + +
    + + + {formatMessage({ id: "gateway.instance.logging.endpoint.label" })} + + + {url.host} + +
    -
    -
    - {!loggingConfig.realtime && pubMessages.length === 0 ?
    Click start button to show real-time logs
    : -
      +
      +
      +
      + + { + message.success( + formatMessage({ + id: "gateway.instance.logging.copy.success", + }) + ); + }} + > +
      + {!loggingConfig.realtime && pubMessages.length === 0 ? ( +
      + + + {formatMessage({ id: "gateway.instance.logging.empty" })} + +
      + ) : ( +
      {pubMessages.map((message, idx) => ( -
    • {message}
      • +
      + {message} +
      ))} -
    } +
    + )}
    @@ -234,9 +359,13 @@ const WebsocketLogViewer = ({instance={}}) => { export default WebsocketLogViewer; -const ConnectionStatus = ({status})=>{ - if(status === "Established"){ - return {status}; - } - return status; -} \ No newline at end of file +const ConnectionStatus = ({ readyState, connectionMeta = {} }) => { + const status = + connectionMeta[readyState] || connectionMeta[ReadyState.UNINSTANTIATED]; + return ( +
    + + {status.text} +
    + ); +}; diff --git a/web/src/pages/Gateway/Instance/Logging/viewer.less b/web/src/pages/Gateway/Instance/Logging/viewer.less new file mode 100644 index 00000000..224763a0 --- /dev/null +++ b/web/src/pages/Gateway/Instance/Logging/viewer.less @@ -0,0 +1,171 @@ +:global { + .realtime-log-viewer { + display: flex; + flex-direction: column; + gap: 12px; + + &__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + } + + &__toolbar { + display: flex; + align-items: center; + gap: 8px; + flex: 1 1 720px; + flex-wrap: wrap; + min-width: 0; + } + + &__level { + width: 110px; + } + + &__input { + width: 210px; + } + + &__input--wide { + width: 260px; + } + + &__meta { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex: 0 0 auto; + flex-wrap: wrap; + margin-left: auto; + } + + &__meta-item, + &__status { + display: inline-flex; + align-items: center; + gap: 6px; + min-height: 32px; + padding: 0 12px; + border-radius: 999px; + border: 1px solid #e8edf3; + background: #fafcff; + color: #445066; + } + + &__meta-item--endpoint { + max-width: 340px; + } + + &__meta-label { + color: #6b778c; + } + + &__meta-value { + max-width: 180px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: #1f2937; + font-weight: 500; + } + + &__status--open { + color: #237804; + background: #f6ffed; + border-color: #b7eb8f; + } + + &__status--connecting { + color: #0958d9; + background: #e6f4ff; + border-color: #91caff; + } + + &__status--closing, + &__status--closed { + color: #cf1322; + background: #fff1f0; + border-color: #ffa39e; + } + + &__status--idle { + color: #8c8c8c; + background: #fafafa; + border-color: #d9d9d9; + } + + &__body { + margin-top: 2px; + } + + &__log-panel { + position: relative; + height: clamp(360px, calc(100vh - 300px), 720px); + overflow-y: auto; + border: 1px solid #e8edf3; + border-radius: 10px; + background: #fafcff; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.6); + padding: 44px 12px 12px; + } + + &__floating-actions { + position: absolute; + top: 10px; + right: 10px; + z-index: 2; + display: inline-flex; + align-items: center; + gap: 8px; + } + + &__copy-button { + color: #5b6b82; + border-color: #d9e2ec; + background: rgba(255, 255, 255, 0.92); + box-shadow: 0 2px 8px rgba(15, 23, 42, 0.08); + + &:hover, + &:focus { + color: #1677ff; + border-color: #91caff; + background: #ffffff; + } + } + + &__log-list { + font-family: SFMono-Regular, Consolas, "Liberation Mono", Menlo, monospace; + font-size: 13px; + line-height: 1.7; + color: #1f2937; + } + + &__log-line { + white-space: pre-wrap; + word-break: break-word; + padding: 2px 0; + } + + &__empty { + display: inline-flex; + align-items: center; + gap: 8px; + color: #6b778c; + min-height: 24px; + } + } + + @media (max-width: 1200px) { + .realtime-log-viewer { + &__meta { + width: 100%; + justify-content: flex-start; + margin-left: 0; + } + } + } +} diff --git a/web/src/pages/Gateway/Instance/Step/extra_step.jsx b/web/src/pages/Gateway/Instance/Step/extra_step.jsx index 616c3030..10efaff5 100644 --- a/web/src/pages/Gateway/Instance/Step/extra_step.jsx +++ b/web/src/pages/Gateway/Instance/Step/extra_step.jsx @@ -12,6 +12,7 @@ import { formatMessage } from "umi/locale"; import useFetch from "@/lib/hooks/use_fetch"; import { useMemo } from "react"; import TagEditor from "@/components/infini/TagEditor"; +import { HealthStatusView } from "@/components/infini/health_status_view"; export const ExtraStep = Form.create({ name: "instance_step_edit" })( (props) => { @@ -33,14 +34,31 @@ export const ExtraStep = Form.create({ name: "instance_step_edit" })( return ( <> - + {initialValue?.endpoint} - + {initialValue?.version.number} - - {initialValue?.status} + + diff --git a/web/src/pages/Gateway/Instance/Step/initial_step.jsx b/web/src/pages/Gateway/Instance/Step/initial_step.jsx index 50e4ca92..7a1a0313 100644 --- a/web/src/pages/Gateway/Instance/Step/initial_step.jsx +++ b/web/src/pages/Gateway/Instance/Step/initial_step.jsx @@ -1,6 +1,11 @@ import { Form, Input, Switch, Icon } from "antd"; import { formatMessage } from "umi/locale"; -import { isTLS, removeHttpSchema } from "@/utils/utils"; +import { + isTLS, + isValidEndpointHost, + normalizeEndpointHost, + removeHttpSchema, +} from "@/utils/utils"; @Form.create() export class InitialStep extends React.Component { @@ -44,11 +49,15 @@ export class InitialStep extends React.Component { }; return ( - + {getFieldDecorator("endpoint", { initialValue: removeHttpSchema(initialValue?.endpoint || ""), normalize: (value) => { - return removeHttpSchema(value || "").trim() + return normalizeEndpointHost(value); }, validateTrigger: ["onChange", "onBlur"], rules: [ @@ -59,18 +68,31 @@ export class InitialStep extends React.Component { }), }, { - type: "string", - pattern: /^([\w.-]+(:\d+)?)([/][\w.-]*)*\/?$/, //(https?:\/\/)? - message: formatMessage({ - id: "cluster.regist.form.verify.valid.endpoint", - }), + validator: (rule, value, callback) => { + if (!value || isValidEndpointHost(value)) { + callback(); + return; + } + callback(formatMessage({ + id: "cluster.regist.form.verify.valid.endpoint", + })); + }, }, ], })( - + )} - + {getFieldDecorator("isTLS", { initialValue: isTLS(initialValue?.endpoint), })( @@ -107,10 +129,19 @@ export class InitialStep extends React.Component { rules: [ { required: true, - message: "Please input auth username!", + message: formatMessage({ + id: "cluster.regist.form.verify.required.auth_username", + }), }, ], - })()} + })( + + )} )} diff --git a/web/src/pages/Gateway/Instance/Step/result_step.jsx b/web/src/pages/Gateway/Instance/Step/result_step.jsx index 7de00051..da5a6b11 100644 --- a/web/src/pages/Gateway/Instance/Step/result_step.jsx +++ b/web/src/pages/Gateway/Instance/Step/result_step.jsx @@ -36,7 +36,10 @@ export const ResultStep = (props) => {
    - Endpoint ： + {formatMessage({ + id: "gateway.instance.field.endpoint.label", + })} + ： {removeHttpSchema(instanceConfig?.endpoint)} @@ -44,7 +47,10 @@ export const ResultStep = (props) => { - TLS： + {formatMessage({ + id: "gateway.instance.field.tls.label", + })} + ： {formatMessage({ @@ -57,7 +63,7 @@ export const ResultStep = (props) => { ); const actions = ( - + diff --git a/web/src/pages/Gateway/Instance/index.jsx b/web/src/pages/Gateway/Instance/index.jsx index 8ab963f0..23010edf 100644 --- a/web/src/pages/Gateway/Instance/index.jsx +++ b/web/src/pages/Gateway/Instance/index.jsx @@ -1,5 +1,5 @@ import PageHeaderWrapper from "@/components/PageHeaderWrapper"; -import { Button, Dropdown, Icon, Menu, message, Modal, Tooltip } from "antd"; +import { Button, Drawer, Dropdown, Icon, Menu, message, Modal, Tooltip } from "antd"; import { useCallback, useEffect, @@ -19,10 +19,72 @@ import moment from "moment"; import { formatter } from "@/lib/format"; import { hasAuthority } from "@/utils/authority"; import Wizard from "./Wizard"; +import InstallGateway from "@/components/InstallGateway"; import { isNumber } from "lodash"; import { getSystemClusterID } from "@/utils/setup"; +const metricTransitionStyle = { + display: "inline-flex", + alignItems: "center", + gap: 6, + transition: "color 0.2s ease, opacity 0.2s ease", +}; + +const metricLoadingStyle = { + ...metricTransitionStyle, + color: "rgba(0, 0, 0, 0.45)", +}; + +const metricValueStyle = { + ...metricTransitionStyle, + color: "rgba(0, 0, 0, 0.85)", +}; + +const metricUnavailableStyle = { + ...metricTransitionStyle, + color: "#cf1322", +}; + +const applicationCellStyle = { + display: "inline-flex", + alignItems: "center", + gap: 8, + minWidth: 0, +}; + +const applicationIconStyle = { + width: 16, + display: "inline-flex", + justifyContent: "center", + color: "rgba(0, 0, 0, 0.65)", +}; + +const menuItemContentStyle = { + display: "inline-flex", + alignItems: "center", + gap: 8, +}; + +const applicationMeta = { + console: { + label: "Console", + icon: "appstore", + }, + gateway: { + label: "Gateway", + icon: "api", + }, + agent: { + label: "Agent", + icon: "deployment-unit", + }, +}; + export default (props) => { + const renderTextOrDash = (value) => { + return value || value === 0 ? value : "-"; + }; + const ref = useRef(null); const [isLoading, setIsLoading] = React.useState(); @@ -44,16 +106,28 @@ export default (props) => { if (res && !res.error) { let tableData = dataSource?.data?.map((item) => { - item.info = res?.[item.id] || {}; - return item; + return { + ...item, + info: res?.[item.id] || {}, + statsFetched: true, + }; }); dataSource.data = tableData; - // update dataSource - setTimeout(() => { - if (ref.current?.setDataSource) { - ref.current.setDataSource({ ...dataSource, data: tableData }); - } - }, 500); + if (ref.current?.setDataSource) { + ref.current.setDataSource({ ...dataSource, data: tableData }); + } + return; + } + + const tableData = dataSource?.data?.map((item) => { + return { + ...item, + info: item.info || {}, + statsFetched: true, + }; + }); + if (ref.current?.setDataSource) { + ref.current.setDataSource({ ...dataSource, data: tableData }); } }; @@ -76,16 +150,21 @@ export default (props) => { const showDeleteConfirm = useCallback((record) => { Modal.confirm({ - title: "Are you sure delete this item?", + title: formatMessage({ id: "gateway.instance.delete.confirm.title" }), content: ( <> -
    Name: {record.name}
    -
    Endpoint: {record.endpoint}
    +
    + {formatMessage({ id: "gateway.instance.column.name" })}: {record.name} +
    +
    + {formatMessage({ id: "gateway.instance.column.endpoint" })}:{" "} + {record.endpoint} +
    ), - okText: "Yes", + okText: formatMessage({ id: "form.button.ok" }), okType: "danger", - cancelText: "No", + cancelText: formatMessage({ id: "form.button.cancel" }), onOk() { onDeleteClick(record.id); }, @@ -94,94 +173,200 @@ export default (props) => { const formatTableData = async (value) => { let dataNew = formatESSearchResult(value); + dataNew.data = (dataNew.data || []).map((item) => { + return { + ...item, + info: item.info || {}, + statsFetched: false, + }; + }); //异步加载&更新扩展数据 fetchInstanceStats(dataNew); return dataNew; }; + const renderPendingMetric = (icon = "loading") => { + return ( + + + -- + + ); + }; + + const renderUnavailableMetric = (icon = "warning") => { + return ( + + + -- + + ); + }; + + const renderMetricValue = (icon, value, color) => { + return ( + + + {value} + + ); + }; + + const renderApplication = (value) => { + const key = `${value || ""}`.trim().toLowerCase(); + const meta = applicationMeta[key] || { + label: key ? `${key.charAt(0).toUpperCase()}${key.slice(1)}` : "-", + icon: "appstore", + }; + return ( + + + + + {meta.label} + + ); + }; + const columns = [ { - title: "Application", + title: formatMessage({ id: "gateway.instance.column.application" }), key: "application.name", sortable: true, searchable: true, aggregable: true, + render: (text) => renderApplication(text), }, { - title: "Name", + title: formatMessage({ id: "gateway.instance.column.name" }), key: "name", sortable: true, searchable: true, + render: (text) => renderTextOrDash(text), }, { - title: "Endpoint", + title: formatMessage({ id: "gateway.instance.column.endpoint" }), key: "endpoint", sortable: true, searchable: true, + render: (text) => renderTextOrDash(text), }, - { - title: "Status", - key: "info.system", - render: (text, record) => { - return text ? ( - Online - ) : ( - N/A - ); + { + title: formatMessage({ id: "gateway.instance.column.status" }), + key: "info.system", + render: (text, record) => { + if (!record.statsFetched) { + return ( + + + + {formatMessage({ id: "gateway.instance.status.checking" })} + + + ); + } + return text ? ( + + + {formatMessage({ id: "gateway.instance.status.online" })} + + ) : ( + + + {formatMessage({ id: "gateway.instance.status.unavailable" })} + + ); + }, }, - }, - { - title: "CPU", - key: "info.system.cpu", - render: (text, record) => { - return text || isNumber(text) ? `${text}%` : null; + { + title: formatMessage({ id: "gateway.instance.column.cpu" }), + key: "info.system.cpu", + render: (text, record) => { + if (!record.statsFetched) { + return renderPendingMetric(); + } + return text || isNumber(text) + ? renderMetricValue("dashboard", `${text}%`, "#1890ff") + : renderUnavailableMetric(); + }, }, - }, - { - title: "Memory", - key: "info.system.mem", - render: (text, record) => { - if (!text) { - return null; - } - const byteFormatted = formatter.bytes(text); - return byteFormatted.size + byteFormatted.unit; + { + title: formatMessage({ id: "gateway.instance.column.memory" }), + key: "info.system.mem", + render: (text, record) => { + if (!record.statsFetched) { + return renderPendingMetric(); + } + if (!text) { + return renderUnavailableMetric(); + } + const byteFormatted = formatter.bytes(text); + return renderMetricValue( + "database", + byteFormatted.size + byteFormatted.unit, + "#722ed1" + ); + }, }, - }, { - title: "Storage", + title: formatMessage({ id: "gateway.instance.column.storage" }), key: "info.disk", render: (text, record) => { + if (!record.statsFetched) { + return renderPendingMetric(); + } if (!text) { - return null; + return renderUnavailableMetric(); } const freeByteFormatted = formatter.bytes(record.info.disk.free); const storeByteFormatted = formatter.bytes(record.info.system.store); const allByteFormatted = formatter.bytes(record.info.disk.all); return ( - - {storeByteFormatted.size + storeByteFormatted.unit} + + {renderMetricValue( + "hdd", + storeByteFormatted.size + storeByteFormatted.unit, + "#fa8c16" + )} - ) + ); }, }, { - title: "Uptime", + title: formatMessage({ id: "gateway.instance.column.uptime" }), key: "info.system.uptime_in_ms", render: (text, record) => { + if (!record.statsFetched) { + return renderPendingMetric(); + } if (!text) { - return null; + return renderUnavailableMetric("clock-circle"); } - return moment.duration(text, "ms").humanize(); + return renderMetricValue( + "clock-circle", + moment.duration(text, "ms").humanize(), + "#13c2c2" + ); }, }, { - title: "Tags", + title: formatMessage({ id: "gateway.instance.column.tags" }), key: "tags", aggregable: true, searchable: true, render: (text, record) => { - return Array.isArray(text) && text.join(","); + if (Array.isArray(text) && text.length > 0) { + return text.join(","); + } + return "-"; }, }, { @@ -198,63 +383,89 @@ export default (props) => { } }; - const menuItems = [ - { - key: "queue", - content: ( - - Queue - - ), - }, - { - key: "task", - content: ( - - Task - - ), - }, - // { - // key: "disk", - // content: ( - // - // Disk - // - // ), - // }, - ]; + const isUnavailable = record.statsFetched && !record.info?.system; + const menuItems = []; + if (!isUnavailable) { + menuItems.push( + { + key: "queue", + content: ( + + + + {formatMessage({ id: "gateway.instance.menu.queue" })} + + + ), + }, + { + key: "task", + content: ( + + + + {formatMessage({ id: "gateway.instance.menu.task" })} + + + ), + } + ); + } if (hasAuthority("gateway.instance:all")) { - menuItems.push({ - key: "logging", - content: ( - - Logging + if (!isUnavailable) { + menuItems.push({ + key: "logging", + content: ( + + + + + {formatMessage({ id: "gateway.instance.menu.logging" })} + + ), }); menuItems.push({ - key: "config", - content: ( - - Config - - ), + key: "config", + content: ( + + + + {formatMessage({ id: "gateway.instance.menu.config" })} + + + ), }); + } menuItems.push({ - key: "edit", - content: ( - - {formatMessage({ id: "form.button.edit" })} - - ), + key: "edit", + content: ( + + + + {formatMessage({ id: "form.button.edit" })} + + + ), }); menuItems.push({ - key: "delete", - content: {formatMessage({ id: "form.button.delete" })}, + key: "delete", + content: ( + + + + {formatMessage({ id: "form.button.delete" })} + + + ), }); } + if (menuItems.length === 0) { + return null; + } + const menu = ( {menuItems.map((item) => { @@ -279,6 +490,35 @@ export default (props) => { ]; const [showEmptyUI, setShowEmptyUI] = useState(false); + const [installVisible, setInstallVisible] = useState(false); + const [installGatewayType, setInstallGatewayType] = useState("migration"); + + const openInstallGateway = useCallback((type = "migration") => { + setInstallGatewayType(type); + setInstallVisible(true); + }, []); + + const installGatewayMenu = ( + { + openInstallGateway(key); + }} + > + + + + {formatMessage({ id: "gateway.install.type.migration" })} + + + + + + {formatMessage({ id: "gateway.install.type.relay" })} + + + + ); + if (showEmptyUI) { return ; } @@ -295,15 +535,26 @@ export default (props) => { }} defaultQueryParams={{ from: 0, - size: 10, + size: 20, }} sortEnable={true} sideEnable={true} sideVisible={false} sidePlacement="left" headerToobarExtra={{ - getExtra: (props) => [ - hasAuthority("gateway.instance:all") ? ( + getExtra: (props) => [ + hasAuthority("gateway.instance:all") ? ( + + + + ) : null, + hasAuthority("gateway.instance:all") ? ( diff --git a/web/src/pages/Gateway/Queue/Persistent.jsx b/web/src/pages/Gateway/Queue/Persistent.jsx index cfe54087..0f369ab3 100644 --- a/web/src/pages/Gateway/Queue/Persistent.jsx +++ b/web/src/pages/Gateway/Queue/Persistent.jsx @@ -26,7 +26,7 @@ import QueueTypeIcon from "./QueueTypeIcon"; import AutoTextEllipsis from "@/components/AutoTextEllipsis"; import commonStyles from "@/common.less"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; export default (props) => { const { @@ -101,7 +101,7 @@ export default (props) => { const columns = [ { - title: "Name", + title: formatMessage({ id: "table.field.name" }), dataIndex: "name", render: (name, row) => ( <> @@ -112,8 +112,12 @@ export default (props) => { content={consumerLabelRender(row?.metadata?.label)} title={ <> -
    {`ID: ${row?.metadata?.id}`}
    -
    {`Name: ${name}`}
    +
    + {formatMessage({ id: "table.field.id" })}: {row?.metadata?.id} +
    +
    + {formatMessage({ id: "table.field.name" })}: {name} +
    } > @@ -138,33 +142,35 @@ export default (props) => { className: commonStyles.maxColumnWidth, }, { - title: "Local Storage", + title: formatMessage({ id: "gateway.queue.field.local_storage" }), dataIndex: "storage.local_usage", sorter: (a, b) => a.storage.local_usage_in_bytes - b.storage.local_usage_in_bytes, sortDirections: ["descend", "ascend"], }, { - title: "Produce Offset", + title: formatMessage({ id: "gateway.queue.field.produce_offset" }), dataIndex: "offset", sorter: (a, b) => a.offset - b.offset, sortDirections: ["descend", "ascend"], }, { - title: "Consume Offset (earliest)", + title: formatMessage({ id: "gateway.queue.field.consume_offset_earliest" }), dataIndex: "earliest_consumer_offset", sorter: (a, b) => a.earliest_consumer_offset - b.earliest_consumer_offset, sortDirections: ["descend", "ascend"], }, { - title: "Synchronization (latest_segment)", + title: formatMessage({ + id: "gateway.queue.field.synchronization_latest_segment", + }), dataIndex: "synchronization.latest_segment", sorter: (a, b) => a.synchronization.latest_segment - b.synchronization.latest_segment, sortDirections: ["descend", "ascend"], }, { - title: "Total Messages", + title: formatMessage({ id: "gateway.queue.field.total_messages" }), dataIndex: "messages", sorter: (a, b) => a.messages - b.messages, sortDirections: ["descend", "ascend"], @@ -178,7 +184,7 @@ export default (props) => { {hasAuthority("gateway.instance:all") ? ( <> onDeleteClick([record.metadata.id])} > {formatMessage({ id: "form.button.delete" })} @@ -229,19 +235,30 @@ export default (props) => { let respSuccessCountText = ""; if (respSuccessCount > 0) { - respSuccessCountText = `Success: ${respSuccessCount}`; + respSuccessCountText = formatMessage( + { id: "gateway.queue.message.partial_success" }, + { count: respSuccessCount } + ); } message.error( - `Delete queue failed; ${ - respSuccessCount > 0 ? respSuccessCountText : "" - }` + formatMessage( + { id: "gateway.queue.delete.error" }, + { + detail: respSuccessCount > 0 ? ` ${respSuccessCountText}` : "", + } + ) ); break; } } if (respSuccessCount > 0) { - message.success(`Deleted ${respSuccessCount} queues successfully`); + message.success( + formatMessage( + { id: "gateway.queue.delete.success" }, + { count: respSuccessCount } + ) + ); //clear select state clearSelectedRows(); setTimeout(() => { @@ -276,7 +293,12 @@ export default (props) => { .flat(Infinity); if (promises.length > 0) { try { - message.success(`Deleted ${promises.length} consumers successfully`); + message.success( + formatMessage( + { id: "gateway.queue.consumer.delete.success" }, + { count: promises.length } + ) + ); await Promise.all(promises); //clear select state @@ -292,7 +314,7 @@ export default (props) => { consumerSelectedRows ); console.log("onConsumerDelete error:", e); - message.error(`Delete failed`); + message.error(formatMessage({ id: "gateway.queue.delete_failed" })); } } }; @@ -331,11 +353,11 @@ export default (props) => { - {formatMessage({ id: "form.button.delete" })} Queues + {formatMessage({ id: "gateway.queue.batch.delete_queues" })} - {formatMessage({ id: "form.button.delete" })} Consumers + {formatMessage({ id: "gateway.queue.batch.delete_consumers" })} ); @@ -360,10 +382,10 @@ export default (props) => { }} >
    - { setSearchValue(value); }} @@ -384,7 +406,7 @@ export default (props) => { {hasAuthority("gateway.instance:all") ? ( - @@ -410,7 +432,10 @@ export default (props) => { dispatch({ type: "pageSizeChange", value: size }); }, showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, + formatMessage( + { id: "gateway.router.pagination.total" }, + { start: range[0], end: range[1], total } + ), }} columns={columns} rowSelection={rowSelection} @@ -432,7 +457,7 @@ export default (props) => { ? "offset-normal" : ""; }} - scroll={{ x: "max-content" }} + scroll={dataSource.total > 0 ? { x: "max-content" } : undefined} />
    ); diff --git a/web/src/pages/Gateway/Queue/ResetOffsetModal.jsx b/web/src/pages/Gateway/Queue/ResetOffsetModal.jsx index 7e75f801..440fe6f1 100644 --- a/web/src/pages/Gateway/Queue/ResetOffsetModal.jsx +++ b/web/src/pages/Gateway/Queue/ResetOffsetModal.jsx @@ -36,7 +36,9 @@ const ReestOffsetModal = Form.create({ name: "resetoffset" })((props) => { } ); if (resetRes && resetRes.acknowledged) { - message.success("reset offset succeed"); + message.success( + formatMessage({ id: "gateway.queue.consumer.reset_offset.success" }) + ); hideModal(); handleRefresh() } @@ -63,7 +65,7 @@ const ReestOffsetModal = Form.create({ name: "resetoffset" })((props) => { return ( { >
    - ID: + {formatMessage({ id: "table.field.id" })}: {record?.id} - Group: + {formatMessage({ id: "gateway.queue.consumer.field.group" })}: {record?.group} - Name: + {formatMessage({ id: "table.field.name" })}: {record?.name} - Offset: + {formatMessage({ id: "gateway.queue.field.offset" })}: {record?.offset}
    - + {getFieldDecorator("offset", { - rules: [{ required: true, message: "offset is required!" }], + rules: [ + { + required: true, + message: formatMessage({ + id: "gateway.queue.consumer.reset_offset.offset_required", + }), + }, + ], })()} diff --git a/web/src/pages/Gateway/Queue/Transient.jsx b/web/src/pages/Gateway/Queue/Transient.jsx index 0c44bbb6..8bacea6a 100644 --- a/web/src/pages/Gateway/Queue/Transient.jsx +++ b/web/src/pages/Gateway/Queue/Transient.jsx @@ -21,7 +21,7 @@ import { hasAuthority } from "@/utils/authority"; import IconText from "@/components/infini/IconText"; import QueueTypeIcon from "./QueueTypeIcon"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; export default (props) => { const { @@ -83,12 +83,15 @@ export default (props) => { const columns = [ { - title: "Name", + title: formatMessage({ id: "table.field.name" }), dataIndex: "name", render: (name, row) => ( + { handleMessageList(row?.metadata?.id); @@ -104,14 +107,14 @@ export default (props) => { sorter: (a, b) => sorter.string(a, b, "name"), }, { - title: "Local Storage", + title: formatMessage({ id: "gateway.queue.field.local_storage" }), dataIndex: "storage.local_usage", sorter: (a, b) => a.storage.local_usage_in_bytes - b.storage.local_usage_in_bytes, sortDirections: ["descend", "ascend"], }, { - title: "Depth", + title: formatMessage({ id: "gateway.queue.field.depth" }), dataIndex: "depth", sorter: (a, b) => a.depth - b.depth, sortDirections: ["descend", "ascend"], @@ -126,7 +129,7 @@ export default (props) => { {hasAuthority("gateway.instance:all") ? ( <> onDeleteClick([record.metadata.id])} > {formatMessage({ id: "form.button.delete" })} @@ -158,19 +161,30 @@ export default (props) => { let respSuccessCountText = ""; if (respSuccessCount > 0) { - respSuccessCountText = `Success: ${respSuccessCount}`; + respSuccessCountText = formatMessage( + { id: "gateway.queue.message.partial_success" }, + { count: respSuccessCount } + ); } message.error( - `Delete queues failed; ${ - respSuccessCount > 0 ? respSuccessCountText : "" - }` + formatMessage( + { id: "gateway.queue.delete.error" }, + { + detail: respSuccessCount > 0 ? ` ${respSuccessCountText}` : "", + } + ) ); break; } } if (respSuccessCount > 0) { - message.success(`Deleted ${respSuccessCount} queues successfully`); + message.success( + formatMessage( + { id: "gateway.queue.delete.success" }, + { count: respSuccessCount } + ) + ); clearSelectedRows(); setTimeout(() => { onRefresh(); @@ -198,7 +212,7 @@ export default (props) => { - {formatMessage({ id: "form.button.delete" })} Queues + {formatMessage({ id: "gateway.queue.batch.delete_queues" })} ); @@ -223,10 +237,10 @@ export default (props) => { }} >
    - { setSearchValue(value); }} @@ -247,7 +261,7 @@ export default (props) => { {hasAuthority("gateway.instance:all") ? ( - @@ -272,7 +286,10 @@ export default (props) => { dispatch({ type: "pageSizeChange", value: size }); }, showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, + formatMessage( + { id: "gateway.router.pagination.total" }, + { start: range[0], end: range[1], total } + ), }} columns={columns} rowSelection={rowSelection} diff --git a/web/src/pages/Gateway/Queue/index.jsx b/web/src/pages/Gateway/Queue/index.jsx index e1b3a7cc..36e22437 100644 --- a/web/src/pages/Gateway/Queue/index.jsx +++ b/web/src/pages/Gateway/Queue/index.jsx @@ -123,8 +123,22 @@ const QueueList = (props) => { }); }; + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "resource", locale: "menu.resource" }, + { + title: "runtime_instance", + locale: "menu.resource.runtime.instance", + href: "/resource/runtime/instance", + }, + { + title: "runtime_queue", + locale: "menu.resource.runtime.queue", + }, + ]; + return ( - + { } > - + { QueueTypeIcon /> - + { const columns = useMemo( () => [ { - title: "Message", + title: formatMessage({ id: "gateway.queue.message.field.message" }), dataIndex: "message", width: 650, render: (text, record, index) => ( @@ -54,11 +54,11 @@ const QueueMessage = (props) => { ), }, { - title: "Offset", + title: formatMessage({ id: "gateway.queue.field.offset" }), dataIndex: "offset", }, { - title: "Size", + title: formatMessage({ id: "gateway.queue.message.field.size" }), dataIndex: "size", }, ], @@ -144,7 +144,7 @@ const QueueMessage = (props) => { }} >
    diff --git a/web/src/pages/Gateway/Router/index.jsx b/web/src/pages/Gateway/Router/index.jsx index 2e2f6fc1..2379654d 100644 --- a/web/src/pages/Gateway/Router/index.jsx +++ b/web/src/pages/Gateway/Router/index.jsx @@ -24,7 +24,7 @@ import "../list.scss"; import "@/assets/headercontent.scss"; import moment from "moment"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; const RouterList = (props) => { const [queryParams, setQueryParams] = React.useState({}); @@ -44,7 +44,7 @@ const RouterList = (props) => { method: "DELETE", }); if (deleteRes && deleteRes.result == "deleted") { - message.success("delete succeed"); + message.success(formatMessage({ id: "app.message.delete.success" })); setQueryParams((params) => { return { ...params, @@ -58,11 +58,11 @@ const RouterList = (props) => { const columns = useMemo( () => [ { - title: "Name", + title: formatMessage({ id: "gateway.router.column.name" }), dataIndex: "name", }, { - title: "Default Flow", + title: formatMessage({ id: "gateway.router.column.default_flow" }), dataIndex: "default_flow", render: (text) => { if (!value || !value.flows) { @@ -81,7 +81,7 @@ const RouterList = (props) => { // }, // }, { - title: "Tracing Flow", + title: formatMessage({ id: "gateway.router.column.tracing_flow" }), dataIndex: "tracing_flow", render: (text) => { if (!value || !value.flows) { @@ -95,7 +95,7 @@ const RouterList = (props) => { }, }, { - title: "Last Updated", + title: formatMessage({ id: "gateway.router.column.updated" }), dataIndex: "updated", render: (text) => { return moment(text).format("YYYY-MM-DD HH:mm:ss"); @@ -110,7 +110,7 @@ const RouterList = (props) => { { onDeleteClick(record.id); }} @@ -169,10 +169,12 @@ const RouterList = (props) => { }} >
    - { onSearchClick(value); }} @@ -201,7 +203,7 @@ const RouterList = (props) => { icon="plus" onClick={() => router.push(`/gateway/router/new`)} > - {formatMessage({ id: "gateway.instance.btn.new" })} + {formatMessage({ id: "gateway.router.btn.new" })}
    @@ -215,13 +217,16 @@ const RouterList = (props) => { pagination={{ size: "small", pageSize: 20, - total: total?.value || total, - showSizeChanger: true, - showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, - }} - columns={columns} - onChange={handleTableChange} + total: total?.value || total, + showSizeChanger: true, + showTotal: (total, range) => + formatMessage( + { id: "gateway.router.pagination.total" }, + { start: range[0], end: range[1], total } + ), + }} + columns={columns} + onChange={handleTableChange} /> diff --git a/web/src/pages/Gateway/Router/rules.jsx b/web/src/pages/Gateway/Router/rules.jsx index 947b0d03..986f28f8 100644 --- a/web/src/pages/Gateway/Router/rules.jsx +++ b/web/src/pages/Gateway/Router/rules.jsx @@ -123,7 +123,7 @@ export default (props) => { { dispatch({ type: "delete", diff --git a/web/src/pages/Gateway/Task/index.jsx b/web/src/pages/Gateway/Task/index.jsx index 5d376130..65e772fb 100644 --- a/web/src/pages/Gateway/Task/index.jsx +++ b/web/src/pages/Gateway/Task/index.jsx @@ -3,6 +3,7 @@ import { Tabs, Card, Table, + Empty, Popconfirm, Divider, Form, @@ -90,25 +91,36 @@ const TaskList = (props) => { } }; + const renderConfirmTitle = (id) => ( + + {formatMessage({ id })} + + ); + const columns = useMemo( () => [ { - title: "Name", + title: formatMessage({ id: "gateway.task.column.name" }), dataIndex: "name", }, { - title: "State", + title: formatMessage({ id: "gateway.task.column.state" }), dataIndex: "state", + render: (text) => + formatMessage({ + id: `gateway.task.state.${String(text || "").toLowerCase()}`, + defaultMessage: text, + }), }, { - title: "Start Time", + title: formatMessage({ id: "gateway.task.column.start_time" }), dataIndex: "start_time", render: (text) => { return moment(text).format("YYYY-MM-DD HH:mm:ss"); }, }, { - title: "End Time", + title: formatMessage({ id: "gateway.task.column.end_time" }), dataIndex: "end_time", render: (text) => { return text ? moment(text).format("YYYY-MM-DD HH:mm:ss") : text; @@ -124,20 +136,20 @@ const TaskList = (props) => {
    {record.state === taskStatus.STOPPED ? ( onStartClick(record.name)} > - Start + {formatMessage({ id: "form.button.start" })} ) : null} {/* */} {record.state !== taskStatus.STOPPED ? ( onStopClick(record.name)} > - Stop + {formatMessage({ id: "form.button.stop" })} ) : null}
    @@ -242,14 +254,14 @@ const TaskList = (props) => { onClick={onBatchStartClick} disabled={state.selectedStartKeys.length == 0} > - Start + {formatMessage({ id: "form.button.start" })} ) : null} @@ -273,19 +285,34 @@ const TaskList = (props) => { loading={false} bordered dataSource={taskList} + locale={{ + emptyText: ( + + ), + }} rowKey={(row) => row.name} rowSelection={{ selectedRowKeys: state.selectedRowKeys, onChange: onSelectChange, }} - pagination={{ - size: "small", - pageSize: 20, - total: taskList.length, - showSizeChanger: true, - showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, - }} + pagination={ + taskList.length > 0 + ? { + size: "small", + pageSize: 20, + total: taskList.length, + showSizeChanger: true, + showTotal: (total, range) => + formatMessage( + { id: "system.security.pagination.total" }, + { start: range[0], end: range[1], total } + ), + } + : false + } columns={columns} onChange={handleTableChange} /> diff --git a/web/src/pages/Guide/Initialization/components/Configuration/index.js b/web/src/pages/Guide/Initialization/components/Configuration/index.js index 75ec2a08..8cccaa56 100644 --- a/web/src/pages/Guide/Initialization/components/Configuration/index.js +++ b/web/src/pages/Guide/Initialization/components/Configuration/index.js @@ -1,124 +1,165 @@ -import { useEffect, useState } from 'react'; +import { useState } from 'react'; import { Alert, Button, Form, Icon, Input, Switch, Select } from 'antd'; import request from '@/utils/request'; import { formatMessage } from "umi/locale"; -import TrimSpaceInput from '@/components/TrimSpaceInput'; +import { + getClusterConnectErrorMessageFromError, + getClusterConnectErrorMessageFromResponse, +} from '@/pages/System/Cluster/utils'; +import { + isValidEndpointHost, + normalizeEndpointHosts, +} from '@/utils/utils'; const formItemLayout = { - labelCol: { - md: { span: 8 }, - }, - wrapperCol: { - md: { span: 10 }, - }, + labelCol: { md: { span: 8 } }, + wrapperCol: { md: { span: 10 } }, +}; + +const normalizeHosts = (hosts = []) => normalizeEndpointHosts(hosts); + +const detectTLS = (hosts = []) => { + let result = null; + for (const h of hosts) { + if (/^https:\/\//i.test(h.trim())) result = true; + else if (/^http:\/\//i.test(h.trim())) result = false; + } + return result; }; export default ({ onNext, form, formData, onFormDataChange }) => { const [testLoading, setTestLoading] = useState(false); - const [testStatus, setTestStatus] = useState(); - const [testError, setTestError] = useState(); + const [testStatus, setTestStatus] = useState(); + const [testError, setTestError] = useState(); + const [clusterDefaults, setClusterDefaults] = useState({}); const checkVersion = (version, distribution) => { if (!version) return false; - if(distribution === "easysearch" || distribution === "opensearch"){ - return true - } - const base = '5.3'; - const baseSplit = base.split('.').map(x => parseInt(x)) - const versionSplit = version.split('.').map(x => parseInt(x)) - if(baseSplit[0] > versionSplit[0]){ - return false - }else if(baseSplit[0] < versionSplit[0]){ - return true - } - if(baseSplit[1] > versionSplit[1]){ - return false - }else if(baseSplit[1] < versionSplit[1]){ - return true - } - return true - } + if (distribution === 'easysearch' || distribution === 'opensearch') return true; + const [bMajor, bMinor] = '5.3'.split('.').map(Number); + const [vMajor, vMinor] = version.split('.').map(Number); + if (vMajor !== bMajor) return vMajor > bMajor; + return vMinor >= bMinor; + }; + + const getRecommendedPrimaryShards = (res) => { + const dataNodes = Number(res?.number_of_data_nodes) || 0; + const totalNodes = Number(res?.number_of_nodes) || 0; + return Math.max(1, dataNodes || totalNodes || 1); + }; + + const resetTestStatus = () => { + if (testStatus) setTestStatus(undefined); + if (testError) setTestError(undefined); + setClusterDefaults({}); + }; + + const handleHostsChange = (values) => { + const tls = detectTLS(values); + const cleaned = normalizeHosts(values); + + if (tls !== null) form.setFieldsValue({ isTLS: tls }); + + const patch = { hosts: cleaned }; + if (tls !== null) patch.isTLS = tls; + onFormDataChange(patch); + + resetTestStatus(); + }; const onTest = async (callback) => { form.validateFields(async (err, values) => { - if (err) { - return false; - } + if (err) return; + try { setTestLoading(true); - setTestStatus(); - setTestError(); + setTestStatus(undefined); + setTestError(undefined); + const { hosts, isTLS, isAuth, username, password } = values; + const body = { - hosts: (hosts || []).map(host=>host.trim()), - schema: isTLS === true ? "https" : "http", - } - if (isAuth) { - body.basic_auth = { - username, - password, - } - } + hosts: normalizeHosts(hosts), + schema: isTLS ? 'https' : 'http', + reject_red: true, + }; + if (isAuth) body.basic_auth = { username, password }; const res = await request('/elasticsearch/try_connect', { - method: "POST", - body: body, - }, undefined, false) + method: 'POST', + body, + }, undefined, false); + if (['green', 'yellow'].includes(res?.status)) { if (checkVersion(res?.version, res?.distribution)) { - setTestStatus('success') - if (callback) callback() + const nextDefaults = { + distribution: res?.distribution, + version: res?.version, + number_of_nodes: res?.number_of_nodes, + number_of_data_nodes: res?.number_of_data_nodes, + primary_shards: getRecommendedPrimaryShards(res), + auto_expand_replicas: formData.auto_expand_replicas || '0-1', + }; + setTestStatus('success'); + setClusterDefaults(nextDefaults); + if (callback) callback(nextDefaults); } else { - setTestStatus('error') - setTestError(formatMessage({ id: 'guide.cluster.test.connection.error.version'})) + setTestStatus('error'); + setTestError(formatMessage({ id: 'guide.cluster.test.connection.error.version' })); } } else { - setTestStatus('error') - setTestError(formatMessage({ id: 'guide.cluster.test.connection.failed'})) + setTestStatus('error'); + setTestError( + getClusterConnectErrorMessageFromResponse(res, 'guide.cluster.test.connection.failed') + ); } - setTestLoading(false); } catch (error) { - console.log(error); - setTestStatus('error') + setTestStatus('error'); + setTestError( + await getClusterConnectErrorMessageFromError(error, 'guide.cluster.test.connection.failed') + ); + } finally { setTestLoading(false); } }); - } - - const resetTestStatus = () => { - !!testStatus && setTestStatus() - !!testError && setTestError() - } + }; - const onSubmit = async (e) => { + const onSubmit = (e) => { e.preventDefault(); - if (!testStatus || testStatus === 'error') { - onTest(onFormDataSave) + onTest(onFormDataSave); return; } - onFormDataSave(); - } + }; - const onFormDataSave = () => { + const onFormDataSave = (defaults = clusterDefaults) => { const values = form.getFieldsValue(); - const { hosts, isAuth, username, password } = values; + const { hosts, isTLS, isAuth, username, password } = values; + onFormDataChange({ - hosts: (hosts || []).map(host=>host.trim()), - isAuth, username, password - }) + hosts: normalizeHosts(hosts), + isTLS, + isAuth, + username, + password, + distribution: defaults?.distribution, + version: defaults?.version, + number_of_nodes: defaults?.number_of_nodes, + number_of_data_nodes: defaults?.number_of_data_nodes, + primary_shards: formData.primary_shards || defaults?.primary_shards, + auto_expand_replicas: formData.auto_expand_replicas || defaults?.auto_expand_replicas, + }); onNext(); - } + }; + const validateHostsRule = (rule, value, callback) => { - let vals = value || []; - for(let i = 0; i < vals.length; i++) { - if (!/^[\w\.\-_~%]+(\:\d+)?$/.test(vals[i])) { - return callback(formatMessage({ id: 'guide.cluster.host.validate'})); + for (const raw of (value || [])) { + if (!isValidEndpointHost(raw)) { + return callback(formatMessage({ id: 'guide.cluster.host.validate' })); } } - // validation passed callback(); }; @@ -126,90 +167,95 @@ export default ({ onNext, form, formData, onFormDataChange }) => { return (
    - - {getFieldDecorator("hosts", { + + {getFieldDecorator('hosts', { initialValue: formData.hosts, + normalize: normalizeHosts, rules: [ { required: true, - message: formatMessage({ id: 'guide.cluster.host.required'}), + message: formatMessage({ id: 'guide.cluster.host.required' }), }, - { - validator: validateHostsRule, - } + { validator: validateHostsRule }, ], - })( + )} + - {getFieldDecorator("isTLS", { + {getFieldDecorator('isTLS', { initialValue: formData.isTLS, - valuePropName: 'checked' - })( { - resetTestStatus(); - onFormDataChange({ isTLS: checked }) - }}/>)} + valuePropName: 'checked', + })( + { + resetTestStatus(); + onFormDataChange({ isTLS: checked }); + }} /> + )} - - {getFieldDecorator("isAuth", { - initialValue: formData.isAuth, - valuePropName: 'checked' - })( { - resetTestStatus(); - onFormDataChange({ isAuth: checked }) - }}/>)} + + + {getFieldDecorator('isAuth', { + initialValue: formData.isAuth !== false, + valuePropName: 'checked', + })( + { + resetTestStatus(); + onFormDataChange({ isAuth: checked }); + }} /> + )} - { - formData.isAuth && ( - <> - - {getFieldDecorator("username", { - initialValue: formData.username, - rules: [ - { - required: true, - message: formatMessage({ id: 'guide.username.required'}), - } - ], - })()} - - - {getFieldDecorator("password", { - rules: [{ - required: true, - message: formatMessage({ id: 'guide.password.required'}), - }], - })()} - - - ) - } - { - testError && ( - - + + {(form.getFieldValue('isAuth') !== undefined ? form.getFieldValue('isAuth') : formData.isAuth) && ( + <> + + {getFieldDecorator('username', { + initialValue: formData.username, + rules: [{ + required: true, + message: formatMessage({ id: 'guide.username.required' }), + }], + })()} - ) - } + + {getFieldDecorator('password', { + rules: [{ + required: true, + message: formatMessage({ id: 'guide.password.required' }), + }], + })()} + + + )} + + {testError && ( + + + + )} + -
    - -
    - ) -} \ No newline at end of file + ); +}; diff --git a/web/src/pages/Guide/Initialization/components/Finish/index.js b/web/src/pages/Guide/Initialization/components/Finish/index.js index 2584d475..b321300d 100644 --- a/web/src/pages/Guide/Initialization/components/Finish/index.js +++ b/web/src/pages/Guide/Initialization/components/Finish/index.js @@ -1,10 +1,39 @@ -import { Alert, Button, Icon, Result, Descriptions } from "antd"; +import { Alert, Button, Icon, Result, Descriptions, Spin } from "antd"; import styles from "./index.less"; -import { Link } from "umi"; +import { router } from "umi"; import { formatMessage } from "umi/locale"; import { useEffect } from "react"; +import { + invalidateApplicationSettingsCache, + refreshApplicationSettings, +} from "@/utils/authority"; +import { setSetupRequired } from "@/utils/setup"; -export default ({ formData }) => { +const setupErrorMessageMap = { + "invalid bootstrap password": + "guide.initialization.finish.error.invalid_bootstrap_password", + "bootstrap password does not meet security requirements": + "guide.initialization.finish.error.bootstrap_password_strength", + "bootstrap password is required when resetting administrator": + "guide.initialization.finish.error.bootstrap_password_required", + "bootstrap username is required when resetting administrator": + "guide.initialization.finish.error.bootstrap_username_required", +}; + +const getSetupErrorMessage = (setupError) => { + if (!setupError) { + return null; + } + + const messageId = setupErrorMessageMap[setupError]; + if (messageId) { + return formatMessage({ id: messageId }); + } + + return setupError; +}; + +export default ({ formData, onPrev }) => { const { host, @@ -17,8 +46,14 @@ export default ({ formData }) => { bootstrap_username, bootstrap_password, credential_secret, + setupStatus = "success", + setupError, } = formData; + const isInitializing = setupStatus === "running"; + const isFailed = setupStatus === "failed"; + const localizedSetupError = getSetupErrorMessage(setupError); + const onDownload = () => { let hostV = host if (!hostV && hosts.length > 0) { @@ -50,64 +85,112 @@ export default ({ formData }) => { useEffect(() => { localStorage.setItem("first-login",true); - }, []) + if (!isInitializing && !isFailed) { + setSetupRequired("false"); + invalidateApplicationSettingsCache(); + } + }, [isFailed, isInitializing]) + + const enterConsole = async () => { + setSetupRequired("false"); + invalidateApplicationSettingsCache(); + try { + await refreshApplicationSettings(true); + } catch (error) { + console.log(error); + } + router.replace("/user/login"); + }; return (
    + isInitializing ? ( + + ) : isFailed ? ( + + ) : ( + + ) } title={ - {formatMessage({ id: "guide.completed" })} + {formatMessage({ + id: isInitializing + ? "guide.initialization.finish.pending" + : isFailed + ? "guide.initialization.finish.failed" + : "guide.completed", + })} } extra={ - - + ) : isFailed ? ( + + ) : ( + - + ) } > - - {isTLS ? `https://${hosts[0]}` : `http://${hosts[0]}`} - { - isAuth && ( - <> - {username} - {password.split("").map(() => "*")} - - ) - } - { - isInit && ( - <> - {bootstrap_username} - {bootstrap_password.split("").map(() => "*")} +
    + {isInitializing ? ( + + ) : null} + {isFailed ? ( + + ) : null} + + {isTLS ? `https://${hosts[0]}` : `http://${hosts[0]}`} + { + isAuth && ( + <> + {username} + {password.split("").map(() => "*")} - ) - } - {credential_secret} - - - {formatMessage({ id: "guide.configuration.tips"})} - -
    - )} type="warning" /> + ) + } + { + isInit && ( + <> + {bootstrap_username} + {bootstrap_password.split("").map(() => "*")} + + ) + } + {credential_secret.split("").map(() => "*")} +     + + {formatMessage({ id: "guide.configuration.tips"})} + +
    + )} type="warning" /> + ); diff --git a/web/src/pages/Guide/Initialization/components/Finish/index.less b/web/src/pages/Guide/Initialization/components/Finish/index.less index 5f4da471..c50fc4b6 100644 --- a/web/src/pages/Guide/Initialization/components/Finish/index.less +++ b/web/src/pages/Guide/Initialization/components/Finish/index.less @@ -1,4 +1,37 @@ .finish { + height: 100%; + + .panels { + display: flex; + flex-direction: column; + gap: 10px; + } + + .panel { + width: 100%; + max-width: 860px; + margin: auto; + } + + .pendingAlert { + text-align: center; + + :global(.ant-alert-message) { + white-space: nowrap; + } + } + + .statusAlert { + text-align: left; + } + + .configTable { + text-align: center; + } + + .tipsAlert { + text-align: center; + } .success { font-size: 46px; @@ -18,17 +51,39 @@ :global { .ant-result { padding: 0; + height: 100%; .ant-result-icon { margin-bottom: 0 } + + .ant-result-title { + margin-top: 8px; + margin-bottom: 0; + } + + .ant-result-subtitle { + margin-top: 6px; + } + + .ant-result-content { + margin-top: 16px; + padding: 0 10%; + background: transparent; + } .ant-result-extra { - margin-top: 56px; + margin-top: 24px; .ant-btn { width: 250px; } } } + + .ant-descriptions-small .ant-descriptions-row > th, + .ant-descriptions-small .ant-descriptions-row > td { + padding-top: 10px; + padding-bottom: 10px; + } } } \ No newline at end of file diff --git a/web/src/pages/Guide/Initialization/components/Initialization/index.js b/web/src/pages/Guide/Initialization/components/Initialization/index.js index f799c2b9..8e58d214 100644 --- a/web/src/pages/Guide/Initialization/components/Initialization/index.js +++ b/web/src/pages/Guide/Initialization/components/Initialization/index.js @@ -1,41 +1,45 @@ -import { useEffect, useState } from 'react'; -import { Button, Form, Icon, Input, message, Result, Spin, Tooltip ,Divider} from 'antd'; +import { useEffect, useMemo, useState } from 'react'; +import { Alert, Button, Divider, Form, Icon, Input, InputNumber, message, Result, Spin, Switch, Tooltip } from 'antd'; import styles from './index.less' import request from '@/utils/request'; import { CopyToClipboard } from 'react-copy-to-clipboard'; -import { formatMessage } from "umi/locale"; +import { formatMessage, getLocale } from "umi/locale"; -const errorReason = { - 'elasticsearch_version_too_old': 'Cluster version is too old', - 'elasticsearch_indices_exists': 'Some related indices are already exists in the target cluster', - 'elasticsearch_template_exists': 'Some related templates are already exists in the target cluster', - 'default': 'Cluster version is too old or something are already exists in the target cluster' -} +const defaultAutoExpandReplicas = "0-1"; +const defaultEnableRollup = true; +const skippableValidateTypes = new Set([ + "elasticsearch_indices_exists", + "elasticsearch_template_exists", +]); + +const getRecommendedPrimaryShards = ({ number_of_data_nodes, number_of_nodes } = {}) => { + const dataNodes = Number(number_of_data_nodes) || 0; + const totalNodes = Number(number_of_nodes) || 0; + return Math.max(1, dataNodes || totalNodes || 1); +}; + +const isValidAutoExpandReplicas = (value) => /^(false|all|\d+-\d+)$/.test((value || '').trim()); + +const compareVersions = (currentVersion, targetVersion) => { + const current = `${currentVersion || ''}`.split('.').map((item) => parseInt(item, 10) || 0); + const target = `${targetVersion || ''}`.split('.').map((item) => parseInt(item, 10) || 0); + const size = Math.max(current.length, target.length); + + for (let i = 0; i < size; i += 1) { + const left = current[i] || 0; + const right = target[i] || 0; + if (left > right) return 1; + if (left < right) return -1; + } + return 0; +}; -const initialTasks = [{ - name: "template_ilm", - desc: "Initialize template and ilm" -},{ - name: "rollup", - desc: "Initialize rollup template" -},{ - name: "insight", - desc: "Initialize dashboard template and chart template" -},{ - name: "alerting", - desc: "Initialize bulitin alerting rule and channel" -},{ - name: "agent", - desc: "Initialize agent setup template" -},{ - name: "view", - desc: "Initialize data view template" -}]; export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { const [checkLoading, setCheckLoading] = useState(false); const [checkResult, setCheckResult] = useState({ success: undefined }); + const [hasStarted, setHasStarted] = useState(false); const handlePrev = () => { const resetValues = { @@ -91,27 +95,111 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { onCheck() }, []) + const recommendedPrimaryShards = getRecommendedPrimaryShards(formData); + const rollupSupported = formData.distribution === "easysearch" && compareVersions(formData.version, "1.12.1") >= 0; + const validationType = checkResult?.type || "default"; + const validationTitleId = `guide.cluster.validate.${validationType}`; + const showLocalhostValidation = validationType === "localhost_address"; + const initialTasks = [ + { + name: "template_ilm", + desc: formatMessage({ id: "guide.initialization.task.template_ilm" }), + }, + ...(rollupSupported && formData.enable_rollup !== false + ? [{ + name: "rollup", + desc: formatMessage({ id: "guide.initialization.task.rollup" }), + }] + : []), + { + name: "insight", + desc: formatMessage({ id: "guide.initialization.task.insight" }), + }, + { + name: "alerting", + desc: formatMessage({ id: "guide.initialization.task.alerting" }), + }, + { + name: "agent", + desc: formatMessage({ id: "guide.initialization.task.agent" }), + }, + { + name: "view", + desc: formatMessage({ id: "guide.initialization.task.view" }), + }, + ]; + + useEffect(() => { + if (!checkResult.success) { + return; + } + const nextValues = {}; + if (!formData.primary_shards) { + nextValues.primary_shards = recommendedPrimaryShards; + } + if (!formData.auto_expand_replicas) { + nextValues.auto_expand_replicas = defaultAutoExpandReplicas; + } + if (rollupSupported && typeof formData.enable_rollup === "undefined") { + nextValues.enable_rollup = defaultEnableRollup; + } + if (Object.keys(nextValues).length > 0) { + onFormDataChange(nextValues); + } + }, [checkResult.success, formData.primary_shards, formData.auto_expand_replicas, formData.enable_rollup, recommendedPrimaryShards, rollupSupported]); + const onSkipClick = () => { onFormDataChange({ ...formData, skip: true}) onNext(); } + const canSkipValidationFailure = skippableValidateTypes.has(checkResult?.type); const [taskState, setTaskState] = useState({ currentIndex: 1, status: "pending", - logs: [], + taskLogs: {}, }); + const taskLogItems = useMemo(() => { + if (!hasStarted) { + return []; + } + return initialTasks.map((task, index) => { + const taskLog = taskState.taskLogs?.[task.name] || {}; + return { + ...task, + order: index + 1, + status: taskLog.status || "pending", + reason: taskLog.reason || "", + }; + }); + }, [hasStarted, initialTasks, taskState.taskLogs]); + const updateTaskLog = (task, nextLog = {}) => { + setTaskState((st) => ({ + ...st, + taskLogs: { + ...(st.taskLogs || {}), + [task.name]: { + name: task.name, + desc: task.desc, + ...(st.taskLogs?.[task.name] || {}), + ...nextLog, + }, + }, + })); + }; const runTask = async ()=>{ if(taskState.currentIndex > initialTasks.length){ return } - setTaskState(st=>{ - st.logs.push(`start to initialize template [${initialTasks[taskState.currentIndex-1].name}]`) - return { - ...st, - status: "running", - } - }) + const currentTask = initialTasks[taskState.currentIndex - 1]; + updateTaskLog(currentTask, { + status: "running", + reason: "", + }); + setTaskState(st => ({ + ...st, + status: "running", + })); const { hosts, isTLS, @@ -128,11 +216,15 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { if (isAuth) { cluster.username = username; cluster.password = password; - } - const body = { - cluster, - initialize_template: initialTasks[taskState.currentIndex-1].name, - } + } + const body = { + cluster, + initialize_template: currentTask.name, + language: getLocale(), + primary_shards: formData.primary_shards, + auto_expand_replicas: formData.auto_expand_replicas, + enable_rollup: rollupSupported ? formData.enable_rollup !== false : false, + } const res = await request( "/setup/_initialize_template", { @@ -143,33 +235,62 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { false ); if(typeof res?.success !== "undefined"){ - setTaskState(st=>{ - st.logs.push(res.log); - if(res?.success === true){ - st.currentIndex = st.currentIndex +1; - } - return { - ...st, - status: res?.success === true ? "success" : "failed", - } + updateTaskLog(currentTask, { + status: res?.success === true ? "success" : "failed", + reason: res?.success === true ? "" : (res?.log || res?.error?.reason || ""), }); + setTaskState(st => ({ + ...st, + currentIndex: res?.success === true ? st.currentIndex + 1 : st.currentIndex, + status: res?.success === true ? "success" : "failed", + })); } } useEffect(()=>{ - if(!checkResult.success){ + if(!checkResult.success || !hasStarted){ return } runTask(); - },[taskState.currentIndex, checkResult.success]) + },[taskState.currentIndex, checkResult.success, hasStarted]) + + const startInitialization = () => { + if (!Number.isInteger(formData.primary_shards) || formData.primary_shards < 1) { + message.error(formatMessage({ id: "guide.initialization.primary_shards.invalid" })); + return; + } + const autoExpandReplicas = (formData.auto_expand_replicas || defaultAutoExpandReplicas).trim(); + if (!isValidAutoExpandReplicas(autoExpandReplicas)) { + message.error(formatMessage({ id: "guide.initialization.auto_expand_replicas.invalid" })); + return; + } + onFormDataChange({ + auto_expand_replicas: autoExpandReplicas, + enable_rollup: rollupSupported ? formData.enable_rollup !== false : false, + }); + setTaskState({ + currentIndex: 1, + status: "pending", + taskLogs: {}, + }); + setHasStarted(true); + } const retryTask = ()=>{ runTask(); } const skipTask = ()=>{ + const currentTask = initialTasks[taskState.currentIndex - 1]; + if (currentTask) { + updateTaskLog(currentTask, { + status: "skipped", + reason: "", + }); + } setTaskState(st=>{ return { ...st, - currentIndex: st.currentIndex + 1 + currentIndex: st.currentIndex + 1, + status: "pending", } }) } @@ -181,35 +302,130 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => {
    { checkResult.success ? ( -
    -
    -
    +
    + {!hasStarted ? ( + <> + +
    +
    + {formatMessage({ id: "guide.initialization.primary_shards" })} +
    + onFormDataChange({ primary_shards: value })} + /> +
    + {formatMessage({ id: "guide.initialization.primary_shards.help" })} +
    +
    +
    +
    + {formatMessage({ id: "guide.initialization.auto_expand_replicas" })} +
    + onFormDataChange({ auto_expand_replicas: event.target.value })} + /> +
    + {formatMessage({ id: "guide.initialization.auto_expand_replicas.help" })} +
    +
    + {rollupSupported ? ( +
    + onFormDataChange({ enable_rollup: checked })} + /> +
    +
    + {formatMessage({ id: "guide.initialization.rollup" })} +
    +
    + {formatMessage({ id: "guide.initialization.rollup.help" })} +
    +
    +
    + ) : null} +
    + + +
    + + ) : ( + <> +
    +
    [{currentIndex+1}/{initialTasks.length}] {taskState.status === "running" ? : null} {taskState.status === "failed" ? : null} {initialTasks[currentIndex].desc}
    {taskState.status === "failed" ? -
    - RetrySkip -
    : null} -
    -
    - {taskState.logs.map(item=>
    {item}
    )} -
    -
    +
    + RetrySkip +
    : null} +
    +
    + {taskLogItems.map((item) => ( +
    +
    + + [{item.order}/{initialTasks.length}] {item.desc} + +
    + + {item.reason ? ( + + + + ) : null} +
    +
    +
    + ))} +
    +
    -
    + + )}
    ) : ( { title={( - { formatMessage({ id: `guide.cluster.validate.${checkResult?.type || 'default'}`}) } + { formatMessage({ id: validationTitleId }) } - { formatMessage({ id: 'guide.step.refresh' }) } + { formatMessage({ id: 'guide.cluster.validate.refresh' }) } )} subTitle={( -
    -
    { formatMessage({ id: 'guide.cluster.validate.sub' }) }
    -
    { formatMessage({ id: 'guide.cluster.validate.sub.strong' }) }
    -
    + showLocalhostValidation ? ( +
    +
    { formatMessage({ id: 'guide.cluster.validate.localhost.sub' }) }
    +
    + ) : ( +
    +
    { formatMessage({ id: 'guide.cluster.validate.sub' }) }
    +
    { formatMessage({ id: 'guide.cluster.validate.sub.strong' }) }
    +
    + ) )} extra={[ , - , + canSkipValidationFailure ? ( + + ) : null, ]} > { @@ -252,9 +476,11 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => {
    ) } -
    - {formatMessage({ id: 'guide.cluster.skip.desc' })} -
    + {canSkipValidationFailure ? ( +
    + {formatMessage({ id: 'guide.cluster.skip.desc' })} +
    + ) : null} ) } @@ -263,4 +489,43 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { ) -} \ No newline at end of file +} + +const StatusTag = ({ status = "pending" }) => { + const statusMetaMap = { + pending: { + icon: "clock-circle", + className: styles.taskStatusTagPending, + }, + running: { + icon: "loading", + className: styles.taskStatusTagRunning, + }, + success: { + icon: "check-circle", + theme: "filled", + className: styles.taskStatusTagSuccess, + }, + failed: { + icon: "close-circle", + theme: "filled", + className: styles.taskStatusTagFailed, + }, + skipped: { + icon: "minus-circle", + theme: "filled", + className: styles.taskStatusTagSkipped, + }, + }; + const statusMeta = statusMetaMap[status] || statusMetaMap.pending; + + return ( + + + {formatMessage({ + id: `guide.initialization.task.status.${status}`, + defaultMessage: status, + })} + + ); +}; diff --git a/web/src/pages/Guide/Initialization/components/Initialization/index.less b/web/src/pages/Guide/Initialization/components/Initialization/index.less index 24083793..1615de9b 100644 --- a/web/src/pages/Guide/Initialization/components/Initialization/index.less +++ b/web/src/pages/Guide/Initialization/components/Initialization/index.less @@ -1,5 +1,51 @@ .initialization { - + height: 100%; + + .contentInner { + width: 100%; + max-width: 860px; + margin: 0 auto; + } + + .topAlert, + .fieldBlock, + .toggleBlock { + margin-bottom: 16px; + } + + .fieldTitle { + margin-bottom: 8px; + font-weight: 500; + } + + .fieldHelp { + margin-top: 8px; + color: rgba(0, 0, 0, 0.45); + } + + .toggleBlock { + display: flex; + align-items: center; + gap: 8px; + } + + .toggleHelp { + margin-top: 4px; + color: rgba(0, 0, 0, 0.45); + } + + .actionRow { + display: flex; + justify-content: space-between; + width: 100%; + max-width: 520px; + margin: 16px auto 0; + } + + .actionButton { + width: calc(50% - 6px); + } + .warning { font-size: 46px; color: rgb(255, 215, 0); @@ -30,6 +76,112 @@ } } + .taskLogPanel { + height: auto; + margin-top: 8px; + padding: 8px 10px; + overflow: visible; + border-radius: 6px; + border: 1px solid #e8edf3; + background: #fafbfc; + } + + .taskLogItem { + color: #445066; + line-height: 1.8; + + & + .taskLogItem { + margin-top: 6px; + } + } + + .taskLogMain { + display: flex; + align-items: center; + gap: 12px; + justify-content: space-between; + min-width: 0; + padding: 7px 10px 7px 12px; + border-radius: 6px; + background: #fff; + border: 1px solid #eef2f7; + } + + .taskHeader { + display: flex; + align-items: center; + gap: 12px; + } + + .taskHeaderMain { + min-width: 0; + } + + .taskHeaderActions { + margin-left: auto; + white-space: nowrap; + } + + .taskLogName { + min-width: 0; + flex: 1 1 auto; + word-break: break-word; + } + + .taskLogStatus { + display: flex; + align-items: center; + gap: 6px; + flex-shrink: 0; + white-space: nowrap; + } + + .taskLogReasonIcon { + color: rgba(0, 0, 0, 0.45); + } + + .taskStatusTag { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 2px 10px; + border-radius: 999px; + font-size: 12px; + font-weight: 600; + line-height: 20px; + border: 1px solid transparent; + } + + .taskStatusTagPending { + color: #5b667a; + background: #f3f5f8; + border-color: #e2e8f0; + } + + .taskStatusTagRunning { + color: #0958d9; + background: #e6f4ff; + border-color: #91caff; + } + + .taskStatusTagSuccess { + color: #237804; + background: #f6ffed; + border-color: #b7eb8f; + } + + .taskStatusTagFailed { + color: #cf1322; + background: #fff1f0; + border-color: #ffa39e; + } + + .taskStatusTagSkipped { + color: #ad6800; + background: #fff7e6; + border-color: #ffd591; + } + .skipDesc { color: rgba(0, 0, 0, 0.45); font-size: 14px; @@ -48,6 +200,17 @@ } } + .credentialSecretTip { + display: inline-block; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: #ff0000; + line-height: 20px; + opacity: 0.7; + } + .refresh { cursor: pointer; } @@ -79,4 +242,4 @@ :global(.ant-list) { color: #ffffff; } -} \ No newline at end of file +} diff --git a/web/src/pages/Guide/Initialization/components/Settings/index.jsx b/web/src/pages/Guide/Initialization/components/Settings/index.jsx index 1f10c661..eafcce52 100644 --- a/web/src/pages/Guide/Initialization/components/Settings/index.jsx +++ b/web/src/pages/Guide/Initialization/components/Settings/index.jsx @@ -12,7 +12,7 @@ import { Row, Col, Modal, - List, + List, } from "antd"; import styles from "../Initialization/index.less"; import request from "@/utils/request"; @@ -32,18 +32,21 @@ const formItemLayout = { export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { const [confirmDirty, setConfirmDirty] = useState(false); - const [resetUser, setResetUser] = useState(false) + const [resetUser, setResetUser] = useState(Boolean(formData.reset_user)) const [loading, setLoading] = useState(false); const [passwordHelp, setPasswordHelp] = useState(null); const handlePrev = () => { const resetValues = { + bootstrap_username: undefined, bootstrap_password: undefined, bootstrap_password_confirm: undefined, credential_secret: undefined, + reset_user: false, skip: false, }; onFormDataChange(resetValues); + setResetUser(false); form.setFieldsValue(resetValues, () => { onPrev(); }); @@ -76,15 +79,17 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { const newValues = { ...formData, ...values, + setupStatus: "running", + setupError: undefined, }; onFormDataChange(newValues); + onNext(); onInitialize(newValues); }); } - + const onInitialize = async (formData) => { try { - setLoading(true); const { hosts, isTLS, @@ -109,8 +114,11 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { } body.cluster = cluster; body.skip = skip; - body.bootstrap_username = bootstrap_username; - body.bootstrap_password = bootstrap_password; + body.reset_user = !skip || resetUser; + if (!skip || resetUser) { + body.bootstrap_username = bootstrap_username; + body.bootstrap_password = bootstrap_password; + } body.credential_secret = credential_secret; const res = await request( "/setup/_initialize", @@ -125,14 +133,25 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { if(res.secret_mismatch === true){ localStorage.setItem("secret_mismatch", "1"); } - onNext(); + onFormDataChange({ + ...formData, + setupStatus: "success", + setupError: undefined, + }); } else { - message.error(res?.error?.reason); + onFormDataChange({ + ...formData, + setupStatus: "failed", + setupError: res?.error?.reason, + }); } - setLoading(false); } catch (error) { console.log(error); - setLoading(false); + onFormDataChange({ + ...formData, + setupStatus: "failed", + setupError: error?.message, + }); } }; @@ -188,8 +207,23 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { generateKey(); }, []); + useEffect(() => { + setResetUser(Boolean(formData.reset_user)); + }, [formData.reset_user]); + const onResetUserChange = (checked)=>{ setResetUser(checked); + onFormDataChange({ + reset_user: checked, + ...(checked ? {} : { + bootstrap_username: undefined, + bootstrap_password: undefined, + bootstrap_password_confirm: undefined, + }), + }); + if (!checked) { + form.resetFields(["bootstrap_username", "bootstrap_password", "bootstrap_password_confirm"]); + } } const [verified, setVerified] = useState(undefined); @@ -341,16 +375,11 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { )} -
    - {formatMessage({ id: "guide.credential_secret.tips" })} -
    + +
    + {formatMessage({ id: "guide.credential_secret.tips" })} +
    +
    @@ -412,14 +441,7 @@ export default ({ onPrev, onNext, form, formData, onFormDataChange }) => { -
    +
    {formatMessage({ id: "guide.credential_secret.tips" })}
    diff --git a/web/src/pages/Guide/Initialization/index.js b/web/src/pages/Guide/Initialization/index.js index 2ede48a4..e7eaa576 100644 --- a/web/src/pages/Guide/Initialization/index.js +++ b/web/src/pages/Guide/Initialization/index.js @@ -36,7 +36,9 @@ export default Form.create()(({ form }) => { const [current, setCurrent] = useState(0); - const [formData, setFormData] = useState({}); + const [formData, setFormData] = useState({ + isAuth: true, + }); const step = steps[current]; diff --git a/web/src/pages/Guide/Initialization/index.less b/web/src/pages/Guide/Initialization/index.less index 5564c8ac..ce567d53 100644 --- a/web/src/pages/Guide/Initialization/index.less +++ b/web/src/pages/Guide/Initialization/index.less @@ -1,17 +1,34 @@ .container { width: 100%; height: 100%; - overflow: auto; - padding: 5% 20%; + min-height: 0; + overflow: hidden; + box-sizing: border-box; + display: flex; + padding: 16px 28px 8px; .box { + width: 100%; + max-width: 1120px; + height: 100%; + max-height: none; + margin: 0 auto; background-color: #fff; - padding: 96px 64px; + padding: 32px 44px 20px; + display: flex; + flex-direction: column; + min-height: 0; + overflow: hidden; .stepsContent { + display: flex; + flex-direction: column; + flex: 1 1 auto; + min-height: 0; + overflow: hidden; .desc { - margin: 48px 48px 24px; + margin: 20px 24px 8px; font-weight: 400; font-size: 14px; color: rgb(16, 16, 16); @@ -20,7 +37,18 @@ .content { border-radius: 12px; - min-height: 200px; + flex: 1 1 auto; + min-height: 0; + display: flex; + align-items: stretch; + justify-content: center; + overflow: auto; + + > * { + width: 100%; + max-width: 920px; + margin: 0 auto; + } } } diff --git a/web/src/pages/List/Articles.js b/web/src/pages/List/Articles.js index 5df46d5f..bc23b446 100644 --- a/web/src/pages/List/Articles.js +++ b/web/src/pages/List/Articles.js @@ -120,7 +120,7 @@ class SearchList extends Component { ) : null; return ( - +
    diff --git a/web/src/pages/List/TableList.js b/web/src/pages/List/TableList.js index 1853dab6..ddcb5e63 100644 --- a/web/src/pages/List/TableList.js +++ b/web/src/pages/List/TableList.js @@ -333,7 +333,7 @@ class TableList extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>配置 订阅警报 diff --git a/web/src/pages/Overview/components/Activities/GenerateDesc.jsx b/web/src/pages/Overview/components/Activities/GenerateDesc.jsx index 8d39a40c..59e5ae95 100644 --- a/web/src/pages/Overview/components/Activities/GenerateDesc.jsx +++ b/web/src/pages/Overview/components/Activities/GenerateDesc.jsx @@ -58,7 +58,7 @@ export default (props) => { > {indexName} {" "} - was created in cluster{" "} + was recorded in cluster{" "} diff --git a/web/src/pages/Overview/components/Activities/index.less b/web/src/pages/Overview/components/Activities/index.less index aad5c44a..3f3aff11 100644 --- a/web/src/pages/Overview/components/Activities/index.less +++ b/web/src/pages/Overview/components/Activities/index.less @@ -1,17 +1,27 @@ .activities { width: 100%; height: 100%; - max-height: 536px; + min-height: 0; + display: flex; + flex-direction: column; :global { + .ant-card { + height: 100%; + } + .ant-card-body { width: 100%; height: 100%; + overflow: hidden; } .ant-timeline { - height: calc(100% - 12px); - overflow: hidden; + height: 100%; + overflow-y: auto; + overflow-x: hidden; + padding-right: 8px; + margin-right: -8px; } .ant-timeline-item { margin-bottom: -20px; @@ -39,6 +49,8 @@ .content { width: 100%; - height: calc(100% - 29px); + flex: 1; + min-height: 0; + overflow: hidden; } } diff --git a/web/src/pages/Overview/components/Disk/index.jsx b/web/src/pages/Overview/components/Disk/index.jsx index a64ca947..84c8cf48 100644 --- a/web/src/pages/Overview/components/Disk/index.jsx +++ b/web/src/pages/Overview/components/Disk/index.jsx @@ -1,28 +1,9 @@ import styles from "./index.less" -import request from "@/utils/request"; -import { pathPrefix } from "@/services/common"; -import React , { useEffect, useState } from "react"; +import React from "react"; import { Spin } from "antd" import { formatter } from "@/lib/format"; -export default () => { - - const [data, setData] = useState(0); - const [loading, setLoading] = useState(false); - - const fetchData = async () => { - setLoading(true) - const res = await request(`${pathPrefix}/elasticsearch/overview`, { - method: "GET", - }); - setData(res?.total_used_store_in_bytes || 0); - setLoading(false) - }; - - useEffect(() => { - fetchData() - }, []) - +export default ({ data = 0, loading = false }) => { const totalStoreSize = formatter.bytes(data); return ( diff --git a/web/src/pages/Overview/components/Message/index.jsx b/web/src/pages/Overview/components/Message/index.jsx index d1c825b3..f104c998 100644 --- a/web/src/pages/Overview/components/Message/index.jsx +++ b/web/src/pages/Overview/components/Message/index.jsx @@ -9,7 +9,7 @@ import { Link } from "umi"; const icon = () => ( ( export default (props) => { const { currentUser } = props; const [stats, setStats] = useState({}); + const [noticeCount, setNoticeCount] = useState(0); const [loading, setLoading] = useState(false); const fetchData = async () => { setLoading(true); - const res = await request(`/alerting/message/_stats`, { - method: "GET", - }); - if (res?.alert?.current && !res?.error) { - setStats(res.alert.current); + const [alertRes, noticeRes] = await Promise.all([ + request(`/alerting/message/_stats`, { + method: "GET", + }), + request(`/notification/_search`, { + method: "POST", + body: { + from: 0, + size: 0, + status: ["new"], + }, + }), + ]); + if (alertRes?.alert?.current && !alertRes?.error) { + setStats(alertRes.alert.current); + } + if (!noticeRes?.error) { + setNoticeCount(noticeRes?.hits?.total?.value || 0); } setLoading(false); }; @@ -76,7 +90,7 @@ export default (props) => { className={styles.num} style={{ color: "rgb(2, 127, 254)" }} > - {currentUser.notifyCount || 0} + {noticeCount || currentUser.notifyCount || 0}
    {formatMessage({ id: "overview.message.notice" })} diff --git a/web/src/pages/Overview/components/PieChart/index.jsx b/web/src/pages/Overview/components/PieChart/index.jsx index 35965ecd..7041c34f 100644 --- a/web/src/pages/Overview/components/PieChart/index.jsx +++ b/web/src/pages/Overview/components/PieChart/index.jsx @@ -24,23 +24,10 @@ export default (props) => { innerRadius: 0.6, legend: false, label: false, - interactions: [ - { - type: "element-active", - }, - ], statistic: { title: false, }, data: filterData, - state: { - active: { - style: { - lineWidth: 0, - cursor: "pointer", - }, - }, - }, }; return ; diff --git a/web/src/pages/Overview/components/Product/index.less b/web/src/pages/Overview/components/Product/index.less index e7123d28..928e01eb 100644 --- a/web/src/pages/Overview/components/Product/index.less +++ b/web/src/pages/Overview/components/Product/index.less @@ -1,6 +1,6 @@ .product { - height: calc(100% - 325px); - max-height: 211px; + display: flex; + flex-direction: column; .title { font-weight: 700; @@ -10,13 +10,10 @@ } .content { - height: calc(100% - 29px); - overflow-y: auto; - :global { .ant-card-body { width: 100%; - height: 100%; + height: auto; } } @@ -41,4 +38,4 @@ } -} \ No newline at end of file +} diff --git a/web/src/pages/Overview/components/Quick/icons/ClusterIcon.jsx b/web/src/pages/Overview/components/Quick/icons/ClusterIcon.jsx index e8baf1a4..0a410ffb 100644 --- a/web/src/pages/Overview/components/Quick/icons/ClusterIcon.jsx +++ b/web/src/pages/Overview/components/Quick/icons/ClusterIcon.jsx @@ -3,24 +3,24 @@ export default () => { - - + + - - + + - - + + - + - - + + diff --git a/web/src/pages/Overview/components/Quick/icons/ComparisonIcon.jsx b/web/src/pages/Overview/components/Quick/icons/ComparisonIcon.jsx new file mode 100644 index 00000000..5a9dd140 --- /dev/null +++ b/web/src/pages/Overview/components/Quick/icons/ComparisonIcon.jsx @@ -0,0 +1,31 @@ +export default () => { + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +}; diff --git a/web/src/pages/Overview/components/Quick/icons/DevToolIcon.jsx b/web/src/pages/Overview/components/Quick/icons/DevToolIcon.jsx index 7876b219..38eb86e8 100644 --- a/web/src/pages/Overview/components/Quick/icons/DevToolIcon.jsx +++ b/web/src/pages/Overview/components/Quick/icons/DevToolIcon.jsx @@ -4,38 +4,38 @@ export default () => { 编组 3 - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - + - + @@ -44,7 +44,7 @@ export default () => { - + diff --git a/web/src/pages/Overview/components/Quick/icons/DiscoverIcon.jsx b/web/src/pages/Overview/components/Quick/icons/DiscoverIcon.jsx index 89cbca2f..4ec7a347 100644 --- a/web/src/pages/Overview/components/Quick/icons/DiscoverIcon.jsx +++ b/web/src/pages/Overview/components/Quick/icons/DiscoverIcon.jsx @@ -4,45 +4,45 @@ export default () => { 编组 5 - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + diff --git a/web/src/pages/Overview/components/Quick/icons/MessageIcon.jsx b/web/src/pages/Overview/components/Quick/icons/MessageIcon.jsx index 5457fd38..9d779aaa 100644 --- a/web/src/pages/Overview/components/Quick/icons/MessageIcon.jsx +++ b/web/src/pages/Overview/components/Quick/icons/MessageIcon.jsx @@ -3,50 +3,50 @@ export default () => { - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + diff --git a/web/src/pages/Overview/components/Quick/icons/MigrationIcon.jsx b/web/src/pages/Overview/components/Quick/icons/MigrationIcon.jsx index 90f0b80c..023f8e0f 100644 --- a/web/src/pages/Overview/components/Quick/icons/MigrationIcon.jsx +++ b/web/src/pages/Overview/components/Quick/icons/MigrationIcon.jsx @@ -4,27 +4,27 @@ export default () => { 编组 2 - - + + - - + + - - + + - + - + - - + + diff --git a/web/src/pages/Overview/components/Quick/icons/MonitorIcon.jsx b/web/src/pages/Overview/components/Quick/icons/MonitorIcon.jsx index fe242c00..f540a68b 100644 --- a/web/src/pages/Overview/components/Quick/icons/MonitorIcon.jsx +++ b/web/src/pages/Overview/components/Quick/icons/MonitorIcon.jsx @@ -2,15 +2,15 @@ export default () => { return ( monitor - + - - - - - - - + + + + + + + ) diff --git a/web/src/pages/Overview/components/Quick/icons/SecurityIcon.jsx b/web/src/pages/Overview/components/Quick/icons/SecurityIcon.jsx index 353c81db..d70be45b 100644 --- a/web/src/pages/Overview/components/Quick/icons/SecurityIcon.jsx +++ b/web/src/pages/Overview/components/Quick/icons/SecurityIcon.jsx @@ -4,25 +4,25 @@ export default () => { 编组 6 - - + + - - + + - - + + - + - - + + diff --git a/web/src/pages/Overview/components/Quick/index.jsx b/web/src/pages/Overview/components/Quick/index.jsx index 8fc74595..13f4de53 100644 --- a/web/src/pages/Overview/components/Quick/index.jsx +++ b/web/src/pages/Overview/components/Quick/index.jsx @@ -1,4 +1,5 @@ import { Card, Col, Icon, Row } from "antd"; +import React, { useEffect, useState } from "react"; import styles from "./index.less"; import { router } from "umi"; import MessageIcon from "./icons/MessageIcon"; @@ -7,51 +8,122 @@ import ClusterIcon from "./icons/ClusterIcon"; import SecurityIcon from "./icons/SecurityIcon"; import DiscoverIcon from "./icons/DiscoverIcon"; import MonitorIcon from "./icons/MonitorIcon"; +import MigrationIcon from "./icons/MigrationIcon"; +import ComparisonIcon from "./icons/ComparisonIcon"; import { formatMessage } from "umi/locale"; +import { + APPLICATION_SETTINGS_UPDATED_EVENT, + getEnterpriseTaskManagerEnabled, + hasAuthority, +} from "@/utils/authority"; -const ROUTES = [ - { - path: "/resource/cluster", - name: "cluster_regist", - icon: ClusterIcon, - }, - { - path: "/alerting/message", - name: "alert", - icon: MessageIcon, - }, - { - path: "/insight/discover", - name: "discover", - icon: DiscoverIcon, - }, - { - path: "/cluster/monitor", - name: "monitor", - icon: MonitorIcon, - }, - { - path: "/system/security", - name: "security", - icon: SecurityIcon, - }, - { - path: "/devtool/console", - name: "dev_tools", - icon: DevToolIcon, - }, -]; +const getRoutes = (taskManagerEnabled) => { + const routes = [ + { + path: "/resource/cluster", + name: "cluster_regist", + icon: ClusterIcon, + }, + { + path: "/alerting/message", + name: "alert", + icon: MessageIcon, + }, + { + path: "/insight/discover", + name: "discover", + icon: DiscoverIcon, + }, + { + path: "/cluster/monitor", + name: "monitor", + icon: MonitorIcon, + }, + { + path: "/system/security", + name: "security", + icon: SecurityIcon, + }, + { + path: "/devtool/console", + name: "dev_tools", + icon: DevToolIcon, + }, + ]; + + if ( + taskManagerEnabled && + (hasAuthority("data_tools.comparison:all") || + hasAuthority("data_tools.comparison:read")) + ) { + routes.unshift({ + path: "/data_tools/comparison", + name: "comparison", + icon: ComparisonIcon, + }); + } + + if ( + taskManagerEnabled && + (hasAuthority("data_tools.migration:all") || + hasAuthority("data_tools.migration:read")) + ) { + routes.unshift({ + path: "/data_tools/migration", + name: "migration", + icon: MigrationIcon, + }); + } + + return routes; +}; export default () => { + const [taskManagerEnabled, setTaskManagerEnabled] = useState( + getEnterpriseTaskManagerEnabled() === "true" + ); + + useEffect(() => { + const syncTaskManagerEnabled = () => { + setTaskManagerEnabled(getEnterpriseTaskManagerEnabled() === "true"); + }; + + syncTaskManagerEnabled(); + window.addEventListener( + APPLICATION_SETTINGS_UPDATED_EVENT, + syncTaskManagerEnabled + ); + return () => { + window.removeEventListener( + APPLICATION_SETTINGS_UPDATED_EVENT, + syncTaskManagerEnabled + ); + }; + }, []); + + const routes = getRoutes(taskManagerEnabled); + const compactLayout = routes.length > 6; + return (
    {formatMessage({ id: "overview.title.quick" })}
    - {ROUTES.map((item, index) => ( -
    - router.push(item.path)} size="small" bodyStyle={{display:"flex"}}> + {routes.map((item, index) => ( + + router.push(item.path)} + size="small" + bodyStyle={{ display: "flex", alignItems: "center", height: "100%" }} + >
    {item.icon && ( diff --git a/web/src/pages/Overview/components/Quick/index.less b/web/src/pages/Overview/components/Quick/index.less index 01e08f63..6c0fb6b8 100644 --- a/web/src/pages/Overview/components/Quick/index.less +++ b/web/src/pages/Overview/components/Quick/index.less @@ -23,11 +23,14 @@ font-size: 14px; color: rgb(16, 16, 16); line-height: 20px; + word-break: break-word; + overflow-wrap: anywhere; } .icon { margin-right: 10px; font-size: 40px; + flex-shrink: 0; } } -} \ No newline at end of file +} diff --git a/web/src/pages/Overview/components/Status/index.jsx b/web/src/pages/Overview/components/Status/index.jsx index 849c2d53..dd5d720e 100644 --- a/web/src/pages/Overview/components/Status/index.jsx +++ b/web/src/pages/Overview/components/Status/index.jsx @@ -4,14 +4,16 @@ import { Link } from "umi"; export default (props) => { const { title, icon, data = [], loading, linkTo } = props; + const Wrapper = linkTo ? Link : "div"; + const wrapperProps = linkTo ? { to: linkTo } : {}; return ( <> {loading ? ( ) : ( - { {/*
    */} {/*
    */} - + )} ); diff --git a/web/src/pages/Overview/index.jsx b/web/src/pages/Overview/index.jsx index ecfce9d9..3106933f 100644 --- a/web/src/pages/Overview/index.jsx +++ b/web/src/pages/Overview/index.jsx @@ -8,7 +8,7 @@ import Disk from "./components/Disk"; import Quick from "./components/Quick"; import Product from "./components/Product"; import Activities from "./components/Activities"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import request from "@/utils/request"; import { GREEN, GREY, RED, YELLOW } from "./components/PieChart"; import IconTitle from "./components/IconTitle"; @@ -16,6 +16,7 @@ import ClustersSvg from "@/components/Icons/Clusters"; import NodesSvg from "@/components/Icons/Nodes"; import HostsSvg from "@/components/Icons/Hosts"; import DiskSvg from "@/components/Icons/DB"; +import useResizeObserver from "@react-hook/resize-observer"; const STATUS = [ { @@ -39,6 +40,7 @@ const STATUS = [ const { node } = data || {}; return [ { group: "available", value: node?.available || 0, color: GREEN }, + { group: "unknown", value: node?.unknown || 0, color: GREY }, { group: "unavailable", value: node?.unavailable || 0, color: GREY }, ]; }, @@ -56,7 +58,9 @@ const STATUS = [ { title: "disk", icon: () => , - render: () => , + render: (data, loading) => ( + + ), }, ]; @@ -67,6 +71,8 @@ export default connect(({ user }) => ({ const [status, setStatus] = useState({}); const [loading, setLoading] = useState(false); + const leftNewsColumnRef = useRef(null); + const [newsColumnHeight, setNewsColumnHeight] = useState(); const fetchStatus = async () => { setLoading(true); @@ -81,6 +87,22 @@ export default connect(({ user }) => ({ fetchStatus(); }, []); + useEffect(() => { + if ( + leftNewsColumnRef.current && + typeof leftNewsColumnRef.current.getBoundingClientRect === "function" + ) { + setNewsColumnHeight(leftNewsColumnRef.current.getBoundingClientRect().height); + } + }, []); + + useResizeObserver(leftNewsColumnRef, (entry) => { + const nextHeight = Math.ceil(entry.contentRect.height); + if (nextHeight > 0) { + setNewsColumnHeight(nextHeight); + } + }); + return (
    @@ -98,7 +120,7 @@ export default connect(({ user }) => ({
    {item.render ? ( - item.render() + item.render(status, loading) ) : ( ({ -
    - - + +
    + + +
    - + diff --git a/web/src/pages/Overview/index.less b/web/src/pages/Overview/index.less index e85e951c..4bd21e3d 100644 --- a/web/src/pages/Overview/index.less +++ b/web/src/pages/Overview/index.less @@ -30,4 +30,23 @@ align-items: center } } -} \ No newline at end of file + + .news { + :global { + .ant-col { + height: auto; + } + } + } + + .stackColumn { + .stackColumnInner { + display: flex; + flex-direction: column; + } + } + + .fillColumn { + display: flex; + } +} diff --git a/web/src/pages/Platform/Notification/index.scss b/web/src/pages/Platform/Notification/index.scss index 249ff675..5da98155 100644 --- a/web/src/pages/Platform/Notification/index.scss +++ b/web/src/pages/Platform/Notification/index.scss @@ -4,7 +4,18 @@ align-items: center; gap: 5px; } - .read { - opacity: 0.4; + .ant-table-tbody > tr.read > td { + background-color: #f5f7fb !important; + color: rgba(0, 0, 0, 0.5); + } + .ant-table-tbody > tr.read > td a { + color: rgba(0, 0, 0, 0.65); + } + :global { + .notice-status-read { + color: rgba(0, 0, 0, 0.65); + background: #eef2f7; + border-color: #d6dee9; + } } } diff --git a/web/src/pages/Platform/Notification/index.tsx b/web/src/pages/Platform/Notification/index.tsx index 82ec0bb6..e991ebdd 100644 --- a/web/src/pages/Platform/Notification/index.tsx +++ b/web/src/pages/Platform/Notification/index.tsx @@ -36,11 +36,82 @@ import { hasAuthority } from "@/utils/authority"; import "./index.scss"; import _ from "lodash"; import Markdown from "@/components/Markdown"; +import SearchInput from "@/components/infini/SearchInput"; const { TabPane } = Tabs; -const { Search } = Input; const { Option } = Select; +const normalizeTimeValue = (value, fallback = "auto", keys = []) => { + if (typeof value === "string" || typeof value === "number") { + return `${value}`; + } + if (value && typeof value === "object") { + for (const key of keys) { + if ( + Object.prototype.hasOwnProperty.call(value, key) && + value[key] !== undefined && + value[key] !== null + ) { + const candidate = value[key]; + if (typeof candidate === "string" || typeof candidate === "number") { + return `${candidate}`; + } + } + } + } + return fallback; +}; + +const tryParseJSON = (value) => { + if (!value || typeof value !== "string") return undefined; + try { + return JSON.parse(value); + } catch (e) { + return undefined; + } +}; + +const normalizeAlertingMessageLink = (rawLink = "") => { + let link = rawLink || ""; + if (link.indexOf("/#") === 0) { + link = link.substr(2); + } + if (!link || !link.includes("/alerting/message")) { + return link; + } + const [pathname, queryString = ""] = link.split("?"); + if (!queryString) { + return pathname; + } + const params = new URLSearchParams(queryString); + const rawG = params.get("_g"); + if (!rawG) { + return link; + } + const parsedG = tryParseJSON(rawG) || tryParseJSON(decodeURIComponent(rawG)); + if (!parsedG || typeof parsedG !== "object") { + return pathname; + } + const normalizedG = { ...parsedG }; + const range = normalizedG?.timeRange; + if (typeof normalizedG.start_time !== "string") { + normalizedG.start_time = normalizeTimeValue( + normalizedG.start_time ?? range?.min ?? range?.from, + "auto", + ["from", "min", "gte", "start"] + ); + } + if (typeof normalizedG.end_time !== "string") { + normalizedG.end_time = normalizeTimeValue( + normalizedG.end_time ?? range?.max ?? range?.to, + "auto", + ["to", "max", "lte", "end"] + ); + } + params.set("_g", JSON.stringify(normalizedG)); + return `${pathname}?${params.toString()}`; +}; + const Index = (props) => { const [param, setParam] = useQueryParam("_g", JsonParam); const [searchValue, setSearchValue] = React.useState(""); @@ -110,31 +181,35 @@ const Index = (props) => { const columns = [ { - title: "Title", + title: formatMessage({ id: "platform.notification.table.title" }), dataIndex: "title", render: (text, record) => {record.title}, }, { - title: "Created", + title: formatMessage({ id: "platform.notification.table.created" }), dataIndex: "created", render: (text, record) => ( {moment(record.created).fromNow()} ), }, { - title: "Status", + title: formatMessage({ id: "platform.notification.table.status" }), dataIndex: "status", render: (text, record) => ( - {text} + + {text == "new" + ? formatMessage({ id: "platform.notification.status.new" }) + : formatMessage({ id: "platform.notification.status.read" })} + ), }, { title: formatMessage({ id: "table.field.actions" }), render: (text, record) => { - let link = record.link || ""; - if (link.indexOf("/#") === 0) { - link = link.substr(2); - } + const link = normalizeAlertingMessageLink(record.link || ""); return (
    @@ -201,41 +276,8 @@ const Index = (props) => { }; return ( - // - // - // - // - // - // {noticeData.notification?.map((item) => { - // return ( - // - //

    {item.body}

    - //     - // ); - // })} - //     - //     - - // - // - // {noticeData.todo?.map((item) => { - // return ( - // - //

    {item.body}

    - //     - // ); - // })} - //     - //     - //     - //     - + +
    { }} >
    - { expandRowByClick={true} onExpand={onExpand} /> - - // + + ); }; diff --git a/web/src/pages/Platform/Overview/Cluster/Card/index.js b/web/src/pages/Platform/Overview/Cluster/Card/index.js index 3e1aef8e..a6bb9c9e 100644 --- a/web/src/pages/Platform/Overview/Cluster/Card/index.js +++ b/web/src/pages/Platform/Overview/Cluster/Card/index.js @@ -10,35 +10,17 @@ import "./index.scss"; import { Providers, ProviderIcon } from "@/lib/providers"; import { formatMessage } from "umi/locale"; import { SearchEngineIcon } from "@/lib/search_engines"; -import request from "@/utils/request"; export default (props) => { - const { infoAction, id, parentLoading } = props; + const { infoAction, id, parentLoading, info: prefetchedInfo, infoLoading } = props; const clusterID = props.data?._id; const metadata = props.data._source || {}; - const [info, setInfo] = useState({}); - const [loading, setLoading] = useState(false) - - const fetchListInfo = async (id) => { - if (!id) return - setLoading(true) - const res = await request(infoAction, { - method: "POST", - body: [id], - ignoreTimeout: true - }, false, false); - if (res) { - setInfo(res[id] || {}); - } - setLoading(false) - }; + const [info, setInfo] = useState(prefetchedInfo || {}); useEffect(() => { - if (!parentLoading) { - fetchListInfo(id) - } - }, [id, parentLoading]) + setInfo(prefetchedInfo || {}); + }, [prefetchedInfo]); const summary = info?.summary || {}; const metrics = info?.metrics || {}; @@ -111,7 +93,7 @@ export default (props) => { const healthStatus = metadata.labels?.health_status; return ( - +
    { const id = props.data?._id @@ -10,7 +11,7 @@ export default (props)=>{ return ( ) diff --git a/web/src/pages/Platform/Overview/Cluster/Detail/MetricTopN.jsx b/web/src/pages/Platform/Overview/Cluster/Detail/MetricTopN.jsx index 353cb4be..79029ec0 100644 --- a/web/src/pages/Platform/Overview/Cluster/Detail/MetricTopN.jsx +++ b/web/src/pages/Platform/Overview/Cluster/Detail/MetricTopN.jsx @@ -3,6 +3,7 @@ import { Spin, Empty } from 'antd'; import { ESPrefix } from "@/services/common"; import useFetch from "@/lib/hooks/use_fetch"; import Treemap from "@/components/infini/charts/Treemap"; +import { getLocalizedTreemapTitle } from "@/utils/treemap_title"; export const MetricTopN = (props) => { const clusterID = props.data?._id || null; @@ -19,7 +20,7 @@ export const MetricTopN = (props) => { return (
    -
    {treemapResult?._source?.name}
    +
    {getLocalizedTreemapTitle(treemapResult?._source?.name)}
    {treemapData.children ? ( { } const overviews = [ - { - key: 'nodes', - title: 'Nodes', + { + key: "nodes", + titleId: "overview.title.node", action: `${ESPrefix}/${clusterID}/nodes`, - component: MetricNodes + component: MetricNodes, }, - { - key: 'indices', - title: 'Indices', + { + key: "indices", + titleId: "overview.title.index", action: `${ESPrefix}/${clusterID}/indices`, - component: MetricIndices - } - ] + component: MetricIndices, + }, + ]; return ( { params={{ clusterID, clusterName }} linkMore={`/cluster/monitor/elasticsearch/${clusterID}`} overviews={overviews} - metrics={['index_throughput', 'search_throughput', 'index_latency', 'search_latency']} + metrics={[ + "index_throughput", + "search_throughput", + "index_latency", + "search_latency", + ]} /> - ) + ); }; diff --git a/web/src/pages/Platform/Overview/Cluster/Monitor/Logs.jsx b/web/src/pages/Platform/Overview/Cluster/Monitor/Logs.jsx index 5cf38e97..9c095578 100644 --- a/web/src/pages/Platform/Overview/Cluster/Monitor/Logs.jsx +++ b/web/src/pages/Platform/Overview/Cluster/Monitor/Logs.jsx @@ -17,6 +17,11 @@ export default (props) => { {...props} aggs={AGGS} queryFilters={[ + { + "term": { + "metadata.category": "elasticsearch" + } + }, { "term": { "metadata.labels.cluster_id": clusterID @@ -42,4 +47,4 @@ export default (props) => { ]} /> ); -} \ No newline at end of file +} diff --git a/web/src/pages/Platform/Overview/Cluster/Monitor/advanced.jsx b/web/src/pages/Platform/Overview/Cluster/Monitor/advanced.jsx index d889dc96..34221daa 100644 --- a/web/src/pages/Platform/Overview/Cluster/Monitor/advanced.jsx +++ b/web/src/pages/Platform/Overview/Cluster/Monitor/advanced.jsx @@ -63,6 +63,7 @@ export default (props) => { }); return ( { @@ -35,24 +37,6 @@ const Indices = ({ setShowRealtime(clusterAvailable); }, [clusterID, clusterAvailable]); - const initialQueryParams = { - from: 0, - size: 20, - }; - - function reducer(queryParams, action) { - switch (action.type) { - case "pageSizeChange": - return { - ...queryParams, - size: action.value, - }; - default: - throw new Error(); - } - } - const [queryParams, dispatch] = React.useReducer(reducer, initialQueryParams); - const { loading: indicesLoading, error: indicesError, @@ -60,9 +44,12 @@ const Indices = ({ } = useFetch( `${ESPrefix}/${clusterID}/indices${showRealtime ? "/realtime" : ""}`, { - queryParams: showRealtime ? {} : formatTimeRange(timeRange), + queryParams: { + ...(showRealtime ? {} : formatTimeRange(timeRange)), + timeout, + }, }, - [clusterID, timeRange, showRealtime] + [clusterID, timeRange, showRealtime, refresh, timeout] ); const [hits, hitsTotal] = useMemo(() => { @@ -96,7 +83,7 @@ const Indices = ({ const [columns] = useMemo(() => { let columns = [ { - title: "Name", + title: formatMessage({ id: "overview.column.name" }), dataIndex: "index", render: (text, record) => ( , sorter: (a, b) => sorter.string(a, b, "health"), }, { - title: "Status", + title: formatMessage({ id: "overview.column.status" }), dataIndex: "status", sorter: (a, b) => sorter.string(a, b, "status"), }, { - title: "Shards", + title: formatMessage({ id: "overview.column.shards" }), dataIndex: "shards", render: (text, record) => {text || 0}, sorter: (a, b) => a?.shards - b?.shards, }, { - title: "Replicas", + title: formatMessage({ id: "overview.column.replicas" }), dataIndex: "replicas", render: (text, record) => {text || 0}, sorter: (a, b) => a?.replicas - b?.replicas, }, { - title: "Document Count", + title: formatMessage({ id: "overview.column.document_count" }), dataIndex: "docs_count", render: (text, record) => {formatter.number(text || 0)}, sorter: (a, b) => a?.docs_count - b?.docs_count, }, { - title: "Data", + title: formatMessage({ id: "overview.column.data" }), dataIndex: "store_size_bytes", render: (text, record) => {record?.store_size || 0}, sorter: (a, b) => a?.store_size_bytes - b?.store_size_bytes, @@ -155,7 +142,7 @@ const Indices = ({ ]; if (showRealtime) { columns.push({ - title: "Pri Indexing Rate", + title: formatMessage({ id: "overview.column.primary_indexing_rate" }), dataIndex: "index_qps", render: (text, record) => ( {text != null ? `${text} /s` : "N/A"} @@ -163,7 +150,7 @@ const Indices = ({ sorter: (a, b) => a?.index_qps - b?.index_qps, }); columns.push({ - title: "Pri Indexing Bytes", + title: formatMessage({ id: "overview.column.primary_indexing_bytes" }), dataIndex: "index_bytes_qps", render: (text, record) => ( @@ -173,7 +160,7 @@ const Indices = ({ sorter: (a, b) => a?.index_bytes_qps - b?.index_bytes_qps, }); columns.push({ - title: "Search Rate", + title: formatMessage({ id: "overview.column.search_rate" }), dataIndex: "query_qps", render: (text, record) => ( {text != null ? `${text} /s` : "N/A"} @@ -182,7 +169,7 @@ const Indices = ({ }); } else { columns.push({ - title: "Timestamp", + title: formatMessage({ id: "overview.column.timestamp" }), dataIndex: "timestamp", render: (text, record) => {formatUtcTimeToLocal(text)}, sorter: (a, b) => sorter.string(a, b, "timestamp"), @@ -206,6 +193,8 @@ const Indices = ({ { let val = value ? [value] : []; setSearchFilterFields(val); @@ -267,12 +256,9 @@ const Indices = ({ columns={columns} pagination={{ size: "small", - pageSize: queryParams.size, + pageSize: 20, total: hitsTotal, - showSizeChanger: true, - onShowSizeChange: (_, size) => { - dispatch({ type: "pageSizeChange", value: size }); - }, + showSizeChanger: false, }} scroll={{x: 'max-content' }} /> diff --git a/web/src/pages/Platform/Overview/Cluster/Monitor/nodes.jsx b/web/src/pages/Platform/Overview/Cluster/Monitor/nodes.jsx index 0a43ab6d..05dfe302 100644 --- a/web/src/pages/Platform/Overview/Cluster/Monitor/nodes.jsx +++ b/web/src/pages/Platform/Overview/Cluster/Monitor/nodes.jsx @@ -23,13 +23,11 @@ export default ({ clusterID, clusterName, timeRange, + refresh, + timeout, clusterAvailable, bucketSize, }) => { - const initialQueryParams = { - from: 0, - size: 20, - }; const filterFields = { name: "Name", ip: "IP", @@ -41,19 +39,6 @@ export default ({ setShowRealtime(clusterAvailable); }, [clusterID, clusterAvailable]); - function reducer(queryParams, action) { - switch (action.type) { - case "pageSizeChange": - return { - ...queryParams, - size: action.value, - }; - default: - throw new Error(); - } - } - const [queryParams, dispatch] = React.useReducer(reducer, initialQueryParams); - const { loading: nodesLoading, error: nodesError, @@ -61,9 +46,12 @@ export default ({ } = useFetch( `${ESPrefix}/${clusterID}/nodes${showRealtime ? "/realtime" : ""}`, { - queryParams: showRealtime ? {} : formatTimeRange(timeRange), + queryParams: { + ...(showRealtime ? {} : formatTimeRange(timeRange)), + timeout, + }, }, - [clusterID, timeRange, showRealtime] + [clusterID, timeRange, showRealtime, refresh, timeout] ); const [hits, hitsTotal] = React.useMemo(() => { @@ -93,8 +81,10 @@ export default ({ const [columns] = React.useMemo(() => { let columns = [ { - title: "Name", + title: formatMessage({ id: "overview.column.name" }), dataIndex: "name", + fixed: "left", + width: 200, render: (text, record) => ( ( @@ -145,14 +135,14 @@ export default ({ } columns = columns.concat([ { - title: "Shards", + title: formatMessage({ id: "overview.column.shards" }), dataIndex: "shards", render: (text, record) => {text || "N/A"}, sorter: (a, b) => a?.shards - b?.shards, className: "verticalAlign", }, { - title: "CPU Usage", + title: formatMessage({ id: "overview.column.cpu_usage" }), dataIndex: "cpu", render: (text, record) => { const number = parseFloat(text); @@ -168,7 +158,7 @@ export default ({ className: "verticalAlign", }, { - title: "Load Average", + title: formatMessage({ id: "overview.column.load_average" }), dataIndex: "load_1m", render: (text, record) => ( @@ -179,7 +169,7 @@ export default ({ className: "verticalAlign", }, { - title: "JVM Heap", + title: formatMessage({ id: "overview.column.jvm_heap" }), dataIndex: "heap.percent", render: (text, record) => { const number = parseFloat(text); @@ -195,14 +185,14 @@ export default ({ className: "verticalAlign", }, { - title: "Disk Free Space", + title: formatMessage({ id: "overview.column.disk_free_space" }), dataIndex: "disk_avail_bytes", render: (text, record) => {record["disk.avail"] || "N/A"}, sorter: (a, b) => a?.disk_avail_bytes - b?.disk_avail_bytes, className: "verticalAlign", }, { - title: "Disk Used Space", + title: formatMessage({ id: "overview.column.disk_used_space" }), dataIndex: "disk_used_bytes", render: (text, record) => {record["disk.used"] || "N/A"}, sorter: (a, b) => a?.disk_used_bytes - b?.disk_used_bytes, @@ -211,14 +201,14 @@ export default ({ ]); if (showRealtime) { columns.push({ - title: "Uptime", + title: formatMessage({ id: "overview.column.uptime" }), dataIndex: "uptime_ms", render: (text, record) => {record["uptime"] || "N/A"}, sorter: (a, b) => a?.uptime_ms - b?.uptime_ms, className: "verticalAlign", }); columns.push({ - title: "Indexing Rate", + title: formatMessage({ id: "overview.column.indexing_rate" }), dataIndex: "index_qps", render: (text, record) => ( {text != null ? `${text} /s` : "N/A"} @@ -227,7 +217,7 @@ export default ({ className: "verticalAlign", }); columns.push({ - title: "Indexing Bytes", + title: formatMessage({ id: "overview.column.indexing_bytes" }), dataIndex: "index_bytes_qps", render: (text, record) => ( @@ -238,7 +228,7 @@ export default ({ className: "verticalAlign", }); columns.push({ - title: "Search Rate", + title: formatMessage({ id: "overview.column.search_rate" }), dataIndex: "query_qps", render: (text, record) => ( {text != null ? `${text} /s` : "N/A"} @@ -248,7 +238,7 @@ export default ({ }); } else { columns.push({ - title: "Timestamp", + title: formatMessage({ id: "overview.column.timestamp" }), dataIndex: "timestamp", render: (text, record) => {formatUtcTimeToLocal(text)}, sorter: (a, b) => sorter.string(a, b, "timestamp"), @@ -273,6 +263,8 @@ export default ({ { let val = value ? [value] : []; setSearchFilterFields(val); @@ -319,12 +311,9 @@ export default ({ columns={columns} pagination={{ size: "small", - pageSize: queryParams.size, + pageSize: 20, total: hitsTotal, - showSizeChanger: true, - onShowSizeChange: (_, size) => { - dispatch({ type: "pageSizeChange", value: size }); - }, + showSizeChanger: false, }} scroll={{x: 'max-content' }} /> diff --git a/web/src/pages/Platform/Overview/Cluster/Monitor/statistic_bar.jsx b/web/src/pages/Platform/Overview/Cluster/Monitor/statistic_bar.jsx index 43183223..ef32b1b0 100644 --- a/web/src/pages/Platform/Overview/Cluster/Monitor/statistic_bar.jsx +++ b/web/src/pages/Platform/Overview/Cluster/Monitor/statistic_bar.jsx @@ -20,6 +20,8 @@ const vstyle = { const StatisticBar = ({ clusterID, timeRange, + refresh, + timeout, setSpinning, clusterAvailable, clusterMonitored, @@ -30,13 +32,20 @@ const StatisticBar = ({ const { loading, error, value } = useFetch( `${ESPrefix}/${clusterID}/metrics`, - {}, - [clusterID, timeRange] + { + queryParams: { + timeout, + }, + }, + [clusterID, timeRange, refresh, timeout] ); React.useEffect(() => { setSpinning(loading); - }, [loading]); + return () => { + setSpinning(false); + }; + }, [loading, setSpinning]); React.useEffect(() => { if (onInfoChange) { @@ -130,10 +139,16 @@ const StatisticBar = ({ {!clusterAvailable ? (
    - Cluster is not availabe since:{" "} - {value?.summary?.timestamp - ? formatUtcTimeToLocal(value?.summary?.timestamp) - : "N/A"} + {formatMessage( + { + id: "cluster.manage.monitoring.notice.unavailable_since", + }, + { + timestamp: value?.summary?.timestamp + ? formatUtcTimeToLocal(value?.summary?.timestamp) + : "N/A", + } + )}
    ) : !clusterMonitored && @@ -142,16 +157,28 @@ const StatisticBar = ({ .isAfter(value?.summary?.timestamp) ? (
    - Cluster is not monitored.{" "} + {formatMessage({ + id: "cluster.manage.monitoring.notice.unmonitored", + })}{" "}
    - Last data collection time:{" "} - {value?.summary?.timestamp - ? formatUtcTimeToLocal(value?.summary?.timestamp) - : "N/A"} + {formatMessage( + { + id: "cluster.manage.monitoring.notice.last_collection_time", + }, + { + timestamp: value?.summary?.timestamp + ? formatUtcTimeToLocal(value?.summary?.timestamp) + : "N/A", + } + )}
    ) : null} diff --git a/web/src/pages/Platform/Overview/Cluster/Table/index.jsx b/web/src/pages/Platform/Overview/Cluster/Table/index.jsx index 69d0c49d..96cf70b5 100644 --- a/web/src/pages/Platform/Overview/Cluster/Table/index.jsx +++ b/web/src/pages/Platform/Overview/Cluster/Table/index.jsx @@ -1,5 +1,5 @@ import React, { useState, useEffect, useMemo } from "react"; -import { Table, Tooltip, Progress, Spin } from "antd"; +import { Table, Tooltip, Progress } from "antd"; import { formatter } from "@/utils/format"; import { formatMessage } from "umi/locale"; import { SearchEngineIcon } from "@/lib/search_engines"; @@ -14,7 +14,7 @@ export default (props) => { {...props} columns={[ { - title: "Name", + title: formatMessage({ id: "overview.column.name" }), dataIndex: "name", render: (text, record) => { return ( @@ -49,20 +49,19 @@ export default (props) => { {record.metadata?.name}
    - ); }, }, { - title: "Version", + title: formatMessage({ id: "overview.column.version" }), dataIndex: "version", render: (text, record) => { return record.metadata?.version ?? ""; }, }, { - title: "Health", + title: formatMessage({ id: "overview.column.health" }), dataIndex: "health_status", render: (text, record) => { return ( @@ -98,28 +97,28 @@ export default (props) => { }, }, { - title: "Nodes", + title: formatMessage({ id: "overview.column.nodes" }), dataIndex: "nodes", render: (text, record) => { return record.summary?.number_of_nodes || 0; }, }, { - title: "Indices", + title: formatMessage({ id: "overview.column.indices" }), dataIndex: "Indices", render: (text, record) => { return record.summary?.number_of_indices || 0; }, }, { - title: "Shards", + title: formatMessage({ id: "overview.column.shards" }), dataIndex: "Shards", render: (text, record) => { return record.summary?.number_of_shards || 0; }, }, { - title: "Docs", + title: formatMessage({ id: "overview.column.docs" }), dataIndex: "Docs", render: (text, record) => { return ( @@ -134,7 +133,7 @@ export default (props) => { }, }, { - title: "Disk Usage", + title: formatMessage({ id: "overview.column.disk_usage" }), dataIndex: "DiskUsage", render: (text, record) => { return ( @@ -161,7 +160,7 @@ export default (props) => { }, }, { - title: "JVM Heap", + title: formatMessage({ id: "overview.column.jvm_heap" }), dataIndex: "JVMHeap", render: (text, record) => { return ( diff --git a/web/src/pages/Platform/Overview/Cluster/index.tsx b/web/src/pages/Platform/Overview/Cluster/index.tsx index d47cc5b1..8bb25418 100644 --- a/web/src/pages/Platform/Overview/Cluster/index.tsx +++ b/web/src/pages/Platform/Overview/Cluster/index.tsx @@ -24,9 +24,24 @@ const aggsParams = [ ]; const details = [ - { title: "Metrics", component: Metrics, key: "metrics" }, - { title: "TopN", component: MetricTopN, key: "topN" }, - { title: "Infos", component: Infos, key: "infos" }, + { + title: "Metrics", + titleId: "overview.detail.metrics", + component: Metrics, + key: "metrics", + }, + { + title: "TopN", + titleId: "cluster.monitor.tabs.topn", + component: MetricTopN, + key: "topN", + }, + { + title: "Infos", + titleId: "overview.detail.infos", + component: Infos, + key: "infos", + }, ]; const sideSorterOptions = [ diff --git a/web/src/pages/Platform/Overview/Host/Detail/Edit.js b/web/src/pages/Platform/Overview/Host/Detail/Edit.js index ca2ac576..f21fbb9a 100644 --- a/web/src/pages/Platform/Overview/Host/Detail/Edit.js +++ b/web/src/pages/Platform/Overview/Host/Detail/Edit.js @@ -105,7 +105,7 @@ export default Form.create()((props)=>{
    +); export default ({ clusterID, @@ -88,7 +94,7 @@ export default ({ const [columns] = React.useMemo(() => { let columns = [ { - title: "Shard", + title: formatMessage({ id: "overview.column.shard" }), dataIndex: "shard", render: (text, record) => { if (!record.shard_id) { @@ -111,7 +117,7 @@ export default ({ sorter: (a, b) => a?.shard - b?.shard, }, { - title: "Prirep", + title: formatMessage({ id: "overview.column.prirep" }), dataIndex: "prirep", render: (text, record) => ( @@ -125,12 +131,12 @@ export default ({ sorter: (a, b) => sorter.string(a, b, "prirep"), }, { - title: "IP", + title: formatMessage({ id: "overview.column.ip" }), dataIndex: "ip", sorter: (a, b) => sorter.string(a, b, "ip"), }, { - title: "Node", + title: formatMessage({ id: "overview.column.node" }), dataIndex: "node", render: (text, record) => { if (!text) { @@ -159,12 +165,12 @@ export default ({ className: commonStyles.maxColumnWidth }, { - title: "State", + title: formatMessage({ id: "overview.column.state" }), dataIndex: "state", sorter: (a, b) => sorter.string(a, b, "state"), }, { - title: "Docs", + title: formatMessage({ id: "overview.column.docs" }), dataIndex: "docs", render: (text, record) => ( {formatter.number(record?.docs || 0)} @@ -172,7 +178,7 @@ export default ({ sorter: (a, b) => a?.docs - b?.docs, }, { - title: "Store", + title: formatMessage({ id: "overview.column.store" }), dataIndex: "store_size_bytes", render: (text, record) => ( {formatter.bytes(record?.store_in_bytes || 0)} @@ -180,7 +186,7 @@ export default ({ sorter: (a, b) => a?.store_in_bytes - b?.store_in_bytes, }, { - title: "Indexing Rate", + title: formatMessage({ id: "overview.column.indexing_rate" }), dataIndex: "index_qps", render: (text, record) => ( {text != null ? `${text} /s` : "N/A"} @@ -188,7 +194,7 @@ export default ({ sorter: (a, b) => a?.index_qps - b?.index_qps, }, { - title: "Indexing Bytes", + title: formatMessage({ id: "overview.column.indexing_bytes" }), dataIndex: "index_bytes_qps", render: (text, record) => ( @@ -212,15 +218,15 @@ export default ({ }} >
    - { - setSearchValue(value); + setSearchValue(value.trim()); }} onChange={(e) => { - setSearchValue(e.target.value); + setSearchValue(e.target.value.trim()); }} />
    @@ -232,11 +238,11 @@ export default ({ }} >
    diff --git a/web/src/pages/Platform/Overview/Indices/Monitor/statistic_bar.jsx b/web/src/pages/Platform/Overview/Indices/Monitor/statistic_bar.jsx index 0c8cb78d..4cebd3c6 100644 --- a/web/src/pages/Platform/Overview/Indices/Monitor/statistic_bar.jsx +++ b/web/src/pages/Platform/Overview/Indices/Monitor/statistic_bar.jsx @@ -5,6 +5,7 @@ import { formatter } from "@/utils/format"; import { HealthStatusCircle } from "@/components/infini/health_status_circle"; import OverviewStatistic from "../../components/overview_statistic"; import { formatUtcTimeToLocal } from "@/utils/utils"; +import { formatMessage } from "umi/locale"; const vstyle = { fontSize: 12, @@ -41,7 +42,7 @@ const StatisticBar = ({ clusterID, indexName, timeRange, setSpinning }) => { overviewStatistic = [ { key: "Health", - title: "Health", + title: formatMessage({ id: "indices.field.health" }), value: indexValue?.index_info?.health, vstyle: { ...vstyle, @@ -52,37 +53,37 @@ const StatisticBar = ({ clusterID, indexName, timeRange, setSpinning }) => { }, { key: "Status", - title: "Status", + title: formatMessage({ id: "indices.field.status" }), value: indexValue?.index_info?.status ?? "N/A", }, { key: "Total", - title: "Total", + title: formatMessage({ id: "indices.field.store_size" }), value: indexValue?.index_info?.store_size?.toUpperCase() ?? "N/A", }, { key: "Primaries", - title: "Primaries", + title: formatMessage({ id: "indices.field.primary_store_size" }), value: indexValue?.index_info?.pri_store_size?.toUpperCase() ?? "N/A", }, { key: "Documents", - title: "Documents", + title: formatMessage({ id: "indices.field.docs_count" }), value: formatter.number(indexValue?.index_info?.docs_count || 0), }, { key: "Total shards", - title: "Total shards", + title: formatMessage({ id: "overview.statistic.total_shards" }), value: indexValue?.index_info?.shards ?? "N/A", }, { key: "Unassigned shards", - title: "Unassigned shards", + title: formatMessage({ id: "cluster.monitor.summary.unassign_shard" }), value: indexValue?.unassigned_shards ?? "N/A", }, { key: "Updated", - title: "Updated", + title: formatMessage({ id: "overview.statistic.updated" }), value: indexValue?.timestamp ? formatUtcTimeToLocal(indexValue?.timestamp) : "N/A", @@ -95,15 +96,20 @@ const StatisticBar = ({ clusterID, indexName, timeRange, setSpinning }) => { {!isAvailable ? (
    - Index is{" "} - {indexValue?.index_info?.status == "delete" || - indexValue?.index_info?.status == "close" - ? `${indexValue?.index_info?.status}d` - : "not availabe"}{" "} - since:{" "} - {indexValue?.timestamp - ? formatUtcTimeToLocal(indexValue?.timestamp) - : "N/A"} + {formatMessage( + { id: "overview.status.index_since" }, + { + status: + indexValue?.index_info?.status == "delete" + ? formatMessage({ id: "overview.status.deleted" }) + : indexValue?.index_info?.status == "close" + ? formatMessage({ id: "overview.status.closed" }) + : formatMessage({ id: "overview.status.unavailable" }), + timestamp: indexValue?.timestamp + ? formatUtcTimeToLocal(indexValue?.timestamp) + : "N/A", + } + )}
    ) : null} diff --git a/web/src/pages/Platform/Overview/Indices/Table/index.jsx b/web/src/pages/Platform/Overview/Indices/Table/index.jsx index d1d31a03..46ab6b71 100644 --- a/web/src/pages/Platform/Overview/Indices/Table/index.jsx +++ b/web/src/pages/Platform/Overview/Indices/Table/index.jsx @@ -1,5 +1,5 @@ import React, { useState, useEffect, useMemo } from "react"; -import { Tooltip, Progress, Icon, Spin } from "antd"; +import { Tooltip, Progress, Icon } from "antd"; import { formatter } from "@/utils/format"; import { formatUtcTimeToLocal } from "@/utils/utils"; import { formatMessage } from "umi/locale"; @@ -14,7 +14,7 @@ export default (props) => { {...props} columns={[ { - title: "Name", + title: formatMessage({ id: "overview.column.name" }), dataIndex: "name", render: (text, record) => { return ( @@ -38,14 +38,20 @@ export default (props) => { {record.metadata?.index_name}
    - ); }, }, - { - title: "Health", - dataIndex: "health_status", +{ + title: formatMessage({ id: "overview.column.cluster" }), + dataIndex: "cluster_name", + render: (text, record) => { + return record.metadata?.cluster_name || "N/A"; + }, +}, +{ + title: formatMessage({ id: "overview.column.health" }), + dataIndex: "health_status", render: (text, record) => { return ( { }, }, { - title: "Status", + title: formatMessage({ id: "overview.column.status" }), dataIndex: "index_status", render: (text, record) => { return record.metadata?.labels?.state ?? "N/A"; }, }, { - title: "Store Size", + title: formatMessage({ id: "overview.column.store_size" }), dataIndex: "store_size", render: (text, record) => { - return record.summary?.index_info?.store_size?.toUpperCase() || "N/A"; + return formatter.bytes(record.summary?.store_in_bytes) || "N/A"; }, }, { - title: "Shards", + title: formatMessage({ id: "overview.column.shards" }), dataIndex: "Shards", render: (text, record) => { + const shards = record.summary?.shards || 0; + const replicas = record.summary?.replicas || 0; return ( - Unassigned Shards: - {record.summary?.unassigned_shards || 0} -
    Shards: - {record.summary?.index_info?.shards || 0} + {shards}
    Replicas: - {record.summary?.index_info?.replicas || 0} + {replicas} } > - {record.summary?.unassigned_shards || 0} /{" "} - {record.summary?.index_info?.shards || - 0 + (record.summary?.index_info?.replicas || 0)} + {shards} / {shards + replicas}     ); }, }, { - title: "Docs", + title: formatMessage({ id: "overview.column.docs" }), dataIndex: "Docs", render: (text, record) => { return ( @@ -128,15 +131,15 @@ export default (props) => { title={ Deleted: - {formatter.number(record.summary?.docs?.deleted || 0)} + {formatter.number(record.summary?.docs_deleted || 0)}
    Total: - {formatter.number(record.summary?.docs?.count || 0)} + {formatter.number(record.summary?.docs_count || 0)}     } > - {formatter.numberToHuman(record.summary?.docs?.deleted)} /{" "} - {formatter.numberToHuman(record.summary?.docs?.count)} + {formatter.number(record.summary?.docs_deleted || 0)} /{" "} + {formatter.number(record.summary?.docs_count || 0)}     ); }, diff --git a/web/src/pages/Platform/Overview/Indices/index.tsx b/web/src/pages/Platform/Overview/Indices/index.tsx index dd34b32e..4bdc90a8 100644 --- a/web/src/pages/Platform/Overview/Indices/index.tsx +++ b/web/src/pages/Platform/Overview/Indices/index.tsx @@ -19,8 +19,18 @@ const aggsParams = [ ]; const details = [ - { title: "Metrics", component: Metrics, key: "metrics" }, - { title: "Infos", component: Infos, key: "infos" }, + { + title: "Metrics", + titleId: "overview.detail.metrics", + component: Metrics, + key: "metrics", + }, + { + title: "Infos", + titleId: "overview.detail.infos", + component: Infos, + key: "infos", + }, ]; const sideSorterOptions = [ diff --git a/web/src/pages/Platform/Overview/Node/Card/index.js b/web/src/pages/Platform/Overview/Node/Card/index.js index ce4fb77a..bf887570 100644 --- a/web/src/pages/Platform/Overview/Node/Card/index.js +++ b/web/src/pages/Platform/Overview/Node/Card/index.js @@ -8,33 +8,15 @@ import { HealthStatusView } from "@/components/infini/health_status_view"; import { formatUtcTimeToLocal } from "@/utils/utils"; import { FieldFilterFacet } from "@/components/Overview/List/FieldFilterFacet"; import "./index.scss"; -import request from "@/utils/request"; export default (props) => { - const { infoAction, id, parentLoading } = props; + const { infoAction, id, parentLoading, info: prefetchedInfo, infoLoading } = props; const metadata = props.data._source?.metadata || {}; - const [info, setInfo] = useState({}); - const [loading, setLoading] = useState(false) - - const fetchListInfo = async (id) => { - if (!id) return - setLoading(true) - const res = await request(infoAction, { - method: "POST", - body: [id], - ignoreTimeout: true - }, false, false); - if (res) { - setInfo(res[id] || {}); - } - setLoading(false) - }; + const [info, setInfo] = useState(prefetchedInfo || {}); useEffect(() => { - if (!parentLoading) { - fetchListInfo(id) - } - }, [id, parentLoading]) + setInfo(prefetchedInfo || {}); + }, [prefetchedInfo]); const summary = info?.summary || {}; const metrics = info?.metrics || {}; @@ -106,7 +88,7 @@ export default (props) => { const healthStatus = metadata?.labels?.status; return ( - +
    { ")"}
    +
    +
    + + {metadata?.cluster_name || "N/A"} + +
    +
    Cluster
    +
    {summary?.shard_info?.indices_count || 0} diff --git a/web/src/pages/Platform/Overview/Node/Card/index.scss b/web/src/pages/Platform/Overview/Node/Card/index.scss index ead9ae86..4e9faa26 100644 --- a/web/src/pages/Platform/Overview/Node/Card/index.scss +++ b/web/src/pages/Platform/Overview/Node/Card/index.scss @@ -6,6 +6,20 @@ .card-metadata-info { min-width: 480px; } + .cluster-metric { + min-width: 140px; + margin-right: 12px; + .value { + max-width: 140px; + } + .cluster-value { + display: inline-block; + max-width: 140px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + } } .card-chart { min-width: 398px; diff --git a/web/src/pages/Platform/Overview/Node/Detail/Infos.js b/web/src/pages/Platform/Overview/Node/Detail/Infos.js index 8c448140..611a3c64 100644 --- a/web/src/pages/Platform/Overview/Node/Detail/Infos.js +++ b/web/src/pages/Platform/Overview/Node/Detail/Infos.js @@ -2,6 +2,7 @@ import React from "react"; import { ESPrefix } from "@/services/common"; import { formatter } from "@/utils/format"; import Infos from "@/components/Overview/Detail/Infos"; +import { formatMessage } from "umi/locale"; export default (props) => { const nodeID = props.data?._source?.metadata?.node_id; @@ -44,7 +45,7 @@ export default (props) => { return ( diff --git a/web/src/pages/Platform/Overview/Node/Detail/Logs.js b/web/src/pages/Platform/Overview/Node/Detail/Logs.js index 9eb31745..45349f9b 100644 --- a/web/src/pages/Platform/Overview/Node/Detail/Logs.js +++ b/web/src/pages/Platform/Overview/Node/Detail/Logs.js @@ -1,13 +1,12 @@ import { Button, - Divider, Icon, - Input, - Progress, + InputNumber, Select, Tooltip, Slider, Popover, + Switch, } from "antd"; import { useMemo, useState, useRef, useEffect, useCallback } from "react"; import { VariableSizeList as List } from "react-window"; @@ -19,7 +18,6 @@ import { } from "@/components/hooks/useFullScreen"; import "./logs.scss"; -import useFetch from "@/lib/hooks/use_fetch"; import request from "@/utils/request"; import moment from "moment"; import { formatter } from "@/utils/format"; @@ -27,37 +25,123 @@ import { formatter } from "@/utils/format"; import InstallAgent from "@/components/InstallAgent"; import { formatMessage } from "umi/locale"; import Location from "@/components/Icons/Location"; -const ButtonGroup = Button.Group; +const PAGE_SIZE = 50; +const DEFAULT_TAIL_LINES = 200; +const TAIL_LINE_OPTIONS = [50, 100, 200, 500, 1000]; + +const getLogFileKey = (logFile = {}) => + `${logFile.logs_path || ""}::${logFile.name || ""}`; + +const getLatestStartLine = (totalRows = 0) => + Math.max((totalRows || 0) - PAGE_SIZE + 1, 1); + +const hasKnownTotalRows = (logFile = {}) => + logFile?.total_rows_known === true && Number.isInteger(logFile?.total_rows); + +const createInitialLogState = (viewerVersion = 0) => ({ + hasNextPage: true, + isNextPageLoading: false, + items: [], + logFiles: [], + file: "", + fileName: "", + logsPath: "", + offset: 0, + totalRows: undefined, + totalRowsKnown: false, + startLineNumber: 1, + loadTailLines: 0, + latestLines: DEFAULT_TAIL_LINES, + followLatestEnabled: false, + autoScrollToBottom: false, + viewerVersion, +}); + +const getSelectedFileState = ( + logFile = {}, + { followLatestEnabled = false, autoScrollToBottom = false } = {} +) => { + const totalRowsKnown = hasKnownTotalRows(logFile); + return { + file: logFile?.name ? getLogFileKey(logFile) : "", + fileName: logFile?.name, + logsPath: logFile?.logs_path, + fileSize: logFile?.size_in_bytes, + totalRows: totalRowsKnown ? logFile?.total_rows : undefined, + totalRowsKnown, + offset: 0, + items: [], + hasNextPage: !!logFile?.name, + isNextPageLoading: false, + startLineNumber: totalRowsKnown ? getLatestStartLine(logFile?.total_rows) : 1, + loadTailLines: 0, + followLatestEnabled, + autoScrollToBottom, + }; +}; + +const formatLogFileLabel = (logFile = {}, includeMeta = false) => { + const parts = [logFile.name]; + if (logFile.logs_path) { + parts.push(logFile.logs_path); + } + if (includeMeta) { + parts.push(formatter.bytes(logFile.size_in_bytes || 0)); + if (logFile.modify_time) { + parts.push(moment(logFile.modify_time).format("YYYY.MM.DD")); + } + } + return parts.filter(Boolean).join(" | "); +}; const Logs = (props) => { const clusterID = props.data?._source?.metadata?.cluster_id; const nodeID = props.data?._source?.metadata?.node_id; - const [logState, setLogState] = useState({ - hasNextPage: true, - isNextPageLoading: false, - items: [], - logFiles: [], - file: "", - offset: 0, - startLineNumber: 1, - autoScrollToBottom: false, - }); + const [logState, setLogState] = useState(() => createInitialLogState()); const [loading, setLoading] = useState(false); + const [gotoLine, setGotoLine] = useState(); + const [gotoPopoverVisible, setGotoPopoverVisible] = useState(false); + const logStateRef = useRef(logState); + const requestSeqRef = useRef(0); - const fetchLogs = async (withLoading) => { + useEffect(() => { + logStateRef.current = logState; + }, [logState]); + + const selectedLogFile = useMemo( + () => + (logState.logFiles || []).find( + (logFile) => getLogFileKey(logFile) === logState.file + ), + [logState.file, logState.logFiles] + ); + + const fetchLogs = async (withLoading = false, preferredFile = logStateRef.current.file) => { if (withLoading) setLoading(true); const res = await request(`/elasticsearch/${clusterID}/node/${nodeID}/logs/_list`); if (res && res.success) { const { log_files: logFiles } = res; + const selectedFile = + logFiles.find((logFile) => getLogFileKey(logFile) === preferredFile) || + logFiles[0] || + {}; + const currentState = logStateRef.current; + const keepFollowLatest = + preferredFile && preferredFile === currentState.file + ? currentState.followLatestEnabled + : false; + requestSeqRef.current += 1; setLogState((st) => { return { ...st, + agentStatus: undefined, logFiles, - file: logFiles[0]?.name, - fileSize: logFiles[0]?.size_in_bytes, - totalRows: logFiles[0]?.total_rows, - startLineNumber: 1, + viewerVersion: st.viewerVersion + 1, + ...getSelectedFileState(selectedFile, { + followLatestEnabled: keepFollowLatest, + autoScrollToBottom: keepFollowLatest, + }), }; }); } else { @@ -74,29 +158,30 @@ const Logs = (props) => { }; useEffect(() => { - if(!nodeID){ - return + if (!nodeID) { + return; } - setLogState({ - hasNextPage: true, - isNextPageLoading: false, - items: [], - logFiles: [], - file: "", - offset: 0, - startLineNumber: 1, - }); + requestSeqRef.current += 1; + setLogState((st) => createInitialLogState(st.viewerVersion + 1)); fetchLogs(); - }, [nodeID]); + }, [clusterID, nodeID]); + const loadNextPage = useCallback( async (...args) => { - if (!logState.file) { + const force = args[0] === true; + const currentState = logStateRef.current; + if (!currentState.file) { + return; + } + if (currentState.isNextPageLoading) { return; } - // console.log("loadNextPage", ...args); - let newLogState = logState; + if (!force && !currentState.hasNextPage) { + return; + } + const requestSeq = requestSeqRef.current + 1; + requestSeqRef.current = requestSeq; setLogState((st) => { - newLogState = st; return { ...st, isNextPageLoading: true, @@ -105,123 +190,199 @@ const Logs = (props) => { const res = await request(`/elasticsearch/${clusterID}/node/${nodeID}/logs/_read`, { method: "POST", body: { - file_name: newLogState.file, - offset: newLogState.offset, - start_line_number: newLogState.startLineNumber, - lines: 50, + file_name: currentState.fileName, + logs_path: currentState.logsPath, + offset: currentState.offset, + start_line_number: currentState.startLineNumber, + tail_lines: currentState.loadTailLines, + lines: PAGE_SIZE, }, }); - if (res && res.lines) { - setLogState((st) => { - const newItems = [...newLogState.items]; - let idx = newItems.length; - res.lines.map((line) => { - newItems.push({ - ...line, - lineNumber: line.line_number, - index: idx, - }); - idx++; - }); - const maxLineNumber = newItems[newItems.length - 1]?.line_number; + if (requestSeq !== requestSeqRef.current) { + return; + } + setLogState((st) => { + if (st.file !== currentState.file) { return { ...st, - hasNextPage: res.has_more, isNextPageLoading: false, - items: newItems, - offset: newItems[newItems.length - 1]?.offset || st.offset, - startLineNumber: maxLineNumber + 1, - totalRows: maxLineNumber > st.totalRows ? maxLineNumber : st.totalRows, }; + } + const incomingLines = (res?.lines || []).map((line, index, arr) => { + if ( + currentState.loadTailLines > 0 && + st.totalRowsKnown && + Number.isInteger(st.totalRows) + ) { + return { + ...line, + line_number: Math.max(st.totalRows - arr.length + index + 1, 1), + }; + } + return line; }); - } else { - setLogState((st) => { - return { - ...st, - isNextPageLoading: false, - hasNextPage: res.has_more, - }; + const newItems = [...st.items]; + let idx = newItems.length; + incomingLines.forEach((line) => { + newItems.push({ + ...line, + lineNumber: line.line_number, + index: idx, + }); + idx += 1; }); - } + const lastLine = newItems[newItems.length - 1]; + const lastLineNumber = lastLine?.line_number; + const nextStartLineNumber = Number.isInteger(lastLineNumber) + ? lastLineNumber + 1 + : 0; + return { + ...st, + hasNextPage: !!res?.has_more, + isNextPageLoading: false, + items: newItems, + offset: lastLine?.offset || st.offset, + startLineNumber: + incomingLines.length > 0 ? nextStartLineNumber : st.startLineNumber, + loadTailLines: 0, + totalRows: + st.totalRowsKnown && + Number.isInteger(lastLineNumber) && + lastLineNumber > (st.totalRows || 0) + ? lastLineNumber + : st.totalRows, + }; + }); }, - [nodeID, logState] + [clusterID, nodeID] ); - const clearAutoRefresh = ()=>{ - if(autoRefreshTimeoutRef.current){ + + const clearAutoRefresh = () => { + if (autoRefreshTimeoutRef.current) { setRefreshStart(false); clearTimeout(autoRefreshTimeoutRef.current); autoRefreshTimeoutRef.current = null; - setLogState(st=>{ + setLogState((st) => { return { ...st, autoScrollToBottom: false, - } - }) + }; + }); } - } + }; + + const resetViewerPosition = useCallback( + ({ + startLineNumber = 1, + autoScrollToBottom = false, + loadTailLines = 0, + followLatestEnabled, + }) => { + requestSeqRef.current += 1; + setLogState((st) => { + return { + ...st, + viewerVersion: st.viewerVersion + 1, + hasNextPage: !!st.fileName, + isNextPageLoading: false, + items: [], + offset: 0, + startLineNumber: + loadTailLines > 0 ? 0 : Math.max(startLineNumber || 1, 1), + loadTailLines, + ...(typeof followLatestEnabled === "boolean" + ? { followLatestEnabled } + : {}), + autoScrollToBottom, + }; + }); + }, + [] + ); + const onLogFileChange = (file) => { clearAutoRefresh(); + requestSeqRef.current += 1; setLogState((st) => { - const logFile = st.logFiles.find((lf) => lf.name == file); + const logFile = st.logFiles.find((lf) => getLogFileKey(lf) === file); return { ...st, - file, - offset: 0, - items: [], - hasNextPage: true, - fileSize: logFile?.size_in_bytes, - totalRows: logFile?.total_rows, - startLineNumber: 1, + viewerVersion: st.viewerVersion + 1, + ...getSelectedFileState(logFile), }; }); }; - const offsetRef = useRef(); + const onGotoOffsetClick = () => { clearAutoRefresh(); - setLogState((st) => { - return { - ...st, - currentLineNumber: parseInt(offsetRef.current?.state.value) - }; + if (!logStateRef.current.totalRowsKnown) { + return false; + } + if (!Number.isInteger(gotoLine) || gotoLine < 1) { + return false; + } + resetViewerPosition({ + startLineNumber: gotoLine, + autoScrollToBottom: false, + followLatestEnabled: false, }); + setGotoPopoverVisible(false); + return true; }; const autoRefreshTimeoutRef = useRef(); const [refreshStart, setRefreshStart] = useState(false); - const autoRefresh = async ()=> { - setLogState(st=>{ + + const autoRefresh = async () => { + setLogState((st) => { return { ...st, autoRefreshLoading: true, - autoScrollToBottom: true, - currentLineNumber: undefined - } - }) - await loadNextPage() - setLogState(st=>{ + autoScrollToBottom: st.followLatestEnabled, + }; + }); + await loadNextPage(true); + setLogState((st) => { return { ...st, autoRefreshLoading: false, - } - }) - autoRefreshTimeoutRef.current = setTimeout(autoRefresh, 5000) - } + }; + }); + autoRefreshTimeoutRef.current = setTimeout(autoRefresh, 5000); + }; - useEffect(()=>{ + const refreshCurrentLog = useCallback(async () => { + const currentState = logStateRef.current; + if (!currentState.fileName) { + await fetchLogs(true, currentState.file); + return; + } + setLogState((st) => ({ + ...st, + autoScrollToBottom: st.followLatestEnabled, + })); + await loadNextPage(true); + }, [loadNextPage]); + + useEffect(() => { return clearAutoRefresh; }, []); - const onViewLatestClick = () => { - setLogState((st) => { - return { + const onViewLatestClick = (checked) => { + if (!checked) { + setLogState((st) => ({ ...st, - startLineNumber: st.totalRows - 20, - items: [], - hasNextPage: true, - }; + followLatestEnabled: false, + autoScrollToBottom: false, + })); + return; + } + clearAutoRefresh(); + resetViewerPosition({ + loadTailLines: logStateRef.current.latestLines || DEFAULT_TAIL_LINES, + autoScrollToBottom: true, + followLatestEnabled: true, }); - // autoRefreshTimeoutRef.current = setTimeout(autoRefresh, 5000) - }; if (logState.agentStatus === "uninstall") { @@ -242,7 +403,6 @@ const Logs = (props) => {
    ); @@ -262,73 +422,171 @@ const Logs = (props) => { return (
    -
    - {formatMessage({ id: "agent.logs.label.log_file" })} - +
    + + {formatMessage({ id: "agent.logs.label.log_file" })} + +
    + + + + {selectedLogFile ? ( + +
    + {selectedLogFile.logs_path ? ( + + {selectedLogFile.logs_path} + + ) : null} + + {moment(selectedLogFile.modify_time).format("YYYY.MM.DD HH:mm")} + +
    +     + ) : null} +
    - - {/* loading={logState.autoRefreshLoading} */} - - -
    - } - trigger="click" + refreshCurrentLog(); + }} > - + +
    + } + trigger="click" + > + + + ) : ( + - - -
    -
    - {logState.file ? ( - - ) : null} + )} +
    +
    + {logState.file ? ( + + ) : null} +
    ); }; @@ -406,26 +664,30 @@ const InfiniteLogViewer = ({ isNextPageLoading, items, loadNextPage, - file, - logFiles, - setLogState, autoScrollToBottom, - currentLineNumber, totalRows, + totalRowsKnown, + resetViewerPosition, }) => { - const maxLineNumber = Math.max( - ...items - .filter((item) => Number.isInteger(item.lineNumber)) - .map((item) => item.lineNumber) + const maxLineNumber = useMemo( + () => + items.reduce((max, item) => { + if (Number.isInteger(item.lineNumber) && item.lineNumber > max) { + return item.lineNumber; + } + return max; + }, totalRows || 1), + [items, totalRows] ); const fullScreenHandle = useFullScreenHandle(); const itemCount = hasNextPage ? items.length + 1 : items.length; - const loadMoreItems = isNextPageLoading ? () => {} : loadNextPage; + const loadMoreItems = isNextPageLoading ? () => Promise.resolve() : loadNextPage; const [progress, setProgress] = useState(0); - const progressCacheRef = useRef() + const progressCacheRef = useRef(); + const sliderChangingRef = useRef(false); // Every row is loaded except for our loading indicator row. const isItemLoaded = (index) => !hasNextPage || index < items.length; @@ -464,44 +726,35 @@ const InfiniteLogViewer = ({ return
    {content}
    ; }; const onSliderChange = (v) => { + sliderChangingRef.current = true; progressCacheRef.current = v; setProgress(v); }; const onSliderAfterChange = () => { - if (Number.isInteger(progressCacheRef.current)) { - setLogState((st) => { - let startLineNumber = parseInt((st.totalRows * progressCacheRef.current) / 100); - return { - ...st, - hasNextPage: true, - currentLineNumber: startLineNumber === 0 ? 1 : startLineNumber - }; + if ( + totalRowsKnown && + Number.isInteger(progressCacheRef.current) && + totalRows > 0 + ) { + const startLineNumber = Math.max( + parseInt((totalRows * progressCacheRef.current) / 100, 10), + 1 + ); + resetViewerPosition({ + startLineNumber, + autoScrollToBottom: false, + followLatestEnabled: false, }); } - } + sliderChangingRef.current = false; + }; useEffect(() => { if (autoScrollToBottom === true) { - listRef.current.scrollToItem(itemCount, "end"); + listRef.current?.scrollToItem(Math.max(itemCount - 1, 0), "end"); } - }, [items]); - - useEffect(() => { - if (autoScrollToBottom || items.length === 0 || !Number.isInteger(currentLineNumber) || currentLineNumber < 1) { - return; - } - const lastItemLineNumber = items[items.length - 1].lineNumber - if (lastItemLineNumber >= currentLineNumber) { - listRef.current.scrollToItem(currentLineNumber - 1, "end"); - } else { - if (hasNextPage) { - loadNextPage() - } else { - listRef.current.scrollToItem(currentLineNumber - 1, "end"); - } - } - }, [JSON.stringify(items), hasNextPage, currentLineNumber, autoScrollToBottom]) + }, [autoScrollToBottom, itemCount, items]); return ( @@ -539,26 +792,21 @@ const InfiniteLogViewer = ({ itemCount={itemCount} onItemsRendered={(props) => { const { visibleStopIndex } = props; - if (visibleStopIndex > 0) { - if (logFiles?.length === 0 || !file) return 0; - const currentFile = logFiles.find( - (item) => item.name === file + if ( + !sliderChangingRef.current && + totalRowsKnown && + totalRows > 0 && + Number.isInteger(visibleStopIndex) && + items[visibleStopIndex] + ) { + setProgress( + parseInt( + Math.floor( + (items[visibleStopIndex].lineNumber / totalRows) * 100 + ).toFixed(0), + 10 + ) ); - if ( - currentFile && - currentFile.size_in_bytes && - Number.isInteger(visibleStopIndex) && - items[visibleStopIndex] - ) { - setProgress( - parseInt( - Math.floor( - (items[visibleStopIndex].lineNumber / totalRows) * - 100 - ).toFixed(0) - ) - ); - } } return onItemsRendered(props); }} @@ -566,12 +814,6 @@ const InfiniteLogViewer = ({ height={height} width={width} itemSize={getRowHeight} - onScroll={() => { - setLogState((logState) => ({ - ...logState, - currentLineNumber: undefined - })) - }} > {Item} @@ -582,8 +824,12 @@ const InfiniteLogViewer = ({
    - {/* */} - +
    diff --git a/web/src/pages/Platform/Overview/Node/Detail/Metrics.js b/web/src/pages/Platform/Overview/Node/Detail/Metrics.js index b3f6162a..37d38dfa 100644 --- a/web/src/pages/Platform/Overview/Node/Detail/Metrics.js +++ b/web/src/pages/Platform/Overview/Node/Detail/Metrics.js @@ -15,7 +15,7 @@ export default (props) => { const overviews = [ { key: "indices", - title: "Indices", + titleId: "overview.title.index", action: `${ESPrefix}/${clusterID}/node/${nodeID}/indices`, component: MetricIndices, }, diff --git a/web/src/pages/Platform/Overview/Node/Detail/logs.scss b/web/src/pages/Platform/Overview/Node/Detail/logs.scss index c8687c43..50a4a677 100644 --- a/web/src/pages/Platform/Overview/Node/Detail/logs.scss +++ b/web/src/pages/Platform/Overview/Node/Detail/logs.scss @@ -1,17 +1,124 @@ .logs { .form-line { display: flex; - align-items: center; - gap: 1em; + align-items: flex-start; + flex-wrap: nowrap; + gap: 16px; + .form-item { display: flex; align-items: center; - gap: 3px; + gap: 8px; + min-width: 0; + .offset { width: 100px; } + + &--log-file { + flex: 1 1 auto; + min-width: 0; + align-items: flex-start; + gap: 10px; + } + + &__label { + line-height: 32px; + white-space: nowrap; + } + } + } + + .log-file-picker { + flex: 1 1 auto; + display: flex; + flex-direction: column; + align-items: stretch; + gap: 4px; + min-width: 0; + } + + .log-actions { + display: flex; + align-items: center; + flex-wrap: nowrap; + gap: 8px; + flex: 0 0 auto; + margin-left: auto; + } + + .log-latest-switch { + display: inline-flex; + align-items: center; + gap: 8px; + height: 32px; + padding: 0 12px; + border: 1px solid #d9d9d9; + border-radius: 4px; + background: #fff; + + &__label { + color: rgba(0, 0, 0, 0.85); + white-space: nowrap; + line-height: 1; + } + + &__select { + width: 88px; + } + + &__suffix { + color: rgba(0, 0, 0, 0.65); + white-space: nowrap; + line-height: 1; + } + } + + .log-file-select { + width: min(100%, 460px); + align-self: flex-start; + + .ant-select-selection-selected-value { + max-width: calc(100% - 24px); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + } + + .log-file-summary { + display: flex; + align-items: center; + justify-content: flex-end; + flex-wrap: nowrap; + gap: 16px; + width: auto; + max-width: min(100%, 520px); + align-self: flex-end; + min-width: 0; + white-space: nowrap; + + &__path { + min-width: 0; + flex: 1 1 auto; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: rgba(0, 0, 0, 0.45); + font-size: 12px; + line-height: 20px; + } + + &__time { + display: inline-flex; + align-items: center; + flex-shrink: 0; + color: rgba(0, 0, 0, 0.45); + font-size: 12px; + line-height: 20px; } } + .viewer { margin-top: 10px; } @@ -19,9 +126,17 @@ height: 100%; width: 100%; background-color: #fff; + border: 1px solid #e8e8e8; + border-radius: 4px; + overflow: hidden; + .v-header { - border: 1px solid #d9d9d9; + background-color: #fafafa; + border-bottom: 1px solid #e8e8e8; display: flex; + min-height: 40px; + align-items: center; + .icon-list { margin-left: auto; margin-right: 1em; @@ -29,30 +144,45 @@ cursor: pointer; } } + } .body { - background-color: black; - color: #fff; + background-color: #fff; + color: rgba(0, 0, 0, 0.85); font-size: 12px; padding-right: 0; + font-family: SFMono-Regular, Consolas, "Liberation Mono", Menlo, monospace; + .log-row { display: flex; gap: 1em; + padding: 4px 12px; + border-bottom: 1px solid #f5f5f5; + + &:hover { + background-color: #fafafa; + } + .number { text-align: right; + color: rgba(0, 0, 0, 0.45); + flex-shrink: 0; } .log-text { display: flex; overflow: hidden; + min-width: 0; .more { cursor: pointer; - margin-right: 4px ; + margin-right: 4px; + color: rgba(0, 0, 0, 0.45); } .text { margin-bottom: 0; word-break: break-all; word-wrap: break-word; + color: inherit; &.collapsed { white-space: nowrap; text-overflow: ellipsis; @@ -63,6 +193,67 @@ } } } + + @media (max-width: 1320px) { + .form-line { + flex-wrap: wrap; + } + + .log-file-picker { + align-items: stretch; + } + + .log-file-summary { + width: 100%; + max-width: 100%; + align-self: stretch; + justify-content: flex-start; + } + + .log-actions { + width: 100%; + margin-left: 0; + } + + .log-file-select { + width: min(100%, 420px); + } + + } +} + +.log-file-select-dropdown { + .log-file-option { + display: flex; + align-items: center; + justify-content: space-between; + width: 100%; + min-width: 0; + gap: 16px; + + &__name { + min-width: 0; + flex: 1 1 auto; + font-weight: 500; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + padding-right: 24px; + } + + &__meta { + display: inline-flex; + align-items: center; + justify-content: flex-end; + min-width: 88px; + flex-shrink: 0; + white-space: nowrap; + color: rgba(0, 0, 0, 0.45); + font-size: 12px; + text-align: right; + margin-left: 24px; + } + } } .install-agent { diff --git a/web/src/pages/Platform/Overview/Node/Monitor/Logs.jsx b/web/src/pages/Platform/Overview/Node/Monitor/Logs.jsx index 2efaad62..f335f11a 100644 --- a/web/src/pages/Platform/Overview/Node/Monitor/Logs.jsx +++ b/web/src/pages/Platform/Overview/Node/Monitor/Logs.jsx @@ -15,6 +15,11 @@ export default (props) => { {...props} aggs={AGGS} queryFilters={[ + { + "term": { + "metadata.category": "elasticsearch" + } + }, { "term": { "metadata.labels.node_uuid": nodeID @@ -23,4 +28,4 @@ export default (props) => { ]} /> ); -} \ No newline at end of file +} diff --git a/web/src/pages/Platform/Overview/Node/Monitor/advanced.jsx b/web/src/pages/Platform/Overview/Node/Monitor/advanced.jsx index 00389ba8..3a43f656 100644 --- a/web/src/pages/Platform/Overview/Node/Monitor/advanced.jsx +++ b/web/src/pages/Platform/Overview/Node/Monitor/advanced.jsx @@ -21,6 +21,7 @@ export default (props) => { }); return ( { isAgent, clusterID, nodeID, + info, } = props + const nodeRoles = Array.isArray(info?.roles) ? info.roles : []; + const hasDataRole = + nodeRoles.length === 0 + ? true + : nodeRoles.some((role) => role === "data" || role?.startsWith?.("data_")); + return ( { "index_latency", "search_latency", "parent_breaker", - isAgent ? "shard_state" : undefined, + isAgent && hasDataRole ? "shard_state" : undefined, ].filter((item) => !!item)} /> ); diff --git a/web/src/pages/Platform/Overview/Node/Monitor/shards.jsx b/web/src/pages/Platform/Overview/Node/Monitor/shards.jsx index a71354d0..98976f23 100644 --- a/web/src/pages/Platform/Overview/Node/Monitor/shards.jsx +++ b/web/src/pages/Platform/Overview/Node/Monitor/shards.jsx @@ -13,7 +13,13 @@ import IconText from "@/components/infini/IconText"; import AutoTextEllipsis from "@/components/AutoTextEllipsis"; import commonStyles from "@/common.less" -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; + +const searchButton = ( + +); export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { const [searchValue, setSearchValue] = React.useState(""); @@ -85,7 +91,7 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { const [columns] = React.useMemo(() => { let columns = [ { - title: "Index", + title: formatMessage({ id: "overview.column.index" }), dataIndex: "index", render: (text, record) => { return ( @@ -109,7 +115,7 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { className: commonStyles.maxColumnWidth }, { - title: "Shard", + title: formatMessage({ id: "overview.column.shard" }), dataIndex: "shard", render: (text, record) => { if (!record.shard_id) { @@ -132,7 +138,7 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { sorter: (a, b) => a?.shard - b?.shard, }, { - title: "Prirep", + title: formatMessage({ id: "overview.column.prirep" }), dataIndex: "prirep", render: (text, record) => ( @@ -146,12 +152,12 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { sorter: (a, b) => sorter.string(a, b, "prirep"), }, { - title: "State", + title: formatMessage({ id: "overview.column.state" }), dataIndex: "state", sorter: (a, b) => sorter.string(a, b, "state"), }, { - title: "Docs", + title: formatMessage({ id: "overview.column.docs" }), dataIndex: "docs", render: (text, record) => ( {formatter.number(record?.docs || 0)} @@ -159,7 +165,7 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { sorter: (a, b) => a?.docs - b?.docs, }, { - title: "Store", + title: formatMessage({ id: "overview.column.store" }), dataIndex: "store_size_bytes", render: (text, record) => ( {formatter.bytes(record?.store_in_bytes || 0)} @@ -167,7 +173,7 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { sorter: (a, b) => a?.store_in_bytes - b?.store_in_bytes, }, { - title: "Indexing Rate", + title: formatMessage({ id: "overview.column.indexing_rate" }), dataIndex: "index_qps", render: (text, record) => ( {text != null ? `${text} /s` : "N/A"} @@ -175,7 +181,7 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { sorter: (a, b) => a?.index_qps - b?.index_qps, }, { - title: "Indexing Bytes", + title: formatMessage({ id: "overview.column.indexing_bytes" }), dataIndex: "index_bytes_qps", render: (text, record) => ( @@ -199,15 +205,15 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { }} >
    - { - setSearchValue(value); + setSearchValue(value.trim()); }} onChange={(e) => { - setSearchValue(e.target.value); + setSearchValue(e.target.value.trim()); }} />
    @@ -234,11 +240,11 @@ export default ({ clusterID, clusterName, nodeID, timeRange, bucketSize }) => { />
    diff --git a/web/src/pages/Platform/Overview/Node/Monitor/statistic_bar.jsx b/web/src/pages/Platform/Overview/Node/Monitor/statistic_bar.jsx index 4c20d496..16e72d54 100644 --- a/web/src/pages/Platform/Overview/Node/Monitor/statistic_bar.jsx +++ b/web/src/pages/Platform/Overview/Node/Monitor/statistic_bar.jsx @@ -6,6 +6,7 @@ import { formatUtcTimeToLocal } from "@/utils/utils"; import moment from "moment"; import { HealthStatusCircle } from "@/components/infini/health_status_circle"; import OverviewStatistic from "../../components/overview_statistic"; +import { formatMessage } from "umi/locale"; const vstyle = { fontSize: 12, @@ -13,7 +14,7 @@ const vstyle = { fontWeight: "bold", }; -const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { +const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning, onInfoChange }) => { if (!clusterID || !nodeID) { return null; } @@ -31,6 +32,12 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { setSpinning(loading); }, [loading]); + React.useEffect(() => { + if (onInfoChange) { + onInfoChange(nodeValue); + } + }, [JSON.stringify(nodeValue)]); + const isAvailable = loading || (nodeValue?.status && @@ -44,7 +51,7 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { { key: "Status", value: nodeValue?.status || "N/A", - title: "Status", + title: formatMessage({ id: "overview.column.status" }), vstyle: { ...vstyle, display: "flex", @@ -57,39 +64,41 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { value: nodeValue?.jvm?.uptime ? moment.duration(nodeValue?.jvm?.uptime).humanize() : "N/A", - title: "Uptime", + title: formatMessage({ id: "overview.column.uptime" }), }, { key: "Type", - value: nodeValue?.is_master_node ? "Master Node" : "Not Master Node", - title: "Type", + value: nodeValue?.is_master_node + ? formatMessage({ id: "overview.statistic.master_node" }) + : formatMessage({ id: "overview.statistic.not_master_node" }), + title: formatMessage({ id: "overview.statistic.type" }), }, { key: "Transport Address", value: nodeValue?.transport_address || "N/A", - title: "Transport Address", + title: formatMessage({ id: "overview.column.transport_address" }), }, { key: "Indices", value: nodeValue?.shard_info?.indices_count, - title: "Indices", + title: formatMessage({ id: "overview.column.indices" }), }, { key: "Shards", value: (nodeValue?.shard_info?.shard_count || 0) + (nodeValue?.shard_info?.replicas_count || 0), - title: "Shards", + title: formatMessage({ id: "overview.column.shards" }), }, { key: "Documents", value: formatter.number(nodeValue?.indices?.docs?.count || 0), - title: "Documents", + title: formatMessage({ id: "indices.field.docs_count" }), }, { key: "Data", value: formatter.bytes(nodeValue?.indices?.store?.size_in_bytes || 0), - title: "Data", + title: formatMessage({ id: "overview.column.data" }), }, { key: "JVM Heap", @@ -104,7 +113,7 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { : 0 ).toFixed(2) + "%)", - title: "JVM Heap", + title: formatMessage({ id: "overview.column.jvm_heap" }), }, { key: "Free Disk Space", @@ -117,7 +126,7 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { : 0 ).toFixed(2) + "%)", - title: "Free Disk Space", + title: formatMessage({ id: "overview.column.disk_free_space" }), }, ]; } @@ -126,10 +135,14 @@ const StatisticBar = ({ clusterID, nodeID, timeRange, setSpinning }) => { {!isAvailable ? (
    - Node is not availabe since:{" "} - {nodeValue?.timestamp - ? formatUtcTimeToLocal(nodeValue?.timestamp) - : "N/A"} + {formatMessage( + { id: "overview.status.node_since" }, + { + timestamp: nodeValue?.timestamp + ? formatUtcTimeToLocal(nodeValue?.timestamp) + : "N/A", + } + )}
    ) : null} diff --git a/web/src/pages/Platform/Overview/Node/Table/index.jsx b/web/src/pages/Platform/Overview/Node/Table/index.jsx index 5410f3dc..27810716 100644 --- a/web/src/pages/Platform/Overview/Node/Table/index.jsx +++ b/web/src/pages/Platform/Overview/Node/Table/index.jsx @@ -1,6 +1,7 @@ import React, { useState, useEffect, useMemo } from "react"; -import { Tooltip, Progress, Icon, Spin } from "antd"; +import { Tooltip, Progress, Icon } from "antd"; import { formatter } from "@/utils/format"; +import { formatMessage } from "umi/locale"; import { HealthStatusView } from "@/components/infini/health_status_view"; import { StatusBlockGroup } from "@/components/infini/status_block"; import CommonTable from "../../components/CommonTable"; @@ -11,7 +12,7 @@ export default (props) => { {...props} columns={[ { - title: "Name", + title: formatMessage({ id: "overview.column.name" }), dataIndex: "name", render: (text, record) => { return ( @@ -45,61 +46,67 @@ export default (props) => { {record.metadata?.node_name} - ); }, }, +{ + title: formatMessage({ id: "overview.column.cluster" }), + dataIndex: "cluster_name", + render: (text, record) => { + return record.metadata?.cluster_name || "N/A"; + }, +}, +{ + title: formatMessage({ id: "overview.column.status" }), + dataIndex: "status", + render: (text, record) => { + return ( + + + + {record.metrics_status?.metric?.label + + "(" + + (record.metrics_status?.data?.length || 14) + + " " + + record.metrics_status?.metric?.units + + ")"} + + + } + > +
    + +
    +     + ); + }, +}, { - title: "Status", - dataIndex: "status", - render: (text, record) => { - return ( - - - - {record.metrics_status?.metric?.label + - "(" + - (record.metrics_status?.data?.length || 14) + - " " + - record.metrics_status?.metric?.units + - ")"} - - - } - > -
    - -
    -     - ); - }, - }, - { - title: "Indices", + title: formatMessage({ id: "overview.column.indices" }), dataIndex: "Indices", render: (text, record) => { return record.summary?.shard_info?.indices_count || 0; }, }, { - title: "Shards", + title: formatMessage({ id: "overview.column.shards" }), dataIndex: "Shards", render: (text, record) => { return record.summary?.shard_info?.shard_count || 0; }, }, { - title: "Docs", + title: formatMessage({ id: "overview.column.docs" }), dataIndex: "Docs", render: (text, record) => { return ( @@ -114,7 +121,7 @@ export default (props) => { }, }, { - title: "Disk Usage", + title: formatMessage({ id: "overview.column.disk_usage" }), dataIndex: "DiskUsage", render: (text, record) => { return ( @@ -141,7 +148,7 @@ export default (props) => { }, }, { - title: "JVM Heap", + title: formatMessage({ id: "overview.column.jvm_heap" }), dataIndex: "JVMHeap", render: (text, record) => { return ( diff --git a/web/src/pages/Platform/Overview/Node/index.tsx b/web/src/pages/Platform/Overview/Node/index.tsx index 7b128ee0..35cceef5 100644 --- a/web/src/pages/Platform/Overview/Node/index.tsx +++ b/web/src/pages/Platform/Overview/Node/index.tsx @@ -28,9 +28,24 @@ const aggsParams = [ ]; const details = [ - { title: "Metrics", component: Metrics, key: "metrics" }, - { title: "Infos", component: Infos, key: "infos" }, - { title: "Logs", component: Logs, key: "logs" }, + { + title: "Metrics", + titleId: "overview.detail.metrics", + component: Metrics, + key: "metrics", + }, + { + title: "Infos", + titleId: "overview.detail.infos", + component: Infos, + key: "infos", + }, + { + title: "Logs", + titleId: "cluster.monitor.tabs.logs", + component: Logs, + key: "logs", + }, ]; const sideSorterOptions = [ diff --git a/web/src/pages/Platform/Overview/components/CommonTable/index.jsx b/web/src/pages/Platform/Overview/components/CommonTable/index.jsx index 49266099..3b1709e4 100644 --- a/web/src/pages/Platform/Overview/components/CommonTable/index.jsx +++ b/web/src/pages/Platform/Overview/components/CommonTable/index.jsx @@ -27,31 +27,29 @@ export default (props) => { } = props; const [infos, setInfos] = useState({}); - const [loadings, setLoadings] = useState({}); const fetchListInfo = async (data) => { - data?.forEach((item) => { - setLoadings((loadings) => ({ - ...loadings, - [item.id]: true - })) - request(infoAction, { + const ids = (data || []) + .map((item) => item?.id) + .filter((id) => !!id && !infos[id]); + if (ids.length === 0) { + return; + } + const res = await request( + infoAction, + { method: "POST", - body: [item.id], - }, false, false).then((res) => { - if (res && !res.error) { - setInfos((infos) => ({ - ...infos, - ...res - })); - } - }).finally(() => { - setLoadings((loadings) => ({ - ...loadings, - [item.id]: false - })) - }) - }) + body: ids, + }, + false, + false + ); + if (res && !res.error) { + setInfos((current) => ({ + ...current, + ...res, + })); + } }; useEffect(() => { @@ -71,6 +69,7 @@ export default (props) => { loading={loading} columns={columns} dataSource={tableData} + scroll={{ x: "max-content" }} rowKey={"id"} pagination={{ size: "small", @@ -91,7 +90,7 @@ export default (props) => { }, }; }} - rowClassName={(record) => `${styles.rowPointer} ${loadings[record.id] && !parentLoading ? styles.loading : ''}`} + rowClassName={styles.rowPointer} /> ); diff --git a/web/src/pages/Platform/Overview/components/CommonTable/index.less b/web/src/pages/Platform/Overview/components/CommonTable/index.less index 5c508966..cdb1794f 100644 --- a/web/src/pages/Platform/Overview/components/CommonTable/index.less +++ b/web/src/pages/Platform/Overview/components/CommonTable/index.less @@ -1,28 +1,3 @@ .rowPointer { cursor: pointer; - :global { - .ant-spin-spinning { - display: none; - } - } -} - -.loading { - position: relative; - cursor: default; - pointer-events: none; - opacity: 0.5; - :global { - .ant-spin-spinning { - position: absolute; - left: 0; - top: 0; - width: 100%; - height: 100%; - display: flex; - justify-content: center; - align-items: center; - z-index: 1; - } - } } \ No newline at end of file diff --git a/web/src/pages/Platform/Overview/components/Logs/index.jsx b/web/src/pages/Platform/Overview/components/Logs/index.jsx index 24860712..ca1fc6bb 100644 --- a/web/src/pages/Platform/Overview/components/Logs/index.jsx +++ b/web/src/pages/Platform/Overview/components/Logs/index.jsx @@ -1,8 +1,8 @@ -import { Empty, Input, Spin, Table } from "antd"; +import { Card, Empty, Icon, Spin, Table } from "antd"; import styles from "./index.less" import DatePicker from "@/common/src/DatePicker"; import { useEffect, useMemo, useRef, useState } from "react"; -import { formatESSearchResult, formatTimeRange } from "@/lib/elasticsearch/util"; +import { buildContainsQueryString, formatESSearchResult, formatTimeRange } from "@/lib/elasticsearch/util"; import { formatMessage } from "umi/locale"; import request from "@/utils/request"; import moment from "moment"; @@ -15,6 +15,7 @@ import { cloneDeep } from "lodash"; import { Link } from "umi"; import InstallAgent from "@/components/InstallAgent"; import { getSystemClusterID } from "@/utils/setup"; +import SearchInput from "@/components/infini/SearchInput"; const COLORS = { 'INFO': '#e8eef2', @@ -22,8 +23,79 @@ const COLORS = { 'ERROR': '#ff3f3f' } +const buildTimestampKeywordFilter = (keyword, timeField) => { + const value = `${keyword ?? ""}`.trim(); + if (!value) { + return null; + } + const parsed = moment.tz( + value, + ["YYYY-MM-DD HH:mm:ss", "YYYY-MM-DDTHH:mm:ss"], + true, + getTimezone() + ); + if (!parsed.isValid()) { + return null; + } + return { + range: { + [timeField]: { + gte: parsed.clone().startOf("second").toISOString(), + lte: parsed.clone().endOf("second").toISOString(), + format: "strict_date_optional_time", + } + } + }; +}; + +const buildLogSearchFilters = (queryParams = {}, queryFilters = [], timeField) => { + const filters = [...queryFilters]; + if (queryParams?.filters) { + Object.keys(queryParams.filters).map((field) => { + const values = queryParams.filters[field]; + if (Array.isArray(values)) { + values.forEach((item) => { + filters.push({ + term: { [field]: item } + }); + }); + } else if (values) { + filters.push({ + term: { [field]: values } + }); + } + }); + } + const keywordFilters = []; + const searchQuery = buildContainsQueryString(queryParams?.keyword); + if (searchQuery) { + keywordFilters.push({ + query_string: { + query: searchQuery, + fields: ["payload.message"], + analyze_wildcard: true, + } + }); + } + const timestampKeywordFilter = buildTimestampKeywordFilter(queryParams?.keyword, timeField); + if (timestampKeywordFilter) { + keywordFilters.push(timestampKeywordFilter); + } + if (keywordFilters.length === 1) { + filters.push(keywordFilters[0]); + } else if (keywordFilters.length > 1) { + filters.push({ + bool: { + should: keywordFilters, + minimum_should_match: 1, + } + }); + } + return filters; +}; + export default (props) => { - const { timeRange, isAgent, refresh, aggs, queryFilters = [], extraColumns = [] } = props; + const { timeRange, isAgent, refresh, aggs, queryFilters = [], extraColumns = [], handleTimeChange } = props; const ref = useRef(null); @@ -32,6 +104,7 @@ export default (props) => { const [loading, setLoading] = useState(false) const [result, setResult] = useState(false) + const [sideVisible, setSideVisible] = useState(true) const [queryParams, setQueryParams] = useState({ size: 20, @@ -41,38 +114,13 @@ export default (props) => { }, keyword: '' }) + const [searchKeyword, setSearchKeyword] = useState("") const fetchData = async (queryParams, timeRange, aggs, queryFilters) => { if (!timeRange) return; setLoading(true) const newTimeRange = formatTimeRange(timeRange); - const filters = [...queryFilters] - if (queryParams?.filters) { - Object.keys(queryParams.filters).map((field) => { - const values = queryParams.filters[field]; - if (Array.isArray(values)) { - values.forEach((item) => { - filters.push({ - 'term': { [field]: item } - }) - }); - } else { - if (values) { - filters.push({ - 'term': { [field]: values } - }) - } - } - }); - } - if (queryParams?.keyword) { - filters.push({ - query_string: { - query: `*${queryParams.keyword}*`, - fields: ['payload.message'], - } - }); - } + const filters = buildLogSearchFilters(queryParams, queryFilters, timeField); const res = await request(`${ESPrefix}/${getSystemClusterID()}/search/ese?timeout=60m`, { method: 'POST', body: { @@ -127,6 +175,16 @@ export default (props) => { fetchData(queryParams, timeRange, aggs, queryFilters) }, [JSON.stringify(queryParams), timeRange, JSON.stringify(aggs), JSON.stringify(queryFilters)]) + const lastTimeRangeKeyRef = useRef(`${timeRange?.min || ""}:${timeRange?.max || ""}`); + + useEffect(() => { + const nextTimeRangeKey = `${timeRange?.min || ""}:${timeRange?.max || ""}`; + if (lastTimeRangeKeyRef.current !== nextTimeRangeKey && queryParams.from !== 0) { + setQueryParams((st) => ({ ...st, from: 0 })); + } + lastTimeRangeKeyRef.current = nextTimeRangeKey; + }, [timeRange?.min, timeRange?.max, queryParams.from]); + const columns = [ { title: formatMessage({ id: "cluster.monitor.logs.timestamp" }), @@ -234,17 +292,66 @@ export default (props) => { ], } + const histogramQuery = useMemo(() => { + const filters = buildLogSearchFilters( + { + keyword: queryParams?.keyword, + }, + queryFilters, + timeField + ); + if (filters.length === 0) { + return undefined; + } + return JSON.stringify({ + bool: { + filter: filters, + should: [], + must_not: [], + } + }); + }, [queryParams?.keyword, JSON.stringify(queryFilters)]); + + const totalHits = useMemo(() => { + return result?.total?.value || result?.total || 0; + }, [result?.total]); + + const showHistogram = useMemo(() => { + const levelBuckets = result?.aggregations?.Level?.buckets; + if (!levelBuckets) return true; + return levelBuckets.length > 0; + }, [result?.aggregations?.Level]); + const isNotEmpty = useMemo(() => { - return result?.data?.length > 0 - }, [result?.data?.length]) + return totalHits > 0 + }, [totalHits]) + + const emptyText = useMemo(() => ( + + ), [isAgent]); + + const onHistogramQueriesChange = (nextQueries = {}) => { + const nextRange = nextQueries?.range; + if (!nextRange?.from || !nextRange?.to || typeof handleTimeChange !== "function") { + return; + } + setQueryParams((st) => ({ ...st, from: 0 })); + handleTimeChange({ + start: nextRange.from, + end: nextRange.to, + }); + } return ( -
    - { - isNotEmpty ? ( - <> -
    +
    + {isAgent || isNotEmpty ? ( +
    +
    { }} />
    -
    -
    - { - setQueryParams((st) => ({ ...st, from: 0, keyword: value })); - }} - enterButton - /> -
    -
    - -
    -
    -
    { - setQueryParams((st) => ({ - ...st, - from: (page - 1) * st.size, - })); - }, - showSizeChanger: true, - onShowSizeChange: (_, size) => { - setQueryParams((st) => ({ ...st, from: 0, size })); - }, - showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, - }} - /> - +
    + setSideVisible((visible) => !visible)} + title={ + sideVisible + ? formatMessage({ id: "listview.side.button.collapse" }) + : formatMessage({ id: "listview.side.button.expand" }) + } + > + + + +
    +
    + { + const nextKeyword = `${value ?? ""}`.trim(); + setSearchKeyword(nextKeyword); + setQueryParams((st) => ({ ...st, from: 0, keyword: nextKeyword })); + }} + onChange={(e) => { + const nextKeyword = e?.target?.value ?? ""; + setSearchKeyword(nextKeyword); + if (nextKeyword === "") { + setQueryParams((st) => ({ ...st, from: 0, keyword: "" })); + } + }} + enterButton={formatMessage({ id: "form.button.search" })} + /> +
    +
    + {showHistogram && ( +
    + +
    + )} +
    +
    { + setQueryParams((st) => ({ + ...st, + from: (page - 1) * st.size, + })); + }, + showSizeChanger: true, + onShowSizeChange: (_, size) => { + setQueryParams((st) => ({ ...st, from: 0, size })); + }, + showTotal: (total, range) => + `${range[0]}-${range[1]} of ${total} items`, + }} + /> + + - - ) : ( + + ) : ( +
    -
    {!isAgent && }
    +
    + +
    - ) - } +
    + )} ); -} \ No newline at end of file +} diff --git a/web/src/pages/Platform/Overview/components/Logs/index.less b/web/src/pages/Platform/Overview/components/Logs/index.less index 475def79..2898c9af 100644 --- a/web/src/pages/Platform/Overview/components/Logs/index.less +++ b/web/src/pages/Platform/Overview/components/Logs/index.less @@ -1,26 +1,107 @@ .logs { - display: flex; - gap: 12px; width: 100%; min-height: calc(100vh - 444px); - - .side { - width: 220px; + + .emptyBlock { + width: 100%; + margin: 0; + text-align: center; + + :global(.ant-empty-image) { + margin-left: auto; + margin-right: auto; + } + + :global(.ant-empty-description) { + text-align: center; + } + } + + .installAgentWrap { + width: 644px; + max-width: 100%; + margin: 0 auto; + text-align: left; + } +} + +.logLayout { + display: flex; + width: 100%; + gap: 10px; + + .sideWrap { + min-height: calc(100vh - 130px); + padding: 24px 15px; + background-color: #ffffff; + } + + .contentWrap { + position: relative; + min-width: 0; + + .expandAndCollapse { + cursor: pointer; + z-index: 10; + position: absolute; + left: 0; + top: 24px; + width: 14px; + height: 32px; + line-height: 32px; + border-radius: 0 4px 4px 0; + background-color: rgba(234, 244, 255, 1); + text-align: center; + } } - .result { - width: calc(100% - 220px - 12px); + .resultCard { + min-width: 0; .header { + display: flex; + align-items: center; + gap: 10px; margin-bottom: 16px; } - .histogram { + .searchBox { + flex: 1; + min-width: 0; + max-width: 600px; + } + + :global(.ant-table-placeholder) { + border-bottom: none; + } + } + + .histogram { + width: 100%; + height: 140px; + border: 1px solid rgb(232, 232, 232); + border-radius: 2px; + margin-bottom: 16px; + } + + &.expand { + .sideWrap { + display: block; + width: 220px; + } + + .contentWrap { + width: calc(100% - 220px); + } + } + + &.collapse { + .sideWrap { + display: none; + } + + .contentWrap { width: 100%; - height: 140px; - border: 1px solid rgb(232, 232, 232); - border-radius: 2px; - margin-bottom: 16px; } } -} \ No newline at end of file +} diff --git a/web/src/pages/Platform/Overview/components/MetricChart.jsx b/web/src/pages/Platform/Overview/components/MetricChart.jsx index 2a89a949..0a21474a 100644 --- a/web/src/pages/Platform/Overview/components/MetricChart.jsx +++ b/web/src/pages/Platform/Overview/components/MetricChart.jsx @@ -150,6 +150,14 @@ export default (props) => { } }; + const chartTimeFormatter = + typeof timeRange?.timeFormatter === "function" + ? (value) => { + const formatted = timeRange.timeFormatter(value); + return formatted == null ? "" : formatted; + } + : formatter.dates(1); + const renderChart = () => { if (error) { return ( @@ -160,14 +168,44 @@ export default (props) => { } const axis = metric?.axis || []; const lines = metric?.lines || []; + + // Fill gaps for Bar charts (auto_date_histogram only returns non-empty buckets) + lines.forEach((line) => { + if (line.type !== "Bar" || !line.data || line.data.length < 2) return; + const timestamps = [...new Set(line.data.map((d) => d.x))].sort( + (a, b) => a - b + ); + if (timestamps.length < 2) return; + const gaps = []; + for (let i = 1; i < timestamps.length; i++) { + gaps.push(timestamps[i] - timestamps[i - 1]); + } + gaps.sort((a, b) => a - b); + const intervalMs = gaps[Math.floor(gaps.length / 2)]; + if (!intervalMs || intervalMs <= 0) return; + const existingSet = new Set(timestamps.map(String)); + const filledData = [...line.data]; + const startTs = timestamps[0]; + const endTs = timestamps[timestamps.length - 1]; + const maxSlots = 200; + let count = 0; + for (let ts = startTs; ts <= endTs && count < maxSlots; ts += intervalMs) { + count++; + if (!existingSet.has(String(ts))) { + filledData.push({ x: ts, y: 100, g: "empty" }); + } + } + filledData.sort((a, b) => a.x - b.x || (a.g || "").localeCompare(b.g || "")); + line.data = filledData; + }); if (lines.every((item) => !item.data || item.data.length === 0)) { const emptyProps = {} if (metric?.min_bucket_size > 0 && metric?.hits_total > 0) { emptyProps.description = ( - <> -
    + + {formatMessage({ id: "cluster.metrics.time_interval.empty" }, { min_bucket_size: metric.min_bucket_size})} -
    + handleTimeIntervalChange(`${metric?.min_bucket_size}s`)}> @@ -178,11 +216,11 @@ export default (props) => { )}> - e.preventDefault()}> + e.preventDefault()}> {formatMessage({ id: `cluster.metrics.time_interval.apply`})} - + ) } return ( @@ -216,8 +254,8 @@ export default (props) => { {metricKey == "cluster_health" ? ( { yAccessor === "y" ) { + if (g === "empty") return "#D3DAE6"; if( ["red", "yellow", "green"].includes(g)){ return g; } @@ -322,20 +361,27 @@ export default (props) => { } return ( -
    +
    { - metric?.request && ( - - - message.success(formatMessage({id: "cluster.metrics.request.copy.success"}))} - /> - - - ) - } + metric?.request && ( +
    + + + message.success(formatMessage({id: "cluster.metrics.request.copy.success"}))} + /> + + +
    + ) + }
    @@ -361,4 +407,4 @@ export default (props) => {
    ); - } \ No newline at end of file + } diff --git a/web/src/pages/Platform/Overview/components/Metrics.scss b/web/src/pages/Platform/Overview/components/Metrics.scss index c844aab4..f4b39be8 100644 --- a/web/src/pages/Platform/Overview/components/Metrics.scss +++ b/web/src/pages/Platform/Overview/components/Metrics.scss @@ -25,26 +25,78 @@ } } -.vizChartContainer, .metric-item{ +.metricChartContainer { position: relative; + .copyAction { + position: absolute; + right: 12px; + bottom: 12px; + z-index: 11; + opacity: 0; + visibility: hidden; + pointer-events: none; + transition: opacity 0.2s ease, visibility 0.2s ease; + } .copyReq { cursor: pointer; - position: absolute; - display: none; - right: 3px; - bottom: 0px; + display: flex; + align-items: center; + justify-content: center; width: 24px; height: 24px; + border-radius: 4px; + box-shadow: rgba(0, 0, 0, 0.16) 0px 0px 5px 0px; + background: #fff; z-index: 11; &:hover { color: #1890ff; + box-shadow: rgba(0, 0, 0, 0.3) 0px 0px 5px 0px; } } &:hover { - .copyReq { - display: block; + .copyAction { + opacity: 1; + visibility: visible; + pointer-events: auto; } + } } + +.vizChartContainer, .metric-item{ + position: relative; + .copyAction { + position: absolute; + right: 12px; + bottom: 12px; + z-index: 11; + opacity: 0; + visibility: hidden; + pointer-events: none; + transition: opacity 0.2s ease, visibility 0.2s ease; + } + .copyReq { + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + border-radius: 4px; + box-shadow: rgba(0, 0, 0, 0.16) 0px 0px 5px 0px; + background: #fff; + z-index: 11; + &:hover { + color: #1890ff; + box-shadow: rgba(0, 0, 0, 0.3) 0px 0px 5px 0px; + } + } + &:hover { + .copyAction { + opacity: 1; + visibility: visible; + pointer-events: auto; + } + } } .vizChartItem { @@ -110,8 +162,10 @@ .px-box{ display: flex; justify-content: space-between; + gap: 6px; + align-items: flex-start; .px { - flex: 1; + flex: 1 1 auto; + min-width: 0; } } - diff --git a/web/src/pages/Platform/Overview/components/Side/SearchFacet.less b/web/src/pages/Platform/Overview/components/Side/SearchFacet.less index be367094..19356f99 100644 --- a/web/src/pages/Platform/Overview/components/Side/SearchFacet.less +++ b/web/src/pages/Platform/Overview/components/Side/SearchFacet.less @@ -11,9 +11,21 @@ .value { display: flex; align-items: center; + gap: 8px; + min-width: 0; :global { label.ant-checkbox-wrapper { - width: 152px; + display: flex; + align-items: center; + flex: 1 1 auto; + min-width: 0; + } + label.ant-checkbox-wrapper > span:first-child { + flex: 0 0 auto; + } + label.ant-checkbox-wrapper > span:last-child { + flex: 1 1 auto; + min-width: 0; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; @@ -21,9 +33,11 @@ } .count { margin-left: auto; + flex: 0 0 auto; color: #888; font-size: 0.85em; text-align: right; + white-space: nowrap; } } } diff --git a/web/src/pages/Platform/Overview/components/TopN/index.jsx b/web/src/pages/Platform/Overview/components/TopN/index.jsx index f8bf6706..958f6c0c 100644 --- a/web/src/pages/Platform/Overview/components/TopN/index.jsx +++ b/web/src/pages/Platform/Overview/components/TopN/index.jsx @@ -594,13 +594,15 @@ export default (props) => {
    { result?.request && ( - - -
    message.success(formatMessage({id: "cluster.metrics.request.copy.success"}))}> - -
    -     -     +
    + + +
    message.success(formatMessage({id: "cluster.metrics.request.copy.success"}))}> + +
    +     +     +
    ) } { isTreemap ? :
    } @@ -608,4 +610,4 @@ export default (props) => { ) -} \ No newline at end of file +} diff --git a/web/src/pages/Platform/Overview/components/TopN/index.less b/web/src/pages/Platform/Overview/components/TopN/index.less index c80fe809..4efb68fc 100644 --- a/web/src/pages/Platform/Overview/components/TopN/index.less +++ b/web/src/pages/Platform/Overview/components/TopN/index.less @@ -49,21 +49,29 @@ height: calc(100vh - 500px); min-height: 500px; position: relative; + overflow: hidden; + + .copyAction { + position: absolute; + right: 12px; + bottom: 12px; + z-index: 11; + opacity: 0; + visibility: hidden; + pointer-events: none; + transition: opacity 0.2s ease, visibility 0.2s ease; + } .info { - position: absolute; - display: none; - right: 2px; - top: 2px; + display: flex; + align-items: center; + justify-content: center; width: 24px; height: 24px; border-radius: 4px; - line-height: 24px; - text-align: center; font-size: 16px; box-shadow: rgba(0, 0, 0, 0.16) 0px 0px 5px 0px; background: #fff; - z-index: 11; cursor: pointer; color: rgba(0, 0, 0, 0.45); transition: all 0.3s ease; @@ -75,8 +83,10 @@ } &:hover { - .info { - display: block; + .copyAction { + opacity: 1; + visibility: visible; + pointer-events: auto; } } } @@ -96,4 +106,4 @@ border-bottom-right-radius: 4px !important; } } -} \ No newline at end of file +} diff --git a/web/src/pages/Platform/Overview/components/index_metric.jsx b/web/src/pages/Platform/Overview/components/index_metric.jsx index 30a26e67..e388298a 100644 --- a/web/src/pages/Platform/Overview/components/index_metric.jsx +++ b/web/src/pages/Platform/Overview/components/index_metric.jsx @@ -11,6 +11,62 @@ import Anchor from "@/components/Anchor"; import MetricChart from "./MetricChart"; import { createRef, useEffect, useMemo, useState } from "react"; +const normalizeIndexSelection = (value, indices = []) => { + if (Array.isArray(value)) { + return value; + } + if (!value) { + return []; + } + if (typeof value === "string") { + return indices.find((item) => item?.index === value) ? indices.filter((item) => item?.index === value) : [{ index: value }]; + } + return [value]; +}; + +const extractIndexNames = (value) => { + if (Array.isArray(value)) { + return value + .map((item) => (typeof item === "string" ? item : item?.index)) + .filter(Boolean); + } + if (typeof value === "string") { + return [value]; + } + if (value?.index) { + return [value.index]; + } + return []; +}; + +const INDEX_GROUP_FALLBACK_METRICS = { + operations: "indexing_rate", + latency: "indexing_latency", + storage: "index_storage", + document: "doc_count", + memory: "segment_memory", + cache: "query_cache", +}; + +const buildChartEntries = (groupKey, metricKeys = []) => { + const normalizedMetricKeys = metricKeys.filter(Boolean); + const entries = normalizedMetricKeys.map((metricKey, index) => ({ + metricKey, + chartId: `${metricKey}__${index}`, + })); + if (normalizedMetricKeys.length % 2 === 1 && normalizedMetricKeys.length > 0) { + const preferredMetric = INDEX_GROUP_FALLBACK_METRICS[groupKey]; + const metricKey = normalizedMetricKeys.includes(preferredMetric) + ? preferredMetric + : normalizedMetricKeys[0]; + entries.push({ + metricKey, + chartId: `${metricKey}__extra`, + }); + } + return entries; +}; + export default (props) => { const { @@ -45,12 +101,11 @@ export default (props) => { }; const indexValueChange = (value) => { - const indexNames = value.map(item=>item.index); setParam((param) => { delete param["top"]; return { ...param, - index_name: indexNames, + index_name: value, }; }); }; @@ -62,8 +117,9 @@ export default (props) => { if (param.top) { newParams.top = param.top; } - if (param.index_name) { - newParams.index_name = param.index_name; + const indexNames = extractIndexNames(param.index_name); + if (indexNames.length > 0) { + newParams.index_name = indexNames; } if (bucketSize) { newParams.bucket_size = bucketSize @@ -79,6 +135,7 @@ export default (props) => { const formatedIndices = useMemo(() => { return Object.values(indices || []); }, [indices]); + const selectedIndices = useMemo(() => normalizeIndexSelection(param.index_name, formatedIndices), [param.index_name, formatedIndices]); const [charts, setCharts] = useState([]) @@ -87,8 +144,8 @@ export default (props) => { const cs = {} metrics.forEach((item) => { if (item[1]?.length > 0) { - item[1].forEach((metricKey) => { - cs[metricKey] = createRef() + buildChartEntries(item[0], item[1]).forEach((entry) => { + cs[entry.chartId] = createRef() }) } }) @@ -123,6 +180,7 @@ export default (props) => { mode="multiple" placeholder="Select index" allowClear + value={selectedIndices} onChange={indexValueChange}/> @@ -135,6 +193,7 @@ export default (props) => {
    {metrics.filter((item) => !!item && !!item[1]).map((item) => { + const chartEntries = buildChartEntries(item[0], item[1] || []); return (
    { >
    { - item[1].map((metricKey) => ( - ( + { + if (Array.isArray(value)) { + return value; + } + if (!value) { + return []; + } + if (typeof value === "string") { + return nodes.find((item) => item?.host === value) + ? nodes.filter((item) => item?.host === value) + : [{ host: value }]; + } + return [value]; +}; + +const extractNodeNames = (value) => { + if (Array.isArray(value)) { + return value + .map((item) => (typeof item === "string" ? item : item?.host)) + .filter(Boolean); + } + if (typeof value === "string") { + return [value]; + } + if (value?.host) { + return [value.host]; + } + return []; +}; + +const NODE_GROUP_FALLBACK_METRICS = { + operations: "indexing_rate", + latency: "indexing_latency", + system: "cpu", + circuit_breaker: "parent_breaker", + io: "total_io_operations", + transport: "transport_rx_bytes", + storage: "segment_count", + document: "docs_count", + http: "http_connect_num", + JVM: "jvm_heap_used_percent", + memory: "segment_memory", + cache: "query_cache", +}; + +const buildChartEntries = (groupKey, metricKeys = []) => { + const normalizedMetricKeys = metricKeys.filter(Boolean); + const entries = normalizedMetricKeys.map((metricKey, index) => ({ + metricKey, + chartId: `${metricKey}__${index}`, + })); + if (normalizedMetricKeys.length % 2 === 1 && normalizedMetricKeys.length > 0) { + const preferredMetric = NODE_GROUP_FALLBACK_METRICS[groupKey]; + const metricKey = normalizedMetricKeys.includes(preferredMetric) + ? preferredMetric + : normalizedMetricKeys[0]; + entries.push({ + metricKey, + chartId: `${metricKey}__extra`, + }); + } + return entries; +}; + export default (props) => { const { @@ -51,12 +115,11 @@ export default (props) => { const nodeValueChange = useCallback( (value) => { - const nodeNames = value.map(item=>item.host); setParam((param) => { delete param["top"]; return { ...param, - node_name: nodeNames, + node_name: value, }; }); }, @@ -74,8 +137,9 @@ export default (props) => { if (param.top) { newParams.top = param.top; } - if (param.node_name) { - newParams.node_name = param.node_name; + const nodeNames = extractNodeNames(param.node_name); + if (nodeNames.length > 0) { + newParams.node_name = nodeNames; } if (bucketSize) { newParams.bucket_size = bucketSize @@ -83,7 +147,7 @@ export default (props) => { return newParams; }, [param, timeRange, bucketSize]); - const formatedNodes = React.useMemo(() => { + const formatedNodes = useMemo(() => { if (!nodes) { return []; } @@ -94,6 +158,10 @@ export default (props) => { } }); }, [nodes]); + const selectedNodes = useMemo( + () => normalizeNodeSelection(param.node_name, formatedNodes), + [param.node_name, formatedNodes] + ); const [charts, setCharts] = useState([]) @@ -102,8 +170,8 @@ export default (props) => { const cs = {} metrics.forEach((item) => { if (item[1]?.length > 0) { - item[1].forEach((metricKey) => { - cs[metricKey] = createRef() + buildChartEntries(item[0], item[1]).forEach((entry) => { + cs[entry.chartId] = createRef() }) } }) @@ -146,6 +214,7 @@ export default (props) => { mode="multiple" placeholder="Select node" allowClear + value={selectedNodes} onChange={nodeValueChange}/>
    @@ -158,6 +227,7 @@ export default (props) => {
    {metrics.filter((item) => !!item && !!item[1]).map((item) => { + const chartEntries = buildChartEntries(item[0], item[1] || []); return (
    { >
    { - item[1].map((metricKey) => ( - ( + {
    { - setParam((param) => { - return { - tab: key, - }; + setParam({ + ...(param || {}), + tab: key, }); }} destroyInactiveTabPane @@ -99,7 +98,7 @@ const NewOverview = (props) => { tab={ <> - {`${pane.title}(${pane.count})`} + {`${formatMessage({ id: pane.titleId })}(${pane.count})`} } key={pane.key} diff --git a/web/src/pages/Profile/AdvancedProfile.js b/web/src/pages/Profile/AdvancedProfile.js index 0006f993..d59849cb 100644 --- a/web/src/pages/Profile/AdvancedProfile.js +++ b/web/src/pages/Profile/AdvancedProfile.js @@ -37,7 +37,7 @@ const menu = ( ); const action = ( - + @@ -90,7 +90,7 @@ const tabList = [ const desc1 = (
    - + 曲丽丽 @@ -100,7 +100,7 @@ const desc1 = ( const desc2 = (
    - + 周毛毛 diff --git a/web/src/pages/Redirect.js b/web/src/pages/Redirect.js index 6c461470..a0a7d5c0 100644 --- a/web/src/pages/Redirect.js +++ b/web/src/pages/Redirect.js @@ -5,24 +5,31 @@ import { router } from "umi"; import { getSetupRequired } from "@/utils/setup"; export default (props) => { - const userAuthority = getAuthority(); - const { menuData } = useGlobal(); + const { menuData, authResolved, sessionValid } = useGlobal() || {}; useEffect(() => { if (getSetupRequired() === 'true') { - router.push("/guide/initialization"); + router.replace("/guide/initialization"); + return; } if (getAuthEnabled() === "true") { + if (!authResolved) { + return; + } + if (!sessionValid) { + router.replace("/user/login"); + return; + } //find authoriy page - const tpath = findFirstAuthorityPath(menuData, userAuthority); + const tpath = findFirstAuthorityPath(menuData, getAuthority()); if (tpath) { - router.push(tpath); + router.replace(tpath); } else { - router.push("/user/login"); + router.replace("/exception/403"); } } else { - router.push("/cluster/overview"); + router.replace("/cluster/overview"); } - }, []); + }, [authResolved, menuData, sessionValid]); return null; }; diff --git a/web/src/pages/Result/Error.js b/web/src/pages/Result/Error.js index fba2da6d..0e04c5aa 100644 --- a/web/src/pages/Result/Error.js +++ b/web/src/pages/Result/Error.js @@ -5,7 +5,7 @@ import Result from '@/components/Result'; import PageHeaderWrapper from '@/components/PageHeaderWrapper'; const extra = ( - +
    +
    + diff --git a/web/src/pages/SearchManage/alias/AliasManage.js b/web/src/pages/SearchManage/alias/AliasManage.js index 0c829730..15bc7c85 100644 --- a/web/src/pages/SearchManage/alias/AliasManage.js +++ b/web/src/pages/SearchManage/alias/AliasManage.js @@ -147,7 +147,7 @@ class AliasManage extends PureComponent { { title: "操作", render: (text, record) => ( - + {/* this.handleUpdateModalVisible(true, record)}>别名设置*/} {/**/} +
    {this.renderForm()}
    @@ -408,16 +408,22 @@ class AliasIndexTable extends React.Component { { title: "索引路由", dataIndex: "index_routing", + render: (text) => { + return text || text === 0 ? text : "-"; + }, }, { title: "搜索路由", dataIndex: "search_routing", + render: (text) => { + return text || text === 0 ? text : "-"; + }, }, { title: "过滤查询", dataIndex: "filter", render: (text) => { - return text ? JSON.stringify(text) : ""; + return text ? JSON.stringify(text) : "-"; }, }, { diff --git a/web/src/pages/SearchManage/alias/Param.js b/web/src/pages/SearchManage/alias/Param.js index a91e8ffd..3cdc372c 100644 --- a/web/src/pages/SearchManage/alias/Param.js +++ b/web/src/pages/SearchManage/alias/Param.js @@ -195,7 +195,7 @@ class Param extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>别名设置 { @@ -391,7 +391,7 @@ class Param extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/alias/Rule.js b/web/src/pages/SearchManage/alias/Rule.js index e25a4749..21510b8a 100644 --- a/web/src/pages/SearchManage/alias/Rule.js +++ b/web/src/pages/SearchManage/alias/Rule.js @@ -195,7 +195,7 @@ class rule extends PureComponent { { title: '操作', render: (text, record) => ( - +     this.handleUpdateModalVisible(true, record)}>设置规则 { @@ -391,7 +391,7 @@ class rule extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/analyzer/AnalyzerTest.js b/web/src/pages/SearchManage/analyzer/AnalyzerTest.js index 7bc49e9f..f76cad06 100644 --- a/web/src/pages/SearchManage/analyzer/AnalyzerTest.js +++ b/web/src/pages/SearchManage/analyzer/AnalyzerTest.js @@ -101,7 +101,7 @@ class AnalyzerTest extends Component { return ( - +
    diff --git a/web/src/pages/SearchManage/analyzer/Manage.js b/web/src/pages/SearchManage/analyzer/Manage.js index ea753f1b..792325f2 100644 --- a/web/src/pages/SearchManage/analyzer/Manage.js +++ b/web/src/pages/SearchManage/analyzer/Manage.js @@ -195,7 +195,7 @@ class Manage extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>设置 { @@ -391,7 +391,7 @@ class Manage extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/dict/Common.js b/web/src/pages/SearchManage/dict/Common.js index cfb54e0a..a80ebf6e 100644 --- a/web/src/pages/SearchManage/dict/Common.js +++ b/web/src/pages/SearchManage/dict/Common.js @@ -178,16 +178,16 @@ class Common extends PureComponent { { title: '操作', render: (text, record) => ( - +     this.handleUpdateModalVisible(true, record)}>修改 - - { - this.state.selectedRows.push(record); - this.handleDeleteClick(); -}}>删除 - -), -}, + + { + this.state.selectedRows.push(record); + this.handleDeleteClick(); + }}>删除 + + ), + }, ]; componentDidMount() { @@ -393,7 +393,7 @@ render() { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/models/alias.js b/web/src/pages/SearchManage/models/alias.js index ced24dbb..2b4105d6 100644 --- a/web/src/pages/SearchManage/models/alias.js +++ b/web/src/pages/SearchManage/models/alias.js @@ -1,6 +1,18 @@ import {getAliasList, doAlias } from '@/services/alias'; import {getIndices } from '@/services/indices'; +const normalizeAliasDetail = (item = {}) => { + const indexes = Array.isArray(item.indexes) ? item.indexes : []; + const explicitWriteIndex = indexes.find((index) => index?.is_write_index)?.index; + const fallbackWriteIndex = indexes.length === 1 ? indexes[0]?.index : undefined; + + return { + ...item, + indexes, + write_index: item.write_index || explicitWriteIndex || fallbackWriteIndex || "", + }; +}; + export default { namespace: 'alias', @@ -11,7 +23,7 @@ export default { const res = yield call(getAliasList, payload); let aliasList = []; for(let k in res){ - aliasList.push(res[k]); + aliasList.push(normalizeAliasDetail(res[k])); } yield put({ type: 'saveData', diff --git a/web/src/pages/SearchManage/nlp/Intention.js b/web/src/pages/SearchManage/nlp/Intention.js index 0f546f4f..a1508a0c 100644 --- a/web/src/pages/SearchManage/nlp/Intention.js +++ b/web/src/pages/SearchManage/nlp/Intention.js @@ -191,7 +191,7 @@ class Intention extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>设置 { @@ -387,7 +387,7 @@ class Intention extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/nlp/Knowledge.js b/web/src/pages/SearchManage/nlp/Knowledge.js index 3992a6ce..7534b5ee 100644 --- a/web/src/pages/SearchManage/nlp/Knowledge.js +++ b/web/src/pages/SearchManage/nlp/Knowledge.js @@ -191,7 +191,7 @@ class Knowledge extends PureComponent { { title: '操作', render: (text, record) => ( - +     this.handleUpdateModalVisible(true, record)}>设置 { @@ -387,7 +387,7 @@ class Knowledge extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/nlp/Query.js b/web/src/pages/SearchManage/nlp/Query.js index ddb14d45..d7139b8d 100644 --- a/web/src/pages/SearchManage/nlp/Query.js +++ b/web/src/pages/SearchManage/nlp/Query.js @@ -195,7 +195,7 @@ class Query extends PureComponent { { title: '操作', render: (text, record) => ( - +     this.handleUpdateModalVisible(true, record)}>设置 { @@ -391,7 +391,7 @@ class Query extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/nlp/Text.js b/web/src/pages/SearchManage/nlp/Text.js index 38f54c6b..46ea710d 100644 --- a/web/src/pages/SearchManage/nlp/Text.js +++ b/web/src/pages/SearchManage/nlp/Text.js @@ -195,7 +195,7 @@ class Text extends PureComponent { { title: '操作', render: (text, record) => ( - +     this.handleUpdateModalVisible(true, record)}>设置 { @@ -391,7 +391,7 @@ class Text extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/template/History.js b/web/src/pages/SearchManage/template/History.js index b6375a52..f3a15eb0 100644 --- a/web/src/pages/SearchManage/template/History.js +++ b/web/src/pages/SearchManage/template/History.js @@ -332,7 +332,7 @@ class History extends PureComponent { { title: '操作', render: (text, record) => ( - +     this.handleUpdateModalVisible(true, record)}>配置模板 订阅警报 @@ -589,7 +589,7 @@ class History extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/template/Param.js b/web/src/pages/SearchManage/template/Param.js index efa43dc5..62a09cd2 100644 --- a/web/src/pages/SearchManage/template/Param.js +++ b/web/src/pages/SearchManage/template/Param.js @@ -328,7 +328,7 @@ class Param extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>配置参数 订阅警报 @@ -585,7 +585,7 @@ class Param extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    diff --git a/web/src/pages/SearchManage/template/SearchTemplate.js b/web/src/pages/SearchManage/template/SearchTemplate.js index a754683c..a08193fd 100644 --- a/web/src/pages/SearchManage/template/SearchTemplate.js +++ b/web/src/pages/SearchManage/template/SearchTemplate.js @@ -137,7 +137,7 @@ class SearchTemplate extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>设置 { @@ -342,7 +342,7 @@ class SearchTemplate extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    diff --git a/web/src/pages/Synchronize/IngestPipeline.js b/web/src/pages/Synchronize/IngestPipeline.js index 396c141c..b146cf32 100644 --- a/web/src/pages/Synchronize/IngestPipeline.js +++ b/web/src/pages/Synchronize/IngestPipeline.js @@ -23,6 +23,7 @@ import { } from 'antd'; import StandardTable from '@/components/StandardTable'; import PageHeaderWrapper from '@/components/PageHeaderWrapper'; +import { formatMessage } from 'umi/locale'; import styles from '../List/TableList.less'; @@ -157,7 +158,7 @@ class IngestPipeline extends PureComponent { { title: '操作', render: (text, record) => ( - + this.handleUpdateModalVisible(true, record)}>编辑 { @@ -402,7 +403,7 @@ class IngestPipeline extends PureComponent { handleUpdate: this.handleUpdate, }; return ( - +
    {this.renderForm()}
    @@ -452,11 +453,11 @@ class IngestPipeline extends PureComponent { rules: [{ required: true }], })()} - + {form.getFieldDecorator('batch_delay', { initialValue: editingRecord.batch_delay, rules: [{ required: true }], - })()} + })()} {form.getFieldDecorator('queue_type', { diff --git a/web/src/pages/Synchronize/LogstashConfig.js b/web/src/pages/Synchronize/LogstashConfig.js index 363c4102..62354799 100644 --- a/web/src/pages/Synchronize/LogstashConfig.js +++ b/web/src/pages/Synchronize/LogstashConfig.js @@ -213,7 +213,7 @@ class LogstashConfig extends Component { ) }; return ( - + { + if (!value) { + callback(); + return; + } + const normalized = value.trim(); + const invalidPattern = /[A-Z\\/*?"<>|,#\s:]/; + const validPattern = /^(?![._+-])(?!\.{1,2}$)[a-z0-9._+-]+$/; + if (normalized !== normalized.toLowerCase() || invalidPattern.test(normalized) || !validPattern.test(normalized)) { + callback(formatMessage({ id: 'synchronize.rebuild.target_index.invalid' })); + return; + } + callback(); + } componentDidMount(){ const {dispatch} = this.props; dispatch({ @@ -165,12 +180,15 @@ class Rebuild extends Component { stepDom = (
    - + {getFieldDecorator('dest_index', { initialValue: configData.dest.index || '', - rules: [{ required: true, message: '请输入目标索引名称' }], + rules: [ + { required: true, message: formatMessage({ id: 'synchronize.rebuild.target_index.required' }) }, + { validator: this.validateTargetIndexName }, + ], })( - + )} diff --git a/web/src/pages/Synchronize/RebuildList.js b/web/src/pages/Synchronize/RebuildList.js index b90a4de8..f7e54278 100644 --- a/web/src/pages/Synchronize/RebuildList.js +++ b/web/src/pages/Synchronize/RebuildList.js @@ -107,14 +107,14 @@ class RebuildList extends React.Component { }, { title: formatMessage({ id: "table.field.actions" }), - render: (text, record) => ( -
    - this.handleDeleteClick(record)} - > - Delete - + render: (text, record) => ( +
    + this.handleDeleteClick(record)} + > + {formatMessage({ id: "form.button.delete" })} + {record.status == "FAILED" ? ( diff --git a/web/src/pages/System/Audit/index.jsx b/web/src/pages/System/Audit/index.jsx index 18af8c58..7f36f79a 100644 --- a/web/src/pages/System/Audit/index.jsx +++ b/web/src/pages/System/Audit/index.jsx @@ -1,5 +1,6 @@ import { useRef, useCallback, useState, useEffect } from "react"; import { Button, Dropdown, Icon, Menu, Modal, Drawer, Descriptions } from "antd"; +import { connect } from "dva"; import { formatMessage } from "umi/locale"; import { formatESSearchResult } from "@/lib/elasticsearch/util"; @@ -12,15 +13,41 @@ import ListView from "@/components/ListView"; import styles from './index.less'; import { getSystemClusterID } from "@/utils/setup"; -export default (props) => { +const AuditPage = (props) => { const listViewRef = useRef(null); + const { clusterList = [] } = props; const clusterID = getSystemClusterID(); const collectionName = "audit_log"; const timeField = "timestamp"; //timestamp + const [histogramState, setHistogramState] = useState({ + enable: true, + visible: false, + widget: {}, + }); const [visible, setVisible] = useState(false); const [selectedItem, setSelectedItem] = useState(); + const getResourceDisplayName = useCallback( + (item) => { + const resourceName = item?.metadata?.labels?.resource_name; + if (!resourceName) { + return "-"; + } + if (item?.metadata?.resource_type !== "cluster_management") { + return resourceName; + } + const matchedCluster = clusterList.find( + (cluster) => + cluster?.id === resourceName || + cluster?.cluster_uuid === resourceName || + cluster?.name === resourceName + ); + return matchedCluster?.name || resourceName; + }, + [clusterList] + ); + const formatTableData = (value) => { let dataNew = formatESSearchResult(value); @@ -138,6 +165,61 @@ export default (props) => { const selectedRows = listViewRef.current?.selectedRows.rows || [] downloadFile(selectedRows, 'text/plain', 'audit_log.json') } + + const initHistogramWidget = async () => { + let res = await request(`/collection/${collectionName}/metadata`); + let indexName = res?.metadata?.index_name || ""; + if (indexName) { + let widget = { + bucket_size: "auto", + is_stack: true, + format: { + type: "number", + pattern: "0.00a", + }, + legend: false, + series: [ + { + metric: { + formula: "a", + groups: [ + { + field: "metadata.log_type", + limit: 10, + }, + ], + items: [ + { + field: "*", + name: "a", + statistic: "count", + }, + ], + sort: [ + { + direction: "desc", + key: "_count", + }, + ], + }, + queries: { + cluster_id: clusterID, + indices: [indexName], + time_field: timeField, + }, + type: "date-histogram", + }, + ], + }; + setHistogramState((st) => ({ ...st, widget })); + } + }; + + useEffect(() => { + if (histogramState.enable) { + initHistogramWidget(); + } + }, []); return ( @@ -152,19 +234,24 @@ export default (props) => { }} defaultQueryParams={{ from: 0, - size: 10, - timeRange: { from: "now-7d", to: "now", timeField: timeField }, + size: 20, + timeRange: { from: "auto", to: "auto", timeField: timeField }, sort: [[timeField, "desc"]], }} dateTimeEnable={true} isRefreshPaused={true} sortEnable={true} sideEnable={true} - sideVisible={true} + sideVisible={false} sidePlacement="left" + datePickerContainerStyle={{ width: 320, maxWidth: "45vw", minWidth: 270 }} + histogramEnable={histogramState.enable} + histogramVisible={histogramState.visible} + histogramWidget={histogramState.widget} rowSelectionExtra={{ getExtra: (props) => [
    资源名称
    -
    {selectedItem.metadata.labels.resource_name || '-'}
    +
    {getResourceDisplayName(selectedItem)}
    操作
    @@ -230,3 +317,7 @@ export default (props) => { ); }; + +export default connect(({ global }) => ({ + clusterList: global.clusterList, +}))(AuditPage); diff --git a/web/src/pages/System/Cluster/AgentCredentialForm.jsx b/web/src/pages/System/Cluster/AgentCredentialForm.jsx index 06a2ed73..7f0f4074 100644 --- a/web/src/pages/System/Cluster/AgentCredentialForm.jsx +++ b/web/src/pages/System/Cluster/AgentCredentialForm.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useMemo, useState } from "react"; -import { Button, Divider, Form, Input, Select, Row, Col } from "antd"; +import { Button, Form, Icon, Input, Select, Tooltip } from "antd"; import { formatMessage } from "umi/locale"; import useFetch from "@/lib/hooks/use_fetch"; import { formatESSearchResult } from "@/lib/elasticsearch/util"; @@ -18,6 +18,13 @@ export default (props) => { } = props; useEffect(() => {}, [credentialRequired]); + const getInitialAgentCredentialValue = (value) => + value?.agent_credential_id + ? value.agent_credential_id + : value?.username + ? MANUAL_VALUE + : undefined; + const onTryConnect = async () => { const values = await props.form.validateFields((errors, values) => { if (errors?.credential_id) { @@ -54,9 +61,17 @@ export default (props) => { tryConnect(values); }; - const [isManual, setIsManual] = useState(); + const [selectedCredential, setSelectedCredential] = useState( + getInitialAgentCredentialValue(initialValue) + ); + const [isManual, setIsManual] = useState( + getInitialAgentCredentialValue(initialValue) === MANUAL_VALUE + ); + const canReadCredential = + hasAuthority("system.credential:all") || + hasAuthority("system.credential:read"); - const { loading, error, value, run } = useFetch( + const { loading, value, run } = useFetch( "/credential/_search", { queryParams: { @@ -69,26 +84,82 @@ export default (props) => { ); const onCredentialChange = (value) => { - if (value === "manual") { - setIsManual(true); - } else { - setIsManual(false); - } + setSelectedCredential(value); + setIsManual(value === MANUAL_VALUE); }; - const { data, total } = useMemo(() => { + const { data } = useMemo(() => { return formatESSearchResult(value); }, [value]); + const credentialActionsStyle = { + display: "flex", + alignItems: "center", + gap: 12, + }; + const credentialGroupStyle = { + display: "flex", + alignItems: "center", + flex: 1, + minWidth: 0, + }; + const credentialSelectWrapStyle = { + flex: 1, + minWidth: 0, + }; + const refreshButtonSize = 32; + const refreshButtonStyle = { + width: refreshButtonSize, + minWidth: refreshButtonSize, + height: refreshButtonSize, + padding: 0, + marginLeft: -1, + borderTopLeftRadius: 0, + borderBottomLeftRadius: 0, + zIndex: 1, + display: "flex", + alignItems: "center", + justifyContent: "center", + }; + const refreshButtonWrapStyle = { + display: "flex", + alignItems: "center", + flex: `0 0 ${refreshButtonSize}px`, + }; + + const credentialOptions = useMemo(() => { + const options = data.map((item) => ({ + id: item.id, + name: item.name, + })); + if ( + initialValue?.agent_credential_id && + !options.find((item) => item.id === initialValue.agent_credential_id) + ) { + options.unshift({ + id: initialValue.agent_credential_id, + name: initialValue.agent_credential_id, + }); + } + return options; + }, [data, initialValue?.agent_credential_id]); useEffect(() => { - setIsManual(props.isManual); - }, [props.isManual]); + const nextValue = getInitialAgentCredentialValue(initialValue); + setSelectedCredential(nextValue); + setIsManual(nextValue === MANUAL_VALUE); + }, [initialValue?.agent_credential_id, initialValue?.username]); + + useEffect(() => { + if (canReadCredential) { + run(); + } + }, [canReadCredential, run]); useEffect(() => { - if (hasAuthority('system.credential:all') || hasAuthority('system.credential:read')) { - run() + if (canReadCredential && initialValue?.agent_credential_id) { + run(); } - }, []) + }, [canReadCredential, initialValue?.agent_credential_id, run]); if (!needAuth) { return null; @@ -97,44 +168,74 @@ export default (props) => { return ( <> + {formatMessage({ + id: "cluster.regist.step.connect.label.agent_credential", + })} + + + + + } > - -
    - {getFieldDecorator("agent_credential_id", { - initialValue: initialValue?.agent_credential_id - ? initialValue?.agent_credential_id - : initialValue?.username - ? MANUAL_VALUE - : undefined, - rules: [ - { - required: credentialRequired, - message: formatMessage({ - id: "cluster.regist.form.verify.required.agent_credential", - }), - }, - ], - })( - - - )} - - + > + + {formatMessage({ + id: "cluster.regist.step.connect.credential.manual", + })} + + {credentialOptions.map((item) => ( + + {item.name} + + ))} + + )} + +
    + + +
    + + {selectedCredential ? ( - - + ) : null} + {isManual && ( <> diff --git a/web/src/pages/System/Cluster/CollectMode.jsx b/web/src/pages/System/Cluster/CollectMode.jsx index 0b172de8..3062edbb 100644 --- a/web/src/pages/System/Cluster/CollectMode.jsx +++ b/web/src/pages/System/Cluster/CollectMode.jsx @@ -30,7 +30,17 @@ export default (props) => { content: ( <>
    - {formatMessage({id: "cluster.manage.metric_collection_mode.confirm.message"}, { mode: value === "agent" ? "Agent" : "Agentless" })} + {formatMessage( + { id: "cluster.manage.metric_collection_mode.confirm.message" }, + { + mode: formatMessage({ + id: + value === "agent" + ? "cluster.manage.metric_collection_mode.option.agent" + : "cluster.manage.metric_collection_mode.option.agentless", + }), + } + )}
    {isAgentless && isLargeCluster && (
    @@ -51,11 +61,19 @@ export default (props) => { }); }} > - Agentless - Agent + + {formatMessage({ + id: "cluster.manage.metric_collection_mode.option.agentless", + })} + + + {formatMessage({ + id: "cluster.manage.metric_collection_mode.option.agent", + })} + )} ); -}; \ No newline at end of file +}; diff --git a/web/src/pages/System/Cluster/CredentialForm.jsx b/web/src/pages/System/Cluster/CredentialForm.jsx index b6abeb14..17c6a7c5 100644 --- a/web/src/pages/System/Cluster/CredentialForm.jsx +++ b/web/src/pages/System/Cluster/CredentialForm.jsx @@ -82,14 +82,14 @@ export default (props) => { ], })( )} diff --git a/web/src/pages/System/Cluster/Form.js b/web/src/pages/System/Cluster/Form.js index f5dd857b..2bb12795 100644 --- a/web/src/pages/System/Cluster/Form.js +++ b/web/src/pages/System/Cluster/Form.js @@ -8,7 +8,7 @@ import { Button, Switch, message, - Spin, Select, + Spin, Select, Tooltip, } from "antd"; import router from "umi/router"; @@ -20,12 +20,17 @@ import { formatMessage } from "umi/locale"; import TagEditor from "@/components/infini/TagEditor"; import MonitorConfigsForm from "./MonitorConfigsForm"; import MetadataConfigsForm from "./MetadataConfigsForm"; -import { formatConfigsValues } from "./utils"; +import { + formatConfigsValues, + getClusterProbePath, + getClusterConnectErrorMessageFromResponse, +} from "./utils"; import CredentialForm from "./CredentialForm"; import AgentCredentialForm from "./AgentCredentialForm"; import { MANUAL_VALUE } from "./steps"; import SearchEngines from "./components/SearchEngines"; import Providers from "./components/Providers"; +import { isValidEndpointHost, normalizeEndpointHosts } from "@/utils/utils"; import TrimSpaceInput from "@/components/TrimSpaceInput"; import CollectMode from "./CollectMode"; @@ -47,6 +52,7 @@ class ClusterForm extends React.Component { btnLoading: false, btnLoadingAgent: false, submitLoading: false, + showProbePath: !!getClusterProbePath(props.clusterConfig?.editValue), }; } @@ -99,6 +105,7 @@ class ClusterForm extends React.Component { isManual, collectMode, monitored: editValue?.hasOwnProperty("monitored") ? editValue?.monitored : false, + showProbePath: !!getClusterProbePath(editValue), }); } }); @@ -158,8 +165,10 @@ class ClusterForm extends React.Component { name: values.name, host: values.host, hosts: values.hosts, + probe_path: values.probe_path, + is_auth: this.state.needAuth === true, credential_id: - values.credential_id !== MANUAL_VALUE + this.state.needAuth === true && values.credential_id !== MANUAL_VALUE ? values.credential_id : undefined, basic_auth: { @@ -171,11 +180,14 @@ class ClusterForm extends React.Component { values.agent_credential_id !== MANUAL_VALUE && isAgentMode ? values.agent_credential_id : undefined, - agent_basic_auth: { - username: values.agent_username, - password: values.agent_password, - }, + agent_basic_auth: isAgentMode + ? { + username: values.agent_username, + password: values.agent_password, + } + : undefined, metric_collection_mode: values.metric_collection_mode || 'agentless', + agent_collection_interval: isAgentMode ? (values.agent_collection_interval || 0) : 0, description: values.description, enabled: values.enabled, @@ -273,6 +285,7 @@ class ClusterForm extends React.Component { hosts: values.hosts, schema: values.isTLS === true ? "https" : "http", + probe_path: values.probe_path, }; if (type === "agent") { newVals = { @@ -293,7 +306,9 @@ class ClusterForm extends React.Component { newVals = { ...newVals, ...{ + is_auth: this.state.needAuth === true, credential_id: + this.state.needAuth === true && values.credential_id !== MANUAL_VALUE ? values.credential_id : undefined, @@ -310,7 +325,7 @@ class ClusterForm extends React.Component { type: "clusterConfig/doTryConnect", payload: newVals, }); - if (res) { + if (res && !res.error) { message.success( formatMessage({ id: "app.message.connect.success", @@ -320,6 +335,13 @@ class ClusterForm extends React.Component { version: res.version, }); this.clusterUUID = res.cluster_uuid; + } else if (res?.error) { + message.error( + getClusterConnectErrorMessageFromResponse( + res, + "cluster.regist.try_connect.failed" + ) + ); } if (type === "agent") { this.setState({ btnLoadingAgent: false }); @@ -334,13 +356,24 @@ class ClusterForm extends React.Component { validateHostsRule = (rule, value, callback) => { let vals = value || []; for(let i = 0; i < vals.length; i++) { - if (!/^[\w\.\-_~%]+(\:\d+)?$/.test(vals[i])) { + if (!isValidEndpointHost(vals[i])) { return callback(formatMessage({ id: "cluster.regist.form.verify.valid.endpoint" })); } } // validation passed callback(); }; + validateProbePathRule = (rule, value, callback) => { + if (!value) { + callback(); + return; + } + if (!String(value).trim().startsWith("/")) { + callback(formatMessage({ id: "cluster.regist.form.verify.valid.probe_path" })); + return; + } + callback(); + }; render() { const { getFieldDecorator } = this.props.form; @@ -367,6 +400,15 @@ class ClusterForm extends React.Component { }, }; const { editValue, editMode } = this.props.clusterConfig; + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "resource", locale: "menu.resource" }, + { title: "cluster", locale: "menu.resource.cluster", href: "/resource/cluster" }, + { + title: "editCluster", + locale: editMode === "NEW" ? "menu.resource.registCluster" : "menu.resource.editCluster", + }, + ]; //add host value to hosts field if it's empty if(editValue.host){ if(!editValue.hosts){ @@ -378,7 +420,7 @@ class ClusterForm extends React.Component { } } return ( - + {getFieldDecorator("hosts", { initialValue: editValue.hosts, + normalize: (value) => normalizeEndpointHosts(value), rules: [ { validator: this.validateHostsRule, @@ -480,7 +523,11 @@ class ClusterForm extends React.Component { rules: [], })()} - + {getFieldDecorator("isTLS", { initialValue: editValue?.schema === "https", valuePropName: "checked", @@ -491,6 +538,47 @@ class ClusterForm extends React.Component { /> )} + + + + {this.state.showProbePath ? ( + + {getFieldDecorator("probe_path", { + initialValue: getClusterProbePath(editValue), + normalize: (value) => (value || "").trim(), + rules: [ + { + validator: this.validateProbePathRule, + }, + ], + })( + + )} + + ) : null} { this.state.collectMode === 'agent' && ( - + <> + + + {formatMessage({ id: "agent.instance.collection_interval.label" })} + {" "} + + } + > + {getFieldDecorator("agent_collection_interval", { + initialValue: editValue?.agent_collection_interval || null, + })( + value ? `${value} ${formatMessage({ id: "agent.instance.collection_interval.unit" })}` : ""} + parser={(value) => `${value || ""}`.replace(/[^\d]/g, "")} + style={{ width: 160 }} + /> + )} + + ) } { @@ -618,7 +730,11 @@ class ClusterForm extends React.Component { form={this.props.form} editValue={editValue} /> - + {getFieldDecorator("tags", { initialValue: editValue.tags, rules: [], diff --git a/web/src/pages/System/Cluster/MetadataConfigsForm.jsx b/web/src/pages/System/Cluster/MetadataConfigsForm.jsx index 5f349a22..808639cd 100644 --- a/web/src/pages/System/Cluster/MetadataConfigsForm.jsx +++ b/web/src/pages/System/Cluster/MetadataConfigsForm.jsx @@ -1,4 +1,4 @@ -import { Form, Input, InputNumber, Icon, Switch } from "antd"; +import { Form, Input, InputNumber, Icon, Switch, Tooltip } from "antd"; import { formatMessage } from "umi/locale"; const InputGroup = Input.Group; @@ -12,17 +12,45 @@ const configs = [ const MetadataConfigsForm = (props) => { const editValue = props.editValue; - const { getFieldDecorator } = props.form; + const enabledLabel = formatMessage({ + id: "cluster.manage.config_item.enabled", + }); + const intervalLabel = formatMessage({ + id: "cluster.manage.config_item.interval", + }); + const intervalUnit = formatMessage({ + id: "cluster.manage.config_item.interval.unit", + }); + const renderConfigLabel = (key) => ( + + {formatMessage({ + id: `cluster.manage.metadata_configs.${key}`, + })} + + + + + ); + + const getIntervalInitialValue = (value) => { + const normalized = `${value || ""}`.replace(/[^\d]/g, ""); + return normalized ? Number(normalized) : 10; + }; return ( <> {configs.map((item, i) => { return ( @@ -34,7 +62,7 @@ const MetadataConfigsForm = (props) => { paddingRight: 10, }} > - enabled + {enabledLabel} {getFieldDecorator(`metadata_configs.${item}.enabled`, { valuePropName: "checked", @@ -56,19 +84,20 @@ const MetadataConfigsForm = (props) => { paddingRight: 10, }} > - interval + {intervalLabel} {getFieldDecorator(`metadata_configs.${item}.interval`, { - initialValue: - editValue?.metadata_configs?.[item]?.interval || 10, + initialValue: getIntervalInitialValue( + editValue?.metadata_configs?.[item]?.interval + ), rules: [], })( `${value}s`} - parser={(value) => value.replace("s", "")} + formatter={(value) => `${value}${intervalUnit}`} + parser={(value) => `${value || ""}`.replace(/[^\d]/g, "")} /> )} diff --git a/web/src/pages/System/Cluster/MonitorConfigsForm.jsx b/web/src/pages/System/Cluster/MonitorConfigsForm.jsx index dc3a1214..c6800af2 100644 --- a/web/src/pages/System/Cluster/MonitorConfigsForm.jsx +++ b/web/src/pages/System/Cluster/MonitorConfigsForm.jsx @@ -1,4 +1,4 @@ -import { Form, Input, InputNumber, Icon, Switch } from "antd"; +import { Form, Input, InputNumber, Icon, Switch, Tooltip } from "antd"; import { formatMessage } from "umi/locale"; const InputGroup = Input.Group; @@ -12,8 +12,39 @@ const configs = [ const MonitorConfigsForm = (props) => { const editValue = props.editValue; - const { getFieldDecorator } = props.form; + const enabledLabel = formatMessage({ + id: "cluster.manage.config_item.enabled", + }); + const intervalLabel = formatMessage({ + id: "cluster.manage.config_item.interval", + }); + const intervalUnit = formatMessage({ + id: "cluster.manage.config_item.interval.unit", + }); + const renderConfigLabel = (key) => ( + + {formatMessage({ + id: `cluster.manage.monitor_configs.${key}`, + })} + + + + + ); + + const getIntervalInitialValue = (value) => { + const normalized = `${value || ""}`.replace(/[^\d]/g, ""); + return normalized ? Number(normalized) : 10; + }; + return (
    { return ( @@ -45,7 +74,7 @@ const MonitorConfigsForm = (props) => { paddingRight: 10, }} > - enabled + {enabledLabel} {getFieldDecorator(`monitor_configs.${item.key}.enabled`, { valuePropName: "checked", @@ -69,19 +98,20 @@ const MonitorConfigsForm = (props) => { paddingRight: 10, }} > - interval + {intervalLabel} {getFieldDecorator(`monitor_configs.${item.key}.interval`, { - initialValue: - editValue?.monitor_configs?.[item.key]?.interval || 10, + initialValue: getIntervalInitialValue( + editValue?.monitor_configs?.[item.key]?.interval + ), rules: [], })( `${value}s`} - parser={(value) => value.replace("s", "")} + formatter={(value) => `${value}${intervalUnit}`} + parser={(value) => `${value || ""}`.replace(/[^\d]/g, "")} /> )} diff --git a/web/src/pages/System/Cluster/Step.js b/web/src/pages/System/Cluster/Step.js index b134be21..381ea04b 100644 --- a/web/src/pages/System/Cluster/Step.js +++ b/web/src/pages/System/Cluster/Step.js @@ -1,4 +1,4 @@ -import { Form, Steps, Button, message, Spin, Card, Row, Col } from "antd"; +import { Alert, Form, Steps, Button, message, Spin, Card, Row, Col } from "antd"; import { connect } from "dva"; import { useState, useRef, useEffect } from "react"; import { InitialStep, ExtraStep, ResultStep, MANUAL_VALUE } from "./steps"; @@ -6,6 +6,10 @@ import PageHeaderWrapper from "@/components/PageHeaderWrapper"; import styles from "./step.less"; import { formatMessage } from "umi/locale"; import { formatConfigsValues } from "./utils"; +import { + getClusterConnectErrorMessageFromError, + getClusterConnectErrorMessageFromResponse, +} from "./utils"; import { Link } from "umi"; import { SearchEngines } from "@/lib/search_engines"; @@ -42,6 +46,7 @@ const ClusterStep = ({ dispatch, history, query }) => { isLoading: false, current: 0, }); + const [connectError, setConnectError] = useState(); const changeStep = (step) => { setState((st) => { return { @@ -65,6 +70,7 @@ const ClusterStep = ({ dispatch, history, query }) => { const createFormPromise = (type, formatPayload, callback) => { return new Promise((resolve, reject) => { setIsLoading(true); + setConnectError(undefined); formRef.current.validateFields((errors, values) => { if (errors) { resolve(false); @@ -82,12 +88,25 @@ const ClusterStep = ({ dispatch, history, query }) => { } resolve(true); } else { + setConnectError( + getClusterConnectErrorMessageFromResponse( + res, + "cluster.regist.try_connect.failed" + ) + ); resolve(false); setIsLoading(false); } }) - .catch((err) => { + .catch(async (err) => { + setConnectError( + await getClusterConnectErrorMessageFromError( + err, + "cluster.regist.try_connect.failed" + ) + ); setIsLoading(false); + resolve(false); }); }); }); @@ -103,12 +122,14 @@ const ClusterStep = ({ dispatch, history, query }) => { username: values.username, password: values.password, }, + is_auth: values.isAuth === true, hosts: values.hosts, credential_id: - values.credential_id !== MANUAL_VALUE + values.isAuth === true && values.credential_id !== MANUAL_VALUE ? values.credential_id : undefined, schema: values.isTLS === true ? "https" : "http", + probe_path: values.probe_path, }), (values, res) => { setClusterConfig({ @@ -135,37 +156,44 @@ const ClusterStep = ({ dispatch, history, query }) => { const metadata_configs_new = formatConfigsValues( values.metadata_configs ); + const isAgentMode = values.metric_collection_mode === "agent"; clusterConfig.location.region = clusterConfig.location.region || "default"; - const newVals = { - name: values.name, - version: clusterConfig.version, - distribution: clusterConfig.distribution, + const newVals = { + name: values.name, + version: clusterConfig.version, + distribution: clusterConfig.distribution, host: clusterConfig.host, hosts: clusterConfig.hosts, + probe_path: clusterConfig.probe_path, location: clusterConfig.location, + is_auth: clusterConfig.isAuth === true, credential_id: + clusterConfig.isAuth === true && clusterConfig.credential_id !== MANUAL_VALUE ? clusterConfig.credential_id : undefined, - basic_auth: { - username: clusterConfig.username || "", - password: clusterConfig.password || "", - }, - agent_credential_id: - values.agent_credential_id !== MANUAL_VALUE - ? values.agent_credential_id + basic_auth: { + username: clusterConfig.username || "", + password: clusterConfig.password || "", + }, + agent_credential_id: + values.agent_credential_id !== MANUAL_VALUE && isAgentMode + ? values.agent_credential_id + : undefined, + agent_basic_auth: isAgentMode + ? { + username: values.agent_username, + password: values.agent_password, + } : undefined, - agent_basic_auth: { - username: values.agent_username, - password: values.agent_password, - }, - description: values.description, - enabled: true, - monitored: values.monitored, - monitor_configs: monitor_configs_new, - metadata_configs: metadata_configs_new, - discovery: { + description: values.description, + enabled: true, + monitored: values.monitored, + metric_collection_mode: values.metric_collection_mode || "agentless", + monitor_configs: monitor_configs_new, + metadata_configs: metadata_configs_new, + discovery: { enabled: values.discovery.enabled, }, schema: clusterConfig.isTLS ? "https" : "http", @@ -241,6 +269,14 @@ const ClusterStep = ({ dispatch, history, query }) => { ))}
    {renderContent(current)}
    + {current === 0 && connectError ? ( + + ) : null} { } return ( - - {name} + + + + {name} ); }; diff --git a/web/src/pages/System/Cluster/components/ClusterName/index.scss b/web/src/pages/System/Cluster/components/ClusterName/index.scss index 220f223b..387e1bf7 100644 --- a/web/src/pages/System/Cluster/components/ClusterName/index.scss +++ b/web/src/pages/System/Cluster/components/ClusterName/index.scss @@ -1,7 +1,35 @@ .cluster-name-link { display: flex; align-items: center; - > span { - margin-left: 3px; + min-width: 0; + gap: 4px; + + &__icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + flex: 0 0 18px; + line-height: 1; + + :global(.anticon) { + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + line-height: 1; + vertical-align: top; + } } -} \ No newline at end of file + + &__text { + flex: 1 1 auto; + min-width: 0; + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } +} diff --git a/web/src/pages/System/Cluster/index.jsx b/web/src/pages/System/Cluster/index.jsx index 2ed1cec3..a6480e1d 100644 --- a/web/src/pages/System/Cluster/index.jsx +++ b/web/src/pages/System/Cluster/index.jsx @@ -1,12 +1,8 @@ import PageHeaderWrapper from "@/components/PageHeaderWrapper"; -import { Button, Dropdown, Icon, Menu, message, Modal } from "antd"; +import { Button, Dropdown, Icon, Menu, message, Modal, Tooltip } from "antd"; import { useCallback, - useEffect, - useMemo, - useState, useRef, - Fragment, } from "react"; import { formatMessage } from "umi/locale"; import { formatESSearchResult } from "@/lib/elasticsearch/util"; @@ -21,6 +17,27 @@ import { useGlobal } from "@/layouts/GlobalContext"; import { ESPrefix } from "@/services/common"; import { getSystemClusterID } from "@/utils/setup"; +// 统一的省略 + Tooltip 渲染 +const renderWithTooltip = (text, maxWidth = "100%") => ( + + + {text} + + +); + +// 让所有列表头不换行 +const noWrapHeaderCell = () => ({ style: { whiteSpace: "nowrap" } }); + export default (props) => { const ref = useRef(null); const [isLoading, setIsLoading] = React.useState(); @@ -38,6 +55,7 @@ export default (props) => { }, }); }, []); + const onEditClick = useCallback(async (record) => { dispatch({ type: "clusterConfig/saveData", @@ -47,82 +65,157 @@ export default (props) => { }, }); }, []); + const onDeleteClick = useCallback(async (id) => { return dispatch({ type: "clusterConfig/deleteCluster", - payload: { - id: id, - }, + payload: { id }, }).then((result) => { - if (result?.result == "deleted") { - message.success( - formatMessage({ - id: "app.message.delete.success", - }) - ); + if (result?.result === "deleted") { + message.success(formatMessage({ id: "app.message.delete.success" })); setIsLoading(true); setTimeout(() => { ref.current?.refresh(); }, 1000); } else { console.log("delete failed:", result); - message.success( - formatMessage({ - id: "app.message.delete.failed", - }) - ); + message.error(formatMessage({ id: "app.message.delete.failed" })); } }); }, []); + const onMonitorClick = useCallback(async (ids, actionKey) => { + let actionUrl = ""; + if (actionKey === "enable") { + actionUrl = `${ESPrefix}/_enable`; + } else if (actionKey === "disable") { + actionUrl = `${ESPrefix}/_disable`; + } + if (!actionUrl || !(ids instanceof Array)) { + message.warn(formatMessage({ id: "app.message.warning.invalid.params" })); + return; + } + + setIsLoading(true); + const res = await request(actionUrl, { + method: "POST", + body: ids, + }); + + if (res?.acknowledged) { + message.success(formatMessage({ id: "app.message.operate.success" })); + ref.current?.refresh(); + dispatch({ + type: "global/fetchClusterStatus", + }); + } else { + message.error( + res?.message || + res?.error || + formatMessage({ id: "app.message.operate.failed" }) + ); + } + setIsLoading(false); + }, []); + const showDeleteConfirm = useCallback((record) => { Modal.confirm({ - title: "Are you sure delete this item?", + title: formatMessage({ id: "cluster.manage.delete.confirm.title" }), content: ( <> -
    Cluster: {record.name}
    -
    Version: {record.version}
    -
    Endpoint: {record.endpoint}
    +
    + {formatMessage( + { id: "cluster.manage.delete.confirm.cluster" }, + { name: record.name } + )} +
    +
    + {formatMessage( + { id: "cluster.manage.delete.confirm.version" }, + { version: record.version } + )} +
    +
    + {formatMessage( + { id: "cluster.manage.delete.confirm.endpoint" }, + { endpoint: record.endpoint } + )} +
    ), - okText: "Yes", + okText: formatMessage({ id: "form.button.ok" }), okType: "danger", - cancelText: "No", + cancelText: formatMessage({ id: "form.button.cancel" }), onOk() { onDeleteClick(record.id); }, }); }, []); + const showMonitorConfirm = useCallback((record, actionKey) => { + const isEnable = actionKey === "enable"; + Modal.confirm({ + title: formatMessage({ + id: isEnable + ? "cluster.manage.monitoring.confirm.enable.title" + : "cluster.manage.monitoring.confirm.disable.title", + }), + content: ( + <> +
    + {formatMessage( + { id: "cluster.manage.monitoring.confirm.cluster" }, + { name: record.name } + )} +
    +
    + {formatMessage( + { id: "cluster.manage.monitoring.confirm.version" }, + { version: record.version } + )} +
    +
    + {formatMessage( + { id: "cluster.manage.monitoring.confirm.endpoint" }, + { endpoint: record.endpoint } + )} +
    + + ), + okText: formatMessage({ id: "form.button.ok" }), + cancelText: formatMessage({ id: "form.button.cancel" }), + onOk() { + onMonitorClick([record.id], actionKey); + }, + }); + }, []); + const onClean = async (type) => { - setIsLoading(true) + setIsLoading(true); const res = await request(`${ESPrefix}/metadata/${type}`, { - method: 'DELETE' - }) + method: "DELETE", + }); if (res?.acknowledged) { - message.success(formatMessage({ id: "app.message.operate.success"})) + message.success(formatMessage({ id: "app.message.operate.success" })); } - setIsLoading(false) - } + setIsLoading(false); + }; const showCleanConfirm = (type) => { - let title - if (type === 'node') { - title = formatMessage({ id: "form.button.clean.unavailable.nodes.desc" }) - } else if (type === 'index') { - title = formatMessage({ id: "form.button.clean.deleted.indices.desc" }) - } + const titleId = + type === "node" + ? "form.button.clean.unavailable.nodes.desc" + : "form.button.clean.deleted.indices.desc"; Modal.confirm({ - title, + title: formatMessage({ id: titleId }), onOk() { - onClean(type) + onClean(type); }, }); }; - const formatTableData = async (value) => { + const formatTableData = async (value) => { let dataNew = formatESSearchResult(value); - //更新扩展数据 let tableData = dataNew?.data?.map((item) => { item.number_of_nodes = clusterStatus?.[item.id]?.health?.number_of_nodes || 0; @@ -135,125 +228,161 @@ export default (props) => { const columns = [ { title: "Distribution", + width: 150, key: "distribution", aggregable: true, visible: false, + onHeaderCell: noWrapHeaderCell, }, { - title: formatMessage({ - id: "cluster.manage.table.column.name", - }), + title: formatMessage({ id: "cluster.manage.table.column.name" }), key: "name", + width: 180, + fixed: "left", sortable: true, searchable: true, - render: (text, record) => { - return ( - - ); - }, + onHeaderCell: noWrapHeaderCell, + onCell: () => ({ + style: { + verticalAlign: "middle", + }, + }), + render: (text, record) => ( + +
    + +
    +     + ), }, { - title: formatMessage({ - id: "cluster.manage.table.column.health", - }), + title: formatMessage({ id: "cluster.manage.table.column.health" }), + width: 100, key: "labels.health_status", sortable: true, searchable: true, aggregable: true, - render: (text, record) => { - return ; - }, + onHeaderCell: noWrapHeaderCell, + render: (text) => , }, { - title: formatMessage({ - id: "cluster.manage.table.column.version", - }), + title: formatMessage({ id: "cluster.manage.table.column.version" }), + width: 100, key: "version", sortable: true, searchable: true, aggregable: true, + ellipsis: true, + onHeaderCell: noWrapHeaderCell, + render: renderWithTooltip, }, { - title: formatMessage({ - id: "cluster.manage.table.column.node_count", - }), + title: formatMessage({ id: "cluster.manage.table.column.node_count" }), + width: 80, key: "number_of_nodes", + onHeaderCell: noWrapHeaderCell, }, { - title: formatMessage({ - id: "cluster.manage.table.column.endpoint", - }), + title: formatMessage({ id: "cluster.manage.table.column.endpoint" }), key: "endpoint", + width: 200, sortable: true, searchable: true, + ellipsis: true, + onHeaderCell: noWrapHeaderCell, + render: (text) => renderWithTooltip(text, 180), }, { - title: formatMessage({ - id: "cluster.manage.table.column.monitored", - }), + title: formatMessage({ id: "cluster.manage.table.column.monitor_toggle" }), + width: 120, key: "monitored", sortable: true, - render: (text, record) => { - return formatMessage({ - id: text - ? "cluster.manage.monitored.on" - : "cluster.manage.monitored.off", - }); - }, + onHeaderCell: noWrapHeaderCell, + render: (text) => + formatMessage({ + id: text ? "cluster.manage.monitored.on" : "cluster.manage.monitored.off", + }), }, { - title: formatMessage({ - id: "cluster.manage.table.column.monitor_mode", - }), + title: formatMessage({ id: "cluster.manage.table.column.monitor_mode" }), + width: 150, key: "metric_collection_mode", sortable: false, - render: (text) => { - return text === "agent" ? "Agent" : "Agentless"; - }, + onHeaderCell: noWrapHeaderCell, + render: (text) => + formatMessage({ + id: + text === "agent" + ? "cluster.manage.metric_collection_mode.option.agent" + : "cluster.manage.metric_collection_mode.option.agentless", + }), }, { - title: formatMessage({ - id: "cluster.manage.table.column.discovery.enabled", - }), + title: formatMessage({ id: "cluster.manage.table.column.discovery.enabled" }), + width: 150, sortable: true, key: "discovery.enabled", - render: (text, record) => { - return formatMessage({ - id: text - ? "cluster.manage.monitored.on" - : "cluster.manage.monitored.off", - }); - }, + onHeaderCell: noWrapHeaderCell, + render: (text) => + formatMessage({ + id: text ? "cluster.manage.monitored.on" : "cluster.manage.monitored.off", + }), }, { title: formatMessage({ - id: "cluster.regist.step.connect.label.auth", + id: "cluster.manage.metadata_configs.node_availability_check", }), + width: 170, + sortable: true, + key: "metadata_configs.node_availability_check.enabled", + onHeaderCell: noWrapHeaderCell, + render: (text) => + formatMessage({ + id: text ? "cluster.manage.monitored.on" : "cluster.manage.monitored.off", + }), + }, + { + title: formatMessage({ id: "cluster.regist.step.connect.label.auth" }), + width: 120, key: "credential_id", - // aggregable: true, - render: (text, record) => { - return record.credential_id || record?.basic_auth?.username + onHeaderCell: noWrapHeaderCell, + render: (text, record) => + record.credential_id || record?.basic_auth?.username ? formatMessage({ id: "cluster.regist.step.complete.tls.yes" }) - : formatMessage({ id: "cluster.regist.step.complete.tls.no" }); - }, + : formatMessage({ id: "cluster.regist.step.complete.tls.no" }), }, { - title: formatMessage({ - id: "table.field.actions", - }), + title: formatMessage({ id: "table.field.actions" }), + width: 80, align: "center", + fixed: "right", // 操作列固定在右侧，不随横向滚动消失 + onHeaderCell: noWrapHeaderCell, render: (text, record) => { const onMenuClick = ({ key }) => { switch (key) { + case "enable_monitoring": + showMonitorConfirm(record, "enable"); + break; + case "disable_monitoring": + showMonitorConfirm(record, "disable"); + break; case "clean_nodes": - showCleanConfirm('node'); + showCleanConfirm("node"); break; case "clean_indices": - showCleanConfirm('index'); + showCleanConfirm("index"); break; case "delete": showDeleteConfirm(record); @@ -268,14 +397,20 @@ export default (props) => { content: ( { - onEditClick(record); - }} + onClick={() => onEditClick(record)} > {formatMessage({ id: "form.button.edit" })} ), }); + menuItems.push({ + key: record.monitored ? "disable_monitoring" : "enable_monitoring", + content: formatMessage({ + id: record.monitored + ? "cluster.manage.monitoring.disable.action" + : "cluster.manage.monitoring.enable.action", + }), + }); menuItems.push({ key: "clean_nodes", content: formatMessage({ id: "form.button.clean.unavailable.nodes" }), @@ -294,61 +429,62 @@ export default (props) => { const menu = ( - {menuItems.map((item) => { - return {item.content}; - })} + {menuItems.map((item) => ( + {item.content} + ))} ); + return ( - + + e.preventDefault()}> + + + ); }, }, ]; if (!hasAuthority("system.cluster:all")) { - columns.splice(columns.length - 1) + columns.splice(columns.length - 1); } return ( - { - return formatTableData(value); - }} - defaultQueryParams={{ - from: 0, - size: 20, - }} - sortEnable={true} - sideEnable={true} - sideVisible={false} - sidePlacement="left" - headerToobarExtra={{ - getExtra: (props) => [ - hasAuthority("system.cluster:all") ? ( - - - - ) : null, - ], - }} - /> +
    + formatTableData(value)} + defaultQueryParams={{ + from: 0, + size: 20, + sort: [ + ["name", "asc"], + ["id", "asc"], + ], + }} + sortEnable={true} + sideEnable={true} + sideVisible={false} + sidePlacement="left" + scroll={{ x: "max-content" }} // 超出横向滚动，避免撑破布局 + headerToobarExtra={{ + getExtra: () => [ + hasAuthority("system.cluster:all") ? ( + + + + ) : null, + ], + }} + /> +
    ); }; diff --git a/web/src/pages/System/Cluster/models/cluster.js b/web/src/pages/System/Cluster/models/cluster.js index 0e1d87fc..4ca9f887 100644 --- a/web/src/pages/System/Cluster/models/cluster.js +++ b/web/src/pages/System/Cluster/models/cluster.js @@ -150,8 +150,8 @@ export default { }, *doTryConnect({ payload }, { call, put, select }) { let res = yield call(tryConnect, payload); - if (res.error) { - return false; + if (!res || res.error) { + return res; } yield put({ diff --git a/web/src/pages/System/Cluster/steps/extra_step.js b/web/src/pages/System/Cluster/steps/extra_step.js index b7ad87d9..e7973f1a 100644 --- a/web/src/pages/System/Cluster/steps/extra_step.js +++ b/web/src/pages/System/Cluster/steps/extra_step.js @@ -6,6 +6,7 @@ import { InputNumber, Divider, Descriptions, + Tooltip, message, } from "antd"; import { HealthStatusView } from "@/components/infini/health_status_view"; @@ -15,7 +16,9 @@ import MonitorConfigsForm from "../MonitorConfigsForm"; import MetadataConfigsForm from "../MetadataConfigsForm"; import "../Form.scss"; import AgentCredentialForm from "../AgentCredentialForm"; +import CollectMode from "../CollectMode"; import { MANUAL_VALUE } from "./initial_step"; +import { getClusterConnectErrorMessageFromResponse } from "../utils"; @Form.create() export class ExtraStep extends React.Component { @@ -27,15 +30,25 @@ export class ExtraStep extends React.Component { agentCredentialRequired: false, isManual: false, needAuth: false, + collectMode: "agentless", }; } componentDidMount() { const { initialValue } = this.props - const needAuth = initialValue?.credential_id ? true : false + let collectMode = initialValue?.metric_collection_mode || "agentless"; + if ( + typeof initialValue?.metric_collection_mode === "undefined" && + initialValue?.monitor_configs?.node_stats?.enabled === false && + initialValue?.monitor_configs?.index_stats?.enabled === false + ) { + collectMode = "agent"; + } + const needAuth = !!(initialValue?.credential_id || initialValue?.username); this.setState({ monitored: initialValue?.monitored ?? true, needAuth, - isManual: needAuth ? !!initialValue?.username : false + isManual: !initialValue?.agent_credential_id && !!initialValue?.agent_username, + collectMode, }); } @@ -66,6 +79,7 @@ export class ExtraStep extends React.Component { let newVals = { hosts: initialValue?.hosts || [], schema: initialValue.isTLS === true ? "https" : "http", + probe_path: initialValue?.probe_path, }; newVals = { ...newVals, @@ -86,12 +100,19 @@ export class ExtraStep extends React.Component { type: "clusterConfig/doTryConnect", payload: newVals, }); - if (res) { + if (res && !res.error) { message.success( formatMessage({ id: "app.message.connect.success", }) ); + } else if (res?.error) { + message.error( + getClusterConnectErrorMessageFromResponse( + res, + "cluster.regist.try_connect.failed" + ) + ); } this.setState({ btnLoadingAgent: false }); } @@ -104,6 +125,12 @@ export class ExtraStep extends React.Component { form: { getFieldDecorator }, initialValue, } = this.props; + const endpointList = Array.isArray(initialValue?.hosts) && initialValue.hosts.length > 0 + ? initialValue.hosts + : initialValue?.host + ? [initialValue.host] + : []; + const endpointText = endpointList.length > 0 ? endpointList.join(", ") : "-"; const formItemLayout = { labelCol: { xs: { span: 24 }, @@ -122,12 +149,32 @@ export class ExtraStep extends React.Component { id: "cluster.manage.table.column.endpoint", })} > - {initialValue?.host} + +
    + {endpointText} +
    +     - + {initialValue?.isTLS ? ( - - ) : null} + <> + + {formatMessage({ id: "cluster.regist.step.complete.tls.yes" })} + + ) : ( + formatMessage({ id: "cluster.regist.step.complete.tls.no" }) + )} )}     - { + this.setState({ collectMode: mode, agentCredentialRequired: false }, () => { + const monitor_configs = this.props.form.getFieldValue("monitor_configs") || {}; + if (mode === "agent") { + monitor_configs.node_stats = { ...(monitor_configs.node_stats || {}), enabled: false }; + monitor_configs.index_stats = { ...(monitor_configs.index_stats || {}), enabled: false }; + } else { + monitor_configs.node_stats = { ...(monitor_configs.node_stats || {}), enabled: true }; + monitor_configs.index_stats = { ...(monitor_configs.index_stats || {}), enabled: true }; + } + this.props.form.setFieldsValue({ + metric_collection_mode: mode, + monitor_configs, + }); + }); }} - isManual={this.state.isManual} - isEdit={true} - tryConnect={this.tryConnect} - credentialRequired={this.state.agentCredentialRequired} /> + {this.state.collectMode === "agent" ? ( + <> + + + ) : null} - + {getFieldDecorator("tags", { initialValue: initialValue?.tags || ["default"], rules: [], diff --git a/web/src/pages/System/Cluster/steps/initial_step.js b/web/src/pages/System/Cluster/steps/initial_step.js index 129b3ea8..ad9bc7be 100644 --- a/web/src/pages/System/Cluster/steps/initial_step.js +++ b/web/src/pages/System/Cluster/steps/initial_step.js @@ -1,10 +1,14 @@ -import { Form, Input, Switch, Icon, Select } from "antd"; +import { Form, Input, Switch, Icon, Select, Button } from "antd"; import { formatMessage } from "umi/locale"; import CredentialForm from "../CredentialForm"; import "../Form.scss"; import SearchEngines from "../components/SearchEngines"; import Providers from "../components/Providers"; -import { isTLS, removeHttpSchema } from "@/utils/utils"; +import { + isTLS, + isValidEndpointHost, + normalizeEndpointHosts, +} from "@/utils/utils"; export const MANUAL_VALUE = "manual"; @@ -13,9 +17,10 @@ export class InitialStep extends React.Component { constructor(props) { super(props); this.state = { - needAuth: props.initialValue?.isAuth !== undefined, + needAuth: props.initialValue?.isAuth === true, isManual: props.initialValue?.credential_id === MANUAL_VALUE, - isPageTLS: isTLS(props.initialValue?.host) + isPageTLS: isTLS(props.initialValue?.host), + showProbePath: !!props.initialValue?.probe_path, }; } handleAuthChange = (val) => { @@ -43,13 +48,24 @@ export class InitialStep extends React.Component { validateHostsRule = (rule, value, callback) => { let vals = value || []; for(let i = 0; i < vals.length; i++) { - if (!/^[\w\.\-_~%]+(\:\d+)?$/.test(vals[i])) { + if (!isValidEndpointHost(vals[i])) { return callback(formatMessage({ id: "cluster.regist.form.verify.valid.endpoint" })); } } // validation passed callback(); }; + validateProbePathRule = (rule, value, callback) => { + if (!value) { + callback(); + return; + } + if (!String(value).trim().startsWith("/")) { + callback(formatMessage({ id: "cluster.regist.form.verify.valid.probe_path" })); + return; + } + callback(); + }; render() { const { form: { getFieldDecorator }, @@ -107,7 +123,7 @@ export class InitialStep extends React.Component { {getFieldDecorator("hosts", { initialValue: initialValue?.hosts || [], normalize: (value) => { - return (value || []).map((v) => removeHttpSchema(v || "").trim()); + return normalizeEndpointHosts(value); }, validateTrigger: ["onChange", "onBlur"], rules: [ @@ -123,7 +139,11 @@ export class InitialStep extends React.Component { ], })( + )} + + ) : null} ( + +
    + {host} +
    +     +); + export const ResultStep = (props) => { const { clusterConfig, oneMoreClick, goToClusterList } = props; + const endpointList = Array.isArray(clusterConfig?.hosts) && clusterConfig.hosts.length > 0 + ? clusterConfig.hosts + : clusterConfig?.host + ? [clusterConfig.host] + : []; const information = (
    @@ -39,12 +52,17 @@ export const ResultStep = (props) => { ：
    - {clusterConfig?.hosts.map((host) =>
    {host}
    )} + {endpointList.length > 0 + ? endpointList.map((host) => renderEndpoint(host)) + : "-"} - TLS： + {formatMessage({ + id: "cluster.manage.field.tls.label", + })} + ： {formatMessage({ @@ -57,7 +75,7 @@ export const ResultStep = (props) => { ); const actions = ( - +
    .ant-table-content > .ant-table-body > table > .ant-table-tbody > tr > td) { + vertical-align: middle; + } + + :global(.ant-table-small > .ant-table-content > .ant-table-body > table > .ant-table-thead > tr > th) { + vertical-align: middle; + } +} diff --git a/web/src/pages/System/Credential/CredentialForm.jsx b/web/src/pages/System/Credential/CredentialForm.jsx index c9040a4e..109c4b1a 100644 --- a/web/src/pages/System/Credential/CredentialForm.jsx +++ b/web/src/pages/System/Credential/CredentialForm.jsx @@ -1,14 +1,6 @@ -import { Form, Input, Select, Button, Drawer, Tag, Icon } from "antd"; +import { Spin, Form, Input, Select, Button, Tag, Icon, Tooltip } from "antd"; import { formatMessage } from "umi/locale"; -import { - forwardRef, - useCallback, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import request from "@/utils/request"; +import { forwardRef, useEffect, useRef, useState } from "react"; const formItemLayout = { labelCol: { @@ -26,21 +18,46 @@ const TYPES = [ type: "basic_auth", name: "basic_auth", }, + { + type: "token", + name: "token", + }, ]; export default Form.create()((props) => { - const { form, record, onSubmit } = props; + const { form, record, onSubmit, submitLoading } = props; const { getFieldDecorator } = form; const { payload = {} } = record || {}; + const isEdit = !!record?.id; const [type, setType] = useState(record?.type); - const [loading, setLoading] = useState(false); - const handleSubmit = () => { + const getRequiredMessage = (id, defaultMessage) => + formatMessage({ id, defaultMessage }); + + const getRequiredRule = (id, defaultMessage) => ({ + validator: (_, value, callback) => { + if (typeof value === "string") { + if (value.trim() === "") { + callback(getRequiredMessage(id, defaultMessage)); + return; + } + } else if (value === undefined || value === null || value === "") { + callback(getRequiredMessage(id, defaultMessage)); + return; + } + callback(); + }, + }); + + const handleSubmit = async () => { + if (submitLoading) { + return; + } form.validateFields(async (err, values) => { if (err) return; - onSubmit(values); + await onSubmit(values); }); }; @@ -56,10 +73,10 @@ export default Form.create()((props) => { {getFieldDecorator("username", { initialValue: payload[type]?.username, rules: [ - { - required: true, - message: "Please inpurt username!", - }, + getRequiredRule( + "credential.manage.form.username.required", + "Please input username!" + ), ], })()} @@ -71,15 +88,24 @@ export default Form.create()((props) => { {getFieldDecorator("password", { initialValue: payload[type]?.password, rules: [ - { - required: record?.id ? false : true, - message: "Please inpurt password!", - }, + ...(isEdit + ? [] + : [ + getRequiredRule( + "credential.manage.form.password.required", + "Please input password!" + ), + ]), ], })( )} @@ -87,6 +113,37 @@ export default Form.create()((props) => { ); } + if (type === "token") { + return ( + + {getFieldDecorator("token_value", { + initialValue: payload[type]?.value, + rules: [ + ...(isEdit + ? [] + : [ + getRequiredRule( + "credential.manage.form.token.required", + "Please input token!" + ), + ]), + ], + })( + + )} + + ); + } }; useEffect(() => { @@ -94,63 +151,69 @@ export default Form.create()((props) => { }, [record?.type]); return ( - - - {getFieldDecorator("type", { - initialValue: record?.type, - rules: [ - { - required: true, - message: "Please select type!", - }, - ], - })( - - )} - - - {getFieldDecorator("name", { - initialValue: record?.name, - rules: [ - { - required: true, - message: "Please input name!", - }, - ], - })()} - - {renderAuth(type)} - - {getFieldDecorator("tags", { - initialValue: record?.tags || [], - })()} - - -
    - -
    -     - + +
    + + {getFieldDecorator("type", { + initialValue: record?.type, + rules: [ + getRequiredRule( + "credential.manage.form.type.required", + "Please select type!" + ), + ], + })( + + )} + + + {getFieldDecorator("name", { + initialValue: record?.name, + rules: [ + getRequiredRule( + "credential.manage.form.name.required", + "Please input name!" + ), + ], + })()} + + {renderAuth(type)} + + {getFieldDecorator("tags", { + initialValue: record?.tags || [], + })()} + + +
    + +
    +     + +     ); }); diff --git a/web/src/pages/System/Credential/Index.js b/web/src/pages/System/Credential/Index.js index 1ceb4f51..b95c4dfd 100644 --- a/web/src/pages/System/Credential/Index.js +++ b/web/src/pages/System/Credential/Index.js @@ -8,6 +8,7 @@ import { Popconfirm, Table, message, + Icon, } from "antd"; import PageHeaderWrapper from "@/components/PageHeaderWrapper"; import styles from "./Index.less"; @@ -21,12 +22,18 @@ import { hasAuthority } from "@/utils/authority"; import AutoTextEllipsis from "@/components/AutoTextEllipsis"; import commonStyles from "@/common.less" -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; + +const firstColumnIconStyle = { + marginRight: 8, + color: "#999", + fontSize: 12, +}; export default () => { const initialQueryParams = { from: 0, - size: 10, + size: 20, }; function reducer(queryParams, action) { @@ -58,6 +65,7 @@ export default () => { const [visible, setVisible] = useState(false); const [selectedItem, setSelectedItem] = useState(); + const [submitLoading, setSubmitLoading] = useState(false); const [queryParams, dispatch] = useReducer(reducer, initialQueryParams); @@ -97,14 +105,19 @@ export default () => { }; const onSubmit = async (value) => { - const { name, type, tags, username, password } = value; + if (submitLoading) { + return; + } + setSubmitLoading(true); + const { name, type, tags, username, password, token_value } = value; + const credentialType = selectedItem?.type || type; const body = { name, - type, + type: credentialType, tags, payload: {}, }; - if (type === "basic_auth") { + if (credentialType === "basic_auth") { body.payload = { basic_auth: { username, @@ -112,71 +125,105 @@ export default () => { }, }; } + if (credentialType === "token") { + body.payload = { + token: { + value: token_value, + }, + }; + } if (selectedItem) { - if (body.payload?.basic_auth?.password === "") { - delete body.payload.basic_auth["password"]; - } - const res = await request(`/credential/${selectedItem.id}`, { - method: "PUT", - body, - }); - if (res) { - if (res.result === "updated") { - message.success( + try { + if (body.payload?.basic_auth?.password === "") { + delete body.payload.basic_auth["password"]; + } + if (body.payload?.token?.value === "") { + delete body.payload.token["value"]; + } + const res = await request(`/credential/${selectedItem.id}`, { + method: "PUT", + body, + }); + if (res?.error) { + message.error(res.error.reason || formatMessage({ id: "app.message.update.failed" })); + return; + } + if (res) { + if (res.result === "updated") { + message.success( + formatMessage({ + id: "app.message.update.success", + }) + ); + setTimeout(() => { + setSelectedItem(); + setVisible(false); + run(); + }, 500); + } else if (res.result === "not_found") { + message.error(`Update failed: not found`); + } + } else { + console.log("Update failed: ", res); + message.error( formatMessage({ - id: "app.message.update.success", + id: "app.message.update.failed", }) ); - setTimeout(() => { - setSelectedItem(); - setVisible(false); - run(); - }, 500); - } else if (res.result === "not_found") { - message.error(`Update failed: not found`); } - } else { - console.log("Update failed: ", res); - message.error( - formatMessage({ - id: "app.message.update.failed", - }) - ); + } finally { + setSubmitLoading(false); } } else { - const res = await request(`/credential`, { - method: "POST", - body, - }); - if (res) { - if (res.result === "created") { - message.success( + try { + const res = await request(`/credential`, { + method: "POST", + body, + }); + if (res?.error) { + message.error(res.error.reason || formatMessage({ id: "app.message.create.failed" })); + return; + } + if (res) { + if (res.result === "created") { + message.success( + formatMessage({ + id: "app.message.create.success", + }) + ); + setTimeout(() => { + setSelectedItem(); + setVisible(false); + run(); + }, 500); + } + } else { + console.log("Create failed: ", res); + message.error( formatMessage({ - id: "app.message.create.success", + id: "app.message.create.failed", }) ); - setTimeout(() => { - setSelectedItem(); - setVisible(false); - run(); - }, 500); } - } else { - console.log("Create failed: ", res); - message.error( - formatMessage({ - id: "app.message.create.failed", - }) - ); + } finally { + setSubmitLoading(false); } } }; const columns = [ { - title: "ID", + title: formatMessage({ + id: "table.field.id", + }), dataIndex: "id", key: "id", + render: (text) => ( +
    + + {text} +
    + ), }, { title: formatMessage({ @@ -231,7 +278,7 @@ export default () => { }), width: 150, render: (text, record) => ( - + { setSelectedItem(record); @@ -244,7 +291,7 @@ export default () => { onRemove(record.id)} > @@ -279,7 +326,7 @@ export default () => { }} >
    - { { + if (submitLoading) { + return; + } setVisible(false); }} visible={visible} @@ -351,7 +401,7 @@ export default () => { })} destroyOnClose > - + ); diff --git a/web/src/pages/System/Email/Components/CredentialForm.jsx b/web/src/pages/System/Email/Components/CredentialForm.jsx index 254924c4..89b3f459 100644 --- a/web/src/pages/System/Email/Components/CredentialForm.jsx +++ b/web/src/pages/System/Email/Components/CredentialForm.jsx @@ -69,16 +69,16 @@ export default (props) => { loading={loading} onChange={onCredentialChange} > - { - data.map((item) => {item.name}) - } - + { formatMessage({ id: "cluster.regist.step.connect.credential.manual", }) } + { + data.map((item) => {item.name}) + } )} @@ -100,7 +100,9 @@ export default (props) => { }), }, ], - })()} + })()} { })( )} @@ -129,11 +133,11 @@ export default (props) => { isEdit && ( <> -
    + {formatMessage({ id: "cluster.regist.form.credential.manual.desc", })} -
    +     ) @@ -144,4 +148,4 @@ export default (props) => { ) -} \ No newline at end of file +} diff --git a/web/src/pages/System/Email/Components/ServerConfig.jsx b/web/src/pages/System/Email/Components/ServerConfig.jsx index 7c963fa8..619e2c08 100644 --- a/web/src/pages/System/Email/Components/ServerConfig.jsx +++ b/web/src/pages/System/Email/Components/ServerConfig.jsx @@ -9,9 +9,32 @@ import { message, } from "antd"; import { useCallback, useState } from "react"; +import { formatMessage } from "umi/locale"; import CredentialForm from "./CredentialForm"; import { hasAuthority } from "@/utils/authority"; +const getEmailServerTestErrorMessage = (response) => { + const genericMessage = formatMessage({ + id: "settings.email.server.message.test.error.send_failed", + }); + if (response instanceof Error) { + return genericMessage; + } + const errorKey = response?.error?.key; + if (errorKey) { + return formatMessage({ + id: errorKey, + defaultMessage: response?.error?.reason, + }); + } + if (response?.error?.reason) { + return genericMessage; + } + return formatMessage({ + id: "app.message.http.status.500", + }); +}; + export default Form.create({ name: "email_server_cfg" })((props) => { const { form, config = {}, setState, onSaveClick } = props; const { getFieldDecorator } = form; @@ -70,22 +93,37 @@ export default Form.create({ name: "email_server_cfg" })((props) => { const { sendTo } = values; const emailRegexp = /^[\w-]+(\.[\w-]+)*@([a-z0-9-]+\.)+[a-z]{2,}$/i; if (!sendTo || !emailRegexp.test(sendTo.trim())) { - message.error("Receive Email is invalid"); + message.error( + formatMessage({ + id: "settings.email.server.form.validation.recipient", + }) + ); return; } const send_to = [sendTo.trim()]; delete values.sendTo; - const testRes = await request("/email/server/_test", { - method: "POST", - body: { - ...config, - ...values, - send_to, + const testRes = await request( + "/email/server/_test", + { + method: "POST", + body: { + ...config, + ...values, + send_to, + }, }, - }); + false, + false + ); if (testRes && testRes.acknowledged === true) { - message.success("send succed"); + message.success( + formatMessage({ + id: "settings.email.server.message.test.success", + }) + ); + return; } + message.error(getEmailServerTestErrorMessage(testRes)); }); }; @@ -94,13 +132,19 @@ export default Form.create({ name: "email_server_cfg" })((props) => { return (
    - + {getFieldDecorator("name", { initialValue: config.name, rules: [ { required: true, - message: "Please input name!", + message: formatMessage({ + id: "settings.email.server.form.validation.name", + }), }, ], onChange: (ev) => { @@ -120,60 +164,102 @@ export default Form.create({ name: "email_server_cfg" })((props) => { }, })()} - + {getFieldDecorator("host", { initialValue: config.host, rules: [ { required: true, - message: "Please input smtp server host!", + message: formatMessage({ + id: "settings.email.server.form.validation.host", + }), }, ], })()} - + {getFieldDecorator("port", { initialValue: config.port, rules: [ { required: true, - message: "Please input smtp server port!", + message: formatMessage({ + id: "settings.email.server.form.validation.port", + }), }, ], })()} - + {getFieldDecorator("tls_min_version", { initialValue: config.tls_min_version, rules: [ ], })()} - + {getFieldDecorator("tls", { initialValue: config.tls, valuePropName: "checked", })()} - + + {getFieldDecorator("sender", { + initialValue: config.sender, + })()} + + {getFieldDecorator("enabled", { initialValue: config.enabled, valuePropName: "checked", })()} - +
    {getFieldDecorator( "sendTo", {} )( - + )}
    @@ -192,7 +280,9 @@ export default Form.create({ name: "email_server_cfg" })((props) => { loading={isLoading} disabled={hasAuthority("alerting.rule:all") ? false : true} > - Save + {formatMessage({ + id: "form.button.save", + })}     diff --git a/web/src/pages/System/Email/Server.jsx b/web/src/pages/System/Email/Server.jsx index ad1d38ff..6a9b39ef 100644 --- a/web/src/pages/System/Email/Server.jsx +++ b/web/src/pages/System/Email/Server.jsx @@ -9,7 +9,7 @@ import { formatESSearchResult } from '@/lib/elasticsearch/util'; import { formatMessage } from "umi/locale"; import EmptyServer from './Components/EmptyServer'; -const ServerList = ({})=>{ +const ServerList = ({ embedded = false })=>{ const [loading, setLoading] = useState(false); @@ -62,7 +62,11 @@ const ServerList = ({})=>{ method: "DELETE", }); if (deleteRes && deleteRes.result == "deleted") { - message.success("delete succed"); + message.success( + formatMessage({ + id: "app.message.delete.success", + }) + ); } } setState(st=>{ @@ -144,15 +148,19 @@ const ServerList = ({})=>{ }; const onEmptyAddClick = ()=>{ setState(st=>{ - const newCfg = {name:"New Config Name", id: "tmp_"+new Date().valueOf()}; + const newCfg = { + name: formatMessage({ + id: "settings.email.server.form.temp_name", + }), + id: "tmp_"+new Date().valueOf() + }; return { servers: [...(st.servers || []), newCfg], activeKey: newCfg.id, } }) } - return ( - + const content = ( {state.servers.length == 0 ? : { {cfg.name} onDeleteClick(cfg.id)} - > - + title={formatMessage({ + id: "app.message.confirm.delete", + })} + onConfirm={() => onDeleteClick(cfg.id)} + > + } key={cfg.id} closable={false} > @@ -182,8 +192,15 @@ const ServerList = ({})=>{ } key="add" closable={false} /> } + ); + if (embedded) { + return content; + } + return ( + + {content} ); } -export default ServerList; \ No newline at end of file +export default ServerList; diff --git a/web/src/pages/System/Role/Data/api_privilege_field.jsx b/web/src/pages/System/Role/Data/api_privilege_field.jsx index 967dc71b..c818481b 100644 --- a/web/src/pages/System/Role/Data/api_privilege_field.jsx +++ b/web/src/pages/System/Role/Data/api_privilege_field.jsx @@ -1,5 +1,6 @@ import { Button, Input, Select, Icon } from "antd"; import { useEffect, useMemo, useState } from "react"; +import { formatMessage } from "umi/locale"; const InputGroup = Input.Group; const Option = Select.Option; @@ -75,12 +76,16 @@ const ApiPrivilegeField = ({ value = [], onChange, options }) => { marginTop: 10, }} > -
    Category
    -
    Privilege
    +
    + {formatMessage({ id: "system.role.data.api_privilege.category" })} +
    +
    + {formatMessage({ id: "system.role.data.api_privilege.privilege" })} +
    {apiPrivilegeEl}
    ); diff --git a/web/src/pages/System/Role/Data/form.jsx b/web/src/pages/System/Role/Data/form.jsx index 019294a8..f73ec180 100644 --- a/web/src/pages/System/Role/Data/form.jsx +++ b/web/src/pages/System/Role/Data/form.jsx @@ -6,6 +6,7 @@ import { InputNumber, Card, Button, + Result, Row, Col, Transfer, @@ -49,6 +50,15 @@ const tailFormItemLayout = { const DataRoleForm = (props) => { const { getFieldDecorator } = props.form; const [isLoading, setIsLoading] = useState(false); + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "system", locale: "menu.system" }, + { title: "security", locale: "menu.system.security" }, + { + title: props.mode === "edit" ? "edit_role" : "new_role", + locale: props.mode === "edit" ? "menu.system.edit_role" : "menu.system.new_role", + }, + ]; const handleSubmit = useCallback( (e) => { @@ -144,7 +154,7 @@ const DataRoleForm = (props) => { }); return ( - + @@ -154,81 +164,131 @@ const DataRoleForm = (props) => { } > - -
    - - {getFieldDecorator("name", { - initialValue: editValue.name, - rules: [ - { - required: true, - message: "Please input name!", - }, - ], - })()} - - - {getFieldDecorator("privilege.elasticsearch.cluster.resources", { - initialValue: editValue.privilege?.elasticsearch?.cluster - ?.resources || [{ id: "*", name: "*" }], - rules: [ - { - required: true, - message: "Please select cluster!", - }, - ], - })( - - )} - - - {getFieldDecorator( - "privilege.elasticsearch.cluster.permissions", - { + {props.createResult ? ( + { + props.history.push( + `/system/security?_g=${encodeURIComponent( + JSON.stringify({ tab: "role" }) + )}` + ); + }} + > + {formatMessage({ + id: "system.security.role.create.button.view_list", + })} + , + , + ]} + /> + ) : ( + + + + {getFieldDecorator("name", { + initialValue: editValue.name, + rules: [ + { + required: true, + message: formatMessage({ + id: "system.role.platform.name.required", + }), + }, + ], + })()} + + + {getFieldDecorator("privilege.elasticsearch.cluster.resources", { initialValue: editValue.privilege?.elasticsearch?.cluster - ?.permissions || [{ "*": ["*"] }], + ?.resources || [{ id: "*", name: "*" }], rules: [ { required: true, - message: "Please select cluster privilege!", + message: formatMessage({ + id: "system.role.data.cluster.required", + }), }, ], - } - )()} - - - {getFieldDecorator("privilege.elasticsearch.index", { - initialValue: editValue.privilege?.elasticsearch?.index || [ - { name: ["*"], permissions: ["*"] }, - ], - rules: [ + })( + + )} + + + {getFieldDecorator( + "privilege.elasticsearch.cluster.permissions", { - required: true, - message: "Please select index privilege!", - }, - ], - })( - - )} - - - {getFieldDecorator("description", { - initialValue: editValue.description, - rules: [], - })()} - - - - - - + initialValue: editValue.privilege?.elasticsearch?.cluster + ?.permissions || [{ "*": ["*"] }], + rules: [ + { + required: true, + message: formatMessage({ + id: "system.role.data.cluster_privilege.required", + }), + }, + ], + } + )()} + + + {getFieldDecorator("privilege.elasticsearch.index", { + initialValue: editValue.privilege?.elasticsearch?.index || [ + { name: ["*"], permissions: ["*"] }, + ], + rules: [ + { + required: true, + message: formatMessage({ + id: "system.role.data.index_privilege.required", + }), + }, + ], + })( + + )} + + + {getFieldDecorator("description", { + initialValue: editValue.description, + rules: [], + })()} + + + + + +     + )}     ); diff --git a/web/src/pages/System/Role/Data/index_privilege_field.jsx b/web/src/pages/System/Role/Data/index_privilege_field.jsx index b984ff4d..3916f46e 100644 --- a/web/src/pages/System/Role/Data/index_privilege_field.jsx +++ b/web/src/pages/System/Role/Data/index_privilege_field.jsx @@ -4,6 +4,7 @@ import { Button, Input, Select, Icon } from "antd"; import { useCallback, useEffect, useMemo, useState } from "react"; import { DataRoleFromContext } from "./context"; import { ESPrefix } from "@/services/common"; +import { formatMessage } from "umi/locale"; const InputGroup = Input.Group; const Option = Select.Option; @@ -76,12 +77,16 @@ const IndexPrivilegeField = ({ privileges = [], value = [], onChange }) => { marginTop: 10, }} > -
    Index
    -
    Privilege
    +
    + {formatMessage({ id: "system.role.data.index_privilege.index" })} +
    +
    + {formatMessage({ id: "system.role.data.index_privilege.privilege" })} +
    {indexPrivilegeEl} ); diff --git a/web/src/pages/System/Role/Data/new.jsx b/web/src/pages/System/Role/Data/new.jsx index e43c97a5..fd3ee8e1 100644 --- a/web/src/pages/System/Role/Data/new.jsx +++ b/web/src/pages/System/Role/Data/new.jsx @@ -1,12 +1,12 @@ import DataRoleForm from "./form"; import { Form } from "antd"; -import { useCallback } from "react"; +import { useCallback, useState } from "react"; import request from "@/utils/request"; -import { message } from "antd"; import { router } from "umi"; import { formatMessage } from "umi/locale"; export default Form.create({ name: "data_role_form_new" })((props) => { + const [createResult, setCreateResult] = useState(null); const onSaveClick = useCallback(async (values) => { const saveRes = await request(`/role/elasticsearch`, { method: "POST", @@ -15,17 +15,18 @@ export default Form.create({ name: "data_role_form_new" })((props) => { }, }); if (saveRes && saveRes.result == "created") { - message.success( - formatMessage({ - id: "app.message.save.success", - }) - ); - props.form.resetFields(); + setCreateResult(saveRes); } }, []); return ( { + props.form.resetFields(); + setCreateResult(null); + }} onSaveClick={onSaveClick} title="Create Data Role" /> diff --git a/web/src/pages/System/Role/Platform/api_permission.jsx b/web/src/pages/System/Role/Platform/api_permission.jsx index b8c9aa55..9ebc8afa 100644 --- a/web/src/pages/System/Role/Platform/api_permission.jsx +++ b/web/src/pages/System/Role/Platform/api_permission.jsx @@ -1,6 +1,8 @@ import { Transfer } from "antd"; +import React from "react"; -const ApiPermission = ({ value = [], onChange, permissions = [] }) => { +const ApiPermission = React.forwardRef( + ({ value = [], onChange, permissions = [] }, ref) => { const filterOption = (inputValue, option) => option.description.indexOf(inputValue) > -1; const dataSource = permissions.map((p) => { @@ -10,14 +12,17 @@ const ApiPermission = ({ value = [], onChange, permissions = [] }) => { }; }); return ( - item.title} - /> +
    + item.title} + /> +
    ); -}; + } +); export default ApiPermission; diff --git a/web/src/pages/System/Role/Platform/form.jsx b/web/src/pages/System/Role/Platform/form.jsx index 817d74e3..09f3aed3 100644 --- a/web/src/pages/System/Role/Platform/form.jsx +++ b/web/src/pages/System/Role/Platform/form.jsx @@ -6,6 +6,7 @@ import { InputNumber, Card, Button, + Result, Row, Col, Transfer, @@ -19,8 +20,9 @@ import TagEditor from "@/components/infini/TagEditor"; import Permission from "./permission"; import ApiPermission from "./api_permission"; import request from "@/utils/request"; -import { menuData } from "./menu"; +import { getMenuData } from "./menu"; import { formatMessage } from "umi/locale"; +import { refreshApplicationSettings } from "@/utils/authority"; const formItemLayout = { labelCol: { @@ -47,6 +49,16 @@ const tailFormItemLayout = { const PlatformRoleForm = (props) => { const { getFieldDecorator } = props.form; const [isLoading, setIsLoading] = useState(false); + const [menuData, setMenuData] = useState(() => getMenuData()); + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "system", locale: "menu.system" }, + { title: "security", locale: "menu.system.security" }, + { + title: props.mode === "edit" ? "edit_role" : "new_role", + locale: props.mode === "edit" ? "menu.system.edit_role" : "menu.system.new_role", + }, + ]; const handleSubmit = useCallback( (e) => { @@ -70,8 +82,20 @@ const PlatformRoleForm = (props) => { const editValue = props.value || {}; + useEffect(() => { + let isMounted = true; + refreshApplicationSettings().finally(() => { + if (isMounted) { + setMenuData(getMenuData()); + } + }); + return () => { + isMounted = false; + }; + }, []); + return ( - + @@ -81,41 +105,90 @@ const PlatformRoleForm = (props) => { } > -
    - - {getFieldDecorator("name", { - initialValue: editValue.name, - rules: [ - { - required: true, - message: "Please input name!", - }, - ], - })()} - - - {getFieldDecorator("privilege.platform", { - initialValue: editValue.privilege?.platform || [], - rules: [ - { - required: true, - message: "Please select platform feature privilege!", - }, - ], - })()} - - - {getFieldDecorator("description", { - initialValue: editValue.description, - rules: [], - })()} - - - - - + {props.createResult ? ( + { + props.history.push( + `/system/security?_g=${encodeURIComponent( + JSON.stringify({ tab: "role" }) + )}` + ); + }} + > + {formatMessage({ + id: "system.security.role.create.button.view_list", + })} + , + , + ]} + /> + ) : ( +
    + + {getFieldDecorator("name", { + initialValue: editValue.name, + rules: [ + { + required: true, + message: formatMessage({ + id: "system.role.platform.name.required", + }), + }, + ], + })()} + + + {getFieldDecorator("privilege.platform", { + initialValue: editValue.privilege?.platform || [], + rules: [ + { + validator: (rule, value, callback) => { + if (Array.isArray(value) && value.length > 0) { + callback(); + return; + } + callback( + formatMessage({ + id: "system.role.platform.feature_privilege.required", + }) + ); + }, + }, + ], + })()} + + + {getFieldDecorator("description", { + initialValue: editValue.description, + rules: [], + })()} + + + + + + )}     ); diff --git a/web/src/pages/System/Role/Platform/menu.js b/web/src/pages/System/Role/Platform/menu.js index 7d0d019f..dade82b3 100644 --- a/web/src/pages/System/Role/Platform/menu.js +++ b/web/src/pages/System/Role/Platform/menu.js @@ -1,4 +1,6 @@ -export const menuData = [ +import { getEnterpriseTaskManagerEnabled } from "@/utils/authority"; + +const baseMenuData = [ { key: "workbench", menuKey: "overview" }, { key: "cluster", @@ -91,12 +93,12 @@ export const menuData = [ }, ], }, - { key: "system", children: [ { key: "system.smtp_server", + menuKey: "system.settings", }, { key: "system.security", @@ -110,3 +112,23 @@ export const menuData = [ ], }, ]; + +const enterpriseTaskMenu = { + key: "data_tools", + children: [ + { + key: "data_tools.migration", + }, + { + key: "data_tools.comparison", + }, + ], +}; + +export const getMenuData = () => { + const menuData = [...baseMenuData]; + if (getEnterpriseTaskManagerEnabled() === "true") { + menuData.splice(menuData.length - 1, 0, enterpriseTaskMenu); + } + return menuData; +}; diff --git a/web/src/pages/System/Role/Platform/new.jsx b/web/src/pages/System/Role/Platform/new.jsx index d9fe70ae..e029b904 100644 --- a/web/src/pages/System/Role/Platform/new.jsx +++ b/web/src/pages/System/Role/Platform/new.jsx @@ -1,12 +1,12 @@ import PlatformRoleForm from "./form"; import { Form } from "antd"; -import { useCallback } from "react"; +import { useCallback, useState } from "react"; import request from "@/utils/request"; -import { message } from "antd"; import { router } from "umi"; import { formatMessage } from "umi/locale"; export default Form.create({ name: "platform_role_form_new" })((props) => { + const [createResult, setCreateResult] = useState(null); const onSaveClick = useCallback(async (values) => { const saveRes = await request(`/role/platform`, { method: "POST", @@ -15,17 +15,18 @@ export default Form.create({ name: "platform_role_form_new" })((props) => { }, }); if (saveRes && saveRes.result == "created") { - message.success( - formatMessage({ - id: "app.message.save.success", - }) - ); - props.form.resetFields(); + setCreateResult(saveRes); } }, []); return ( { + props.form.resetFields(); + setCreateResult(null); + }} onSaveClick={onSaveClick} title="Create Platform Role" /> diff --git a/web/src/pages/System/Role/Platform/permission.jsx b/web/src/pages/System/Role/Platform/permission.jsx index 62240f73..6dad8636 100644 --- a/web/src/pages/System/Role/Platform/permission.jsx +++ b/web/src/pages/System/Role/Platform/permission.jsx @@ -46,9 +46,9 @@ const renderTree = ({ key, children, menuKey }, onValueChange) => { ); }; -export default ({ data, value, onChange }) => { +const Permission = React.forwardRef(({ data, value, onChange }, ref) => { const onValueChange = (pitem) => { - let permissions = value || []; + const permissions = [...(value || [])]; const newTemps = (pitem || "").split(":"); const srcIndex = permissions.findIndex((p) => { return ( @@ -62,22 +62,26 @@ export default ({ data, value, onChange }) => { } else { permissions.push(pitem); } - permissions = permissions.filter((p) => !p.endsWith(":none")); + const nextPermissions = permissions.filter((p) => !p.endsWith(":none")); if (typeof onChange == "function") { - onChange(permissions); + onChange(nextPermissions); } }; data = data || []; return ( - - }> - {data.map((item) => { - return renderTree(item, onValueChange); - })} - - +
    + + }> + {data.map((item) => { + return renderTree(item, onValueChange); + })} + + +
    ); -}; +}); + +export default Permission; const PermissionTitle = ({ id, title, onChange, showOptions }) => { const { value } = React.useContext(PermissionContext); @@ -112,7 +116,7 @@ const PermissionTitle = ({ id, title, onChange, showOptions }) => { {enumValues.map((item) => { diff --git a/web/src/pages/System/Role/index.jsx b/web/src/pages/System/Role/index.jsx index b9ecfb68..fe7e209e 100644 --- a/web/src/pages/System/Role/index.jsx +++ b/web/src/pages/System/Role/index.jsx @@ -11,6 +11,7 @@ import { message, Menu, Dropdown, + Icon, } from "antd"; import { formatMessage } from "umi/locale"; import useFetch from "@/lib/hooks/use_fetch"; @@ -27,7 +28,19 @@ import moment from "moment"; import { formatter } from "@/lib/format"; import { hasAuthority } from "@/utils/authority"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; + +const firstColumnIconStyle = { + marginRight: 8, + color: "#999", + fontSize: 12, +}; + +const displayOrDash = (value) => { + if (value === null || value === undefined) return "-"; + const text = `${value}`.trim(); + return text ? text : "-"; +}; const RoleList = (props) => { const [queryParams, setQueryParams] = React.useState({}); @@ -44,7 +57,7 @@ const RoleList = (props) => { async (roleID) => { const deleteRes = await request(`/role/${roleID}`, { method: "DELETE", - }); + }, false, false); if (deleteRes && deleteRes.result == "deleted") { message.success( formatMessage({ @@ -54,6 +67,17 @@ const RoleList = (props) => { setTimeout(() => { onRefreshClick(); }, 1000); + } else if (deleteRes && deleteRes.error) { + const reason = deleteRes.error.reason || ""; + if (reason === "role is still assigned to users") { + message.error( + formatMessage({ + id: "system.security.role.delete.error.assigned_to_users", + }) + ); + } else { + message.error(reason || formatMessage({ id: "app.message.error" })); + } } }, [setQueryParams] @@ -61,51 +85,62 @@ const RoleList = (props) => { const columns = useMemo( () => [ { - title: "Name", + title: formatMessage({ id: "system.security.role.table.name" }), dataIndex: "name", + render: (text) => ( +
    + + {displayOrDash(text)} +
    + ), }, { - title: "Type", + title: formatMessage({ id: "system.security.role.table.type" }), dataIndex: "type", + render: (val) => displayOrDash(val), }, { - title: "Builtin", + title: formatMessage({ id: "system.security.role.table.builtin" }), dataIndex: "builtin", render: (val) => { return val === true ? "true" : "false"; }, }, { - title: "Description", + title: formatMessage({ id: "system.security.role.table.description" }), dataIndex: "description", + render: (val) => displayOrDash(val), }, { title: formatMessage({ id: "table.field.actions" }), - render: (text, record) => ( -     - ), + render: (text, record) => { + if (!(hasAuthority("system.security:all") && record.builtin === false)) { + return "-"; + } + return ( +
    + + {formatMessage({ id: "form.button.edit" })} + + + onDeleteClick(record.id)} + > + {formatMessage({ id: "form.button.delete" })} + +
    + ); + }, }, ], @@ -160,8 +195,12 @@ const RoleList = (props) => { }; const menu = ( - Add Platform Role - Add Data Role + + {formatMessage({ id: "system.security.role.menu.add_platform" })} + + + {formatMessage({ id: "system.security.role.menu.add_data" })} + ); @@ -176,10 +215,12 @@ const RoleList = (props) => { }} >
    - { onSearchClick(value); }} @@ -229,7 +270,10 @@ const RoleList = (props) => { total: total?.value || total, showSizeChanger: true, showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, + formatMessage( + { id: "system.security.pagination.total" }, + { start: range[0], end: range[1], total } + ), }} columns={columns} onChange={handleTableChange} diff --git a/web/src/pages/System/Security/Token.jsx b/web/src/pages/System/Security/Token.jsx new file mode 100644 index 00000000..8c67fbf0 --- /dev/null +++ b/web/src/pages/System/Security/Token.jsx @@ -0,0 +1,528 @@ +import React, { useMemo, useReducer, useState } from "react"; +import { + Button, + Card, + DatePicker, + Divider, + Drawer, + Form, + Icon, + Input, + Modal, + Popconfirm, + Switch, + Table, + message, +} from "antd"; +import { formatMessage } from "umi/locale"; +import request from "@/utils/request"; +import useFetch from "@/lib/hooks/use_fetch"; +import { formatESSearchResult } from "@/lib/elasticsearch/util"; +import { hasAuthority } from "@/utils/authority"; +import SearchInput from "@/components/infini/SearchInput"; +import moment from "moment"; + +const firstColumnIconStyle = { + marginRight: 8, + color: "#999", + fontSize: 12, +}; + +const ellipsisTextStyle = { + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", +}; + +const tokenPreviewStyle = { + position: "relative", + marginTop: 12, + padding: "16px 52px 16px 16px", + borderRadius: 6, + background: "rgb(241, 242, 245)", + color: "rgba(0, 0, 0, 0.85)", + fontFamily: + '"SFMono-Regular", Monaco, Menlo, Consolas, "Liberation Mono", "Ubuntu Mono", monospace', + lineHeight: 1.6, + wordBreak: "break-all", +}; + +const tokenPreviewCopyButtonStyle = { + position: "absolute", + top: 12, + right: 12, +}; + +const getDefaultTokenExpireTime = () => moment().add(1, "year"); + +const copyText = async (text) => { + if (!text) { + return false; + } + if (navigator?.clipboard?.writeText) { + await navigator.clipboard.writeText(text); + return true; + } + + const input = document.createElement("textarea"); + input.value = text; + input.setAttribute("readonly", "readonly"); + input.style.position = "fixed"; + input.style.top = "-9999px"; + document.body.appendChild(input); + input.select(); + const copied = document.execCommand("copy"); + document.body.removeChild(input); + return copied; +}; + +const TokenForm = Form.create()(({ form, record, onSubmit, submitLoading }) => { + const { getFieldDecorator, validateFields } = form; + const isEdit = !!record; + const [neverExpire, setNeverExpire] = useState(!(Number(record?.expire_in) > 0)); + + const submit = (e) => { + e.preventDefault(); + validateFields((err, values) => { + if (!err) { + onSubmit({ + name: values.name, + description: values.description, + expire_in: values.never_expire ? 0 : values.expire_in.unix(), + }); + } + }); + }; + + return ( +
    + + {getFieldDecorator("name", { + initialValue: record?.name || "", + rules: [ + { + required: true, + message: formatMessage({ id: "system.security.token.form.name.required" }), + }, + ], + })()} + + + {getFieldDecorator("description", { + initialValue: record?.description || "", + })()} + + + {getFieldDecorator("never_expire", { + initialValue: !(Number(record?.expire_in) > 0), + valuePropName: "checked", + })( + { + setNeverExpire(checked); + if (!checked && !form.getFieldValue("expire_in")) { + form.setFieldsValue({ + expire_in: getDefaultTokenExpireTime(), + }); + } + }} + /> + )} + + + {getFieldDecorator("expire_in", { + initialValue: + Number(record?.expire_in) > 0 + ? moment.unix(record.expire_in) + : getDefaultTokenExpireTime(), + rules: neverExpire + ? [] + : [ + { + validator: (_, value, callback) => { + if (!value) { + callback( + formatMessage({ + id: "system.security.token.form.expire.required", + }) + ); + return; + } + if (!value.isAfter(moment())) { + callback( + formatMessage({ + id: "system.security.token.form.expire.future", + }) + ); + return; + } + callback(); + }, + }, + ], + })( + + current && current < moment().startOf("day") + } + /> + )} + +
    + + +
    + + ); +}); + +const Token = () => { + const initialQueryParams = { + from: 0, + size: 20, + keyword: "", + }; + + function reducer(queryParams, action) { + switch (action.type) { + case "search": + return { + ...queryParams, + from: 0, + keyword: action.value, + }; + case "pagination": + return { + ...queryParams, + from: (action.current - 1) * action.pageSize, + size: action.pageSize, + }; + case "refresh": + return { + ...queryParams, + _t: Date.now(), + }; + default: + return queryParams; + } + } + + const [queryParams, dispatch] = useReducer(reducer, initialQueryParams); + const [visible, setVisible] = useState(false); + const [submitLoading, setSubmitLoading] = useState(false); + const [selectedItem, setSelectedItem] = useState(); + + const canSearch = + hasAuthority("system.security:all") || hasAuthority("system.security:read"); + const canCreate = hasAuthority("system.security:all"); + const canUpdate = hasAuthority("system.security:all"); + const canDelete = hasAuthority("system.security:all"); + + const { loading, value, run } = useFetch( + "/auth/access_token/_search", + { + queryParams, + noticeable: false, + }, + [queryParams], + canSearch + ); + + const openCreateTokenResult = (token) => { + Modal.success({ + title: formatMessage({ id: "system.security.token.create.result.title" }), + content: ( +
    +

    {formatMessage({ id: "system.security.token.create.result.tip" })}

    +
    + +
    {token}
    +
    +
    + ), + okText: formatMessage({ id: "form.button.ok" }), + width: 640, + }); + }; + + const onCopy = async (token) => { + try { + const copied = await copyText(token); + if (!copied) { + throw new Error("copy failed"); + } + message.success(formatMessage({ id: "system.security.token.copy.success" })); + } catch (e) { + message.error(formatMessage({ id: "system.security.token.copy.failed" })); + } + }; + + const onDelete = async (tokenID) => { + const res = await request(`/auth/access_token/${tokenID}`, { + method: "DELETE", + }); + if (res?.result === "deleted") { + message.success(formatMessage({ id: "app.message.delete.success" })); + run(); + return; + } + message.error(res?.error?.reason || formatMessage({ id: "app.message.delete.failed" })); + }; + + const onSubmit = async (formValue) => { + if (formValue == null) { + setVisible(false); + setSelectedItem(undefined); + return; + } + if (submitLoading) { + return; + } + + setSubmitLoading(true); + try { + if (selectedItem?.id) { + const res = await request(`/auth/access_token/${selectedItem.id}`, { + method: "PUT", + body: { + name: formValue.name, + description: formValue.description, + expire_in: formValue.expire_in, + }, + }); + if (res?.result === "updated") { + message.success(formatMessage({ id: "app.message.update.success" })); + setVisible(false); + setSelectedItem(undefined); + run(); + return; + } + message.error(res?.error?.reason || formatMessage({ id: "app.message.update.failed" })); + return; + } + + const res = await request(`/auth/access_token`, { + method: "POST", + body: { + name: formValue.name, + description: formValue.description, + expire_in: formValue.expire_in, + }, + }); + if (res?._id && res?.access_token) { + message.success(formatMessage({ id: "app.message.create.success" })); + setVisible(false); + setSelectedItem(undefined); + run(); + openCreateTokenResult(res.access_token); + return; + } + message.error(res?.error?.reason || formatMessage({ id: "app.message.create.failed" })); + } finally { + setSubmitLoading(false); + } + }; + + const { data, total } = useMemo(() => { + if (!value || value.error) { + return { + data: [], + total: 0, + }; + } + return formatESSearchResult(value); + }, [value]); + + const columns = [ + { + title: formatMessage({ id: "table.field.id" }), + dataIndex: "id", + width: 240, + render: (text) => ( +
    + + + {text || "-"} + +
    + ), + }, + { + title: formatMessage({ id: "system.security.token.table.name" }), + dataIndex: "name", + render: (text) => ( +
    + {text || "-"} +
    + ), + }, + { + title: formatMessage({ id: "system.security.token.table.description" }), + dataIndex: "description", + render: (text) => ( +
    + {text || "-"} +
    + ), + }, + { + title: formatMessage({ id: "system.security.token.table.permissions" }), + dataIndex: "permissions", + width: 260, + render: (permissions) => { + const text = (permissions || []).join(", ") || "-"; + return ( + +
    {text}
    +     + ); + }, + }, + { + title: formatMessage({ id: "system.security.token.table.expire" }), + dataIndex: "expire_in", + width: 180, + render: (expireIn) => + Number(expireIn) > 0 + ? moment.unix(expireIn).format("YYYY.MM.DD HH:mm:ss") + : formatMessage({ id: "system.security.token.never_expire" }), + }, + { + title: formatMessage({ id: "table.field.actions" }), + width: 150, + render: (_, record) => ( + <> + {canUpdate ? ( + { + setSelectedItem(record); + setVisible(true); + }} + > + {formatMessage({ id: "form.button.edit" })} + + ) : null} + {canUpdate && canDelete ? : null} + {canDelete ? ( + onDelete(record.id)} + > + {formatMessage({ id: "form.button.delete" })} + + ) : null} + + ), + }, + ]; + + if (!canUpdate && !canDelete) { + columns.splice(columns.length - 1, 1); + } + + return ( + +
    +
    + dispatch({ type: "search", value: keyword })} + /> +
    +
    + + {canCreate ? ( + + ) : null} +
    +
    +
    + formatMessage( + { id: "system.security.pagination.total" }, + { start: range[0], end: range[1], total: t } + ), + }} + onChange={(pagination) => + dispatch({ + type: "pagination", + current: pagination.current, + pageSize: pagination.pageSize, + }) + } + /> + { + if (!submitLoading) { + setVisible(false); + setSelectedItem(undefined); + } + }} + visible={visible} + title={formatMessage({ + id: selectedItem?.id + ? "system.security.token.drawer.edit.title" + : "system.security.token.drawer.create.title", + })} + destroyOnClose + > + + + + ); +}; + +export default Token; diff --git a/web/src/pages/System/Security/index.jsx b/web/src/pages/System/Security/index.jsx index a0cb02ba..130a811c 100644 --- a/web/src/pages/System/Security/index.jsx +++ b/web/src/pages/System/Security/index.jsx @@ -25,6 +25,7 @@ import "@/assets/headercontent.scss"; const { TabPane } = Tabs; import User from "../User/index"; import Role from "../Role/index"; +import Token from "./Token"; const Security = (props) => { const [param, setParam] = useQueryParam("_g", JsonParam); @@ -37,12 +38,24 @@ const Security = (props) => { setParam({ ...param, tab: key }); }} > - User} key="user"> + {formatMessage({ id: "system.security.tab.user" })}} + key="user" + > - Role} key="role"> + {formatMessage({ id: "system.security.tab.role" })}} + key="role" + > + {formatMessage({ id: "system.security.tab.token" })}} + key="token" + > + + diff --git a/web/src/pages/System/Settings/index.jsx b/web/src/pages/System/Settings/index.jsx new file mode 100644 index 00000000..133f6619 --- /dev/null +++ b/web/src/pages/System/Settings/index.jsx @@ -0,0 +1,448 @@ +import PageHeaderWrapper from "@/components/PageHeaderWrapper"; +import { + Alert, + Button, + Card, + Empty, + Icon, + InputNumber, + Spin, + Switch, + Tabs, + message, +} from "antd"; +import { formatMessage } from "umi/locale"; +import router from "umi/router"; +import { useEffect, useMemo, useState } from "react"; +import request from "@/utils/request"; +import ServerList from "../Email/Server"; +import { hasAuthority, refreshApplicationSettings } from "@/utils/authority"; + +const { TabPane } = Tabs; + +const SystemSettings = (props) => { + const query = new URLSearchParams(props.location?.search || ""); + const pathDefaultTab = props.location?.pathname?.includes("/email_server") + ? "email" + : "general"; + const canReadCluster = + hasAuthority("system.cluster:all") || hasAuthority("system.cluster:read"); + const canWriteCluster = hasAuthority("system.cluster:all"); + const canReadEmail = + hasAuthority("system.smtp_server:all") || + hasAuthority("system.smtp_server:read"); + const tabs = useMemo(() => { + const nextTabs = []; + if (canReadCluster) { + nextTabs.push("general"); + } + if (canReadEmail) { + nextTabs.push("email"); + } + return nextTabs; + }, [canReadCluster, canReadEmail]); + const [activeTab, setActiveTab] = useState( + query.get("tab") || pathDefaultTab || tabs[0] || "general" + ); + const defaultRetentionMaxSize = 50; + const [retentionDays, setRetentionDays] = useState(30); + const [retentionDraftDays, setRetentionDraftDays] = useState(30); + const [retentionMaxSize, setRetentionMaxSize] = useState(defaultRetentionMaxSize); + const [retentionDraftMaxSize, setRetentionDraftMaxSize] = useState( + defaultRetentionMaxSize + ); + const [retentionLoading, setRetentionLoading] = useState(false); + const [rollupEnabled, setRollupEnabled] = useState(false); + const [rollupSupported, setRollupSupported] = useState(false); + const [rollupLoading, setRollupLoading] = useState(false); + const [advancedVisible, setAdvancedVisible] = useState(false); + const [localTemplatesLoading, setLocalTemplatesLoading] = useState(false); + + const normalizeRetentionSize = (value) => + `${value || ""}`.replace(/\s+/g, "").toLowerCase(); + + const parseRetentionSizeToGb = (value) => { + const normalizedValue = normalizeRetentionSize(value); + const matches = normalizedValue.match(/^(\d+)(b|kb|mb|gb|tb|k|m|g|t)$/i); + if (!matches) { + return defaultRetentionMaxSize; + } + const size = Number(matches[1]); + const unit = matches[2].toLowerCase(); + const multiplier = { + b: 1 / 1024 / 1024 / 1024, + kb: 1 / 1024 / 1024, + k: 1 / 1024 / 1024, + mb: 1 / 1024, + m: 1 / 1024, + gb: 1, + g: 1, + tb: 1024, + t: 1024, + }[unit]; + if (!multiplier) { + return defaultRetentionMaxSize; + } + return Math.max(1, Math.ceil(size * multiplier)); + }; + + const isValidRetentionSize = (value) => + Number.isInteger(value) && value > 0; + + useEffect(() => { + if (tabs.length > 0 && !tabs.includes(activeTab)) { + setActiveTab(tabs[0]); + } + }, [tabs, activeTab]); + + useEffect(() => { + if (!canReadCluster) { + return; + } + const fetchRetentionSetting = async () => { + setRetentionLoading(true); + const res = await request("/setting/system/retention", { + method: "GET", + }); + if (!res?.error && Number.isInteger(res.days) && res.days > 0) { + setRetentionDays(res.days); + setRetentionDraftDays(res.days); + if (res.max_size) { + const normalizedSize = parseRetentionSizeToGb(res.max_size); + setRetentionMaxSize(normalizedSize); + setRetentionDraftMaxSize(normalizedSize); + } + } + setRetentionLoading(false); + }; + const fetchRollupSetting = async () => { + setRollupLoading(true); + const applicationSettings = await refreshApplicationSettings(true); + const supported = !!applicationSettings?.system_cluster?.rollup_supported; + setRollupSupported(supported); + if (!supported) { + setRollupEnabled(false); + setRollupLoading(false); + return; + } + const res = await request("/setting/system/rollup", { + method: "GET", + }); + if (!res?.error) { + setRollupEnabled(!!res.enabled); + } + setRollupLoading(false); + }; + fetchRetentionSetting(); + fetchRollupSetting(); + }, [canReadCluster]); + + const onTabChange = (key) => { + setActiveTab(key); + router.replace(`/system/settings?tab=${key}`); + }; + + const onRollupToggle = async (checked) => { + const previousValue = rollupEnabled; + setRollupEnabled(checked); + setRollupLoading(true); + const res = await request("/setting/system/rollup", { + method: "PUT", + body: { + enabled: checked, + }, + }); + if (res?.error) { + setRollupEnabled(previousValue); + setRollupLoading(false); + return; + } + await refreshApplicationSettings(true); + setRollupLoading(false); + message.success( + formatMessage({ id: "settings.system.rollup.update.success" }) + ); + }; + + const onRetentionSave = async () => { + if (!Number.isInteger(retentionDraftDays) || retentionDraftDays <= 0) { + message.warning( + formatMessage({ id: "settings.system.retention.validation.days" }) + ); + return; + } + if (!isValidRetentionSize(retentionDraftMaxSize)) { + message.warning( + formatMessage({ id: "settings.system.retention.validation.max_size" }) + ); + return; + } + setRetentionLoading(true); + const normalizedMaxSize = normalizeRetentionSize(`${retentionDraftMaxSize}gb`); + const res = await request("/setting/system/retention", { + method: "PUT", + body: { + days: retentionDraftDays, + max_size: normalizedMaxSize, + }, + }); + if (res?.error) { + setRetentionLoading(false); + return; + } + setRetentionDays(res.days); + setRetentionDraftDays(res.days); + setRetentionMaxSize(parseRetentionSizeToGb(res.max_size || normalizedMaxSize)); + setRetentionDraftMaxSize( + parseRetentionSizeToGb(res.max_size || normalizedMaxSize) + ); + setRetentionLoading(false); + message.success( + formatMessage({ id: "settings.system.retention.update.success" }) + ); + }; + + const onLocalTemplatesRefresh = async () => { + setLocalTemplatesLoading(true); + const res = await request("/setting/system/local_templates/_refresh", { + method: "POST", + }); + setLocalTemplatesLoading(false); + if (res?.error) { + return; + } + message.success( + formatMessage({ id: "settings.system.local_templates.update.success" }) + ); + }; + + const renderRetentionSettings = () => { + if (!canReadCluster) { + return null; + } + return ( + + +
    +
    +
    + {formatMessage({ id: "settings.system.retention.title" })} +
    +
    + {formatMessage({ + id: "settings.system.retention.description", + })} +
    +
    +
    + { + setRetentionDraftDays( + Number.isInteger(value) ? value : retentionDraftDays + ); + }} + /> + {formatMessage({ id: "settings.system.retention.unit" })} + + {formatMessage({ id: "settings.system.retention.size.label" })} + + { + setRetentionDraftMaxSize( + Number.isInteger(value) ? value : retentionDraftMaxSize + ); + }} + /> + {formatMessage({ id: "settings.system.retention.size.unit" })} + +
    +
    + {formatMessage({ id: "settings.system.retention.help" })}} + /> +     +     + ); + }; + + const renderRollupSettings = () => { + if (!canReadCluster || !rollupSupported) { + return null; + } + return ( + + +
    +
    +
    + {formatMessage({ id: "settings.system.rollup.title" })} +
    +
    + {formatMessage({ id: "settings.system.rollup.description" })} +
    +
    + +
    + +     +     + ); + }; + + const renderLocalTemplateSettings = () => { + if (!canReadCluster) { + return null; + } + return ( + + +
    +
    +
    + {formatMessage({ id: "settings.system.local_templates.title" })} +
    +
    + {formatMessage({ + id: "settings.system.local_templates.description", + })} +
    +
    + +
    + +     +     + ); + }; + + const renderAdvancedSettings = () => { + if (!canReadCluster) { + return null; + } + return ( + + + {advancedVisible ? ( +
    + {renderRollupSettings()} + {renderLocalTemplateSettings()} +
    + ) : null} +     + ); + }; + + const renderGeneralSettings = () => { + if (!canReadCluster) { + return ; + } + return ( + <> + {renderRetentionSettings()} + {renderAdvancedSettings()} + + ); + }; + + return ( + + + + {canReadCluster ? ( + + {renderGeneralSettings()} + + ) : null} + {canReadEmail ? ( + + + + ) : null} + + + + ); +}; + +export default SystemSettings; diff --git a/web/src/pages/System/User/form.jsx b/web/src/pages/System/User/form.jsx index 960dedc8..7590f862 100644 --- a/web/src/pages/System/User/form.jsx +++ b/web/src/pages/System/User/form.jsx @@ -9,6 +9,7 @@ import { Row, Col, Result, + message, } from "antd"; import PageHeaderWrapper from "@/components/PageHeaderWrapper"; // import "./form.scss"; @@ -45,8 +46,18 @@ const tailFormItemLayout = { const UserForm = (props) => { const { getFieldDecorator } = props.form; const [isLoading, setIsLoading] = useState(false); + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "system", locale: "menu.system" }, + { title: "security", locale: "menu.system.security" }, + { + title: props.mode === "edit" ? "edit_user" : "new_user", + locale: + props.mode === "edit" ? "menu.system.edit_user" : "menu.system.new_user", + }, + ]; - const { value: roleRes } = useFetch( + const { loading: rolesLoading, error: rolesError, value: roleRes } = useFetch( `/role/_search`, { queryParams: { size: 10000 } }, [] @@ -67,24 +78,35 @@ const UserForm = (props) => { e.preventDefault(); setIsLoading(true); props.form.validateFields(async (err, values) => { - if (err) { - return false; - } - if (typeof props.onSaveClick == "function") { - let newVals = { - ...values, - }; - if (newVals.roles) { - newVals.roles = newVals.roles.map((rid) => { - return roles.find((role) => role.id == rid); - }); + try { + if (err || rolesLoading || rolesError) { + return; + } + if (typeof props.onSaveClick == "function") { + let newVals = { + ...values, + }; + if (newVals.roles) { + newVals.roles = newVals.roles + .map((rid) => { + return roles.find((role) => role.id == rid); + }) + .filter(Boolean); + } + await props.onSaveClick(newVals); } - await props.onSaveClick(newVals); + } catch (error) { + message.error( + formatMessage({ + id: "app.message.save.failed", + }) + ); + } finally { setIsLoading(false); } }); }, - [props.form, roles] + [props.form, props.onSaveClick, roles, rolesError, rolesLoading] ); const editValue = props.value || {}; if (editValue.roles && editValue.roles.length && editValue.roles[0]?.name) { @@ -95,7 +117,7 @@ const UserForm = (props) => { }; return ( - + @@ -107,18 +129,24 @@ const UserForm = (props) => { > {!props.createResult ? (
    - + {getFieldDecorator("name", { initialValue: editValue.name, rules: [ { required: true, - message: "Please input name!", + message: formatMessage({ + id: "system.security.user.form.name.required", + }), }, ], })()} - + {getFieldDecorator("nick_name", { initialValue: editValue.nick_name, rules: [ @@ -136,7 +164,9 @@ const UserForm = (props) => { rules: [], })()} */} - + {getFieldDecorator("phone", { initialValue: editValue.phone, rules: [ @@ -152,7 +182,9 @@ const UserForm = (props) => { ], })()} - + {getFieldDecorator("email", { initialValue: editValue.email, rules: [ @@ -162,24 +194,31 @@ const UserForm = (props) => { // }, { type: "email", - message: "The input is not valid email!", + message: formatMessage({ + id: "system.security.user.form.email.invalid", + }), }, ], })()} - + {getFieldDecorator("roles", { initialValue: editValue.roles, rules: [ { required: true, - message: "Please select roles!", + message: formatMessage({ + id: "system.security.user.form.roles.required", + }), }, ], })( )} - + {getFieldDecorator("tags", { initialValue: editValue.tags || [], rules: [], })()} - @@ -221,13 +267,16 @@ export default UserForm; const CreateResult = ({ password }) => ( + {(copy) => ( )} , diff --git a/web/src/pages/System/User/index.jsx b/web/src/pages/System/User/index.jsx index edfe1bb9..a261364b 100644 --- a/web/src/pages/System/User/index.jsx +++ b/web/src/pages/System/User/index.jsx @@ -9,8 +9,10 @@ import { Button, Input, message, + Icon, + Switch, } from "antd"; -import { formatMessage } from "umi/locale"; +import { formatMessage, getLocale } from "umi/locale"; import useFetch from "@/lib/hooks/use_fetch"; import { ESPrefix } from "@/services/common"; import { useGlobal } from "@/layouts/GlobalContext"; @@ -25,7 +27,24 @@ import moment from "moment"; import { formatter } from "@/lib/format"; import { hasAuthority } from "@/utils/authority"; -const { Search } = Input; +import SearchInput from "@/components/infini/SearchInput"; + +const firstColumnIconStyle = { + marginRight: 8, + color: "#999", + fontSize: 12, +}; + +const displayOrDash = (value) => { + if (value === null || value === undefined) return "-"; + const text = `${value}`.trim(); + return text ? text : "-"; +}; + +const isAdministratorUser = (record) => + (record?.roles || []).some( + (role) => role?.id === "Administrator" || role?.name === "Administrator" + ); const UserList = (props) => { const [queryParams, setQueryParams] = React.useState({}); @@ -58,40 +77,90 @@ const UserList = (props) => { }, [setQueryParams] ); + const onToggleEnabledClick = useCallback( + async (userID, enabled) => { + const actionUrl = enabled ? "/user/_enable" : "/user/_disable"; + const res = await request(actionUrl, { + method: "POST", + body: [userID], + }); + if (res?.acknowledged) { + message.success(formatMessage({ id: "app.message.operate.success" })); + onRefreshClick(); + return; + } + message.error(formatMessage({ id: "app.message.operate.failed" })); + }, + [setQueryParams] + ); + + const normalizeEnabled = (value) => value !== false; + const columns = useMemo( () => [ { - title: "Name", + title: formatMessage({ id: "system.security.user.table.name" }), dataIndex: "name", + render: (text) => ( +
    + + {displayOrDash(text)} +
    + ), }, { - title: "Nickname", + title: formatMessage({ id: "system.security.user.table.nickname" }), dataIndex: "nick_name", + render: (val) => displayOrDash(val), }, { - title: "Roles", + title: formatMessage({ id: "system.security.user.table.roles" }), dataIndex: "roles", render: (val) => { - return (val || []).map((role) => role.name).join(","); + const text = (val || []).map((role) => role.name).join(","); + return displayOrDash(text); }, }, { - title: "Phone", + title: formatMessage({ id: "system.security.user.table.phone" }), dataIndex: "phone", + render: (val) => displayOrDash(val), }, { - title: "Email", + title: formatMessage({ id: "system.security.user.table.email" }), dataIndex: "email", + render: (val) => displayOrDash(val), }, { - title: "Tags", + title: formatMessage({ id: "system.security.user.table.tags" }), dataIndex: "tags", render: (text) => { - return text; + return displayOrDash(text); + }, + }, + { + title: formatMessage({ id: "system.security.user.table.status" }), + dataIndex: "enabled", + render: (enabled, record) => { + if (isAdministratorUser(record)) { + return "-"; + } + const checked = normalizeEnabled(enabled); + return ( + + onToggleEnabledClick(record.id, nextChecked) + } + /> + ); }, }, { title: formatMessage({ id: "table.field.actions" }), + width: getLocale() === "zh-CN" ? 180 : 240, render: (text, record) => (
    {hasAuthority("system.security:all") ? ( @@ -100,21 +169,25 @@ const UserList = (props) => { key="permission" to={`/system/security/user/edit/${record.id}`} > - Edit + {formatMessage({ id: "form.button.edit" })} onDeleteClick(record.id)} > - Delete + {formatMessage({ id: "form.button.delete" })} - Reset Password + {formatMessage({ + id: "system.security.user.action.reset_password", + })} ) : null} @@ -123,7 +196,7 @@ const UserList = (props) => { }, ], - [value] + [value, onToggleEnabledClick] ); const { data: users, total } = React.useMemo(() => { setIsLoading(loading); @@ -174,10 +247,12 @@ const UserList = (props) => { }} >
    - { onSearchClick(value); }} @@ -224,7 +299,10 @@ const UserList = (props) => { total: total?.value || total, showSizeChanger: true, showTotal: (total, range) => - `${range[0]}-${range[1]} of ${total} items`, + formatMessage( + { id: "system.security.pagination.total" }, + { start: range[0], end: range[1], total } + ), }} columns={columns} onChange={handleTableChange} diff --git a/web/src/pages/System/User/new.jsx b/web/src/pages/System/User/new.jsx index 606e1e71..d02f8a04 100644 --- a/web/src/pages/System/User/new.jsx +++ b/web/src/pages/System/User/new.jsx @@ -22,12 +22,21 @@ export default Form.create({ name: "user_form_new" })((props) => { }) ); setCreateResult(saveRes); + return; + } + if (saveRes && !saveRes.error) { + message.error( + formatMessage({ + id: "app.message.save.failed", + }) + ); } }, []); return (
    { const { form, match } = props; const { getFieldDecorator } = form; const [passwordHelp, setPasswordHelp] = useState(null); + const breadcrumbList = [ + { title: "home", locale: "menu.home", href: "/" }, + { title: "system", locale: "menu.system" }, + { title: "security", locale: "menu.system.security" }, + { title: "reset_password", locale: "menu.system.reset_password" }, + ]; const onCancelClick = () => { props.history.go(-1); @@ -118,7 +124,7 @@ export default Form.create({ name: "user_form_new" })((props) => { }); }; return ( - + diff --git a/web/src/pages/User/Login.js b/web/src/pages/User/Login.js index 48f89c6f..b748b754 100644 --- a/web/src/pages/User/Login.js +++ b/web/src/pages/User/Login.js @@ -3,7 +3,10 @@ import { connect } from "dva"; import { formatMessage, FormattedMessage } from "umi/locale"; import Link from "umi/link"; import { Checkbox, Alert, Icon,Button } from "antd"; +import router from "umi/router"; import Login from "@/components/Login"; +import { refreshApplicationSettings } from "@/utils/authority"; +import { getSetupRequired, setSetupRequired } from "@/utils/setup"; import styles from "./Login.less"; import "./LoginPage.scss"; @@ -17,6 +20,33 @@ class LoginPage extends Component { state = { type: "account", autoLogin: true, + ssoProviders: [], + }; + + componentDidMount() { + if (getSetupRequired() === "true") { + router.replace("/guide/initialization"); + return; + } + + this.syncSetupState(); + } + + syncSetupState = async () => { + try { + const res = await refreshApplicationSettings(); + setSetupRequired(`${!!res?.setup_required}`); + const oauthProviders = res?.security?.auth?.oauth || {}; + const ssoProviders = Object.keys(oauthProviders) + .map((key) => oauthProviders[key]) + .filter((item) => !!item?.url); + this.setState({ ssoProviders }); + if (res?.setup_required) { + router.replace("/guide/initialization"); + } + } catch (error) { + console.log(error); + } }; onTabChange = (type) => { @@ -71,7 +101,7 @@ class LoginPage extends Component { render() { const { login, submitting } = this.props; - const { type, autoLogin } = this.state; + const { type, autoLogin, ssoProviders } = this.state; return (
    - {
    - - -
    } + {ssoProviders.length > 0 ? ( +
    + + {ssoProviders.map((provider, index) => ( + + ))} +
    + ) : null}
    ); diff --git a/web/src/pages/User/SSOSuccess.js b/web/src/pages/User/SSOSuccess.js index ebeb09e1..593c4299 100644 --- a/web/src/pages/User/SSOSuccess.js +++ b/web/src/pages/User/SSOSuccess.js @@ -6,6 +6,10 @@ import Result from "@/components/Result"; import { router } from "umi"; import { setAuthority } from "@/utils/authority"; import { reloadAuthorized } from "@/utils/Authorized"; +import { + clearStoredLoginResponse, + storeLoginResponse, +} from "@/utils/auth_session"; const actions = (
    @@ -20,9 +24,9 @@ const actions = ( const SSOSuccess = ({ location }) => { useEffect(() => { if (location?.query?.payload) { - localStorage.setItem("login-response", location.query.payload); + const loginResponse = storeLoginResponse(location.query.payload); try { - const query = JSON.parse(location.query.payload); + const query = loginResponse || JSON.parse(location.query.payload); if (query?.privilege) { setAuthority(query.privilege); reloadAuthorized(); @@ -33,7 +37,7 @@ const SSOSuccess = ({ location }) => { localStorage.setItem("infini-console-authority", ""); } } else { - localStorage.setItem("login-response", ""); + clearStoredLoginResponse(); } setTimeout(() => { router.push("/"); diff --git a/web/src/services/api.js b/web/src/services/api.js index 0706519b..023bb5c6 100644 --- a/web/src/services/api.js +++ b/web/src/services/api.js @@ -1,5 +1,53 @@ import { stringify } from "qs"; -import request from "@/utils/request"; +import request, { formatResponse } from "@/utils/request"; + +const normalizeAuthResponse = async (response) => { + if (!response || typeof response.text !== "function") { + if (response && typeof response === "object") { + return response; + } + + return { + status: "error", + success: false, + error: { + reason: typeof response === "string" && response ? response : "", + }, + }; + } + + let payload = {}; + const body = await response.text(); + if (body) { + try { + payload = JSON.parse(body); + } catch (error) { + payload = { + error: { + reason: body, + }, + }; + } + } + + const normalized = formatResponse({ + ...payload, + status: payload?.status || (response.ok ? "ok" : "error"), + success: + typeof payload?.success === "boolean" ? payload.success : response.ok, + }); + + if (!normalized?.error && !response.ok) { + normalized.error = { + reason: payload?.message || response.statusText || `HTTP ${response.status}`, + }; + } + + return { + ...normalized, + httpStatus: response.status, + }; +}; export async function queryConsoleInfo() { return request("/_info"); @@ -108,10 +156,45 @@ export async function updateFakeList(params) { } export async function fakeAccountLogin(params) { - return request("/account/login", { - method: "POST", - body: params, - }); + const response = await request( + "/account/login", + { + method: "POST", + body: params, + skipAuthRedirect: true, + }, + true, + false + ); + + return normalizeAuthResponse(response); +} + +export async function getAccountLoginChallenge(params) { + const response = await request( + "/account/login/challenge", + { + method: "POST", + body: params, + skipAuthRedirect: true, + }, + true, + false + ); + + return normalizeAuthResponse(response); +} + +export async function fakeAccountLogout() { + return request( + "/account/logout", + { + method: "POST", + skipAuthRedirect: true, + }, + false, + false + ); } export async function fakeRegister(params) { diff --git a/web/src/services/cluster.js b/web/src/services/cluster.js index 943526ee..ef59161c 100644 --- a/web/src/services/cluster.js +++ b/web/src/services/cluster.js @@ -10,8 +10,22 @@ export async function getClusterVersion(params) { export async function getClusterMetrics(params) { let id = params.cluster_id; delete params["cluster_id"]; + const rawMin = params?.timeRange?.min; + const rawMax = params?.timeRange?.max; + const min = + `${rawMin ?? ""}`.toLowerCase() === "auto" + ? "auto" + : Number.isFinite(rawMin) + ? rawMin + : Date.now() - 15 * 60 * 1000; + const max = + `${rawMax ?? ""}`.toLowerCase() === "auto" + ? "auto" + : Number.isFinite(rawMax) + ? rawMax + : Date.now(); return request( - `${ESPrefix}/${id}/metrics?min=${params.timeRange.min}&max=${params.timeRange.max}`, + `${ESPrefix}/${id}/metrics?min=${min}&max=${max}`, { method: "GET", } diff --git a/web/src/utils/auth_session.js b/web/src/utils/auth_session.js new file mode 100644 index 00000000..2e036ea6 --- /dev/null +++ b/web/src/utils/auth_session.js @@ -0,0 +1,432 @@ +const LOGIN_RESPONSE_KEY = "login-response"; +const LOGIN_ACTIVITY_KEY = "login-last-activity-at"; +const REFRESH_ENDPOINT = "/account/refresh"; +const MIN_REFRESH_THRESHOLD_MS = 5 * 60 * 1000; +const ACTIVITY_IDLE_TIMEOUT_MS = 15 * 60 * 1000; +const ACTIVITY_PERSIST_INTERVAL_MS = 30 * 1000; +const REFRESH_RETRY_INTERVAL_MS = 30 * 1000; + +let refreshPromise = null; +let refreshTimer = null; +let managerStarted = false; +let lastRecordedActivityAt = 0; +let lastRefreshAttemptAt = 0; + +function canUseWindow() { + return typeof window !== "undefined"; +} + +function parseJSON(value) { + if (!value) { + return null; + } + try { + return JSON.parse(value); + } catch (e) { + return null; + } +} + +function decodeJwtPayload(token) { + if (!token || !canUseWindow()) { + return null; + } + const segments = token.split("."); + if (segments.length < 2) { + return null; + } + try { + const base64 = segments[1].replace(/-/g, "+").replace(/_/g, "/"); + const padded = base64.padEnd(base64.length + ((4 - (base64.length % 4)) % 4), "="); + return JSON.parse(window.atob(padded)); + } catch (e) { + return null; + } +} + +function toUnixMs(value) { + const numericValue = Number(value); + if (!Number.isFinite(numericValue) || numericValue <= 0) { + return 0; + } + return numericValue > 1e12 ? numericValue : numericValue * 1000; +} + +function normalizeLoginResponse(response, now = Date.now()) { + if (!response || typeof response !== "object") { + return null; + } + const normalized = { + ...response, + }; + let expiresAt = toUnixMs(normalized.expires_at); + + if (!expiresAt && normalized.access_token) { + const payload = decodeJwtPayload(normalized.access_token); + expiresAt = toUnixMs(payload?.exp); + } + + if (!expiresAt) { + const expireInSeconds = Number(normalized.expire_in); + if (Number.isFinite(expireInSeconds) && expireInSeconds > 0) { + expiresAt = now + expireInSeconds * 1000; + } + } + + if (expiresAt) { + normalized.expires_at = expiresAt; + } + + return normalized; +} + +function getStoredActivityAt() { + if (!canUseWindow()) { + return 0; + } + return Number(window.localStorage.getItem(LOGIN_ACTIVITY_KEY) || 0); +} + +function setStoredActivityAt(activityAt) { + if (!canUseWindow()) { + return; + } + window.localStorage.setItem(LOGIN_ACTIVITY_KEY, `${activityAt}`); +} + +function clearRefreshTimer() { + if (refreshTimer) { + window.clearTimeout(refreshTimer); + refreshTimer = null; + } +} + +function getRefreshThresholdMs(loginResponse) { + const tokenLifetimeMs = Number(loginResponse?.expire_in || 0) * 1000; + if (tokenLifetimeMs > 0) { + return Math.max(MIN_REFRESH_THRESHOLD_MS, tokenLifetimeMs / 2); + } + return MIN_REFRESH_THRESHOLD_MS; +} + +function getNormalizedRequestPath(requestUrl) { + if (!canUseWindow()) { + return requestUrl; + } + + const resolvedUrl = new URL(requestUrl, window.location.origin); + let pathname = resolvedUrl.pathname; + const basePath = + window.routerBase && window.routerBase !== "/" + ? window.routerBase.replace(/\/$/, "") + : ""; + + if (basePath && pathname.startsWith(`${basePath}/`)) { + pathname = pathname.slice(basePath.length); + } else if (basePath && pathname === basePath) { + pathname = "/"; + } + + return pathname; +} + +function buildUrlWithBasePath(relativeUrl) { + if (!canUseWindow() || /^(https?:)?\/\//.test(relativeUrl)) { + return relativeUrl; + } + const basePath = + window.routerBase && window.routerBase !== "/" + ? window.routerBase.replace(/\/$/, "") + : ""; + const cleanUrl = relativeUrl.replace(/^\//, ""); + return `${basePath}/${cleanUrl}`; +} + +function isRefreshPath(requestUrl) { + const path = getNormalizedRequestPath(requestUrl); + return ( + path === REFRESH_ENDPOINT || + path === "/account/login" || + path === "/account/login/challenge" || + path === "/account/logout" + ); +} + +function hasRecentUserActivity(now = Date.now()) { + const lastActivityAt = getStoredActivityAt(); + return lastActivityAt > 0 && now-lastActivityAt <= ACTIVITY_IDLE_TIMEOUT_MS; +} + +function dispatchLogout() { + if ( + !canUseWindow() || + window.location.href.indexOf("user/login") !== -1 || + !window.g_app?._store + ) { + return; + } + window.g_app._store.dispatch({ + type: "login/logout", + payload: { + skipServerLogout: true, + }, + }); +} + +function scheduleNextRefresh() { + if (!canUseWindow()) { + return; + } + clearRefreshTimer(); + const loginResponse = getStoredLoginResponse(); + const expiresAt = Number(loginResponse?.expires_at || 0); + if (!loginResponse?.access_token || !expiresAt) { + return; + } + + const now = Date.now(); + const remainingMs = expiresAt - now; + if (remainingMs <= 0) { + return; + } + const refreshThresholdMs = getRefreshThresholdMs(loginResponse); + + const delayMs = + remainingMs > refreshThresholdMs + ? remainingMs - refreshThresholdMs + : Math.min(REFRESH_RETRY_INTERVAL_MS, remainingMs); + + refreshTimer = window.setTimeout(() => { + refreshTimer = null; + if (!hasRecentUserActivity()) { + scheduleNextRefresh(); + return; + } + refreshAccessToken().catch(() => { + scheduleNextRefresh(); + }); + }, Math.max(1000, delayMs)); +} + +async function requestTokenRefresh(currentToken) { + if (!canUseWindow() || !currentToken) { + return null; + } + + const refreshUrl = buildUrlWithBasePath(REFRESH_ENDPOINT); + if (new URL(refreshUrl, window.location.origin).protocol !== "https:") { + return null; + } + + const response = await window.fetch(refreshUrl, { + method: "POST", + credentials: "include", + headers: { + Accept: "application/json", + "Content-Type": "application/json; charset=utf-8", + Authorization: `Bearer ${currentToken}`, + }, + }); + + let payload = null; + try { + payload = await response.json(); + } catch (e) { + payload = null; + } + + if (!response.ok) { + const error = new Error( + payload?.error?.reason || response.statusText || "failed to refresh token" + ); + error.status = response.status; + throw error; + } + + return payload; +} + +export function getStoredLoginResponse() { + if (!canUseWindow()) { + return null; + } + const parsed = parseJSON(window.localStorage.getItem(LOGIN_RESPONSE_KEY)); + const normalized = normalizeLoginResponse(parsed); + if (!normalized?.access_token) { + return null; + } + + if (parsed?.expires_at !== normalized.expires_at) { + window.localStorage.setItem(LOGIN_RESPONSE_KEY, JSON.stringify(normalized)); + } + + return normalized; +} + +export function storeLoginResponse(response, { recordActivity = true } = {}) { + if (!canUseWindow()) { + return null; + } + const parsed = + typeof response === "string" + ? parseJSON(response) + : response; + + if (!parsed?.access_token) { + clearStoredLoginResponse(); + return null; + } + + const normalized = normalizeLoginResponse(parsed); + if (!normalized) { + clearStoredLoginResponse(); + return null; + } + + window.localStorage.setItem(LOGIN_RESPONSE_KEY, JSON.stringify(normalized)); + + if (recordActivity) { + setStoredActivityAt(Date.now()); + } + + scheduleNextRefresh(); + return normalized; +} + +export function clearStoredLoginResponse() { + if (!canUseWindow()) { + return; + } + window.localStorage.removeItem(LOGIN_RESPONSE_KEY); + window.localStorage.removeItem(LOGIN_ACTIVITY_KEY); + clearRefreshTimer(); +} + +export function getAuthorizationToken() { + return getStoredLoginResponse()?.access_token || ""; +} + +export function recordUserActivity({ force = false } = {}) { + if (!canUseWindow() || !getStoredLoginResponse()?.access_token) { + return; + } + + const now = Date.now(); + if (!force && now - lastRecordedActivityAt < ACTIVITY_PERSIST_INTERVAL_MS) { + return; + } + + lastRecordedActivityAt = now; + setStoredActivityAt(now); + + const loginResponse = getStoredLoginResponse(); + const refreshThresholdMs = getRefreshThresholdMs(loginResponse); + if ( + loginResponse?.expires_at && + loginResponse.expires_at - now <= refreshThresholdMs + ) { + refreshAccessToken().catch(() => {}); + return; + } + + scheduleNextRefresh(); +} + +export async function refreshAccessToken({ force = false } = {}) { + const loginResponse = getStoredLoginResponse(); + if (!loginResponse?.access_token) { + return null; + } + + const now = Date.now(); + const expiresAt = Number(loginResponse.expires_at || 0); + const refreshThresholdMs = getRefreshThresholdMs(loginResponse); + if ( + !force && + ( + !expiresAt || + expiresAt - now > refreshThresholdMs || + !hasRecentUserActivity(now) || + now - lastRefreshAttemptAt < REFRESH_RETRY_INTERVAL_MS + ) + ) { + return loginResponse; + } + + if (refreshPromise) { + return refreshPromise; + } + + lastRefreshAttemptAt = now; + const previousToken = loginResponse.access_token; + + refreshPromise = requestTokenRefresh(previousToken) + .then((response) => { + if (response?.status === "ok" && response.access_token) { + return storeLoginResponse(response, { recordActivity: false }); + } + return getStoredLoginResponse(); + }) + .catch((error) => { + const latest = getStoredLoginResponse(); + if (latest?.access_token && latest.access_token !== previousToken) { + return latest; + } + if (Number(error?.status) === 401) { + dispatchLogout(); + return null; + } + return latest; + }) + .finally(() => { + refreshPromise = null; + scheduleNextRefresh(); + }); + + return refreshPromise; +} + +export function ensureFreshAccessToken(requestUrl) { + if (!getStoredLoginResponse()?.access_token || isRefreshPath(requestUrl)) { + return Promise.resolve(getStoredLoginResponse()); + } + return refreshAccessToken(); +} + +export function startActivityAwareTokenRefresh() { + if (!canUseWindow() || managerStarted) { + return; + } + + managerStarted = true; + + const handleActivity = () => { + recordUserActivity(); + }; + + ["mousedown", "keydown", "scroll", "touchstart", "mousemove"].forEach( + (eventName) => { + window.addEventListener(eventName, handleActivity, { passive: true }); + } + ); + + document.addEventListener("visibilitychange", () => { + if (document.visibilityState === "visible") { + recordUserActivity({ force: true }); + refreshAccessToken().catch(() => {}); + } else { + scheduleNextRefresh(); + } + }); + + window.addEventListener("storage", (event) => { + if ( + event.key === LOGIN_RESPONSE_KEY || + event.key === LOGIN_ACTIVITY_KEY + ) { + scheduleNextRefresh(); + } + }); + + scheduleNextRefresh(); +} diff --git a/web/src/utils/authority.js b/web/src/utils/authority.js index 73f07abd..925dbe9f 100644 --- a/web/src/utils/authority.js +++ b/web/src/utils/authority.js @@ -1,33 +1,113 @@ import request from "./request"; +import { + getAuthorizationToken, + getStoredLoginResponse, +} from "./auth_session"; +import { setSetupRequired } from "./setup"; + +const APPLICATION_AUTH_KEY = "infini-auth"; +const APPLICATION_ROLLUP_KEY = "infini-rollup-enabled"; +const ENTERPRISE_TASK_MANAGER_KEY = "infini-enterprise-task-manager-enabled"; +export const APPLICATION_SETTINGS_UPDATED_EVENT = + "console:application-settings-updated"; +let applicationSettingsPromise = null; +let applicationSettingsCache = null; + +export function invalidateApplicationSettingsCache() { + applicationSettingsPromise = null; + applicationSettingsCache = null; +} + +function persistApplicationSettings(res) { + localStorage.setItem(APPLICATION_AUTH_KEY, `${res.auth_enabled}`); + if (typeof res.setup_required !== "undefined") { + setSetupRequired(`${!!res.setup_required}`); + } + localStorage.setItem( + APPLICATION_ROLLUP_KEY, + `${!!res.system_cluster?.rollup_enabled}` + ); + localStorage.setItem( + ENTERPRISE_TASK_MANAGER_KEY, + `${!!res.enterprise_plugins?.task_manager}` + ); +} + +function normalizeAuthorityValue(authority) { + if (authority == null || authority === "") { + return []; + } + + let normalized = authority; + if (typeof normalized === "string") { + try { + normalized = JSON.parse(normalized); + } catch (e) { + normalized = authority; + } + } + + if (typeof normalized === "string") { + return [normalized]; + } + + if (Array.isArray(normalized)) { + return normalized.filter((item) => typeof item === "string" && item); + } + + return []; +} + +export function extractAuthorityFromResponse(payload) { + const source = payload?._source || payload; + const candidates = [ + source?.privilege, + payload?.privilege, + source?.permissions, + payload?.permissions, + source?.currentAuthority, + payload?.currentAuthority, + ]; + + for (const candidate of candidates) { + const authority = normalizeAuthorityValue(candidate); + if (authority.length > 0) { + return authority; + } + } + + return []; +} // use localStorage to store the authority info, which might be sent from server in actual project. export function getAuthority(str) { - // return localStorage.getItem('infini-console-authority') || ['admin', 'user']; const authorityString = typeof str === "undefined" ? localStorage.getItem("infini-console-authority") : str; - // authorityString could be admin, "admin", ["admin"] - let authority; - try { - authority = JSON.parse(authorityString); - } catch (e) { - authority = authorityString; + const authority = normalizeAuthorityValue(authorityString); + if (authority.length > 0 || typeof str !== "undefined") { + return authority; } - if (typeof authority === "string") { - return [authority]; - } - return authority; + return extractAuthorityFromResponse(getStoredLoginResponse()); } export function setAuthority(authority) { - const proAuthority = typeof authority === "string" ? [authority] : authority; + const proAuthority = normalizeAuthorityValue(authority); return localStorage.setItem( "infini-console-authority", JSON.stringify(proAuthority) ); } +export function syncAuthorityFromResponse(payload) { + const authority = extractAuthorityFromResponse(payload); + if (authority.length > 0) { + setAuthority(authority); + } + return authority; +} + export function hasAuthority(authority) { if (getAuthEnabled() === "true") { const userAuthority = getAuthority() || []; @@ -37,49 +117,60 @@ export function hasAuthority(authority) { } export function getAuthEnabled() { - return localStorage.getItem("infini-auth"); + return localStorage.getItem(APPLICATION_AUTH_KEY); } export function isLogin() { - const responseStr = localStorage.getItem("login-response"); - if (responseStr) { - let loginResponse = null; - try { - loginResponse = JSON.parse(responseStr); - if (loginResponse?.username && loginResponse?.status == "ok") { - return true; - } - } catch (err) { - console.error(err); - } + const loginResponse = getStoredLoginResponse(); + if (loginResponse?.username && loginResponse?.status == "ok") { + return true; } return false; } export function getAuthorizationHeader() { - const responseStr = localStorage.getItem("login-response"); - if (responseStr) { - let loginResponse = null; - try { - loginResponse = JSON.parse(responseStr); - } catch (err) { - console.error(err); - } - if (loginResponse) { - return "Bearer " + loginResponse.access_token; - } + const accessToken = getAuthorizationToken(); + if (accessToken) { + return "Bearer " + accessToken; } return ""; } export function getRollupEnabled() { - return localStorage.getItem("infini-rollup-enabled"); + return localStorage.getItem(APPLICATION_ROLLUP_KEY); } -(async function() { - const res = await request("/setting/application"); - if (res && !res.error) { - localStorage.setItem("infini-auth", res.auth_enabled); - localStorage.setItem('infini-rollup-enabled', res.system_cluster?.rollup_enabled || false) +export function getEnterpriseTaskManagerEnabled() { + return localStorage.getItem(ENTERPRISE_TASK_MANAGER_KEY); +} + +export async function refreshApplicationSettings(force = false) { + if (applicationSettingsPromise) { + return applicationSettingsPromise; } -})(); + + if (!force && applicationSettingsCache) { + return applicationSettingsCache; + } + + applicationSettingsPromise = request("/setting/application") + .then((res) => { + if (res && !res.error) { + persistApplicationSettings(res); + applicationSettingsCache = res; + if (typeof window !== "undefined") { + window.dispatchEvent( + new CustomEvent(APPLICATION_SETTINGS_UPDATED_EVENT, { + detail: res, + }) + ); + } + } + return res; + }) + .finally(() => { + applicationSettingsPromise = null; + }); + + return applicationSettingsPromise; +} diff --git a/web/src/utils/authority.test.js b/web/src/utils/authority.test.js index 8a6cd41f..9ad0d2dd 100644 --- a/web/src/utils/authority.test.js +++ b/web/src/utils/authority.test.js @@ -1,8 +1,8 @@ -import { getAuthority } from './authority'; +import { extractAuthorityFromResponse, getAuthority } from './authority'; describe('getAuthority should be strong', () => { it('empty', () => { - expect(getAuthority(null)).toEqual(['admin']); // default value + expect(getAuthority(null)).toEqual([]); }); it('string', () => { expect(getAuthority('admin')).toEqual(['admin']); @@ -17,3 +17,23 @@ describe('getAuthority should be strong', () => { expect(getAuthority('["admin", "guest"]')).toEqual(['admin', 'guest']); }); }); + +describe('extractAuthorityFromResponse', () => { + it('prefers privilege from login response', () => { + expect( + extractAuthorityFromResponse({ + privilege: ['workbench:all', 'cluster.overview:all'], + }) + ).toEqual(['workbench:all', 'cluster.overview:all']); + }); + + it('falls back to permissions from current user profile', () => { + expect( + extractAuthorityFromResponse({ + _source: { + permissions: ['system.security:all'], + }, + }) + ).toEqual(['system.security:all']); + }); +}); diff --git a/web/src/utils/password.js b/web/src/utils/password.js new file mode 100644 index 00000000..dca229a9 --- /dev/null +++ b/web/src/utils/password.js @@ -0,0 +1,68 @@ +const encoder = new TextEncoder(); + +const toHex = (buffer) => + Array.from(buffer) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); + +const fromHex = (hex) => { + const bytes = new Uint8Array(hex.length / 2); + for (let i = 0; i < bytes.length; i += 1) { + bytes[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); + } + return bytes; +}; + +async function derivePasswordVerifier(password, salt, iterations) { + if (!window.crypto?.subtle) { + throw new Error("Web Crypto API unavailable"); + } + + const passwordKey = await window.crypto.subtle.importKey( + "raw", + encoder.encode(password), + "PBKDF2", + false, + ["deriveBits"] + ); + const bits = await window.crypto.subtle.deriveBits( + { + name: "PBKDF2", + salt: encoder.encode(salt), + iterations, + hash: "SHA-256", + }, + passwordKey, + 256 + ); + + return toHex(new Uint8Array(bits)); +} + +export async function buildPasswordProof({ + password, + username, + challengeId, + nonce, + salt, + iterations, +}) { + const verifier = await derivePasswordVerifier(password, salt, iterations); + const hmacKey = await window.crypto.subtle.importKey( + "raw", + fromHex(verifier), + { + name: "HMAC", + hash: "SHA-256", + }, + false, + ["sign"] + ); + const signature = await window.crypto.subtle.sign( + "HMAC", + hmacKey, + encoder.encode(`${username}:${challengeId}:${nonce}`) + ); + + return toHex(new Uint8Array(signature)); +} diff --git a/web/src/utils/request.js b/web/src/utils/request.js index 876aaaea..4470804f 100644 --- a/web/src/utils/request.js +++ b/web/src/utils/request.js @@ -5,6 +5,7 @@ import hash from "hash.js"; import { isAntdPro } from "./utils"; import { formatMessage } from "umi/locale"; import { getAuthorizationHeader } from "./authority"; +import { ensureFreshAccessToken } from "./auth_session"; import * as uuid from 'uuid'; export const formatResponse = (response) => { @@ -31,6 +32,215 @@ export const formatResponse = (response) => { } } +const secureTransportErrorReason = + "Sensitive requests require HTTPS. Enable Console HTTPS or put Console behind an HTTPS reverse proxy."; +const ERROR_NOTIFICATION_DEDUPE_MS = 4000; +const DATATOOLS_LICENSE_REQUIRED_REASON = + "a valid license is required to use DataTools"; +const DATATOOLS_LICENSE_MODAL_EVENT = "console:datatools-license-required"; +const DATATOOLS_LICENSE_MODAL_DEDUPE_MS = 3000; +const recentErrorNotifications = new Map(); +let lastDataToolsLicenseModalAt = 0; + +const sensitiveRequestRules = [ + { method: "POST", pattern: /^\/account\/login\/challenge$/ }, + { method: "POST", pattern: /^\/account\/login$/ }, + { method: "POST", pattern: /^\/account\/refresh$/ }, + { method: "PUT", pattern: /^\/account\/password$/ }, + { method: "POST", pattern: /^\/user$/ }, + { method: "PUT", pattern: /^\/user\/[^/]+\/password$/ }, + { method: "POST", pattern: /^\/credential$/ }, + { method: "PUT", pattern: /^\/credential\/[^/]+$/ }, + { method: "POST", pattern: /^\/setup\/_validate$/ }, + { method: "POST", pattern: /^\/setup\/_initialize$/ }, + { method: "POST", pattern: /^\/setup\/_validate_secret$/ }, + { method: "POST", pattern: /^\/setup\/_initialize_template$/ }, + { method: "POST", pattern: /^\/elasticsearch\/$/ }, + { method: "PUT", pattern: /^\/elasticsearch\/[^/]+$/ }, + { method: "POST", pattern: /^\/elasticsearch\/try_connect$/ }, + { method: "POST", pattern: /^\/email\/server$/ }, + { method: "POST", pattern: /^\/email\/server\/_test$/ }, + { method: "PUT", pattern: /^\/email\/server\/[^/]+$/ }, + { method: "PUT", pattern: /^\/setting\/system\/rollup$/ }, + { method: "PUT", pattern: /^\/setting\/system\/retention$/ }, + { method: "POST", pattern: /^\/setting\/system\/local_templates\/_refresh$/ }, +]; + +const publicRequestRules = [ + { method: "GET", pattern: /^\/setting\/application$/ }, + { method: "GET", pattern: /^\/_info$/ }, + { method: "GET", pattern: /^\/_license\/info$/ }, + { method: "POST", pattern: /^\/account\/login\/challenge$/ }, + { method: "POST", pattern: /^\/account\/login$/ }, +]; + +const getNormalizedRequestPath = (requestUrl) => { + if (typeof window === "undefined") { + return requestUrl; + } + + const resolvedUrl = new URL(requestUrl, window.location.origin); + let pathname = resolvedUrl.pathname; + const basePath = + window.routerBase && window.routerBase !== "/" + ? window.routerBase.replace(/\/$/, "") + : ""; + + if (basePath && pathname.startsWith(`${basePath}/`)) { + pathname = pathname.slice(basePath.length); + } else if (basePath && pathname === basePath) { + pathname = "/"; + } + + return pathname; +}; + +const requestUsesSecureTransport = (requestUrl) => { + if (typeof window === "undefined") { + return true; + } + + return new URL(requestUrl, window.location.origin).protocol === "https:"; +}; + +const getCurrentRoutePath = () => { + if (typeof window === "undefined") { + return "/"; + } + + const hash = window.location.hash || ""; + const hashPath = hash.startsWith("#") ? hash.slice(1) : hash; + if (hashPath.startsWith("/")) { + return getNormalizedRequestPath(hashPath); + } + + return getNormalizedRequestPath(window.location.pathname || "/"); +}; + +const isDataToolsRoute = () => { + return /^\/data_tools(\/|$)/.test(getCurrentRoutePath()); +}; + +const openDataToolsLicenseModal = () => { + if (typeof window === "undefined") { + return; + } + + const now = Date.now(); + if (now - lastDataToolsLicenseModalAt < DATATOOLS_LICENSE_MODAL_DEDUPE_MS) { + return; + } + + lastDataToolsLicenseModalAt = now; + window.dispatchEvent(new CustomEvent(DATATOOLS_LICENSE_MODAL_EVENT)); +}; + +const requestRequiresSecureTransport = (requestUrl, method = "GET") => { + const normalizedMethod = method.toUpperCase(); + const normalizedPath = getNormalizedRequestPath(requestUrl); + + return sensitiveRequestRules.some( + ({ method: sensitiveMethod, pattern }) => + sensitiveMethod === normalizedMethod && pattern.test(normalizedPath) + ); +}; + +const requestShouldSkipAuthorization = ( + requestUrl, + method = "GET", + option = {} +) => { + if (option?.skipAuthorization === true) { + return true; + } + + const normalizedMethod = method.toUpperCase(); + const normalizedPath = getNormalizedRequestPath(requestUrl); + + return publicRequestRules.some( + ({ method: publicMethod, pattern }) => + publicMethod === normalizedMethod && pattern.test(normalizedPath) + ); +}; + +const getInsecureTransportResponse = () => ({ + status: "error", + success: false, + currentAuthority: "guest", + error: { + reason: secureTransportErrorReason, + }, +}); + +const cleanupRecentErrorNotifications = (now = Date.now()) => { + recentErrorNotifications.forEach((timestamp, key) => { + if (now - timestamp >= ERROR_NOTIFICATION_DEDUPE_MS) { + recentErrorNotifications.delete(key); + } + }); +}; + +const showErrorNotification = ({ + message, + description, + style, + dedupeKey, +}) => { + const now = Date.now(); + cleanupRecentErrorNotifications(now); + const key = dedupeKey || `${message}`; + const lastShownAt = recentErrorNotifications.get(key); + if (lastShownAt && now - lastShownAt < ERROR_NOTIFICATION_DEDUPE_MS) { + return; + } + recentErrorNotifications.set(key, now); + notification.error({ + key, + placement: "topRight", + message, + description, + style, + }); +}; + +const fetchReplayNonce = async (requestUrl, requestMethod, authorizationHeader) => { + const nonceEndpoint = buildUrlWithBasePath("/account/replay_nonce"); + const headers = { + Accept: "application/json", + "Content-Type": "application/json; charset=utf-8", + }; + if (authorizationHeader) { + headers.Authorization = authorizationHeader; + } + + const response = await fetch(nonceEndpoint, { + method: "POST", + credentials: "include", + headers, + body: JSON.stringify({ + method: requestMethod, + path: getNormalizedRequestPath(requestUrl), + }), + }); + + let payload = null; + try { + payload = await response.json(); + } catch (error) { + payload = null; + } + + if (!response.ok || !payload?.nonce) { + const reason = + payload?.error?.reason || + payload?.message || + "failed to fetch replay nonce"; + throw new Error(reason); + } + + return payload.nonce; +}; + const checkStatus = async (response, noticeable, option={}) => { const codeMessage = { 200: formatMessage({ id: "app.message.http.status.200" }), @@ -53,25 +263,42 @@ const checkStatus = async (response, noticeable, option={}) => { if (response.status >= 200 && response.status < 300) { return response; } + if (response.status === 403 && isDataToolsRoute()) { + let jsonRes = null; + try { + jsonRes = await response.clone().json(); + } catch (error) { + jsonRes = null; + } + if (jsonRes?.error?.reason === DATATOOLS_LICENSE_REQUIRED_REASON) { + openDataToolsLicenseModal(); + return response; + } + } + if (response.status === 409) { + let jsonRes = null; + try { + jsonRes = await response.clone().json(); + } catch (error) { + jsonRes = null; + } + if (jsonRes?.error?.reason) { + return response; + } + } if (response.status == 500) { - const jsonRes = await response.clone().json(); - if (jsonRes.error && !jsonRes.stack) { + let jsonRes = null; + try { + jsonRes = await response.clone().json(); + } catch (error) { + jsonRes = null; + } + if (jsonRes?.error && !jsonRes.stack) { if (noticeable) { - let desc = ""; - if (typeof jsonRes.error == "string") { - desc = jsonRes.error; - } else { - if (jsonRes.error?.reason) { - desc = ( -
    - {jsonRes.error.reason} -
    - ); - } else { - desc = JSON.stringify(jsonRes.error); - } - } - desc = ( + const friendlyMessage = jsonRes.error?.key + ? formatMessage({ id: jsonRes.error.key }) + : formatMessage({ id: "app.message.http.status.500" }); + const desc = (
    { {response.status}
    - {desc} +
    + {friendlyMessage} +
    ); - notification.error({ + showErrorNotification({ message: formatMessage({ id: "app.message.http.request.error" }), description: desc, style: { wordBreak: "break-all" }, + dedupeKey: `http-500:${jsonRes.error?.key || jsonRes.error?.reason || friendlyMessage}`, }); } return response; @@ -104,10 +334,11 @@ const checkStatus = async (response, noticeable, option={}) => { option.hasOwnProperty("showErrorInner") && option.showErrorInner === true ) { - notification.error({ + showErrorNotification({ message: response.statusText, description: errortext, style: { wordBreak: "break-all" }, + dedupeKey: `http-inner:${response.status}:${response.statusText}:${errortext}`, }); return response; } @@ -136,10 +367,11 @@ const checkStatus = async (response, noticeable, option={}) => { {errortext}
    ); - notification.error({ + showErrorNotification({ message: `${formatMessage({ id: "app.message.http.request.error" })}`, description: desc, style: { wordBreak: "break-all" }, + dedupeKey: `http-status:${response.status}:${errortext}`, }); } } @@ -264,10 +496,11 @@ export default function request( signal, }; const newOptions = { ...defaultOptions, ...options }; + const requestMethod = (newOptions.method || "GET").toUpperCase(); if ( - newOptions.method === "POST" || - newOptions.method === "PUT" || - newOptions.method === "DELETE" + requestMethod === "POST" || + requestMethod === "PUT" || + requestMethod === "DELETE" ) { if (!(newOptions.body instanceof FormData)) { newOptions.headers = { @@ -290,9 +523,17 @@ export default function request( "Accept-Encoding": "gzip, deflate, br", ...newOptions.headers, }; - const authorizationHeader = getAuthorizationHeader(); - if (authorizationHeader) { - newOptions.headers["Authorization"] = authorizationHeader; + const requiresSecureTransport = requestRequiresSecureTransport(url, requestMethod); + if (requiresSecureTransport && !requestUsesSecureTransport(url)) { + if (noticeable) { + showErrorNotification({ + message: "HTTPS required", + description: secureTransportErrorReason, + style: { wordBreak: "break-all" }, + dedupeKey: `https-required:${getNormalizedRequestPath(url)}`, + }); + } + return Promise.resolve(getInsecureTransportResponse()); } const expirys = options.expirys && 60; @@ -311,8 +552,11 @@ export default function request( } } - return ( - fetch(url, newOptions) + const sendRequest = (nonce) => { + if (nonce) { + newOptions.headers["X-Request-Nonce"] = nonce; + } + return fetch(url, newOptions) .then((res) => checkStatus(res, noticeable, option)) // .then(response => cachedSave(response, hashcode)) .then((response) => { @@ -332,11 +576,15 @@ export default function request( //connection refused const err = new Error(); err.name = "ERR_CONNECTION_REFUSED"; - err.message = "Failed to connnect server"; + err.message = formatMessage({ + id: "error.request.connection_refused", + }); if (typeof setGlobalHealth === "function") { setGlobalHealth({ error: err.name, - desc: err.message, + desc: formatMessage({ + id: "error.request.connection_refused.tip", + }), }); } return err; @@ -344,7 +592,7 @@ export default function request( if (status === "AbortError") { if (noticeable) { - notification.error({ + showErrorNotification({ message: formatMessage({ id: "app.message.http.request.timeout", }), @@ -355,6 +603,7 @@ export default function request( "\r\nURL:" + url, style: { wordBreak: "break-all" }, + dedupeKey: `request-timeout:${url}`, }); } return; @@ -364,9 +613,15 @@ export default function request( if (status === 401) { // @HACK /* eslint-disable no-underscore-dangle */ - if (location.href.indexOf("user/login") === -1) { + if ( + option?.skipAuthRedirect !== true && + location.href.indexOf("user/login") === -1 + ) { window.g_app._store.dispatch({ type: "login/logout", + payload: { + skipServerLogout: true, + }, }); } } @@ -375,7 +630,9 @@ export default function request( if (location.href.includes('/insight/dashboard') && url.includes('/visualization/data')) { return e.rawResponse; } - router.push("/exception/403"); + if (option?.skipAuthRedirect !== true) { + router.push("/exception/403"); + } } if (status == 500) { router.push({ @@ -392,6 +649,43 @@ export default function request( return e.rawResponse; } return e.response; - }) - ); + }); + }; + + const executeRequest = () => { + const authorizationHeader = requestShouldSkipAuthorization( + url, + requestMethod, + option + ) + ? "" + : getAuthorizationHeader(); + if (authorizationHeader) { + newOptions.headers["Authorization"] = authorizationHeader; + } else { + delete newOptions.headers["Authorization"]; + } + + const noncePromise = requiresSecureTransport + ? fetchReplayNonce(url, requestMethod, authorizationHeader) + : Promise.resolve(null); + + const handleNonceError = (error) => { + if (noticeable) { + showErrorNotification({ + message: "Request rejected", + description: error?.message || "failed to fetch replay nonce", + style: { wordBreak: "break-all" }, + dedupeKey: `request-rejected:${getNormalizedRequestPath(url)}:${error?.message || "failed to fetch replay nonce"}`, + }); + } + return getInsecureTransportResponse(); + }; + + return noncePromise.then((nonce) => sendRequest(nonce), handleNonceError); + }; + + return Promise.resolve( + option?.skipAuthRefresh === true ? null : ensureFreshAccessToken(url) + ).then(executeRequest); } diff --git a/web/src/utils/setup.js b/web/src/utils/setup.js index 79807e34..165bba0e 100644 --- a/web/src/utils/setup.js +++ b/web/src/utils/setup.js @@ -12,4 +12,22 @@ export function isSystemCluster(clusterID){ export function getSystemClusterID() { return 'infini_default_system_cluster'; -} \ No newline at end of file +} + +export function getPreferredCluster(clusters = [], options = {}) { + if (!Array.isArray(clusters) || clusters.length === 0) { + return null; + } + + const { selectedClusterID, targetClusterID } = options; + const findCluster = (clusterID) => + clusterID ? clusters.find((item) => item.id === clusterID) : null; + + return ( + findCluster(selectedClusterID) || + findCluster(targetClusterID) || + findCluster(getSystemClusterID()) || + clusters[0] || + null + ); +} diff --git a/web/src/utils/treemap_title.js b/web/src/utils/treemap_title.js new file mode 100644 index 00000000..008113ee --- /dev/null +++ b/web/src/utils/treemap_title.js @@ -0,0 +1,23 @@ +import { formatMessage } from "umi/locale"; + +const TREEMAP_TITLE_MAP = { + "Search latency by index": "cluster.monitor.treemap.search_latency_by_index", + "Avg search latency by index": + "cluster.monitor.treemap.search_latency_by_index", +}; + +export const getLocalizedTreemapTitle = (title) => { + if (!title) { + return title; + } + + const localeId = TREEMAP_TITLE_MAP[title]; + if (!localeId) { + return title; + } + + return formatMessage({ + id: localeId, + defaultMessage: title, + }); +}; diff --git a/web/src/utils/utils.js b/web/src/utils/utils.js index 5f5062c4..2acc4993 100644 --- a/web/src/utils/utils.js +++ b/web/src/utils/utils.js @@ -243,6 +243,35 @@ export function removeHttpSchema(val) { return val?.replace(/^https?:\/\//i, ""); } +export function normalizeEndpointHost(val) { + const raw = `${val || ""}`.trim(); + if (!raw) { + return ""; + } + + const candidate = /^[a-z][a-z\d+\-.]*:\/\//i.test(raw) + ? raw + : `http://${raw}`; + + try { + const parsed = new URL(candidate); + if (parsed.host) { + return parsed.host.trim(); + } + } catch (e) {} + + return removeHttpSchema(raw).replace(/[/?#].*$/, "").trim(); +} + +export function normalizeEndpointHosts(vals = []) { + return (vals || []).map((item) => normalizeEndpointHost(item)).filter(Boolean); +} + +export function isValidEndpointHost(val) { + const host = normalizeEndpointHost(val); + return /^(?:[\w.\-_~%]+(?::\d+)?|\[[0-9A-Fa-f:.%]+\](?::\d+)?)$/.test(host); +} + export function addHttpSchema(val, isTLS = false) { let schema = isTLS ? "https" : "http"; val = schema + "://" + removeHttpSchema(val);