Relay

Makefile

@@ -2,7 +2,7 @@ all: build
 .PHONY: all build
 
 BUILD_IMAGE ?= colonyos/colonies
-PUSH_IMAGE ?= colonyos/colonies:v1.9.12
+PUSH_IMAGE ?= colonyos/colonies:v1.9.13-beta9
 
 VERSION := $(shell git rev-parse --short HEAD)
 BUILDTIME := $(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
@@ -87,8 +87,9 @@ endif
 	@cd pkg/cluster; go test -v --race
 	@cd pkg/cron; go test -v --race
 	@cd pkg/fs; go test -v --race
+ifeq ($(COLONIES_FILE_STORAGE_TYPE),coloniesfs)
 	@cd pkg/fs/localstore; go test -v --race
-ifneq ($(COLONIES_FILE_STORAGE_TYPE),coloniesfs)
+else
 	@cd pkg/fs/s3; go test -v --race
 endif
 

docker-compose.env

@@ -19,7 +19,9 @@ export COLONIES_MONITOR_INTERVAL="1"
 # ============================================================================
 # DATABASE CONFIGURATION
 # ============================================================================
-export COLONIES_DB_HOST="timescaledb"
+export COLONIES_DB_TYPE="embedded"
+
+#export COLONIES_DB_HOST="timescaledb"
 export COLONIES_DB_USER="postgres"
 export COLONIES_DB_PASSWORD="rFcLGNkgsNtksg6Pgtn9CumL4xXBQ7"
 
@@ -58,6 +60,7 @@ export EXECUTOR_GPU="false"
 # ============================================================================
 # COLONYFS
 # ============================================================================
+export COLONIES_FILE_STORAGE_TYPE="coloniesfs"
 export MINIO_USER="admin"
 export MINIO_PASSWORD="admin12345"
 export AWS_S3_ENDPOINT="localhost:9000"

docker-compose.yml

@@ -51,7 +51,7 @@ services:
       "
 
   colonies-server:
-    image: colonyos/colonies:v1.9.12
+    image: colonyos/colonies:v1.9.13
     depends_on:
       - timescaledb
     environment:
@@ -87,7 +87,7 @@ services:
     command: sh -c "colonies server start --initdb --port ${COLONIES_SERVER_PORT} --relayport 25100 --etcdname server1 --etcdhost colonies-server --etcdclientport 23100 --etcdpeerport 24100 --initial-cluster server1=colonies-server:24100:25100:${COLONIES_SERVER_PORT} --etcddatadir /var/colonies/etcd --insecure"
 
   colonies-setup:
-    image: colonyos/colonies:v1.9.12
+    image: colonyos/colonies:v1.9.13
     depends_on:
       - colonies-server
     environment:

go.mod

@@ -5,11 +5,12 @@ go 1.24.0
 toolchain go1.24.8
 
 require (
-	github.com/btcsuite/btcd/btcec/v2 v2.3.2
+	github.com/btcsuite/btcd/btcec/v2 v2.3.6
 	github.com/gin-contrib/cors v1.5.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-playground/assert/v2 v2.2.0
 	github.com/go-resty/resty/v2 v2.11.0
+	github.com/google/btree v1.1.2
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.3
 	github.com/jedib0t/go-pretty/v6 v6.5.4
@@ -25,6 +26,7 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.12
 	go.etcd.io/etcd/server/v3 v3.5.12
 	golang.org/x/crypto v0.42.0
+	golang.org/x/term v0.35.0
 )
 
 require (
@@ -51,7 +53,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/google/btree v1.1.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
@@ -105,7 +106,6 @@ require (
 	golang.org/x/net v0.44.0 // indirect
 	golang.org/x/oauth2 v0.31.0 // indirect
 	golang.org/x/sys v0.36.0 // indirect
-	golang.org/x/term v0.35.0 // indirect
 	golang.org/x/text v0.29.0 // indirect
 	golang.org/x/time v0.12.0 // indirect
 	google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect

go.sum

@@ -11,8 +11,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U=
-github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
+github.com/btcsuite/btcd/btcec/v2 v2.3.6 h1:IzlsEr9olcSRKB/n7c4351F3xHKxS2lma+1UFGCYd4E=
+github.com/btcsuite/btcd/btcec/v2 v2.3.6/go.mod h1:m22FrOAiuxl/tht9wIqAoGHcbnCCaPWyauO8y2LGGtQ=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=

internal/cli/common.go

@@ -354,6 +354,19 @@ func parseEnv() {
 		FileStorageDir = fileStorageDirEnv
 	}
 
+	if RelayHost == "" {
+		RelayHost = os.Getenv("COLONIES_RELAY_HOST")
+	}
+
+	RelayPortEnvStr := os.Getenv("COLONIES_RELAY_PORT")
+	if RelayPortEnvStr != "" {
+		RelayPort, err = strconv.Atoi(RelayPortEnvStr)
+		if err != nil {
+			log.Error("Failed to parse COLONIES_RELAY_PORT")
+		}
+		CheckError(err)
+	}
+
 	monitorPortStr := os.Getenv("COLONIES_MONITOR_PORT")
 	if monitorPortStr != "" {
 		MonitorPort, err = strconv.Atoi(monitorPortStr)
@@ -486,6 +499,7 @@ func checkDevEnv() {
 		envProposal += "export COLONIES_COLONY_PRVKEY=\"ba949fa134981372d6da62b6a56f336ab4d843b22c02a4257dcf7d0d73097514\"\n"
 		envProposal += "export COLONIES_PRVKEY=\"ddf7f7791208083b6a9ed975a72684f6406a269cfa36f1b1c32045c0a71fff05\"\n"
 		envProposal += "export COLONIES_EXECUTOR_TYPE=\"cli\"\n"
+		envProposal += "export COLONIES_RELAY_HOST=\"\"\n"
 
 		fmt.Println(envProposal)
 		os.Exit(-1)

internal/cli/db.go

@@ -70,7 +70,12 @@ func parseDBEnv() {
 	if DataDir == "" {
 		home, err := os.UserHomeDir()
 		if err == nil {
-			DataDir = filepath.Join(home, ".colonies")
+			serverID := os.Getenv("COLONIES_SERVER_ID")
+			if serverID != "" {
+				DataDir = filepath.Join(home, ".colonies", serverID)
+			} else {
+				DataDir = filepath.Join(home, ".colonies")
+			}
 		}
 	}
 

internal/cli/metrics.go

@@ -0,0 +1,238 @@
+package cli
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/colonyos/colonies/pkg/core"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	metricsCmd.AddCommand(lsMetricsCmd)
+	metricsCmd.AddCommand(getMetricCmd)
+	metricsCmd.AddCommand(historyMetricsCmd)
+	metricsCmd.AddCommand(rmMetricCmd)
+	rootCmd.AddCommand(metricsCmd)
+
+	metricsCmd.PersistentFlags().StringVarP(&ColonyName, "colonyname", "", "", "Colony name")
+	metricsCmd.PersistentFlags().StringVarP(&ServerHost, "host", "", "localhost", "Server host")
+	metricsCmd.PersistentFlags().IntVarP(&ServerPort, "port", "", -1, "Server HTTP port")
+
+	lsMetricsCmd.Flags().StringVarP(&TargetExecutorName, "name", "", "", "Executor name")
+	lsMetricsCmd.Flags().BoolVarP(&JSON, "json", "", false, "Output as JSON")
+
+	getMetricCmd.Flags().StringVarP(&TargetExecutorName, "name", "", "", "Executor name")
+	getMetricCmd.MarkFlagRequired("name")
+	getMetricCmd.Flags().StringVarP(&Key, "key", "", "", "Metric key")
+	getMetricCmd.MarkFlagRequired("key")
+	getMetricCmd.Flags().BoolVarP(&JSON, "json", "", false, "Output as JSON")
+
+	historyMetricsCmd.Flags().StringVarP(&TargetExecutorName, "name", "", "", "Executor name")
+	historyMetricsCmd.MarkFlagRequired("name")
+	historyMetricsCmd.Flags().StringVarP(&Key, "key", "", "", "Metric key")
+	historyMetricsCmd.MarkFlagRequired("key")
+	historyMetricsCmd.Flags().StringVarP(&MetricPeriod, "period", "", "", "Period: day, week, or month")
+	historyMetricsCmd.MarkFlagRequired("period")
+	historyMetricsCmd.Flags().StringVarP(&FromDate, "from", "", "", "Start date (YYYY-MM-DD)")
+	historyMetricsCmd.Flags().StringVarP(&ToDate, "to", "", "", "End date (YYYY-MM-DD)")
+	historyMetricsCmd.Flags().BoolVarP(&JSON, "json", "", false, "Output as JSON")
+
+	rmMetricCmd.Flags().StringVarP(&TargetExecutorName, "name", "", "", "Executor name")
+	rmMetricCmd.MarkFlagRequired("name")
+	rmMetricCmd.Flags().StringVarP(&Key, "key", "", "", "Metric key (omit to remove all)")
+}
+
+func parsePeriod(s string) (int, error) {
+	switch s {
+	case "day":
+		return core.PERIOD_DAY, nil
+	case "week":
+		return core.PERIOD_WEEK, nil
+	case "month":
+		return core.PERIOD_MONTH, nil
+	default:
+		return 0, errors.New("Invalid period, must be: day, week, or month")
+	}
+}
+
+func defaultFrom(period int) time.Time {
+	now := time.Now().UTC()
+	switch period {
+	case core.PERIOD_DAY:
+		return now.AddDate(0, 0, -7)
+	case core.PERIOD_WEEK:
+		return now.AddDate(0, 0, -28)
+	case core.PERIOD_MONTH:
+		return now.AddDate(0, -6, 0)
+	default:
+		return now.AddDate(0, 0, -7)
+	}
+}
+
+var metricsCmd = &cobra.Command{
+	Use:   "metrics",
+	Short: "Manage executor metrics",
+	Long:  "Manage executor metrics",
+}
+
+var lsMetricsCmd = &cobra.Command{
+	Use:   "ls",
+	Short: "List metrics",
+	Long:  "List metrics for an executor, or list executors with metric counts",
+	Run: func(cmd *cobra.Command, args []string) {
+		client := setup()
+
+		if TargetExecutorName != "" {
+			metrics, err := client.GetAllMetrics(ColonyName, TargetExecutorName, PrvKey)
+			CheckError(err)
+
+			if len(metrics) == 0 {
+				log.Info("No metrics found")
+				os.Exit(0)
+			}
+
+			if JSON {
+				jsonString, err := core.ConvertMetricArrayToJSON(metrics)
+				CheckError(err)
+				fmt.Println(jsonString)
+				os.Exit(0)
+			}
+
+			printMetricKeysTable(metrics)
+		} else {
+			executors, err := client.GetExecutors(ColonyName, PrvKey)
+			CheckError(err)
+
+			if len(executors) == 0 {
+				log.Info("No executors found")
+				os.Exit(0)
+			}
+
+			var results []executorMetricCount
+			for _, executor := range executors {
+				metrics, err := client.GetMetrics(ColonyName, executor.Name, PrvKey)
+				CheckError(err)
+				results = append(results, executorMetricCount{
+					Name:  executor.Name,
+					Count: len(metrics),
+				})
+			}
+
+			if JSON {
+				type jsonResult struct {
+					Executor string `json:"executor"`
+					Metrics  int    `json:"metrics"`
+				}
+				var jsonResults []jsonResult
+				for _, r := range results {
+					jsonResults = append(jsonResults, jsonResult{
+						Executor: r.Name,
+						Metrics:  r.Count,
+					})
+				}
+				printMetricsJSON(jsonResults)
+				os.Exit(0)
+			}
+
+			printExecutorMetricCountTable(results)
+		}
+	},
+}
+
+func printMetricsJSON(v interface{}) {
+	jsonBytes, err := json.MarshalIndent(v, "", "  ")
+	CheckError(err)
+	fmt.Println(string(jsonBytes))
+}
+
+var getMetricCmd = &cobra.Command{
+	Use:   "get",
+	Short: "Get a metric value",
+	Long:  "Get a single metric value for an executor",
+	Run: func(cmd *cobra.Command, args []string) {
+		client := setup()
+
+		metric, err := client.GetMetric(ColonyName, TargetExecutorName, Key, PrvKey)
+		CheckError(err)
+
+		if JSON {
+			jsonString, err := metric.ToJSON()
+			CheckError(err)
+			fmt.Println(jsonString)
+			os.Exit(0)
+		}
+
+		printMetricDetailTable(metric)
+	},
+}
+
+var historyMetricsCmd = &cobra.Command{
+	Use:   "history",
+	Short: "Show metric history",
+	Long:  "Show time-series history for a metric",
+	Run: func(cmd *cobra.Command, args []string) {
+		client := setup()
+
+		period, err := parsePeriod(MetricPeriod)
+		CheckError(err)
+
+		var from time.Time
+		if FromDate != "" {
+			from, err = time.Parse("2006-01-02", FromDate)
+			CheckError(err)
+		} else {
+			from = defaultFrom(period)
+		}
+
+		var to time.Time
+		if ToDate != "" {
+			to, err = time.Parse("2006-01-02", ToDate)
+			CheckError(err)
+			// Set to end of day
+			to = to.Add(23*time.Hour + 59*time.Minute + 59*time.Second)
+		} else {
+			to = time.Now().UTC()
+		}
+
+		metrics, err := client.GetMetricHistory(ColonyName, TargetExecutorName, Key, period, from, to, PrvKey)
+		CheckError(err)
+
+		if len(metrics) == 0 {
+			log.Info("No metric history found")
+			os.Exit(0)
+		}
+
+		if JSON {
+			jsonString, err := core.ConvertMetricArrayToJSON(metrics)
+			CheckError(err)
+			fmt.Println(jsonString)
+			os.Exit(0)
+		}
+
+		printMetricHistoryTable(metrics)
+	},
+}
+
+var rmMetricCmd = &cobra.Command{
+	Use:   "rm",
+	Short: "Remove metrics",
+	Long:  "Remove metrics for an executor",
+	Run: func(cmd *cobra.Command, args []string) {
+		client := setup()
+
+		if Key != "" {
+			err := client.RemoveMetric(ColonyName, TargetExecutorName, Key, PrvKey)
+			CheckError(err)
+			log.WithFields(log.Fields{"ExecutorName": TargetExecutorName, "Key": Key}).Info("Metric removed")
+		} else {
+			err := client.RemoveAllMetrics(ColonyName, TargetExecutorName, PrvKey)
+			CheckError(err)
+			log.WithFields(log.Fields{"ExecutorName": TargetExecutorName}).Info("All metrics removed")
+		}
+	},
+}