From 2e04c0f122fc7a212f8fe5fed8e237147f53d3a3 Mon Sep 17 00:00:00 2001 From: "D. Can Celasun" Date: Sat, 24 Sep 2022 20:34:56 +0100 Subject: [PATCH] Support transient SMART failures As discussed in [1] some SMART errors are transient and should not be treated as permanent. This commit adds support for a configurable list of ATA SMART attribute IDs, failures of which will be treated as transient. Drive health history is still recorded and notifications are sent, but the device itself is not marked as failed. Fixes #374. [1] https://github.com/AnalogJ/scrutiny/issues/374 --- collector/pkg/config/interface.go | 1 + collector/pkg/config/mock/mock_config.go | 14 +++++ example.scrutiny.yaml | 4 ++ webapp/backend/pkg/config/config.go | 9 ++- webapp/backend/pkg/config/interface.go | 1 + webapp/backend/pkg/config/mock/mock_config.go | 14 +++++ webapp/backend/pkg/constants.go | 6 +- ...tiny_repository_device_smart_attributes.go | 11 ++-- .../scrutiny_repository_migrations.go | 11 ++-- .../backend/pkg/models/measurements/smart.go | 36 +++++++---- .../pkg/models/measurements/smart_test.go | 60 +++++++++++++++---- webapp/backend/pkg/web/server_test.go | 29 ++++++--- 12 files changed, 150 insertions(+), 46 deletions(-) diff --git a/collector/pkg/config/interface.go b/collector/pkg/config/interface.go index e810cec49..a548d423a 100644 --- a/collector/pkg/config/interface.go +++ b/collector/pkg/config/interface.go @@ -20,6 +20,7 @@ type Interface interface { GetInt(key string) int GetString(key string) string GetStringSlice(key string) []string + GetIntSlice(key string) []int UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error GetDeviceOverrides() []models.ScanOverride diff --git a/collector/pkg/config/mock/mock_config.go b/collector/pkg/config/mock/mock_config.go index 1b135b623..7f8de99d4 100644 --- a/collector/pkg/config/mock/mock_config.go +++ b/collector/pkg/config/mock/mock_config.go @@ -133,6 +133,20 @@ func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key) } +// GetIntSlice mocks base method. +func (m *MockInterface) GetIntSlice(key string) []int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIntSlice", key) + ret0, _ := ret[0].([]int) + return ret0 +} + +// GetIntSlice indicates an expected call of GetIntSlice. +func (mr *MockInterfaceMockRecorder) GetIntSlice(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIntSlice", reflect.TypeOf((*MockInterface)(nil).GetIntSlice), key) +} + // GetString mocks base method. func (m *MockInterface) GetString(key string) string { m.ctrl.T.Helper() diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index c93725eb0..154a29952 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -52,6 +52,10 @@ log: file: '' #absolute or relative paths allowed, eg. web.log level: INFO +failures: + transient: + ata: + - 195 # Hardware_ECC_Recovered, see https://superuser.com/a/1511916/169872 # Notification "urls" look like the following. For more information about service specific configuration see # Shoutrrr's documentation: https://containrrr.dev/shoutrrr/services/overview/ diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index 39613736a..a32c5b83a 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -1,12 +1,13 @@ package config import ( - "github.com/analogj/go-util/utils" - "github.com/analogj/scrutiny/webapp/backend/pkg/errors" - "github.com/spf13/viper" "log" "os" "strings" + + "github.com/analogj/go-util/utils" + "github.com/analogj/scrutiny/webapp/backend/pkg/errors" + "github.com/spf13/viper" ) const DB_USER_SETTINGS_SUBKEY = "user" @@ -51,6 +52,8 @@ func (c *configuration) Init() error { c.SetDefault("web.influxdb.token", "scrutiny-default-admin-token") c.SetDefault("web.influxdb.retention_policy", true) + c.SetDefault("failures.transient.ata", []int{195}) + //c.SetDefault("disks.include", []string{}) //c.SetDefault("disks.exclude", []string{}) diff --git a/webapp/backend/pkg/config/interface.go b/webapp/backend/pkg/config/interface.go index d041dc221..ceca3e933 100644 --- a/webapp/backend/pkg/config/interface.go +++ b/webapp/backend/pkg/config/interface.go @@ -25,5 +25,6 @@ type Interface interface { GetInt64(key string) int64 GetString(key string) string GetStringSlice(key string) []string + GetIntSlice(key string) []int UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error } diff --git a/webapp/backend/pkg/config/mock/mock_config.go b/webapp/backend/pkg/config/mock/mock_config.go index 1b61b2cbc..aed5136d8 100644 --- a/webapp/backend/pkg/config/mock/mock_config.go +++ b/webapp/backend/pkg/config/mock/mock_config.go @@ -119,6 +119,20 @@ func (mr *MockInterfaceMockRecorder) GetInt64(key interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt64", reflect.TypeOf((*MockInterface)(nil).GetInt64), key) } +// GetIntSlice mocks base method. +func (m *MockInterface) GetIntSlice(key string) []int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIntSlice", key) + ret0, _ := ret[0].([]int) + return ret0 +} + +// GetIntSlice indicates an expected call of GetIntSlice. +func (mr *MockInterfaceMockRecorder) GetIntSlice(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIntSlice", reflect.TypeOf((*MockInterface)(nil).GetIntSlice), key) +} + // GetString mocks base method. func (m *MockInterface) GetString(key string) string { m.ctrl.T.Helper() diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go index a82c9c35c..009f13f82 100644 --- a/webapp/backend/pkg/constants.go +++ b/webapp/backend/pkg/constants.go @@ -4,8 +4,9 @@ const DeviceProtocolAta = "ATA" const DeviceProtocolScsi = "SCSI" const DeviceProtocolNvme = "NVMe" -//go:generate stringer -type=AttributeStatus // AttributeStatus bitwise flag, 1,2,4,8,16,32,etc +// +//go:generate stringer -type=AttributeStatus type AttributeStatus uint8 const ( @@ -23,8 +24,9 @@ func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus { return b & func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag } func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 } -//go:generate stringer -type=DeviceStatus // DeviceStatus bitwise flag, 1,2,4,8,16,32,etc +// +//go:generate stringer -type=DeviceStatus type DeviceStatus uint8 const ( diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index 96015bb65..46b52a8ca 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -3,21 +3,22 @@ package database import ( "context" "fmt" + "strings" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" influxdb2 "github.com/influxdata/influxdb-client-go/v2" "github.com/influxdata/influxdb-client-go/v2/api" log "github.com/sirupsen/logrus" - "strings" - "time" ) -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // SMART -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) { deviceSmartData := measurements.Smart{} - err := deviceSmartData.FromCollectorSmartInfo(wwn, collectorSmartData) + err := deviceSmartData.FromCollectorSmartInfo(sr.appConfig, wwn, collectorSmartData) if err != nil { sr.logger.Errorln("Could not process SMART metrics", err) return measurements.Smart{}, err diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 015428cc6..733429db2 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "strconv" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20201107210306" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220503120000" @@ -17,8 +20,6 @@ import ( "github.com/influxdata/influxdb-client-go/v2/api/http" log "github.com/sirupsen/logrus" "gorm.io/gorm" - "strconv" - "time" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -384,8 +385,8 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { // helpers -//When adding data to influxdb, an error may be returned if the data point is outside the range of the retention policy. -//This function will ignore retention policy errors, and allow the migration to continue. +// When adding data to influxdb, an error may be returned if the data point is outside the range of the retention policy. +// This function will ignore retention policy errors, and allow the migration to continue. func ignorePastRetentionPolicyError(err error) error { var influxDbWriteError *http.Error if errors.As(err, &influxDbWriteError) { @@ -468,7 +469,7 @@ func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(d }) } - postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable) + postDeviceSmartData.ProcessAtaSmartInfo(nil, preAtaSmartAttributesTable) } else if preDevice.IsNvme() { //info collector.SmartInfo diff --git a/webapp/backend/pkg/models/measurements/smart.go b/webapp/backend/pkg/models/measurements/smart.go index 40c139700..b44a286d3 100644 --- a/webapp/backend/pkg/models/measurements/smart.go +++ b/webapp/backend/pkg/models/measurements/smart.go @@ -2,13 +2,15 @@ package measurements import ( "fmt" - "github.com/analogj/scrutiny/webapp/backend/pkg" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" - "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" "log" "strconv" "strings" "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg" + "github.com/analogj/scrutiny/webapp/backend/pkg/config" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" ) type Smart struct { @@ -100,8 +102,8 @@ func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) { return &sm, nil } -//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries) -func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error { +// Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries) +func (sm *Smart) FromCollectorSmartInfo(cfg config.Interface, wwn string, info collector.SmartInfo) error { sm.DeviceWWN = wwn sm.Date = time.Unix(info.LocalTime.TimeT, 0) @@ -117,7 +119,7 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er // process ATA/NVME/SCSI protocol data sm.Attributes = map[string]SmartAttribute{} if sm.DeviceProtocol == pkg.DeviceProtocolAta { - sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table) + sm.ProcessAtaSmartInfo(cfg, info.AtaSmartAttributes.Table) } else if sm.DeviceProtocol == pkg.DeviceProtocolNvme { sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog) } else if sm.DeviceProtocol == pkg.DeviceProtocolScsi { @@ -127,8 +129,8 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er return nil } -//generate SmartAtaAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) { +// generate SmartAtaAttribute entries from Scrutiny Collector Smart data. +func (sm *Smart) ProcessAtaSmartInfo(cfg config.Interface, tableItems []collector.AtaSmartAttributesTableItem) { for _, collectorAttr := range tableItems { attrModel := SmartAtaAttribute{ AttributeId: collectorAttr.ID, @@ -149,13 +151,25 @@ func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTa attrModel.PopulateAttributeStatus() sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel - if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) { + var transient bool + + if cfg != nil { + transients := cfg.GetIntSlice("failures.transient.ata") + for i := range transients { + if collectorAttr.ID == transients[i] { + transient = true + break + } + } + } + + if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) && !transient { sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny) } } } -//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data. +// generate SmartNvmeAttribute entries from Scrutiny Collector Smart data. func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) { sm.Attributes = map[string]SmartAttribute{ @@ -185,7 +199,7 @@ func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.Nv } } -//generate SmartScsiAttribute entries from Scrutiny Collector Smart data. +// generate SmartScsiAttribute entries from Scrutiny Collector Smart data. func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) { sm.Attributes = map[string]SmartAttribute{ "scsi_grown_defect_list": (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(), diff --git a/webapp/backend/pkg/models/measurements/smart_test.go b/webapp/backend/pkg/models/measurements/smart_test.go index ad7b73e93..8eadcfa46 100644 --- a/webapp/backend/pkg/models/measurements/smart_test.go +++ b/webapp/backend/pkg/models/measurements/smart_test.go @@ -2,14 +2,17 @@ package measurements_test import ( "encoding/json" - "github.com/analogj/scrutiny/webapp/backend/pkg" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" - "github.com/stretchr/testify/require" "io/ioutil" "os" "testing" "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg" + mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" ) func TestSmart_Flatten(t *testing.T) { @@ -306,6 +309,11 @@ func TestNewSmartFromInfluxDB_SCSI(t *testing.T) { func TestFromCollectorSmartInfo(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-ata.json") require.NoError(t, err) defer smartDataFile.Close() @@ -319,7 +327,7 @@ func TestFromCollectorSmartInfo(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -338,6 +346,11 @@ func TestFromCollectorSmartInfo(t *testing.T) { func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-fail.json") require.NoError(t, err) defer smartDataFile.Close() @@ -351,7 +364,7 @@ func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -362,6 +375,11 @@ func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) { func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-fail2.json") require.NoError(t, err) defer smartDataFile.Close() @@ -375,7 +393,7 @@ func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -386,6 +404,11 @@ func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) { func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-ata-failed-scrutiny.json") require.NoError(t, err) defer smartDataFile.Close() @@ -399,7 +422,7 @@ func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -419,6 +442,11 @@ func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) { func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-nvme-failed.json") require.NoError(t, err) defer smartDataFile.Close() @@ -432,7 +460,7 @@ func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -450,6 +478,11 @@ func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) { func TestFromCollectorSmartInfo_Nvme(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-nvme.json") require.NoError(t, err) defer smartDataFile.Close() @@ -463,7 +496,7 @@ func TestFromCollectorSmartInfo_Nvme(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -477,6 +510,11 @@ func TestFromCollectorSmartInfo_Nvme(t *testing.T) { func TestFromCollectorSmartInfo_Scsi(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-scsi.json") require.NoError(t, err) defer smartDataFile.Close() @@ -490,7 +528,7 @@ func TestFromCollectorSmartInfo_Scsi(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index 227d3a2fe..1627a78ab 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -4,6 +4,16 @@ import ( "bytes" "encoding/json" "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path" + "strings" + "testing" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/config" mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock" @@ -14,15 +24,6 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path" - "strings" - "testing" - "time" ) /* @@ -104,6 +105,8 @@ func (suite *ServerTestSuite) TestHealthRoute() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -146,6 +149,7 @@ func (suite *ServerTestSuite) TestRegisterDevicesRoute() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -188,6 +192,7 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -245,6 +250,7 @@ func (suite *ServerTestSuite) TestPopulateMultiple() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -343,6 +349,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -388,6 +395,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -433,6 +441,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -478,6 +487,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -522,6 +532,7 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll))