From e3f9f97a7d9172699dd0a8a24df9fc22314d00cf Mon Sep 17 00:00:00 2001 From: Harsh Rajput Date: Fri, 23 Feb 2024 15:02:15 +0530 Subject: [PATCH 01/11] replace es with light image --- canned/elasticsearch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index cd76aa0..4d7a2d4 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -22,12 +22,13 @@ type ElasticSearch struct { func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { _ = os.Setenv("TC_HOST", "localhost") req := testcontainers.ContainerRequest{ - Image: getEnvString("ES_CONTAINER_IMAGE", "elasticsearch:7.17.9"), + Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2"), Env: map[string]string{ "discovery.type": "single-node"}, ExposedPorts: []string{"9200/tcp"}, WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute RegistryCred: getBasicAuth(), + AutoRemove: true, SkipReaper: skipReaper(), } container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ From a36d509f74ed2460db61add5b76620c4096b8718 Mon Sep 17 00:00:00 2001 From: Harsh Rajput Date: Fri, 23 Feb 2024 17:46:36 +0530 Subject: [PATCH 02/11] add env for es --- canned/elasticsearch.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 4d7a2d4..19f682c 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -22,10 +22,13 @@ type ElasticSearch struct { func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { _ = os.Setenv("TC_HOST", "localhost") req := testcontainers.ContainerRequest{ - Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2"), + Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), Env: map[string]string{ - "discovery.type": "single-node"}, - ExposedPorts: []string{"9200/tcp"}, + "discovery.type": "single-node", + "network.host": "0.0.0.0", + "network.bind_host": "0.0.0.0", + }, + ExposedPorts: []string{"9200/tcp", "9300/tcp"}, WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute RegistryCred: getBasicAuth(), AutoRemove: true, From ba08a3bfcaa33d8582a992dba9eeb1d3f038b3ed Mon Sep 17 00:00:00 2001 From: Harsh Rajput Date: Tue, 5 Mar 2024 16:25:44 +0530 Subject: [PATCH 03/11] rem 9300 port --- canned/elasticsearch.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 19f682c..1f39ccf 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -22,14 +22,16 @@ type ElasticSearch struct { func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { _ = os.Setenv("TC_HOST", "localhost") req := testcontainers.ContainerRequest{ - Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), + //Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), + Image: getEnvString("ES_CONTAINER_IMAGE", "elasticsearch:7.0.0"), Env: map[string]string{ "discovery.type": "single-node", "network.host": "0.0.0.0", "network.bind_host": "0.0.0.0", }, - ExposedPorts: []string{"9200/tcp", "9300/tcp"}, - WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute + ExposedPorts: []string{"9200/tcp"}, + //WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute + WaitingFor: wait.ForHTTP("/").WithPort("9200/tcp").WithStartupTimeout(time.Minute * 3), RegistryCred: getBasicAuth(), AutoRemove: true, SkipReaper: skipReaper(), From b4a39e22190bff4e997012189231b4c1f9686d60 Mon Sep 17 00:00:00 2001 From: harsh-awg <138427417+harsh-awg@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:51:06 +0530 Subject: [PATCH 04/11] add 9300 port --- canned/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 1f39ccf..12dbddc 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -29,7 +29,7 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { "network.host": "0.0.0.0", "network.bind_host": "0.0.0.0", }, - ExposedPorts: []string{"9200/tcp"}, + ExposedPorts: []string{"9200/tcp", "9300/tcp"}, //WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute WaitingFor: wait.ForHTTP("/").WithPort("9200/tcp").WithStartupTimeout(time.Minute * 3), RegistryCred: getBasicAuth(), From 20605b58b066d1f8208d88688a47941cdeb0b6c7 Mon Sep 17 00:00:00 2001 From: harsh-awg <138427417+harsh-awg@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:29:20 +0530 Subject: [PATCH 05/11] add v6.4.2 es image --- canned/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 12dbddc..18942af 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -23,7 +23,7 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { _ = os.Setenv("TC_HOST", "localhost") req := testcontainers.ContainerRequest{ //Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), - Image: getEnvString("ES_CONTAINER_IMAGE", "elasticsearch:7.0.0"), + Image: getEnvString("ES_CONTAINER_IMAGE", "157529275398.dkr.ecr.ap-south-1.amazonaws.com/ci-libraries/docker.elastic.co/elasticsearch/elasticsearch:6.4.2"), Env: map[string]string{ "discovery.type": "single-node", "network.host": "0.0.0.0", From 95e60d203a09032205097a7d7c13932efcf14d8b Mon Sep 17 00:00:00 2001 From: harsh-awg <138427417+harsh-awg@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:55:42 +0530 Subject: [PATCH 06/11] add oss v7 es image --- canned/elasticsearch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 18942af..6ecc2eb 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -22,8 +22,8 @@ type ElasticSearch struct { func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { _ = os.Setenv("TC_HOST", "localhost") req := testcontainers.ContainerRequest{ - //Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), - Image: getEnvString("ES_CONTAINER_IMAGE", "157529275398.dkr.ecr.ap-south-1.amazonaws.com/ci-libraries/docker.elastic.co/elasticsearch/elasticsearch:6.4.2"), + Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), + //Image: getEnvString("ES_CONTAINER_IMAGE", "157529275398.dkr.ecr.ap-south-1.amazonaws.com/ci-libraries/docker.elastic.co/elasticsearch/elasticsearch:6.4.2"), Env: map[string]string{ "discovery.type": "single-node", "network.host": "0.0.0.0", From 172051cf33f893fffb1e4792c5209a35559f5a51 Mon Sep 17 00:00:00 2001 From: harsh-awg <138427417+harsh-awg@users.noreply.github.com> Date: Tue, 5 Mar 2024 18:33:49 +0530 Subject: [PATCH 07/11] Update elasticsearch.go --- canned/elasticsearch.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 6ecc2eb..b963414 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -30,10 +30,10 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { "network.bind_host": "0.0.0.0", }, ExposedPorts: []string{"9200/tcp", "9300/tcp"}, - //WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute - WaitingFor: wait.ForHTTP("/").WithPort("9200/tcp").WithStartupTimeout(time.Minute * 3), + WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute + //WaitingFor: wait.ForHTTP("/").WithPort("9200/tcp").WithStartupTimeout(time.Minute * 3), RegistryCred: getBasicAuth(), - AutoRemove: true, + //AutoRemove: true, SkipReaper: skipReaper(), } container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ From c253ec12dcefed0c44eb474e28302dacaf0895c7 Mon Sep 17 00:00:00 2001 From: Harsh Rajput Date: Tue, 5 Mar 2024 20:05:03 +0530 Subject: [PATCH 08/11] set 3min for indexinit --- canned/elasticsearch.go | 11 ++++++----- pkg/grillelasticsearch/stubs.go | 6 ++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index b963414..b8d1a79 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -25,16 +25,16 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), //Image: getEnvString("ES_CONTAINER_IMAGE", "157529275398.dkr.ecr.ap-south-1.amazonaws.com/ci-libraries/docker.elastic.co/elasticsearch/elasticsearch:6.4.2"), Env: map[string]string{ - "discovery.type": "single-node", - "network.host": "0.0.0.0", - "network.bind_host": "0.0.0.0", + "discovery.type": "single-node", + //"network.host": "0.0.0.0", + //"network.bind_host": "0.0.0.0", }, ExposedPorts: []string{"9200/tcp", "9300/tcp"}, WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute //WaitingFor: wait.ForHTTP("/").WithPort("9200/tcp").WithStartupTimeout(time.Minute * 3), RegistryCred: getBasicAuth(), //AutoRemove: true, - SkipReaper: skipReaper(), + SkipReaper: skipReaper(), } container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, @@ -54,7 +54,8 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { } client, err := elasticsearch.NewClient(elasticsearch.Config{ - Addresses: []string{endpoint}, + Addresses: []string{endpoint}, + EnableRetryOnTimeout: true, }) if err != nil { return nil, err diff --git a/pkg/grillelasticsearch/stubs.go b/pkg/grillelasticsearch/stubs.go index d3c1d32..c642ad9 100644 --- a/pkg/grillelasticsearch/stubs.go +++ b/pkg/grillelasticsearch/stubs.go @@ -12,8 +12,10 @@ import ( func (ge *ElasticSearch) CreateIndex(index string, mapping string) grill.Stub { return grill.StubFunc(func() error { req := esapi.IndicesCreateRequest{ - Index: index, - Body: strings.NewReader(mapping), + Index: index, + MasterTimeout: 3000, + Timeout: 3000, + Body: strings.NewReader(mapping), } res, err := req.Do(context.Background(), ge.elasticSearch.Client) if err != nil { From 63ed330e2c84041db47966aab2d7f9368cc2ba23 Mon Sep 17 00:00:00 2001 From: Harsh Rajput Date: Wed, 6 Mar 2024 14:46:23 +0530 Subject: [PATCH 09/11] exposed 9300 endpoint as well --- canned/elasticsearch.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index b8d1a79..22c0ac3 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -25,9 +25,9 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), //Image: getEnvString("ES_CONTAINER_IMAGE", "157529275398.dkr.ecr.ap-south-1.amazonaws.com/ci-libraries/docker.elastic.co/elasticsearch/elasticsearch:6.4.2"), Env: map[string]string{ - "discovery.type": "single-node", - //"network.host": "0.0.0.0", - //"network.bind_host": "0.0.0.0", + "discovery.type": "single-node", + "network.host": "0.0.0.0", + "network.bind_host": "0.0.0.0", }, ExposedPorts: []string{"9200/tcp", "9300/tcp"}, WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute @@ -46,6 +46,7 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { } host, _ := container.Host(ctx) port, _ := container.MappedPort(ctx, "9200") + port2, _ := container.MappedPort(ctx, "9300") endpoint, err := container.Endpoint(ctx, "") endpoint = fmt.Sprintf("http://%s", endpoint) @@ -54,7 +55,7 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { } client, err := elasticsearch.NewClient(elasticsearch.Config{ - Addresses: []string{endpoint}, + Addresses: []string{endpoint, fmt.Sprintf("http://%s:%s", host, port2.Port())}, EnableRetryOnTimeout: true, }) if err != nil { From 1cfca5a718e2640d26dd15e264feb940b7c133bb Mon Sep 17 00:00:00 2001 From: Harsh Rajput Date: Wed, 6 Mar 2024 19:16:56 +0530 Subject: [PATCH 10/11] endpoint randomisation stopped --- canned/elasticsearch.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 22c0ac3..4b6755d 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -21,14 +21,27 @@ type ElasticSearch struct { func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { _ = os.Setenv("TC_HOST", "localhost") + //volumeConfig := map[string]string{ + // "host-path": "my_es_data", + // "container-path": "/usr/share/elasticsearch/data", + //} req := testcontainers.ContainerRequest{ Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), - //Image: getEnvString("ES_CONTAINER_IMAGE", "157529275398.dkr.ecr.ap-south-1.amazonaws.com/ci-libraries/docker.elastic.co/elasticsearch/elasticsearch:6.4.2"), + //Image: getEnvString("ES_CONTAINER_IMAGE", "elasticsearch:7.0.0"), Env: map[string]string{ "discovery.type": "single-node", "network.host": "0.0.0.0", "network.bind_host": "0.0.0.0", }, + //Mounts: testcontainers.ContainerMounts{ + // { + // Source: testcontainers.GenericVolumeMountSource{ + // Name: volumeConfig["host-path"], + // }, + // Target: testcontainers.ContainerMountTarget(volumeConfig["container-path"]), + // ReadOnly: false, + // }, + //}, ExposedPorts: []string{"9200/tcp", "9300/tcp"}, WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute //WaitingFor: wait.ForHTTP("/").WithPort("9200/tcp").WithStartupTimeout(time.Minute * 3), @@ -46,16 +59,16 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { } host, _ := container.Host(ctx) port, _ := container.MappedPort(ctx, "9200") - port2, _ := container.MappedPort(ctx, "9300") - endpoint, err := container.Endpoint(ctx, "") - endpoint = fmt.Sprintf("http://%s", endpoint) + //port2, _ := container.MappedPort(ctx, "9300") + //endpoint, err := container.Endpoint(ctx, "") + endpoint := fmt.Sprintf("http://%s:%s", host, port.Port()) if err != nil { return nil, err } client, err := elasticsearch.NewClient(elasticsearch.Config{ - Addresses: []string{endpoint, fmt.Sprintf("http://%s:%s", host, port2.Port())}, + Addresses: []string{endpoint}, //, fmt.Sprintf("http://%s:%s", host, port2.Port())}, EnableRetryOnTimeout: true, }) if err != nil { From bf3895bdbcce127df393ff4e1520428dadced087 Mon Sep 17 00:00:00 2001 From: Harsh Rajput Date: Tue, 12 Mar 2024 11:43:11 +0530 Subject: [PATCH 11/11] clean up --- canned/elasticsearch.go | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/canned/elasticsearch.go b/canned/elasticsearch.go index 4b6755d..0ca90cc 100644 --- a/canned/elasticsearch.go +++ b/canned/elasticsearch.go @@ -21,33 +21,17 @@ type ElasticSearch struct { func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { _ = os.Setenv("TC_HOST", "localhost") - //volumeConfig := map[string]string{ - // "host-path": "my_es_data", - // "container-path": "/usr/share/elasticsearch/data", - //} req := testcontainers.ContainerRequest{ - Image: getEnvString("ES_CONTAINER_IMAGE", "docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0"), - //Image: getEnvString("ES_CONTAINER_IMAGE", "elasticsearch:7.0.0"), + Image: getEnvString("ES_CONTAINER_IMAGE", "elasticsearch:7.0.0"), Env: map[string]string{ "discovery.type": "single-node", "network.host": "0.0.0.0", "network.bind_host": "0.0.0.0", }, - //Mounts: testcontainers.ContainerMounts{ - // { - // Source: testcontainers.GenericVolumeMountSource{ - // Name: volumeConfig["host-path"], - // }, - // Target: testcontainers.ContainerMountTarget(volumeConfig["container-path"]), - // ReadOnly: false, - // }, - //}, ExposedPorts: []string{"9200/tcp", "9300/tcp"}, WaitingFor: wait.ForListeningPort("9200").WithStartupTimeout(time.Minute * 3), // Default timeout is 1 minute - //WaitingFor: wait.ForHTTP("/").WithPort("9200/tcp").WithStartupTimeout(time.Minute * 3), RegistryCred: getBasicAuth(), - //AutoRemove: true, - SkipReaper: skipReaper(), + SkipReaper: skipReaper(), } container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, @@ -59,8 +43,6 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { } host, _ := container.Host(ctx) port, _ := container.MappedPort(ctx, "9200") - //port2, _ := container.MappedPort(ctx, "9300") - //endpoint, err := container.Endpoint(ctx, "") endpoint := fmt.Sprintf("http://%s:%s", host, port.Port()) if err != nil { @@ -68,7 +50,7 @@ func NewElasticSearch(ctx context.Context) (*ElasticSearch, error) { } client, err := elasticsearch.NewClient(elasticsearch.Config{ - Addresses: []string{endpoint}, //, fmt.Sprintf("http://%s:%s", host, port2.Port())}, + Addresses: []string{endpoint}, EnableRetryOnTimeout: true, }) if err != nil {