Discovery Engine failing due to OOM and race conditions

[Ankurk99]

We have observed Discovery Engine crashing due to OOM and Goroutine race conditions with the stable image (v0.8). But the reason is not clear and currently the issue is not reproducible.
Until the issue is reproduced and fixed, if anyone encounters it please comment on this issue.

[rksharma95]

i encountered this error, reporting the same, here's the env info and stack trace.
Environment: k3s, 5.15.0-60-generic, k8s v1.25.6,

fatal error: concurrent map writes
fatal error: concurrent map writes

goroutine 1143 [running]:
runtime.throw({0x22193a5?, 0xc00058ea08?})
        /usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc000c22520 sp=0xc000c224f0 pc=0x4a9cb1
runtime.mapassign(0x1f64ee0, 0xc0001be7e0, 0x7f58a5d43280?)
        /usr/local/go/src/runtime/map.go:686 +0x4c5 fp=0xc000c225a0 sp=0xc000c22520 pc=0x480d45
k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypeWithName(0xc0000d6d90, {{0x221b66b, 0x16}, {0x21facfa, 0x2}, {0x1dfea57, 0x13}}, {0x24ed390?, 0xc0001e7c00})
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme.go:175 +0x2f7 fp=0xc000c22838 sp=0xc000c225a0 pc=0x91adb7
k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypes(0xc0000d6d90, {{0x221b66b?, 0x0?}, {0x21facfa?, 0x0?}}, {0xc0004617e0?, 0x2, 0x495?})
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme.go:149 +0x237 fp=0xc000c22940 sp=0xc000c22838 pc=0x91a577
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).Register.func1(0x23a?)
        /home/hp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/scheme/scheme.go:71 +0x45 fp=0xc000c229a0 sp=0xc000c22940 pc=0xe499c5
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme(...)
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme_builder.go:29
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).AddToScheme(0x0?, 0x0?)
        /home/hp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/scheme/scheme.go:86 +0x5d fp=0xc000c229d8 sp=0xc000c229a0 pc=0xe49a7d
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).AddToScheme-fm(0x0?)
        <autogenerated>:1 +0x2c fp=0xc000c229f8 sp=0xc000c229d8 pc=0xe513cc
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme(0x0?, 0x0?)
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme_builder.go:29 +0x5c fp=0xc000c22a30 sp=0xc000c229f8 pc=0x92053c
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme-fm(0x0?)
        <autogenerated>:1 +0x2c fp=0xc000c22a50 sp=0xc000c22a30 pc=0xe5ddac
github.com/accuknox/auto-policy-discovery/src/cluster.ConnectLocalAPIClient()
        /home/hp/Documents/discovery-engine/src/cluster/k8sClientHandler.go:95 +0x3ea fp=0xc000c22b28 sp=0xc000c22a50 pc=0x172032a
github.com/accuknox/auto-policy-discovery/src/cluster.ConnectK8sClient()
        /home/hp/Documents/discovery-engine/src/cluster/k8sClientHandler.go:47 +0x3d fp=0xc000c22b48 sp=0xc000c22b28 pc=0x171ff1d
github.com/accuknox/auto-policy-discovery/src/cluster.GetPodsFromK8sClient()
        /home/hp/Documents/discovery-engine/src/cluster/k8sClientHandler.go:201 +0x2e fp=0xc000c23150 sp=0xc000c22b48 pc=0x1720a6e
github.com/accuknox/auto-policy-discovery/src/cluster.GetPods({0xc000c069d0?, 0x22618c1?})
        /home/hp/Documents/discovery-engine/src/cluster/clusterResourceHandler.go:16 +0x59 fp=0xc000c23218 sp=0xc000c23150 pc=0x171f4f9
github.com/accuknox/auto-policy-discovery/src/systempolicy.PopulateSystemPoliciesFromSystemLogs({0xc00002d500?, 0x0?, 0x0?})
        /home/hp/Documents/discovery-engine/src/systempolicy/systemPolicy.go:1111 +0x351 fp=0xc000c23ec8 sp=0xc000c23218 pc=0x18ed371
github.com/accuknox/auto-policy-discovery/src/systempolicy.DiscoverSystemPolicyMain()
        /home/hp/Documents/discovery-engine/src/systempolicy/systemPolicy.go:1470 +0x2a5 fp=0xc000c23f70 sp=0xc000c23ec8 pc=0x18f1565
github.com/robfig/cron.FuncJob.Run(0x0?)
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:92 +0x1a fp=0xc000c23f80 sp=0xc000c23f70 pc=0xdc9e1a
github.com/robfig/cron.(*Cron).runWithRecovery(0x0?, {0x24eaae0?, 0x2311468?})
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:165 +0x62 fp=0xc000c23fb8 sp=0xc000c23f80 pc=0xdca182
github.com/robfig/cron.(*Cron).run.func1()
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:199 +0x2e fp=0xc000c23fe0 sp=0xc000c23fb8 pc=0xdcad4e
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc000c23fe8 sp=0xc000c23fe0 pc=0x4dc7c1
created by github.com/robfig/cron.(*Cron).run
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:199 +0x79f

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f58a54bfe18, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc00042a700?, 0xc00006cf00?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Accept(0xc00042a700)
        /usr/local/go/src/internal/poll/fd_unix.go:614 +0x22c
net.(*netFD).accept(0xc00042a700)
        /usr/local/go/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc00062b890)
        /usr/local/go/src/net/tcpsock_posix.go:139 +0x28
net.(*TCPListener).Accept(0xc00062b890)
        /usr/local/go/src/net/tcpsock.go:288 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0000001e0, {0x25004d8?, 0xc00062b890})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/server.go:796 +0x477
main.main()
        /home/hp/Documents/discovery-engine/src/main.go:60 +0x125

goroutine 6 [select]:
io.(*pipe).read(0xc000100a80, {0xc0000ae000, 0x1000, 0x1?})
        /usr/local/go/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0x0?, {0xc0000ae000?, 0x0?, 0x0?})
        /usr/local/go/src/io/pipe.go:136 +0x25
bufio.(*Scanner).Scan(0xc000088f28)
        /usr/local/go/src/bufio/scan.go:215 +0x865
github.com/sirupsen/logrus.(*Entry).writerScanner(0x0?, 0xc000010428, 0xc0001a4190)
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:59 +0xa5
created by github.com/sirupsen/logrus.(*Entry).WriterLevel
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:51 +0x3d1

goroutine 7 [select]:
io.(*pipe).read(0xc000100ae0, {0xc00020e000, 0x1000, 0x1?})
        /usr/local/go/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0x0?, {0xc00020e000?, 0x0?, 0x0?})
        /usr/local/go/src/io/pipe.go:136 +0x25
bufio.(*Scanner).Scan(0xc000214f28)
        /usr/local/go/src/bufio/scan.go:215 +0x865
github.com/sirupsen/logrus.(*Entry).writerScanner(0x0?, 0xc000010438, 0xc0001a41b0)
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:59 +0xa5
created by github.com/sirupsen/logrus.(*Entry).WriterLevel
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:51 +0x3d1

goroutine 8 [select]:
io.(*pipe).read(0xc000100b40, {0xc000280000, 0x1000, 0x1?})
        /usr/local/go/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0x0?, {0xc000280000?, 0x0?, 0x0?})
        /usr/local/go/src/io/pipe.go:136 +0x25
bufio.(*Scanner).Scan(0xc000210f28)
        /usr/local/go/src/bufio/scan.go:215 +0x865
github.com/sirupsen/logrus.(*Entry).writerScanner(0x0?, 0xc000010448, 0xc0001a41d0)
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:59 +0xa5
created by github.com/sirupsen/logrus.(*Entry).WriterLevel
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:51 +0x3d1

goroutine 9 [select]:
io.(*pipe).read(0xc000100ba0, {0xc0000af000, 0x1000, 0x1?})
        /usr/local/go/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0x0?, {0xc0000af000?, 0x0?, 0x0?})
        /usr/local/go/src/io/pipe.go:136 +0x25
bufio.(*Scanner).Scan(0xc000089f28)
        /usr/local/go/src/bufio/scan.go:215 +0x865
github.com/sirupsen/logrus.(*Entry).writerScanner(0x0?, 0xc000010458, 0xc0001a41f0)
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:59 +0xa5
created by github.com/sirupsen/logrus.(*Entry).WriterLevel
        /home/hp/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/writer.go:51 +0x3d1

goroutine 36 [IO wait]:
internal/poll.runtime_pollWait(0x7f58a54bff08, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000693300?, 0xc0000547b0?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).ReadMsg(0xc000693300, {0xc0000547b0, 0x10, 0x10}, {0xc000644020, 0x1000, 0x1000}, 0x1?)
        /usr/local/go/src/internal/poll/fd_unix.go:304 +0x31c
net.(*netFD).readMsg(0xc000693300, {0xc0000547b0?, 0x7f58a4c3ee40?, 0x0?}, {0xc000644020?, 0x4888a5?, 0x34ec020?}, 0x346a6f8?)
        /usr/local/go/src/net/fd_posix.go:78 +0x37
net.(*UnixConn).readMsg(0xc000688650, {0xc0000547b0?, 0xc00064fd80?, 0x47ef05?}, {0xc000644020?, 0xb?, 0xc0000e8090?})
        /usr/local/go/src/net/unixsock_posix.go:115 +0x4f
net.(*UnixConn).ReadMsgUnix(0xc000688650, {0xc0000547b0?, 0xc00064fdf8?, 0x47ef05?}, {0xc000644020?, 0x0?, 0x0?})
        /usr/local/go/src/net/unixsock.go:143 +0x3c
github.com/godbus/dbus.(*oobReader).Read(0xc000644000, {0xc0000547b0?, 0xc00064fe20?, 0x47f267?})
        /home/hp/go/pkg/mod/github.com/godbus/dbus@v0.0.0-20190726142602-4481cbc300e2/transport_unix.go:21 +0x45
io.ReadAtLeast({0x24e5de0, 0xc000644000}, {0xc0000547b0, 0x10, 0x10}, 0x10)
        /usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
github.com/godbus/dbus.(*unixTransport).ReadMessage(0xc00068aa80)
        /home/hp/go/pkg/mod/github.com/godbus/dbus@v0.0.0-20190726142602-4481cbc300e2/transport_unix.go:91 +0x11e
github.com/godbus/dbus.(*Conn).inWorker(0xc000234360)
        /home/hp/go/pkg/mod/github.com/godbus/dbus@v0.0.0-20190726142602-4481cbc300e2/conn.go:294 +0x3b
created by github.com/godbus/dbus.(*Conn).Auth
        /home/hp/go/pkg/mod/github.com/godbus/dbus@v0.0.0-20190726142602-4481cbc300e2/auth.go:118 +0x9ed

goroutine 69 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0001259e0)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/transport.go:324 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/transport.go:339
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00029c180, {0x21a3060?, 0xc000416000}, 0x0?)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:986 +0xbe
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000416000?)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:851 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000123680, 0xc00064be48, 0xc00064be18)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:715 +0x144
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000123680, {0x21a3060?, 0xc000416000?})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:850 +0x11f
github.com/kubearmor/KubeArmor/protobuf.(*logServiceWatchAlertsClient).Recv(0xc000537dd0)
        /home/hp/go/pkg/mod/github.com/kubearmor/!kube!armor/protobuf@v0.0.0-20220504043216-6451e04be58b/kubearmor_grpc.pb.go:106 +0x4c
github.com/accuknox/auto-policy-discovery/src/plugin.StartKubeArmorRelay.func2()
        /home/hp/Documents/discovery-engine/src/plugin/kubearmor.go:499 +0x26b
created by github.com/accuknox/auto-policy-discovery/src/plugin.StartKubeArmorRelay
        /home/hp/Documents/discovery-engine/src/plugin/kubearmor.go:481 +0x4aa

goroutine 68 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00043b5e0, {0xc00068ac58, 0x5, 0x5})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/transport.go:191 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00043b5e0, {0xc00068ac58?, 0xc000465218?, 0xc0006c17a0?})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/transport.go:171 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00062fda0, {0xc00068ac58?, 0xc0006c1818?, 0xd6a167?})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/transport.go:485 +0x32
io.ReadAtLeast({0x24e8ca0, 0xc00062fda0}, {0xc00068ac58, 0x5, 0x5}, 0x5)
        /usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00068fd40, {0xc00068ac58, 0x5, 0x5})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/transport.go:469 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc00068ac48, 0x400000)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/rpc_util.go:559 +0x47
google.golang.org/grpc.recvAndDecompress(0x0?, 0xc00068fd40, {0x0, 0x0}, 0x400000, 0x0, {0x0, 0x0})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/rpc_util.go:690 +0x66
google.golang.org/grpc.recv(0x486146?, {0x7f58a54ff2a0, 0x34fb3d0}, 0x7f58a8273fff?, {0x0?, 0x0?}, {0x218ac00, 0xc000ae2000}, 0x7f58a80c6a00?, 0x0, ...)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/rpc_util.go:756 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00029c240, {0x218ac00?, 0xc000ae2000}, 0x0?)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:1000 +0x2b0
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000ae2000?)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:851 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0001257a0, 0xc0006c1c48, 0xc0006c1c18)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:699 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0001257a0, {0x218ac00?, 0xc000ae2000?})
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:850 +0x11f
github.com/kubearmor/KubeArmor/protobuf.(*logServiceWatchLogsClient).Recv(0xc0000cf5d0)
        /home/hp/go/pkg/mod/github.com/kubearmor/!kube!armor/protobuf@v0.0.0-20220504043216-6451e04be58b/kubearmor_grpc.pb.go:138 +0x4c
github.com/accuknox/auto-policy-discovery/src/plugin.StartKubeArmorRelay.func1({0x2501a88?, 0xc000537620?})
        /home/hp/Documents/discovery-engine/src/plugin/kubearmor.go:418 +0x284
created by github.com/accuknox/auto-policy-discovery/src/plugin.StartKubeArmorRelay
        /home/hp/Documents/discovery-engine/src/plugin/kubearmor.go:400 +0x345

goroutine 72 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:349 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:348 +0xb70

goroutine 66 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000296040)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/balancer_conn_wrappers.go:112 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/balancer_conn_wrappers.go:73 +0x22a

goroutine 37 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:349 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/stream.go:348 +0xb70

goroutine 55 [sleep]:
time.Sleep(0x77359400)
        /usr/local/go/src/runtime/time.go:194 +0x12e
github.com/accuknox/auto-policy-discovery/src/networkpolicy.StartNetworkLogRcvr()
        /home/hp/Documents/discovery-engine/src/networkpolicy/networkPolicy.go:2343 +0x25
created by github.com/accuknox/auto-policy-discovery/src/networkpolicy.StartNetworkCronJob
        /home/hp/Documents/discovery-engine/src/networkpolicy/networkPolicy.go:2348 +0x25

goroutine 56 [select]:
github.com/robfig/cron.(*Cron).run(0xc0000d8370)
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:191 +0x585
created by github.com/robfig/cron.(*Cron).Start
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:144 +0x65

goroutine 57 [chan receive]:
github.com/accuknox/auto-policy-discovery/src/systempolicy.StartSystemLogRcvr()
        /home/hp/Documents/discovery-engine/src/systempolicy/systemPolicy.go:1481 +0x95
created by github.com/accuknox/auto-policy-discovery/src/systempolicy.StartSystemCronJob
        /home/hp/Documents/discovery-engine/src/systempolicy/systemPolicy.go:1497 +0x25

goroutine 58 [select]:
github.com/robfig/cron.(*Cron).run(0xc0000d8410)
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:191 +0x585
created by github.com/robfig/cron.(*Cron).Start
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:144 +0x65

goroutine 59 [select]:
github.com/robfig/cron.(*Cron).run(0xc0000d84b0)
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:191 +0x585
created by github.com/robfig/cron.(*Cron).Start
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:144 +0x65

goroutine 60 [select]:
github.com/robfig/cron.(*Cron).run(0xc0000d8550)
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:191 +0x585
created by github.com/robfig/cron.(*Cron).Start
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:144 +0x65

goroutine 94 [IO wait]:
internal/poll.runtime_pollWait(0x7f58a54bf968, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc0005d8c00?, 0xc0009e8000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc0005d8c00, {0xc0009e8000, 0xb535, 0xb535})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0005d8c00, {0xc0009e8000?, 0xc000065dc0?, 0xc0009e80c7?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000623670, {0xc0009e8000?, 0x34ea?, 0xc000065dc0?})
        /usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc000a5e7f8, {0xc0009e8000?, 0x0?, 0x203000?})
        /usr/local/go/src/crypto/tls/conn.go:785 +0x3d
bytes.(*Buffer).ReadFrom(0xc000507e78, {0x24e2ec0, 0xc000a5e7f8})
        /usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000507c00, {0x24e9b20?, 0xc000623670}, 0xb473?)
        /usr/local/go/src/crypto/tls/conn.go:807 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc000507c00, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:614 +0x116
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc000507c00, {0xc000aa1000, 0x1000, 0x0?})
        /usr/local/go/src/crypto/tls/conn.go:1285 +0x16f
bufio.(*Reader).Read(0xc000219bc0, {0xc0008d4580, 0x9, 0xc000256f88?})
        /usr/local/go/src/bufio/bufio.go:236 +0x1b4
io.ReadAtLeast({0x24e2d20, 0xc000219bc0}, {0xc0008d4580, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
golang.org/x/net/http2.readFrameHeader({0xc0008d4580?, 0x9?, 0xc000000000?}, {0x24e2d20?, 0xc000219bc0?})
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008d4540)
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000288f98)
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:2224 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc000349980)
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:2119 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:817 +0xc1c

goroutine 70 [IO wait]:
internal/poll.runtime_pollWait(0x7f58a54bfd28, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc0006e2080?, 0xc0002be000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc0006e2080, {0xc0002be000, 0x8000, 0x8000})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0006e2080, {0xc0002be000?, 0x7f58a4364298?, 0x7?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000232330, {0xc0002be000?, 0x7f58ceba5f18?, 0x800010601?})
        /usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000218ea0, {0xc0002d6040, 0x9, 0x18?})
        /usr/local/go/src/bufio/bufio.go:236 +0x1b4
io.ReadAtLeast({0x24e2d20, 0xc000218ea0}, {0xc0002d6040, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
golang.org/x/net/http2.readFrameHeader({0xc0002d6040?, 0x9?, 0xde7aa3a1a?}, {0x24e2d20?, 0xc000218ea0?})
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0002d6000)
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0002d8000)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/http2_client.go:1503 +0x414
created by google.golang.org/grpc/internal/transport.newHTTP2Client
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/http2_client.go:367 +0x1865

goroutine 88 [IO wait]:
internal/poll.runtime_pollWait(0x7f58a54bfa58, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc0006e2e80?, 0xc0006d2500?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc0006e2e80, {0xc0006d2500, 0x202c, 0x202c})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0006e2e80, {0xc0006d2500?, 0xc0001ddec0?, 0xc0006d3a0b?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000232208, {0xc0006d2500?, 0xfdc?, 0xc0001ddec0?})
        /usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0003e8330, {0xc0006d2500?, 0x0?, 0x271?})
        /usr/local/go/src/crypto/tls/conn.go:785 +0x3d
bytes.(*Buffer).ReadFrom(0xc000507778, {0x24e2ec0, 0xc0003e8330})
        /usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000507500, {0x24e9b20?, 0xc000232208}, 0xb26?)
        /usr/local/go/src/crypto/tls/conn.go:807 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc000507500, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:614 +0x116
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc000507500, {0xc000aa0000, 0x1000, 0x7f7640?})
        /usr/local/go/src/crypto/tls/conn.go:1285 +0x16f
bufio.(*Reader).Read(0xc000219680, {0xc0008d4200, 0x9, 0x813fe2?})
        /usr/local/go/src/bufio/bufio.go:236 +0x1b4
io.ReadAtLeast({0x24e2d20, 0xc000219680}, {0xc0008d4200, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
net/http.http2readFrameHeader({0xc0008d4200?, 0x9?, 0xc00099e4e0?}, {0x24e2d20?, 0xc000219680?})
        /usr/local/go/src/net/http/h2_bundle.go:1566 +0x6e
net/http.(*http2Framer).ReadFrame(0xc0008d41c0)
        /usr/local/go/src/net/http/h2_bundle.go:1830 +0x95
net/http.(*http2clientConnReadLoop).run(0xc00028df98)
        /usr/local/go/src/net/http/h2_bundle.go:8820 +0x130
net/http.(*http2ClientConn).readLoop(0xc000a9e000)
        /usr/local/go/src/net/http/h2_bundle.go:8716 +0x6f
created by net/http.(*http2Transport).newClientConn
        /usr/local/go/src/net/http/h2_bundle.go:7444 +0xa65

goroutine 71 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000102370, 0x1)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000218fc0)
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/http2_client.go:417 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
        /home/hp/go/pkg/mod/google.golang.org/grpc@v1.49.0/internal/transport/http2_client.go:415 +0x1eb1

goroutine 83 [IO wait]:
internal/poll.runtime_pollWait(0x7f58a54bfc38, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc00042a980?, 0xc0002eb000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc00042a980, {0xc0002eb000, 0xe63, 0xe63})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00042a980, {0xc0002eb000?, 0x201bf01?, 0xc0000df7a0?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000232360, {0xc0002eb000?, 0x8f3?, 0xc0000df7a0?})
        /usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0008c50e0, {0xc0002eb000?, 0x0?, 0x34?})
        /usr/local/go/src/crypto/tls/conn.go:785 +0x3d
bytes.(*Buffer).ReadFrom(0xc000118cf8, {0x24e2ec0, 0xc0008c50e0})
        /usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000118a80, {0x24e9b20?, 0xc000232360}, 0xc0002eb575?)
        /usr/local/go/src/crypto/tls/conn.go:807 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc000118a80, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:614 +0x116
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc000118a80, {0xc0008e6000, 0x1000, 0x7f7640?})
        /usr/local/go/src/crypto/tls/conn.go:1285 +0x16f
bufio.(*Reader).Read(0xc0008c7e60, {0xc0008d4ba0, 0x9, 0x813fe2?})
        /usr/local/go/src/bufio/bufio.go:236 +0x1b4
io.ReadAtLeast({0x24e2d20, 0xc0008c7e60}, {0xc0008d4ba0, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
net/http.http2readFrameHeader({0xc0008d4ba0?, 0x9?, 0xc0008d97d0?}, {0x24e2d20?, 0xc0008c7e60?})
        /usr/local/go/src/net/http/h2_bundle.go:1566 +0x6e
net/http.(*http2Framer).ReadFrame(0xc0008d4b60)
        /usr/local/go/src/net/http/h2_bundle.go:1830 +0x95
net/http.(*http2clientConnReadLoop).run(0xc00008bf98)
        /usr/local/go/src/net/http/h2_bundle.go:8820 +0x130
net/http.(*http2ClientConn).readLoop(0xc000256300)
        /usr/local/go/src/net/http/h2_bundle.go:8716 +0x6f
created by net/http.(*http2Transport).newClientConn
        /usr/local/go/src/net/http/h2_bundle.go:7444 +0xa65

goroutine 77 [IO wait]:
internal/poll.runtime_pollWait(0x7f58a54bfb48, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc00042a200?, 0xc0008e7000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc00042a200, {0xc0008e7000, 0xe64, 0xe64})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00042a200, {0xc0008e7000?, 0x201bf01?, 0xc0000de6a0?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000688228, {0xc0008e7000?, 0xc000216810?, 0x488f13?})
        /usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0008c4378, {0xc0008e7000?, 0x0?, 0x34?})
        /usr/local/go/src/crypto/tls/conn.go:785 +0x3d
bytes.(*Buffer).ReadFrom(0xc000118978, {0x24e2ec0, 0xc0008c4378})
        /usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000118700, {0x24e9b20?, 0xc000688228}, 0x37?)
        /usr/local/go/src/crypto/tls/conn.go:807 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc000118700, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:614 +0x116
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc000118700, {0xc000a2b000, 0x1000, 0x47ef05?})
        /usr/local/go/src/crypto/tls/conn.go:1285 +0x16f
bufio.(*Reader).Read(0xc0008e9920, {0xc00092e3c0, 0x9, 0xc000216cd8?})
        /usr/local/go/src/bufio/bufio.go:236 +0x1b4
io.ReadAtLeast({0x24e2d20, 0xc0008e9920}, {0xc00092e3c0, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:331 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
net/http.http2readFrameHeader({0xc00092e3c0?, 0x9?, 0xc000216da0?}, {0x24e2d20?, 0xc0008e9920?})
        /usr/local/go/src/net/http/h2_bundle.go:1566 +0x6e
net/http.(*http2Framer).ReadFrame(0xc00092e380)
        /usr/local/go/src/net/http/h2_bundle.go:1830 +0x95
net/http.(*http2clientConnReadLoop).run(0xc000216f98)
        /usr/local/go/src/net/http/h2_bundle.go:8820 +0x130
net/http.(*http2ClientConn).readLoop(0xc0002da480)
        /usr/local/go/src/net/http/h2_bundle.go:8716 +0x6f
created by net/http.(*http2Transport).newClientConn
        /usr/local/go/src/net/http/h2_bundle.go:7444 +0xa65

goroutine 427 [chan receive]:
github.com/accuknox/auto-policy-discovery/src/recommendpolicy.initDeploymentWatcher()
        /home/hp/Documents/discovery-engine/src/recommendpolicy/helperFunctions.go:160 +0x18a
created by github.com/accuknox/auto-policy-discovery/src/recommendpolicy.StartRecommendCronJob
        /home/hp/Documents/discovery-engine/src/recommendpolicy/recommendPolicy.go:105 +0xd5

goroutine 426 [select]:
github.com/robfig/cron.(*Cron).run(0xc000918000)
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:191 +0x585
created by github.com/robfig/cron.(*Cron).Start
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:144 +0x65

goroutine 358 [select]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc000a9f980, 0xc000338900)
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:1437 +0xa89
golang.org/x/net/http2.(*clientStream).doRequest(0x2501468?, 0xc00091c800?)
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:1299 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:1228 +0x30a

goroutine 359 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000a9f9c8, 0x2)
        /usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc000345d40?)
        /usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc000a9f9b0, {0xc000a02001, 0x3dff, 0x3dff})
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000a02001?, 0x2?, 0x2204a00?})
        /home/hp/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:2507 +0x75
encoding/json.(*Decoder).refill(0xc0008e4a00)
        /usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc0008e4a00)
        /usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc0008e4a00, {0x1f89ca0, 0xc000b35770})
        /usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00099e960, {0xc000a0c000, 0x4000, 0x5000})
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0008cf810, 0xc00096e000?, {0x24f0bf8, 0xc000b56a00})
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000065c60)
        /home/hp/go/pkg/mod/k8s.io/client-go@v0.26.0/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000aff0c0)
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/watch/streamwatcher.go:105 +0xdc
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/watch/streamwatcher.go:76 +0x130

goroutine 1154 [running]:
        goroutine running on other thread; stack unavailable
created by github.com/robfig/cron.(*Cron).run
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:199 +0x79f

goroutine 1154 [running]:
runtime.throw({0x22193a5?, 0xc00058ea08?})
        /usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc000821520 sp=0xc0008214f0 pc=0x4a9cb1
runtime.mapassign(0x1f64ee0, 0xc0001be7e0, 0x7f58a6d85e30?)
        /usr/local/go/src/runtime/map.go:686 +0x4c5 fp=0xc0008215a0 sp=0xc000821520 pc=0x480d45
k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypeWithName(0xc0000d6d90, {{0x221b66b, 0x16}, {0x21facfa, 0x2}, {0x1dfea57, 0x13}}, {0x24ed390?, 0xc0001e7c00})
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme.go:175 +0x2f7 fp=0xc000821838 sp=0xc0008215a0 pc=0x91adb7
k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypes(0xc0000d6d90, {{0x221b66b?, 0x0?}, {0x21facfa?, 0x0?}}, {0xc0004617e0?, 0x2, 0x495?})
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme.go:149 +0x237 fp=0xc000821940 sp=0xc000821838 pc=0x91a577
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).Register.func1(0x23a?)
        /home/hp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/scheme/scheme.go:71 +0x45 fp=0xc0008219a0 sp=0xc000821940 pc=0xe499c5
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme(...)
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme_builder.go:29
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).AddToScheme(0x0?, 0x0?)
        /home/hp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.13.0/pkg/scheme/scheme.go:86 +0x5d fp=0xc0008219d8 sp=0xc0008219a0 pc=0xe49a7d
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).AddToScheme-fm(0x0?)
        <autogenerated>:1 +0x2c fp=0xc0008219f8 sp=0xc0008219d8 pc=0xe513cc
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme(0x0?, 0x0?)
        /home/hp/go/pkg/mod/k8s.io/apimachinery@v0.26.0/pkg/runtime/scheme_builder.go:29 +0x5c fp=0xc000821a30 sp=0xc0008219f8 pc=0x92053c
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme-fm(0x0?)
        <autogenerated>:1 +0x2c fp=0xc000821a50 sp=0xc000821a30 pc=0xe5ddac
github.com/accuknox/auto-policy-discovery/src/cluster.ConnectLocalAPIClient()
        /home/hp/Documents/discovery-engine/src/cluster/k8sClientHandler.go:95 +0x3ea fp=0xc000821b28 sp=0xc000821a50 pc=0x172032a
github.com/accuknox/auto-policy-discovery/src/cluster.ConnectK8sClient()
        /home/hp/Documents/discovery-engine/src/cluster/k8sClientHandler.go:47 +0x3d fp=0xc000821b48 sp=0xc000821b28 pc=0x171ff1d
github.com/accuknox/auto-policy-discovery/src/cluster.GetServicesFromK8sClient()
        /home/hp/Documents/discovery-engine/src/cluster/k8sClientHandler.go:308 +0x2e fp=0xc000822038 sp=0xc000821b48 pc=0x172100e
github.com/accuknox/auto-policy-discovery/src/cluster.GetAllClusterResources({0xc00092cf68?, 0xc00087cbe0?})
        /home/hp/Documents/discovery-engine/src/cluster/clusterResourceHandler.go:45 +0x85 fp=0xc000822180 sp=0xc000822038 pc=0x171f7c5
github.com/accuknox/auto-policy-discovery/src/observability.convertSysLogToSysSummaryMap({0xc000cdd000, 0x3c, 0x40?})
        /home/hp/Documents/discovery-engine/src/observability/summarizer.go:138 +0x537 fp=0xc000822db8 sp=0xc000822180 pc=0x17495b7
github.com/accuknox/auto-policy-discovery/src/observability.ProcessSystemLogs()
        /home/hp/Documents/discovery-engine/src/observability/kubearmor.go:140 +0x1ca fp=0xc000823f60 sp=0xc000822db8 pc=0x174522a
github.com/accuknox/auto-policy-discovery/src/observability.ObservabilityCronJob()
        /home/hp/Documents/discovery-engine/src/observability/observability.go:103 +0x25 fp=0xc000823f70 sp=0xc000823f60 pc=0x1747d05
github.com/robfig/cron.FuncJob.Run(0x100010000?)
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:92 +0x1a fp=0xc000823f80 sp=0xc000823f70 pc=0xdc9e1a
github.com/robfig/cron.(*Cron).runWithRecovery(0xc000b0e1a0?, {0x24eaae0?, 0x2311388?})
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:165 +0x62 fp=0xc000823fb8 sp=0xc000823f80 pc=0xdca182
github.com/robfig/cron.(*Cron).run.func1()
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:199 +0x2e fp=0xc000823fe0 sp=0xc000823fb8 pc=0xdcad4e
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc000823fe8 sp=0xc000823fe0 pc=0x4dc7c1
created by github.com/robfig/cron.(*Cron).run
        /home/hp/go/pkg/mod/github.com/robfig/cron@v1.2.0/cron.go:199 +0x79f

[rksharma95]

i am trying to figure out if this issue is bcoz of the changes that I'm working on. will update here if it reappear without my changes too.