Any plans for recent kernels? (6.11)

[iafilius]

Hi,

Just starting in remote kernel debugging, i tried the kgdboe and followed your instructions.
compiled agains 6.11-rc3 kandidate, and apart from few warnings it looked and loaded OK.
however my journey stopped quit quickly after getting:

Malformed response to offset query, QCfffffffe
after trying to set-up the connect on gdb prompt.
The remote physical kernel/system didn't respond anymore, and i was not able to recover some logging.

compiling agains 6.9.9 gives:

$ make -C /lib/modules/$(uname -r)/build M=$(pwd)
make: Entering directory '/usr/src/linux-headers-6.9.9-custom'
  CC [M]  /home/arjan/kernel/t/kgdboe/irqsync.o
  CC [M]  /home/arjan/kernel/t/kgdboe/kgdboe_main.o
  CC [M]  /home/arjan/kernel/t/kgdboe/kgdboe_io.o
/home/arjan/kernel/t/kgdboe/kgdboe_io.c:28:6: warning: no previous prototype for ‘kgdb_schedule_breakpoint’ [-Wmissing-prototypes]
   28 | void kgdb_schedule_breakpoint(void)
      |      ^~~~~~~~~~~~~~~~~~~~~~~~
/home/arjan/kernel/t/kgdboe/kgdboe_io.c: In function ‘kgdboe_read_char’:
/home/arjan/kernel/t/kgdboe/kgdboe_io.c:88:5: warning: this ‘while’ clause does not guard... [-Wmisleading-indentation]
   88 |     while (s_IncomingRingBufferReadPosition == s_IncomingRingBufferWritePosition)
      |     ^~~~~
/home/arjan/kernel/t/kgdboe/kgdboe_io.c:91:9: note: ...this statement, but the latter is misleadingly indented as if it were guarded by the ‘while’
   91 |         result = s_IncomingRingBuffer[s_IncomingRingBufferReadPosition++];
      |         ^~~~~~
/home/arjan/kernel/t/kgdboe/kgdboe_io.c: At top level:
/home/arjan/kernel/t/kgdboe/kgdboe_io.c:126:5: warning: no previous prototype for ‘force_single_cpu_mode’ [-Wmissing-prototypes]
  126 | int force_single_cpu_mode(void)
      |     ^~~~~~~~~~~~~~~~~~~~~
In file included from /home/arjan/kernel/t/kgdboe/kgdboe_io.c:8:
/home/arjan/kernel/t/kgdboe/kgdboe_io.c: In function ‘kgdboe_io_init’:
/home/arjan/kernel/t/kgdboe/tracewrapper.h:8:30: warning: initialization of ‘int *’ from ‘long unsigned int’ makes pointer from integer without a cast [-Wint-conversion]
    8 | #define kallsyms_lookup_name ((unsigned long(*)(const char *))kallsyms_lookup_name_address)
      |                              ^
/home/arjan/kernel/t/kgdboe/kgdboe_io.c:175:30: note: in expansion of macro ‘kallsyms_lookup_name’
  175 |     int *_gro_normal_batch = kallsyms_lookup_name("gro_normal_batch");
      |                              ^~~~~~~~~~~~~~~~~~~~
/home/arjan/kernel/t/kgdboe/tracewrapper.h: At top level:
/home/arjan/kernel/t/kgdboe/tracewrapper.h:23:27: warning: ‘do_lookup_tracepoint’ defined but not used [-Wunused-function]
   23 | static struct tracepoint *do_lookup_tracepoint(const char *tracepointName, const char *tracepointPtrName)
      |                           ^~~~~~~~~~~~~~~~~~~~
  CC [M]  /home/arjan/kernel/t/kgdboe/nethook.o
/home/arjan/kernel/t/kgdboe/nethook.c:96:24: warning: no previous prototype for ‘ndo_get_stats64_hook’ [-Wmissing-prototypes]
   96 | DECLARE_NET_API_HOOK2V(ndo_get_stats64, void, struct net_device *, dev, struct rtnl_link_stats64 *, storage)
      |                        ^~~~~~~~~~~~~~~
/home/arjan/kernel/t/kgdboe/nethook.c:86:21: note: in definition of macro ‘DECLARE_NET_API_HOOK2V’
   86 |         return_type name ## _hook(type1 arg1, type2 arg2)       \
      |                     ^~~~
/home/arjan/kernel/t/kgdboe/nethook.c:99:23: warning: no previous prototype for ‘ndo_get_stats_hook’ [-Wmissing-prototypes]
   99 | DECLARE_NET_API_HOOK1(ndo_get_stats, struct net_device_stats*, struct net_device *, dev)
      |                       ^~~~~~~~~~~~~
/home/arjan/kernel/t/kgdboe/nethook.c:62:21: note: in definition of macro ‘DECLARE_NET_API_HOOK1’
   62 |         return_type name ## _hook(type1 arg1)   \
      |                     ^~~~
In file included from /home/arjan/kernel/t/kgdboe/nethook.c:13:
/home/arjan/kernel/t/kgdboe/tracewrapper.h:23:27: warning: ‘do_lookup_tracepoint’ defined but not used [-Wunused-function]
   23 | static struct tracepoint *do_lookup_tracepoint(const char *tracepointName, const char *tracepointPtrName)
      |                           ^~~~~~~~~~~~~~~~~~~~
  CC [M]  /home/arjan/kernel/t/kgdboe/netpoll_wrapper.o
  CC [M]  /home/arjan/kernel/t/kgdboe/poll_copy.o
  CC [M]  /home/arjan/kernel/t/kgdboe/spinhook.o
  CC [M]  /home/arjan/kernel/t/kgdboe/timerhook.o
  LD [M]  /home/arjan/kernel/t/kgdboe/kgdboe.o
  MODPOST /home/arjan/kernel/t/kgdboe/Module.symvers
  CC [M]  /home/arjan/kernel/t/kgdboe/kgdboe.mod.o
  LD [M]  /home/arjan/kernel/t/kgdboe/kgdboe.ko
make: Leaving directory '/usr/src/linux-headers-6.9.9-custom'

And similar on 6.11-rc3.

root@arjan-ThinkPad-T430:~/nouveau-debug/t/kgdboe# make -C /lib/modules/$(uname -r)/build M=$(pwd)
make: Entering directory '/usr/src/linux-headers-6.11.0-rc3-custom'
  CC [M]  /root/nouveau-debug/t/kgdboe/irqsync.o
  CC [M]  /root/nouveau-debug/t/kgdboe/kgdboe_main.o
  CC [M]  /root/nouveau-debug/t/kgdboe/kgdboe_io.o
/root/nouveau-debug/t/kgdboe/kgdboe_io.c:28:6: warning: no previous prototype for ‘kgdb_schedule_breakpoint’ [-Wmissing-prototypes]
   28 | void kgdb_schedule_breakpoint(void)
      |      ^~~~~~~~~~~~~~~~~~~~~~~~
/root/nouveau-debug/t/kgdboe/kgdboe_io.c: In function ‘kgdboe_read_char’:
/root/nouveau-debug/t/kgdboe/kgdboe_io.c:88:5: warning: this ‘while’ clause does not guard... [-Wmisleading-indentation]
   88 |     while (s_IncomingRingBufferReadPosition == s_IncomingRingBufferWritePosition)
      |     ^~~~~
/root/nouveau-debug/t/kgdboe/kgdboe_io.c:91:9: note: ...this statement, but the latter is misleadingly indented as if it were guarded by the ‘while’
   91 |         result = s_IncomingRingBuffer[s_IncomingRingBufferReadPosition++];
      |         ^~~~~~
/root/nouveau-debug/t/kgdboe/kgdboe_io.c: At top level:
/root/nouveau-debug/t/kgdboe/kgdboe_io.c:126:5: warning: no previous prototype for ‘force_single_cpu_mode’ [-Wmissing-prototypes]
  126 | int force_single_cpu_mode(void)
      |     ^~~~~~~~~~~~~~~~~~~~~
In file included from /root/nouveau-debug/t/kgdboe/kgdboe_io.c:8:
/root/nouveau-debug/t/kgdboe/kgdboe_io.c: In function ‘kgdboe_io_init’:
/root/nouveau-debug/t/kgdboe/tracewrapper.h:8:30: warning: initialization of ‘int *’ from ‘long unsigned int’ makes pointer from integer without a cast [-Wint-conversion]
    8 | #define kallsyms_lookup_name ((unsigned long(*)(const char *))kallsyms_lookup_name_address)
      |                              ^
/root/nouveau-debug/t/kgdboe/kgdboe_io.c:175:30: note: in expansion of macro ‘kallsyms_lookup_name’
  175 |     int *_gro_normal_batch = kallsyms_lookup_name("gro_normal_batch");
      |                              ^~~~~~~~~~~~~~~~~~~~
/root/nouveau-debug/t/kgdboe/tracewrapper.h: At top level:
/root/nouveau-debug/t/kgdboe/tracewrapper.h:23:27: warning: ‘do_lookup_tracepoint’ defined but not used [-Wunused-function]
   23 | static struct tracepoint *do_lookup_tracepoint(const char *tracepointName, const char *tracepointPtrName)
      |                           ^~~~~~~~~~~~~~~~~~~~
  CC [M]  /root/nouveau-debug/t/kgdboe/nethook.o
/root/nouveau-debug/t/kgdboe/nethook.c:96:24: warning: no previous prototype for ‘ndo_get_stats64_hook’ [-Wmissing-prototypes]
   96 | DECLARE_NET_API_HOOK2V(ndo_get_stats64, void, struct net_device *, dev, struct rtnl_link_stats64 *, storage)
      |                        ^~~~~~~~~~~~~~~
/root/nouveau-debug/t/kgdboe/nethook.c:86:21: note: in definition of macro ‘DECLARE_NET_API_HOOK2V’
   86 |         return_type name ## _hook(type1 arg1, type2 arg2)       \
      |                     ^~~~
/root/nouveau-debug/t/kgdboe/nethook.c:99:23: warning: no previous prototype for ‘ndo_get_stats_hook’ [-Wmissing-prototypes]
   99 | DECLARE_NET_API_HOOK1(ndo_get_stats, struct net_device_stats*, struct net_device *, dev)
      |                       ^~~~~~~~~~~~~
/root/nouveau-debug/t/kgdboe/nethook.c:62:21: note: in definition of macro ‘DECLARE_NET_API_HOOK1’
   62 |         return_type name ## _hook(type1 arg1)   \
      |                     ^~~~
In file included from /root/nouveau-debug/t/kgdboe/nethook.c:13:
/root/nouveau-debug/t/kgdboe/tracewrapper.h:23:27: warning: ‘do_lookup_tracepoint’ defined but not used [-Wunused-function]
   23 | static struct tracepoint *do_lookup_tracepoint(const char *tracepointName, const char *tracepointPtrName)
      |                           ^~~~~~~~~~~~~~~~~~~~
  CC [M]  /root/nouveau-debug/t/kgdboe/netpoll_wrapper.o
  CC [M]  /root/nouveau-debug/t/kgdboe/poll_copy.o
  CC [M]  /root/nouveau-debug/t/kgdboe/spinhook.o
  CC [M]  /root/nouveau-debug/t/kgdboe/timerhook.o
  LD [M]  /root/nouveau-debug/t/kgdboe/kgdboe.o
  MODPOST /root/nouveau-debug/t/kgdboe/Module.symvers
  CC [M]  /root/nouveau-debug/t/kgdboe/kgdboe.mod.o
  LD [M]  /root/nouveau-debug/t/kgdboe/kgdboe.ko
make: Leaving directory '/usr/src/linux-headers-6.11.0-rc3-custom'

As you may see i'm trying to setup a remote kernel debug for debugging the nouveau driver.

regards

[KatCote]

Same problem.

[KatCote]

After run in gdb "target remote" kernel stopped and i have in gdb this error:
Bogus trace status reply from target: OK

[KatCote]

@sysprogs

[musamaanjum]

I'm running against v6.11 and getting the following error:

sudo insmod kgdboe.ko kallsyms_lookup_name_address=0x$(sudo cat /proc/kallsyms | grep 'T kallsyms_lookup_name' | awk '{print $1}') device_namewlan0
insmod: ERROR: could not insert module kgdboe.ko: Invalid parameters

Any idea how to connect gdb with kdb over the internet without kgdboe?

[Koston-0xDEADBEEF]

I'm seeing the very same problem, on a much older kernel (5.15.1/armv7l).

@iafilius You can ignore compiler warnings. They tend to increase along with compiler version, as the compiler gets more nitpicky.

@musamaanjum Take a look at dmesg, there should be more detailed explanation for the insmod failure.

Still working on triaging the issue, but at least I now know it's not directly caused by KGDBoE. I can replicate it also without even loading the module and using kgdboc instead. As long as there's traffic on the system's network interface, communication from the kernel gdb stub gets corrupted (and whatever else in kernelspace). Disabling the NIC entirely restores KGDB back into working order.

Any idea how to connect gdb with kdb over the internet without kgdboe?

Use a secondary system as the debug master, using a serial link. You will anyway need to rig up some way to remotely powercycle the debugging target, because kernel debugging is pretty unsafe and it's quite easy to end up hardlocking it, or DoS'ing the slow serial link. Some small/cheap embedded platform should do the trick, and you can utilise some GPIO pins for the powercycle circuit.

It doesn't help either that many Linux devs foster deep (and deeply misguided) contempt for kernel debuggers, causing a chronic problem with keeping the source tree debugger-friendly. There are quite a few alternative/complementary debugging systems though, I suggest also checking those through (debugfs, compile-time debug options for specific subsystems, /dev/mem, more dynamic tracing probe gizmos than you can shake a dick at, dedicated tools like usbmon etc.).

[Koston-0xDEADBEEF]

Still working on triaging the issue, but at least I now know it's not directly caused by KGDBoE. I can replicate it also without even loading the module and using kgdboc instead. As long as there's traffic on the system's network interface, communication from the kernel gdb stub gets corrupted (and whatever else in kernelspace). Disabling the NIC entirely restores KGDB back into working order.

I was unable to pinpoint the cause, but it seems something in the way some network drivers implement netpoll support is causing it to write on kgdb heap. This renders kgdb severely handicapped to debug the issue. It would be interesting to solve, but due to project backlog I'm cutting my losses here and moving on with debugging over a serial link with 3000000 baud speed.

[LikeAWalkInThePark]

diff --git a/irqsync.c b/irqsync.c
index 85ddd11..14470ae 100644
--- a/irqsync.c
+++ b/irqsync.c
@@ -45,7 +45,11 @@ void irqsync_free(struct irqsync_manager *mgr)
struct managed_irq *irq, *tmp;

BUG_ON(!mgr);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
del_timer(&mgr->timer);
+#else

• timer_delete_sync(&mgr->timer);
  +#endif
  spin_lock(&mgr->lock);
  if (mgr->irqs_disabled)
  irqsync_enable_all_irqs_locked(mgr);
  @@ -108,8 +112,10 @@ static void irqsync_timer_func(struct timer_list * t)
  {
  #if LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)
  struct irqsync_manager *mgr = (struct irqsync_manager *)ctx;
  -#else
  +#elif LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
  struct irqsync_manager *mgr = from_timer(mgr, t, timer);
  +#else
• struct irqsync_manager *mgr = timer_container_of(mgr, t, timer);
  #endif
  BUG_ON(!mgr);
  if (spin_trylock(&mgr->lock))
  diff --git a/kgdboe_io.c b/kgdboe_io.c
  index 75940c7..ddb5897 100644
  --- a/kgdboe_io.c
  +++ b/kgdboe_io.c
  @@ -6,6 +6,9 @@
  #include "netpoll_wrapper.h"
  #include "nethook.h"
  #include "tracewrapper.h"
  +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0)
• #include <net/hotdata.h>
  +#endif

struct netpoll_wrapper *s_pKgdboeNetpoll;

@@ -171,15 +174,19 @@ int kgdboe_io_init(const char *device_name, int port, const char *local_ip, bool
s_pKgdboeNetpoll = netpoll_wrapper_create(device_name, port, local_ip);
if (!s_pKgdboeNetpoll)
return -EINVAL;

• int *_gro_normal_batch = kallsyms_lookup_name("gro_normal_batch");
• if (_gro_normal_batch)
• {

•    //Unless we do this, thet network stack will internally accumulate packets before processing, greatly increasing KGDBoE latency.
  

•    //See gro_normal_one() in dev.c for details.
  

•    *_gro_normal_batch = 1;
  

• }

• //Unless we do this, thet network stack will internally accumulate packets before processing, greatly increasing KGDBoE latency.
• //See gro_normal_one() in dev.c for details.
• #if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)

• int *_gro_normal_batch = kallsyms_lookup_name("gro_normal_batch");
  

• if (_gro_normal_batch)
  

• 	*_gro_normal_batch = 1;
  

• #else

• struct net_hotdata * _net_hotdata = kallsyms_lookup_name("net_hotdata");
  

• if (_net_hotdata)
  

• 	_net_hotdata->gro_normal_batch = 1;
  

• #endif
• if (force_single_core)
  {
  err = force_single_cpu_mode();
  diff --git a/kgdboe_main.c b/kgdboe_main.c
  index f68abbb..9d2b55c 100644
  --- a/kgdboe_main.c
  +++ b/kgdboe_main.c
  @@ -38,6 +38,7 @@
  #include "kgdboe_io.h"

MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("KGDBoE - A Network Based KGDB Extension");

#include <linux/timer.h>

diff --git a/nethook.c b/nethook.c
index c082f9a..db425aa 100644
--- a/nethook.c
+++ b/nethook.c
@@ -202,7 +202,11 @@ bool nethook_initialize(struct net_device *dev)

nethook.hooked_device = dev;

+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
for (i = 0; i < nr_irqs; i++)
+#else

• for (i = 0; i < irq_get_nr_irqs(); i++)
  +#endif
  {
  struct irq_desc *desc = pirq_to_desc(i);
  if (!desc || !desc->action)

[LikeAWalkInThePark]

Pasted above are my fixes for Linux 6.9+.

This was done ask quickly and cleanly as possible.

Copy and past the message raw and apply the diff.

[smaudet]

diff --git a/irqsync.c b/irqsync.c
index 85ddd11..14470ae 100644
--- a/irqsync.c
+++ b/irqsync.c
@@ -45,7 +45,11 @@ void irqsync_free(struct irqsync_manager *mgr)
 \tstruct managed_irq *irq, *tmp;
 
 \tBUG_ON(!mgr);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)\t
 \tdel_timer(&mgr->timer);
+#else
+\ttimer_delete_sync(&mgr->timer);
+#endif
 \tspin_lock(&mgr->lock);
 \tif (mgr->irqs_disabled)
 \t\tirqsync_enable_all_irqs_locked(mgr);
@@ -108,8 +112,10 @@ static void irqsync_timer_func(struct timer_list * t)
 {
 #if LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)
 \tstruct irqsync_manager *mgr = (struct irqsync_manager *)ctx;
-#else
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
 \tstruct irqsync_manager *mgr = from_timer(mgr, t, timer);
+#else
+\tstruct irqsync_manager *mgr = timer_container_of(mgr, t, timer);
 #endif
 \tBUG_ON(!mgr);
 \tif (spin_trylock(&mgr->lock))
diff --git a/kgdboe_io.c b/kgdboe_io.c
index 75940c7..ddb5897 100644
--- a/kgdboe_io.c
+++ b/kgdboe_io.c
@@ -6,6 +6,9 @@
 #include "netpoll_wrapper.h"
 #include "nethook.h"
 #include "tracewrapper.h"
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0)
+\t#include <net/hotdata.h>
+#endif
 
 struct netpoll_wrapper *s_pKgdboeNetpoll;
 
@@ -171,15 +174,19 @@ int kgdboe_io_init(const char *device_name, int port, const char *local_ip, bool
 \ts_pKgdboeNetpoll = netpoll_wrapper_create(device_name, port, local_ip);
 \tif (!s_pKgdboeNetpoll)
 \t\treturn -EINVAL;
-    
-    int *_gro_normal_batch = kallsyms_lookup_name("gro_normal_batch");
-    if (_gro_normal_batch)
-    {
-        //Unless we do this, thet network stack will internally accumulate packets before processing, greatly increasing KGDBoE latency.
-        //See gro_normal_one() in dev.c for details.
-        *_gro_normal_batch = 1;
-    }
-\t
+
+\t//Unless we do this, thet network stack will internally accumulate packets before processing, greatly increasing KGDBoE latency.
+\t//See gro_normal_one() in dev.c for details.
+\t#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
+\t\tint *_gro_normal_batch = kallsyms_lookup_name("gro_normal_batch");
+\t\tif (_gro_normal_batch)
+\t\t\t*_gro_normal_batch = 1;
+\t#else
+\t\tstruct net_hotdata * _net_hotdata = kallsyms_lookup_name("net_hotdata");
+\t\tif (_net_hotdata)
+\t\t\t_net_hotdata->gro_normal_batch = 1;
+\t#endif
+
 \tif (force_single_core)
 \t{
 \t\terr = force_single_cpu_mode();
diff --git a/kgdboe_main.c b/kgdboe_main.c
index f68abbb..9d2b55c 100644
--- a/kgdboe_main.c
+++ b/kgdboe_main.c
@@ -38,6 +38,7 @@
 #include "kgdboe_io.h"
 
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("KGDBoE - A Network Based KGDB Extension");
 
 #include <linux/timer.h>
 
diff --git a/nethook.c b/nethook.c
index c082f9a..db425aa 100644
--- a/nethook.c
+++ b/nethook.c
@@ -202,7 +202,11 @@ bool nethook_initialize(struct net_device *dev)
 
 \tnethook.hooked_device = dev;
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
 \tfor (i = 0; i < nr_irqs; i++)
+#else
+\tfor (i = 0; i < irq_get_nr_irqs(); i++)
+#endif
 \t{
 \t\tstruct irq_desc *desc = pirq_to_desc(i);
 \t\tif (!desc || !desc->action)

Reposting @LikeAWalkInThePark 's diff since github garbled it.