diff --git a/src/components/helpers/dom_helpers.ts b/src/components/helpers/dom_helpers.ts
index 242bdd891f..ea711a5be6 100644
--- a/src/components/helpers/dom_helpers.ts
+++ b/src/components/helpers/dom_helpers.ts
@@ -288,3 +288,25 @@ function isOtherMobileOS() {
 export function isMobileOS() {
   return isAndroid() || isIOS() || isOtherMobileOS();
 }
+
+const INTERACTIVE_ELEMENT_SELECTOR =
+  "select, input, textarea, button, .o-select, .os-input, .o-input, .o-button, .o-button-icon, .o-button-link, .o-composer";
+
+/** Check if there is an interactive element (input, select, button, ...) between the event.target and event.currentTarget (inclusive)*/
+export function hasInteractiveElementInEventTree(event: Event) {
+  const target = event.target as HTMLElement | null;
+  const root = event.currentTarget as HTMLElement | null;
+  if (!root || !target || !root.contains(target)) {
+    return false;
+  }
+
+  let node: HTMLElement | null = target;
+  while (node && node !== root) {
+    if (node.matches(INTERACTIVE_ELEMENT_SELECTOR)) {
+      return true;
+    }
+    node = node.parentElement;
+  }
+
+  return root.matches(INTERACTIVE_ELEMENT_SELECTOR);
+}
diff --git a/src/components/side_panel/pivot/pivot_custom_groups_collapsible/pivot_custom_groups_collapsible.xml b/src/components/side_panel/pivot/pivot_custom_groups_collapsible/pivot_custom_groups_collapsible.xml
index c907786ea9..710de1eed1 100644
--- a/src/components/side_panel/pivot/pivot_custom_groups_collapsible/pivot_custom_groups_collapsible.xml
+++ b/src/components/side_panel/pivot/pivot_custom_groups_collapsible/pivot_custom_groups_collapsible.xml
@@ -11,9 +11,7 @@
             t-as="group"
             t-key="group_index"
             class="o-pivot-custom-group pb-1">
-            <div
-              class="d-flex align-items-center justify-content-between small"
-              t-on-pointerdown.stop="">
+            <div class="d-flex align-items-center justify-content-between small">
               <TextInput
                 value="group.name"
                 onChange="(newName) => this.onRenameGroup(group_index, newName)"
diff --git a/src/components/side_panel/pivot/pivot_layout_configurator/pivot_dimension/pivot_dimension.xml b/src/components/side_panel/pivot/pivot_layout_configurator/pivot_dimension/pivot_dimension.xml
index f39308f8c3..b2d89b9c65 100644
--- a/src/components/side_panel/pivot/pivot_layout_configurator/pivot_dimension/pivot_dimension.xml
+++ b/src/components/side_panel/pivot/pivot_layout_configurator/pivot_dimension/pivot_dimension.xml
@@ -19,7 +19,7 @@
           />
           <span t-else="1" class="o-fw-bold text-truncate" t-esc="dimensionDisplayName"/>
         </div>
-        <div class="d-flex flex-rows" t-on-pointerdown.stop="">
+        <div class="d-flex flex-rows">
           <t t-slot="upper-right-icons"/>
           <i
             class="o-button-icon fa fa-trash pe-1 ps-2"
diff --git a/src/components/side_panel/pivot/pivot_layout_configurator/pivot_layout_configurator.ts b/src/components/side_panel/pivot/pivot_layout_configurator/pivot_layout_configurator.ts
index 100279844b..143d644a08 100644
--- a/src/components/side_panel/pivot/pivot_layout_configurator/pivot_layout_configurator.ts
+++ b/src/components/side_panel/pivot/pivot_layout_configurator/pivot_layout_configurator.ts
@@ -17,6 +17,7 @@ import {
 } from "../../../../types/pivot";
 import { SpreadsheetChildEnv } from "../../../../types/spreadsheet_env";
 import { ComposerFocusStore } from "../../../composer/composer_focus_store";
+import { hasInteractiveElementInEventTree } from "../../../helpers/dom_helpers";
 import { useDragAndDropListItems } from "../../../helpers/drag_and_drop_dom_items_hook";
 import { PivotCustomGroupsCollapsible } from "../pivot_custom_groups_collapsible/pivot_custom_groups_collapsible";
 import { AddDimensionButton } from "./add_dimension_button/add_dimension_button";
@@ -73,7 +74,7 @@ export class PivotLayoutConfigurator extends Component<Props, SpreadsheetChildEn
   }
 
   startDragAndDrop(dimension: PivotDimensionType, event: MouseEvent) {
-    if (event.button !== 0 || (event.target as HTMLElement).tagName === "SELECT") {
+    if (event.button !== 0 || hasInteractiveElementInEventTree(event)) {
       return;
     }
 
@@ -137,8 +138,7 @@ export class PivotLayoutConfigurator extends Component<Props, SpreadsheetChildEn
   startDragAndDropMeasures(measure: PivotMeasure, event: MouseEvent) {
     if (
       event.button !== 0 ||
-      (event.target as HTMLElement).tagName === "SELECT" ||
-      (event.target as HTMLElement).tagName === "INPUT" ||
+      hasInteractiveElementInEventTree(event) ||
       this.composerFocus.focusMode !== "inactive"
     ) {
       return;
diff --git a/src/components/side_panel/pivot/pivot_layout_configurator/pivot_measure/pivot_measure.xml b/src/components/side_panel/pivot/pivot_layout_configurator/pivot_measure/pivot_measure.xml
index d8c6b1cb27..27f5edc568 100644
--- a/src/components/side_panel/pivot/pivot_layout_configurator/pivot_measure/pivot_measure.xml
+++ b/src/components/side_panel/pivot/pivot_layout_configurator/pivot_measure/pivot_measure.xml
@@ -18,7 +18,7 @@
         />
       </t>
       <div t-if="measure.computedBy" class="d-flex flex-row small">
-        <div class="d-flex flex-column py-2 px-2 w-100" t-on-pointerdown.stop="">
+        <div class="d-flex flex-column py-2 px-2 w-100">
           <StandaloneComposer
             onConfirm.bind="updateMeasureFormula"
             composerContent="measure.computedBy.formula"
diff --git a/tests/pivots/spreadsheet_pivot/spreadsheet_pivot_side_panel.test.ts b/tests/pivots/spreadsheet_pivot/spreadsheet_pivot_side_panel.test.ts
index 5cfd3a036d..6d0ca73249 100644
--- a/tests/pivots/spreadsheet_pivot/spreadsheet_pivot_side_panel.test.ts
+++ b/tests/pivots/spreadsheet_pivot/spreadsheet_pivot_side_panel.test.ts
@@ -862,34 +862,49 @@ describe("Spreadsheet pivot side panel", () => {
     ]);
   });
 
-  test("Cannot drag a dimension when clicking its upper right icons", async () => {
+  test("Cannot drag a dimension when clicking on one of its clickable element", async () => {
     extendMockGetBoundingClientRect({
-      /**
-       * 'pt-1' is the class of the main div of the pivot dimension
-       */
-      "pt-1": () => ({
-        height: 10,
-        y: 0,
-      }),
-      "o-section-title": () => ({
-        height: 10,
-        y: 10,
-      }),
-      "pivot-dimensions": () => ({
-        height: 40,
-        y: 0,
-      }),
+      "h-100": () => ({ height: 40, y: 0 }), // side panel, it is the parent container of the drag & drop
+      "pt-1": () => ({ height: 10, y: 0 }), // main div of a pivot dimension
+      "o-section-title": () => ({ height: 10, y: 10 }),
     });
-    await click(fixture.querySelector(".add-dimension")!);
-    await click(fixture.querySelectorAll(".o-autocomplete-value")[0]);
-    await editSelectComponent(".pivot-dimension .o-select", "desc");
-    expect(model.getters.getPivotCoreDefinition("1").columns).toEqual([
-      { fieldName: "Amount", order: "desc" },
-    ]);
-    await clickAndDrag(".pivot-dimension .fa-trash", { x: 0, y: 30 }, undefined, true);
-    expect(model.getters.getPivotCoreDefinition("1").columns).toEqual([
-      { fieldName: "Amount", order: "desc" },
-    ]);
+    updatePivot(model, "1", { columns: [{ fieldName: "Amount" }] });
+    await nextTick();
+
+    const dragAndDropContainer = () => fixture.querySelector(".pivot-dimension")?.parentElement;
+
+    expect(dragAndDropContainer()).not.toHaveStyle({ position: "relative" }); // this style is added during the drag & drop
+
+    await clickAndDrag(".pivot-dimension .fa-trash", { x: 0, y: 30 }, undefined, false);
+    expect(dragAndDropContainer()).not.toHaveStyle({ position: "relative" });
+
+    await clickAndDrag(".pivot-dimension .o-select", { x: 0, y: 30 }, undefined, false);
+    expect(dragAndDropContainer()).not.toHaveStyle({ position: "relative" });
+  });
+
+  test("Cannot drag a measure when clicking on one of its clickable element", async () => {
+    extendMockGetBoundingClientRect({
+      "h-100": () => ({ height: 40, y: 0 }), // side panel, it is the parent container of the drag & drop
+      "pt-1": () => ({ height: 10, y: 0 }), // main div of a pivot measure
+    });
+    updatePivot(model, "1", {
+      measures: [
+        { id: "amount:sum", fieldName: "amount", aggregator: "sum" },
+        { id: "amount:avg", fieldName: "person", aggregator: "avg" },
+      ],
+    });
+    await nextTick();
+
+    expect(".pivot-measure").not.toHaveStyle({ position: "relative" }); // this style is added during the drag & drop
+
+    await clickAndDrag(".pivot-measure .fa-trash", { x: 0, y: 30 }, undefined, false);
+    expect(".pivot-measure").not.toHaveStyle({ position: "relative" });
+
+    await clickAndDrag(".pivot-measure .o-select", { x: 0, y: 30 }, undefined, false);
+    expect(".pivot-measure").not.toHaveStyle({ position: "relative" });
+
+    await clickAndDrag(".pivot-measure .os-input", { x: 0, y: 30 }, undefined, false);
+    expect(".pivot-measure").not.toHaveStyle({ position: "relative" });
   });
 
   test("Pivot with multiple time the same dimension does not crash the side panel", async () => {