diff --git a/.changeset/solid-facts-flow.md b/.changeset/solid-facts-flow.md
new file mode 100644
index 0000000..440a75d
--- /dev/null
+++ b/.changeset/solid-facts-flow.md
@@ -0,0 +1,5 @@
+---
+"react-flexy-panels": minor
+---
+
+feat: enforce dragging on handle only
diff --git a/packages/react-flexy-panels/src/hooks/usePanelDrag.ts b/packages/react-flexy-panels/src/hooks/usePanelDrag.ts
index fc578d2..8ee5b7d 100644
--- a/packages/react-flexy-panels/src/hooks/usePanelDrag.ts
+++ b/packages/react-flexy-panels/src/hooks/usePanelDrag.ts
@@ -14,6 +14,7 @@ export const usePanelDrag = ({ direction, handleId }: UsePanelDragOptions) => {
   const [isDragging, setIsDragging] = useState(false);
   const dragStartRef = useRef<number>(0);
   const handleElementRef = useRef<HTMLElement | null>(null);
+  const unappliedDragDeltaRef = useRef<number>(0);
 
   const updateDragPosition = useCallback(
     (clientX: number, clientY: number) => {
@@ -36,12 +37,17 @@ export const usePanelDrag = ({ direction, handleId }: UsePanelDragOptions) => {
         return;
       }
 
-      updatePanelSizes({
+      const appliedDragDelta = updatePanelSizes({
         panel1,
         panel2,
         dragDelta,
         direction,
+        unappliedDragDelta: unappliedDragDeltaRef.current,
       });
+
+      unappliedDragDeltaRef.current =
+        unappliedDragDeltaRef.current + (dragDelta - appliedDragDelta);
+
       dragStartRef.current = dragCurrent;
     },
     [direction, handleId]
@@ -70,28 +76,35 @@ export const usePanelDrag = ({ direction, handleId }: UsePanelDragOptions) => {
     [isDragging, updateDragPosition]
   );
 
-  const handleMouseDown = useCallback(
-    (e: React.MouseEvent<HTMLDivElement>) => {
-      // Cache handle element reference on drag start
+  const resetDraggingStates = useCallback(
+    (
+      e: React.MouseEvent<HTMLDivElement> | React.TouchEvent<HTMLDivElement>
+    ) => {
       handleElementRef.current = e.currentTarget;
+      unappliedDragDeltaRef.current = 0;
       setIsDragging(true);
+    },
+    []
+  );
+
+  const handleMouseDown = useCallback(
+    (e: React.MouseEvent<HTMLDivElement>) => {
+      resetDraggingStates(e);
       dragStartRef.current = direction === "horizontal" ? e.clientX : e.clientY;
     },
-    [direction]
+    [direction, resetDraggingStates]
   );
 
   const handleTouchStart = useCallback(
     (e: React.TouchEvent<HTMLDivElement>) => {
       if (e.touches.length > 0) {
-        // Cache handle element reference on drag start
-        handleElementRef.current = e.currentTarget;
-        setIsDragging(true);
+        resetDraggingStates(e);
         const touch = e.touches[0];
         dragStartRef.current =
           direction === "horizontal" ? touch.clientX : touch.clientY;
       }
     },
-    [direction]
+    [direction, resetDraggingStates]
   );
 
   useEffect(() => {
diff --git a/packages/react-flexy-panels/src/utils/getPanelSizeByDirection.ts b/packages/react-flexy-panels/src/utils/getPanelSizeByDirection.ts
new file mode 100644
index 0000000..654234e
--- /dev/null
+++ b/packages/react-flexy-panels/src/utils/getPanelSizeByDirection.ts
@@ -0,0 +1,13 @@
+import { Direction } from "../types";
+
+export const getPanelSizeByDirection = ({
+  panel,
+  direction,
+}: {
+  panel: HTMLElement;
+  direction: Direction;
+}) => {
+  return direction === "horizontal"
+    ? panel.getBoundingClientRect().width
+    : panel.getBoundingClientRect().height;
+};
diff --git a/packages/react-flexy-panels/src/utils/index.ts b/packages/react-flexy-panels/src/utils/index.ts
index 1561491..7f0d989 100644
--- a/packages/react-flexy-panels/src/utils/index.ts
+++ b/packages/react-flexy-panels/src/utils/index.ts
@@ -1,3 +1,5 @@
 export * from "./attachSetSizeFunction";
 export * from "./findAdjacentPanels";
-export * from "./panelSize";
+export * from "./getPanelSizeByDirection";
+export * from "./updatePanelSizes";
+export * from "./isSameSign";
\ No newline at end of file
diff --git a/packages/react-flexy-panels/src/utils/isSameSign.ts b/packages/react-flexy-panels/src/utils/isSameSign.ts
new file mode 100644
index 0000000..553b0d3
--- /dev/null
+++ b/packages/react-flexy-panels/src/utils/isSameSign.ts
@@ -0,0 +1,3 @@
+export const isSameSign = (a: number, b: number) => {
+  return (a === 0 && b === 0) || (a > 0 && b > 0) || (a < 0 && b < 0);
+};
diff --git a/packages/react-flexy-panels/src/utils/panelSize.ts b/packages/react-flexy-panels/src/utils/updatePanelSizes.ts
similarity index 76%
rename from packages/react-flexy-panels/src/utils/panelSize.ts
rename to packages/react-flexy-panels/src/utils/updatePanelSizes.ts
index 03b57f1..1d688a3 100644
--- a/packages/react-flexy-panels/src/utils/panelSize.ts
+++ b/packages/react-flexy-panels/src/utils/updatePanelSizes.ts
@@ -1,4 +1,6 @@
 import { Direction } from "../types";
+import { getPanelSizeByDirection } from "./getPanelSizeByDirection";
+import { isSameSign } from "./isSameSign";
 
 type PanelUnit = "px" | "%" | "auto";
 
@@ -10,32 +12,40 @@ export function updatePanelSizes(props: {
   panel2: HTMLDivElement;
   dragDelta: number;
   direction: Direction;
-}): void {
-  const { panel1, panel2, dragDelta, direction } = props;
+  unappliedDragDelta: number;
+}): number {
+  const { panel1, panel2, direction, unappliedDragDelta } = props;
+  let dragDelta = props.dragDelta;
+
+  if (dragDelta === 0) {
+    return 0;
+  }
+
+  if (unappliedDragDelta !== 0 && dragDelta !== 0) {
+    if (!isSameSign(unappliedDragDelta, dragDelta)) {
+      if (Math.abs(unappliedDragDelta) > Math.abs(dragDelta)) {
+        return 0;
+      } else {
+        dragDelta += unappliedDragDelta;
+      }
+    }
+  }
 
   const panel1Unit = (panel1.dataset.unit || "auto") as PanelUnit;
   const panel2Unit = (panel2.dataset.unit || "auto") as PanelUnit;
 
-  const panel1Size =
-    direction === "horizontal"
-      ? panel1.getBoundingClientRect().width
-      : panel1.getBoundingClientRect().height;
-  const panel2Size =
-    direction === "horizontal"
-      ? panel2.getBoundingClientRect().width
-      : panel2.getBoundingClientRect().height;
+  const panel1Size = getPanelSizeByDirection({ panel: panel1, direction });
+  const panel2Size = getPanelSizeByDirection({ panel: panel2, direction });
 
   // Get the parent container to check bounds
   const container = panel1.parentElement;
   const containerSize = container
-    ? direction === "horizontal"
-      ? container.getBoundingClientRect().width
-      : container.getBoundingClientRect().height
+    ? getPanelSizeByDirection({ panel: container, direction })
     : null;
 
   // Calculate new sizes for the two panels being resized
-  let panel1NewSize = panel1Size + dragDelta;
-  let panel2NewSize = panel2Size - dragDelta;
+  let panel1NewSize = Math.abs(panel1Size + dragDelta);
+  let panel2NewSize = Math.abs(panel2Size - dragDelta);
 
   // Constrain sizes to container bounds
   const minPanelSize = 0;
@@ -55,10 +65,7 @@ export function updatePanelSizes(props: {
 
     // Calculate current sizes of all other panels
     const otherPanelsTotalSize = allPanels.reduce((sum, panel) => {
-      const size =
-        direction === "horizontal"
-          ? panel.getBoundingClientRect().width
-          : panel.getBoundingClientRect().height;
+      const size = getPanelSizeByDirection({ panel, direction });
       return sum + size;
     }, 0);
 
@@ -110,4 +117,9 @@ export function updatePanelSizes(props: {
     panel1.style.flex = `0 1 ${panel1NewSize}px`;
     panel2.style.flex = `0 1 ${panel2NewSize}px`;
   }
+
+  const appliedDragDelta1 =
+    getPanelSizeByDirection({ panel: panel1, direction }) - panel1Size;
+
+  return appliedDragDelta1;
 }