diff --git a/CHANGES.md b/CHANGES.md
index f2ea3df0..a5aa3731 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -42,6 +42,12 @@
     EWMH taskbars and pagers. Useful for `NamedScratchpad` windows, since
     you will usually be taken to the `NSP` workspace by them.
 
+  * `XMonad.Actions.Submap`
+
+    Establish pointer grab to avoid freezing X, when button press occurs after
+    submap key press.  And terminate submap at button press in the same way,
+    as we do for wrong key press.
+
 ### Minor Changes
 
   * `XMonad.Layout.LayoutBuilder`
diff --git a/XMonad/Actions/Submap.hs b/XMonad/Actions/Submap.hs
index cd85880a..d4caa26c 100644
--- a/XMonad/Actions/Submap.hs
+++ b/XMonad/Actions/Submap.hs
@@ -75,17 +75,23 @@ submapDefaultWithKey defAction keys = do
     XConf { theRoot = root, display = d } <- ask
 
     io $ grabKeyboard d root False grabModeAsync grabModeAsync currentTime
+    io $ grabPointer d root False buttonPressMask grabModeAsync grabModeAsync
+                     none none currentTime
 
     (m, s) <- io $ allocaXEvent $ \p -> fix $ \nextkey -> do
-        maskEvent d keyPressMask p
-        KeyEvent { ev_keycode = code, ev_state = m } <- getEvent p
-        keysym <- keycodeToKeysym d code 0
-        if isModifierKey keysym
-            then nextkey
-            else return (m, keysym)
+        maskEvent d (keyPressMask .|. buttonPressMask) p
+        ev <- getEvent p
+        case ev of
+          KeyEvent { ev_keycode = code, ev_state = m } -> do
+            keysym <- keycodeToKeysym d code 0
+            if isModifierKey keysym
+                then nextkey
+                else return (m, keysym)
+          _ -> return (0, 0)
     -- Remove num lock mask and Xkb group state bits
     m' <- cleanMask $ m .&. ((1 `shiftL` 12) - 1)
 
+    io $ ungrabPointer d currentTime
     io $ ungrabKeyboard d currentTime
 
     fromMaybe (defAction (m', s)) (M.lookup (m', s) keys)