diff --git a/src/prompt_toolkit/key_binding/key_bindings.py b/src/prompt_toolkit/key_binding/key_bindings.py
index cf37c6d90..3e529114c 100644
--- a/src/prompt_toolkit/key_binding/key_bindings.py
+++ b/src/prompt_toolkit/key_binding/key_bindings.py
@@ -362,7 +362,8 @@ def remove(self, *args: Keys | str | KeyHandlerCallable) -> None:
             self._clear_cache()
         else:
             # No key binding found for this function. Raise ValueError.
-            raise ValueError(f"Binding not found: {function!r}")
+            binding_repr = ", ".join(map(repr, args))
+            raise ValueError(f"Binding not found: {binding_repr}")
 
     # For backwards-compatibility.
     add_binding = add
diff --git a/tests/test_key_binding.py b/tests/test_key_binding.py
index 1c60880af..e5262324b 100644
--- a/tests/test_key_binding.py
+++ b/tests/test_key_binding.py
@@ -77,6 +77,7 @@ def test_remove_bindings(handlers):
     with set_dummy_app():
         h = handlers.controlx_controlc
         h2 = handlers.controld
+        h3 = handlers.controla
 
         # Test passing a handler to the remove() function.
         bindings = KeyBindings()
@@ -94,6 +95,12 @@ def test_remove_bindings(handlers):
         bindings.remove(Keys.ControlX, Keys.ControlC)
         assert len(bindings.bindings) == 1
 
+        # Test passing unbound key sequence to the remove() function
+        with pytest.raises(ValueError):
+            bindings.remove(h3)
+        with pytest.raises(ValueError):
+            bindings.remove(Keys.ControlA)
+
 
 def test_feed_simple(processor, handlers):
     with set_dummy_app():