diff --git a/packages/format-library/src/link/index.js b/packages/format-library/src/link/index.js
index e1aa17e78f8d0..4657cc093201c 100644
--- a/packages/format-library/src/link/index.js
+++ b/packages/format-library/src/link/index.js
@@ -39,28 +39,22 @@ function Edit( {
 	contentRef,
 } ) {
 	const [ addingLink, setAddingLink ] = useState( false );
-	const [ clickedLink, setClickedLink ] = useState( false );
-
-	function setClickedLinkTrue() {
-		setClickedLink( true );
-	}
-
-	function setClickedLinkFalse() {
-		setClickedLink( false );
-	}
 
 	useLayoutEffect( () => {
 		const editableContentElement = contentRef.current;
 
-		function handleClick() {
-			setClickedLinkTrue();
+		function handleClick( event ) {
+			if ( event.target.tagName !== 'A' ) {
+				return;
+			}
+			setAddingLink( true );
 		}
 
 		editableContentElement.addEventListener( 'click', handleClick );
 
 		return () => {
 			editableContentElement.removeEventListener( 'click', handleClick );
-			setClickedLinkFalse();
+			setAddingLink( false );
 		};
 	}, [ contentRef, isActive ] );
 
@@ -117,7 +111,7 @@ function Edit( {
 				aria-haspopup="true"
 				aria-expanded={ addingLink }
 			/>
-			{ ( addingLink || ( isActive && clickedLink ) ) && (
+			{ addingLink && (
 				<InlineLinkUI
 					addingLink={ addingLink }
 					stopAddingLink={ stopAddingLink }