diff --git a/packages/block-library/src/group/edit.js b/packages/block-library/src/group/edit.js
index d4706725812164..4b06ec7945ccc8 100644
--- a/packages/block-library/src/group/edit.js
+++ b/packages/block-library/src/group/edit.js
@@ -17,13 +17,13 @@ import {
 	withColors,
 } from '@wordpress/block-editor';
 
+const renderAppender = () => <InnerBlocks.ButtonBlockAppender />;
+
 function GroupEdit( {
 	className,
 	setBackgroundColor,
 	backgroundColor,
-	isSelected,
 	hasInnerBlocks,
-	isInnerBlockSelected,
 } ) {
 	const styles = {
 		backgroundColor: backgroundColor.color,
@@ -33,12 +33,6 @@ function GroupEdit( {
 		'has-background': !! backgroundColor.color,
 	} );
 
-	const isAppenderVisible = (
-		isSelected ||
-		isInnerBlockSelected ||
-		! hasInnerBlocks
-	);
-
 	return (
 		<Fragment>
 			<InspectorControls>
@@ -55,9 +49,7 @@ function GroupEdit( {
 			</InspectorControls>
 			<div className={ classes } style={ styles }>
 				<InnerBlocks
-					renderAppender={ () => (
-						isAppenderVisible && <InnerBlocks.ButtonBlockAppender />
-					) }
+					renderAppender={ ! hasInnerBlocks && renderAppender }
 				/>
 			</div>
 		</Fragment>
@@ -69,14 +61,12 @@ export default compose( [
 	withSelect( ( select, { clientId } ) => {
 		const {
 			getBlock,
-			hasSelectedInnerBlock,
 		} = select( 'core/block-editor' );
 
 		const block = getBlock( clientId );
 
 		return {
 			hasInnerBlocks: !! ( block && block.innerBlocks.length ),
-			isInnerBlockSelected: hasSelectedInnerBlock( clientId, true ),
 		};
 	} ),
 ] )( GroupEdit );
diff --git a/packages/e2e-tests/specs/blocks/group.test.js b/packages/e2e-tests/specs/blocks/group.test.js
index dcf10b7418e608..7ce9021d26bd4b 100644
--- a/packages/e2e-tests/specs/blocks/group.test.js
+++ b/packages/e2e-tests/specs/blocks/group.test.js
@@ -31,13 +31,8 @@ describe( 'Group', () => {
 	it( 'can have other blocks appended to it using the button appender', async () => {
 		await searchForBlock( 'Group' );
 		await page.click( '.editor-block-list-item-group' );
-
-		await page.waitForSelector( '.block-editor-button-block-appender' );
 		await page.click( '.block-editor-button-block-appender' );
-
-		await page.waitForSelector( '.editor-block-list-item-paragraph' );
 		await page.click( '.editor-block-list-item-paragraph' );
-
 		await page.keyboard.type( 'Group Block with a Paragraph' );
 
 		expect( await getEditedPostContent() ).toMatchSnapshot();