diff --git a/packages/edit-site/src/components/global-styles/shadows-edit-panel.js b/packages/edit-site/src/components/global-styles/shadows-edit-panel.js
index f26a8a5ed1743..0de1f2c99362c 100644
--- a/packages/edit-site/src/components/global-styles/shadows-edit-panel.js
+++ b/packages/edit-site/src/components/global-styles/shadows-edit-panel.js
@@ -35,7 +35,7 @@ import {
 	reset,
 	moreVertical,
 } from '@wordpress/icons';
-import { useState, useMemo, useEffect } from '@wordpress/element';
+import { useState, useMemo, useEffect, useRef } from '@wordpress/element';
 
 /**
  * Internal dependencies
@@ -300,6 +300,7 @@ function ShadowsPreview( { shadow } ) {
 }
 
 function ShadowEditor( { shadow, onChange } ) {
+	const addShadowButtonRef = useRef();
 	const shadowParts = useMemo( () => getShadowParts( shadow ), [ shadow ] );
 
 	const onChangeShadowPart = ( index, part ) => {
@@ -314,6 +315,7 @@ function ShadowEditor( { shadow, onChange } ) {
 
 	const onRemoveShadowPart = ( index ) => {
 		onChange( shadowParts.filter( ( p, i ) => i !== index ).join( ', ' ) );
+		addShadowButtonRef.current.focus();
 	};
 
 	return (
@@ -334,6 +336,7 @@ function ShadowEditor( { shadow, onChange } ) {
 							onClick={ () => {
 								onAddShadowPart();
 							} }
+							ref={ addShadowButtonRef }
 						/>
 					</FlexItem>
 				</HStack>
@@ -393,28 +396,24 @@ function ShadowItem( { shadow, onChange, canRemove, onRemove } ) {
 				};
 
 				return (
-					<HStack align="center" justify="flex-start" spacing={ 0 }>
-						<FlexItem style={ { flexGrow: 1 } }>
-							<Button
-								__next40pxDefaultSize
-								icon={ shadowIcon }
-								{ ...toggleProps }
-							>
-								{ shadowObj.inset
-									? __( 'Inner shadow' )
-									: __( 'Drop shadow' ) }
-							</Button>
-						</FlexItem>
+					<>
+						<Button
+							__next40pxDefaultSize
+							icon={ shadowIcon }
+							{ ...toggleProps }
+						>
+							{ shadowObj.inset
+								? __( 'Inner shadow' )
+								: __( 'Drop shadow' ) }
+						</Button>
 						{ canRemove && (
-							<FlexItem>
-								<Button
-									__next40pxDefaultSize
-									icon={ reset }
-									{ ...removeButtonProps }
-								/>
-							</FlexItem>
+							<Button
+								size="small"
+								icon={ reset }
+								{ ...removeButtonProps }
+							/>
 						) }
-					</HStack>
+					</>
 				);
 			} }
 			renderContent={ () => (
diff --git a/packages/edit-site/src/components/global-styles/style.scss b/packages/edit-site/src/components/global-styles/style.scss
index 11dbae7bc66b1..68cc40c4b6206 100644
--- a/packages/edit-site/src/components/global-styles/style.scss
+++ b/packages/edit-site/src/components/global-styles/style.scss
@@ -146,20 +146,34 @@
 
 .edit-site-global-styles__shadow-editor__dropdown {
 	width: 100%;
+	position: relative;
+}
 
-	.edit-site-global-styles__shadow-editor__dropdown-toggle,
-	.edit-site-global-styles__shadow-editor__remove-button {
-		width: 100%;
-		height: auto;
-		padding-top: $grid-unit;
-		padding-bottom: $grid-unit;
-		text-align: left;
-		border-radius: inherit;
-
-		&.is-open {
-			background: $gray-100;
-			color: var(--wp-admin-theme-color);
-		}
+.edit-site-global-styles__shadow-editor__dropdown-toggle {
+	width: 100%;
+	height: auto;
+	padding-top: $grid-unit;
+	padding-bottom: $grid-unit;
+	text-align: left;
+	border-radius: inherit;
+
+	&.is-open {
+		background: $gray-100;
+		color: var(--wp-admin-theme-color);
+	}
+}
+
+.edit-site-global-styles__shadow-editor__remove-button {
+	position: absolute;
+	right: $grid-unit;
+	top: $grid-unit;
+	opacity: 0;
+
+	.edit-site-global-styles__shadow-editor__dropdown-toggle:hover + &,
+	&:focus,
+	&:hover {
+		border: none;
+		opacity: 1;
 	}
 }