diff --git a/packages/block-library/src/cover/edit/index.js b/packages/block-library/src/cover/edit/index.js
index 513e222294dbd9..83dd7b76658231 100644
--- a/packages/block-library/src/cover/edit/index.js
+++ b/packages/block-library/src/cover/edit/index.js
@@ -130,9 +130,7 @@ function CoverEdit( {
 	const isUploadingMedia = isTemporaryMedia( id, url );
 
 	const onUploadError = ( message ) => {
-		createErrorNotice( Array.isArray( message ) ? message[ 2 ] : message, {
-			type: 'snackbar',
-		} );
+		createErrorNotice( message, { type: 'snackbar' } );
 	};
 
 	const mediaElement = useRef();
diff --git a/packages/block-library/src/post-featured-image/edit.js b/packages/block-library/src/post-featured-image/edit.js
index 4ee479b42be257..a133b8846a4ee1 100644
--- a/packages/block-library/src/post-featured-image/edit.js
+++ b/packages/block-library/src/post-featured-image/edit.js
@@ -118,7 +118,7 @@ function PostFeaturedImageDisplay( {
 
 	const { createErrorNotice } = useDispatch( noticesStore );
 	const onUploadError = ( message ) => {
-		createErrorNotice( message[ 2 ], { type: 'snackbar' } );
+		createErrorNotice( message, { type: 'snackbar' } );
 	};
 
 	const controls = (
diff --git a/packages/block-library/src/site-logo/edit.js b/packages/block-library/src/site-logo/edit.js
index 74e721183b0b5d..a08e3520af4b39 100644
--- a/packages/block-library/src/site-logo/edit.js
+++ b/packages/block-library/src/site-logo/edit.js
@@ -469,7 +469,7 @@ export default function LogoEdit( {
 
 	const { createErrorNotice } = useDispatch( noticesStore );
 	const onUploadError = ( message ) => {
-		createErrorNotice( message[ 2 ], { type: 'snackbar' } );
+		createErrorNotice( message, { type: 'snackbar' } );
 	};
 
 	const controls = canUserEdit && logoUrl && (
diff --git a/packages/media-utils/CHANGELOG.md b/packages/media-utils/CHANGELOG.md
index 3c99109b55fa3a..973e0cd472e965 100644
--- a/packages/media-utils/CHANGELOG.md
+++ b/packages/media-utils/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Breaking Change
+
+-   The `onError` now always receives the `message` as a string ([#39448](https://github.com/WordPress/gutenberg/pull/39448)).
+
 ## 3.6.0 (2022-05-04)
 
 ## 3.5.0 (2022-04-21)
diff --git a/packages/media-utils/src/utils/upload-media.js b/packages/media-utils/src/utils/upload-media.js
index e0d484ab76281d..1d55327bcb53ac 100644
--- a/packages/media-utils/src/utils/upload-media.js
+++ b/packages/media-utils/src/utils/upload-media.js
@@ -104,17 +104,6 @@ export async function uploadMedia( {
 		return includes( allowedMimeTypesForUser, fileType );
 	};
 
-	// Build the error message including the filename.
-	const triggerError = ( error ) => {
-		error.message = [
-			<strong key="filename">{ error.file.name }</strong>,
-			': ',
-			error.message,
-		];
-
-		onError( error );
-	};
-
 	const validFiles = [];
 
 	for ( const mediaFile of files ) {
@@ -125,10 +114,14 @@ export async function uploadMedia( {
 			mediaFile.type &&
 			! isAllowedMimeTypeForUser( mediaFile.type )
 		) {
-			triggerError( {
+			onError( {
 				code: 'MIME_TYPE_NOT_ALLOWED_FOR_USER',
-				message: __(
-					'Sorry, you are not allowed to upload this file type.'
+				message: sprintf(
+					// translators: %s: file name.
+					__(
+						'%s: Sorry, you are not allowed to upload this file type.'
+					),
+					mediaFile.name
 				),
 				file: mediaFile,
 			} );
@@ -138,9 +131,13 @@ export async function uploadMedia( {
 		// Check if the block supports this mime type.
 		// Defer to the server when type not detected.
 		if ( mediaFile.type && ! isAllowedType( mediaFile.type ) ) {
-			triggerError( {
+			onError( {
 				code: 'MIME_TYPE_NOT_SUPPORTED',
-				message: __( 'Sorry, this file type is not supported here.' ),
+				message: sprintf(
+					// translators: %s: file name.
+					__( '%s: Sorry, this file type is not supported here.' ),
+					mediaFile.name
+				),
 				file: mediaFile,
 			} );
 			continue;
@@ -148,10 +145,14 @@ export async function uploadMedia( {
 
 		// Verify if file is greater than the maximum file upload size allowed for the site.
 		if ( maxUploadFileSize && mediaFile.size > maxUploadFileSize ) {
-			triggerError( {
+			onError( {
 				code: 'SIZE_ABOVE_LIMIT',
-				message: __(
-					'This file exceeds the maximum upload size for this site.'
+				message: sprintf(
+					// translators: %s: file name.
+					__(
+						'%s: This file exceeds the maximum upload size for this site.'
+					),
+					mediaFile.name
 				),
 				file: mediaFile,
 			} );
@@ -160,9 +161,13 @@ export async function uploadMedia( {
 
 		// Don't allow empty files to be uploaded.
 		if ( mediaFile.size <= 0 ) {
-			triggerError( {
+			onError( {
 				code: 'EMPTY_FILE',
-				message: __( 'This file is empty.' ),
+				message: sprintf(
+					// translators: %s: file name.
+					__( '%s: This file is empty.' ),
+					mediaFile.name
+				),
 				file: mediaFile,
 			} );
 			continue;