diff --git a/packages/components/src/color-edit/index.js b/packages/components/src/color-edit/index.js
index fc38c443f5a915..972ead0825737e 100644
--- a/packages/components/src/color-edit/index.js
+++ b/packages/components/src/color-edit/index.js
@@ -41,6 +41,7 @@ function ColorOption( {
 	isEditingNameOnMount = false,
 	isEditingColorOnMount = false,
 	onCancel,
+	immutableColorSlugs = [],
 } ) {
 	const [ isHover, setIsHover ] = useState( false );
 	const [ isFocused, setIsFocused ] = useState( false );
@@ -118,7 +119,9 @@ function ColorOption( {
 							onChange={ ( newColorName ) =>
 								onChange( {
 									color,
-									slug: kebabCase( newColorName ),
+									slug: immutableColorSlugs.includes( slug )
+										? slug
+										: kebabCase( newColorName ),
 									name: newColorName,
 								} )
 							}
@@ -222,7 +225,12 @@ function ColorInserter( { onInsert, onCancel } ) {
 	);
 }
 
-export default function ColorEdit( { colors, onChange, emptyUI } ) {
+export default function ColorEdit( {
+	colors,
+	onChange,
+	emptyUI,
+	immutableColorSlugs,
+} ) {
 	const [ isInsertingColor, setIsInsertingColor ] = useState( false );
 	return (
 		<BaseControl>
@@ -254,6 +262,7 @@ export default function ColorEdit( { colors, onChange, emptyUI } ) {
 									color={ color.color }
 									name={ color.name }
 									slug={ color.slug }
+									immutableColorSlugs={ immutableColorSlugs }
 									onChange={ ( newColor ) => {
 										onChange(
 											colors.map(
diff --git a/packages/edit-site/src/components/sidebar/color-palette-panel.js b/packages/edit-site/src/components/sidebar/color-palette-panel.js
index 5de6cab52addae..4a656a2c45958c 100644
--- a/packages/edit-site/src/components/sidebar/color-palette-panel.js
+++ b/packages/edit-site/src/components/sidebar/color-palette-panel.js
@@ -1,44 +1,67 @@
+/**
+ * External dependencies
+ */
+import { get } from 'lodash';
+
 /**
  * WordPress dependencies
  */
-import {
-	Button,
-	__experimentalColorEdit as ColorEdit,
-} from '@wordpress/components';
+import { __experimentalColorEdit as ColorEdit } from '@wordpress/components';
 import { __ } from '@wordpress/i18n';
+import { useSelect } from '@wordpress/data';
+
+/**
+ * Internal dependencies
+ */
+import { useEditorFeature, GLOBAL_CONTEXT } from '../editor/utils';
 
-export default function ColorPalettePanel( {
-	contextName,
-	getSetting,
-	setSetting,
-} ) {
-	const colors = getSetting( contextName, 'color.palette' );
-	let emptyUI;
-	if ( colors === undefined ) {
-		emptyUI = __(
-			'Using theme or core default colors. Add some colors to create your own color palette instead.'
-		);
-	} else if ( colors && colors.length === 0 ) {
-		emptyUI = (
-			<>
-				<p>{ __( 'Using an empty color palette.' ) }</p>
-				<Button
-					isSmall
-					isSecondary
-					onClick={ () => setSetting( contextName, 'color.palette' ) }
-				>
-					{ __( 'Reset to theme/core defaults' ) }
-				</Button>
-			</>
-		);
-	}
+/**
+ * Shared reference to an empty array for cases where it is important to avoid
+ * returning a new array reference on every invocation, as in a connected or
+ * other pure component which performs `shouldComponentUpdate` check on props.
+ * This should be used as a last resort, since the normalized data should be
+ * maintained by the reducer result in state.
+ *
+ * @type {Array}
+ */
+const EMPTY_ARRAY = [];
+
+export default function ColorPalettePanel( { contextName, setSetting } ) {
+	const colors = useEditorFeature( 'color.palette', contextName );
+	const immutableColorSlugs = useSelect(
+		( select ) => {
+			const baseStyles = select( 'core/edit-site' ).getSettings()
+				.__experimentalGlobalStylesBaseStyles;
+			const basePalette =
+				get( baseStyles, [
+					contextName,
+					'settings',
+					'color',
+					'palette',
+				] ) ??
+				get( baseStyles, [
+					GLOBAL_CONTEXT,
+					'settings',
+					'color',
+					'palette',
+				] );
+			if ( ! basePalette ) {
+				return EMPTY_ARRAY;
+			}
+			return basePalette.map( ( { slug } ) => slug );
+		},
+		[ contextName ]
+	);
 	return (
 		<ColorEdit
+			immutableColorSlugs={ immutableColorSlugs }
 			colors={ colors }
 			onChange={ ( newColors ) => {
 				setSetting( contextName, 'color.palette', newColors );
 			} }
-			emptyUI={ emptyUI }
+			emptyUI={ __(
+				'Colors are empty! Add some colors to create your own color palette.'
+			) }
 		/>
 	);
 }