diff --git a/packages/reactivity/__tests__/reactive.spec.ts b/packages/reactivity/__tests__/reactive.spec.ts
index bd4ec402bb2..1c25c49121b 100644
--- a/packages/reactivity/__tests__/reactive.spec.ts
+++ b/packages/reactivity/__tests__/reactive.spec.ts
@@ -2,6 +2,8 @@ import { isRef, ref } from '../src/ref'
 import {
   isProxy,
   isReactive,
+  isReadonly,
+  isShallow,
   markRaw,
   reactive,
   readonly,
@@ -359,4 +361,25 @@ describe('reactivity/reactive', () => {
     const c = computed(() => {})
     expect(isProxy(c)).toBe(false)
   })
+
+  test('The results of the shallow and readonly assignments are the same (Map)', () => {
+    const map = reactive(new Map())
+    map.set('foo', shallowReactive({ a: 2 }))
+    expect(isShallow(map.get('foo'))).toBe(true)
+
+    map.set('bar', readonly({ b: 2 }))
+    expect(isReadonly(map.get('bar'))).toBe(true)
+  })
+
+  test('The results of the shallow and readonly assignments are the same (Set)', () => {
+    const set = reactive(new Set())
+    set.add(shallowReactive({ a: 2 }))
+    set.add(readonly({ b: 2 }))
+    let count = 0
+    for (const i of set) {
+      if (count === 0) expect(isShallow(i)).toBe(true)
+      else expect(isReadonly(i)).toBe(true)
+      count++
+    }
+  })
 })
diff --git a/packages/reactivity/__tests__/shallowReactive.spec.ts b/packages/reactivity/__tests__/shallowReactive.spec.ts
index e9b64d39b36..cb5ef8e6d97 100644
--- a/packages/reactivity/__tests__/shallowReactive.spec.ts
+++ b/packages/reactivity/__tests__/shallowReactive.spec.ts
@@ -123,6 +123,29 @@ describe('shallowReactive', () => {
       shallowSet.forEach(x => expect(isReactive(x)).toBe(false))
     })
 
+    test('Setting a reactive object on a shallowReactive map', () => {
+      const msg = ref('ads')
+      const bar = reactive({ msg })
+      const foo = shallowReactive(new Map([['foo1', bar]]))
+      foo.set('foo2', bar)
+
+      expect(isReactive(foo.get('foo2'))).toBe(true)
+      expect(isReactive(foo.get('foo1'))).toBe(true)
+    })
+
+    test('Setting a reactive object on a shallowReactive set', () => {
+      const msg = ref(1)
+      const bar = reactive({ msg })
+      const foo = reactive({ msg })
+
+      const deps = shallowReactive(new Set([bar]))
+      deps.add(foo)
+
+      deps.forEach(dep => {
+        expect(isReactive(dep)).toBe(true)
+      })
+    })
+
     // #1210
     test('onTrack on called on objectSpread', () => {
       const onTrackFn = vi.fn()
diff --git a/packages/reactivity/src/collectionHandlers.ts b/packages/reactivity/src/collectionHandlers.ts
index 7c4b36fbe94..c39e3ed48cd 100644
--- a/packages/reactivity/src/collectionHandlers.ts
+++ b/packages/reactivity/src/collectionHandlers.ts
@@ -1,4 +1,10 @@
-import { toRaw, toReactive, toReadonly } from './reactive'
+import {
+  isReadonly,
+  isShallow,
+  toRaw,
+  toReactive,
+  toReadonly,
+} from './reactive'
 import {
   ITERATE_KEY,
   MAP_KEY_ITERATE_KEY,
@@ -72,8 +78,10 @@ function size(target: IterableCollections, isReadonly = false) {
   return Reflect.get(target, 'size', target)
 }
 
-function add(this: SetTypes, value: unknown) {
-  value = toRaw(value)
+function add(this: SetTypes, value: unknown, _isShallow = false) {
+  if (!_isShallow && !isShallow(value) && !isReadonly(value)) {
+    value = toRaw(value)
+  }
   const target = toRaw(this)
   const proto = getProto(target)
   const hadKey = proto.has.call(target, value)
@@ -84,8 +92,10 @@ function add(this: SetTypes, value: unknown) {
   return this
 }
 
-function set(this: MapTypes, key: unknown, value: unknown) {
-  value = toRaw(value)
+function set(this: MapTypes, key: unknown, value: unknown, _isShallow = false) {
+  if (!_isShallow && !isShallow(value) && !isReadonly(value)) {
+    value = toRaw(value)
+  }
   const target = toRaw(this)
   const { has, get } = getProto(target)
 
@@ -263,8 +273,12 @@ function createInstrumentations() {
       return size(this as unknown as IterableCollections)
     },
     has,
-    add,
-    set,
+    add(this: SetTypes, value: unknown) {
+      return add.call(this, value, true)
+    },
+    set(this: MapTypes, key: unknown, value: unknown) {
+      return set.call(this, key, value, true)
+    },
     delete: deleteEntry,
     clear,
     forEach: createForEach(false, true),