diff --git a/.chronus/changes/fix-expression-cast-to-object-2024-6-11-19-32-52.md b/.chronus/changes/fix-expression-cast-to-object-2024-6-11-19-32-52.md
new file mode 100644
index 0000000000..968edd1ca6
--- /dev/null
+++ b/.chronus/changes/fix-expression-cast-to-object-2024-6-11-19-32-52.md
@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/compiler"
+---
+
+Do not cast model expression to object value if the constraint is allowing the type
diff --git a/packages/compiler/src/core/checker.ts b/packages/compiler/src/core/checker.ts
index fe52cdf802..4bd74c586d 100644
--- a/packages/compiler/src/core/checker.ts
+++ b/packages/compiler/src/core/checker.ts
@@ -971,6 +971,18 @@ export function createChecker(program: Program): Checker {
     kind: "argument" | "assignment";
     type: Type;
   }
+
+  function canTryLegacyCast(
+    target: Type,
+    constraint: MixedParameterConstraint | undefined
+  ): constraint is MixedParameterConstraint &
+    Required<Pick<MixedParameterConstraint, "valueType">> {
+    return Boolean(
+      constraint?.valueType &&
+        !(constraint.type && ignoreDiagnostics(isTypeAssignableTo(target, constraint.type, target)))
+    );
+  }
+
   /**
    * Gets a type or value depending on the node and current constraint.
    * For nodes that can be both type or values(e.g. string), the value will be returned if the constraint expect a value of that type even if the constrain also allows the type.
@@ -986,7 +998,7 @@ export function createChecker(program: Program): Checker {
     if (entity === null) {
       return entity;
     } else if (isType(entity)) {
-      if (valueConstraint) {
+      if (canTryLegacyCast(entity, constraint?.constraint)) {
         return legacy_tryTypeToValueCast(entity, valueConstraint, node);
       } else {
         return entity;
@@ -1548,7 +1560,7 @@ export function createChecker(program: Program): Checker {
             ? finalMap.get(param.constraint.type)!
             : param.constraint;
 
-        if (isType(type) && param.constraint?.valueType) {
+        if (isType(type) && canTryLegacyCast(type, param.constraint)) {
           const converted = legacy_tryTypeToValueCast(
             type,
             { kind: "argument", type: param.constraint.valueType },
diff --git a/packages/compiler/test/checker/values/array-values.test.ts b/packages/compiler/test/checker/values/array-values.test.ts
index ebdb75147e..d5e1a59235 100644
--- a/packages/compiler/test/checker/values/array-values.test.ts
+++ b/packages/compiler/test/checker/values/array-values.test.ts
@@ -1,8 +1,14 @@
 import { ok, strictEqual } from "assert";
 import { describe, expect, it } from "vitest";
 import { isValue } from "../../../src/index.js";
-import { expectDiagnostics } from "../../../src/testing/index.js";
-import { compileValue, compileValueOrType, diagnoseUsage, diagnoseValue } from "./utils.js";
+import { expectDiagnosticEmpty, expectDiagnostics } from "../../../src/testing/index.js";
+import {
+  compileAndDiagnoseValueOrType,
+  compileValue,
+  compileValueOrType,
+  diagnoseUsage,
+  diagnoseValue,
+} from "./utils.js";
 
 it("no values", async () => {
   const object = await compileValue(`#[]`);
@@ -111,6 +117,17 @@ describe("(LEGACY) cast tuple to array value", () => {
     });
   });
 
+  it("doesn't cast or emit diagnostic if constraint also allow tuples", async () => {
+    const [entity, diagnostics] = await compileAndDiagnoseValueOrType(
+      `(valueof unknown) | unknown`,
+      `["foo"]`,
+      { disableDeprecatedSuppression: true }
+    );
+    expectDiagnosticEmpty(diagnostics);
+    strictEqual(entity?.entityKind, "Type");
+    strictEqual(entity.kind, "Tuple");
+  });
+
   it("emit a error if element in tuple expression are not castable to value", async () => {
     const { diagnostics, pos } = await diagnoseUsage(`
       model Test<T extends valueof string[]> {}
diff --git a/packages/compiler/test/checker/values/object-values.test.ts b/packages/compiler/test/checker/values/object-values.test.ts
index 2cfa75562d..12a52e03bd 100644
--- a/packages/compiler/test/checker/values/object-values.test.ts
+++ b/packages/compiler/test/checker/values/object-values.test.ts
@@ -1,8 +1,14 @@
 import { ok, strictEqual } from "assert";
 import { describe, expect, it } from "vitest";
 import { isValue } from "../../../src/index.js";
-import { expectDiagnostics } from "../../../src/testing/index.js";
-import { compileValue, compileValueOrType, diagnoseUsage, diagnoseValue } from "./utils.js";
+import { expectDiagnosticEmpty, expectDiagnostics } from "../../../src/testing/index.js";
+import {
+  compileAndDiagnoseValueOrType,
+  compileValue,
+  compileValueOrType,
+  diagnoseUsage,
+  diagnoseValue,
+} from "./utils.js";
 
 it("no properties", async () => {
   const object = await compileValue(`#{}`);
@@ -163,6 +169,17 @@ describe("(LEGACY) cast model to object value", () => {
     strictEqual(b.value, "bar");
   });
 
+  it("doesn't cast or emit diagnostic if constraint also allow models", async () => {
+    const [entity, diagnostics] = await compileAndDiagnoseValueOrType(
+      `{a: string} | valueof {a: string}`,
+      `{a: "b"}`,
+      { disableDeprecatedSuppression: true }
+    );
+    expectDiagnosticEmpty(diagnostics);
+    strictEqual(entity?.entityKind, "Type");
+    strictEqual(entity.kind, "Model");
+  });
+
   it("emit a warning diagnostic", async () => {
     const { diagnostics, pos } = await diagnoseUsage(`
       model Test<T extends valueof {a: string}> {}
diff --git a/packages/compiler/test/checker/values/utils.ts b/packages/compiler/test/checker/values/utils.ts
index c1c895ca66..cf0fc7991b 100644
--- a/packages/compiler/test/checker/values/utils.ts
+++ b/packages/compiler/test/checker/values/utils.ts
@@ -65,7 +65,10 @@ export async function diagnoseValue(code: string, other?: string): Promise<reado
 export async function compileAndDiagnoseValueOrType(
   constraint: string,
   code: string,
-  other?: string
+  {
+    other,
+    disableDeprecatedSuppression,
+  }: { other?: string; disableDeprecatedSuppression?: boolean }
 ): Promise<[Type | Value | undefined, readonly Diagnostic[]]> {
   const host = await createTestHost();
   host.addJsFile("collect.js", {
@@ -78,7 +81,7 @@ export async function compileAndDiagnoseValueOrType(
       import "./collect.js";
       extern dec collect(target, value: ${constraint});
 
-      #suppress "deprecated" "for testing"
+      ${disableDeprecatedSuppression ? "" : `#suppress "deprecated" "for testing"`}
       @collect(${code})
       @test model Test {}
       ${other ?? ""}
@@ -99,7 +102,7 @@ export async function compileValueOrType(
   code: string,
   other?: string
 ): Promise<Value | Type> {
-  const [called, diagnostics] = await compileAndDiagnoseValueOrType(constraint, code, other);
+  const [called, diagnostics] = await compileAndDiagnoseValueOrType(constraint, code, { other });
   expectDiagnosticEmpty(diagnostics);
   ok(called, "Decorator was not called");
 
@@ -111,6 +114,6 @@ export async function diagnoseValueOrType(
   code: string,
   other?: string
 ): Promise<readonly Diagnostic[]> {
-  const [_, diagnostics] = await compileAndDiagnoseValueOrType(constraint, code, other);
+  const [_, diagnostics] = await compileAndDiagnoseValueOrType(constraint, code, { other });
   return diagnostics;
 }