diff --git a/qiskit/circuit/instructionset.py b/qiskit/circuit/instructionset.py
index 2b1a3b756de..d77a71c9d80 100644
--- a/qiskit/circuit/instructionset.py
+++ b/qiskit/circuit/instructionset.py
@@ -15,6 +15,8 @@
 """
 
 from __future__ import annotations
+
+from collections.abc import Sequence, MutableSequence
 from typing import Callable
 
 from qiskit.circuit.exceptions import CircuitError
@@ -52,7 +54,7 @@ def __init__(  # pylint: disable=bad-docstring-quotes
                     used.  It may throw an error if the resource is not valid for usage.
 
         """
-        self._instructions: list[CircuitInstruction] = []
+        self._instructions: list[CircuitInstruction | (Sequence[CircuitInstruction], int)] = []
         self._requester = resource_requester
 
     def __len__(self):
@@ -61,7 +63,11 @@ def __len__(self):
 
     def __getitem__(self, i):
         """Return instruction at index"""
-        return self._instructions[i]
+        inst = self._instructions[i]
+        if isinstance(inst, CircuitInstruction):
+            return inst
+        data, idx = inst
+        return data[idx]
 
     def add(self, instruction, qargs=None, cargs=None):
         """Add an instruction and its context (where it is attached)."""
@@ -73,10 +79,22 @@ def add(self, instruction, qargs=None, cargs=None):
             instruction = CircuitInstruction(instruction, tuple(qargs), tuple(cargs))
         self._instructions.append(instruction)
 
+    def _add_ref(self, data: MutableSequence[CircuitInstruction], pos: int):
+        """Add a reference to an instruction and its context within a mutable sequence.
+        Updates to the instruction set will modify the specified sequence in place."""
+        self._instructions.append((data, pos))
+
     def inverse(self):
         """Invert all instructions."""
         for i, instruction in enumerate(self._instructions):
-            self._instructions[i] = instruction.replace(operation=instruction.operation.inverse())
+            if isinstance(instruction, CircuitInstruction):
+                self._instructions[i] = instruction.replace(
+                    operation=instruction.operation.inverse()
+                )
+            else:
+                data, idx = instruction
+                instruction = data[idx]
+                data[idx] = instruction.replace(operation=instruction.operation.inverse())
         return self
 
     def c_if(self, classical: Clbit | ClassicalRegister | int, val: int) -> "InstructionSet":
@@ -132,26 +150,40 @@ def c_if(self, classical: Clbit | ClassicalRegister | int, val: int) -> "Instruc
         if self._requester is not None:
             classical = self._requester(classical)
         for instruction in self._instructions:
-            instruction.operation = instruction.operation.c_if(classical, val)
+            if isinstance(instruction, CircuitInstruction):
+                updated = instruction.operation.c_if(classical, val)
+                if updated is not instruction.operation:
+                    raise CircuitError(
+                        "SingletonGate instances can only be added to InstructionSet via _add_ref"
+                    )
+            else:
+                data, idx = instruction
+                instruction = data[idx]
+                data[idx] = instruction.replace(
+                    operation=instruction.operation.c_if(classical, val)
+                )
         return self
 
     # Legacy support for properties.  Added in Terra 0.21 to support the internal switch in
     # `QuantumCircuit.data` from the 3-tuple to `CircuitInstruction`.
 
+    def _instructions_iter(self):
+        return (i if isinstance(i, CircuitInstruction) else i[0][i[1]] for i in self._instructions)
+
     @property
     def instructions(self):
         """Legacy getter for the instruction components of an instruction set.  This does not
         support mutation."""
-        return [instruction.operation for instruction in self._instructions]
+        return [instruction.operation for instruction in self._instructions_iter()]
 
     @property
     def qargs(self):
         """Legacy getter for the qargs components of an instruction set.  This does not support
         mutation."""
-        return [list(instruction.qubits) for instruction in self._instructions]
+        return [list(instruction.qubits) for instruction in self._instructions_iter()]
 
     @property
     def cargs(self):
         """Legacy getter for the cargs components of an instruction set.  This does not support
         mutation."""
-        return [list(instruction.clbits) for instruction in self._instructions]
+        return [list(instruction.clbits) for instruction in self._instructions_iter()]
diff --git a/qiskit/circuit/quantumcircuit.py b/qiskit/circuit/quantumcircuit.py
index 1cca0f044d9..6b32c5d9fcb 100644
--- a/qiskit/circuit/quantumcircuit.py
+++ b/qiskit/circuit/quantumcircuit.py
@@ -1308,27 +1308,29 @@ def append(
         expanded_cargs = [self.cbit_argument_conversion(carg) for carg in cargs or []]
 
         if self._control_flow_scopes:
+            circuit_data = self._control_flow_scopes[-1].instructions
             appender = self._control_flow_scopes[-1].append
             requester = self._control_flow_scopes[-1].request_classical_resource
         else:
+            circuit_data = self._data
             appender = self._append
             requester = self._resolve_classical_resource
         instructions = InstructionSet(resource_requester=requester)
         if isinstance(operation, Instruction):
             for qarg, carg in operation.broadcast_arguments(expanded_qargs, expanded_cargs):
                 self._check_dups(qarg)
-                instruction = CircuitInstruction(operation, qarg, carg)
-                appender(instruction)
-                instructions.add(instruction)
+                data_idx = len(circuit_data)
+                appender(CircuitInstruction(operation, qarg, carg))
+                instructions._add_ref(circuit_data, data_idx)
         else:
             # For Operations that are non-Instructions, we use the Instruction's default method
             for qarg, carg in Instruction.broadcast_arguments(
                 operation, expanded_qargs, expanded_cargs
             ):
                 self._check_dups(qarg)
-                instruction = CircuitInstruction(operation, qarg, carg)
-                appender(instruction)
-                instructions.add(instruction)
+                data_idx = len(circuit_data)
+                appender(CircuitInstruction(operation, qarg, carg))
+                instructions._add_ref(circuit_data, data_idx)
         return instructions
 
     # Preferred new style.
diff --git a/test/python/circuit/test_circuit_operations.py b/test/python/circuit/test_circuit_operations.py
index 43c60ae4c76..c5f174af67e 100644
--- a/test/python/circuit/test_circuit_operations.py
+++ b/test/python/circuit/test_circuit_operations.py
@@ -1263,11 +1263,11 @@ def test_pop_previous_instruction_removes_parameters(self):
         x, y = Parameter("x"), Parameter("y")
         test = QuantumCircuit(1, 1)
         test.rx(y, 0)
-        last_instructions = test.u(x, y, 0, 0)
+        last_instructions = list(test.u(x, y, 0, 0))
         self.assertEqual({x, y}, set(test.parameters))
 
         instruction = test._pop_previous_instruction_in_scope()
-        self.assertEqual(list(last_instructions), [instruction])
+        self.assertEqual(last_instructions, [instruction])
         self.assertEqual({y}, set(test.parameters))
 
     def test_decompose_gate_type(self):