Updates for SamplerV2 support

* Patch ExperimentData.completion_times to handle Job.time_per_step not
  existing
* Pass noise_model run option through to Sampler
* Add support for level 1 data to Sampler execution
* Handle case where the Sampler strips circuit metadata from results
* Mark some test backends as simulators so the sampler does not try to
  validate circuits
* Change some tests to be more consistent about shots since the Sampler
  can not handle the requested number of shots differing from the shots
in the actual results.
* Support level 2 bitstrings in MockIQBackend
* Fix PulseBackend returning data as numpy array instead of a list
* Pass run options through to backend in T2HahnBackend
* Do not set block_for_results timeout to 0 when TEST_TIMEOUT is 0
  (TEST_TIMEOUT=0 indicates no timeout, not immediate timeout).
* Monkey-patch BackendSamplerV2's run circuits function in the tests so
  that it does not strip circuit metadata that the test backends need to
generate results.
* Fix inconsistency between bitstring format and number of qubits in
  restless test.

qiskit_experiments/framework/base_experiment.py

@@ -404,6 +404,8 @@ def _run_jobs(
                             sampler.options.execution.meas_type = "kerneled"
                     else:
                         raise QiskitError("Only meas level 1 + 2 supported by sampler")
+                if "noise_model" in run_options:
+                    sampler.options.simulator.noise_model = run_options["noise_model"]
 
                 if run_options.get("shots") is not None:
                     sampler.options.default_shots = run_options.get("shots")

qiskit_experiments/framework/experiment_data.py

@@ -316,8 +316,22 @@ def completion_times(self) -> Dict[str, datetime]:
         """Returns the completion times of the jobs."""
         job_times = {}
         for job_id, job in self._jobs.items():
-            if job is not None and "COMPLETED" in job.time_per_step():
-                job_times[job_id] = job.time_per_step().get("COMPLETED")
+            if job is not None:
+                if hasattr(job, "time_per_step") and "COMPLETED" in job.time_per_step():
+                    job_times[job_id] = job.time_per_step().get("COMPLETED")
+                elif (execution := job.result().metadata.get("execution")) and "execution_spans" in execution:
+                    job_times[job_id] = execution["execution_spans"].stop
+                elif (client := getattr(job, "_api_client", None)) and hasattr(client, "job_metadata"):
+                    metadata = client.job_metadata(job.job_id())
+                    finished = metadata.get("timestamps", {}).get("finished", {})
+                    if finished:
+                        job_times[job_id] = datetime.fromisoformat(finished)
+                if job_id not in job_times:
+                    warnings.warn(
+                        "Could not determine job completion time. Using current timestamp.",
+                        UserWarning,
+                    )
+                    job_times[job_id] = datetime.now()
 
         return job_times
 
@@ -1026,18 +1040,62 @@ def _add_result_data(self, result: Result, job_id: Optional[str] = None) -> None
                     # convert to a Sampler Pub Result (can remove this later when the bug is fixed)
                     testres = SamplerPubResult(result[i].data, result[i].metadata)
                     data["job_id"] = job_id
-                    if isinstance(testres.data[next(iter(testres.data))], BitArray):
+                    if testres.data:
+                        inner_data = testres.data[next(iter(testres.data))]
+                    if not testres.data:
+                        # No data, usually this only happens in tests
+                        pass
+                    elif isinstance(inner_data, BitArray):
                         # bit results so has counts
                         data["meas_level"] = 2
-                        data["meas_return"] = "avg"
+                        # The sampler result always contains bitstrings. At
+                        # this point, we have lost track of whether the job
+                        # requested memory/meas_return=single. Here we just
+                        # hope that nothing breaks if we always return single
+                        # shot results since the counts dict is also returned
+                        # any way.
+                        data["meas_return"] = "single"
                         # join the data
                         data["counts"] = testres.join_data(testres.data.keys()).get_counts()
+                        data["memory"] = testres.join_data(testres.data.keys()).get_bitstrings()
                         # number of shots
-                        data["shots"] = testres.data[next(iter(testres.data))].num_shots
+                        data["shots"] = inner_data.num_shots
+                    elif isinstance(inner_data, np.ndarray):
+                        data["meas_level"] = 1
+                        joined_data = testres.join_data(testres.data.keys())
+                        # Need to split off the pub dimension representing
+                        # different parameter binds which is trivial because
+                        # qiskit-experiments does not support parameter binding
+                        # to pubs currently.
+                        joined_data = joined_data[0]
+                        if joined_data.ndim == 1:
+                            data["meas_return"] = "avg"
+                            # TODO: we either need to track shots in the
+                            # circuit metadata and pull it out here or get
+                            # upstream to report the number of shots in the
+                            # sampler result for level 1 avg data.
+                            data["shots"] = 1
+                            data["memory"] = np.zeros((len(joined_data), 2), dtype=float)
+                            data["memory"][:, 0] = np.real(joined_data)
+                            data["memory"][:, 1] = np.imag(joined_data)
+                        else:
+                            data["meas_return"] = "single"
+                            data["shots"] = joined_data.shape[0]
+                            data["memory"] = np.zeros((*joined_data.shape, 2), dtype=float)
+                            data["memory"][:, :, 0] = np.real(joined_data)
+                            data["memory"][:, :, 1] = np.imag(joined_data)
                     else:
-                        raise QiskitError("Sampler with meas level 1 support TBD")
+                        raise QiskitError(f"Unexpected result format: {type(inner_data)}")
 
-                    data["metadata"] = testres.metadata["circuit_metadata"]
+                    # Some Sampler implementations remove the circuit metadata
+                    # which some experiment Analysis classes need. Here we try
+                    # to put it back from the circuits themselves.
+                    if "circuit_metadata" in testres.metadata:
+                        data["metadata"] = testres.metadata["circuit_metadata"]
+                    else:
+                        corresponding_pub = job.inputs["pubs"][i]
+                        circuit = corresponding_pub[0]
+                        data["metadata"] = circuit.metadata
 
                     self._result_data.append(data)
 

qiskit_experiments/test/fake_backend.py

@@ -43,6 +43,7 @@ def __init__(
         num_qubits=1,
         max_experiments=100,
     ):
+        self.simulator = True
         super().__init__(provider=provider, name=backend_name)
         self._target = Target(num_qubits=num_qubits)
         # Add a measure for each qubit so a simple measure circuit works
@@ -62,13 +63,13 @@ def _default_options(cls):
     def target(self) -> Target:
         return self._target
 
-    def run(self, run_input, **options):
+    def run(self, run_input, shots=100, **options):
         if not isinstance(run_input, list):
             run_input = [run_input]
         results = [
             {
-                "data": {"0": 100},
-                "shots": 100,
+                "data": {"0": shots},
+                "shots": shots,
                 "success": True,
                 "header": {"metadata": circ.metadata},
                 "meas_level": 2,

qiskit_experiments/test/mock_iq_backend.py

@@ -239,6 +239,7 @@ def __init__(
 
         self._experiment_helper = experiment_helper
         self._rng = np.random.default_rng(rng_seed)
+        self.simulator = True
 
         super().__init__()
 
@@ -456,6 +457,12 @@ def _generate_data(
                 result_in_str = str(format(result, "b").zfill(output_length))
                 counts[result_in_str] = num_occurrences
             run_result["counts"] = counts
+            if meas_return == "single" or self.options.get("memory"):
+                run_result["memory"] = [
+                    format(result, "x")
+                    for result, num in enumerate(results)
+                    for _ in range(num)
+                ]
         else:
             # Phase has meaning only for IQ shot, so we calculate it here
             phase = self.experiment_helper.iq_phase([circuit])[0]

qiskit_experiments/test/pulse_backend.py

@@ -319,7 +319,7 @@ def _state_to_measurement_data(
             centers = self._iq_cluster_centers(circuit=circuit)
             measurement_data = self._iq_data(state.probabilities(), shots, centers, 0.2)
             if meas_return == "avg":
-                measurement_data = np.average(np.array(measurement_data), axis=0)
+                measurement_data = np.average(np.array(measurement_data), axis=0).tolist()
         else:
             raise QiskitError(f"Unsupported measurement level {meas_level}.")
 

qiskit_experiments/test/t2hahn_backend.py

@@ -198,6 +198,6 @@ def run(
 
         sim = AerSimulator(noise_model=noise_model, seed_simulator=self._seed)
 
-        job = sim.run(new_circuits, shots=shots)
+        job = sim.run(new_circuits, shots=shots, **options)
 
         return FakeJob(self, job.result())

test/base.py

@@ -25,6 +25,11 @@
 import uncertainties
 from qiskit.utils.deprecation import deprecate_func
 import qiskit_aer.backends.aerbackend
+# The following imports are just for _patched_run_circuits
+import qiskit.primitives.backend_sampler_v2
+from qiskit.circuit import QuantumCircuit
+from qiskit.providers import BackendV1, BackendV2
+from qiskit.result import Result
 
 from qiskit_experiments.framework import (
     ExperimentDecoder,
@@ -106,6 +111,9 @@ def setUpClass(cls):
             """Set-up test class."""
             super().setUpClass()
 
+            # Hack to pass metadata through to test backends
+            qiskit.primitives.backend_sampler_v2._run_circuits = _patched_run_circuits
+
             warnings.filterwarnings("error", category=DeprecationWarning)
             # Tests should not generate any warnings unless testing those
             # warnings. In that case, the test should catch the warning
@@ -127,6 +135,13 @@ def setUpClass(cls):
                 message=".*The curve data representation has been replaced by the `DataFrame` format.*",
                 category=PendingDeprecationWarning,
             )
+            warnings.filterwarnings(
+                "default",
+                module="qiskit_experiments",
+                message=".*Could not determine job completion time.*",
+                category=UserWarning,
+            )
+
 
             # Some functionality may be deprecated in Qiskit Experiments. If
             # the deprecation warnings aren't filtered, the tests will fail as
@@ -161,7 +176,7 @@ def assertExperimentDone(
                 timeout: The maximum time in seconds to wait for executor to
                     complete. Defaults to the value of ``TEST_TIMEOUT``.
             """
-            if timeout is None:
+            if timeout is None and TEST_TIMEOUT != 0:
                 timeout = TEST_TIMEOUT
             experiment_data.block_for_results(timeout=timeout)
 
@@ -321,3 +336,41 @@ def experiment_data_equiv(cls, data1, data2):
 
 
 QiskitExperimentsTestCase = create_base_test_case(USE_TESTTOOLS)
+
+
+def _patched_run_circuits(
+    circuits: QuantumCircuit | list[QuantumCircuit],
+    backend: BackendV1 | BackendV2,
+    **run_options,
+) -> tuple[list[Result], list[dict]]:
+    """Remove metadata of circuits and run the circuits on a backend.
+    Args:
+        circuits: The circuits
+        backend: The backend
+        monitor: Enable job minotor if True
+        **run_options: run_options
+    Returns:
+        The result and the metadata of the circuits
+    """
+    if isinstance(circuits, QuantumCircuit):
+        circuits = [circuits]
+    metadata = []
+    for circ in circuits:
+        metadata.append(circ.metadata)
+        # Commenting out this line is only change from qiskit.primitives.backend_estimator._run_circuits
+        # circ.metadata = {}
+    if isinstance(backend, BackendV1):
+        max_circuits = getattr(backend.configuration(), "max_experiments", None)
+    elif isinstance(backend, BackendV2):
+        max_circuits = backend.max_circuits
+    else:
+        raise RuntimeError("Backend version not supported")
+    if max_circuits:
+        jobs = [
+            backend.run(circuits[pos : pos + max_circuits], **run_options)
+            for pos in range(0, len(circuits), max_circuits)
+        ]
+        result = [x.result() for x in jobs]
+    else:
+        result = [backend.run(circuits, **run_options).result()]
+    return result, metadata

test/data_processing/test_restless_experiment.py

@@ -82,7 +82,7 @@ def test_end_to_end_restless_standard_processor(self, pi_ratio):
 
         amp_exp = FineXAmplitude([0], backend)
         # standard data processor.
-        standard_processor = DataProcessor("counts", [Probability("01")])
+        standard_processor = DataProcessor("counts", [Probability("1")])
         amp_exp.analysis.set_options(data_processor=standard_processor)
         # enable a restless measurement setting.
         amp_exp.enable_restless(rep_delay=1e-6, override_processor_by_restless=False)

test/framework/test_composite.py

@@ -454,6 +454,7 @@ def test_composite_subexp_data(self):
                 "1111": 5,
             },
             {
+                "0000": 6,
                 "0001": 3,
                 "0010": 4,
                 "0011": 5,
@@ -499,7 +500,7 @@ def test_composite_subexp_data(self):
                 "1111": 3,
             },
             {
-                "0000": 3,
+                "0000": 12,
                 "0001": 6,
                 "0010": 7,
                 "0011": 1,
@@ -524,10 +525,13 @@ def run(self, run_input, **options):
                 for circ, cnt in zip(run_input, counts):
                     results.append(
                         {
-                            "shots": -1,
+                            "shots": sum(cnt.values()),
                             "success": True,
                             "header": {"metadata": circ.metadata},
-                            "data": {"counts": cnt},
+                            "data": {
+                                "counts": cnt,
+                                "memory": [format(int(f"0b{s}", 2), "x") for s, n in cnt.items() for _ in range(n)]
+                            },
                         }
                     )
 
@@ -573,7 +577,7 @@ def circuits(self):
             ],
             flatten_results=False,
         )
-        expdata = par_exp.run(Backend(num_qubits=4))
+        expdata = par_exp.run(Backend(num_qubits=4), shots=sum(counts[0].values()))
         self.assertExperimentDone(expdata)
 
         self.assertEqual(len(expdata.data()), len(counts))
@@ -583,17 +587,17 @@ def circuits(self):
         counts1 = [
             [
                 {"00": 14, "10": 19, "11": 11, "01": 8},
-                {"01": 14, "10": 7, "11": 13, "00": 12},
+                {"01": 14, "10": 7, "11": 13, "00": 18},
                 {"00": 14, "01": 5, "10": 16, "11": 17},
                 {"00": 4, "01": 16, "10": 19, "11": 13},
-                {"00": 12, "01": 15, "10": 11, "11": 5},
+                {"00": 21, "01": 15, "10": 11, "11": 5},
             ],
             [
                 {"00": 10, "01": 10, "10": 12, "11": 20},
-                {"00": 12, "01": 10, "10": 7, "11": 17},
+                {"00": 18, "01": 10, "10": 7, "11": 17},
                 {"00": 17, "01": 7, "10": 14, "11": 14},
                 {"00": 9, "01": 14, "10": 22, "11": 7},
-                {"00": 17, "01": 10, "10": 9, "11": 7},
+                {"00": 26, "01": 10, "10": 9, "11": 7},
             ],
         ]
 
@@ -604,11 +608,11 @@ def circuits(self):
                 self.assertDictEqual(circ_data["counts"], circ_counts)
 
         counts2 = [
-            [{"00": 10, "01": 10, "10": 12, "11": 20}, {"00": 12, "01": 10, "10": 7, "11": 17}],
+            [{"00": 10, "01": 10, "10": 12, "11": 20}, {"00": 18, "01": 10, "10": 7, "11": 17}],
             [
                 {"00": 17, "01": 7, "10": 14, "11": 14},
                 {"00": 9, "01": 14, "10": 22, "11": 7},
-                {"00": 17, "01": 10, "10": 9, "11": 7},
+                {"00": 26, "01": 10, "10": 9, "11": 7},
             ],
         ]
 
@@ -618,8 +622,8 @@ def circuits(self):
                 self.assertDictEqual(circ_data["counts"], circ_counts)
 
         counts3 = [
-            [{"0": 22, "1": 30}, {"0": 19, "1": 27}],
-            [{"0": 20, "1": 32}, {"0": 22, "1": 24}],
+            [{"0": 22, "1": 30}, {"0": 25, "1": 27}],
+            [{"0": 20, "1": 32}, {"0": 28, "1": 24}],
         ]
 
         self.assertEqual(len(expdata.child_data(1).child_data(0).child_data()), len(counts3))
@@ -947,7 +951,7 @@ def test_batch_transpile_options_integrated(self):
         noise_model = noise.NoiseModel()
         noise_model.add_all_qubit_quantum_error(noise.depolarizing_error(0.5, 2), ["cx", "swap"])
 
-        expdata = self.batch2.run(backend, noise_model=noise_model, shots=1000)
+        expdata = self.batch2.run(backend, noise_model=noise_model, shots=1000, memory=True)
         self.assertExperimentDone(expdata)
 
         self.assertEqual(expdata.child_data(0).analysis_results("non-zero counts").value, 8)

test/library/characterization/test_cross_resonance_hamiltonian.py

@@ -151,8 +151,9 @@ def test_integration(self, ix, iy, iz, zx, zy, zz):
 
         dt = 0.222e-9
         sigma = 64
+        shots=2000
 
-        backend = AerSimulator(seed_simulator=123, shots=2000)
+        backend = AerSimulator(seed_simulator=123, shots=shots)
         backend._configuration.dt = dt
 
         # Note that Qiskit is Little endian, i.e. [q1, q0]
@@ -179,7 +180,7 @@ def test_integration(self, ix, iy, iz, zx, zy, zz):
         )
         expr.backend = backend
 
-        exp_data = expr.run()
+        exp_data = expr.run(shots=shots)
         self.assertExperimentDone(exp_data, timeout=1000)
 
         self.assertEqual(exp_data.analysis_results("omega_ix").quality, "good")

test/library/tomography/test_process_tomography.py

@@ -55,7 +55,7 @@ def test_full_qpt_random_unitary(self, num_qubits):
         backend = AerSimulator(seed_simulator=seed, shots=shots)
         target = qi.random_unitary(2**num_qubits, seed=seed)
         exp = ProcessTomography(target)
-        expdata = exp.run(backend, analysis=None)
+        expdata = exp.run(backend, analysis=None, shots=shots)
         self.assertExperimentDone(expdata)
 
         # Run each tomography fitter analysis as a subtest so

test/library/tomography/test_state_tomography.py

@@ -55,7 +55,7 @@ def test_full_qst(self, num_qubits):
         backend = AerSimulator(seed_simulator=seed, shots=shots)
         target = qi.random_statevector(2**num_qubits, seed=seed)
         exp = StateTomography(target)
-        expdata = exp.run(backend, analysis=None)
+        expdata = exp.run(backend, analysis=None, shots=shots)
         self.assertExperimentDone(expdata)
 
         # Run each tomography fitter analysis as a subtest so