From f16bfdd577725c9b7c7426cacb94b3431a744955 Mon Sep 17 00:00:00 2001 From: Jessie Yu Date: Wed, 22 Apr 2020 13:50:05 -0400 Subject: [PATCH] [Stable] backport fixes for large qobj and complex in qobj (#635) * Fix tests for terra 0.13 (#623) * stop using Qobj class * fix type hint * print all diff * use to_dict * copy return data * update requirement * remove deleted file * don't use terra master * use newer osx * use pyenv * make test more tolerant (cherry picked from commit da6c78e5b513817aa373d7bfa005d16467464140) # Conflicts: # CHANGELOG.md * increase put timeout (#626) (cherry picked from commit 3af24f77c140c4ac1f0b91bbd1e1012f9429cb7d) * Convert list to complex for Qobj (#631) * convert list to complex for qobj * increase timeout for qobj download * change error type (cherry picked from commit 2d32a1ddbd043c22e7693fbcdd8dfb18a3d43139) # Conflicts: # CHANGELOG.md --- .travis.yml | 18 ++--- CHANGELOG.md | 5 ++ qiskit/providers/ibmq/api/rest/job.py | 4 +- .../providers/ibmq/api/rest/schemas/root.py | 3 +- qiskit/providers/ibmq/api/session.py | 2 +- qiskit/providers/ibmq/ibmqbackend.py | 10 +-- qiskit/providers/ibmq/job/ibmqjob.py | 11 +-- qiskit/providers/ibmq/jupyter/__init__.py | 9 --- qiskit/providers/ibmq/managed/managedjob.py | 10 +-- .../providers/ibmq/managed/managedjobset.py | 4 +- qiskit/providers/ibmq/utils/qobj_utils.py | 61 ++++++++++++++-- requirements.txt | 2 +- setup.py | 2 +- test/fake_account_client.py | 2 +- test/ibmq/test_ibmq_job.py | 2 +- test/ibmq/test_ibmq_job_model.py | 3 +- test/ibmq/test_ibmq_jobmanager.py | 3 +- test/ibmq/test_serialization.py | 72 +++++++++++++++++++ test/utils.py | 4 +- 19 files changed, 174 insertions(+), 53 deletions(-) create mode 100644 test/ibmq/test_serialization.py diff --git a/.travis.yml b/.travis.yml index 346018954..62ac918c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,6 @@ stage_linux: &stage_linux stage_osx: &stage_osx <<: *stage_generic os: osx - osx_image: xcode9.2 language: generic cache: pip: true @@ -60,15 +59,16 @@ stage_osx: &stage_osx # installed manually. | if [ ${TRAVIS_OS_NAME} = "osx" ]; then - if [[ ! -d ~/python-interpreters/$PYTHON_VERSION ]]; then - git clone git://github.com/pyenv/pyenv.git - cd pyenv/plugins/python-build - ./install.sh - cd ../../.. - python-build $PYTHON_VERSION ~/python-interpreters/$PYTHON_VERSION + if [[ ! -d ~/.pyenv ]]; then + git clone git://github.com/pyenv/pyenv.git ~/.pyenv fi - virtualenv --python ~/python-interpreters/$PYTHON_VERSION/bin/python venv - source venv/bin/activate + export PYENV_ROOT="$HOME/.pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init -)" + pyenv install -s $PYTHON_VERSION + pyenv local $PYTHON_VERSION + pip install --upgrade pip + pip install wheel setuptools fi diff --git a/CHANGELOG.md b/CHANGELOG.md index a39a37a71..0656c37cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ The format is based on [Keep a Changelog]. ## [UNRELEASED] +### Fixed + +- Increased timeout value to allow large Qobj to be uploaded. (\#626) +- Added a JSON decoder to convert lists in Qobj to complex. (\#631) + ## [0.6.0] - 2020-03-26 ### Added diff --git a/qiskit/providers/ibmq/api/rest/job.py b/qiskit/providers/ibmq/api/rest/job.py index 2df2b9dc8..f86aac1e7 100644 --- a/qiskit/providers/ibmq/api/rest/job.py +++ b/qiskit/providers/ibmq/api/rest/job.py @@ -173,7 +173,7 @@ def put_object_storage(self, url: str, qobj_dict: Dict[str, Any]) -> str: """ data = json.dumps(qobj_dict, cls=json_encoder.IQXJsonEconder) logger.debug('Uploading Qobj to object storage.') - response = self.session.put(url, data=data, bare=True) + response = self.session.put(url, data=data, bare=True, timeout=600) return response.text def get_object_storage(self, url: str) -> Dict[str, Any]: @@ -186,5 +186,5 @@ def get_object_storage(self, url: str) -> Dict[str, Any]: JSON response. """ logger.debug('Downloading Qobj from object storage.') - response = self.session.get(url, bare=True).json() + response = self.session.get(url, bare=True, timeout=600).json() return response diff --git a/qiskit/providers/ibmq/api/rest/schemas/root.py b/qiskit/providers/ibmq/api/rest/schemas/root.py index e2a1371b4..835f238ef 100644 --- a/qiskit/providers/ibmq/api/rest/schemas/root.py +++ b/qiskit/providers/ibmq/api/rest/schemas/root.py @@ -16,7 +16,6 @@ from marshmallow.validate import OneOf from qiskit.providers import JobStatus -from qiskit.providers.models.backendconfiguration import BackendConfigurationSchema from qiskit.providers.ibmq.apiconstants import ApiJobKind from qiskit.validation import BaseSchema from qiskit.validation.fields import String, Dict, Nested, Boolean, List, Number @@ -120,7 +119,7 @@ class BackendsResponseSchema(BaseSchema): """Schema for BackendResponse""" # Required properties - backends = List(Nested(BackendConfigurationSchema, required=True)) + backends = Dict(required=True, many=True) class JobsRequestSchema(BaseSchema): diff --git a/qiskit/providers/ibmq/api/session.py b/qiskit/providers/ibmq/api/session.py index d61dfc391..793e136a6 100644 --- a/qiskit/providers/ibmq/api/session.py +++ b/qiskit/providers/ibmq/api/session.py @@ -221,7 +221,7 @@ def request( # type: ignore[override] final_url = self.base_url + url # Add a timeout to the connection for non-proxy connections. - if not self.proxies: + if not self.proxies and 'timeout' not in kwargs: kwargs.update({'timeout': self._timeout}) try: diff --git a/qiskit/providers/ibmq/ibmqbackend.py b/qiskit/providers/ibmq/ibmqbackend.py index db80ee0e6..30b81ad33 100644 --- a/qiskit/providers/ibmq/ibmqbackend.py +++ b/qiskit/providers/ibmq/ibmqbackend.py @@ -21,7 +21,7 @@ from datetime import datetime as python_datetime from marshmallow import ValidationError -from qiskit.qobj import Qobj, validate_qobj_against_schema +from qiskit.qobj import QasmQobj, PulseQobj, validate_qobj_against_schema from qiskit.providers.basebackend import BaseBackend # type: ignore[attr-defined] from qiskit.providers.jobstatus import JobStatus from qiskit.providers.models import (BackendStatus, BackendProperties, @@ -114,7 +114,7 @@ def __init__( def run( self, - qobj: Qobj, + qobj: Union[QasmQobj, PulseQobj], job_name: Optional[str] = None, job_share_level: Optional[str] = None, job_tags: Optional[List[str]] = None, @@ -174,7 +174,7 @@ def run( def _submit_job( self, - qobj: Qobj, + qobj: Union[QasmQobj, PulseQobj], job_name: Optional[str] = None, job_share_level: Optional[ApiJobShareLevel] = None, job_tags: Optional[List[str]] = None @@ -517,7 +517,7 @@ def properties( def run( self, - qobj: Qobj, + qobj: Union[QasmQobj, PulseQobj], job_name: Optional[str] = None, job_share_level: Optional[str] = None, job_tags: Optional[List[str]] = None, @@ -606,7 +606,7 @@ def active_jobs(self, limit: int = 10) -> None: def run( self, - qobj: Qobj, + qobj: Union[QasmQobj, PulseQobj], job_name: Optional[str] = None, job_share_level: Optional[str] = None, job_tags: Optional[List[str]] = None, diff --git a/qiskit/providers/ibmq/job/ibmqjob.py b/qiskit/providers/ibmq/job/ibmqjob.py index 3e2be8906..6dbc3147d 100644 --- a/qiskit/providers/ibmq/job/ibmqjob.py +++ b/qiskit/providers/ibmq/job/ibmqjob.py @@ -15,7 +15,7 @@ """IBM Quantum Experience job.""" import logging -from typing import Dict, Optional, Tuple, Any, List, Callable +from typing import Dict, Optional, Tuple, Any, List, Callable, Union import warnings from datetime import datetime from concurrent import futures @@ -28,7 +28,7 @@ BaseBackend) from qiskit.providers.jobstatus import JOB_FINAL_STATES, JobStatus from qiskit.providers.models import BackendProperties -from qiskit.qobj import Qobj +from qiskit.qobj import QasmQobj, PulseQobj from qiskit.result import Result from qiskit.validation import BaseModel, ModelValidationError, bind_schema @@ -36,6 +36,7 @@ from ..api.clients import AccountClient from ..api.exceptions import ApiError, UserTimeoutExceededError from ..utils.utils import RefreshQueue +from ..utils.qobj_utils import dict_to_qobj from .exceptions import (IBMQJobApiError, IBMQJobFailureError, IBMQJobTimeoutError, IBMQJobInvalidStateError) from .queueinfo import QueueInfo @@ -123,7 +124,7 @@ def __init__(self, # Convert qobj from dictionary to Qobj. if isinstance(kwargs.get('_qobj', None), dict): - self._qobj = Qobj.from_dict(kwargs.pop('_qobj')) + self._qobj = dict_to_qobj(kwargs.pop('_qobj')) BaseModel.__init__(self, _backend=_backend, _job_id=_job_id, _creation_date=_creation_date, @@ -141,7 +142,7 @@ def __init__(self, self._cancelled = False self._job_error_msg = None # type: Optional[str] - def qobj(self) -> Optional[Qobj]: + def qobj(self) -> Optional[Union[QasmQobj, PulseQobj]]: """Return the Qobj for this job. Returns: @@ -159,7 +160,7 @@ def qobj(self) -> Optional[Qobj]: with api_to_job_error(): qobj = self._api.job_download_qobj( self.job_id(), self._use_object_storage) - self._qobj = Qobj.from_dict(qobj) + self._qobj = dict_to_qobj(qobj) return self._qobj diff --git a/qiskit/providers/ibmq/jupyter/__init__.py b/qiskit/providers/ibmq/jupyter/__init__.py index 4afa701a9..d80c1f568 100644 --- a/qiskit/providers/ibmq/jupyter/__init__.py +++ b/qiskit/providers/ibmq/jupyter/__init__.py @@ -38,15 +38,6 @@ from qiskit.test.ibmq_mock import mock_get_backend mock_get_backend('FakeVigo') - # TODO Remove after terra 0.13 is released. - from unittest.mock import MagicMock - from qiskit import IBMQ - backend = IBMQ.get_provider().get_backend() - backend._credentials = MagicMock(token='123456', url='https://', - hub='hub', group='group', project='project') - backend.configuration().max_experiments = 75 - backend.jobs = MagicMock(return_value=[]) - .. jupyter-execute:: from qiskit import IBMQ diff --git a/qiskit/providers/ibmq/managed/managedjob.py b/qiskit/providers/ibmq/managed/managedjob.py index 3524f7710..f8a9a3a2c 100644 --- a/qiskit/providers/ibmq/managed/managedjob.py +++ b/qiskit/providers/ibmq/managed/managedjob.py @@ -16,12 +16,12 @@ import warnings import logging -from typing import List, Optional +from typing import List, Optional, Union from concurrent.futures import ThreadPoolExecutor from threading import Lock from qiskit.providers.ibmq import IBMQBackend -from qiskit.qobj import Qobj +from qiskit.qobj import QasmQobj, PulseQobj from qiskit.result import Result from qiskit.providers.jobstatus import JobStatus from qiskit.providers.exceptions import JobError @@ -60,7 +60,7 @@ def __init__( def submit( self, - qobj: Qobj, + qobj: Union[QasmQobj, PulseQobj], job_name: str, backend: IBMQBackend, executor: ThreadPoolExecutor, @@ -87,7 +87,7 @@ def submit( def _async_submit( self, - qobj: Qobj, + qobj: Union[QasmQobj, PulseQobj], job_name: str, backend: IBMQBackend, submit_lock: Lock, @@ -216,7 +216,7 @@ def cancel(self) -> None: logger.warning("Unable to cancel job %s for experiments %d-%d: %s", self.job.job_id(), self.start_index, self.end_index, cancel_error) - def qobj(self) -> Optional[Qobj]: + def qobj(self) -> Optional[Union[QasmQobj, PulseQobj]]: """Return the Qobj for this job. Returns: diff --git a/qiskit/providers/ibmq/managed/managedjobset.py b/qiskit/providers/ibmq/managed/managedjobset.py index 8b1970203..4c75547ff 100644 --- a/qiskit/providers/ibmq/managed/managedjobset.py +++ b/qiskit/providers/ibmq/managed/managedjobset.py @@ -26,7 +26,7 @@ from qiskit.circuit import QuantumCircuit from qiskit.pulse import Schedule from qiskit.compiler import assemble -from qiskit.qobj import Qobj +from qiskit.qobj import QasmQobj, PulseQobj from qiskit.providers.jobstatus import JobStatus from qiskit.providers.ibmq.apiconstants import ApiJobShareLevel from qiskit.providers.ibmq.accountprovider import AccountProvider @@ -401,7 +401,7 @@ def job( 'Unable to find the job for experiment {}.'.format(experiment)) @requires_submit - def qobjs(self) -> List[Qobj]: + def qobjs(self) -> List[Union[QasmQobj, PulseQobj]]: """Return the Qobjs for the jobs in this set. Returns: diff --git a/qiskit/providers/ibmq/utils/qobj_utils.py b/qiskit/providers/ibmq/utils/qobj_utils.py index c4fbf782a..e30ec9379 100644 --- a/qiskit/providers/ibmq/utils/qobj_utils.py +++ b/qiskit/providers/ibmq/utils/qobj_utils.py @@ -14,9 +14,9 @@ """Utilities related to Qobj.""" -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, Union, List -from qiskit.qobj import QobjHeader, Qobj +from qiskit.qobj import QobjHeader, QasmQobj, PulseQobj def _serialize_noise_model(config: Dict[str, Any]) -> Dict[str, Any]: @@ -45,10 +45,10 @@ def _serialize_noise_model(config: Dict[str, Any]) -> Dict[str, Any]: def update_qobj_config( - qobj: Qobj, + qobj: Union[QasmQobj, PulseQobj], backend_options: Optional[Dict] = None, noise_model: Any = None -) -> Qobj: +) -> Union[QasmQobj, PulseQobj]: """Update a ``Qobj`` configuration from backend options and a noise model. Args: @@ -77,3 +77,56 @@ def update_qobj_config( qobj.config = QobjHeader.from_dict(config) return qobj + + +def dict_to_qobj(qobj_dict: Dict) -> Union[QasmQobj, PulseQobj]: + """Convert a Qobj in dictionary format to an instance. + + Args: + qobj_dict: Qobj in dictionary format. + + Returns: + The corresponding QasmQobj or PulseQobj instance. + """ + if qobj_dict['type'] == 'PULSE': + _decode_pulse_qobj(qobj_dict) # Convert to proper types. + return PulseQobj.from_dict(qobj_dict) + return QasmQobj.from_dict(qobj_dict) + + +def _decode_pulse_qobj(pulse_qobj: Dict) -> None: + """Decode a pulse Qobj. + + Args: + pulse_qobj: Qobj to be decoded. + """ + pulse_library = pulse_qobj['config']['pulse_library'] + for lib in pulse_library: + lib['samples'] = [_to_complex(sample) for sample in lib['samples']] + + for exp in pulse_qobj['experiments']: + for instr in exp['instructions']: + if 'val' in instr: + instr['val'] = _to_complex(instr['val']) + if 'parameters' in instr and 'amp' in instr['parameters']: + instr['parameters']['amp'] = _to_complex(instr['parameters']['amp']) + + +def _to_complex(value: Union[List[float], complex]) -> complex: + """Convert the input value to type ``complex``. + + Args: + value: Value to be converted. + + Returns: + Input value in ``complex``. + + Raises: + TypeError: If the input value is not in the expected format. + """ + if isinstance(value, list) and len(value) == 2: + return complex(value[0], value[1]) + elif isinstance(value, complex): + return value + + raise TypeError("{} is not in a valid complex number format.".format(value)) diff --git a/requirements.txt b/requirements.txt index 1d4bacd7b..2d825f135 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ nest-asyncio>=1.0.0,!=1.1.0 -qiskit-terra>=0.10 +qiskit-terra>=0.13 requests>=2.19 requests_ntlm>=1.1.0 websockets>=7,<8 diff --git a/setup.py b/setup.py index 1e3cc03f4..39c70d5b4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ REQUIREMENTS = [ "nest-asyncio>=1.0.0,!=1.1.0", - "qiskit-terra>=0.10", + "qiskit-terra>=0.13", "requests>=2.19", "requests-ntlm>=1.1.0", "websockets>=7,<8", diff --git a/test/fake_account_client.py b/test/fake_account_client.py index 2302fa20f..f5c9454c7 100644 --- a/test/fake_account_client.py +++ b/test/fake_account_client.py @@ -182,7 +182,7 @@ def job_submit(self, backend_name, qobj_dict, job_name, job_share_level, def job_download_qobj(self, job_id, *_args, **_kwargs): """Retrieve and return a Qobj.""" - return self._get_job(job_id).qobj + return copy.deepcopy(self._get_job(job_id).qobj) def job_result(self, job_id, *_args, **_kwargs): """Return a random job result.""" diff --git a/test/ibmq/test_ibmq_job.py b/test/ibmq/test_ibmq_job.py index 35657b2ee..8b9bdd71b 100644 --- a/test/ibmq/test_ibmq_job.py +++ b/test/ibmq/test_ibmq_job.py @@ -659,7 +659,7 @@ def final_state_callback(c_job_id, c_status, c_job, **kwargs): # Check called within wait time. if callback_info['last call time'] and job._status not in JOB_FINAL_STATES: self.assertAlmostEqual( - time.time() - callback_info['last call time'], wait_time, delta=0.1) + time.time() - callback_info['last call time'], wait_time, delta=0.2) callback_info['last call time'] = time.time() # Put callback data in a dictionary to make it mutable. diff --git a/test/ibmq/test_ibmq_job_model.py b/test/ibmq/test_ibmq_job_model.py index ab7d93850..2fb60b94c 100644 --- a/test/ibmq/test_ibmq_job_model.py +++ b/test/ibmq/test_ibmq_job_model.py @@ -17,7 +17,6 @@ from qiskit import ClassicalRegister, QuantumCircuit, QuantumRegister from qiskit.compiler import assemble, transpile from qiskit.validation import ModelValidationError -from qiskit.validation.jsonschema import SchemaValidationError from qiskit.providers.ibmq import IBMQJob @@ -54,7 +53,7 @@ def test_invalid_qobj(self, provider): backend=backend) delattr(qobj, 'qobj_id') - with self.assertRaises(SchemaValidationError): + with self.assertRaises(AttributeError): backend.run(qobj, validate_qobj=True) def test_valid_job(self): diff --git a/test/ibmq/test_ibmq_jobmanager.py b/test/ibmq/test_ibmq_jobmanager.py index a4265a308..fd7aa5065 100644 --- a/test/ibmq/test_ibmq_jobmanager.py +++ b/test/ibmq/test_ibmq_jobmanager.py @@ -126,7 +126,8 @@ def test_job_qobjs(self, provider): job_set.results() for i, qobj in enumerate(job_set.qobjs()): rjob = provider.backends.retrieve_job(jobs[i].job_id()) - self.assertDictEqual(qobj.__dict__, rjob.qobj().__dict__) + self.maxDiff = None # pylint: disable=invalid-name + self.assertDictEqual(qobj.to_dict(), rjob.qobj().to_dict()) @requires_provider def test_error_message(self, provider): diff --git a/test/ibmq/test_serialization.py b/test/ibmq/test_serialization.py new file mode 100644 index 000000000..3abbbbd84 --- /dev/null +++ b/test/ibmq/test_serialization.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# This code is part of Qiskit. +# +# (C) Copyright IBM 2020. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +"""Test serializing and deserializing data sent to the server.""" + +from qiskit.compiler import assemble + +from ..decorators import requires_provider +from ..utils import bell_in_qobj, cancel_job +from ..ibmqtestcase import IBMQTestCase + + +class TestSerialization(IBMQTestCase): + """Test data serialization.""" + + @requires_provider + def test_qasm_qobj(self, provider): + """Test serializing qasm qobj data.""" + backend = provider.get_backend('ibmq_qasm_simulator') + qobj = bell_in_qobj(backend=backend) + job = backend.run(qobj, validate_qobj=True) + rqobj = backend.retrieve_job(job.job_id()).qobj() + + self.assertEqual(_array_to_list(qobj.to_dict()), rqobj.to_dict()) + + @requires_provider + def test_pulse_qobj(self, provider): + """Test serializing pulse qobj data.""" + backend = provider.get_backend('ibmq_armonk') + config = backend.configuration() + defaults = backend.defaults() + inst_map = defaults.circuit_instruction_map + + x = inst_map.get('x', 0) + measure = inst_map.get('measure', range(config.n_qubits)) << x.duration + schedules = x | measure + + qobj = assemble(schedules, backend, meas_level=1, shots=256) + job = backend.run(qobj, validate_qobj=True) + rqobj = backend.retrieve_job(job.job_id()).qobj() + + # Convert numpy arrays to lists since they now get converted right + # before being sent to the server. + self.assertEqual(_array_to_list(qobj.to_dict()), rqobj.to_dict()) + + cancel_job(job) + + +def _array_to_list(data): + """Convert numpy arrays to lists.""" + for key, value in data.items(): + if hasattr(value, 'tolist'): + data[key] = value.tolist() + elif isinstance(value, dict): + _array_to_list(value) + elif isinstance(value, list): + for index, item in enumerate(value): + if isinstance(item, dict): + value[index] = _array_to_list(item) + + return data diff --git a/test/utils.py b/test/utils.py index 7f1d083b8..4fe3bad44 100644 --- a/test/utils.py +++ b/test/utils.py @@ -15,7 +15,7 @@ """General utility functions for testing.""" from qiskit import QuantumCircuit -from qiskit.qobj import Qobj +from qiskit.qobj import QasmQobj from qiskit.compiler import assemble, transpile from qiskit.test.reference_circuits import ReferenceCircuits from qiskit.providers.exceptions import JobError @@ -62,7 +62,7 @@ def get_large_circuit(backend: IBMQBackend) -> QuantumCircuit: return circuit -def bell_in_qobj(backend: IBMQBackend, shots: int = 1024) -> Qobj: +def bell_in_qobj(backend: IBMQBackend, shots: int = 1024) -> QasmQobj: """Return a bell circuit in Qobj format. Args: