Skip to content
This repository has been archived by the owner on Jul 28, 2023. It is now read-only.

Add job limit exception #629

Merged
merged 1 commit into from
Apr 16, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ The format is based on [Keep a Changelog].

## [UNRELEASED]

### Added

- A new exception, `IBMQBackendJobLimitError`, is now raised if a job
could not be submitted because the limit on active jobs has been reached.

### Changed

- `IBMQFactory.save_account()` and `IBMQFactory.enable_account()` now accept
Expand Down
5 changes: 5 additions & 0 deletions qiskit/providers/ibmq/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,8 @@ class IBMQBackendApiProtocolError(IBMQBackendApiError):
class IBMQBackendValueError(IBMQBackendError, ValueError):
"""Value errors raised by the backend modules."""
pass


class IBMQBackendJobLimitError(IBMQBackendError):
"""Errors raised when job limit is reached."""
pass
7 changes: 6 additions & 1 deletion qiskit/providers/ibmq/ibmqbackend.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@
from .backendjoblimit import BackendJobLimit
from .credentials import Credentials
from .exceptions import (IBMQBackendError, IBMQBackendValueError,
IBMQBackendApiError, IBMQBackendApiProtocolError)
IBMQBackendApiError, IBMQBackendApiProtocolError,
IBMQBackendJobLimitError)
from .job import IBMQJob
from .utils import update_qobj_config, validate_job_tags

Expand Down Expand Up @@ -203,6 +204,8 @@ def _submit_job(
the job.
IBMQBackendApiProtocolError: If an unexpected value is received from
the server.
IBMQBackendJobLimitError: If the job could not be submitted because
the job limit has been reached.
"""
try:
qobj_dict = qobj.to_dict()
Expand All @@ -213,6 +216,8 @@ def _submit_job(
job_share_level=job_share_level,
job_tags=job_tags)
except ApiError as ex:
if 'Error code: 3458' in str(ex):
raise IBMQBackendJobLimitError('Error submitting job: {}'.format(str(ex))) from ex
raise IBMQBackendApiError('Error submitting job: {}'.format(str(ex))) from ex

# Error in the job after submission:
Expand Down
26 changes: 12 additions & 14 deletions qiskit/providers/ibmq/managed/managedjob.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

from ..job.ibmqjob import IBMQJob
from ..job.exceptions import IBMQJobTimeoutError
from ..exceptions import IBMQBackendApiError
from ..exceptions import IBMQBackendJobLimitError

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -116,19 +116,17 @@ def _async_submit(
job_name=job_name,
job_share_level=job_share_level.value,
job_tags=job_tags)
except IBMQBackendApiError as api_err:
if 'Error code: 3458' in str(api_err):
final_states = [state.value for state in API_JOB_FINAL_STATES]
oldest_running = backend.jobs(limit=1, descending=False,
db_filter={"status": {"nin": final_states}})
if oldest_running:
oldest_running = oldest_running[0]
logger.warning("Job limit reached, waiting for job %s to finish "
"before submitting the next one.",
oldest_running.job_id())
oldest_running.wait_for_final_state(timeout=300)
else:
raise
except IBMQBackendJobLimitError:
final_states = [state.value for state in API_JOB_FINAL_STATES]
oldest_running = backend.jobs(limit=1, descending=False,
db_filter={"status": {"nin": final_states}})
if oldest_running:
oldest_running = oldest_running[0]
logger.warning("Job limit reached, waiting for job %s to finish "
"before submitting the next one.",
oldest_running.job_id())
oldest_running.wait_for_final_state(timeout=300)

except Exception as err: # pylint: disable=broad-except
warnings.warn("Unable to submit job for experiments {}-{}: {}".format(
self.start_index, self.end_index, err))
Expand Down
26 changes: 20 additions & 6 deletions test/ibmq/test_basic_server_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@

"""Tests that hit all the basic server endpoints using both a public and premium provider."""

from qiskit.test import slow_test
import time

from qiskit.test import slow_test
from qiskit.providers.ibmq import least_busy
from qiskit.providers.ibmq.exceptions import IBMQBackendJobLimitError

from ..decorators import requires_providers
from ..ibmqtestcase import IBMQTestCase
from ..utils import cancel_job, bell_in_qobj
Expand All @@ -41,7 +44,7 @@ def test_job_submission(self):
filters=lambda b: b.configuration().n_qubits >= 5))
with self.subTest(desc=desc, backend=backend):
qobj = bell_in_qobj(backend)
job = backend.run(qobj, validate_qobj=True)
job = self._submit_job_with_retry(qobj, backend)

# Fetch the results.
result = job.result()
Expand All @@ -59,8 +62,7 @@ def test_job_backend_properties_and_status(self):
filters=lambda b: b.configuration().n_qubits >= 5)[0]
with self.subTest(desc=desc, backend=backend):
qobj = bell_in_qobj(backend)
job = backend.run(qobj, validate_qobj=True)

job = self._submit_job_with_retry(qobj, backend)
self.assertIsNotNone(job.properties())
self.assertTrue(job.status())
# Cancel job so it doesn't consume more resources.
Expand All @@ -73,8 +75,7 @@ def test_retrieve_jobs(self):
backend = provider.get_backend(backend_name)
with self.subTest(desc=desc, backend=backend):
qobj = bell_in_qobj(backend)

job = backend.run(qobj, validate_qobj=True)
job = self._submit_job_with_retry(qobj, backend)
job_id = job.job_id()

retrieved_jobs = provider.backends.jobs(backend_name=backend_name)
Expand Down Expand Up @@ -105,3 +106,16 @@ def test_device_status_and_job_limit(self):
if desc == 'public_provider':
self.assertEqual(job_limit.maximum_jobs, 5)
self.assertTrue(job_limit)

def _submit_job_with_retry(self, qobj, backend, max_retry=5):
"""Retry submitting a job if limit is reached."""
limit_error = None
for _ in range(max_retry):
try:
job = backend.run(qobj, validate_qobj=True)
return job
except IBMQBackendJobLimitError as err:
limit_error = err
time.sleep(1)

self.fail("Unable to submit job after {} retries: {}".format(max_retry, limit_error))