Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

introduce -p/--print-status to ocrd network client blocking commands #1277

Merged
merged 28 commits into from
Oct 10, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
f44e28b
introduce: OCRD_NETWORK_CLIENT_POLLING_PRINT
MehmedGIT Oct 1, 2024
7177eb1
fix: config value description
MehmedGIT Oct 1, 2024
df8e8ee
add default value param to preserver backwards compatibility
MehmedGIT Oct 1, 2024
b183cfc
make -b/--block as flags
MehmedGIT Oct 1, 2024
342ef3a
implement feedback
MehmedGIT Oct 1, 2024
0e80a7c
fix: missed params
MehmedGIT Oct 1, 2024
d7df200
fix: integration client tests
MehmedGIT Oct 1, 2024
0bfef64
post_ps_workflow_request: pagewise configurable
kba Oct 1, 2024
19aad83
Merge remote-tracking branch 'github/network_client_block_prints' int…
kba Oct 1, 2024
611b6b5
deployer: Remove any pre-existing socket file before starting the ser…
kba Oct 1, 2024
9a71d04
remove UDS socket files
MehmedGIT Oct 2, 2024
854403d
remove shortcuts for page-wise
MehmedGIT Oct 2, 2024
4d01e66
fix: pass page-wise argument to relevant methods
MehmedGIT Oct 2, 2024
97427e0
Update src/ocrd_network/client_utils.py
MehmedGIT Oct 2, 2024
7454845
add endpoint DELETE /workflow/kill-mets-server-zombies to kill -SIGTE…
kba Oct 2, 2024
0506e9d
move mets-zombie killer to / and return list of killed PIDs
kba Oct 2, 2024
ad81356
/kill_mets_server_zombies use underscores not slashes
kba Oct 2, 2024
7a3be1e
Merge pull request #1278 from OCR-D/page-wise-param
kba Oct 2, 2024
4862d72
use 3.8 compatible typing
kba Oct 2, 2024
2cb3e2a
Merge branch 'network_client_block_prints' into mets-server-kill-zombies
kba Oct 2, 2024
8b6a49c
Merge pull request #1282 from OCR-D/mets-server-rm-socket
kba Oct 2, 2024
0d297e7
Merge branch 'network_client_block_prints' into mets-server-kill-zombies
kba Oct 2, 2024
4f6775f
OcrdMetsServer.kill_process: try the easy way (SIGINT) then the hard …
kba Oct 2, 2024
3882e7a
fix: add default to page_wise param
MehmedGIT Oct 2, 2024
a8bfbe4
Merge branch 'network_client_block_prints' into mets-server-kill-zombies
kba Oct 2, 2024
c5fd843
Merge pull request #1283 from OCR-D/mets-server-kill-zombies
kba Oct 2, 2024
d39c3d7
kill_mets_server_zombies: actually return List[int]
kba Oct 10, 2024
7512bd6
kill_mets_server_zombies: allow dry_run to test
kba Oct 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions src/ocrd_network/cli/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,10 @@ def check_processing_job_status(address: Optional[str], processing_job_id: str):
@click.option('--result-queue-name')
@click.option('--callback-url')
@click.option('--agent-type', default='worker')
@click.option('-b', '--block', default=False,
@click.option('-b', '--block', default=False, is_flag=True,
help='If set, the client will block till job timeout, fail or success.')
@click.option('-p', '--print-state', default=False, is_flag=True,
help='If set, the client will print job states by each iteration.')
def send_processing_job_request(
address: Optional[str],
processor_name: str,
Expand All @@ -120,7 +122,8 @@ def send_processing_job_request(
# TODO: This is temporally available to toggle
# between the ProcessingWorker/ProcessorServer
agent_type: Optional[str],
block: Optional[bool]
block: Optional[bool],
print_state: Optional[bool]
):
"""
Submit a processing job to the processing server.
Expand All @@ -146,7 +149,7 @@ def send_processing_job_request(
assert processing_job_id
print(f"Processing job id: {processing_job_id}")
if block:
client.poll_job_status(job_id=processing_job_id)
client.poll_job_status(job_id=processing_job_id, print_state=print_state)


@client_cli.group('workflow')
Expand Down Expand Up @@ -176,13 +179,16 @@ def check_workflow_job_status(address: Optional[str], workflow_job_id: str):
'the "OCRD_NETWORK_SERVER_ADDR_PROCESSING" env variable is used by default')
@click.option('-m', '--path-to-mets', required=True)
@click.option('-w', '--path-to-workflow', required=True)
@click.option('-b', '--block', default=False,
@click.option('-b', '--block', default=False, is_flag=True,
help='If set, the client will block till job timeout, fail or success.')
@click.option('-p', '--print-state', default=False, is_flag=True,
help='If set, the client will print job states by each iteration.')
def send_workflow_job_request(
address: Optional[str],
path_to_mets: str,
path_to_workflow: str,
block: Optional[bool]
block: Optional[bool],
print_state: Optional[bool]
):
"""
Submit a workflow job to the processing server.
Expand All @@ -192,7 +198,7 @@ def send_workflow_job_request(
assert workflow_job_id
print(f"Workflow job id: {workflow_job_id}")
if block:
client.poll_workflow_status(job_id=workflow_job_id)
client.poll_workflow_status(job_id=workflow_job_id, print_state=print_state)


@client_cli.group('workspace')
Expand Down
10 changes: 6 additions & 4 deletions src/ocrd_network/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@ def check_job_status(self, job_id: str):
def check_workflow_status(self, workflow_job_id: str):
return get_ps_workflow_job_status(self.server_addr_processing, workflow_job_id=workflow_job_id)

def poll_job_status(self, job_id: str) -> str:
def poll_job_status(self, job_id: str, print_state: bool) -> str:
return poll_job_status_till_timeout_fail_or_success(
ps_server_host=self.server_addr_processing, job_id=job_id, tries=self.polling_tries, wait=self.polling_wait)
ps_server_host=self.server_addr_processing, job_id=job_id, tries=self.polling_tries, wait=self.polling_wait,
print_state=print_state)

def poll_workflow_status(self, job_id: str) -> str:
def poll_workflow_status(self, job_id: str, print_state: bool) -> str:
return poll_wf_status_till_timeout_fail_or_success(
ps_server_host=self.server_addr_processing, job_id=job_id, tries=self.polling_tries, wait=self.polling_wait)
ps_server_host=self.server_addr_processing, job_id=job_id, tries=self.polling_tries, wait=self.polling_wait,
print_state=print_state)

def send_processing_job_request(self, processor_name: str, req_params: dict) -> str:
return post_ps_processing_request(
Expand Down
15 changes: 10 additions & 5 deletions src/ocrd_network/client_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
from .constants import JobState, NETWORK_PROTOCOLS


def _poll_endpoint_status(ps_server_host: str, job_id: str, job_type: str, tries: int, wait: int):
def _poll_endpoint_status(
ps_server_host: str, job_id: str, job_type: str, tries: int, wait: int, print_state: bool = False):
if job_type not in ["workflow", "processor"]:
raise ValueError(f"Unknown job type '{job_type}', expected 'workflow' or 'processor'")
job_state = JobState.unset
Expand All @@ -13,18 +14,22 @@ def _poll_endpoint_status(ps_server_host: str, job_id: str, job_type: str, tries
job_state = get_ps_processing_job_status(ps_server_host, job_id)
if job_type == "workflow":
job_state = get_ps_workflow_job_status(ps_server_host, job_id)
if print_state:
print(f"State of the {job_type} job {job_id}: {job_state}")
if job_state == JobState.success or job_state == JobState.failed:
break
tries -= 1
return job_state


def poll_job_status_till_timeout_fail_or_success(ps_server_host: str, job_id: str, tries: int, wait: int) -> JobState:
return _poll_endpoint_status(ps_server_host, job_id, "processor", tries, wait)
def poll_job_status_till_timeout_fail_or_success(
ps_server_host: str, job_id: str, tries: int, wait: int, print_state: bool = False) -> JobState:
return _poll_endpoint_status(ps_server_host, job_id, "processor", tries, wait, print_state)


def poll_wf_status_till_timeout_fail_or_success(ps_server_host: str, job_id: str, tries: int, wait: int) -> JobState:
return _poll_endpoint_status(ps_server_host, job_id, "workflow", tries, wait)
def poll_wf_status_till_timeout_fail_or_success(
ps_server_host: str, job_id: str, tries: int, wait: int, print_state: bool = False) -> JobState:
return _poll_endpoint_status(ps_server_host, job_id, "workflow", tries, wait, print_state)


def get_ps_deployed_processors(ps_server_host: str):
Expand Down
2 changes: 1 addition & 1 deletion src/ocrd_utils/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def _ocrd_download_timeout_parser(val):
config.add("OCRD_NETWORK_CLIENT_POLLING_SLEEP",
description="How many seconds to sleep before trying again.",
parser=int,
default=(True, 30))
default=(True, 10))

config.add("OCRD_NETWORK_CLIENT_POLLING_TIMEOUT",
description="Timeout for a blocking ocrd network client (in seconds).",
Expand Down
Loading