Skip to content

Commit

Permalink
Add option --since in the logs command (#2257)
Browse files Browse the repository at this point in the history
  • Loading branch information
serhiy-storchaka authored Aug 18, 2021
1 parent 16a5c10 commit 0af876d
Show file tree
Hide file tree
Showing 9 changed files with 80 additions and 3 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.D/1964.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add option `--since` in command `neuro logs`. Add parameter *since* in jobs.monitor().
2 changes: 2 additions & 0 deletions CLI.md
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,7 @@ neuro job logs [OPTIONS] JOB
Name | Description|
|----|------------|
|_--help_|Show this message and exit.|
|_--since DATE\_OR_TIMEDELTA_|Only return logs after a specific date \(including). Use value of format '1d2h3m4s' to specify moment in past relatively to current time.|
|_--timestamps_|Include timestamps on each line in the log output.|


Expand Down Expand Up @@ -2626,6 +2627,7 @@ neuro logs [OPTIONS] JOB
Name | Description|
|----|------------|
|_--help_|Show this message and exit.|
|_--since DATE\_OR_TIMEDELTA_|Only return logs after a specific date \(including). Use value of format '1d2h3m4s' to specify moment in past relatively to current time.|
|_--timestamps_|Include timestamps on each line in the log output.|


Expand Down
1 change: 1 addition & 0 deletions neuro-cli/docs/job.md
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ Print the logs for a job.
| Name | Description |
| :--- | :--- |
| _--help_ | Show this message and exit. |
| _--since DATE\_OR\_TIMEDELTA_ | Only return logs after a specific date \(including\). Use value of format '1d2h3m4s' to specify moment in past relatively to current time. |
| _--timestamps_ | Include timestamps on each line in the log output. |


Expand Down
1 change: 1 addition & 0 deletions neuro-cli/docs/shortcuts.md
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ Print the logs for a job.
| Name | Description |
| :--- | :--- |
| _--help_ | Show this message and exit. |
| _--since DATE\_OR\_TIMEDELTA_ | Only return logs after a specific date \(including\). Use value of format '1d2h3m4s' to specify moment in past relatively to current time. |
| _--timestamps_ | Include timestamps on each line in the log output. |


Expand Down
3 changes: 3 additions & 0 deletions neuro-cli/src/neuro_cli/ael.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import signal
import sys
import threading
from datetime import datetime
from typing import Any, Awaitable, Callable, List, Optional, Sequence, Tuple

import aiohttp
Expand Down Expand Up @@ -80,6 +81,7 @@ async def process_logs(
helper: Optional[AttachHelper],
*,
cluster_name: Optional[str],
since: Optional[datetime] = None,
timestamps: bool = False,
) -> None:
codec_info = codecs.lookup("utf8")
Expand All @@ -88,6 +90,7 @@ async def process_logs(
async with root.client.jobs.monitor(
job,
cluster_name=cluster_name,
since=since,
timestamps=timestamps,
separator=separator,
debug=root.verbosity >= 2,
Expand Down
18 changes: 16 additions & 2 deletions neuro-cli/src/neuro_cli/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,19 @@ async def port_forward(

@command()
@argument("job", type=JOB)
@option(
"--since",
metavar="DATE_OR_TIMEDELTA",
help="Only return logs after a specific date (including). "
"Use value of format '1d2h3m4s' to specify moment in "
"past relatively to current time.",
)
@option(
"--timestamps",
is_flag=True,
help="Include timestamps on each line in the log output.",
)
async def logs(root: Root, job: str, timestamps: bool) -> None:
async def logs(root: Root, since: str, job: str, timestamps: bool) -> None:
"""
Print the logs for a job.
"""
Expand All @@ -246,7 +253,14 @@ async def logs(root: Root, job: str, timestamps: bool) -> None:
client=root.client,
status=JobStatus.items(),
)
await process_logs(root, id, None, cluster_name=cluster_name, timestamps=timestamps)
await process_logs(
root,
id,
None,
cluster_name=cluster_name,
since=_parse_date(since),
timestamps=timestamps,
)


@command()
Expand Down
7 changes: 7 additions & 0 deletions neuro-sdk/docs/jobs_reference.rst
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ Jobs

.. comethod:: monitor(id: str, *, \
cluster_name: Optional[str] = None, \
since: Optional[datetime] = None,
timestamps: bool = False,
separator: Optional[str] = None,
) -> AsyncContextManager[AsyncIterator[bytes]]
Expand All @@ -204,6 +205,12 @@ Jobs

``None`` means the current cluster (default).

:param ~datetime.datetime since: Retrieves only logs after the specified date
(including) if it is not ``None``. If the parameter
is a naive datetime object, it represents local time.

``None`` means that no filter is applied (default).

:param bool timestamps: if true, include timestamps on each line in the log output.

:param str separator: string which will separate archive and live logs
Expand Down
6 changes: 6 additions & 0 deletions neuro-sdk/src/neuro_sdk/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -497,11 +497,17 @@ async def monitor(
id: str,
*,
cluster_name: Optional[str] = None,
since: Optional[datetime] = None,
timestamps: bool = False,
separator: Optional[str] = None,
debug: bool = False,
) -> AsyncIterator[bytes]:
url = self._get_monitoring_url(cluster_name) / id / "log"
if since is not None:
if since.tzinfo is None:
# Interpret naive datetime object as local time.
since = since.astimezone(timezone.utc)
url = url.update_query(since=since.isoformat())
if timestamps:
url = url.update_query(timestamps="true")
if separator is not None:
Expand Down
44 changes: 43 additions & 1 deletion neuro-sdk/tests/test_jobs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import asyncio
import json
import sys
from datetime import datetime, timezone
from typing import Any, Callable, Dict, List, Optional

import pytest
Expand Down Expand Up @@ -46,6 +47,7 @@ async def test_jobs_monitor(
) -> None:
async def log_stream(request: web.Request) -> web.StreamResponse:
assert request.headers["Accept-Encoding"] == "identity"
assert "since" not in request.query
assert request.query.get("timestamps", "false") == "false"
resp = web.StreamResponse()
resp.enable_chunked_encoding()
Expand Down Expand Up @@ -82,12 +84,52 @@ async def log_stream(request: web.Request) -> web.StreamResponse:
)


async def test_jobs_monitor_since(
aiohttp_server: _TestServerFactory, make_client: _MakeClient
) -> None:
async def log_stream(request: web.Request) -> web.StreamResponse:
assert request.headers["Accept-Encoding"] == "identity"
assert request.query["since"] == "2021-08-17T00:00:00+00:00"
assert request.query.get("timestamps", "false") == "false"
resp = web.StreamResponse()
resp.enable_chunked_encoding()
resp.enable_compression(web.ContentCoding.identity)
await resp.prepare(request)
for i in range(5, 10):
await resp.write(b"chunk " + str(i).encode("ascii") + b"\n")
return resp

app = web.Application()
app.router.add_get("/jobs/job-id/log", log_stream)

srv = await aiohttp_server(app)

lst = []
async with make_client(srv.make_url("/")) as client:
async with client.jobs.monitor(
"job-id", since=datetime(2021, 8, 17, tzinfo=timezone.utc)
) as it:
async for data in it:
lst.append(data)

assert b"".join(lst) == b"".join(
[
b"chunk 5\n",
b"chunk 6\n",
b"chunk 7\n",
b"chunk 8\n",
b"chunk 9\n",
]
)


async def test_jobs_monitor_timestamps(
aiohttp_server: _TestServerFactory, make_client: _MakeClient
) -> None:
async def log_stream(request: web.Request) -> web.StreamResponse:
assert request.headers["Accept-Encoding"] == "identity"
assert request.query.get("timestamps", "false") == "true"
assert "since" not in request.query
assert request.query["timestamps"] == "true"
resp = web.StreamResponse()
resp.enable_chunked_encoding()
resp.enable_compression(web.ContentCoding.identity)
Expand Down

0 comments on commit 0af876d

Please sign in to comment.