Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Hparams: Allow sessions without name to pull all run,tag combinations as metrics. #6546

Merged
merged 1 commit into from
Aug 16, 2023
Merged

Hparams: Allow sessions without name to pull all run,tag combinations as metrics. #6546

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions tensorboard/plugins/hparams/backend_context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ def compute_metric_infos_from_data_provider_session_groups(
self, ctx, experiment_id, session_groups
):
session_runs = set(
generate_data_provider_session_name(experiment_id, s)
generate_data_provider_session_name(s)
for sg in session_groups
for s in sg.sessions
)
Expand Down Expand Up @@ -446,7 +446,7 @@ def _compute_metric_names(self, ctx, experiment_id, session_runs):
)
for run, tags in scalars_run_to_tag_to_content.items():
session = _find_longest_parent_path(session_runs, run)
if not session:
if session is None:
continue
group = os.path.relpath(run, session)
# relpath() returns "." for the 'session' directory, we use an empty
Expand All @@ -460,14 +460,14 @@ def _compute_metric_names(self, ctx, experiment_id, session_runs):
return metric_names_list


def generate_data_provider_session_name(experiment_id, session):
def generate_data_provider_session_name(session):
"""Generates a name from a HyperparameterSesssionRun.

If the HyperparameterSessionRun contains no experiment or run information
then the name is set to the original experiment_id.
"""
if not session.experiment_id and not session.run:
return experiment_id
return ""
elif not session.experiment_id:
return session.run
elif not session.run:
Expand Down
100 changes: 100 additions & 0 deletions tensorboard/plugins/hparams/backend_context_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,106 @@ def test_experiment_from_data_provider_session_group_without_experiment_name(
"""
self.assertProtoEquals(expected_exp, actual_exp)

def test_experiment_from_data_provider_session_group_without_session_names(
self,
):
self._mock_tb_context.data_provider.list_tensors.side_effect = None
self._hyperparameters = provider.ListHyperparametersResult(
hyperparameters=[],
session_groups=[
provider.HyperparameterSessionGroup(
root=provider.HyperparameterSessionRun(
experiment_id="", run=""
),
sessions=[
provider.HyperparameterSessionRun(
experiment_id="", run=""
),
],
hyperparameter_values=[],
),
],
)
actual_exp = self._experiment_from_metadata()
# The result specifies a single session without explicit identifier. It
# therefore represents a session that includes all run/tag combinations
# as separate metric values.
expected_exp = """
metric_infos {
name {
group: "exp/session_1"
tag: "accuracy"
}
}
metric_infos {
name {
group: "exp/session_2"
tag: "accuracy"
}
}
metric_infos {
name {
group: "exp/session_3"
tag: "accuracy"
}
}
metric_infos {
name {
group: "exp/session_1"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_1/eval"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_1/train"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_2"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_2/eval"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_2/train"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_3"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_3/eval"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_3xyz"
tag: "loss2"
}
}
"""
self.assertProtoEquals(expected_exp, actual_exp)

def test_experiment_from_data_provider_old_response_type(self):
self._hyperparameters = [
provider.Hyperparameter(
Expand Down
6 changes: 4 additions & 2 deletions tensorboard/plugins/hparams/list_session_groups.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def _session_groups_from_data_provider(self):
for session in provider_group.sessions:
session_name = (
backend_context_lib.generate_data_provider_session_name(
self._experiment_id, session
session
)
)
sessions.append(
Expand All @@ -150,8 +150,10 @@ def _session_groups_from_data_provider(self):
)

name = backend_context_lib.generate_data_provider_session_name(
self._experiment_id, provider_group.root
provider_group.root
)
if not name:
name = self._experiment_id
session_group = api_pb2.SessionGroup(
name=name,
sessions=sessions,
Expand Down
33 changes: 33 additions & 0 deletions tensorboard/plugins/hparams/list_session_groups_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2066,6 +2066,39 @@ def test_experiment_from_data_provider_with_metric_values_from_run_name(
response.session_groups[0].sessions[0],
)

def test_experiment_from_data_provider_with_metric_values_empty_session_names(
self,
):
self._mock_tb_context.data_provider.list_tensors.side_effect = None
self._hyperparameters = [
provider.HyperparameterSessionGroup(
root=provider.HyperparameterSessionRun(
experiment_id="", run=""
),
sessions=[
provider.HyperparameterSessionRun(experiment_id="", run="")
],
hyperparameter_values=[],
),
]
request = """
start_index: 0
slice_size: 10
"""
response = self._run_handler(request)
self.assertLen(response.session_groups, 1)
# The name comes from the experiment id.
self.assertEquals(response.session_groups[0].name, "123")
self.assertLen(response.session_groups[0].sessions, 1)
self.assertEquals(response.session_groups[0].sessions[0].name, "")
# The result specifies a single session without explicit identifier. It
# therefore represents a session that includes all run/tag combinations
# as separate metric values.
# There are 11 total run/tag combinations across session_1, _2, _3, _4,
# and _5.
self.assertLen(response.session_groups[0].metric_values, 11)
self.assertLen(response.session_groups[0].sessions[0].metric_values, 11)

def test_experiment_from_data_provider_with_metric_values_aggregates(
self,
):
Expand Down