Skip to content

Commit

Permalink
Allow sessions without name to pull all run,tag combinations as metrics.
Browse files Browse the repository at this point in the history
  • Loading branch information
@bmd3k
bmd3k committed Aug 15, 2023
1 parent 42545de commit 6bbde00
Show file tree
Hide file tree
Showing 4 changed files with 141 additions and 6 deletions.
8 changes: 4 additions & 4 deletions tensorboard/plugins/hparams/backend_context.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ def compute_metric_infos_from_data_provider_session_groups(
self, ctx, experiment_id, session_groups
):
session_runs = set(
generate_data_provider_session_name(experiment_id, s)
generate_data_provider_session_name(s)
for sg in session_groups
for s in sg.sessions
)
Expand Down Expand Up @@ -446,7 +446,7 @@ def _compute_metric_names(self, ctx, experiment_id, session_runs):
)
for run, tags in scalars_run_to_tag_to_content.items():
session = _find_longest_parent_path(session_runs, run)
if not session:
if session is None:
continue
group = os.path.relpath(run, session)
# relpath() returns "." for the 'session' directory, we use an empty
Expand All @@ -460,14 +460,14 @@ def _compute_metric_names(self, ctx, experiment_id, session_runs):
return metric_names_list


def generate_data_provider_session_name(experiment_id, session):
def generate_data_provider_session_name(session):
"""Generates a name from a HyperparameterSesssionRun.
If the HyperparameterSessionRun contains no experiment or run information
then the name is set to the original experiment_id.
"""
if not session.experiment_id and not session.run:
return experiment_id
return ""
elif not session.experiment_id:
return session.run
elif not session.run:
Expand Down
100 changes: 100 additions & 0 deletions tensorboard/plugins/hparams/backend_context_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,106 @@ def test_experiment_from_data_provider_session_group_without_experiment_name(
"""
self.assertProtoEquals(expected_exp, actual_exp)

def test_experiment_from_data_provider_session_group_without_session_names(
self,
):
self._mock_tb_context.data_provider.list_tensors.side_effect = None
self._hyperparameters = provider.ListHyperparametersResult(
hyperparameters=[],
session_groups=[
provider.HyperparameterSessionGroup(
root=provider.HyperparameterSessionRun(
experiment_id="", run=""
),
sessions=[
provider.HyperparameterSessionRun(
experiment_id="", run=""
),
],
hyperparameter_values=[],
),
],
)
actual_exp = self._experiment_from_metadata()
# The result specifies a single session without explicit identifier. It
# therefore represents a session that includes all run/tag combinations
# as separate metric values.
expected_exp = """
metric_infos {
name {
group: "exp/session_1"
tag: "accuracy"
}
}
metric_infos {
name {
group: "exp/session_2"
tag: "accuracy"
}
}
metric_infos {
name {
group: "exp/session_3"
tag: "accuracy"
}
}
metric_infos {
name {
group: "exp/session_1"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_1/eval"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_1/train"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_2"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_2/eval"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_2/train"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_3"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_3/eval"
tag: "loss"
}
}
metric_infos {
name {
group: "exp/session_3xyz"
tag: "loss2"
}
}
"""
self.assertProtoEquals(expected_exp, actual_exp)

def test_experiment_from_data_provider_old_response_type(self):
self._hyperparameters = [
provider.Hyperparameter(
Expand Down
6 changes: 4 additions & 2 deletions tensorboard/plugins/hparams/list_session_groups.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def _session_groups_from_data_provider(self):
for session in provider_group.sessions:
session_name = (
backend_context_lib.generate_data_provider_session_name(
self._experiment_id, session
session
)
)
sessions.append(
Expand All @@ -150,8 +150,10 @@ def _session_groups_from_data_provider(self):
)

name = backend_context_lib.generate_data_provider_session_name(
self._experiment_id, provider_group.root
provider_group.root
)
if not name:
name = self._experiment_id
session_group = api_pb2.SessionGroup(
name=name,
sessions=sessions,
Expand Down
33 changes: 33 additions & 0 deletions tensorboard/plugins/hparams/list_session_groups_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2066,6 +2066,39 @@ def test_experiment_from_data_provider_with_metric_values_from_run_name(
response.session_groups[0].sessions[0],
)

def test_experiment_from_data_provider_with_metric_values_empty_session_names(
self,
):
self._mock_tb_context.data_provider.list_tensors.side_effect = None
self._hyperparameters = [
provider.HyperparameterSessionGroup(
root=provider.HyperparameterSessionRun(
experiment_id="", run=""
),
sessions=[
provider.HyperparameterSessionRun(experiment_id="", run="")
],
hyperparameter_values=[],
),
]
request = """
start_index: 0
slice_size: 10
"""
response = self._run_handler(request)
self.assertLen(response.session_groups, 1)
# The name comes from the experiment id.
self.assertEquals(response.session_groups[0].name, "123")
self.assertLen(response.session_groups[0].sessions, 1)
self.assertEquals(response.session_groups[0].sessions[0].name, "")
# The result specifies a single session without explicit identifier. It
# therefore represents a session that includes all run/tag combinations
# as separate metric values.
# There are 11 total run/tag combinations across session_1, _2, _3, _4,
# and _5.
self.assertLen(response.session_groups[0].metric_values, 11)
self.assertLen(response.session_groups[0].sessions[0].metric_values, 11)

def test_experiment_from_data_provider_with_metric_values_aggregates(
self,
):
Expand Down

0 comments on commit 6bbde00

Please sign in to comment.