From d499ef0f62f5ad6923a5a5b694d0833ccc8e3edb Mon Sep 17 00:00:00 2001
From: yagnaa <53996715+Yagna24@users.noreply.github.com>
Date: Wed, 27 Apr 2022 00:55:07 +0530
Subject: [PATCH 01/41] Python 3.7 compatibility (#29)
Co-authored-by: Juan Coria
---
src/diart/functional.py | 4 +++-
src/diart/sinks.py | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/src/diart/functional.py b/src/diart/functional.py
index 1d340c74..04ba8d50 100644
--- a/src/diart/functional.py
+++ b/src/diart/functional.py
@@ -1,4 +1,6 @@
-from typing import Union, Optional, List, Literal, Iterable, Tuple
+from typing import Union, Optional, List, Iterable, Tuple
+from typing_extensions import Literal
+
import numpy as np
import torch
diff --git a/src/diart/sinks.py b/src/diart/sinks.py
index 0f54f8a1..7d0bb8d2 100644
--- a/src/diart/sinks.py
+++ b/src/diart/sinks.py
@@ -1,6 +1,7 @@
from pathlib import Path
from traceback import print_exc
-from typing import Literal, Union, Text, Optional, Tuple
+from typing import Union, Text, Optional, Tuple
+from typing_extensions import Literal
import matplotlib.pyplot as plt
from pyannote.core import Annotation, Segment, SlidingWindowFeature, notebook
From 935b6304220aabe4823d6c2731f38a67dae24206 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Fri, 22 Apr 2022 14:18:07 +0200
Subject: [PATCH 02/41] Start refactoring for batched diarization pipeline
---
README.md | 2 +-
src/diart/demo.py | 15 +++++++------
src/diart/functional.py | 48 ++++++++++++++++++++++++++++++----------
src/diart/pipelines.py | 48 +++++++++++++++++++++++++++++-----------
src/diart/sources.py | 49 ++++++++++++++++++++++++++++++++---------
5 files changed, 119 insertions(+), 43 deletions(-)
diff --git a/README.md b/README.md
index 85d9dfdf..317a8142 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ torch.Size([4, 512])
1) Create environment:
```shell
-conda create -n diarization python==3.8
+conda create -n diarization python=3.8
conda activate diarization
```
diff --git a/src/diart/demo.py b/src/diart/demo.py
index 8442e86e..a9c2570e 100644
--- a/src/diart/demo.py
+++ b/src/diart/demo.py
@@ -4,7 +4,7 @@
import diart.operators as dops
import diart.sources as src
import rx.operators as ops
-from diart.pipelines import OnlineSpeakerDiarization
+from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
from diart.sinks import RealTimePlot, RTTMWriter
# Define script arguments
@@ -27,7 +27,7 @@
args = parser.parse_args()
# Define online speaker diarization pipeline
-pipeline = OnlineSpeakerDiarization(
+config = PipelineConfig(
step=args.step,
latency=args.latency,
tau_active=args.tau,
@@ -37,6 +37,7 @@
beta=args.beta,
max_speakers=args.max_speakers,
)
+pipeline = OnlineSpeakerDiarization(config)
# Manage audio source
if args.source != "microphone":
@@ -47,7 +48,7 @@
file=args.source,
uri=uri,
reader=src.RegularAudioFileReader(
- args.sample_rate, pipeline.duration, pipeline.step
+ args.sample_rate, config.duration, config.step
),
)
else:
@@ -58,12 +59,12 @@
pipeline.from_source(audio_source).pipe(
ops.do(RTTMWriter(path=output_dir / "output.rttm")),
dops.buffer_output(
- duration=pipeline.duration,
- step=pipeline.step,
- latency=pipeline.latency,
+ duration=config.duration,
+ step=config.step,
+ latency=config.latency,
sample_rate=audio_source.sample_rate
),
-).subscribe(RealTimePlot(pipeline.duration, pipeline.latency))
+).subscribe(RealTimePlot(config.duration, config.latency))
# Read audio source as a stream
if args.source == "microphone":
diff --git a/src/diart/functional.py b/src/diart/functional.py
index 04ba8d50..2ad23ee9 100644
--- a/src/diart/functional.py
+++ b/src/diart/functional.py
@@ -11,6 +11,25 @@
from .mapping import SpeakerMap, SpeakerMapBuilder
+Waveform = Union[SlidingWindowFeature, np.ndarray, torch.Tensor]
+
+
+def resolve_waveform(waveform: Waveform) -> torch.Tensor:
+ # As torch.Tensor with shape (..., 1, samples)
+ if isinstance(waveform, SlidingWindowFeature):
+ data = torch.from_numpy(waveform.data).transpose(-1, -2)
+ elif isinstance(waveform, np.ndarray):
+ data = torch.from_numpy(waveform)
+ else:
+ data = waveform
+ # Make sure there's a batch dimension
+ msg = "Waveform must be 2D (1, samples) or 3D (batch, 1, samples)"
+ assert data.ndim in (2, 3), msg
+ if data.ndim == 2:
+ data = data.unsqueeze(0)
+ return data
+
+
class FrameWiseModel:
def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
self.model = get_model(model)
@@ -19,19 +38,24 @@ def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
device = get_devices(needs=1)[0]
self.model.to(device)
- def __call__(self, waveform: SlidingWindowFeature) -> SlidingWindowFeature:
+ def __call__(self, waveform: Waveform) -> Union[SlidingWindowFeature, np.ndarray]:
with torch.no_grad():
- wave = torch.from_numpy(waveform.data.T[np.newaxis])
- output = self.model(wave.to(self.model.device)).cpu().numpy()[0]
- # Temporal resolution of the output
- resolution = self.model.specifications.duration / output.shape[0]
- # Temporal shift to keep track of current start time
- resolution = SlidingWindow(
- start=waveform.sliding_window.start,
- duration=resolution,
- step=resolution
- )
- return SlidingWindowFeature(output, resolution)
+ wave = resolve_waveform(waveform).to(self.model.device)
+ output = self.model(wave).cpu().numpy()[0]
+
+ # Wrap if a SlidingWindowFeature was given as input
+ if isinstance(waveform, SlidingWindowFeature):
+ # Temporal resolution of the output
+ resolution = self.model.specifications.duration / output.shape[0]
+ # Temporal shift to keep track of current start time
+ resolution = SlidingWindow(
+ start=waveform.sliding_window.start,
+ duration=resolution,
+ step=resolution
+ )
+ return SlidingWindowFeature(output, resolution)
+
+ return output
class ChunkWiseModel:
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index e31d4d84..baed5da2 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -1,15 +1,17 @@
-from typing import Optional
+from pathlib import Path
+from typing import Optional, Union, Text
import rx
import rx.operators as ops
from pyannote.audio.pipelines.utils import PipelineModel
+from pyannote.core import Annotation
from . import functional as fn
from . import operators as dops
from . import sources as src
-class OnlineSpeakerDiarization:
+class PipelineConfig:
def __init__(
self,
segmentation: PipelineModel = "pyannote/segmentation",
@@ -50,34 +52,43 @@ def get_end_time(self, source: src.AudioSource) -> Optional[float]:
return source.duration - source.duration % self.step
return None
+
+class OnlineSpeakerDiarization:
+ def __init__(self, config: PipelineConfig):
+ self.config = config
+
def from_source(self, source: src.AudioSource, output_waveform: bool = True) -> rx.Observable:
- msg = f"Audio source has sample rate {source.sample_rate}, expected {self.sample_rate}"
- assert source.sample_rate == self.sample_rate, msg
+ msg = f"Audio source has sample rate {source.sample_rate}, expected {self.config.sample_rate}"
+ assert source.sample_rate == self.config.sample_rate, msg
# Regularize the stream to a specific chunk duration and step
regular_stream = source.stream
if not source.is_regular:
regular_stream = source.stream.pipe(
- dops.regularize_stream(self.duration, self.step, source.sample_rate)
+ dops.regularize_stream(self.config.duration, self.config.step, source.sample_rate)
)
# Branch the stream to calculate chunk segmentation
segmentation_stream = regular_stream.pipe(
- ops.map(self.segmentation)
+ ops.map(self.config.segmentation)
)
# Join audio and segmentation stream to calculate speaker embeddings
- osp = fn.OverlappedSpeechPenalty(gamma=self.gamma, beta=self.beta)
+ osp = fn.OverlappedSpeechPenalty(gamma=self.config.gamma, beta=self.config.beta)
embedding_stream = rx.zip(regular_stream, segmentation_stream).pipe(
ops.starmap(lambda wave, seg: (wave, osp(seg))),
- ops.starmap(self.embedding),
+ ops.starmap(self.config.embedding),
ops.map(fn.EmbeddingNormalization(norm=1))
)
# Join segmentation and embedding streams to update a background clustering model
# while regulating latency and binarizing the output
clustering = fn.OnlineSpeakerClustering(
- self.tau_active, self.rho_update, self.delta_new, "cosine", self.max_speakers
+ self.config.tau_active,
+ self.config.rho_update,
+ self.config.delta_new,
+ "cosine",
+ self.config.max_speakers,
)
- end_time = self.get_end_time(source)
+ end_time = self.config.get_end_time(source)
aggregation = fn.DelayedAggregation(
- self.step, self.latency, strategy="hamming", stream_end=end_time
+ self.config.step, self.config.latency, strategy="hamming", stream_end=end_time
)
pipeline = rx.zip(segmentation_stream, embedding_stream).pipe(
ops.starmap(clustering),
@@ -86,12 +97,12 @@ def from_source(self, source: src.AudioSource, output_waveform: bool = True) ->
# Aggregate overlapping output windows
ops.map(aggregation),
# Binarize output
- ops.map(fn.Binarize(source.uri, self.tau_active)),
+ ops.map(fn.Binarize(source.uri, self.config.tau_active)),
)
# Add corresponding waveform to the output
if output_waveform:
window_selector = fn.DelayedAggregation(
- self.step, self.latency, strategy="first", stream_end=end_time
+ self.config.step, self.config.latency, strategy="first", stream_end=end_time
)
waveform_stream = regular_stream.pipe(
dops.buffer_slide(window_selector.num_overlapping_windows),
@@ -100,3 +111,14 @@ def from_source(self, source: src.AudioSource, output_waveform: bool = True) ->
return rx.zip(pipeline, waveform_stream)
# No waveform needed, add None for consistency
return pipeline.pipe(ops.map(lambda ann: (ann, None)))
+
+
+class BatchedOnlineSpeakerDiarization:
+ def __init__(self, config: PipelineConfig):
+ self.config = config
+ self.chunk_loader = src.ChunkLoader(
+ self.config.sample_rate, self.config.duration, self.config.step
+ )
+
+ def run_offline(self, file: Union[Text, Path]) -> Annotation:
+ pass
diff --git a/src/diart/sources.py b/src/diart/sources.py
index 3fa11b12..9160939d 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -3,6 +3,7 @@
from queue import SimpleQueue
from typing import Tuple, Text, Optional, Iterable
+import numpy as np
import sounddevice as sd
from einops import rearrange
from pyannote.audio.core.io import Audio, AudioFile
@@ -10,6 +11,39 @@
from rx.subject import Subject
+class ChunkLoader:
+ """Loads an audio file and chunks it according to a given window and step size.
+
+ Parameters
+ ----------
+ sample_rate: int
+ Sample rate to load audio.
+ window_duration: float
+ Duration of the chunk in seconds.
+ step_duration: float
+ Duration of the step between chunks in seconds.
+ """
+
+ def __init__(
+ self,
+ sample_rate: int,
+ window_duration: float,
+ step_duration: float,
+ ):
+ self.audio = Audio(sample_rate, mono=True)
+ self.window_duration = window_duration
+ self.step_duration = step_duration
+ self.window_samples = int(round(window_duration * sample_rate))
+ self.step_samples = int(round(step_duration * sample_rate))
+
+ def get_chunks(self, file: AudioFile) -> np.ndarray:
+ waveform, _ = self.audio(file)
+ return rearrange(
+ waveform.unfold(1, self.window_samples, self.step_samples),
+ "channel chunk frame -> chunk channel frame",
+ ).numpy()
+
+
class AudioSource:
"""Represents a source of audio that can start streaming via the `stream` property.
@@ -91,24 +125,19 @@ def __init__(
step_duration: float,
):
super().__init__(sample_rate)
- self.window_duration = window_duration
- self.step_duration = step_duration
- self.window_samples = int(round(self.window_duration * self.sample_rate))
- self.step_samples = int(round(self.step_duration * self.sample_rate))
+ self.chunk_loader = ChunkLoader(
+ sample_rate, window_duration, step_duration
+ )
@property
def is_regular(self) -> bool:
return True
def iterate(self, file: AudioFile) -> Iterable[SlidingWindowFeature]:
- waveform, _ = self.audio(file)
- chunks = rearrange(
- waveform.unfold(1, self.window_samples, self.step_samples),
- "channel chunk frame -> chunk channel frame",
- ).numpy()
+ chunks = self.chunk_loader.get_chunks(file)
for i, chunk in enumerate(chunks):
w = SlidingWindow(
- start=i * self.step_duration,
+ start=i * self.chunk_loader.step_duration,
duration=self.resolution,
step=self.resolution
)
From 68126c8d2596a70f2e7933ab71f397de7dc71f41 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Fri, 22 Apr 2022 15:59:35 +0200
Subject: [PATCH 03/41] Batchify FrameWiseModel and ChunkWiseModel
---
src/diart/functional.py | 66 +++++++++++++++++++++++++----------------
src/diart/pipelines.py | 22 ++++++++++----
2 files changed, 58 insertions(+), 30 deletions(-)
diff --git a/src/diart/functional.py b/src/diart/functional.py
index 2ad23ee9..b40d30cb 100644
--- a/src/diart/functional.py
+++ b/src/diart/functional.py
@@ -7,27 +7,28 @@
from pyannote.audio.pipelines.utils import PipelineModel, get_model, get_devices
from pyannote.audio.utils.signal import Binarize as PyanBinarize
from pyannote.core import Annotation, Segment, SlidingWindow, SlidingWindowFeature
+from einops import rearrange
from .mapping import SpeakerMap, SpeakerMapBuilder
-Waveform = Union[SlidingWindowFeature, np.ndarray, torch.Tensor]
+TemporalFeatures = Union[SlidingWindowFeature, np.ndarray, torch.Tensor]
-def resolve_waveform(waveform: Waveform) -> torch.Tensor:
- # As torch.Tensor with shape (..., 1, samples)
- if isinstance(waveform, SlidingWindowFeature):
- data = torch.from_numpy(waveform.data).transpose(-1, -2)
- elif isinstance(waveform, np.ndarray):
- data = torch.from_numpy(waveform)
+def resolve_features(features: TemporalFeatures) -> torch.Tensor:
+ # As torch.Tensor with shape (..., channels, frames)
+ if isinstance(features, SlidingWindowFeature):
+ data = torch.from_numpy(features.data).transpose(-1, -2)
+ elif isinstance(features, np.ndarray):
+ data = torch.from_numpy(features)
else:
- data = waveform
+ data = features
# Make sure there's a batch dimension
- msg = "Waveform must be 2D (1, samples) or 3D (batch, 1, samples)"
+ msg = "Temporal features must be 2D or 3D"
assert data.ndim in (2, 3), msg
if data.ndim == 2:
data = data.unsqueeze(0)
- return data
+ return data.float()
class FrameWiseModel:
@@ -38,24 +39,30 @@ def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
device = get_devices(needs=1)[0]
self.model.to(device)
- def __call__(self, waveform: Waveform) -> Union[SlidingWindowFeature, np.ndarray]:
+ def __call__(self, waveform: TemporalFeatures) -> Union[SlidingWindowFeature, np.ndarray]:
with torch.no_grad():
- wave = resolve_waveform(waveform).to(self.model.device)
- output = self.model(wave).cpu().numpy()[0]
+ wave = resolve_features(waveform).to(self.model.device)
+ output = self.model(wave)
+
+ batch_size, num_frames, _ = output.shape
+
+ # Remove batch dimension if batch size is 1
+ if output.shape[0] == 1:
+ output = output[0]
# Wrap if a SlidingWindowFeature was given as input
if isinstance(waveform, SlidingWindowFeature):
# Temporal resolution of the output
- resolution = self.model.specifications.duration / output.shape[0]
+ resolution = self.model.specifications.duration / num_frames
# Temporal shift to keep track of current start time
resolution = SlidingWindow(
start=waveform.sliding_window.start,
duration=resolution,
step=resolution
)
- return SlidingWindowFeature(output, resolution)
+ return SlidingWindowFeature(output.cpu().numpy(), resolution)
- return output
+ return output.transpose(-1, -2).cpu().numpy()
class ChunkWiseModel:
@@ -66,17 +73,24 @@ def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
device = get_devices(needs=1)[0]
self.model.to(device)
- def __call__(self, waveform: SlidingWindowFeature, weights: Optional[SlidingWindowFeature]) -> torch.Tensor:
+ def __call__(self, waveform: TemporalFeatures, weights: Optional[TemporalFeatures]) -> torch.Tensor:
with torch.no_grad():
- chunk = torch.from_numpy(waveform.data.T).float()
- inputs = chunk.unsqueeze(0).to(self.model.device)
+ inputs = resolve_features(waveform).to(self.model.device)
if weights is not None:
- # weights has shape (num_local_speakers, num_frames)
- weights = torch.from_numpy(weights.data.T).float().to(self.model.device)
- inputs = inputs.repeat(weights.shape[0], 1, 1)
- # Shape (num_speakers, emb_dimension)
- output = self.model(inputs, weights=weights).cpu()
- return output
+ weights = resolve_features(weights).to(self.model.device)
+ batch_size, num_speakers, _ = weights.shape
+ inputs = inputs.repeat(1, num_speakers, 1)
+ weights = rearrange(weights, "batch spk frame -> (batch spk) frame")
+ inputs = rearrange(inputs, "batch spk sample -> (batch spk) 1 sample")
+ output = rearrange(
+ self.model(inputs, weights=weights),
+ "(batch spk) feat -> batch spk feat",
+ batch=batch_size,
+ spk=num_speakers
+ )
+ else:
+ output = self.model(inputs)
+ return output.squeeze().cpu()
class OverlappedSpeechPenalty:
@@ -95,6 +109,7 @@ def __init__(self, gamma: float = 3, beta: float = 10):
self.beta = beta
def __call__(self, segmentation: SlidingWindowFeature) -> SlidingWindowFeature:
+ # TODO batchify
weights = torch.from_numpy(segmentation.data).float().T
with torch.no_grad():
probs = torch.softmax(self.beta * weights, dim=0)
@@ -108,6 +123,7 @@ def __init__(self, norm: Union[float, torch.Tensor] = 1):
self.norm = norm
def __call__(self, embeddings: torch.Tensor) -> torch.Tensor:
+ # TODO batchify
if isinstance(self.norm, torch.Tensor):
assert self.norm.shape[0] == embeddings.shape[0]
with torch.no_grad():
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index baed5da2..f424010e 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -1,9 +1,9 @@
-from pathlib import Path
-from typing import Optional, Union, Text
+from typing import Optional
import rx
import rx.operators as ops
from pyannote.audio.pipelines.utils import PipelineModel
+from pyannote.audio.core.io import AudioFile
from pyannote.core import Annotation
from . import functional as fn
@@ -114,11 +114,23 @@ def from_source(self, source: src.AudioSource, output_waveform: bool = True) ->
class BatchedOnlineSpeakerDiarization:
- def __init__(self, config: PipelineConfig):
+ def __init__(self, config: PipelineConfig, batch_size: int = 32):
self.config = config
+ self.batch_size = batch_size
self.chunk_loader = src.ChunkLoader(
self.config.sample_rate, self.config.duration, self.config.step
)
- def run_offline(self, file: Union[Text, Path]) -> Annotation:
- pass
+ def run(self, file: AudioFile) -> Annotation:
+ chunks = self.chunk_loader.get_chunks(file)
+ num_chunks = chunks.shape[0]
+ segmentation, embeddings = [], []
+ for i in range(0, num_chunks, self.batch_size):
+ i_end = i + self.batch_size
+ if i_end > num_chunks:
+ i_end = num_chunks
+ batch = chunks[i:i_end]
+ seg = self.config.segmentation(batch)
+ segmentation.append(seg)
+ # TODO add overlapped speech penalty
+ embeddings.append(self.config.embedding(batch, seg))
From 5ac42e4be2078dc05068492d105d946e94fa33d6 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Mon, 25 Apr 2022 21:04:16 +0200
Subject: [PATCH 04/41] Add batched pipeline implementation
---
src/diart/functional.py | 60 ++++++++++++++++++++---------
src/diart/pipelines.py | 83 +++++++++++++++++++++++++++++++++++++----
2 files changed, 119 insertions(+), 24 deletions(-)
diff --git a/src/diart/functional.py b/src/diart/functional.py
index b40d30cb..a3285858 100644
--- a/src/diart/functional.py
+++ b/src/diart/functional.py
@@ -16,9 +16,21 @@
def resolve_features(features: TemporalFeatures) -> torch.Tensor:
+ """
+ Transform features into a `torch.Tensor` and add batch dimension if missing.
+
+ Parameters
+ ----------
+ features: Union[SlidingWindowFeature, np.ndarray, torch.Tensor]
+ Shape (frames, channels) or (batch, frames, channels)
+
+ Returns
+ -------
+ transformed_features: torch.Tensor, shape (batch, frames, channels)
+ """
# As torch.Tensor with shape (..., channels, frames)
if isinstance(features, SlidingWindowFeature):
- data = torch.from_numpy(features.data).transpose(-1, -2)
+ data = torch.from_numpy(features.data)
elif isinstance(features, np.ndarray):
data = torch.from_numpy(features)
else:
@@ -39,10 +51,10 @@ def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
device = get_devices(needs=1)[0]
self.model.to(device)
- def __call__(self, waveform: TemporalFeatures) -> Union[SlidingWindowFeature, np.ndarray]:
+ def __call__(self, waveform: TemporalFeatures) -> TemporalFeatures:
with torch.no_grad():
- wave = resolve_features(waveform).to(self.model.device)
- output = self.model(wave)
+ wave = rearrange(resolve_features(waveform), "batch sample channel -> batch channel sample")
+ output = self.model(wave.to(self.model.device)).cpu()
batch_size, num_frames, _ = output.shape
@@ -60,9 +72,12 @@ def __call__(self, waveform: TemporalFeatures) -> Union[SlidingWindowFeature, np
duration=resolution,
step=resolution
)
- return SlidingWindowFeature(output.cpu().numpy(), resolution)
+ return SlidingWindowFeature(output.numpy(), resolution)
+
+ if isinstance(waveform, np.ndarray):
+ return output.numpy()
- return output.transpose(-1, -2).cpu().numpy()
+ return output
class ChunkWiseModel:
@@ -76,11 +91,12 @@ def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
def __call__(self, waveform: TemporalFeatures, weights: Optional[TemporalFeatures]) -> torch.Tensor:
with torch.no_grad():
inputs = resolve_features(waveform).to(self.model.device)
+ inputs = rearrange(inputs, "batch sample channel -> batch channel sample")
if weights is not None:
weights = resolve_features(weights).to(self.model.device)
- batch_size, num_speakers, _ = weights.shape
+ batch_size, _, num_speakers = weights.shape
inputs = inputs.repeat(1, num_speakers, 1)
- weights = rearrange(weights, "batch spk frame -> (batch spk) frame")
+ weights = rearrange(weights, "batch frame spk -> (batch spk) frame")
inputs = rearrange(inputs, "batch spk sample -> (batch spk) 1 sample")
output = rearrange(
self.model(inputs, weights=weights),
@@ -108,27 +124,37 @@ def __init__(self, gamma: float = 3, beta: float = 10):
self.gamma = gamma
self.beta = beta
- def __call__(self, segmentation: SlidingWindowFeature) -> SlidingWindowFeature:
- # TODO batchify
- weights = torch.from_numpy(segmentation.data).float().T
+ def __call__(self, segmentation: TemporalFeatures) -> TemporalFeatures:
+ weights = resolve_features(segmentation) # shape (batch, frames, speakers)
with torch.no_grad():
- probs = torch.softmax(self.beta * weights, dim=0)
+ probs = torch.softmax(self.beta * weights, dim=-1)
weights = torch.pow(weights, self.gamma) * torch.pow(probs, self.gamma)
weights[weights < 1e-8] = 1e-8
- return SlidingWindowFeature(weights.T.numpy(), segmentation.sliding_window)
+ if isinstance(segmentation, SlidingWindowFeature):
+ return SlidingWindowFeature(weights.cpu().numpy(), segmentation.sliding_window)
+ if isinstance(segmentation, np.ndarray):
+ return weights.cpu().numpy()
+ return weights
class EmbeddingNormalization:
def __init__(self, norm: Union[float, torch.Tensor] = 1):
self.norm = norm
+ # Add batch dimension if missing
+ if isinstance(self.norm, torch.Tensor) and self.norm.ndim == 2:
+ self.norm = self.norm.unsqueeze(0)
def __call__(self, embeddings: torch.Tensor) -> torch.Tensor:
- # TODO batchify
+ # Add batch dimension if missing
+ if embeddings.ndim == 2:
+ embeddings = embeddings.unsqueeze(0)
if isinstance(self.norm, torch.Tensor):
- assert self.norm.shape[0] == embeddings.shape[0]
+ batch_size1, num_speakers1, _ = self.norm.shape
+ batch_size2, num_speakers2, _ = embeddings.shape
+ assert batch_size1 == batch_size2 and num_speakers1 == num_speakers2
with torch.no_grad():
- norm_embs = self.norm * embeddings / torch.norm(embeddings, p=2, dim=1, keepdim=True)
- return norm_embs
+ norm_embs = self.norm * embeddings / torch.norm(embeddings, p=2, dim=-1, keepdim=True)
+ return norm_embs.squeeze()
class AggregationStrategy:
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index f424010e..73a22b40 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -1,10 +1,13 @@
-from typing import Optional
+from pathlib import Path
+from typing import Optional, Union, Text
+import numpy as np
import rx
import rx.operators as ops
+import torch
+from einops import rearrange
from pyannote.audio.pipelines.utils import PipelineModel
-from pyannote.audio.core.io import AudioFile
-from pyannote.core import Annotation
+from pyannote.core import SlidingWindowFeature, SlidingWindow
from . import functional as fn
from . import operators as dops
@@ -121,8 +124,27 @@ def __init__(self, config: PipelineConfig, batch_size: int = 32):
self.config.sample_rate, self.config.duration, self.config.step
)
- def run(self, file: AudioFile) -> Annotation:
- chunks = self.chunk_loader.get_chunks(file)
+ def run(self, file: Union[Text, Path], output_waveform: bool = False) -> rx.Observable:
+ print("Preprocessing...")
+ file = Path(file)
+ osp = fn.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
+ emb_norm = fn.EmbeddingNormalization(norm=1)
+ clustering = fn.OnlineSpeakerClustering(
+ self.config.tau_active,
+ self.config.rho_update,
+ self.config.delta_new,
+ "cosine",
+ self.config.max_speakers,
+ )
+ end_time = self.chunk_loader.audio.get_duration(file) % self.config.step
+ aggregation = fn.DelayedAggregation(
+ self.config.step, self.config.latency, strategy="hamming", stream_end=end_time
+ )
+
+ chunks = rearrange(
+ self.chunk_loader.get_chunks(file),
+ "chunk channel sample -> chunk sample channel"
+ )
num_chunks = chunks.shape[0]
segmentation, embeddings = [], []
for i in range(0, num_chunks, self.batch_size):
@@ -132,5 +154,52 @@ def run(self, file: AudioFile) -> Annotation:
batch = chunks[i:i_end]
seg = self.config.segmentation(batch)
segmentation.append(seg)
- # TODO add overlapped speech penalty
- embeddings.append(self.config.embedding(batch, seg))
+ embeddings.append(emb_norm(self.config.embedding(batch, osp(seg))))
+ segmentation = np.vstack(segmentation)
+ embeddings = torch.vstack(embeddings)
+ print("Done")
+
+ # Join segmentation and embedding streams to update a background clustering model
+ # while regulating latency and binarizing the output
+ resolution = self.config.duration / segmentation.shape[1]
+ segmentation_stream = rx.range(0, num_chunks).pipe(
+ ops.map(lambda i: SlidingWindowFeature(
+ segmentation[i],
+ SlidingWindow(
+ start=i * self.config.step,
+ duration=resolution,
+ step=resolution,
+ )
+ ))
+ )
+ embedding_stream = rx.range(0, num_chunks).pipe(ops.map(lambda i: embeddings[i]))
+ pipeline = rx.zip(segmentation_stream, embedding_stream).pipe(
+ ops.starmap(clustering),
+ # Buffer 'num_overlapping' sliding chunks with a step of 1 chunk
+ dops.buffer_slide(aggregation.num_overlapping_windows),
+ # Aggregate overlapping output windows
+ ops.map(aggregation),
+ # Binarize output
+ ops.map(fn.Binarize(file.name, self.config.tau_active)),
+ )
+ # Add corresponding waveform to the output
+ if output_waveform:
+ window_selector = fn.DelayedAggregation(
+ self.config.step, self.config.latency, strategy="first", stream_end=end_time
+ )
+ waveform_resolution = 1 / self.config.sample_rate
+ waveform_stream = rx.range(0, num_chunks).pipe(
+ ops.map(lambda i: SlidingWindowFeature(
+ chunks[i],
+ SlidingWindow(
+ start=i * self.config.step,
+ duration=waveform_resolution,
+ step=waveform_resolution,
+ )
+ )),
+ dops.buffer_slide(window_selector.num_overlapping_windows),
+ ops.map(window_selector),
+ )
+ return rx.zip(pipeline, waveform_stream)
+ # No waveform needed, add None for consistency
+ return pipeline.pipe(ops.map(lambda ann: (ann, None)))
From 6aeab0481812dd34a086353f2500c857dcaf6554 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Mon, 25 Apr 2022 22:19:14 +0200
Subject: [PATCH 05/41] Move pre-calculated pipeline to
OnlineSpeakerDiarization.from_file()
---
src/diart/pipelines.py | 160 ++++++++++++++++++++---------------------
1 file changed, 76 insertions(+), 84 deletions(-)
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 73a22b40..9220baf3 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -56,32 +56,19 @@ def get_end_time(self, source: src.AudioSource) -> Optional[float]:
return None
-class OnlineSpeakerDiarization:
+class OnlineSpeakerTracking:
def __init__(self, config: PipelineConfig):
self.config = config
- def from_source(self, source: src.AudioSource, output_waveform: bool = True) -> rx.Observable:
- msg = f"Audio source has sample rate {source.sample_rate}, expected {self.config.sample_rate}"
- assert source.sample_rate == self.config.sample_rate, msg
- # Regularize the stream to a specific chunk duration and step
- regular_stream = source.stream
- if not source.is_regular:
- regular_stream = source.stream.pipe(
- dops.regularize_stream(self.config.duration, self.config.step, source.sample_rate)
- )
- # Branch the stream to calculate chunk segmentation
- segmentation_stream = regular_stream.pipe(
- ops.map(self.config.segmentation)
- )
- # Join audio and segmentation stream to calculate speaker embeddings
- osp = fn.OverlappedSpeechPenalty(gamma=self.config.gamma, beta=self.config.beta)
- embedding_stream = rx.zip(regular_stream, segmentation_stream).pipe(
- ops.starmap(lambda wave, seg: (wave, osp(seg))),
- ops.starmap(self.config.embedding),
- ops.map(fn.EmbeddingNormalization(norm=1))
- )
- # Join segmentation and embedding streams to update a background clustering model
- # while regulating latency and binarizing the output
+ def from_model_streams(
+ self,
+ uri: Text,
+ end_time: Optional[float],
+ segmentation_stream: rx.Observable,
+ embedding_stream: rx.Observable,
+ audio_chunk_stream: Optional[rx.Observable] = None,
+ ) -> rx.Observable:
+ # Initialize clustering and aggregation modules
clustering = fn.OnlineSpeakerClustering(
self.config.tau_active,
self.config.rho_update,
@@ -89,10 +76,13 @@ def from_source(self, source: src.AudioSource, output_waveform: bool = True) ->
"cosine",
self.config.max_speakers,
)
- end_time = self.config.get_end_time(source)
aggregation = fn.DelayedAggregation(
self.config.step, self.config.latency, strategy="hamming", stream_end=end_time
)
+ binarize = fn.Binarize(uri, self.config.tau_active)
+
+ # Join segmentation and embedding streams to update a background clustering model
+ # while regulating latency and binarizing the output
pipeline = rx.zip(segmentation_stream, embedding_stream).pipe(
ops.starmap(clustering),
# Buffer 'num_overlapping' sliding chunks with a step of 1 chunk
@@ -100,14 +90,14 @@ def from_source(self, source: src.AudioSource, output_waveform: bool = True) ->
# Aggregate overlapping output windows
ops.map(aggregation),
# Binarize output
- ops.map(fn.Binarize(source.uri, self.config.tau_active)),
+ ops.map(binarize),
)
# Add corresponding waveform to the output
- if output_waveform:
+ if audio_chunk_stream is not None:
window_selector = fn.DelayedAggregation(
self.config.step, self.config.latency, strategy="first", stream_end=end_time
)
- waveform_stream = regular_stream.pipe(
+ waveform_stream = audio_chunk_stream.pipe(
dops.buffer_slide(window_selector.num_overlapping_windows),
ops.map(window_selector),
)
@@ -116,39 +106,66 @@ def from_source(self, source: src.AudioSource, output_waveform: bool = True) ->
return pipeline.pipe(ops.map(lambda ann: (ann, None)))
-class BatchedOnlineSpeakerDiarization:
- def __init__(self, config: PipelineConfig, batch_size: int = 32):
+class OnlineSpeakerDiarization:
+ def __init__(self, config: PipelineConfig):
self.config = config
- self.batch_size = batch_size
- self.chunk_loader = src.ChunkLoader(
- self.config.sample_rate, self.config.duration, self.config.step
+ self.speaker_tracking = OnlineSpeakerTracking(config)
+
+ def from_source(
+ self,
+ source: src.AudioSource,
+ output_waveform: bool = True
+ ) -> rx.Observable:
+ msg = f"Audio source has sample rate {source.sample_rate}, expected {self.config.sample_rate}"
+ assert source.sample_rate == self.config.sample_rate, msg
+ # Regularize the stream to a specific chunk duration and step
+ regular_stream = source.stream
+ if not source.is_regular:
+ regular_stream = source.stream.pipe(
+ dops.regularize_stream(self.config.duration, self.config.step, source.sample_rate)
+ )
+ # Branch the stream to calculate chunk segmentation
+ segmentation_stream = regular_stream.pipe(ops.map(self.config.segmentation))
+ # Join audio and segmentation stream to calculate speaker embeddings
+ osp = fn.OverlappedSpeechPenalty(gamma=self.config.gamma, beta=self.config.beta)
+ embedding_stream = rx.zip(regular_stream, segmentation_stream).pipe(
+ ops.starmap(lambda wave, seg: (wave, osp(seg))),
+ ops.starmap(self.config.embedding),
+ ops.map(fn.EmbeddingNormalization(norm=1))
+ )
+ end_time = self.config.get_end_time(source)
+ chunk_stream = regular_stream if output_waveform else None
+ return self.speaker_tracking.from_model_streams(
+ source.uri, end_time, segmentation_stream, embedding_stream, chunk_stream
)
- def run(self, file: Union[Text, Path], output_waveform: bool = False) -> rx.Observable:
- print("Preprocessing...")
+ def from_file(
+ self,
+ file: Union[Text, Path],
+ output_waveform: bool = False,
+ batch_size: int = 32
+ ) -> rx.Observable:
+ # Audio file information
file = Path(file)
+ chunk_loader = src.ChunkLoader(
+ self.config.sample_rate, self.config.duration, self.config.step
+ )
+ uri = file.name.split(".")[0]
+ end_time = chunk_loader.audio.get_duration(file) % self.config.step
+
+ # Initialize pipeline modules
osp = fn.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
emb_norm = fn.EmbeddingNormalization(norm=1)
- clustering = fn.OnlineSpeakerClustering(
- self.config.tau_active,
- self.config.rho_update,
- self.config.delta_new,
- "cosine",
- self.config.max_speakers,
- )
- end_time = self.chunk_loader.audio.get_duration(file) % self.config.step
- aggregation = fn.DelayedAggregation(
- self.config.step, self.config.latency, strategy="hamming", stream_end=end_time
- )
+ # Pre-calculate segmentation and embeddings
chunks = rearrange(
- self.chunk_loader.get_chunks(file),
+ chunk_loader.get_chunks(file),
"chunk channel sample -> chunk sample channel"
)
num_chunks = chunks.shape[0]
segmentation, embeddings = [], []
- for i in range(0, num_chunks, self.batch_size):
- i_end = i + self.batch_size
+ for i in range(0, num_chunks, batch_size):
+ i_end = i + batch_size
if i_end > num_chunks:
i_end = num_chunks
batch = chunks[i:i_end]
@@ -157,49 +174,24 @@ def run(self, file: Union[Text, Path], output_waveform: bool = False) -> rx.Obse
embeddings.append(emb_norm(self.config.embedding(batch, osp(seg))))
segmentation = np.vstack(segmentation)
embeddings = torch.vstack(embeddings)
- print("Done")
- # Join segmentation and embedding streams to update a background clustering model
- # while regulating latency and binarizing the output
+ # Stream pre-calculated segmentation, embeddings and chunks
resolution = self.config.duration / segmentation.shape[1]
segmentation_stream = rx.range(0, num_chunks).pipe(
ops.map(lambda i: SlidingWindowFeature(
- segmentation[i],
- SlidingWindow(
- start=i * self.config.step,
- duration=resolution,
- step=resolution,
- )
+ segmentation[i], SlidingWindow(resolution, resolution, i * self.config.step)
))
)
embedding_stream = rx.range(0, num_chunks).pipe(ops.map(lambda i: embeddings[i]))
- pipeline = rx.zip(segmentation_stream, embedding_stream).pipe(
- ops.starmap(clustering),
- # Buffer 'num_overlapping' sliding chunks with a step of 1 chunk
- dops.buffer_slide(aggregation.num_overlapping_windows),
- # Aggregate overlapping output windows
- ops.map(aggregation),
- # Binarize output
- ops.map(fn.Binarize(file.name, self.config.tau_active)),
- )
- # Add corresponding waveform to the output
+ wav_resolution = 1 / self.config.sample_rate
+ chunk_stream = None
if output_waveform:
- window_selector = fn.DelayedAggregation(
- self.config.step, self.config.latency, strategy="first", stream_end=end_time
- )
- waveform_resolution = 1 / self.config.sample_rate
- waveform_stream = rx.range(0, num_chunks).pipe(
+ chunk_stream = rx.range(0, num_chunks).pipe(
ops.map(lambda i: SlidingWindowFeature(
- chunks[i],
- SlidingWindow(
- start=i * self.config.step,
- duration=waveform_resolution,
- step=waveform_resolution,
- )
- )),
- dops.buffer_slide(window_selector.num_overlapping_windows),
- ops.map(window_selector),
+ chunks[i], SlidingWindow(wav_resolution, wav_resolution, i * self.config.step)
+ ))
)
- return rx.zip(pipeline, waveform_stream)
- # No waveform needed, add None for consistency
- return pipeline.pipe(ops.map(lambda ann: (ann, None)))
+
+ return self.speaker_tracking.from_model_streams(
+ uri, end_time, segmentation_stream, embedding_stream, chunk_stream
+ )
From 46dc35374a8983973e7340abd2f6a30ff34331f5 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Mon, 25 Apr 2022 22:31:09 +0200
Subject: [PATCH 06/41] Add argument to skip plotting for faster inference in
demo script
---
src/diart/demo.py | 28 ++++++++++++++++++----------
1 file changed, 18 insertions(+), 10 deletions(-)
diff --git a/src/diart/demo.py b/src/diart/demo.py
index a9c2570e..dacceac6 100644
--- a/src/diart/demo.py
+++ b/src/diart/demo.py
@@ -19,6 +19,7 @@
parser.add_argument("--gamma", default=3, type=float, help="Parameter gamma for overlapped speech penalty")
parser.add_argument("--beta", default=10, type=float, help="Parameter beta for overlapped speech penalty")
parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers")
+parser.add_argument("--no-plot", dest="no_plot", action="store_true", help="Skip plotting for faster inference")
parser.add_argument(
"--output", type=str,
help="Output directory to store the RTTM. Defaults to home directory "
@@ -55,16 +56,23 @@
output_dir = Path("~/").expanduser() if args.output is None else Path(args.output)
audio_source = src.MicrophoneAudioSource(args.sample_rate)
-# Build pipeline from audio source and stream predictions to a real-time plot
-pipeline.from_source(audio_source).pipe(
- ops.do(RTTMWriter(path=output_dir / "output.rttm")),
- dops.buffer_output(
- duration=config.duration,
- step=config.step,
- latency=config.latency,
- sample_rate=audio_source.sample_rate
- ),
-).subscribe(RealTimePlot(config.duration, config.latency))
+# Build pipeline from audio source and stream predictions
+rttm_writer = RTTMWriter(path=output_dir / "output.rttm")
+observable = pipeline.from_source(audio_source)
+if args.no_plot:
+ # Write RTTM file only
+ observable.subscribe(rttm_writer)
+else:
+ # Write RTTM file + buffering and real-time plot
+ observable.pipe(
+ ops.do(rttm_writer),
+ dops.buffer_output(
+ duration=config.duration,
+ step=config.step,
+ latency=config.latency,
+ sample_rate=audio_source.sample_rate
+ ),
+ ).subscribe(RealTimePlot(config.duration, config.latency))
# Read audio source as a stream
if args.source == "microphone":
From 53f0f049c20f4051e27538f85d9622b0d7671473 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Tue, 26 Apr 2022 21:30:11 +0200
Subject: [PATCH 07/41] Remove empty line
---
src/diart/functional.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/diart/functional.py b/src/diart/functional.py
index a3285858..4157df55 100644
--- a/src/diart/functional.py
+++ b/src/diart/functional.py
@@ -1,6 +1,5 @@
from typing import Union, Optional, List, Iterable, Tuple
-from typing_extensions import Literal
-
+from typing_extensions import Literal
import numpy as np
import torch
From 7bb7cda536fe96107c0d2497222a6852a03a3700 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Tue, 26 Apr 2022 23:08:45 +0200
Subject: [PATCH 08/41] Add benchmark script. Add optional verbosity to
from_file(). Add tqdm progress operator. Add chunk number estimation without
loading chunks
---
requirements.txt | 1 +
src/diart/benchmark.py | 60 ++++++++++++++++++++++++++++++++++++++++++
src/diart/operators.py | 19 ++++++++++++-
src/diart/pipelines.py | 20 +++++++++++---
src/diart/sources.py | 6 +++++
5 files changed, 102 insertions(+), 4 deletions(-)
create mode 100644 src/diart/benchmark.py
diff --git a/requirements.txt b/requirements.txt
index 3ac4a51f..184d8a51 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,4 +4,5 @@ rx>=3.2.0
scipy>=1.6.0
sounddevice>=0.4.2
einops>=0.3.0
+tqdm>=4.64.0
git+https://github.com/pyannote/pyannote-audio.git@develop#egg=pyannote-audio
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
new file mode 100644
index 00000000..79f566bf
--- /dev/null
+++ b/src/diart/benchmark.py
@@ -0,0 +1,60 @@
+import argparse
+from pathlib import Path
+
+import diart.operators as dops
+import diart.sources as src
+from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
+from diart.sinks import RTTMWriter
+from tqdm import tqdm
+
+# Define script arguments
+parser = argparse.ArgumentParser()
+parser.add_argument("root", type=str, help="Path to a directory with audio files")
+parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step")
+parser.add_argument("--latency", default=0.5, type=float, help="System latency")
+parser.add_argument("--sample-rate", default=16000, type=int, help="Source sample rate")
+parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active")
+parser.add_argument("--rho", default=0.3, type=float, help="Speech duration threshold rho update")
+parser.add_argument("--delta", default=1, type=float, help="Maximum distance threshold delta new")
+parser.add_argument("--gamma", default=3, type=float, help="Parameter gamma for overlapped speech penalty")
+parser.add_argument("--beta", default=10, type=float, help="Parameter beta for overlapped speech penalty")
+parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers")
+parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation")
+parser.add_argument("--output", type=str, help="Output directory to store the RTTMs. Defaults to `root`")
+args = parser.parse_args()
+
+args.root = Path(args.root)
+msg = "Root argument must be a directory"
+assert args.root.is_dir(), msg
+
+args.output = args.root if args.output is None else Path(args.output)
+msg = "Output argument must be a directory"
+assert args.output.is_dir(), msg
+
+# Define online speaker diarization pipeline
+config = PipelineConfig(
+ step=args.step,
+ latency=args.latency,
+ tau_active=args.tau,
+ rho_update=args.rho,
+ delta_new=args.delta,
+ gamma=args.gamma,
+ beta=args.beta,
+ max_speakers=args.max_speakers,
+)
+pipeline = OnlineSpeakerDiarization(config)
+
+audio_files = list(args.root.expanduser().iterdir())
+pbar = tqdm(total=len(audio_files), unit="file")
+chunk_loader = src.ChunkLoader(config.sample_rate, config.duration, config.step)
+for filepath in audio_files:
+ pbar.set_description(f"Processing {filepath.stem}")
+ num_chunks = chunk_loader.num_chunks(filepath)
+ # TODO run fully online if batch_size < 2
+ pipeline.from_file(filepath, batch_size=args.batch_size, verbose=True).pipe(
+ dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=False)
+ ).subscribe(
+ RTTMWriter(path=args.output / f"{filepath.stem}.rttm")
+ )
+ pbar.update()
+pbar.close()
diff --git a/src/diart/operators.py b/src/diart/operators.py
index c84708d6..23341e4a 100644
--- a/src/diart/operators.py
+++ b/src/diart/operators.py
@@ -1,11 +1,12 @@
from dataclasses import dataclass
-from typing import Callable, Optional, List, Any, Tuple
+from typing import Callable, Optional, List, Any, Tuple, Text
import numpy as np
import rx
from pyannote.core import Annotation, SlidingWindow, SlidingWindowFeature, Segment
from rx import operators as ops
from rx.core import Observable
+from tqdm import tqdm
Operator = Callable[[Observable], Observable]
@@ -280,3 +281,19 @@ def accumulate(
ops.scan(accumulate, OutputAccumulationState.initial()),
ops.map(OutputAccumulationState.to_tuple),
)
+
+
+def progress(
+ desc: Optional[Text] = None,
+ total: Optional[int] = None,
+ unit: Text = "it",
+ leave: bool = True
+) -> Operator:
+ pbar = tqdm(desc=desc, total=total, unit=unit, leave=leave)
+ return rx.pipe(
+ ops.do_action(
+ on_next=lambda _: pbar.update(),
+ on_error=lambda _: pbar.close(),
+ on_completed=lambda: pbar.close(),
+ )
+ )
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 9220baf3..fcf1e146 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -1,3 +1,4 @@
+import math
from pathlib import Path
from typing import Optional, Union, Text
@@ -8,6 +9,7 @@
from einops import rearrange
from pyannote.audio.pipelines.utils import PipelineModel
from pyannote.core import SlidingWindowFeature, SlidingWindow
+from tqdm import tqdm
from . import functional as fn
from . import operators as dops
@@ -29,6 +31,8 @@ def __init__(
beta: float = 10,
max_speakers: int = 20,
):
+ # TODO move models to pipeline
+ # TODO support gpu
self.segmentation = fn.FrameWiseModel(segmentation)
self.duration = duration
if self.duration is None:
@@ -143,7 +147,8 @@ def from_file(
self,
file: Union[Text, Path],
output_waveform: bool = False,
- batch_size: int = 32
+ batch_size: int = 32,
+ verbose: bool = False,
) -> rx.Observable:
# Audio file information
file = Path(file)
@@ -157,14 +162,23 @@ def from_file(
osp = fn.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
emb_norm = fn.EmbeddingNormalization(norm=1)
- # Pre-calculate segmentation and embeddings
+ # Split audio into chunks
chunks = rearrange(
chunk_loader.get_chunks(file),
"chunk channel sample -> chunk sample channel"
)
num_chunks = chunks.shape[0]
+
+ # Set progress if needed
+ iterator = range(0, num_chunks, batch_size)
+ if verbose:
+ desc = "Extracting segmentation and embeddings"
+ total = int(math.ceil(num_chunks / batch_size))
+ iterator = tqdm(iterator, desc=desc, total=total, unit="batch", leave=False)
+
+ # Pre-calculate segmentation and embeddings
segmentation, embeddings = [], []
- for i in range(0, num_chunks, batch_size):
+ for i in iterator:
i_end = i + batch_size
if i_end > num_chunks:
i_end = num_chunks
diff --git a/src/diart/sources.py b/src/diart/sources.py
index 9160939d..a3380ea3 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -37,12 +37,18 @@ def __init__(
self.step_samples = int(round(step_duration * sample_rate))
def get_chunks(self, file: AudioFile) -> np.ndarray:
+ # FIXME last chunk should be padded instead of ignored
waveform, _ = self.audio(file)
return rearrange(
waveform.unfold(1, self.window_samples, self.step_samples),
"channel chunk frame -> chunk channel frame",
).numpy()
+ def num_chunks(self, file: AudioFile) -> int:
+ # FIXME last chunk should be padded instead of ignored
+ num_samples = self.audio.get_duration(file) * self.audio.sample_rate
+ return int((num_samples - self.window_samples + self.step_samples) // self.step_samples)
+
class AudioSource:
"""Represents a source of audio that can start streaming via the `stream` property.
From 5089d524778dfaff155317a0a33d34c632b5425d Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 27 Apr 2022 11:54:36 +0200
Subject: [PATCH 09/41] Dumb down PipelineConfig. Make sample rate completely
depend on the segmentation model. Cleaning and refactoring
---
src/diart/benchmark.py | 3 +-
src/diart/demo.py | 12 ++++----
src/diart/functional.py | 10 ++++++-
src/diart/pipelines.py | 65 +++++++++++++++++++++++------------------
src/diart/sources.py | 13 +++++++++
5 files changed, 66 insertions(+), 37 deletions(-)
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index 79f566bf..9cac2989 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -12,7 +12,6 @@
parser.add_argument("root", type=str, help="Path to a directory with audio files")
parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step")
parser.add_argument("--latency", default=0.5, type=float, help="System latency")
-parser.add_argument("--sample-rate", default=16000, type=int, help="Source sample rate")
parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active")
parser.add_argument("--rho", default=0.3, type=float, help="Speech duration threshold rho update")
parser.add_argument("--delta", default=1, type=float, help="Maximum distance threshold delta new")
@@ -46,7 +45,7 @@
audio_files = list(args.root.expanduser().iterdir())
pbar = tqdm(total=len(audio_files), unit="file")
-chunk_loader = src.ChunkLoader(config.sample_rate, config.duration, config.step)
+chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, config.step)
for filepath in audio_files:
pbar.set_description(f"Processing {filepath.stem}")
num_chunks = chunk_loader.num_chunks(filepath)
diff --git a/src/diart/demo.py b/src/diart/demo.py
index dacceac6..a1e6005e 100644
--- a/src/diart/demo.py
+++ b/src/diart/demo.py
@@ -12,7 +12,6 @@
parser.add_argument("source", type=str, help="Path to an audio file | 'microphone'")
parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step")
parser.add_argument("--latency", default=0.5, type=float, help="System latency")
-parser.add_argument("--sample-rate", default=16000, type=int, help="Source sample rate")
parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active")
parser.add_argument("--rho", default=0.3, type=float, help="Speech duration threshold rho update")
parser.add_argument("--delta", default=1, type=float, help="Maximum distance threshold delta new")
@@ -37,6 +36,7 @@
gamma=args.gamma,
beta=args.beta,
max_speakers=args.max_speakers,
+ device=None, # TODO support GPU
)
pipeline = OnlineSpeakerDiarization(config)
@@ -49,12 +49,12 @@
file=args.source,
uri=uri,
reader=src.RegularAudioFileReader(
- args.sample_rate, config.duration, config.step
+ pipeline.sample_rate, pipeline.duration, config.step
),
)
else:
output_dir = Path("~/").expanduser() if args.output is None else Path(args.output)
- audio_source = src.MicrophoneAudioSource(args.sample_rate)
+ audio_source = src.MicrophoneAudioSource(pipeline.sample_rate)
# Build pipeline from audio source and stream predictions
rttm_writer = RTTMWriter(path=output_dir / "output.rttm")
@@ -67,12 +67,12 @@
observable.pipe(
ops.do(rttm_writer),
dops.buffer_output(
- duration=config.duration,
+ duration=pipeline.duration,
step=config.step,
latency=config.latency,
- sample_rate=audio_source.sample_rate
+ sample_rate=pipeline.sample_rate
),
- ).subscribe(RealTimePlot(config.duration, config.latency))
+ ).subscribe(RealTimePlot(pipeline.duration, config.latency))
# Read audio source as a stream
if args.source == "microphone":
diff --git a/src/diart/functional.py b/src/diart/functional.py
index 4157df55..60ea9e46 100644
--- a/src/diart/functional.py
+++ b/src/diart/functional.py
@@ -50,6 +50,14 @@ def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
device = get_devices(needs=1)[0]
self.model.to(device)
+ @property
+ def sample_rate(self) -> int:
+ return self.model.audio.sample_rate
+
+ @property
+ def duration(self) -> float:
+ return self.model.specifications.duration
+
def __call__(self, waveform: TemporalFeatures) -> TemporalFeatures:
with torch.no_grad():
wave = rearrange(resolve_features(waveform), "batch sample channel -> batch channel sample")
@@ -64,7 +72,7 @@ def __call__(self, waveform: TemporalFeatures) -> TemporalFeatures:
# Wrap if a SlidingWindowFeature was given as input
if isinstance(waveform, SlidingWindowFeature):
# Temporal resolution of the output
- resolution = self.model.specifications.duration / num_frames
+ resolution = self.duration / num_frames
# Temporal shift to keep track of current start time
resolution = SlidingWindow(
start=waveform.sliding_window.start,
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index fcf1e146..fcf927b0 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -30,34 +30,24 @@ def __init__(
gamma: float = 3,
beta: float = 10,
max_speakers: int = 20,
+ device: Optional[torch.device] = None,
):
- # TODO move models to pipeline
- # TODO support gpu
- self.segmentation = fn.FrameWiseModel(segmentation)
- self.duration = duration
- if self.duration is None:
- self.duration = self.segmentation.model.specifications.duration
- self.embedding = fn.ChunkWiseModel(embedding)
+ self.segmentation = segmentation
+ self.embedding = embedding
+ self.requested_duration = duration
self.step = step
self.latency = latency
if self.latency is None:
self.latency = self.step
- assert self.step <= self.latency <= self.duration, "Invalid latency requested"
self.tau_active = tau_active
self.rho_update = rho_update
self.delta_new = delta_new
self.gamma = gamma
self.beta = beta
self.max_speakers = max_speakers
-
- @property
- def sample_rate(self) -> int:
- return self.segmentation.model.audio.sample_rate
-
- def get_end_time(self, source: src.AudioSource) -> Optional[float]:
- if source.duration is not None:
- return source.duration - source.duration % self.step
- return None
+ self.device = device
+ if self.device is None:
+ self.device = torch.device("cpu")
class OnlineSpeakerTracking:
@@ -113,31 +103,49 @@ def from_model_streams(
class OnlineSpeakerDiarization:
def __init__(self, config: PipelineConfig):
self.config = config
+ # TODO support gpu
+ self.segmentation = fn.FrameWiseModel(config.segmentation)
+ self.embedding = fn.ChunkWiseModel(config.embedding)
self.speaker_tracking = OnlineSpeakerTracking(config)
+ msg = "Invalid latency requested"
+ assert self.config.step <= self.config.latency <= self.duration, msg
+
+ @property
+ def sample_rate(self) -> int:
+ """Sample rate expected by the segmentation model"""
+ return self.segmentation.sample_rate
+
+ @property
+ def duration(self) -> float:
+ """Chunk duration (in seconds). Defaults to segmentation model duration"""
+ duration = self.config.requested_duration
+ if duration is None:
+ duration = self.segmentation.duration
+ return duration
def from_source(
self,
source: src.AudioSource,
output_waveform: bool = True
) -> rx.Observable:
- msg = f"Audio source has sample rate {source.sample_rate}, expected {self.config.sample_rate}"
- assert source.sample_rate == self.config.sample_rate, msg
+ msg = f"Audio source has sample rate {source.sample_rate}, expected {self.sample_rate}"
+ assert source.sample_rate == self.sample_rate, msg
# Regularize the stream to a specific chunk duration and step
regular_stream = source.stream
if not source.is_regular:
regular_stream = source.stream.pipe(
- dops.regularize_stream(self.config.duration, self.config.step, source.sample_rate)
+ dops.regularize_stream(self.duration, self.config.step, source.sample_rate)
)
# Branch the stream to calculate chunk segmentation
- segmentation_stream = regular_stream.pipe(ops.map(self.config.segmentation))
+ segmentation_stream = regular_stream.pipe(ops.map(self.segmentation))
# Join audio and segmentation stream to calculate speaker embeddings
osp = fn.OverlappedSpeechPenalty(gamma=self.config.gamma, beta=self.config.beta)
embedding_stream = rx.zip(regular_stream, segmentation_stream).pipe(
ops.starmap(lambda wave, seg: (wave, osp(seg))),
- ops.starmap(self.config.embedding),
+ ops.starmap(self.embedding),
ops.map(fn.EmbeddingNormalization(norm=1))
)
- end_time = self.config.get_end_time(source)
+ end_time = source.end_time(self.config.step)
chunk_stream = regular_stream if output_waveform else None
return self.speaker_tracking.from_model_streams(
source.uri, end_time, segmentation_stream, embedding_stream, chunk_stream
@@ -153,7 +161,7 @@ def from_file(
# Audio file information
file = Path(file)
chunk_loader = src.ChunkLoader(
- self.config.sample_rate, self.config.duration, self.config.step
+ self.sample_rate, self.duration, self.config.step
)
uri = file.name.split(".")[0]
end_time = chunk_loader.audio.get_duration(file) % self.config.step
@@ -183,21 +191,21 @@ def from_file(
if i_end > num_chunks:
i_end = num_chunks
batch = chunks[i:i_end]
- seg = self.config.segmentation(batch)
+ seg = self.segmentation(batch)
segmentation.append(seg)
- embeddings.append(emb_norm(self.config.embedding(batch, osp(seg))))
+ embeddings.append(emb_norm(self.embedding(batch, osp(seg))))
segmentation = np.vstack(segmentation)
embeddings = torch.vstack(embeddings)
# Stream pre-calculated segmentation, embeddings and chunks
- resolution = self.config.duration / segmentation.shape[1]
+ resolution = self.duration / segmentation.shape[1]
segmentation_stream = rx.range(0, num_chunks).pipe(
ops.map(lambda i: SlidingWindowFeature(
segmentation[i], SlidingWindow(resolution, resolution, i * self.config.step)
))
)
embedding_stream = rx.range(0, num_chunks).pipe(ops.map(lambda i: embeddings[i]))
- wav_resolution = 1 / self.config.sample_rate
+ wav_resolution = 1 / self.sample_rate
chunk_stream = None
if output_waveform:
chunk_stream = rx.range(0, num_chunks).pipe(
@@ -206,6 +214,7 @@ def from_file(
))
)
+ # Build speaker tracking pipeline
return self.speaker_tracking.from_model_streams(
uri, end_time, segmentation_stream, embedding_stream, chunk_stream
)
diff --git a/src/diart/sources.py b/src/diart/sources.py
index a3380ea3..a3df1d39 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -77,6 +77,19 @@ def duration(self) -> Optional[float]:
"""The duration of the stream if known. Defaults to None (unknown duration)"""
return None
+ def end_time(self, step: float) -> Optional[float]:
+ """
+ If the duration is known, return the end time of the last chunk
+
+ Parameters
+ ----------
+ step: float
+ Step duration in seconds.
+ """
+ if self.duration is not None:
+ return self.duration - self.duration % step
+ return None
+
def read(self):
"""Start reading the source and yielding samples through the stream"""
raise NotImplementedError
From e02ace3398c5ab56548275c8c45b61f832f18df5 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 27 Apr 2022 14:32:55 +0200
Subject: [PATCH 10/41] Fix segmentation resolution not being adapted to chunk
duration
---
src/diart/functional.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/diart/functional.py b/src/diart/functional.py
index 60ea9e46..d1704f74 100644
--- a/src/diart/functional.py
+++ b/src/diart/functional.py
@@ -72,7 +72,8 @@ def __call__(self, waveform: TemporalFeatures) -> TemporalFeatures:
# Wrap if a SlidingWindowFeature was given as input
if isinstance(waveform, SlidingWindowFeature):
# Temporal resolution of the output
- resolution = self.duration / num_frames
+ duration = wave.shape[-1] / self.sample_rate
+ resolution = duration / num_frames
# Temporal shift to keep track of current start time
resolution = SlidingWindow(
start=waveform.sliding_window.start,
From ecaea2436720b5373d6292c9ff28f805a8fc8536 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 27 Apr 2022 17:12:37 +0200
Subject: [PATCH 11/41] Add DER evaluation to benchmark script. Add
FileAudioSource parameter to emit the current time for profiling
---
src/diart/benchmark.py | 27 ++++++++++++++++++++-------
src/diart/sources.py | 12 ++++++++++--
2 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index 9cac2989..53635658 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -1,15 +1,19 @@
import argparse
from pathlib import Path
+from pyannote.database.util import load_rttm
+from pyannote.metrics.diarization import DiarizationErrorRate
+from tqdm import tqdm
+
import diart.operators as dops
import diart.sources as src
from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
from diart.sinks import RTTMWriter
-from tqdm import tqdm
# Define script arguments
parser = argparse.ArgumentParser()
-parser.add_argument("root", type=str, help="Path to a directory with audio files")
+parser.add_argument("root", type=str, help="Directory with audio files .(wav|flac|m4a|...)")
+parser.add_argument("reference", type=str, help="Directory with RTTM files .rttm")
parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step")
parser.add_argument("--latency", default=0.5, type=float, help="System latency")
parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active")
@@ -23,12 +27,13 @@
args = parser.parse_args()
args.root = Path(args.root)
-msg = "Root argument must be a directory"
-assert args.root.is_dir(), msg
-
+args.reference = Path(args.reference)
args.output = args.root if args.output is None else Path(args.output)
-msg = "Output argument must be a directory"
-assert args.output.is_dir(), msg
+args.output.mkdir(parents=True, exist_ok=True)
+
+assert args.root.is_dir(), "Root argument must be a directory"
+assert args.reference.is_dir(), "Reference argument must be a directory"
+assert args.output.is_dir(), "Output argument must be a directory"
# Define online speaker diarization pipeline
config = PipelineConfig(
@@ -43,6 +48,7 @@
)
pipeline = OnlineSpeakerDiarization(config)
+# Run inference
audio_files = list(args.root.expanduser().iterdir())
pbar = tqdm(total=len(audio_files), unit="file")
chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, config.step)
@@ -57,3 +63,10 @@
)
pbar.update()
pbar.close()
+
+# Run evaluation
+metric = DiarizationErrorRate(collar=0, skip_overlap=False)
+for ref_path in args.reference.iterdir():
+ ref = load_rttm(ref_path).popitem()[1]
+ hyp = load_rttm(args.output / ref_path.name).popitem()[1]
+print(f"Diarization Error Rate: {100 * abs(metric):.1f}")
diff --git a/src/diart/sources.py b/src/diart/sources.py
index a3df1d39..f0205601 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -212,17 +212,22 @@ class FileAudioSource(AudioSource):
Unique identifier of the audio source.
reader: AudioFileReader
Determines how the file will be read.
+ emit_timestamp: bool
+ If True, emit the current time (time.monotonic())
+ alongside the waveform. Defaults to False.
"""
def __init__(
self,
file: AudioFile,
uri: Text,
- reader: AudioFileReader
+ reader: AudioFileReader,
+ emit_timestamp: bool = False
):
super().__init__(uri, reader.sample_rate)
self.reader = reader
self._duration = self.reader.get_duration(file)
self.file = file
+ self.emit_timestamp = emit_timestamp
@property
def is_regular(self) -> bool:
@@ -238,7 +243,10 @@ def read(self):
"""Send each chunk of samples through the stream"""
for waveform in self.reader.iterate(self.file):
try:
- self.stream.on_next(waveform)
+ item = waveform
+ if self.emit_timestamp:
+ item = (waveform, time.monotonic())
+ self.stream.on_next(item)
except Exception as e:
self.stream.on_error(e)
self.stream.on_completed()
From 63525df80eac61886695f16d993cd1c5a0e76087 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 27 Apr 2022 18:26:49 +0200
Subject: [PATCH 12/41] Add optional processing time profiling in
FileAudioSource
---
src/diart/benchmark.py | 32 ++++++++++++++++++++++----------
src/diart/pipelines.py | 4 ++--
src/diart/sources.py | 32 ++++++++++++++++++++++----------
3 files changed, 46 insertions(+), 22 deletions(-)
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index 53635658..6bdd83a8 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -3,7 +3,6 @@
from pyannote.database.util import load_rttm
from pyannote.metrics.diarization import DiarizationErrorRate
-from tqdm import tqdm
import diart.operators as dops
import diart.sources as src
@@ -49,24 +48,37 @@
pipeline = OnlineSpeakerDiarization(config)
# Run inference
-audio_files = list(args.root.expanduser().iterdir())
-pbar = tqdm(total=len(audio_files), unit="file")
chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, config.step)
-for filepath in audio_files:
- pbar.set_description(f"Processing {filepath.stem}")
+for filepath in args.root.expanduser().iterdir():
num_chunks = chunk_loader.num_chunks(filepath)
- # TODO run fully online if batch_size < 2
- pipeline.from_file(filepath, batch_size=args.batch_size, verbose=True).pipe(
- dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=False)
+
+ # Stream fully online if batch size is 1 or lower
+ source = None
+ if args.batch_size < 2:
+ source = src.FileAudioSource(
+ filepath,
+ filepath.stem,
+ src.RegularAudioFileReader(pipeline.sample_rate, pipeline.duration, config.step),
+ # Benchmark the processing time of a single chunk
+ profile=True,
+ )
+ observable = pipeline.from_source(source, output_waveform=False)
+ else:
+ observable = pipeline.from_file(filepath, batch_size=args.batch_size, verbose=True)
+
+ observable.pipe(
+ dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=source is None)
).subscribe(
RTTMWriter(path=args.output / f"{filepath.stem}.rttm")
)
- pbar.update()
-pbar.close()
+
+ if source is not None:
+ source.read()
# Run evaluation
metric = DiarizationErrorRate(collar=0, skip_overlap=False)
for ref_path in args.reference.iterdir():
ref = load_rttm(ref_path).popitem()[1]
hyp = load_rttm(args.output / ref_path.name).popitem()[1]
+ metric(ref, hyp)
print(f"Diarization Error Rate: {100 * abs(metric):.1f}")
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index fcf927b0..b24c2131 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -163,7 +163,7 @@ def from_file(
chunk_loader = src.ChunkLoader(
self.sample_rate, self.duration, self.config.step
)
- uri = file.name.split(".")[0]
+ uri = file.stem
end_time = chunk_loader.audio.get_duration(file) % self.config.step
# Initialize pipeline modules
@@ -180,7 +180,7 @@ def from_file(
# Set progress if needed
iterator = range(0, num_chunks, batch_size)
if verbose:
- desc = "Extracting segmentation and embeddings"
+ desc = f"Pre-calculating {uri}"
total = int(math.ceil(num_chunks / batch_size))
iterator = tqdm(iterator, desc=desc, total=total, unit="batch", leave=False)
diff --git a/src/diart/sources.py b/src/diart/sources.py
index f0205601..80dee123 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -1,7 +1,7 @@
import random
import time
from queue import SimpleQueue
-from typing import Tuple, Text, Optional, Iterable
+from typing import Tuple, Text, Optional, Iterable, List
import numpy as np
import sounddevice as sd
@@ -212,22 +212,21 @@ class FileAudioSource(AudioSource):
Unique identifier of the audio source.
reader: AudioFileReader
Determines how the file will be read.
- emit_timestamp: bool
- If True, emit the current time (time.monotonic())
- alongside the waveform. Defaults to False.
+ profile: bool
+ If True, prints the average processing time of emitting a chunk. Defaults to False.
"""
def __init__(
self,
file: AudioFile,
uri: Text,
reader: AudioFileReader,
- emit_timestamp: bool = False
+ profile: bool = False,
):
super().__init__(uri, reader.sample_rate)
self.reader = reader
self._duration = self.reader.get_duration(file)
self.file = file
- self.emit_timestamp = emit_timestamp
+ self.profile = profile
@property
def is_regular(self) -> bool:
@@ -239,17 +238,30 @@ def duration(self) -> Optional[float]:
"""The duration of a file is known"""
return self._duration
+ def _check_print_time(self, times: List[float]):
+ if self.profile:
+ print(
+ f"File {self.uri}: took {np.mean(times).item():.2f} seconds/chunk "
+ f"(+/- {np.std(times).item():.2f} seconds/chunk) "
+ f"-- based on {len(times)} inputs"
+ )
+
def read(self):
"""Send each chunk of samples through the stream"""
+ times = []
for waveform in self.reader.iterate(self.file):
try:
- item = waveform
- if self.emit_timestamp:
- item = (waveform, time.monotonic())
- self.stream.on_next(item)
+ if self.profile:
+ start_time = time.monotonic()
+ self.stream.on_next(waveform)
+ times.append(time.monotonic() - start_time)
+ else:
+ self.stream.on_next(waveform)
except Exception as e:
+ self._check_print_time(times)
self.stream.on_error(e)
self.stream.on_completed()
+ self._check_print_time(times)
class MicrophoneAudioSource(AudioSource):
From b15db1b7ce391264215620dd278a593fc4fed14b Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 27 Apr 2022 18:33:59 +0200
Subject: [PATCH 13/41] Add GPU support in demo and benchmarking
---
src/diart/benchmark.py | 3 +++
src/diart/demo.py | 7 +++++--
src/diart/pipelines.py | 5 ++---
3 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index 6bdd83a8..d823502f 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -1,6 +1,7 @@
import argparse
from pathlib import Path
+import torch
from pyannote.database.util import load_rttm
from pyannote.metrics.diarization import DiarizationErrorRate
@@ -23,6 +24,7 @@
parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers")
parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation")
parser.add_argument("--output", type=str, help="Output directory to store the RTTMs. Defaults to `root`")
+parser.add_argument("--gpu", dest="gpu", action="store_true", help="Add this flag to run on GPU")
args = parser.parse_args()
args.root = Path(args.root)
@@ -44,6 +46,7 @@
gamma=args.gamma,
beta=args.beta,
max_speakers=args.max_speakers,
+ device=torch.device("cuda") if args.gpu else None,
)
pipeline = OnlineSpeakerDiarization(config)
diff --git a/src/diart/demo.py b/src/diart/demo.py
index a1e6005e..767afdce 100644
--- a/src/diart/demo.py
+++ b/src/diart/demo.py
@@ -1,9 +1,11 @@
import argparse
from pathlib import Path
+import rx.operators as ops
+import torch
+
import diart.operators as dops
import diart.sources as src
-import rx.operators as ops
from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
from diart.sinks import RealTimePlot, RTTMWriter
@@ -19,6 +21,7 @@
parser.add_argument("--beta", default=10, type=float, help="Parameter beta for overlapped speech penalty")
parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers")
parser.add_argument("--no-plot", dest="no_plot", action="store_true", help="Skip plotting for faster inference")
+parser.add_argument("--gpu", dest="gpu", action="store_true", help="Add this flag to run on GPU")
parser.add_argument(
"--output", type=str,
help="Output directory to store the RTTM. Defaults to home directory "
@@ -36,7 +39,7 @@
gamma=args.gamma,
beta=args.beta,
max_speakers=args.max_speakers,
- device=None, # TODO support GPU
+ device=torch.device("cuda") if args.gpu else None,
)
pipeline = OnlineSpeakerDiarization(config)
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index b24c2131..493e2f14 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -103,9 +103,8 @@ def from_model_streams(
class OnlineSpeakerDiarization:
def __init__(self, config: PipelineConfig):
self.config = config
- # TODO support gpu
- self.segmentation = fn.FrameWiseModel(config.segmentation)
- self.embedding = fn.ChunkWiseModel(config.embedding)
+ self.segmentation = fn.FrameWiseModel(config.segmentation, self.config.device)
+ self.embedding = fn.ChunkWiseModel(config.embedding, self.config.device)
self.speaker_tracking = OnlineSpeakerTracking(config)
msg = "Invalid latency requested"
assert self.config.step <= self.config.latency <= self.duration, msg
From dad7153704464dbf491570cdd6498850128c6774 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 27 Apr 2022 18:48:33 +0200
Subject: [PATCH 14/41] Make reference optional in benchmarking script
---
src/diart/benchmark.py | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index d823502f..1adb2da6 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -13,7 +13,7 @@
# Define script arguments
parser = argparse.ArgumentParser()
parser.add_argument("root", type=str, help="Directory with audio files .(wav|flac|m4a|...)")
-parser.add_argument("reference", type=str, help="Directory with RTTM files .rttm")
+parser.add_argument("--reference", type=str, help="Directory with RTTM files .rttm")
parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step")
parser.add_argument("--latency", default=0.5, type=float, help="System latency")
parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active")
@@ -28,14 +28,13 @@
args = parser.parse_args()
args.root = Path(args.root)
-args.reference = Path(args.reference)
+assert args.root.is_dir(), "Root argument must be a directory"
+if args.reference is not None:
+ args.reference = Path(args.reference)
+ assert args.reference.is_dir(), "Reference argument must be a directory"
args.output = args.root if args.output is None else Path(args.output)
args.output.mkdir(parents=True, exist_ok=True)
-assert args.root.is_dir(), "Root argument must be a directory"
-assert args.reference.is_dir(), "Reference argument must be a directory"
-assert args.output.is_dir(), "Output argument must be a directory"
-
# Define online speaker diarization pipeline
config = PipelineConfig(
step=args.step,
@@ -79,9 +78,10 @@
source.read()
# Run evaluation
-metric = DiarizationErrorRate(collar=0, skip_overlap=False)
-for ref_path in args.reference.iterdir():
- ref = load_rttm(ref_path).popitem()[1]
- hyp = load_rttm(args.output / ref_path.name).popitem()[1]
- metric(ref, hyp)
-print(f"Diarization Error Rate: {100 * abs(metric):.1f}")
+if args.reference is not None:
+ metric = DiarizationErrorRate(collar=0, skip_overlap=False)
+ for ref_path in args.reference.iterdir():
+ ref = load_rttm(ref_path).popitem()[1]
+ hyp = load_rttm(args.output / ref_path.name).popitem()[1]
+ metric(ref, hyp)
+ print(f"Diarization Error Rate: {100 * abs(metric):.1f}")
From 8287de971184b00e37b3a0fee01ad094ec5cbf22 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Mon, 2 May 2022 10:32:27 +0200
Subject: [PATCH 15/41] Calculate number of chunks from duration instead of
samples in ChunkLoader
---
src/diart/sources.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/diart/sources.py b/src/diart/sources.py
index 80dee123..e47c59bb 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -46,8 +46,8 @@ def get_chunks(self, file: AudioFile) -> np.ndarray:
def num_chunks(self, file: AudioFile) -> int:
# FIXME last chunk should be padded instead of ignored
- num_samples = self.audio.get_duration(file) * self.audio.sample_rate
- return int((num_samples - self.window_samples + self.step_samples) // self.step_samples)
+ numerator = self.audio.get_duration(file) - self.window_duration + self.step_duration
+ return int(numerator // self.step_duration)
class AudioSource:
From a19041c0ee840ebf967e0cc1a81f8ad01fe9013d Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Tue, 3 May 2022 16:48:05 +0200
Subject: [PATCH 16/41] Fix bug in batched pipeline: an edge case was causing
the batch dimension to disappear. Improve benchmark script help
---
src/diart/benchmark.py | 20 ++++++++++----------
src/diart/pipelines.py | 3 +++
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index 1adb2da6..b64953c9 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -14,16 +14,16 @@
parser = argparse.ArgumentParser()
parser.add_argument("root", type=str, help="Directory with audio files .(wav|flac|m4a|...)")
parser.add_argument("--reference", type=str, help="Directory with RTTM files .rttm")
-parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step")
-parser.add_argument("--latency", default=0.5, type=float, help="System latency")
-parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active")
-parser.add_argument("--rho", default=0.3, type=float, help="Speech duration threshold rho update")
-parser.add_argument("--delta", default=1, type=float, help="Maximum distance threshold delta new")
-parser.add_argument("--gamma", default=3, type=float, help="Parameter gamma for overlapped speech penalty")
-parser.add_argument("--beta", default=10, type=float, help="Parameter beta for overlapped speech penalty")
-parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers")
-parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation")
-parser.add_argument("--output", type=str, help="Output directory to store the RTTMs. Defaults to `root`")
+parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step in seconds. Defaults to 0.5")
+parser.add_argument("--latency", default=0.5, type=float, help="System latency in seconds. Defaults to 0.5")
+parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active in [0,1]. Defaults to 0.5")
+parser.add_argument("--rho", default=0.3, type=float, help="Speech ratio threshold rho update in [0,1]. Defaults to 0.3")
+parser.add_argument("--delta", default=1, type=float, help="Maximum distance threshold delta new in [0,2]. Defaults to 1")
+parser.add_argument("--gamma", default=3, type=float, help="Parameter gamma for overlapped speech penalty. Defaults to 3")
+parser.add_argument("--beta", default=10, type=float, help="Parameter beta for overlapped speech penalty. Defaults to 10")
+parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers. Defaults to 20")
+parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation. If lower than 2, run fully online and estimate real-time latency. Defaults to 32")
+parser.add_argument("--output", type=str, help="Output directory to store RTTM files. Defaults to `root`")
parser.add_argument("--gpu", dest="gpu", action="store_true", help="Add this flag to run on GPU")
args = parser.parse_args()
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 493e2f14..8892b9d4 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -191,6 +191,9 @@ def from_file(
i_end = num_chunks
batch = chunks[i:i_end]
seg = self.segmentation(batch)
+ # Edge case: add batch dimension if i == i_end + 1
+ if seg.ndim == 2:
+ seg.unsqueeze(0)
segmentation.append(seg)
embeddings.append(emb_norm(self.embedding(batch, osp(seg))))
segmentation = np.vstack(segmentation)
From d8362fb60f5c4e80f7fd72357f0aa0d5ad3f7de8 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Tue, 3 May 2022 17:13:46 +0200
Subject: [PATCH 17/41] Fix bug in from_file(): segmentation and embedding
remove batch dimension when batch size is 1
---
src/diart/pipelines.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 8892b9d4..84e5bed8 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -193,9 +193,13 @@ def from_file(
seg = self.segmentation(batch)
# Edge case: add batch dimension if i == i_end + 1
if seg.ndim == 2:
- seg.unsqueeze(0)
+ seg = seg[np.newaxis]
+ emb = emb_norm(self.embedding(batch, osp(seg)))
+ # Edge case: add batch dimension if i == i_end + 1
+ if emb.ndim == 2:
+ emb = emb.unsqueeze(0)
segmentation.append(seg)
- embeddings.append(emb_norm(self.embedding(batch, osp(seg))))
+ embeddings.append(emb)
segmentation = np.vstack(segmentation)
embeddings = torch.vstack(embeddings)
From 4861578e4a77ae630b2847e25c579aae74f27d69 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 4 May 2022 12:13:48 +0200
Subject: [PATCH 18/41] Fix end time bug in batched pipeline
---
src/diart/pipelines.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 84e5bed8..3f33de9e 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -163,7 +163,8 @@ def from_file(
self.sample_rate, self.duration, self.config.step
)
uri = file.stem
- end_time = chunk_loader.audio.get_duration(file) % self.config.step
+ duration = chunk_loader.audio.get_duration(file)
+ end_time = duration - duration % self.config.step
# Initialize pipeline modules
osp = fn.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
From 1830d221c348629b7e976fe8e5770a5948bb9aff Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 4 May 2022 12:21:38 +0200
Subject: [PATCH 19/41] Centralize stream end time calculation
---
src/diart/pipelines.py | 27 +++++++++++++++++++--------
src/diart/sources.py | 13 -------------
2 files changed, 19 insertions(+), 21 deletions(-)
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 3f33de9e..861d4f27 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -49,6 +49,17 @@ def __init__(
if self.device is None:
self.device = torch.device("cpu")
+ def last_chunk_end_time(self, conv_duration: float) -> Optional[float]:
+ """
+ If the duration is known, return the end time of the last chunk
+
+ Parameters
+ ----------
+ conv_duration: float
+ Duration of a conversation in seconds.
+ """
+ return conv_duration - conv_duration % self.step
+
class OnlineSpeakerTracking:
def __init__(self, config: PipelineConfig):
@@ -57,11 +68,14 @@ def __init__(self, config: PipelineConfig):
def from_model_streams(
self,
uri: Text,
- end_time: Optional[float],
+ source_duration: Optional[float],
segmentation_stream: rx.Observable,
embedding_stream: rx.Observable,
audio_chunk_stream: Optional[rx.Observable] = None,
) -> rx.Observable:
+ end_time = None
+ if source_duration is not None:
+ end_time = self.config.last_chunk_end_time(source_duration)
# Initialize clustering and aggregation modules
clustering = fn.OnlineSpeakerClustering(
self.config.tau_active,
@@ -144,10 +158,9 @@ def from_source(
ops.starmap(self.embedding),
ops.map(fn.EmbeddingNormalization(norm=1))
)
- end_time = source.end_time(self.config.step)
chunk_stream = regular_stream if output_waveform else None
return self.speaker_tracking.from_model_streams(
- source.uri, end_time, segmentation_stream, embedding_stream, chunk_stream
+ source.uri, source.duration, segmentation_stream, embedding_stream, chunk_stream
)
def from_file(
@@ -162,9 +175,6 @@ def from_file(
chunk_loader = src.ChunkLoader(
self.sample_rate, self.duration, self.config.step
)
- uri = file.stem
- duration = chunk_loader.audio.get_duration(file)
- end_time = duration - duration % self.config.step
# Initialize pipeline modules
osp = fn.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
@@ -180,7 +190,7 @@ def from_file(
# Set progress if needed
iterator = range(0, num_chunks, batch_size)
if verbose:
- desc = f"Pre-calculating {uri}"
+ desc = f"Pre-calculating {file.stem}"
total = int(math.ceil(num_chunks / batch_size))
iterator = tqdm(iterator, desc=desc, total=total, unit="batch", leave=False)
@@ -222,6 +232,7 @@ def from_file(
)
# Build speaker tracking pipeline
+ duration = chunk_loader.audio.get_duration(file)
return self.speaker_tracking.from_model_streams(
- uri, end_time, segmentation_stream, embedding_stream, chunk_stream
+ file.stem, duration, segmentation_stream, embedding_stream, chunk_stream
)
diff --git a/src/diart/sources.py b/src/diart/sources.py
index e47c59bb..eee19f3d 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -77,19 +77,6 @@ def duration(self) -> Optional[float]:
"""The duration of the stream if known. Defaults to None (unknown duration)"""
return None
- def end_time(self, step: float) -> Optional[float]:
- """
- If the duration is known, return the end time of the last chunk
-
- Parameters
- ----------
- step: float
- Step duration in seconds.
- """
- if self.duration is not None:
- return self.duration - self.duration % step
- return None
-
def read(self):
"""Start reading the source and yielding samples through the stream"""
raise NotImplementedError
From 24dd00954f3ab43b2d8ec587dc6ba89a058378cf Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 4 May 2022 14:40:31 +0200
Subject: [PATCH 20/41] Add diart.benchmark in readme file
---
README.md | 27 +++++++++------------------
1 file changed, 9 insertions(+), 18 deletions(-)
diff --git a/README.md b/README.md
index 317a8142..2f984f00 100644
--- a/README.md
+++ b/README.md
@@ -130,7 +130,10 @@ Awaiting paper publication (ASRU 2021).
![Results table](/table1.png)
-To reproduce the results of the paper, use the following hyper-parameters:
+Diart aims to be lightweight and capable of real-time streaming in practical scenarios.
+Its performance is very close to what is reported in the paper (and sometimes even a bit better).
+
+To obtain the best results, make sure to use the following hyper-parameters:
Dataset | latency | tau | rho | delta
------------|---------|--------|--------|------
@@ -140,28 +143,16 @@ VoxConverse | any | 0.576 | 0.915 | 0.648
DIHARD II | 1s | 0.619 | 0.326 | 0.997
DIHARD II | 5s | 0.555 | 0.422 | 1.517
-For instance, for a DIHARD III configuration:
+`diart.benchmark` can quickly run and evaluate the pipeline, and even measure its real-time latency. For instance, for a DIHARD III configuration:
```shell
-python -m diart.demo /path/to/file.wav --tau=0.555 --rho=0.422 --delta=1.517 --output /output/dir
+python -m diart.benchmark /wav/dir --reference /rttm/dir --tau=0.555 --rho=0.422 --delta=1.517 --output /out/dir
```
-And then to obtain the diarization error rate:
-
-```python
-from pyannote.metrics.diarization import DiarizationErrorRate
-from pyannote.database.util import load_rttm
-
-metric = DiarizationErrorRate()
-hypothesis = load_rttm("/output/dir/output.rttm")
-hypothesis = list(hypothesis.values())[0] # Extract hypothesis from dictionary
-reference = load_rttm("/path/to/reference.rttm")
-reference = list(reference.values())[0] # Extract reference from dictionary
-
-der = metric(reference, hypothesis)
-```
+`diart.benchmark` runs a faster inference and evaluation by pre-calculating model outputs in batches.
+More options about benchmarking can be found by running `python -m diart.benchmark -h`.
-For convenience and to facilitate future comparisons, we also provide the [expected outputs](/expected_outputs) in RTTM format for every entry of Table 1 and Figure 5 in the paper. This includes the VBx offline topline as well as our proposed online approach with latencies 500ms, 1s, 2s, 3s, 4s, and 5s.
+For convenience and to facilitate future comparisons, we also provide the [expected outputs](/expected_outputs) of the paper implementation in RTTM format for every entry of Table 1 and Figure 5. This includes the VBx offline topline as well as our proposed online approach with latencies 500ms, 1s, 2s, 3s, 4s, and 5s.
![Figure 5](/figure5.png)
From 8a80c2d72dd861e26143f868a3640eaefd87db46 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 4 May 2022 17:24:33 +0200
Subject: [PATCH 21/41] Add pyannote.metrics performance report in
diart.benchmark
---
src/diart/benchmark.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index b64953c9..7d20a50d 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -84,4 +84,6 @@
ref = load_rttm(ref_path).popitem()[1]
hyp = load_rttm(args.output / ref_path.name).popitem()[1]
metric(ref, hyp)
- print(f"Diarization Error Rate: {100 * abs(metric):.1f}")
+ print()
+ metric.report(display=True)
+ print()
From b3dfebe536dae00fdc48f8b0d370fd936acf19ac Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 4 May 2022 17:48:46 +0200
Subject: [PATCH 22/41] Add progress bar to demo script
---
src/diart/demo.py | 11 +++++------
src/diart/sources.py | 22 ++++++++++++++++++++--
2 files changed, 25 insertions(+), 8 deletions(-)
diff --git a/src/diart/demo.py b/src/diart/demo.py
index 767afdce..25e5548c 100644
--- a/src/diart/demo.py
+++ b/src/diart/demo.py
@@ -46,11 +46,10 @@
# Manage audio source
if args.source != "microphone":
args.source = Path(args.source).expanduser()
- uri = args.source.name.split(".")[0]
output_dir = args.source.parent if args.output is None else Path(args.output)
audio_source = src.FileAudioSource(
file=args.source,
- uri=uri,
+ uri=args.source.stem,
reader=src.RegularAudioFileReader(
pipeline.sample_rate, pipeline.duration, config.step
),
@@ -60,8 +59,10 @@
audio_source = src.MicrophoneAudioSource(pipeline.sample_rate)
# Build pipeline from audio source and stream predictions
-rttm_writer = RTTMWriter(path=output_dir / "output.rttm")
-observable = pipeline.from_source(audio_source)
+rttm_writer = RTTMWriter(path=output_dir / f"{audio_source.uri}.rttm")
+observable = pipeline.from_source(audio_source).pipe(
+ dops.progress(f"Streaming {audio_source.uri}", total=audio_source.length, leave=True)
+)
if args.no_plot:
# Write RTTM file only
observable.subscribe(rttm_writer)
@@ -78,6 +79,4 @@
).subscribe(RealTimePlot(pipeline.duration, config.latency))
# Read audio source as a stream
-if args.source == "microphone":
- print("Recording...")
audio_source.read()
diff --git a/src/diart/sources.py b/src/diart/sources.py
index eee19f3d..0ab701e0 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -77,6 +77,11 @@ def duration(self) -> Optional[float]:
"""The duration of the stream if known. Defaults to None (unknown duration)"""
return None
+ @property
+ def length(self) -> Optional[int]:
+ """Return the number of audio chunks emitted by this source"""
+ return None
+
def read(self):
"""Start reading the source and yielding samples through the stream"""
raise NotImplementedError
@@ -107,6 +112,9 @@ def is_regular(self) -> bool:
def get_duration(self, file: AudioFile) -> float:
return self.audio.get_duration(file)
+ def get_num_chunks(self, file: AudioFile) -> Optional[int]:
+ return None
+
def iterate(self, file: AudioFile) -> Iterable[SlidingWindowFeature]:
"""Return an iterable over the file's samples"""
raise NotImplementedError
@@ -139,6 +147,10 @@ def __init__(
def is_regular(self) -> bool:
return True
+ def get_num_chunks(self, file: AudioFile) -> Optional[int]:
+ """Return the number of chunks emitted for `file`"""
+ return self.chunk_loader.num_chunks(file)
+
def iterate(self, file: AudioFile) -> Iterable[SlidingWindowFeature]:
chunks = self.chunk_loader.get_chunks(file)
for i, chunk in enumerate(chunks):
@@ -217,14 +229,19 @@ def __init__(
@property
def is_regular(self) -> bool:
- """The regularity depends on the reader"""
+ # The regularity depends on the reader
return self.reader.is_regular
@property
def duration(self) -> Optional[float]:
- """The duration of a file is known"""
+ # The duration of a file is known
return self._duration
+ @property
+ def length(self) -> Optional[int]:
+ # Only the reader can know how many chunks are going to be emitted
+ return self.reader.get_num_chunks(self.file)
+
def _check_print_time(self, times: List[float]):
if self.profile:
print(
@@ -239,6 +256,7 @@ def read(self):
for waveform in self.reader.iterate(self.file):
try:
if self.profile:
+ # Profiling assumes that on_next is blocking
start_time = time.monotonic()
self.stream.on_next(waveform)
times.append(time.monotonic() - start_time)
From 7ecae2551a9b1f952f97d93db209bb105343b553 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 4 May 2022 18:16:55 +0200
Subject: [PATCH 23/41] Fix method docstring
---
src/diart/pipelines.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 861d4f27..f076cda1 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -51,7 +51,7 @@ def __init__(
def last_chunk_end_time(self, conv_duration: float) -> Optional[float]:
"""
- If the duration is known, return the end time of the last chunk
+ Return the end time of the last chunk for a given conversation duration.
Parameters
----------
From 5cdeea23aed84b702798f0e43470a6fbe189b2ac Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Sun, 8 May 2022 18:19:53 +0200
Subject: [PATCH 24/41] Add tqdm requirement to setup.cfg
---
setup.cfg | 1 +
1 file changed, 1 insertion(+)
diff --git a/setup.cfg b/setup.cfg
index 92ea2d4b..fcd08b34 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -26,6 +26,7 @@ install_requires =
scipy>=1.6.0
sounddevice>=0.4.2
einops>=0.3.0
+ tqdm>=4.64.0
pyannote-audio @ git+https://github.com/pyannote/pyannote-audio.git@develop#egg=pyannote-audio
From aa8de083cb219fdd8c9a7ee7cbae7d44c2cf68df Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Sun, 8 May 2022 19:07:14 +0200
Subject: [PATCH 25/41] Rename demo.py to stream.py. Unify script argument
docs. Update README.md accordingly
---
README.md | 24 +++++++++++-------------
src/diart/argdoc.py | 10 ++++++++++
src/diart/benchmark.py | 27 ++++++++++++++-------------
src/diart/{demo.py => stream.py} | 25 +++++++++++--------------
4 files changed, 46 insertions(+), 40 deletions(-)
create mode 100644 src/diart/argdoc.py
rename src/diart/{demo.py => stream.py} (71%)
diff --git a/README.md b/README.md
index 3d1f29a2..98b0d3eb 100644
--- a/README.md
+++ b/README.md
@@ -18,23 +18,21 @@
-## Demo
-
-You can visualize the real-time speaker diarization of an audio stream with the built-in demo script.
+## Getting started
### Stream a recorded conversation
```shell
-python -m diart.demo /path/to/audio.wav
+python -m diart.stream /path/to/audio.wav
```
### Stream from your microphone
```shell
-python -m diart.demo microphone
+python -m diart.stream microphone
```
-See `python -m diart.demo -h` for more information.
+See `python -m diart.stream -h` for more options.
## Build your own pipeline
@@ -142,13 +140,13 @@ Its performance is very close to what is reported in the paper (and sometimes ev
To obtain the best results, make sure to use the following hyper-parameters:
-Dataset | latency | tau | rho | delta
-------------|---------|--------|--------|------
-DIHARD III | any | 0.555 | 0.422 | 1.517
-AMI | any | 0.507 | 0.006 | 1.057
-VoxConverse | any | 0.576 | 0.915 | 0.648
-DIHARD II | 1s | 0.619 | 0.326 | 0.997
-DIHARD II | 5s | 0.555 | 0.422 | 1.517
+| Dataset | latency | tau | rho | delta |
+|-------------|---------|--------|--------|-------|
+| DIHARD III | any | 0.555 | 0.422 | 1.517 |
+| AMI | any | 0.507 | 0.006 | 1.057 |
+| VoxConverse | any | 0.576 | 0.915 | 0.648 |
+| DIHARD II | 1s | 0.619 | 0.326 | 0.997 |
+| DIHARD II | 5s | 0.555 | 0.422 | 1.517 |
`diart.benchmark` can quickly run and evaluate the pipeline, and even measure its real-time latency. For instance, for a DIHARD III configuration:
diff --git a/src/diart/argdoc.py b/src/diart/argdoc.py
new file mode 100644
index 00000000..933f96ca
--- /dev/null
+++ b/src/diart/argdoc.py
@@ -0,0 +1,10 @@
+STEP = "Sliding window step (in seconds)"
+LATENCY = "System latency (in seconds). STEP <= LATENCY <= CHUNK_DURATION"
+TAU = "Probability threshold to consider a speaker as active. 0 <= TAU <= 1"
+RHO = "Speech ratio threshold to decide if centroids are updated with a given speaker. 0 <= RHO <= 1"
+DELTA = "Embedding-to-centroid distance threshold to flag a speaker as known or new. 0 <= DELTA <= 2"
+GAMMA = "Parameter gamma for overlapped speech penalty"
+BETA = "Parameter beta for overlapped speech penalty"
+MAX_SPEAKERS = "Maximum number of speakers"
+GPU = "Run on GPU"
+OUTPUT = "Directory to store the system's output in RTTM format"
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index 7d20a50d..cf0fdc81 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -5,6 +5,7 @@
from pyannote.database.util import load_rttm
from pyannote.metrics.diarization import DiarizationErrorRate
+import diart.argdoc as argdoc
import diart.operators as dops
import diart.sources as src
from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
@@ -12,19 +13,19 @@
# Define script arguments
parser = argparse.ArgumentParser()
-parser.add_argument("root", type=str, help="Directory with audio files .(wav|flac|m4a|...)")
-parser.add_argument("--reference", type=str, help="Directory with RTTM files .rttm")
-parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step in seconds. Defaults to 0.5")
-parser.add_argument("--latency", default=0.5, type=float, help="System latency in seconds. Defaults to 0.5")
-parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active in [0,1]. Defaults to 0.5")
-parser.add_argument("--rho", default=0.3, type=float, help="Speech ratio threshold rho update in [0,1]. Defaults to 0.3")
-parser.add_argument("--delta", default=1, type=float, help="Maximum distance threshold delta new in [0,2]. Defaults to 1")
-parser.add_argument("--gamma", default=3, type=float, help="Parameter gamma for overlapped speech penalty. Defaults to 3")
-parser.add_argument("--beta", default=10, type=float, help="Parameter beta for overlapped speech penalty. Defaults to 10")
-parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers. Defaults to 20")
-parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation. If lower than 2, run fully online and estimate real-time latency. Defaults to 32")
-parser.add_argument("--output", type=str, help="Output directory to store RTTM files. Defaults to `root`")
-parser.add_argument("--gpu", dest="gpu", action="store_true", help="Add this flag to run on GPU")
+parser.add_argument("root", type=str, help="Directory with audio files CONVERSATION.(wav|flac|m4a|...)")
+parser.add_argument("--reference", type=str, help="Optional. Directory with RTTM files CONVERSATION.rttm. Names must match audio files")
+parser.add_argument("--step", default=0.5, type=float, help=f"{argdoc.STEP}. Defaults to 0.5")
+parser.add_argument("--latency", default=0.5, type=float, help=f"{argdoc.LATENCY}. Defaults to 0.5")
+parser.add_argument("--tau", default=0.5, type=float, help=f"{argdoc.TAU}. Defaults to 0.5")
+parser.add_argument("--rho", default=0.3, type=float, help=f"{argdoc.RHO}. Defaults to 0.3")
+parser.add_argument("--delta", default=1, type=float, help=f"{argdoc.DELTA}. Defaults to 1")
+parser.add_argument("--gamma", default=3, type=float, help=f"{argdoc.GAMMA}. Defaults to 3")
+parser.add_argument("--beta", default=10, type=float, help=f"{argdoc.BETA}. Defaults to 10")
+parser.add_argument("--max-speakers", default=20, type=int, help=f"{argdoc.MAX_SPEAKERS}. Defaults to 20")
+parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation. If BATCH_SIZE < 2, run fully online and estimate real-time latency. Defaults to 32")
+parser.add_argument("--gpu", dest="gpu", action="store_true", help=argdoc.GPU)
+parser.add_argument("--output", type=str, help=f"{argdoc.OUTPUT}. Defaults to `root`")
args = parser.parse_args()
args.root = Path(args.root)
diff --git a/src/diart/demo.py b/src/diart/stream.py
similarity index 71%
rename from src/diart/demo.py
rename to src/diart/stream.py
index 25e5548c..253cf272 100644
--- a/src/diart/demo.py
+++ b/src/diart/stream.py
@@ -4,6 +4,7 @@
import rx.operators as ops
import torch
+import diart.argdoc as argdoc
import diart.operators as dops
import diart.sources as src
from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
@@ -12,21 +13,17 @@
# Define script arguments
parser = argparse.ArgumentParser()
parser.add_argument("source", type=str, help="Path to an audio file | 'microphone'")
-parser.add_argument("--step", default=0.5, type=float, help="Source sliding window step")
-parser.add_argument("--latency", default=0.5, type=float, help="System latency")
-parser.add_argument("--tau", default=0.5, type=float, help="Activity threshold tau active")
-parser.add_argument("--rho", default=0.3, type=float, help="Speech duration threshold rho update")
-parser.add_argument("--delta", default=1, type=float, help="Maximum distance threshold delta new")
-parser.add_argument("--gamma", default=3, type=float, help="Parameter gamma for overlapped speech penalty")
-parser.add_argument("--beta", default=10, type=float, help="Parameter beta for overlapped speech penalty")
-parser.add_argument("--max-speakers", default=20, type=int, help="Maximum number of identifiable speakers")
+parser.add_argument("--step", default=0.5, type=float, help=f"{argdoc.STEP}. Defaults to 0.5")
+parser.add_argument("--latency", default=0.5, type=float, help=f"{argdoc.LATENCY}. Defaults to 0.5")
+parser.add_argument("--tau", default=0.5, type=float, help=f"{argdoc.TAU}. Defaults to 0.5")
+parser.add_argument("--rho", default=0.3, type=float, help=f"{argdoc.RHO}. Defaults to 0.3")
+parser.add_argument("--delta", default=1, type=float, help=f"{argdoc.DELTA}. Defaults to 1")
+parser.add_argument("--gamma", default=3, type=float, help=f"{argdoc.GAMMA}. Defaults to 3")
+parser.add_argument("--beta", default=10, type=float, help=f"{argdoc.BETA}. Defaults to 10")
+parser.add_argument("--max-speakers", default=20, type=int, help=f"{argdoc.MAX_SPEAKERS}. Defaults to 20")
parser.add_argument("--no-plot", dest="no_plot", action="store_true", help="Skip plotting for faster inference")
-parser.add_argument("--gpu", dest="gpu", action="store_true", help="Add this flag to run on GPU")
-parser.add_argument(
- "--output", type=str,
- help="Output directory to store the RTTM. Defaults to home directory "
- "if source is microphone or parent directory if source is a file"
-)
+parser.add_argument("--gpu", dest="gpu", action="store_true", help=argdoc.GPU)
+parser.add_argument("--output", type=str, help=f"{argdoc.OUTPUT}. Defaults to home directory if SOURCE == 'microphone' or parent directory if SOURCE is a file")
args = parser.parse_args()
# Define online speaker diarization pipeline
From 1fb18ca7f3752e08fd74ade7536c4c53984fa5be Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Sun, 8 May 2022 19:22:10 +0200
Subject: [PATCH 26/41] Rename functional.py to blocks.py. Rename
FrameWiseModel and ChunkWiseModel to FramewiseModel and ChunkwiseModel
---
README.md | 12 ++++++------
src/diart/{functional.py => blocks.py} | 4 ++--
src/diart/pipelines.py | 22 +++++++++++-----------
3 files changed, 19 insertions(+), 19 deletions(-)
rename src/diart/{functional.py => blocks.py} (99%)
diff --git a/README.md b/README.md
index 98b0d3eb..623c98b5 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ See `python -m diart.stream -h` for more options.
## Build your own pipeline
Diart provides building blocks that can be combined to do speaker diarization on an audio stream.
-The streaming implementation is powered by [RxPY](https://github.com/ReactiveX/RxPY), but the `functional` module is completely independent.
+Streaming is powered by [RxPY](https://github.com/ReactiveX/RxPY), but the `blocks` module is completely independent and can be used separately.
### Example
@@ -48,16 +48,16 @@ import rx
import rx.operators as ops
import diart.operators as myops
from diart.sources import MicrophoneAudioSource
-import diart.functional as fn
+import diart.blocks as blocks
sample_rate = 16000
mic = MicrophoneAudioSource(sample_rate)
# Initialize independent modules
-segmentation = fn.FrameWiseModel("pyannote/segmentation")
-embedding = fn.ChunkWiseModel("pyannote/embedding")
-osp = fn.OverlappedSpeechPenalty(gamma=3, beta=10)
-normalization = fn.EmbeddingNormalization(norm=1)
+segmentation = blocks.FramewiseModel("pyannote/segmentation")
+embedding = blocks.ChunkwiseModel("pyannote/embedding")
+osp = blocks.OverlappedSpeechPenalty(gamma=3, beta=10)
+normalization = blocks.EmbeddingNormalization(norm=1)
# Reformat microphone stream. Defaults to 5s duration and 500ms shift
regular_stream = mic.stream.pipe(myops.regularize_stream(sample_rate))
diff --git a/src/diart/functional.py b/src/diart/blocks.py
similarity index 99%
rename from src/diart/functional.py
rename to src/diart/blocks.py
index d1704f74..2c02623c 100644
--- a/src/diart/functional.py
+++ b/src/diart/blocks.py
@@ -42,7 +42,7 @@ def resolve_features(features: TemporalFeatures) -> torch.Tensor:
return data.float()
-class FrameWiseModel:
+class FramewiseModel:
def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
self.model = get_model(model)
self.model.eval()
@@ -88,7 +88,7 @@ def __call__(self, waveform: TemporalFeatures) -> TemporalFeatures:
return output
-class ChunkWiseModel:
+class ChunkwiseModel:
def __init__(self, model: PipelineModel, device: Optional[torch.device] = None):
self.model = get_model(model)
self.model.eval()
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index f076cda1..919a371e 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -11,7 +11,7 @@
from pyannote.core import SlidingWindowFeature, SlidingWindow
from tqdm import tqdm
-from . import functional as fn
+from . import blocks
from . import operators as dops
from . import sources as src
@@ -77,17 +77,17 @@ def from_model_streams(
if source_duration is not None:
end_time = self.config.last_chunk_end_time(source_duration)
# Initialize clustering and aggregation modules
- clustering = fn.OnlineSpeakerClustering(
+ clustering = blocks.OnlineSpeakerClustering(
self.config.tau_active,
self.config.rho_update,
self.config.delta_new,
"cosine",
self.config.max_speakers,
)
- aggregation = fn.DelayedAggregation(
+ aggregation = blocks.DelayedAggregation(
self.config.step, self.config.latency, strategy="hamming", stream_end=end_time
)
- binarize = fn.Binarize(uri, self.config.tau_active)
+ binarize = blocks.Binarize(uri, self.config.tau_active)
# Join segmentation and embedding streams to update a background clustering model
# while regulating latency and binarizing the output
@@ -102,7 +102,7 @@ def from_model_streams(
)
# Add corresponding waveform to the output
if audio_chunk_stream is not None:
- window_selector = fn.DelayedAggregation(
+ window_selector = blocks.DelayedAggregation(
self.config.step, self.config.latency, strategy="first", stream_end=end_time
)
waveform_stream = audio_chunk_stream.pipe(
@@ -117,8 +117,8 @@ def from_model_streams(
class OnlineSpeakerDiarization:
def __init__(self, config: PipelineConfig):
self.config = config
- self.segmentation = fn.FrameWiseModel(config.segmentation, self.config.device)
- self.embedding = fn.ChunkWiseModel(config.embedding, self.config.device)
+ self.segmentation = blocks.FramewiseModel(config.segmentation, self.config.device)
+ self.embedding = blocks.ChunkwiseModel(config.embedding, self.config.device)
self.speaker_tracking = OnlineSpeakerTracking(config)
msg = "Invalid latency requested"
assert self.config.step <= self.config.latency <= self.duration, msg
@@ -152,11 +152,11 @@ def from_source(
# Branch the stream to calculate chunk segmentation
segmentation_stream = regular_stream.pipe(ops.map(self.segmentation))
# Join audio and segmentation stream to calculate speaker embeddings
- osp = fn.OverlappedSpeechPenalty(gamma=self.config.gamma, beta=self.config.beta)
+ osp = blocks.OverlappedSpeechPenalty(gamma=self.config.gamma, beta=self.config.beta)
embedding_stream = rx.zip(regular_stream, segmentation_stream).pipe(
ops.starmap(lambda wave, seg: (wave, osp(seg))),
ops.starmap(self.embedding),
- ops.map(fn.EmbeddingNormalization(norm=1))
+ ops.map(blocks.EmbeddingNormalization(norm=1))
)
chunk_stream = regular_stream if output_waveform else None
return self.speaker_tracking.from_model_streams(
@@ -177,8 +177,8 @@ def from_file(
)
# Initialize pipeline modules
- osp = fn.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
- emb_norm = fn.EmbeddingNormalization(norm=1)
+ osp = blocks.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
+ emb_norm = blocks.EmbeddingNormalization(norm=1)
# Split audio into chunks
chunks = rearrange(
From 9488368f9f0fc8cb195e9b017788c4be27fbde89 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Sun, 8 May 2022 19:29:30 +0200
Subject: [PATCH 27/41] Update README.md
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 623c98b5..2a73fb32 100644
--- a/README.md
+++ b/README.md
@@ -155,7 +155,7 @@ python -m diart.benchmark /wav/dir --reference /rttm/dir --tau=0.555 --rho=0.422
```
`diart.benchmark` runs a faster inference and evaluation by pre-calculating model outputs in batches.
-More options about benchmarking can be found by running `python -m diart.benchmark -h`.
+See `python -m diart.benchmark -h` for more options.
For convenience and to facilitate future comparisons, we also provide the [expected outputs](/expected_outputs) of the paper implementation in RTTM format for every entry of Table 1 and Figure 5. This includes the VBx offline topline as well as our proposed online approach with latencies 500ms, 1s, 2s, 3s, 4s, and 5s.
From bdcb2427ad8fb1cac64e1bb2144a5d741b8a8734 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Mon, 9 May 2022 17:32:39 +0200
Subject: [PATCH 28/41] Add OverlapAwareSpeakerEmbedding block
---
src/diart/blocks.py | 37 +++++++++++++++++++++++++++++++++++++
src/diart/pipelines.py | 33 +++++++++++++--------------------
2 files changed, 50 insertions(+), 20 deletions(-)
diff --git a/src/diart/blocks.py b/src/diart/blocks.py
index 2c02623c..19b334d5 100644
--- a/src/diart/blocks.py
+++ b/src/diart/blocks.py
@@ -165,6 +165,43 @@ def __call__(self, embeddings: torch.Tensor) -> torch.Tensor:
return norm_embs.squeeze()
+class OverlapAwareSpeakerEmbedding:
+ """
+ Extract overlap-aware speaker embeddings given an audio chunk and its segmentation.
+
+ Parameters
+ ----------
+ model: pyannote.audio.Model, Text or Dict
+ The embedding model. It must take a waveform and weights as input.
+ gamma: float, optional
+ Exponent to lower low-confidence predictions.
+ Defaults to 3.
+ beta: float, optional
+ Softmax's temperature parameter (actually 1/beta) to lower joint speaker activations.
+ Defaults to 10.
+ norm: float or torch.Tensor of shape (batch, speakers, 1) where batch is optional
+ The target norm for the embeddings. It can be different for each speaker.
+ Defaults to 1.
+ device: Optional[torch.device]
+ The device on which to run the embedding model.
+ Defaults to GPU if available or CPU if not.
+ """
+ def __init__(
+ self,
+ model: PipelineModel,
+ gamma: float = 3,
+ beta: float = 10,
+ norm: Union[float, torch.Tensor] = 1,
+ device: Optional[torch.device] = None,
+ ):
+ self.embedding = ChunkwiseModel(model, device)
+ self.osp = OverlappedSpeechPenalty(gamma, beta)
+ self.normalize = EmbeddingNormalization(norm)
+
+ def __call__(self, waveform: TemporalFeatures, segmentation: TemporalFeatures) -> torch.Tensor:
+ return self.normalize(self.embedding(waveform, self.osp(segmentation)))
+
+
class AggregationStrategy:
"""Abstract class representing a strategy to aggregate overlapping buffers"""
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 919a371e..649b6a4f 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -117,11 +117,13 @@ def from_model_streams(
class OnlineSpeakerDiarization:
def __init__(self, config: PipelineConfig):
self.config = config
- self.segmentation = blocks.FramewiseModel(config.segmentation, self.config.device)
- self.embedding = blocks.ChunkwiseModel(config.embedding, self.config.device)
+ self.segmentation = blocks.FramewiseModel(config.segmentation, config.device)
+ self.embedding = blocks.OverlapAwareSpeakerEmbedding(
+ config.embedding, config.gamma, config.beta, norm=1, device=config.device
+ )
self.speaker_tracking = OnlineSpeakerTracking(config)
msg = "Invalid latency requested"
- assert self.config.step <= self.config.latency <= self.duration, msg
+ assert config.step <= config.latency <= self.duration, msg
@property
def sample_rate(self) -> int:
@@ -150,17 +152,12 @@ def from_source(
dops.regularize_stream(self.duration, self.config.step, source.sample_rate)
)
# Branch the stream to calculate chunk segmentation
- segmentation_stream = regular_stream.pipe(ops.map(self.segmentation))
- # Join audio and segmentation stream to calculate speaker embeddings
- osp = blocks.OverlappedSpeechPenalty(gamma=self.config.gamma, beta=self.config.beta)
- embedding_stream = rx.zip(regular_stream, segmentation_stream).pipe(
- ops.starmap(lambda wave, seg: (wave, osp(seg))),
- ops.starmap(self.embedding),
- ops.map(blocks.EmbeddingNormalization(norm=1))
- )
+ seg_stream = regular_stream.pipe(ops.map(self.segmentation))
+ # Join audio and segmentation stream to calculate overlap-aware speaker embeddings
+ emb_stream = rx.zip(regular_stream, seg_stream).pipe(ops.starmap(self.embedding))
chunk_stream = regular_stream if output_waveform else None
return self.speaker_tracking.from_model_streams(
- source.uri, source.duration, segmentation_stream, embedding_stream, chunk_stream
+ source.uri, source.duration, seg_stream, emb_stream, chunk_stream
)
def from_file(
@@ -176,10 +173,6 @@ def from_file(
self.sample_rate, self.duration, self.config.step
)
- # Initialize pipeline modules
- osp = blocks.OverlappedSpeechPenalty(self.config.gamma, self.config.beta)
- emb_norm = blocks.EmbeddingNormalization(norm=1)
-
# Split audio into chunks
chunks = rearrange(
chunk_loader.get_chunks(file),
@@ -205,7 +198,7 @@ def from_file(
# Edge case: add batch dimension if i == i_end + 1
if seg.ndim == 2:
seg = seg[np.newaxis]
- emb = emb_norm(self.embedding(batch, osp(seg)))
+ emb = self.embedding(batch, seg)
# Edge case: add batch dimension if i == i_end + 1
if emb.ndim == 2:
emb = emb.unsqueeze(0)
@@ -216,12 +209,12 @@ def from_file(
# Stream pre-calculated segmentation, embeddings and chunks
resolution = self.duration / segmentation.shape[1]
- segmentation_stream = rx.range(0, num_chunks).pipe(
+ seg_stream = rx.range(0, num_chunks).pipe(
ops.map(lambda i: SlidingWindowFeature(
segmentation[i], SlidingWindow(resolution, resolution, i * self.config.step)
))
)
- embedding_stream = rx.range(0, num_chunks).pipe(ops.map(lambda i: embeddings[i]))
+ emb_stream = rx.range(0, num_chunks).pipe(ops.map(lambda i: embeddings[i]))
wav_resolution = 1 / self.sample_rate
chunk_stream = None
if output_waveform:
@@ -234,5 +227,5 @@ def from_file(
# Build speaker tracking pipeline
duration = chunk_loader.audio.get_duration(file)
return self.speaker_tracking.from_model_streams(
- file.stem, duration, segmentation_stream, embedding_stream, chunk_stream
+ file.stem, duration, seg_stream, emb_stream, chunk_stream
)
From 0510d5a5ebe188d47f37c2d0b2db45c7e9a14c9e Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Mon, 9 May 2022 18:52:40 +0200
Subject: [PATCH 29/41] Update README.md
---
README.md | 26 +++++++++++---------------
1 file changed, 11 insertions(+), 15 deletions(-)
diff --git a/README.md b/README.md
index 2a73fb32..8d65ea50 100644
--- a/README.md
+++ b/README.md
@@ -46,29 +46,25 @@ Obtain overlap-aware speaker embeddings from a microphone stream
```python
import rx
import rx.operators as ops
-import diart.operators as myops
+import diart.operators as dops
from diart.sources import MicrophoneAudioSource
-import diart.blocks as blocks
+from diart.blocks import FramewiseModel, OverlapAwareSpeakerEmbedding
sample_rate = 16000
mic = MicrophoneAudioSource(sample_rate)
# Initialize independent modules
-segmentation = blocks.FramewiseModel("pyannote/segmentation")
-embedding = blocks.ChunkwiseModel("pyannote/embedding")
-osp = blocks.OverlappedSpeechPenalty(gamma=3, beta=10)
-normalization = blocks.EmbeddingNormalization(norm=1)
+segmentation = FramewiseModel("pyannote/segmentation")
+embedding = OverlapAwareSpeakerEmbedding("pyannote/embedding")
# Reformat microphone stream. Defaults to 5s duration and 500ms shift
-regular_stream = mic.stream.pipe(myops.regularize_stream(sample_rate))
+regular_stream = mic.stream.pipe(dops.regularize_stream(sample_rate))
# Branch the microphone stream to calculate segmentation
segmentation_stream = regular_stream.pipe(ops.map(segmentation))
# Join audio and segmentation stream to calculate speaker embeddings
-embedding_stream = rx.zip(regular_stream, segmentation_stream).pipe(
- ops.starmap(lambda wave, seg: (wave, osp(seg))),
- ops.starmap(embedding),
- ops.map(normalization)
-)
+embedding_stream = rx.zip(
+ regular_stream, segmentation_stream
+).pipe(ops.starmap(embedding))
embedding_stream.subscribe(on_next=lambda emb: print(emb.shape))
@@ -89,11 +85,11 @@ torch.Size([4, 512])
1) Create environment:
```shell
-conda create -n diarization python=3.8
-conda activate diarization
+conda create -n diart python=3.8
+conda activate diart
```
-2) Install the latest PyTorch version following the [official instructions](https://pytorch.org/get-started/locally/#start-locally)
+2) [Install PyTorch](https://pytorch.org/get-started/locally/#start-locally)
3) Install pyannote.audio 2.0 (currently in development)
```shell
From c943ef308adbbebf2feab27db0bdd0c2576458ea Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Thu, 12 May 2022 16:50:33 +0200
Subject: [PATCH 30/41] Add Benchmark class
---
requirements.txt | 1 +
src/diart/benchmark.py | 186 +++++++++++++++++++++++------------------
2 files changed, 106 insertions(+), 81 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index 184d8a51..bc7472a7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,5 @@ scipy>=1.6.0
sounddevice>=0.4.2
einops>=0.3.0
tqdm>=4.64.0
+pandas>=1.4.2
git+https://github.com/pyannote/pyannote-audio.git@develop#egg=pyannote-audio
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index cf0fdc81..8dbe9b11 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -1,90 +1,114 @@
-import argparse
from pathlib import Path
+from typing import Union, Text, Optional
-import torch
+import pandas as pd
from pyannote.database.util import load_rttm
from pyannote.metrics.diarization import DiarizationErrorRate
-import diart.argdoc as argdoc
import diart.operators as dops
import diart.sources as src
-from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
+from diart.pipelines import OnlineSpeakerDiarization
from diart.sinks import RTTMWriter
-# Define script arguments
-parser = argparse.ArgumentParser()
-parser.add_argument("root", type=str, help="Directory with audio files CONVERSATION.(wav|flac|m4a|...)")
-parser.add_argument("--reference", type=str, help="Optional. Directory with RTTM files CONVERSATION.rttm. Names must match audio files")
-parser.add_argument("--step", default=0.5, type=float, help=f"{argdoc.STEP}. Defaults to 0.5")
-parser.add_argument("--latency", default=0.5, type=float, help=f"{argdoc.LATENCY}. Defaults to 0.5")
-parser.add_argument("--tau", default=0.5, type=float, help=f"{argdoc.TAU}. Defaults to 0.5")
-parser.add_argument("--rho", default=0.3, type=float, help=f"{argdoc.RHO}. Defaults to 0.3")
-parser.add_argument("--delta", default=1, type=float, help=f"{argdoc.DELTA}. Defaults to 1")
-parser.add_argument("--gamma", default=3, type=float, help=f"{argdoc.GAMMA}. Defaults to 3")
-parser.add_argument("--beta", default=10, type=float, help=f"{argdoc.BETA}. Defaults to 10")
-parser.add_argument("--max-speakers", default=20, type=int, help=f"{argdoc.MAX_SPEAKERS}. Defaults to 20")
-parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation. If BATCH_SIZE < 2, run fully online and estimate real-time latency. Defaults to 32")
-parser.add_argument("--gpu", dest="gpu", action="store_true", help=argdoc.GPU)
-parser.add_argument("--output", type=str, help=f"{argdoc.OUTPUT}. Defaults to `root`")
-args = parser.parse_args()
-
-args.root = Path(args.root)
-assert args.root.is_dir(), "Root argument must be a directory"
-if args.reference is not None:
- args.reference = Path(args.reference)
- assert args.reference.is_dir(), "Reference argument must be a directory"
-args.output = args.root if args.output is None else Path(args.output)
-args.output.mkdir(parents=True, exist_ok=True)
-
-# Define online speaker diarization pipeline
-config = PipelineConfig(
- step=args.step,
- latency=args.latency,
- tau_active=args.tau,
- rho_update=args.rho,
- delta_new=args.delta,
- gamma=args.gamma,
- beta=args.beta,
- max_speakers=args.max_speakers,
- device=torch.device("cuda") if args.gpu else None,
-)
-pipeline = OnlineSpeakerDiarization(config)
-
-# Run inference
-chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, config.step)
-for filepath in args.root.expanduser().iterdir():
- num_chunks = chunk_loader.num_chunks(filepath)
-
- # Stream fully online if batch size is 1 or lower
- source = None
- if args.batch_size < 2:
- source = src.FileAudioSource(
- filepath,
- filepath.stem,
- src.RegularAudioFileReader(pipeline.sample_rate, pipeline.duration, config.step),
- # Benchmark the processing time of a single chunk
- profile=True,
- )
- observable = pipeline.from_source(source, output_waveform=False)
- else:
- observable = pipeline.from_file(filepath, batch_size=args.batch_size, verbose=True)
-
- observable.pipe(
- dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=source is None)
- ).subscribe(
- RTTMWriter(path=args.output / f"{filepath.stem}.rttm")
- )
-
- if source is not None:
- source.read()
-
-# Run evaluation
-if args.reference is not None:
- metric = DiarizationErrorRate(collar=0, skip_overlap=False)
- for ref_path in args.reference.iterdir():
- ref = load_rttm(ref_path).popitem()[1]
- hyp = load_rttm(args.output / ref_path.name).popitem()[1]
- metric(ref, hyp)
- print()
- metric.report(display=True)
- print()
+
+class Benchmark:
+ def __init__(
+ self,
+ speech_path: Union[Text, Path],
+ reference_path: Optional[Union[Text, Path]] = None,
+ output_path: Optional[Union[Text, Path]] = None,
+ ):
+ self.speech_path = Path(speech_path).expanduser()
+ assert self.speech_path.is_dir(), "Speech path must be a directory"
+
+ self.reference_path = reference_path
+ if reference_path is not None:
+ self.reference_path = Path(self.reference_path).expanduser()
+ assert self.reference_path.is_dir(), "Reference path must be a directory"
+
+ if output_path is None:
+ self.output_path = self.speech_path
+ else:
+ self.output_path = Path(output_path).expanduser()
+ self.output_path.mkdir(parents=True, exist_ok=True)
+
+ def __call__(self, pipeline: OnlineSpeakerDiarization, batch_size: int = 32) -> Optional[pd.DataFrame]:
+ # Run inference
+ chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, pipeline.config.step)
+ for filepath in self.speech_path.iterdir():
+ num_chunks = chunk_loader.num_chunks(filepath)
+
+ # Stream fully online if batch size is 1 or lower
+ source = None
+ if batch_size < 2:
+ source = src.FileAudioSource(
+ filepath,
+ filepath.stem,
+ src.RegularAudioFileReader(pipeline.sample_rate, pipeline.duration, pipeline.config.step),
+ # Benchmark the processing time of a single chunk
+ profile=True,
+ )
+ observable = pipeline.from_source(source, output_waveform=False)
+ else:
+ observable = pipeline.from_file(filepath, batch_size=batch_size, verbose=True)
+
+ observable.pipe(
+ dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=source is None)
+ ).subscribe(
+ RTTMWriter(path=self.output_path / f"{filepath.stem}.rttm")
+ )
+
+ if source is not None:
+ source.read()
+
+ # Run evaluation
+ if self.reference_path is not None:
+ metric = DiarizationErrorRate(collar=0, skip_overlap=False)
+ for ref_path in self.reference_path.iterdir():
+ ref = load_rttm(ref_path).popitem()[1]
+ hyp = load_rttm(self.output_path / ref_path.name).popitem()[1]
+ metric(ref, hyp)
+
+ return metric.report(display=True)
+
+
+if __name__ == "__main__":
+ import argparse
+ import torch
+ import diart.argdoc as argdoc
+ from diart.pipelines import PipelineConfig
+
+ # Define script arguments
+ parser = argparse.ArgumentParser()
+ parser.add_argument("root", type=str, help="Directory with audio files CONVERSATION.(wav|flac|m4a|...)")
+ parser.add_argument("--reference", type=str, help="Optional. Directory with RTTM files CONVERSATION.rttm. Names must match audio files")
+ parser.add_argument("--step", default=0.5, type=float, help=f"{argdoc.STEP}. Defaults to 0.5")
+ parser.add_argument("--latency", default=0.5, type=float, help=f"{argdoc.LATENCY}. Defaults to 0.5")
+ parser.add_argument("--tau", default=0.5, type=float, help=f"{argdoc.TAU}. Defaults to 0.5")
+ parser.add_argument("--rho", default=0.3, type=float, help=f"{argdoc.RHO}. Defaults to 0.3")
+ parser.add_argument("--delta", default=1, type=float, help=f"{argdoc.DELTA}. Defaults to 1")
+ parser.add_argument("--gamma", default=3, type=float, help=f"{argdoc.GAMMA}. Defaults to 3")
+ parser.add_argument("--beta", default=10, type=float, help=f"{argdoc.BETA}. Defaults to 10")
+ parser.add_argument("--max-speakers", default=20, type=int, help=f"{argdoc.MAX_SPEAKERS}. Defaults to 20")
+ parser.add_argument("--batch-size", default=32, type=int, help="For segmentation and embedding pre-calculation. If BATCH_SIZE < 2, run fully online and estimate real-time latency. Defaults to 32")
+ parser.add_argument("--gpu", dest="gpu", action="store_true", help=argdoc.GPU)
+ parser.add_argument("--output", type=str, help=f"{argdoc.OUTPUT}. Defaults to `root`")
+ args = parser.parse_args()
+
+ # Set benchmark configuration
+ benchmark = Benchmark(args.root, args.reference, args.output)
+
+ # Define online speaker diarization pipeline
+ pipeline = OnlineSpeakerDiarization(PipelineConfig(
+ step=args.step,
+ latency=args.latency,
+ tau_active=args.tau,
+ rho_update=args.rho,
+ delta_new=args.delta,
+ gamma=args.gamma,
+ beta=args.beta,
+ max_speakers=args.max_speakers,
+ device=torch.device("cuda") if args.gpu else None,
+ ))
+
+ benchmark(pipeline, args.batch_size)
From f0de692577b58ee98cdfd7547f8e7e60aecc2b1d Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Thu, 12 May 2022 17:56:04 +0200
Subject: [PATCH 31/41] Add RealTimeInference class. Minor bug fix in
buffer_output()
---
src/diart/blocks.py | 3 +
src/diart/operators.py | 5 +-
src/diart/stream.py | 147 ++++++++++++++++++++++-------------------
3 files changed, 87 insertions(+), 68 deletions(-)
diff --git a/src/diart/blocks.py b/src/diart/blocks.py
index 19b334d5..2da18939 100644
--- a/src/diart/blocks.py
+++ b/src/diart/blocks.py
@@ -368,6 +368,9 @@ def _prepend_or_append(
)
# Append rest of the outputs
elif self.stream_end is not None and last_buffer.end == self.stream_end:
+ # FIXME instead of appending a larger chunk than expected when latency > step,
+ # keep emitting windows until the signal ends.
+ # This should be fixed at the observable level and not within the aggregation block.
num_frames = output_window.data.shape[0]
last_region = Segment(output_region.start, last_buffer.end)
last_output = buffers[-1].crop(
diff --git a/src/diart/operators.py b/src/diart/operators.py
index 23341e4a..646add85 100644
--- a/src/diart/operators.py
+++ b/src/diart/operators.py
@@ -269,7 +269,10 @@ def accumulate(
else:
# The buffer is full, shift values to the left and copy into last buffer chunk
waveform = np.roll(state.waveform.data, -num_step_samples, axis=0)
- waveform[-num_step_samples:] = value.waveform.data
+ # If running on a file, the online prediction may be shorter depending on the latency
+ # The remaining audio at the end is appended, so value.waveform may be longer than num_step_samples
+ # In that case, we simply ignore the appended samples.
+ waveform[-num_step_samples:] = value.waveform.data[:num_step_samples]
# Wrap waveform in a sliding window feature to include timestamps
window = SlidingWindow(start=start_time, duration=resolution, step=resolution)
diff --git a/src/diart/stream.py b/src/diart/stream.py
index 253cf272..12cfb742 100644
--- a/src/diart/stream.py
+++ b/src/diart/stream.py
@@ -1,79 +1,92 @@
-import argparse
from pathlib import Path
+from typing import Union, Text
import rx.operators as ops
-import torch
-import diart.argdoc as argdoc
import diart.operators as dops
import diart.sources as src
-from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
+from diart.pipelines import OnlineSpeakerDiarization
from diart.sinks import RealTimePlot, RTTMWriter
-# Define script arguments
-parser = argparse.ArgumentParser()
-parser.add_argument("source", type=str, help="Path to an audio file | 'microphone'")
-parser.add_argument("--step", default=0.5, type=float, help=f"{argdoc.STEP}. Defaults to 0.5")
-parser.add_argument("--latency", default=0.5, type=float, help=f"{argdoc.LATENCY}. Defaults to 0.5")
-parser.add_argument("--tau", default=0.5, type=float, help=f"{argdoc.TAU}. Defaults to 0.5")
-parser.add_argument("--rho", default=0.3, type=float, help=f"{argdoc.RHO}. Defaults to 0.3")
-parser.add_argument("--delta", default=1, type=float, help=f"{argdoc.DELTA}. Defaults to 1")
-parser.add_argument("--gamma", default=3, type=float, help=f"{argdoc.GAMMA}. Defaults to 3")
-parser.add_argument("--beta", default=10, type=float, help=f"{argdoc.BETA}. Defaults to 10")
-parser.add_argument("--max-speakers", default=20, type=int, help=f"{argdoc.MAX_SPEAKERS}. Defaults to 20")
-parser.add_argument("--no-plot", dest="no_plot", action="store_true", help="Skip plotting for faster inference")
-parser.add_argument("--gpu", dest="gpu", action="store_true", help=argdoc.GPU)
-parser.add_argument("--output", type=str, help=f"{argdoc.OUTPUT}. Defaults to home directory if SOURCE == 'microphone' or parent directory if SOURCE is a file")
-args = parser.parse_args()
-# Define online speaker diarization pipeline
-config = PipelineConfig(
- step=args.step,
- latency=args.latency,
- tau_active=args.tau,
- rho_update=args.rho,
- delta_new=args.delta,
- gamma=args.gamma,
- beta=args.beta,
- max_speakers=args.max_speakers,
- device=torch.device("cuda") if args.gpu else None,
-)
-pipeline = OnlineSpeakerDiarization(config)
+class RealTimeInference:
+ def __init__(self, output_path: Union[Text, Path], do_plot: bool = True):
+ self.output_path = Path(output_path).expanduser()
+ self.output_path.mkdir(parents=True, exist_ok=True)
+ self.do_plot = do_plot
-# Manage audio source
-if args.source != "microphone":
- args.source = Path(args.source).expanduser()
- output_dir = args.source.parent if args.output is None else Path(args.output)
- audio_source = src.FileAudioSource(
- file=args.source,
- uri=args.source.stem,
- reader=src.RegularAudioFileReader(
- pipeline.sample_rate, pipeline.duration, config.step
- ),
- )
-else:
- output_dir = Path("~/").expanduser() if args.output is None else Path(args.output)
- audio_source = src.MicrophoneAudioSource(pipeline.sample_rate)
+ def __call__(self, pipeline: OnlineSpeakerDiarization, source: src.AudioSource):
+ rttm_writer = RTTMWriter(path=self.output_path / f"{source.uri}.rttm")
+ observable = pipeline.from_source(source).pipe(
+ dops.progress(f"Streaming {source.uri}", total=source.length, leave=True)
+ )
+ if not self.do_plot:
+ # Write RTTM file only
+ observable.subscribe(rttm_writer)
+ else:
+ # Write RTTM file + buffering and real-time plot
+ observable.pipe(
+ ops.do(rttm_writer),
+ dops.buffer_output(
+ duration=pipeline.duration,
+ step=pipeline.config.step,
+ latency=pipeline.config.latency,
+ sample_rate=pipeline.sample_rate
+ ),
+ ).subscribe(RealTimePlot(pipeline.duration, pipeline.config.latency))
+ # Stream audio through the pipeline
+ source.read()
-# Build pipeline from audio source and stream predictions
-rttm_writer = RTTMWriter(path=output_dir / f"{audio_source.uri}.rttm")
-observable = pipeline.from_source(audio_source).pipe(
- dops.progress(f"Streaming {audio_source.uri}", total=audio_source.length, leave=True)
-)
-if args.no_plot:
- # Write RTTM file only
- observable.subscribe(rttm_writer)
-else:
- # Write RTTM file + buffering and real-time plot
- observable.pipe(
- ops.do(rttm_writer),
- dops.buffer_output(
- duration=pipeline.duration,
- step=config.step,
- latency=config.latency,
- sample_rate=pipeline.sample_rate
- ),
- ).subscribe(RealTimePlot(pipeline.duration, config.latency))
-# Read audio source as a stream
-audio_source.read()
+if __name__ == "__main__":
+ import argparse
+ import torch
+ import diart.argdoc as argdoc
+ from diart.pipelines import PipelineConfig
+
+ # Define script arguments
+ parser = argparse.ArgumentParser()
+ parser.add_argument("source", type=str, help="Path to an audio file | 'microphone'")
+ parser.add_argument("--step", default=0.5, type=float, help=f"{argdoc.STEP}. Defaults to 0.5")
+ parser.add_argument("--latency", default=0.5, type=float, help=f"{argdoc.LATENCY}. Defaults to 0.5")
+ parser.add_argument("--tau", default=0.5, type=float, help=f"{argdoc.TAU}. Defaults to 0.5")
+ parser.add_argument("--rho", default=0.3, type=float, help=f"{argdoc.RHO}. Defaults to 0.3")
+ parser.add_argument("--delta", default=1, type=float, help=f"{argdoc.DELTA}. Defaults to 1")
+ parser.add_argument("--gamma", default=3, type=float, help=f"{argdoc.GAMMA}. Defaults to 3")
+ parser.add_argument("--beta", default=10, type=float, help=f"{argdoc.BETA}. Defaults to 10")
+ parser.add_argument("--max-speakers", default=20, type=int, help=f"{argdoc.MAX_SPEAKERS}. Defaults to 20")
+ parser.add_argument("--no-plot", dest="no_plot", action="store_true", help="Skip plotting for faster inference")
+ parser.add_argument("--gpu", dest="gpu", action="store_true", help=argdoc.GPU)
+ parser.add_argument("--output", type=str, help=f"{argdoc.OUTPUT}. Defaults to home directory if SOURCE == 'microphone' or parent directory if SOURCE is a file")
+ args = parser.parse_args()
+
+ # Define online speaker diarization pipeline
+ pipeline = OnlineSpeakerDiarization(PipelineConfig(
+ step=args.step,
+ latency=args.latency,
+ tau_active=args.tau,
+ rho_update=args.rho,
+ delta_new=args.delta,
+ gamma=args.gamma,
+ beta=args.beta,
+ max_speakers=args.max_speakers,
+ device=torch.device("cuda") if args.gpu else None,
+ ))
+
+ # Manage audio source
+ if args.source != "microphone":
+ args.source = Path(args.source).expanduser()
+ args.output = args.source.parent if args.output is None else Path(args.output)
+ audio_source = src.FileAudioSource(
+ file=args.source,
+ uri=args.source.stem,
+ reader=src.RegularAudioFileReader(
+ pipeline.sample_rate, pipeline.duration, pipeline.config.step
+ ),
+ )
+ else:
+ args.output = Path("~/").expanduser() if args.output is None else Path(args.output)
+ audio_source = src.MicrophoneAudioSource(pipeline.sample_rate)
+
+ # Run online inference
+ RealTimeInference(args.output, do_plot=not args.no_plot)(pipeline, audio_source)
From 091cbe22749289e60698c8b4bfe948e925c82532 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Thu, 12 May 2022 18:09:17 +0200
Subject: [PATCH 32/41] Add 'inference' module containing RealTimeInference and
Benchmark
---
src/diart/benchmark.py | 81 ++------------------------------
src/diart/inference.py | 102 +++++++++++++++++++++++++++++++++++++++++
src/diart/stream.py | 45 ++----------------
3 files changed, 112 insertions(+), 116 deletions(-)
create mode 100644 src/diart/inference.py
diff --git a/src/diart/benchmark.py b/src/diart/benchmark.py
index 8dbe9b11..fc07f754 100644
--- a/src/diart/benchmark.py
+++ b/src/diart/benchmark.py
@@ -1,83 +1,12 @@
-from pathlib import Path
-from typing import Union, Text, Optional
+import argparse
-import pandas as pd
-from pyannote.database.util import load_rttm
-from pyannote.metrics.diarization import DiarizationErrorRate
-
-import diart.operators as dops
-import diart.sources as src
-from diart.pipelines import OnlineSpeakerDiarization
-from diart.sinks import RTTMWriter
-
-
-class Benchmark:
- def __init__(
- self,
- speech_path: Union[Text, Path],
- reference_path: Optional[Union[Text, Path]] = None,
- output_path: Optional[Union[Text, Path]] = None,
- ):
- self.speech_path = Path(speech_path).expanduser()
- assert self.speech_path.is_dir(), "Speech path must be a directory"
-
- self.reference_path = reference_path
- if reference_path is not None:
- self.reference_path = Path(self.reference_path).expanduser()
- assert self.reference_path.is_dir(), "Reference path must be a directory"
-
- if output_path is None:
- self.output_path = self.speech_path
- else:
- self.output_path = Path(output_path).expanduser()
- self.output_path.mkdir(parents=True, exist_ok=True)
-
- def __call__(self, pipeline: OnlineSpeakerDiarization, batch_size: int = 32) -> Optional[pd.DataFrame]:
- # Run inference
- chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, pipeline.config.step)
- for filepath in self.speech_path.iterdir():
- num_chunks = chunk_loader.num_chunks(filepath)
-
- # Stream fully online if batch size is 1 or lower
- source = None
- if batch_size < 2:
- source = src.FileAudioSource(
- filepath,
- filepath.stem,
- src.RegularAudioFileReader(pipeline.sample_rate, pipeline.duration, pipeline.config.step),
- # Benchmark the processing time of a single chunk
- profile=True,
- )
- observable = pipeline.from_source(source, output_waveform=False)
- else:
- observable = pipeline.from_file(filepath, batch_size=batch_size, verbose=True)
-
- observable.pipe(
- dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=source is None)
- ).subscribe(
- RTTMWriter(path=self.output_path / f"{filepath.stem}.rttm")
- )
-
- if source is not None:
- source.read()
-
- # Run evaluation
- if self.reference_path is not None:
- metric = DiarizationErrorRate(collar=0, skip_overlap=False)
- for ref_path in self.reference_path.iterdir():
- ref = load_rttm(ref_path).popitem()[1]
- hyp = load_rttm(self.output_path / ref_path.name).popitem()[1]
- metric(ref, hyp)
-
- return metric.report(display=True)
+import torch
+import diart.argdoc as argdoc
+from diart.inference import Benchmark
+from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
if __name__ == "__main__":
- import argparse
- import torch
- import diart.argdoc as argdoc
- from diart.pipelines import PipelineConfig
-
# Define script arguments
parser = argparse.ArgumentParser()
parser.add_argument("root", type=str, help="Directory with audio files CONVERSATION.(wav|flac|m4a|...)")
diff --git a/src/diart/inference.py b/src/diart/inference.py
new file mode 100644
index 00000000..7078da45
--- /dev/null
+++ b/src/diart/inference.py
@@ -0,0 +1,102 @@
+from pathlib import Path
+from typing import Union, Text, Optional
+
+import pandas as pd
+import rx.operators as ops
+from pyannote.database.util import load_rttm
+from pyannote.metrics.diarization import DiarizationErrorRate
+
+import diart.operators as dops
+import diart.sources as src
+from diart.pipelines import OnlineSpeakerDiarization
+from diart.sinks import RTTMWriter, RealTimePlot
+
+
+class RealTimeInference:
+ def __init__(self, output_path: Union[Text, Path], do_plot: bool = True):
+ self.output_path = Path(output_path).expanduser()
+ self.output_path.mkdir(parents=True, exist_ok=True)
+ self.do_plot = do_plot
+
+ def __call__(self, pipeline: OnlineSpeakerDiarization, source: src.AudioSource):
+ rttm_writer = RTTMWriter(path=self.output_path / f"{source.uri}.rttm")
+ observable = pipeline.from_source(source).pipe(
+ dops.progress(f"Streaming {source.uri}", total=source.length, leave=True)
+ )
+ if not self.do_plot:
+ # Write RTTM file only
+ observable.subscribe(rttm_writer)
+ else:
+ # Write RTTM file + buffering and real-time plot
+ observable.pipe(
+ ops.do(rttm_writer),
+ dops.buffer_output(
+ duration=pipeline.duration,
+ step=pipeline.config.step,
+ latency=pipeline.config.latency,
+ sample_rate=pipeline.sample_rate
+ ),
+ ).subscribe(RealTimePlot(pipeline.duration, pipeline.config.latency))
+ # Stream audio through the pipeline
+ source.read()
+
+
+class Benchmark:
+ def __init__(
+ self,
+ speech_path: Union[Text, Path],
+ reference_path: Optional[Union[Text, Path]] = None,
+ output_path: Optional[Union[Text, Path]] = None,
+ ):
+ self.speech_path = Path(speech_path).expanduser()
+ assert self.speech_path.is_dir(), "Speech path must be a directory"
+
+ self.reference_path = reference_path
+ if reference_path is not None:
+ self.reference_path = Path(self.reference_path).expanduser()
+ assert self.reference_path.is_dir(), "Reference path must be a directory"
+
+ if output_path is None:
+ self.output_path = self.speech_path
+ else:
+ self.output_path = Path(output_path).expanduser()
+ self.output_path.mkdir(parents=True, exist_ok=True)
+
+ def __call__(self, pipeline: OnlineSpeakerDiarization, batch_size: int = 32) -> Optional[pd.DataFrame]:
+ # Run inference
+ chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, pipeline.config.step)
+ for filepath in self.speech_path.iterdir():
+ num_chunks = chunk_loader.num_chunks(filepath)
+
+ # Stream fully online if batch size is 1 or lower
+ source = None
+ if batch_size < 2:
+ source = src.FileAudioSource(
+ filepath,
+ filepath.stem,
+ src.RegularAudioFileReader(pipeline.sample_rate, pipeline.duration, pipeline.config.step),
+ # Benchmark the processing time of a single chunk
+ profile=True,
+ )
+ observable = pipeline.from_source(source, output_waveform=False)
+ else:
+ observable = pipeline.from_file(filepath, batch_size=batch_size, verbose=True)
+
+ observable.pipe(
+ dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=source is None)
+ ).subscribe(
+ RTTMWriter(path=self.output_path / f"{filepath.stem}.rttm")
+ )
+
+ if source is not None:
+ source.read()
+
+ # Run evaluation
+ if self.reference_path is not None:
+ metric = DiarizationErrorRate(collar=0, skip_overlap=False)
+ for ref_path in self.reference_path.iterdir():
+ ref = load_rttm(ref_path).popitem()[1]
+ hyp = load_rttm(self.output_path / ref_path.name).popitem()[1]
+ metric(ref, hyp)
+
+ return metric.report(display=True)
diff --git a/src/diart/stream.py b/src/diart/stream.py
index 12cfb742..d98d2c8f 100644
--- a/src/diart/stream.py
+++ b/src/diart/stream.py
@@ -1,49 +1,14 @@
+import argparse
from pathlib import Path
-from typing import Union, Text
-import rx.operators as ops
+import torch
-import diart.operators as dops
+import diart.argdoc as argdoc
import diart.sources as src
-from diart.pipelines import OnlineSpeakerDiarization
-from diart.sinks import RealTimePlot, RTTMWriter
-
-
-class RealTimeInference:
- def __init__(self, output_path: Union[Text, Path], do_plot: bool = True):
- self.output_path = Path(output_path).expanduser()
- self.output_path.mkdir(parents=True, exist_ok=True)
- self.do_plot = do_plot
-
- def __call__(self, pipeline: OnlineSpeakerDiarization, source: src.AudioSource):
- rttm_writer = RTTMWriter(path=self.output_path / f"{source.uri}.rttm")
- observable = pipeline.from_source(source).pipe(
- dops.progress(f"Streaming {source.uri}", total=source.length, leave=True)
- )
- if not self.do_plot:
- # Write RTTM file only
- observable.subscribe(rttm_writer)
- else:
- # Write RTTM file + buffering and real-time plot
- observable.pipe(
- ops.do(rttm_writer),
- dops.buffer_output(
- duration=pipeline.duration,
- step=pipeline.config.step,
- latency=pipeline.config.latency,
- sample_rate=pipeline.sample_rate
- ),
- ).subscribe(RealTimePlot(pipeline.duration, pipeline.config.latency))
- # Stream audio through the pipeline
- source.read()
-
+from diart.inference import RealTimeInference
+from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
if __name__ == "__main__":
- import argparse
- import torch
- import diart.argdoc as argdoc
- from diart.pipelines import PipelineConfig
-
# Define script arguments
parser = argparse.ArgumentParser()
parser.add_argument("source", type=str, help="Path to an audio file | 'microphone'")
From db3fd9dc3df9584822269acd962f4d376e5330a8 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Fri, 13 May 2022 10:01:50 +0200
Subject: [PATCH 33/41] Add docstrings to RealTimeInference and Benchmark
---
src/diart/inference.py | 67 ++++++++++++++++++++++++++++++++++++++++--
1 file changed, 64 insertions(+), 3 deletions(-)
diff --git a/src/diart/inference.py b/src/diart/inference.py
index 7078da45..ad10afba 100644
--- a/src/diart/inference.py
+++ b/src/diart/inference.py
@@ -3,6 +3,7 @@
import pandas as pd
import rx.operators as ops
+from pyannote.core import Annotation
from pyannote.database.util import load_rttm
from pyannote.metrics.diarization import DiarizationErrorRate
@@ -13,13 +14,40 @@
class RealTimeInference:
+ """
+ Streams an audio source to an online speaker diarization pipeline.
+ It writes predictions to an output directory in RTTM format, and also plot predictions in real time.
+
+ Parameters
+ ----------
+ output_path: Text or Path
+ Output directory to store predictions in RTTM format.
+ do_plot: bool
+ Whether to draw predictions in a moving plot. Defaults to True.
+ """
def __init__(self, output_path: Union[Text, Path], do_plot: bool = True):
self.output_path = Path(output_path).expanduser()
self.output_path.mkdir(parents=True, exist_ok=True)
self.do_plot = do_plot
- def __call__(self, pipeline: OnlineSpeakerDiarization, source: src.AudioSource):
- rttm_writer = RTTMWriter(path=self.output_path / f"{source.uri}.rttm")
+ def __call__(self, pipeline: OnlineSpeakerDiarization, source: src.AudioSource) -> Annotation:
+ """
+ Stream audio chunks from `source` to `pipeline` and write predictions to disk.
+
+ Parameters
+ ----------
+ pipeline: OnlineSpeakerDiarization
+ Configured speaker diarization pipeline.
+ source: AudioSource
+ Audio source to be read and streamed.
+
+ Returns
+ -------
+ predictions: Annotation
+ Speaker diarization pipeline predictions
+ """
+ rttm_path = self.output_path / f"{source.uri}.rttm"
+ rttm_writer = RTTMWriter(path=rttm_path)
observable = pipeline.from_source(source).pipe(
dops.progress(f"Streaming {source.uri}", total=source.length, leave=True)
)
@@ -40,8 +68,25 @@ def __call__(self, pipeline: OnlineSpeakerDiarization, source: src.AudioSource):
# Stream audio through the pipeline
source.read()
+ return load_rttm(rttm_path)[source.uri]
+
class Benchmark:
+ """
+ Run an online speaker diarization pipeline on a set of audio files in batches.
+ It writes predictions to a given output directory.
+
+ If the reference is given, it calculates the average diarization error rate.
+
+ Parameters
+ ----------
+ speech_path: Text or Path
+ Directory with audio files.
+ reference_path: Text or Path
+ Directory with reference RTTM files (same names as audio files).
+ output_path: Text or Path
+ Output directory to store predictions in RTTM format.
+ """
def __init__(
self,
speech_path: Union[Text, Path],
@@ -63,7 +108,23 @@ def __init__(
self.output_path.mkdir(parents=True, exist_ok=True)
def __call__(self, pipeline: OnlineSpeakerDiarization, batch_size: int = 32) -> Optional[pd.DataFrame]:
- # Run inference
+ """
+ Run a given pipeline on a set of audio files using
+ pre-calculated segmentation and embeddings in batches.
+
+ Parameters
+ ----------
+ pipeline: OnlineSpeakerDiarization
+ Configured speaker diarization pipeline.
+ batch_size: int
+ Batch size. Defaults to 32.
+
+ Returns
+ -------
+ performance: pandas.DataFrame, optional
+ DataFrame with detailed performance on each file, as well as average performance.
+ None if the reference is not provided.
+ """
chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, pipeline.config.step)
for filepath in self.speech_path.iterdir():
num_chunks = chunk_loader.num_chunks(filepath)
From f450455470c962a10731ff7951a516471e47169d Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Sat, 14 May 2022 16:31:30 +0200
Subject: [PATCH 34/41] Use last incomplete chunk with padding
---
src/diart/sources.py | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/src/diart/sources.py b/src/diart/sources.py
index 0ab701e0..472c2a09 100644
--- a/src/diart/sources.py
+++ b/src/diart/sources.py
@@ -37,17 +37,23 @@ def __init__(
self.step_samples = int(round(step_duration * sample_rate))
def get_chunks(self, file: AudioFile) -> np.ndarray:
- # FIXME last chunk should be padded instead of ignored
waveform, _ = self.audio(file)
- return rearrange(
+ _, num_samples = waveform.shape
+ chunks = rearrange(
waveform.unfold(1, self.window_samples, self.step_samples),
"channel chunk frame -> chunk channel frame",
).numpy()
+ # Add padded last chunk
+ if num_samples - self.window_samples % self.step_samples > 0:
+ last_chunk = waveform[:, chunks.shape[0] * self.step_samples:].unsqueeze(0).numpy()
+ diff_samples = self.window_samples - last_chunk.shape[-1]
+ last_chunk = np.concatenate([last_chunk, np.zeros((1, 1, diff_samples))], axis=-1)
+ return np.vstack([chunks, last_chunk])
+ return chunks
def num_chunks(self, file: AudioFile) -> int:
- # FIXME last chunk should be padded instead of ignored
numerator = self.audio.get_duration(file) - self.window_duration + self.step_duration
- return int(numerator // self.step_duration)
+ return int(np.ceil(numerator / self.step_duration))
class AudioSource:
From dee15c4e5b085b46d9a59c70fdbf9ea4431a64f3 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Wed, 18 May 2022 14:30:50 +0200
Subject: [PATCH 35/41] Improve benchmark progress logging
---
src/diart/inference.py | 16 +++++++++++++---
src/diart/pipelines.py | 5 ++---
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/src/diart/inference.py b/src/diart/inference.py
index ad10afba..627cdad3 100644
--- a/src/diart/inference.py
+++ b/src/diart/inference.py
@@ -126,7 +126,9 @@ def __call__(self, pipeline: OnlineSpeakerDiarization, batch_size: int = 32) ->
None if the reference is not provided.
"""
chunk_loader = src.ChunkLoader(pipeline.sample_rate, pipeline.duration, pipeline.config.step)
- for filepath in self.speech_path.iterdir():
+ audio_file_paths = list(self.speech_path.iterdir())
+ num_audio_files = len(audio_file_paths)
+ for i, filepath in enumerate(audio_file_paths):
num_chunks = chunk_loader.num_chunks(filepath)
# Stream fully online if batch size is 1 or lower
@@ -141,10 +143,18 @@ def __call__(self, pipeline: OnlineSpeakerDiarization, batch_size: int = 32) ->
)
observable = pipeline.from_source(source, output_waveform=False)
else:
- observable = pipeline.from_file(filepath, batch_size=batch_size, verbose=True)
+ observable = pipeline.from_file(
+ filepath,
+ batch_size=batch_size,
+ desc=f"Pre-calculating {filepath.stem} ({i + 1}/{num_audio_files})",
+ )
observable.pipe(
- dops.progress(f"Streaming {filepath.stem}", total=num_chunks, leave=source is None)
+ dops.progress(
+ desc=f"Streaming {filepath.stem} ({i + 1}/{num_audio_files})",
+ total=num_chunks,
+ leave=source is None
+ )
).subscribe(
RTTMWriter(path=self.output_path / f"{filepath.stem}.rttm")
)
diff --git a/src/diart/pipelines.py b/src/diart/pipelines.py
index 649b6a4f..b56b235a 100644
--- a/src/diart/pipelines.py
+++ b/src/diart/pipelines.py
@@ -165,7 +165,7 @@ def from_file(
file: Union[Text, Path],
output_waveform: bool = False,
batch_size: int = 32,
- verbose: bool = False,
+ desc: Optional[Text] = None,
) -> rx.Observable:
# Audio file information
file = Path(file)
@@ -182,8 +182,7 @@ def from_file(
# Set progress if needed
iterator = range(0, num_chunks, batch_size)
- if verbose:
- desc = f"Pre-calculating {file.stem}"
+ if desc is not None:
total = int(math.ceil(num_chunks / batch_size))
iterator = tqdm(iterator, desc=desc, total=total, unit="batch", leave=False)
From 13d52c23a593da4f33ac2dfd7390c5bb9f2ca941 Mon Sep 17 00:00:00 2001
From: Juan Coria
Date: Fri, 20 May 2022 16:40:59 +0200
Subject: [PATCH 36/41] Add docstring correction
---
src/diart/inference.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/diart/inference.py b/src/diart/inference.py
index 627cdad3..a9c79589 100644
--- a/src/diart/inference.py
+++ b/src/diart/inference.py
@@ -16,7 +16,7 @@
class RealTimeInference:
"""
Streams an audio source to an online speaker diarization pipeline.
- It writes predictions to an output directory in RTTM format, and also plot predictions in real time.
+ It writes predictions to an output directory in RTTM format and plots them in real time.
Parameters
----------
From cae45805fcb0a5b1bb856895e2a170e103763d24 Mon Sep 17 00:00:00 2001
From: Juan Coria
Date: Fri, 20 May 2022 18:01:23 +0200
Subject: [PATCH 37/41] Add inference API to README.md
---
README.md | 99 ++++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 69 insertions(+), 30 deletions(-)
diff --git a/README.md b/README.md
index 8d65ea50..550262c2 100644
--- a/README.md
+++ b/README.md
@@ -14,19 +14,42 @@
-
-
+
+
+
+
-## Getting started
+## Install
-### Stream a recorded conversation
+1) Create environment:
+
+```shell
+conda create -n diart python=3.8
+conda activate diart
+```
+
+2) [Install PyTorch](https://pytorch.org/get-started/locally/#start-locally)
+
+3) Install pyannote.audio 2.0 (currently in development)
+```shell
+pip install git+https://github.com/pyannote/pyannote-audio.git@develop#egg=pyannote-audio
+```
+
+4) Install diart:
+```shell
+pip install diart
+```
+
+## Stream your own audio
+
+### A recorded conversation
```shell
python -m diart.stream /path/to/audio.wav
```
-### Stream from your microphone
+### From your microphone
```shell
python -m diart.stream microphone
@@ -34,14 +57,32 @@ python -m diart.stream microphone
See `python -m diart.stream -h` for more options.
+## Inference API
+
+Run a customized real-time speaker diarization pipeline over an audio stream with `diart.inference.RealTimeInference`:
+
+```python
+from diart.sources import MicrophoneAudioSource
+from diart.inference import RealTimeInference
+from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
+
+pipeline = OnlineSpeakerDiarization(PipelineConfig())
+audio_source = MicrophoneAudioSource(pipeline.sample_rate)
+inference = RealTimeInference("/output/path", do_plot=True)
+
+inference(pipeline, audio_source)
+```
+
+For faster inference and evaluation on a dataset we recommend to use `Benchmark` (see our notes on [reproducibility](#reproducibility))
+
## Build your own pipeline
-Diart provides building blocks that can be combined to do speaker diarization on an audio stream.
+Diart also provides building blocks that can be combined to create your own pipeline.
Streaming is powered by [RxPY](https://github.com/ReactiveX/RxPY), but the `blocks` module is completely independent and can be used separately.
### Example
-Obtain overlap-aware speaker embeddings from a microphone stream
+Obtain overlap-aware speaker embeddings from a microphone stream:
```python
import rx
@@ -80,27 +121,6 @@ torch.Size([4, 512])
...
```
-## Install
-
-1) Create environment:
-
-```shell
-conda create -n diart python=3.8
-conda activate diart
-```
-
-2) [Install PyTorch](https://pytorch.org/get-started/locally/#start-locally)
-
-3) Install pyannote.audio 2.0 (currently in development)
-```shell
-pip install git+https://github.com/pyannote/pyannote-audio.git@develop#egg=pyannote-audio
-```
-
-4) Install diart:
-```shell
-pip install diart
-```
-
## Powered by research
Diart is the official implementation of the paper *[Overlap-aware low-latency online speaker diarization based on end-to-end local segmentation](/paper.pdf)* by [Juan Manuel Coria](https://juanmc2005.github.io/), [Hervé Bredin](https://herve.niderb.fr), [Sahar Ghannay](https://saharghannay.github.io/) and [Sophie Rosset](https://perso.limsi.fr/rosset/).
@@ -144,13 +164,32 @@ To obtain the best results, make sure to use the following hyper-parameters:
| DIHARD II | 1s | 0.619 | 0.326 | 0.997 |
| DIHARD II | 5s | 0.555 | 0.422 | 1.517 |
-`diart.benchmark` can quickly run and evaluate the pipeline, and even measure its real-time latency. For instance, for a DIHARD III configuration:
+`diart.benchmark` and `diart.inference.Benchmark` can quickly run and evaluate the pipeline, and even measure its real-time latency. For instance, for a DIHARD III configuration:
```shell
python -m diart.benchmark /wav/dir --reference /rttm/dir --tau=0.555 --rho=0.422 --delta=1.517 --output /out/dir
```
-`diart.benchmark` runs a faster inference and evaluation by pre-calculating model outputs in batches.
+or using the inference API:
+
+```python
+from diart.inference import Benchmark
+from diart.pipelines import OnlineSpeakerDiarization, PipelineConfig
+
+config = PipelineConfig(
+ step=0.5,
+ latency=0.5,
+ tau_active=0.555,
+ rho_update=0.422,
+ delta_new=1.517
+)
+pipeline = OnlineSpeakerDiarization(config)
+benchmark = Benchmark("/wav/dir", "/rttm/dir", "/out/dir")
+
+benchmark(pipeline, batch_size=32)
+```
+
+This runs a faster inference by pre-calculating model outputs in batches.
See `python -m diart.benchmark -h` for more options.
For convenience and to facilitate future comparisons, we also provide the [expected outputs](/expected_outputs) of the paper implementation in RTTM format for every entry of Table 1 and Figure 5. This includes the VBx offline topline as well as our proposed online approach with latencies 500ms, 1s, 2s, 3s, 4s, and 5s.
From d3130ac02db1c3d37263de8be79344184a4703a4 Mon Sep 17 00:00:00 2001
From: Juan Manuel Coria
Date: Fri, 20 May 2022 18:52:32 +0200
Subject: [PATCH 38/41] Add demo gif combining snippet and visualization
---
demo.gif | Bin 0 -> 5236215 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 demo.gif
diff --git a/demo.gif b/demo.gif
new file mode 100644
index 0000000000000000000000000000000000000000..cade3719fbe3d2778f84e93f73dc838e489119c2
GIT binary patch
literal 5236215
zcmce7`9GB3`~Q94vzQrUjIlJB80(Cs8f&&{tl1l~g*0dpDwRq?_l&_1Wi7k0uW2Ze
zq_L)`R4VO5LgiJoQK|X#{{9i)^TTz1IOp*==Q`(HkMlg|`MkEeI9XVRoCmVy5j
zEp0U_U0qjKgGSfX*VkURUeAQCYfabRNT;ti*4n_(*EQ2Jwbn8=S!ZOfWoB-)(Zblu
z+HBKiOEXhzqxD8>DXNb2b?$VAKh1QzIX!}AeME1Qm4o&sXYDOaD;*ad?I?Z6?K-Z(
zI(GJ(>^-!7qjbIZ=>{I9g`A{$hUtbT(h@?oj|OSmm^(Px_{7s}jf3J<+zvWA*}A$p
zdU&{Qb++5aa`oQs;p^)i805P%IN0Cc*NqvniyasizSGy~;NHEvBct~0-?#tZfrI-Z
z!j2q05OXl*=;5RBr;a5hor*hg{7_6}V8Dqds+8M`O7qI9A0X(ix<)!}>jj#71&!4}
z+uorYa920<%C69F0p8Q$p(TDs+D+zrlIc;I^nI!H!v%WBOY{!q=q1$Z({67$4ad_)
znLMF>X9oTE^>q(R^|;B2xr2t}K)%k-6s5hT77=H(qANBWDASHOzdq`M>Y)bt=!P|i
zTi3>3T6?Tr^H7~e>_u8!tNDpW^OKi!Pj#A|?%tTx5$Tid6Mtdrk=(eQb&g4OiJ_PH
z5q+j9y}=om0#fQ6(>jCmx_9$yqb^)Kb-w9jWp4y0e{1frPw|*<_2l8khf(eC68gTR
z@RBpmq-14fr181Ad07Sdxy2;~Wo1Q$g$3ubGb#kdRn=vg<;|59l{xjDb#>KE&9xU>
zo9i2@+Am-1=(yb7edYS~?n}a^nJXQV&aM|lCH?(9{Ep|D*I(wgk2YQ(=^lMiD|%dT
z?`!XU@uf$f8>jzvz5CmF1+~o1*Uj
zcQF5d7!8TEr=!nyds~*H@p=XV{5Ph?Pvrk9;9rx=fzW+>JVN*F508@B7kc18=qAQ`
zQ!`UDGr;hFJ^mL%07e3U7>ct{(3gyn*AE^MRt}^QwOmV`+p31nkWC_AjRHvsI<6f}kDCK7N4~n#(fsrhd%x>ix2rABE=N3S+WFD6sp3Q`Giuw9q8#_1HDtclQPMXVg5Nhs`}|
z*xnp+S0(EGQ(^dM`Hjww|6bT#d_R2m`iZAEkz8qY&z`QOwSeQi)e3>iB9w>o`PRQWo*)pJr#+P93GW38AGn(1|Mjkru{HW
z&UB&Wj!?Zu+=U35$$~hHZ({NFp}uv`e03xtgg%q)ptOG`$9}JcKixSyHq51;Qpr0T
zmso{#)vhBU4Y&N|smt1CBW>xzm9#9~`!!*eoUx+htrw`5uGjttB
zeC{^2T>bE&NpGFx`8=bUXG}G#XJ=&6t6Emlbtk**=4uQG2sio&CJ!Op_-lu}85n0J
z&>gMIkU{lh!ux3b#PB{sqYrTonV`8HFmq)Jab@?emXHN$!v>#wzC8)rS*5=da?$zy
zWAyXQ-ytLy<4K?G-X>->xfupv0@tCb^jl}L$zbUb*K_p|khJXm)emh%%Hn)kfzubcK1@~^)M!JBESSBRWN~3s;HY(
zXJXkh|IqC52x;g;OXsI}g&jUeJzPe{fcFeDw!O?-1$rMAIl%fFg}Pfq0_2{s?EBt|
zv@~4#lMeXA7pArb2fmFSZ0cYy|MGT$`Uj6du
zfaK50+d~^&yPS`tOMul1;Li!{c4l&nl-p3|lhiDV_C3XCRl{|NoJr(S6%8W?Nh&Fwj4^EsIT;KaHaqt?9Ue5yF;>Dm2
zy$?jQN(%m>4Dc$yjSRDU^tGi@2iS+X1vpDk>kk*
zR{NNV%h&AWs)w=In5q%mPrOjT!>|t;5Dg=*`V1*6ij#}kVFZXMfFk<0UmvUU6Ol_1
zF`PZYUChIQLbO?fhKx>OcnliguJ5`mtaJ+^8=PSwsaNksGuJ(D3HEghDw4p50hRe?B@1BE4EO
zm)C9ACOj_1Km9$_ORWdI_B>M}3VoFTf0_5e`>gM0PZBd9_W*>Lm!Di@v?(AK<0<5>
z855(eSNe!&g*=-36fS6`pL|c4x^ZlZaB5|MN@(M6b)P1ctPHA#*y*{Ew9TKK1J-cjIQ<1+pM$g`StGa^n-rh
z3AopOzH{u|`cpqg*@Vl5x7_!xlSiB|JpS40sbMKH8jzqVf(?gX20%(z1jxO+nHW0&
zQkVXDA6A;!RPjLThsv*U{*QDs+@{?6)~`bcYATf9FTZ65{d)Lg!OhkC{WFJCza9xf
zu=Os}^B;Xw9yNF#OW=>qv#$SoT%mTX`fQ6ka?d=ezd1(9n
z{xvyLw@$dTy_*s17)vKLl`+IA6<4*{kUC&bgyAkvI<$Th)
zwp({U9lzP|GTx=LXYOv9ZRhh>EB7`Y65myd|NZ;TqaB^SQl2r77f-DNwsrMusUFC4
zzb+=8vj``&o=*u{o#sczxro*+ojtY6P}n9tytO>$)alh3x}@H1L%Kj4QIwLct+=`@0`7y*zKO{c^=z%
z6aX#03pEa`ZM`%%6tb`TF!HSW0Xl-#nC#mZkRmF)mZ*Iw?UdP7jh$oB;i^%m3tlhx
zp6ELCR$1}YHPf?GOMM^fANLIdrSD41P+}3#e+-bN2)#2n#`aQ-=BQA6XaoiApUyO#=jWMU8(wG$J#|-{rgRyT@u|&LdIx2#
zN&~$|PdiYt>D(SmOjrN(<>c?!E1FP1J!t5Be|Bo*5I_)cBr(1>A1`g#*OFcLt-2r_
zMUJL{WQ%)%E(3?fi!gSAH#D2W-zD5vl4u3MMTFfkmk30_{h>ff_UW(kfRpHSz%6vG
zD4^eLD}s$EC*oWHP$@2HewpJ@e^}edPQ=FRM&P@$r5ohq7+$AD8>JcKWMhkD(}3h1
z@QJQlHS%~a4CrNA|+6Ty{
zfg~n?UPM%i0RoUVums2mfR8?aj6}RoVI*MB3_o+k=}gyNnMGjzEMEquU}1nv-G_<+
zq*wr!7$7B1eK8cb*BOY2NL6K`Y9^h`sdz^ra7lEU$N)_lpwc`*67S2N1Js2;JU#Om
z8AMXy5+NWjO6x~~L`D*t1;{h9`uAi}=)vZTxLQ$K5}ERtOi^b-E@CVw2IScYZQ))y
z=1FlU&ILxeES^LNfm0(+Z^>u5&cGn%1QB}C%SagoB8Ef7$h?=*AK}&?
zmFbW|DV@wA7v-p2%TYRsiraYVd>hJJl%u^Pm+~Ij&X#c~Jc1s;pA%=}pJn44F-FRH
z88LZg63aY`qCA_Ed8S2qn|I{dV)Lz^<=NlP<1FUc?Z`L3o$u_D@A)j>Rj;=OQjt&>u6!I<&dPdy8yC$62x4p*3)d<_fK;%8_|GfkXdyD1Z3bo|%7~O`
zs=#3!SCxGRq#`SYxN2&!^f@Gt>ziMJ6gYj!Wk$|x*KIed`fGNk16atw{EJLZs{`Y2D8T5qdbAN^XE7^&t$
za*ZbkEh1STkh=nF!>sDOaJAd0((MCB(eg5RVyV;Ox|m;eM^x%#t?G{l*2l%y#~0Tp
zbk`?6uZMrtb5$C6Rt*lWHC(Y&7K0ijc`hGvZY(jz4`vC_LE|t&kvqjSdzy&h;W<0c;
z1?++$#5k~f96&6hBE%q$zm39XVOlK<;mSt8-+}A%
z!flaI!Amd#IgXvD3+9~rI**^~j62nNv;+lr
zDqjH-PO5fs4Z1EO!PTf2T~VW5XH~Cdt6s|wx^^z=T7mVo!t2-0HebvC
zc&${myF|MCyjyo!^VRFZ6sn%WxnByS;Dvs<#LTUzB`aXxZE!mb-q&jy4E;A%k0)t~
z;xu724^cLby~uGaejqQe)PWnlDJ)4z=Y*=*=`cgtN#M$6LC8)GV4
zP%t_SK(mE6P%xCQ2Yo$y{@Sw=pP16QKve&+>nqkk?yvLy;zSA)^<^ufij8ccB7qV?
zb@s6eYICk1?o0YHYatHBXda8Y{C)*D>kOlWxQ!xo2@!z03NhA~y8>{3GK3u%T=O^%
zwTLiv11c90qfr21EqV|_ln?WMzw1~4~{@Vj!Lq=1r7~yk@-^A!xvPUSZ
zZb4Y#1tM&P05^Fe%3+wPT`Aoq>^AZ`mDnu*m?^WdK^eV0A*)#aiiqYUC6gmactjm$
zKIryb>h>dE52sT`Zaevjz^>-&rB+RxVB8LYV>lz#bQjtbssKvpmjqd^GknA`7G5g8N^6rtw}2TZMj(s3+|
z>O=jz3)tYZXKm`^0^d^tB@tpPr32CrUTckjhbmfG$fcG{j2>5O3U9-r&e_Lp%sFm2
zseh#Ygl?OD#2w`mTNPeNMC%<-$V?*TBmL#A?~*oocpYfjYAZ8gu=GejbfSJc-fOOb
ze=GV{4!EL9X*ZyhW|Zbxja;~M{q-{965M`C#95jjksHTF3)|3Hh!)oEsyW=uwJMYt
z*TFoA6Qk&iqLD~Qh#y7#d!Vq2CttKQ^>_|uKnBzjO>r}w4I!yeV>I;Q%;-|n=(QK4
zs_e$2E*XAI)bMUSH)jj`%E|={z>0d#g^iSE;nu4FW!XseMO?`^R@)V*V&S40W%s0U
zFP)!MOGLQqBe)MQaXP;$T!c6WBH$`4TRh3Hjd?)SKUcy;BB;n_A<_jFX#97e`}iU6
z`Z(Tqb9CYK9PPBy*v2QU)2_E{6D)r_`HU69DPW8`051oW>IeIqHE3~Nj<`IZc$17
zD6vApZ{No~25T`}_DhjB7Np%ie`4YB($bjP{_zK&9`${h=~!H&{#9LaVad6VqAYkO
zbuf}ZMGnuubAe~o;ubU+(A@{UAp+<%4RXG@76Tvx{vP{_$vRrsR>GOgD}Q@~UF
zwHqjoq9(^fd~bD%qqzEB=s!f9So!q&{x{#hAio-`FB7+Z-C9XOh#Hl%LC=Gc=N532
z%T-OlkKxv&5YHF&<_k;7P{`s($oEij7I&{z2Filo0VP7*
zn8$up1YncRQChiQviR{o&o77=geeYZ%G|F`{^V6c>wEoc=5N(S_<|kFDf-*M-K1kP
zYA}I?yW+7hbdZxAF#vYG6nZSQeEt!GSUz(4&rhdir%K8iiKw)M_Lx`>=J7$f@{D-(
zjJ*z!<09fByV+v#@5ul_8%F$~@jvqsd)9+a0-QN_yK92qNI0#fm09MJ-%+Cz8@hUz@-y(d*#Sze)w
zaT!Q8P+dRiUX<<8h;3Hm$1DbbvlJ%aZhBbs~C^Yna
z-@w6`M~^kQikD_VLTp!m=m02*c%Y~w7QM7un9OlWl%UmuS7N4z5)sVu;Gaik@0M*z
zyGMBu`{7a9f1c(j)^^n!LFYoSFY^@0S6aVACS2xqtcl}NH_5Hs$=T6^<
zkNx}S&sOz=$p%6Jlv>hK*pJgES_p6kL_;Ripfzs16O4&Tk!MA6V5H~fsbK@?Jm8q`
z8;ud9dfQ-Y<)wHLj^u3?eTF(IgXFi;#!h4=iZQY>T`4~T*k>{nsO%Jm2xG5(hT$5)
z)4b*Ej4~?JehWe@%)l9j6&W5?>|GReW-2{1yzUKhMvGGlB852~8<@W`#}3^WIDsD0pgO
z#2TJHOWNl>F)%ZKo6XW7h?oRW3VzyDCPT_WuBzSJ39EyJa4QZztcZq6MFhmC%7-!+
zXB2h3Ye9u=enwYeWjC-EEj77VbAGM&pUS~d8x0tFz&|t`rnqW21s%S9@m-qFv4A0m
zM6-7j+fA~xy4cCGp6|D$tod}``p;srMOZqmt}E=!hGC2Ftc|miJ!$pf7Q1r_R1d=V
zmJtkSh}XuTEoA8#2z^GwG}nNzcrjlGqV^SFdLtbju-b(jMRQ1GOI9~l5%9g??U*c*
z1Z`oP$!j1OyJsp;-%^dOSfVouYTYBl?Txzj9H{#TjF(ji=sN=rFIIsa$@c8KAiMC|
ztl1gkpR@Y-E7$6!4(m3+28@*h048K967b3#nNkdKlhcHqX&!Vy(}81(^~1-A6jkCd
zDv;;dN2Jao&|y_C%J0+j&CwESnm)YX+&;G#l8p>R40Y0#q^L!lFh2-!+vRJX?3sTRy7N6>8ZlaL9(gYR>sBWgPg(1V252
z!)vJ-y~=N2JJ82_9(XYiPm2H?ZL3wCc2+0_rdfF@kGvP3<4|UNYibJt2AGqvFsmN}
zcViVqe&`VnqP0B(f~y3rFq33wq&hn#NU06%2gEv{?>vy?$^hPIis-;79cG*8
zqvW03yy%+0ZeWCvLTMq$4g{rd&;gmyaj&APSOQstZ-M07!)y(k8Gkl8cY`>
zSilkaB2|%;pBzprpcdnn^5g-In4zK|kZ)Ph#!?oszA!fl1X2t(Kfq~P5ELq(^ANh@
z4IV9;0p%=-M%r37!f#9;i=uzrC?A#_$m1%OjZ1I1kj?W^4pl6fR5G%y?5~LnW1!iN
z>m1~)5f}|6_SqC9Freo-X{J$X@f1iJZ}1(tkJ)Mh=tVM~KUDg4`s$7P$j6pJP8**;
zIF}k*v$Q?P;XK8l14MW+Zc>ia)<1FDNK5;hgR-<+L&`+?xeyH;C*
zB+d-L#-a?HiVh+_Oanb}+oovReRyZhij+;4Ee)6va}+s#6oHyM>hetS8$MJj$gWpixRnpg8%x5hs#=#8!0
z`0D)=q0{{~qpaY`mmh?+-^QfKUvoDNx=*-2w(s3n>2qU)29B7Vw^&G&r{tvVT4VjF
zcOxls_)jEHiEwo(iQ(*Z^wa0&-*4pmue6`OwBUkj%fh`m(f0es(ZlM6PnQBtjBF$<
zX3JVnNpFeke^mA@J;YO&K0`&n{=GmTT9)m|xDD!7?2t7^53Fud`fd|bNQ%YLP&w~~Hz+)Vs*;?3_j
zQm0pjwj_PI@Md*x>z3t{htr+MLwme^
zD=!M&_{o+`5z_qY}bzUt^L-ghO|>t=vSxoQ?>iGO#8JB
z`*j@bbbb1DdHu9}J6cUYy`^6-mSCM@ZnV+h9v*1_+T?wzZ}Yc-al}AMQQ!Jfvd>~l6pVJrM@IxWIQTd?
zxeq!A4?5*LILA6T*9>kQb8yde@K_k!Ryw$?XV9a^VVmKQtId$t6$dZmkgtuSceG{&Or>>`bI+~$I`LXIynXPOR}8lGh=afkQA
z8)7iYmPPxu)|p?)!LD%BG0iuw7pOE#3!j5)$2n_9vi3wfO~KOYE1WhA9DS`#95=jW
z@K%C0Idt`$?W*G{Go|$RR^kNOC*TUl!FdJaloWh6S0kfBJltkbn8vP@RA6CD0C#3|
z;Mzy)kakYgv!2eV9+1e96FXm0)yp8e+>R=h7Q@rY_F{4?0|)5DrPR)sr;YU7dqCmF
zWciJsIlP1nh1(9{0L+MUnR`KhkgKyEH*g-QqNOMWq?8%i)J=f8^Dc=qcUs3V|GfWH
zDBxW=w|@lPp0Fiq9DQZC+m8Tep~G3A4Ki#HBE;#k#pZYzBzFOo&tbh0K(s1@U_f5;
zJbxI*vXecXxer$0Ex$pgewPZ{9U%t28d7u^P`jua&EzOrfN5loVgRV73=`-a`9f-Z<$$Tcex{`*)p
zrvMw?aDVH!4B4I_7?tVL
zN)JokmJO>HmSNc_iQ=9V6)K<=0FiHBRvEdsv$@%4iAD5nP=`_GLZvGiN^4oraXGFx
zu{~h?LQT6XVq8gM-UM2#-uxSkczNGtQ{P`3TAMo!dOYFr&d0-yB{VF;le7$3TC&9u
zzX2j(8hZ44p#W53qqDP1!zB?&o~o1gZP@*b!xGx3yXP{jkN~pyJofwq*ILEt7DK$CA
z(^=87Y~Y5m?v4W~${0BIC%^()lw(llhstrignEiq@AK@t5s`xSn64s%v
zn*jf519AZy6dN$vIC%z)F=AZ(tddOQV&t4bBMUGl!>w1mRh)0=ZpQs
zh21Hn_%>ieICLTYZ96t8)R>!ro?{1xYMXQO(Gw1@UL~!m}y6S`M
z4=pI(kswkWN~I9k-lk#FlRZ)h-=sG`05j!)5~KG!*8X`WkV#iR02}B7l4riadMu$N
z$4`WBy*V|yNdq>u1;C=K&1U)pvi+OM=Rg3o|8R3oH3voAsk#D?-4q`VaZXvE_hr9W
z)BiN~@EUAHimaIDl?`J>{u&t6^%6BND%cp2qFD$G@DwyyDIaqt1{B#-ZZBu~n@1eQ
zNI&t(+~81c!UdmnJw_!2bEK5>@nUa+Gw^1x8UyI)0QZl(L6!#a^C=|how5#VAZUO4
zf<7Qqv{C;TwA{p>NWjPiJYNT@D#Tz^)&a_BE%79q`5dSgd%uZhAluTB&crSIl@E^c
z^IUwTe{$|l2X4|zQPR-WVJPYd@Lb|rlfsQU{~$O6mdJU^U1e|&Yjk)5Ywk2~*lltr
zuqpO^92PyrgL>u(pyQc+8AR`XYruX#Pe25U?VV;E&fY{oMYg&{FbVp&nvxrIZ^h+NK9hm+t!!kMR_dKW~*1XOL=4TE+5ot6O&3s>1ygUFcbM`8hPfVdv&Ph^i6pH@oMcZHOmfm(py+`PuzSvSDcZy+5}256>Q$
z-8+7Cb|Ua)a?R{v-MJ%IXUPk*v3_&M&@+e7K*ajFxbt)NoA;3>Ze9wX`?J08Y>zAJ
z-rQ+(!imk|le+Uev<18w+ZyihXj2^+Ko5V**f{$YV;|0-E71yk}R(^_)!7`Xe7js@6=Ebw%TQc}2>_HENfd`1jrTwpqI&AxBvE
z&bL>^f4%fIwH`lGN<{H>jlho?D}Wb&Ps6Nh<7-0G@Q3^81H6e3FQv0VWlEtK#!t$Z
zba`X6U=f9)KO(Ps81Q7HyqNB!MV^G@vY9fEy!=a=MC9#Y0gmLbF@8T+JawZ1{(L
z>UmILhV4A7@EwO8{eqtBC2%Cs*nVxClm(
z9{+&V6wvFk7gxuboiiXe0UwN7m~B@{?edtnlWL78WF
zXcAFgh4<^GIJJ|UkpXqqJTD^S77zdr*KjI()_Vm)q@TP%6DTEq3<~g9?puHQ$-?a0
zg%67h<3E?qr#6w}k(#fP8^>NTr
z2da@nd-Ag&)|kYvixV)Qb;TZ(#UR$>k~b`Udaa0%T?CAPyba7``9&ay%y}C9_0cMq
zAvpSUK}B(;5@h+wPacJ`y|N6<}~Tyu9f*m2vXx^BdVWc;1%!IJ`4(eK1eU-_x9r
zA~jG*0o-jPO^0%Ujpr=_^tk>FoWSO7Dqa65H~LE@t_tf@{vByRp
zUBJ^K&a1sOfm}M4%KgW39`0MBr1c$2-A8@;+Pji=qL$Mq%rKrV4d?khdNO*_8q
ztW<)EM{)oS78h@;GH7@`q3n%F%SGePGBk3q^*#L64pf-T0Rs+Q(lS?Mf)u?ftzy+`
znPdqKLZp6{`$hr}+-o$}e;uvZbknQ4sX&X|PD+JC$P3#G_a+SYJF#rXvY
zpnNBd{l_m%K65#I6s2$&_Algkh>bCr;-BetYwU7fbAVpgoeK%)BRh}3dvW-~{NaNa
zel_khjb!h}{n~p+aXEX_b4LIfZ$+^O}YyDE{-2nb{dK7QU0pK;y*K2p7R_es>dI8gwl
z$`xzDYb{<6B6P-cEvZ*{!UlWBE_6PBfqAIk1w8^t|AgIywuC<$kC?0@f2qtmxZIg
zfD8l)7RLuqqBZZ1kq;odXN?IFN
z1_^-|8YhPjbNx$Vg|5cTe7c$-(6&JAiaCpQX#Vow$K?%T5m!ngzdkxn2Oi8=V{&j|
z#b{6=gg{qFogwTVS+IGFI`)65;SIp*2d87)2BbzcZ)E_Ha5bk
znE|{{1DgzM*mLo-(k&DrGa{#6N@g{Ny_iOGSryp>@(Q+Z>`@9!E;M!WZU@pgnQ^8)
zO@SdIvs>VP1Uel+ll6lwxKgLm7aeh0nK3Lo`evpGvn$SP8qmyN$;KGKIb3FfVvHL2
zIcwi*(BIt%B}G#d*}7pUSNc!}I~gcN{m1?TveB`c=Ov&EUvp44UWS1nILH8f1PnrN0_e2r|nd*x1Z
zOK-Qzg76AXx}|AkxT&R0Xk4;zX;9N%hE&0_cfZu3=C*M8hST-vw#x^l4|cXDeKc&I
zYyIF(cL)wL=vlgBz2V09=c6AOmKMnuZ`?n7=HN9+yUIb~2VLny*Mv`)_@-s6=+75l
z=2#gGWpG?40dRMZzn84C!~89Zy26){Q*zIZBw%PJaO5_!cca7qg(G8u^CZVX{
z9PnP*!Qi7~>jv--ZtpF&M9^#LchNrDx1zt?B3ubWuuNZ2z76e&x?%Pd_FXk=tjpMB
z{%lW4;*V$c#ANdq2d@7ye|fm%;%)sSFA_JtK3Vc#1}XmcpN-;Zt;_b}6xEOw!V^l8
z#bnm$r)D!HF?sM_M^+qBb3FK6GjZvOS(<%@q;Mz3$VDQR4IaewoluWu@>|Nh+m|Gt6a+D8ibI0eGI1*CizqNV|1I{5mhN$W*l&OmvI!F*NJ;X
z6e-)bXgh_II$?p5dve@%Ig4>jV^xzpKp!^ZHl5LKVG}n`Wvn%|C-<}Z@x^o-52`KM
zl8Tm2H%fkS`|$CMpdkWJ4(U%lOy-z|sPEW{_hPXRxCQjPKW@Z4jy^3?1{c;(>LMUk+uA+7gll
zSg!~!**YL-!vod`&^k}o5ZKHj;zy%jCItZQ!f%s!fFls7R}GL0_sP>JNq=Ua01
zPNr`TNV;3_DS>A5_N3KuzZPEUBu5t~w%h_I#N?g
zq%e=n&Snif29)nLaHP}48Co+jinfIfuN69zDYN3^RfV&}hwqv3-tOh=6ZPD~t;pv+
z7g7ePXxYs!$ZWMwbeo7{j#6+Tj*)rb`1Ue)4^^41
z88SMC1?he=>K-mSkJ@rOT9)u1B@M20e1OfP6cf>m?;j_;Ka2EFv+K~y`1!c2_@Z*_
zKiw;vgZFQlKKpvc;JLya-k;1=m|*pjyfs{0y>nAfIWFws$vxOt6ILVr@fE(EZ+nG%
ztEFz!x=35+z{Q4wX77a5FN`xA!xWZ|kBE7?ff_;AE#Jz!3N#Lw_1${aeNj1kCzsJM
z6|}oG3~Q;q{&~Y;d);zP{vAh%^x^8m$MLj|*MHW>b=-R8ZQI#1yJGuBpvjN_B=gqD8P7!)DgL~6bd<)stM>xL7U7IJy
zo;FGPQ_V<~R|%cp)HR?+_?)UN$(xc}{-V5EJ@#e2+q7cmhQTvG8JXeHerpU;hs@6k
z7!HRse0X8xM&^80ad#u$q2{Rd&PHht6OjFCDA!DVYiEG&3rqy-kJ$djy(7l#5s93EKLPw@&~{_LFBbj$K^1yzW;#
zNQCIEdo9o3&7^}3eRxy9d4Diy(049~plOBeSoK@$|FlhPGYJVZZbo{f;QlOLLf`m|o0Acm(F%|w!D83j*
zVu&0Su?VeYAi_JxR9>_eptYG_p-2{@RaJ~=q4rB@vJ|Am;!lZ30BocXIIOM>kRnh$
zST?LYCK2073v3NynUIu%BXrV;kFcnZt8$0Y$y_FqQGVR%=?!%0R_FIa1dg(^;3zT%
zn2s9%CuGul!akSa-s8ztHS7hng?-b%|1OJU|DsP^as^|~{XRR;EybU=l?|pArd<~i
z4YoQD$b&q!9;QbkRwRvIg0e~^8XImUp^#r82dbT&2W!hug!oITEcAyT#~D>-qNosY
ztU_8)0)7vkSSNl#zel6=$v%cJ70Jsf>?GFot87YUWGW+$Q=hr-Y+Pq+5Y07kP_04&
z*laFB0j9-rABq7mcqd)hkF;RY@B;=3cDw+bfe`YsfNA-VK@1mkVCuz5AbJnC`j2A;
zzn|uifpCUtD`)8p+e%8P9=Y%GMJ`PU(GyhVJ%JfHS0r_cs07HskWoHFM*%^|N&*rl
z!CcxEE=_?6YIF2qhz^(06_~&(lg58hG#)Cgn?u6)#ss6tT#ulRPa5+smJA|yxU>?zaQ9EQMh9UD+llL5{vX%T5@qE2%(*v19P2)iYnfb+-rBbM{UOf
zV5h*;)3$xp0U-@-^~mtVphk7NMvpRV|ADz~Oa$13ynCg#-9jWQgrqYd&t5KaMr1JZ
z3TY^U@*zAI@(8()7eO~}yq20M*VhKIT@Y5tl@UUO>@d(lWUxA@6PdYl5k?Sg>G}WC
zS|C~r)y_g&CVM782)RC#q6;|)c6msP!#9^*3^Y@=hXNWG8}%|Z;qXNATWSyG`_Ccc
zkcwYAT6Xyq%|S^p+eRy~i>YZ-`b$|PG1Jv_FCFDjfb*1*P+hPfeDkpGYRkDjfUH>;
zD|-6v=6JkJ7Z+g@#3bwn#zeLxIHY?Ah~d!2k*X16w1~=E0orEqToQm%jkZHF)nbyy
zELwR4Z5cNL6BOD}ez>fogc6;A%Kh%wSwqbii#}sLyH=SG4Z%6Zz9X&u=UceQZyG-BwntTHwoXGnNPvJD
zs2tbW;3^~2RRm&Csy$pHQWST7U!btRGM}4Fyo-#6XbTZChC4u(R`~Fnv6f;ZZ;43S
zqRxFRedbTaspIv(*Xp6--g4S0i8jG`n~V36fQa+EfP=HrE1HqL%=}*9_BH06@4KMB
zy$4(MO0G6GF3gU&iwtTSNW6X^zu#w7WDuivvU?66rvn<&SV-r8dc#;Wlhjk|s$zS*
zTvg5?Xq$dt{SeQcB$B!U$+J>oG?4}tcc9hT$7L=C`B-oPmP5WBnMBPaMgJlTWWolk
z$c#vLWEYn(GIQvx9SJ`#wJNejGv~i!^_*)?9SLL~*PJ`Hr?3m!cBEf78%T|u4>-~f
zj5LT)>U$ER)GDf_$uv)sKmRU+et9e@D6LQNzwp#tQ;Wc=7w;~Clb-*m2;IFtli
z?2V36+HX!08F$YDjg2MeTW%L?m2T!KPfiihLk6q$opJ!3=t7M;FMI&zVS{M9
zI3=7OGez!;hMqvpz!~V2aRg?VCb1ET*&^C&hax;6cg2oY+T{4DK@QK{7OO#{@>RTB
z*#C5MEAPBxAzGK+ITDj5w4<#ubzA#&;~?9Rb;9<3DKc}-L5Th>f`zwdqhVRof!6bZ
zD#z_!JQq1#73e!R*~N2_{QiI^Tv|W_Z>5QrFG-hcu%l6Nl|ahyH>|o%YJ_o*k9x->oYyD{J}H`WdP-
zJR@9-jM)i_VZuLpY<#Osd?s=b`#}yRvkJ&i`Cv;(#Uc=tG56=k?GJJQP2r%gGfY?z
zwJC{YR{qgu<%Gq5Iap1!ae!c|b^2<5R;$VsiUE$pNH?Z;rX#_5(KZ{F<-GzN;7J`J
zC>F_%>>S1ng(!%QW*CC*FkxN-RD9ExIKkk{OGpJ6nK`XH|0+eumpc6dzbcYaWy;I|
zgow5w57F*i5g{9pQAMJH80fFCIE7FY`DP?w_or5#^fv&dJtB?S4-PI##|Bq4K#k{%
z*LTCkh41aj``3OR(zW={l?)dpRc8+ysmcSW-EZvY!+!pEE2n(JuZQH{2aRcS#lOwf
zuBEK+I-q(ptNON&$3;aM0&lvsviO#E@MEDqjVh+=gYx~x_XNgz(s%XbUC*9vElQv*
zK2BRN$!t7KOFLeloO`4q77FG=w3_k}@>heB^!4h)#>=4X^*a9r(e<&HT8
zQ>pJRGJ+3v3h+mbE6q7DvtH@;!^@7lVmZ&gZnD49ta0gw;n5zBYeUMobGXUY&n2gP
zO`R^8x>a0oQdf4bxaV>!wqfw7nS+59CU0fT9Gi~1{Lb_`
z;rcD~$UE_r`=f798m@Tycld5o*mf$Dn84kxI$UJ)29I~GfVr{`$9#phgu-K7n{VH>
zL63Ivlh)@lX*+A5QQ!X>lg;nf=U@ECP}4J$52(^XZt&H^N9%_h%$qdI4PG)M
zJCE$&*xA!&ixdRyQRi7mb+G*jp(e+Z)+vz0vy=2ENDG89Tj6A0z#ZrtX$2-Kfd2wP
zna=4YgvinR2yA22EvHgy3lTU^zz>v=Ii#3ciEbb1GUGC##I-c*rRtmx3V2sRS4M<_g#6YSoTp|%B&YY+tG6~wYa;tUT
zx{~H`V_VWlI;f31?Y#^sgplZc0t@EpEshl!P_@Rz%W%F^5%$pTL&=)*Ms6b);2>4d
zb((oy$5u}D9||}z=V5{G*U7uA4Ni!((ubtJAyvtoocg#&WWcBCSlrP&aW^{?_3jY^
z{*R)2@n`z`;{g8o?0z$|xz25KFSof1)m(EaMsA6kOA=zP3F$L;a+yo$K2izQ=%%RV
znl2(;NUD*fBBPRSU%&nSf$j15?ChM+d7sz&d6>kC;c@F;gs@DuUa{UQFmsG)m<+SJ
zHn$`P>bCZr64YCNO94stx=}r^$!V|J!7cK@p?le%zcTf;z}EOvn$c;9t-|%3rYsx4
zATyiPnoXYEr?1$pOWH&1N+-PxIj79^BLYYMFrn{m>z)heP(mHw3(sAz>0>Z^DG*VX
zdnKmirZTsvxL$W0h_7{kmq%x1tw~xmxO&!a^{cbV81QQzOs_EMgEKBSF^QEN1$5k&
zW^|Xn=4U&emKOHtbUEmDzz}oSNrgTBCH==AUFiClqMkv(b~)&2aV7b_GOE3nwA7dvkkrEd7&-~GPCO?vj&Ew
zyEzXsv$ow{@5Iz|g#uyDM3VN+1RD|sK)6CL@2Rz7xn>zXI+LmUFT{XNJ^SBb_vU(lroDH^FG>ZmI2^owjecWM97G6YkCWEj-|vjd
z_@Ly35J99&t=ud?&y{JgRk&{_b93T@zN8c6Sx7Gy7PYRNZZ5@-w_8`Q36=9xP1*Z|!;#=+`vp
z4CaMxAWUm%h8Y2I+D9ruNgPpYKbvw4B@uE$FhIm@6*Szz-HsMUY4FG-;3{Eb%8Ih)v!P2g?F>
zIcBX|hevtSk?rzI7RN*5-`(S)6*iqrOL%|3LU#)hI+*lvxPD9Rrc*Y_v14s}CHa($
z1B>sv3luh=&OG?-ZV&nSfHTo
z{2ioH>_|77Ucz#~w6u)$;9KxCuI@Cu2o({*@Ff|+`l#|~Qy9Y(~U)RG8$gN{mZ$s(r(Fss-opw~o@1o0}vG(o(
z{7X1&7&dTueKx&HPdi&9pAHs`MM21xGviHz6i?b}y_{UtD)2xfl7mEJ-QB{#fUG*c
zEUPbZkI|9Xv^{7!w?fDSy{S-iR>RKi5=uqB36v*i1tPPRcI}%tVQEEu$nmkPph@DB
zihRJou1*M0trH={^K|_M^U6w~A5@86ispK#1_E16bZHl>gw-uCCt})${C9p}%)i`;
z*Chl@COj}4n@srkh>pReMn$6jc)seU;tIQGl`udqo4LBOZB0CR7$gRYxXOQDigHdp
z;~^}YlcCZIlz?8Oc$j^TAzz^k6=-}VYZN>!AxT1>HS6~j!BpmEykR&-_Qj%F*>emG
z3bO&iD0;q{R!0SnJJC7E?IT%fbwUCh6$C>&@9=im1FK>8S9v;JZNi*J^MqAC?hojY
zw4JxJoTa=X&CuV*zO>Na$*rfb!m9olU%lhM2vn`ZI_GNY*Y^AFNozu2M`dSw$}2W~^l=Eyp%flwIiDZRpPjml4eU;P
z{c7KpH!Bk|PyX%9KDhYi=R2%Fk8&q?#~xUyPSaX#p{gK(tQ=@CsBl8HOgNwf&s|bj
zUAVRQ_Rn&%^jgmpM_;`IBtXKx*zoV$XlQ89hQQTDTv6`GLCsPqa?24Dmz{Lhjd%QE
zaQSK>{H2wU_Tr(&AIELc@+{#~^GDDECFJ)@G!qrnXJQVg34rlU-=@yXZiZyyM+(f3
zumyW#=W^Xj*qgFRoGi>WSKdQQ0G;N5`PKPGnyctUCpu+1H$M5vt8>!UNr1HoJ&Cu?
z;hrqo=8F)dlSbMYHs_%z8<>5&M$yHj>}jxzxn^?Y$6ocf&epNd>-8EeR4i_Ll|5Xd
zEB~$t?bx>cF3(DaWy4JFD0wE7Rh|5B
zP0OmZXl7(WS>mzc<NqfDoUp{(ie@aWjNU`8+U^v>zx+i2Q
zJwRjr=#@i5Uv`&|1awcx3DQ*!cH@tX^?tQz%MXbShyNKLFsf;*IvN|c&SShMQog-<
zp>?le#`y5A)T@n1gS{sBiRZ;zp==f-x4%V%3AgSge`OR5}(H*p(k`wdfKOG
zi+8I%D0O|QG~r~ps%;_F;%-?<03n<%eQeM<62mibdar7H?7_Cw#IjRU;jy%XnJzb$
zkG4(vug^JR>n#ZscB@31yc>Nq{wguA{bZ!Ym0!2^ZVIbCr*b*k_dWHFxZkuoq10sY
z*zlp*q+^HYXFmy!jpSK=*G6ts)vk(pGFpewHnqn@t{xjbdgFU3xbcJ2mX9N+R=!`j
zdi~R!L3(gfB#KU3$luLAORY`)(Gt3CXzLN`S}Fg=k4p&)%0Zh8o==;msOWKvtXd~d
zwON**E(qntoMraKu2KWtp(x)hg-Flj
zS^Y7RbJl
z8WHkYy{!g}B|WY=oNLvIJC%^^3U51O%pU+z@vbUL$nuuwq%9
zXhM2A9Z-6H#E8eZV6W>E>acP4=^@uj<%PXz3Tm7NiW5M`Mtfg(jC4i|0*|Sf^efjs
zcStR3wb5&?L0GoF(ZG@5imVY(HY{ebd*z
zq8FrzP*A_q9GW+$8xusw7PBd94M-~$h-viMUXuztXSO=)hy>qIuNei%x^`146GDS$DL&^tYxbZ|Hp=>gVTtS*(O}XzBw0!WIkfmh
z4j|8`kh2v+!eW@AF#fDuzP+rtk$P(&Cl!k6Y6J>3WP`f#a0n4d&vHS?!WI#Oe5oEb
z62}C6M2H?5feHZ(Q-UW1*op9$jR`{GAvGa>fKAv`scANf!0>0
z=t9%H!vs-GglQr&bJURURA3FFH`vP+lF%OAIguiYyt6b`2)J|Ov%;M=fied?Wj+j#
z2|}r`Q0eagNuy%8VM^vJQyODW`AAUC<`R?bmzgrkS#+YB331I`M(&5tV_)x)>~JUl
zZYQH0gmtJ&l1NUiAGQMp(_qSR8i4AsgyTJ!O%W(xE&{VGXBn!c#g{JzV0PUUjEJJB
zLsJlb)?s3Hbq54>qnJ|>DonX*t?wuZm0M47WgFSld%}d2O1|R#p7jYL6zMF5?4#(p
z1$LSN-_4E0v0<)kihrn)#9AQ1-Bt2sGb1+aQUQj{AS)bg(kmM3&w!!H8^M?B&`{PFg(o(2I=07klQsV*Od=Mv1NBnp$wd1FbIAA
zsywrvVr_3kkd&fB6*r0C(RBIfdc)YNpAqMQ+s|^`v#lt-n&vt2={i`OP`M}B2vU_I
zx)*s>BZW2#H}{j9Z!oh1eeDH?bu_kZHPdKi{^4424NRiJ}H
zL2EkpVYPE~g}nN9xvp7MI!xBH0O2YGG%(suT&X8cFay3(O)e&L1&*O%MRV}{nKWjv
zWHMUqF|&&KX1S80?QKnDQS_AtRYWUt|xz>k-js-^L*{HxWxIzbk)U(
z!xs$iTzGNWUV&2|(XX&)gV#g(IBOelYY?3gmKwR3EhR*RyLY3*NPFSTy?Bv!2_F-N
zLHp9NHGH{58Z4SiA*P{ia^#AED40m$FrQMl2>0v-n4$EtWpr$Qz?z;w%Yh|lLwD(^
zadzgY4yWdi-q;(>wWtKpfxg&cNY1reBTgh&$j6wU^V@>jU&5Dza<}b!q08R8C)Us0
z`97sER4hRmvR-JA6_|!Fdv5?Kq9h)Bv9mt~!IRTKR}(JR5p0;?ARB3LGw|vUBdsyK
zFO(!lBXqG5TbX|&jS1_f5?{)Tz!8)cZQnT%|4<)
z0l1__Ff)EDwbGt61EfDQmHrJVW`#<(284UOw9~UE-VTr!FTa?bH^9@W&wqXz|M_Mj
zie(#$-kOUtg@5@9i>notM~l44X%UIE+M_}O?&smPSCAA30$ys!C$VWbCc1Ds{-HmX
zLqmI#(9!+Q7ko5I*v?8=pArazyLmWb)hG&JY;rD0V;)d|PHeo7sS{#Kp!itm)QN+%
zrdODm0iffRh7R-h@moM56iMUP89HF25Yl<^07r;vVu5yinrU*rji~tg70~)QTsR`7
zPHXfI09{1I`oHZhejC}bk)9$Znh845u!apf+a=Fv29yLCeQC+6kZkaMAdTj$!bhQ`
z`4uh&!e8Dsd`f<=p$%9e<5|&`Q=zS;C~qJ34U(Q;l)=)eqX!PQ*(=dNMHonA(A_!}
zNNP(mZokHROpmG#XMB%US~Hgc`p=H5T&6aFWL)UzdHV9@_pig0z`Z-(9H(jr8n+o{
z&m}3Qt(vLec7HJCAW7C2uMZmtLGobn$
zothdt`uwcxO-Cj6v2ZglY8jUd0~O|NGF!MxigaQPovLGhM=n=ElqmSKbu(PxyT(-~70NOj*WQUMU3?Eby$mnv&2Kz$;Jq>X%6=tGXxcY+=Mul=K(}HtVOi~$
z%v46__FqI=edYbLFN(R+yMm=3pOBu16jV-<`SsOsl5*gr!pt(s2(%kfu+@{HLQ^C$
zApIMfd-=y^qCNBHP6yrsHG7JbZ}gfqMU`!o5wR88YwHb^dM?}+NIf}YH2F~NL}j1F
z@D=Uhq{|haD*d;7`c=z}Hu_78UU%)b8#p#aqE6j^Rir@eCgx{$D&5Ig63KL>Ke-AY
z{Kag!YV+Lj4jBN6dS6?XN{!w@Kyey
z;(9tME=#G>UPdH(+xE{!G4#c3=bGN8`q{28V~*4(RI>kmc0cA#MeIq)_Jc2{+~tqa
zG=snPd~o>9g?7#2j#DuUh8JhI5-Z-kxvh}Rsx#b$I8Y)Zbd{vDx4^a~(mO_67X-(u916T3fs6h3O(NG%B^#!gXbp>x2L
z=uWzAvYIkghG-g^u>_E2XcVAZ3{DMR36lOxQ#9)Sd8J!=JzXh_8+PyHM)B|6eWkAr
zzf!)a|NL{t7U-eg;Ep56T6(Nkr#BveNwuaj8v2T4^@9x35*oToO41?>+J+;yKV6zDuzue7~PT%y1mfaWQCh{
z24Zx=@r9G-gUDnynV?fAc5tI{B9`&o;%j4=gJfjV5zTunSbJczQT=Dz=>Nfq4_JKQ%$
zI>U!dYY_)|<{jfb6Xp*aU%s|@)b(o6q_xfE7*Bp3+*Lgj2>WQ?^XnJi=&959N7`_=
z@W<8CJ5qkAq=b
zw9`B0P9{D{kUy)&F<(yBOuUwQj7Qqxu!U>>WhuXQe03UcIl@lFY8L~PZJHT_T&(5I
z$?mr=tA$Z!>MQ?XkJVDgxdmhC;94l
zRk$7wn1v8Hb#y$BMzzPj`lms?(d489B@}&Cjdr`_Jz$8+E8Qc~aJ^A9V3@Up?|%i7
z9(D8^_B^vI{+ms#S;J<+elR~i}Y@G~DMp1|^Vd1-F0obOdJQ+yiL0c_
z0Bc-@AW8u0k2B$HHzv$l#J#!NRE1T9Dld~9q)ac$`q$-YThJV2ZcC;Nfzweq1n3-M
z!(oz>ce}c7qbfl$b%cG?Zp5D0rLk3chF^{t*CHm144bSUqsg*~xw{F~ArF^sa&)FtN{Zj2k~K=j>1RAG2CZCDKur$eIS
z*q{BVr$Mp1aKK=UD*XwJ$W9GwmKT>5jJ=xbap7)4|B)IWDk`w0M8lA7QHyF~Pc5oa;f~WPi
zzF|tEdcN8QCIziwIZ8*waGeH_5YB*8GRC*-$l4cdb%K*QtlXfnXJ`*-mU)BiqV1FX
zVt0fF5mJnMX)}ZV2UDPYj~Si;xJ52?E6k@4^mPoTuCfgpJmA=TECIoVpc8&
z48>ZP7%Fy&of)`}yPNK$Ts^VDiSc}*KsnS=CIe_!r-D+k5n;qOCAQ9QLBY49AEi3j
zlmtop=wH2J+8qe4jcPA(WWesgKo?D|C36C~jpW-`?X^}yHhj4U(4*xj^pNsl=4|9{
z=9=S0HD1`Z!5<-bE=nG|=AQ8Em@Kp|R-D^X
zHWW{n<;$K+L+b7F&ZTzsP|ig46AHBv<k{#P&E)#Ma;W$NzM$?hvp
z1D#&BKBsQ&pvO*rI_|gGuHfxOJ+4i2?lLZ=11l$xKH8M4T4T%9u!UWF%+FQM@}7QI
zI-u~hgQI==WJwg`tbD+ZS1BQUyE^x{9Z7-$LT$}jLH5fU$S#7tgLyvpVML&
zm9YD7gG|VJ_MyZ-$7GIQNUag}jahy>CU&g({`8#DT8qtE$c-Gc10$HXqY%2cAQVh67Mi|!d?o+#<32qKBYH;
zTA8dp?T$OnIokfw+_{!uoz26p^9kMw$9yS3LriSDv-*iCmAj7E^d6-|78Z-JeE2l{
z$5Y7$Q)6ShYQvI+@o1>D9;REUevpYh$~t&J6rquZy>kHCscE+oiajVmhZRN^^VK(0
z6Yi2CHuk9DeXxi4=mTu}A&%LpsSty9bhUuc#nX^_xiMG+v*UwK*+{{UkC>xE>%AEI
zi3Gy3BRcGZ>F5Y1j&x|~XeRd@!Gr|&OHR9}xJH&F2H?V|Fej7-PZq~Sof`$pECC%}
zR2Um3O@l>LMJCkrm-sQVKmRu2oO)I=}{Addi&P|uHeM=0UAq(;mCq~NLSdz|T
z2}G%Xg$9Bn*pm!X1~>7zIcPD0^I+)n1?2{G+zA?JCQ8{nVubBBS`YSlSv$!LZ4}+_QA?^zu@b1oE
zbMRlfH4d}shlS`wHvV^2ST;Ym)FKOPLeFy#-}s=EN<(wS_!@T5iSB$ojfiBL?W2+W
zu&Zb-6grJZe(ZHG<3H3I71QRq!SV4viXoGT>VB-1`}P%#&&Zs
z-Dj=6tg|RlJ9j#&cfMDWTEXwb96kkNj*7E>Z#iCa{c!n@+%iA)?H_``v=Gdk()N_K
z5Sgfuo!dje+ovgdkZ%gDniP>a0zoWx0f5ccMwbHUWC1!0+MFcZkt#rUaj*tX=(|F~
zyA2z}MQ8k@!d~?&rLwWb9JAGcGgtB&Oww)7(9wt4Y8=rS3poGy`-VJ6n^0$Z88btx
z*mn7s3RkF3m?)?g;%5V_O%sp5Z9}J<`Yp?;XVK8g4%k-*Lk|nGPUfNS7pNGLL%Ne_
zM`?bCXy~@5nAs$Z@eRTaCi5yo;PW`eS&?xOC
zYD2=lo7O0a7P^~MVk_xqxX){v#WHV|%=#+^y&@Ay_ybPyy<&Mja}6=6XkzA#`=ban
zNX$MKE(pYZb_MDMwGVr+);oYp5Im9{hZ@2C9LHHQ6;JZZXwPup1`NRsJS2})z)V#QiP~;G+bjdV)ndTgBW*`fy@3?f?~#p
z)vh=7#@PyuQ8eCQV{})0)t!g`H%|F1pO)eli4T0HMt0r7LH`eC%O7
zeLFms4`&~{2y-)z#F}1b#s+WMW-*1eenoZ`VcppAscw~v)GMSa_}*8w1~R~g>)RPJ
zz}69L?k6%gI!zXok`<{Vb!cCPJn~Qme+?r$feD(}%*SBvvM>)np*SRX-l%1V
z<^~goGb_hVU3Wy6LG1Z%wL~F0sMxH4gOwyQf_~GJ8R#+w+F=%*@%k9L18ozHPGVt`
zm}rj3I!#d2Ny2I(&;c&!TnXioMxLOnMX-j2bBN(&r`CP(@(s2uKrRl
z;D@F2Po^EYfVDxxKU=On4g8)E{PS1t>bfkH`x!}ksL$fud}!m6(hk3VrNcR$_xmjx
zU+>tX-}z=K3z)`~T|Ji|z;`h*-E3^G2w%>_caqR`Ol$%`DBxfY)6ltif{9>rqJXpv
zY9dF`t=M~$3AYU!RBn45*N|P)yq?#mmz(odpK%;B3zhw=n?AqmaLqrqfs9B`}P%dT1mb{kae^k;7lRnA{bEW%>-1TBS)esQ6d+80EMKXbZJ|0d^vL#Zr9TKDlI=}&?=3aLL
ziEEtR=QdHSh_rIOB%%OHrjs%p&l{lk9$9X>Z`oPZ$ynnBkB>mx+X&@57MKbh_l>tY3G)mwOXGy*WkXXrE
zKAGHVl>77<>j;{j5~Z3fMGx8~BKWw^deuZePW4Y)EU6|T7-w@Xrd9lK`%Q%K}
z7P=*;Z3V=VOn$E43THniLXxvXu$3`{py6yruG-OKKLn-u&Tp4ZkdhT#n~}plWMdPU
zci~)oKMNaALKmy|(F)_Iud<6qLCdQj3HU3cAv3Q{~|HvACfP_gT
zF-@=KCNVKQ4n9v%bbEw$mxMkDpwF{+oM$}0KY~vpp>IO?cqTgNPSJUe)NMA7$Hs3y
zg}==rSp;J`fV){D)RO|K`+WSw2qvCSxGy}PM^heRJkKOGbTFwREF_0PSBmftm0mRQ
z@NTD09SY)CygDv2-j9i2`}|hiZX9~VKMzUOil6a#MeKi@D#UsZ&9F*^xN#<7m`BJZ!JAC55FZmS
zz|SCaELyK6v()2RSRM;=>OsByWAq`h@E|maEj&XB!iGLt;}-}ph7Hywtlo1+(NoF<
z6R~>AVBbS*j@EB0mE)%#-oi|DqZ0&}jSDyOSOsw**nSdrX3stR(_^KI$Lg4bG3NLt
zr;Y?bVj1Jr?pox0LI2?T6)@1*q+7)#_=7E&FZKj@1s>gC_JH*UVxseVrkeQpP9etX
z5GGfEA0Hv4vaoq1!U*q0EemsGgrH$9`6xm;Zzh3}TfxHI2p!$Dr{B!GrjKU44npl_
zO61-3IEG|~KT_zjUZUTGiIKaZ0R$VU=fTrjaScKoA9~6sCD%h$?n_wgb)bQ7wEjL$
z?c@5*K$W?(@pn_8M;^Dw%C&{{>5t8tzKw+W<|B|o>=hdB5*v}XgliOG-_*!@a6UT;
z23%`p`^RJI1%Q?aH#L9*!x7Cd$+0X!gJh`7f~_RnS#b(2M_^NdXqGVI?AFu3s5<`(
z5x`$zrZrCkRyVP8CuOxrc)qAgVu()-V2NyaFMvBo!X4@>9pW_?CU|_g{q&FJ$kiVB
zCC(+3a6z^9vn>N>V1oO5d+}2G*B;@OAUfDf(ogHgo?-1*>qRwlmL!UO0~;dZ&ruDJ(
zZIr!lx69f~neo@~O?<*_LBLZMX6%jf2!!ilYxfIqmnx*r^YASdQb*be?xl?$m#p)Z
z50{H)#zh$|466jvhKdpV7*9&)G`@qe(wjh7ujVV&fDb(*C6WH0K~lF!EB1M~W7aC%
zD^i}%2%T(P#tY?J3~a_8Y&?W}woz(~gU@BJ&Uw5V`eX45D>Xtw=dyGnPZf0lXL^R}
z*{S#f61SaHa>%7VmE<^^RDBn%OTJjw?wB;k>^c94uWgf8>6U@e9kJM7p8iI|7ejX5bO>Xg9>Jj2
zyN*>`G@SiHwT!Nuo@>^T@}ut=zuhpsJyq+Bb*<5ZzVGJwCT-&a`=p~D6OAE>RWiS=
zw~UyCB+Yg$d*f?q_ui%duu*pdIE`NqDA
zs}C;eDAbt@&kt>3yRaY`3@3L4+0zpy4M-?H7*=KaW0$m)kwGP6Hc6d{{I_!HlcO_a-
z53JCH_As=((CZ#(?UtP@oqGa6qgB{t5G+pWZvGB?MtoufLUVrp5*W
z*|PF6zif>M6GcxU)o*cga`(d#ih6tA15hKChuDY;Jb;gry&k+ZGmRO)laHxTz}m6d;tnvro+
zYu6^){>C7JHqG#QiQ$W?7o}}6J0fH
z^8PdH`%Je-@GlXcp%Y%{9`ldU>(q@jojLq&$kJD^@c90qPj`@yx?X+1eGSwVsIyb=@MHv%B@b+f0|^$;B_>``GHF8(>~aHSAS
zN^P(sZ1q8TSVKT^4+hjbiuDRStCcCrmPwvM^qxH@KOtE29jXtlJZ=Vh((rCh1u1
zK{cu%Y@BQ{RvZ?QMk=~HQq$@%uA%TTJR*i@`5=6u
zTYAJArOBb|hq~%hAAid;$8Z_lt}({{(w$5tQMO}t4oWX=Tr4k#;?By(hj-TXJ=nBt
z;XXnqmW`{%G>=jJR_M}q3w!Cbn@2dRIb_Q<3c^SuRB6WsM}>zRCnw0Hz}mM@I!io$
zSGK00T~CTqBFIstn#gSMbs*;IZM!MI!+l4HZ!lGLEHZJ)V<%GE;1Na9L7skTFzPHR
zzvZo&_4<
z;c{)j?@Idqa>CDa6*!~FS__)qEpWl1z&pQOo#|0_|7<7KM||2vfQve-YrHU0Z?XWaRtoPsa-IoRra5pipw>;jMEEDK~f6c^Q1e6H`YV)qk^meQvlY
zL*tIGI4uR%QTf+zi9>=Lxx2a?=aEmu^B7s^j9!Y;A!L*@rk
zf+%|V=)11*EG?NQhvH@`HXoeW_4cR{|ERIq;uNWNE=S$wzl5d(&&j9%Q{*lQlAG-Y
zuD;+EpE#1zYW=u-J)of7_-SXTLdLQA4Tn%CY6A*mSdNMhna;02t$c3!7_yl0(Kg0W
zEt$HO$;F9#;F(*6Fenu9+TI>Z+zhB_fp)*_$w6)s!Il>dkecj2%JrYrNp$#_({FzQ
zMa%W6p1wS|6rM58fa|!hDR~=Pno$fcOPDoCoI~i33!0yP`+;_FB}-Kc)Vg)=wLxc=
zMnViR&6+J5wNXy#uV4MVb8`))yL-#au*NXOj=qV23D(Y6o5+##7v449;w?_RSj$?y
zzxMv{-mJ(o`yJK=p|kz2g{`>_rpb(#F#_$@HPEqol3QA&{n$vaLL}W)Q~q{6#vH<~
zA}rbU$|R&eDPMA94Hg3dBjAG=BI%?)xr7PB(HbGc1Cc5iKs^vS+r-r(
z!6fa1N;H^`Yb!vKG2B;1jY9r{y0ijesu&`59MtUKYD2PUy=*NZSEQF`aEu$U%EZ)F
zvW@{7r0it~(tz?j^|Gl^WlB)hf*z!9?HCYfpgdkC_eqxA1Zt0iFb1T;$W{-8z$Wgt
zG$dUQhA#X+YpFrwfq{@ZBm0mRK&657I-a=}w0#|NiUDlu83fk^pmq~dLoa(-?|#w@
zPnia4GGG|CG?oO@lmsdR$t!gr3k#`W@}$WojZI1x+eAs!e1DduZ2vL$1g9+({j9GxcQX+`C!Vj!=;JS_mJtw9TqcrYsje;FV>
z6(MDDZYV2Tdli6-5uVJAs(O$gEl-<;l;HVFQAkgTt;&GD835jTBQtE|o0u}%M0qyP$+w;Q}iw<3MbUxuBPeJv!DTP(QH+NW{oxW-%L`K#ju4?(LzvQETXVj?0
zLpASc53RrrU?r11P#sF%VVK5r%;@1P(z`5H{+Gc`R#650V}PEz|O%&vE_q0McVH
zknzHkgmiiV;zit~AGzBAdvPJ)PXoP*C@c{-ngD6BV9Nd8*>YuuZ@d$*8=}8l%BB`w
zQgJdk)gH@6IPh{@7=^ebpWs5rBPxzgESL>{;oUlIwFr9-W2PFu1E^)l;^JG~lVvCmr;O_^sWWg{3
zFz6pQpaYcc@im%!uJ#=CQUwe`VLGc^uW@cDJ4f{|2-C~M{(2B_sv8g^xZCO%RR)w9
z4wixS`LTH_BmS}TL%_IS=+PHq5{yO5218z|vLE>Dfcy*nRjmUoeEhvhFkyey({KJO
zs~~I6E%B&-gj2n`BFZfAb*O=h3Uk1R273S-*D4HAmdXp8w|`|AI4MP}{HLzA(EgHz
zO6)YGFO2h4wV!ixU05ee`tvuv9=~j`Yp!(Hr#*d=m=o33a&;{kY4aIc6hoW*K+`lp
zhM7Z-0?ZY;XvG{^8nmnphz$YpP?=}RF#u{)9wI`7861cysUrF2c`djN|Fc))Gk>wRXe2}aBOqyAwqM=<0UCA1
z$+2s)F{*j2C4`sY{<8TLW{?}IHyJ(YM`wEn1p2E))b{1=eB-ai0X%`1eJ}h2
z^rYcHo+p!dPBx6R4ynn2GPFo!p0AYKl!}-QFGRlBIHqkKs(kS!mX+sM*FDRDE*Kp3
z3Y?nk5Bc(@HFv>a$Cpbn?`xUt(N7lZKd;%gy#zciYrSQ9MmX=9S=aK`7F8CGheo#v
zyZ&YyE5apD0gZu6rg!5(qA`&3AQSj`>KSNWZXrRL8D0Gb8Wao@w)#jiOGz{Lm~=QV
zU;qtK?yMfPO~3`1FJXm%mtG!=0fW_n;bqzf?cZ81|gXtvbh*QG$}5gQ|sgtLBm3
zd=TaeyQHnD5!j6x*YIac*qC8IVHlzU%G^^fjmh&F&wX_S^5OV;0<-F_4mckqelqt5
z1DcqRJimaL`wmY1&0ThttR@gflzYZtXt}RtMT|(+fz(+@Z+4D%l#3B8FYGxOw~4o{
zjtdCD=^4pB906N9_L@a114uA0SF*Amvg1;C_MsX7Pa#HsLNeRn&g_{N6LPLmx$CZ?
zLV<|-UD4;`qR%EqoVr^Yakg9xuzkLM0Mv+|F>+pwWnyuk*?Jl>N^>O*YW;7}i+1<;
z`*O2l2t%tt@*r5<7xA35MncPgoe+(6^we=pTJ?*&gxW=QSY%8OXZ?Rq8r=8MxEHRy
zm}tC~$k=SD6Thtd^9FleuIl0veJIf~KFP}Kp5e{6j5A3#SCVYsRhn#WG+nf8U7zbN
z%(Yru&9gY1JC3`M@M~sEL
z>RN75Pw_EMkt#@b+x<35oWvYX_BxyrSpI|Mo?K~>vh!X_@ZlRQG~~M`=oBD#V!qqg
zri2-R#LZlTZCH!7$?17|+ccYPusqt$o_PCU&3yj2L
zDxqWFW4#WfsDIr*=FK^LAnoVM-gikV@r14FD=EVV4#j^?z4<6@?ErN1$H7JUybs@w
z(>e}$rRD8Cz>@v}9ZoB}_l-NOyYXd5@XoZ7;nhzI2bxRL%GXwn&aCdZm{w|huqr;O
z?1B?-ZTG>Ny$KawOF=T5-#P!Rx^nQu;d|Bby*2L+p1v2KaN*|(>GXOlj$ooB!wLcqGIQe_iP{A$fE%(w}htn^;`+ey``o*8=m!&h>)HB+RGp=sV2>zBX
z*phK0KI7)$jLQKT9cMEBN71>*Gxh&*{G79kZHAdk!(y(ROEuT#Qq48@Ms6VuT_n|%
zN}{$IMiQai%`Mk7N~F}xrG-i*={}dFQc-^#oS~DNj%ReOS4?TEa0bN}jl#^728-
zBQjTWH08}oinM-Nlf;$Y`KwEUdNUGwppc!(@$cLPvqBD|$Wd-(f%h{ZdH&83$sOeF
z$`coy!eZx6F6yjw-AOk@A@+rw_`2iN;<3i9G$<|x`J-~>$AufR^!b1Fr~W?pL!_tw
zU3Y>lNZ(Jg0gy_1-VH3>zH~)3Jf9LDvzLoeC$`O+%8Su+d_nMBF
zK5dP-tf>3rh}pCD-4Cii4ebqle(k^;@uv@-TVHk`{gUIV4qx}?R`RisZy!5NCdFG*
zUeBn;7SwN%Kgo-F(kSx<&|C_it46-_P%`r7FLIS#i_G<{Dpr>8gD$(&q6P(-8l=sX
zA!V4*0}q&dd41~?OlfU0ZE*^Y)!&lCpnelImqf~sB-?TN~Hh2AaNUig)HatjC7IK{dO1nQDaLdJufS`*0bEus=;pYfZwYiXn9+m-N
zOa3?S+SRK~!n>v8m$viW6U-t>ovOGO^mZ%@z~dP?d&HYN$SK5lRu9
z)#TN}1@5CWg$^znY+MVDy=#}s82uTbCp3OYU?xZ%(+&B4VpvqRaM!Mg+y-7*`3FH%
z<&Ce;n=jusLcFgz6Xl$Zcjp~5Q+Ds)JfHL+hTCsEplyHY%J*&hL&}vQuusa3;ms9o
zF0@o1OHZ7+cS1ETa(~y`l=jq+F_sg&Ym6^WJ(`30v*%=!cc`)A1vgtZ{8HQ$Ov@T8
zD~A=E(PA4F&c%ySWSsCV%BZ#x&@e5-h4sP4kR6fkS
z->Sw#;Y-dc0=@)S^F_JL%Chg4!`4+R6UO$W=*vc)xGSmM?prgmO;!1>Jfpr5<~grdW2ZV$PHtqM$h#}-1LwfMfn*fTd`C@rr03+5yMH&BJ*pik4=FEn7TGUqWgWlj`gv;{`;hHxHQVL|M;%;
z8TS<9J_+s`dx^wWpE@?3xkq+@n;*q_<^HwbVNHtmQFt)15sQ~zJmm6q=t_pq;>3cW
z)hHp-^6!6}nj)sRCb&3BKQ%}*liua~ZfNX)!c!+9VHH}5reJ~#FATe^xRJ>sR!g!-&)4HD87
zo8(fj1W))`s?d7a{+Qjh(cAcKSz8xG8t!zlNgU0?
zGpSCaajuri&&z_x;tFEo2aU0U%ss3q)GzTM^-wP6T8e)myei$4gi{Y*kgD4BrJEWF
zQ)|#3c@jqh%azjiBYXb26sPN+r>g4XzRls+N|tz(35ln#_nOj}szGB~Rr&stklmD(
zmnE$iEr^^}eNU87uU)&}CjD4GS$z7ap6WSno3peFw}KjcKSg=Dc8}GU;2IUoO#+)^0|oAdx{Oy%
z=sabe?=0D@cld)-YwY-ypu!do-ZV@@tthE`t4gc8>-Fcc!L^Onx-qXzckC^B`u3%@
zP-g5fPJi)FJ4WeYPRdx_ou1ft*?}%&F)h0fA-)XMa{2;V`*g$
z`i{_5Qc0gIciPHKGd9S5s%@R#oN2K(^0W&p4S$sS^zQEUJ^Syj0h&I({`otr
z=R^6yaOfNQqir;@u1;Q=R@DKWu(#)z4i678T}a_Qg$W!7zh%_lJ^2
zn*K1~FNAKIwng6QxEcL?n#420B43U>Lw2?u@~=2FQWE#pG^1!J^tFX-+1q87YCH1&
zKa1kwxLM2L;E~vfZ+jljv#Yz5h1avi#%7I2!xY@
zKCE%iH`zXPwefL1qsGF}v!m%rbI6t0S3@1mPcJj~dam}m*7EF1%;To*AFWzmbh4(0
z%Dp;UU)>--`|xVG^YWY9Y?w~JSC??IkFR1HQMOe$eJ5AntHPmJb{1P?x$pJJb)&Zr
z1kR_meV#|Xd)%!
z&%T*9JNdf3Pdt{#QO@_=fvJo~mi>gHH{46bqXMT2n=s7eyjl
zxP!xys@5MoqxFIkL>?p16Iz`cVf>um+BJ?TPB`Tc991VP52>6e?|
z$HTT)n{0(W&Ay))IazwD6b+4>krUg^Q|
zhoOEiZz1OFk#YK4uNOP328PT!=PlMCFSvLsG@B9G6c?*Tbd3$>a2935N%7q~_gzNx
zt(`|$o|Y4Qamx*wwF*$al;)pm6dYxlT90r@RWL`6z8`{5jEQzxswT@sopD;o%RQ_+
zYks~=ohtR$KQUL;`8s)FrLFT#(j8^*UbzP(PfPI#E=r-_Ti8!l>x#N}Ir2sn9XA$Y
z3F0g1z%WVFjlXDV+cNja`iTTT@S&yibCT+|fWtl>k`drN)tc$7Fs&-{bi?}?-}WKy
z)#3r=afWm6v`AmqlRx2tO?hGEXuALV@PVcd>bNSso~nv<_eP8to$;~6C70O$H~;v*
zqY1G7v*F8MAMaoJyEJ`py=RyEq0ALvxf1w$484_}%98UM$R1BWLo~Rr_N;nP+3F!J
zvWp9;)F7|7r=2k@gS7fWN-e$BRd()-l&5%CVS?S|3#_K2N~Df_B8?O2g7cN?51LTj
z750qfrn)aYV6V9JX1?!OXJk!-JSe2BEm+5658Knhn{L@LDOqt=(*@wzvHrEU-lp_l
zXCq~F_B2)s8d7>eB%=&emAMa@fY!IockG_$G5$DvjKqF*ikzpT)g&Qja|OJ}GO^5M
zU4)P=jfMt_8lE9sUBe}Z*crO2*Jy?J?qHRjWl@AvJi30RJKP7WuXex4&(v%bTi;uq
z;7)yO+@j1=3atsSmR79V&ee%xFj3VK$IrY-z{ji?>JtsZuLTTfs7X^xm{E!lPFC+M
z-g;x?VvvYX24it{g?G{8Lxo7|23-ZiqG%Qp;I8euBCp$3E|np^gG_p`=4oU=+bA-5@_{m!}hJvgDv!|h*%z6b`WK!)jf
z)mdyLYpsP=g>+JaWa+v+~1=EXv^3w5&%YoTP}!h83INvp^Sj8-$<
z$s&3vpox_ayou9TrARGPFZ$NBZ@LNrCEd)^H|b->RX+@j<&EZ9^XWwj7<#
z-g!+z@9v%dZn3XldH=5U_k=C^-Y$A0pD$k36Puo9z)9Ec$I;h3ZwU|d3M*b=#{-&k
z&FVB@!t#8+@STB(*i$lLk>Hq<>(hE@wdu3xcE>2=v}cFwOI7gkmYpfD*eu0`F!7;D
z;ehqpLl?YWe|_C{eaz=_($hZ!w@CGX->zrsIANeV5UlV%e2@Ku+zDLvlbe7d>*Z$;
z1p^|8M)?Z4t45?q`=wr7j*v(8p2UPLi3#_G>FY-(6;CYnkrdnbo4u#-rAz%<5l+Tl
zcmtd9v%o5b#_-kdz=Kgwg#BnRei+AvmV-rf#Y0t}ZN+gKza;Ly?dL9x%oy4Ky35FE
zFR1r^YZCTrDCB>
zz?LqtzSjPH?;BFwSz5GW>-n^>rRhe@QusY**<$E
zrQ7?Q?|F3NVF0B4C)C})y7ltAfrvfb@$ej-{S+9vF*UF^Bu!LwJ;mT)#tG|dy4y$M
zqP6y&Dt!I(*0HXG(;8~8u3P0yF7|$`f6_Y$l)Wu0N65e_@ex3YDEQAd_|}FCI=;w$
z4_(a((zg)}0KM^x_QU*$@76_?ryVb|^oE570}=$ZcPiU#j-PLHeQ>@d^29x9hWZs$
z*5@`y`EA72nhlhj#s;BkO=ox6qPoOWS5p5&U)6&`>OnL22eXrsgMgxdv%(6z{bX_7
zo|fE;-)jn^5rjB0NTA&ZjHnniR)lgAydgWp|F^<@wGyWac!<#3-lYIBY;hz|ntSS-
zii|%w6>cM>tNMlo1fDy=~!k?MCsv|gS{
zNVTA)T7{%;V;}3fL%7|Ec)>{BxRgroLJVF=wY-4Mlgg8ec%EH6?+Kpo5|2Sh^QWZ+
zgro%}rg_fE=Zca8CX@ZEh;l+Om<NuVW#03DkWDrjWkP9$ba!(P+=vxF71i;4tKw(9?M^zHCYlfdKaXa0$TVVlMJIhyK
z6VYsdK-z
z`wq|%0!ftYBLommVwVa56;Xy50ph49kqkhEdPcnK43QEp%a_-QGENeRPY6UEIwS)y
zKq;WYgc%5TE7OllJ24(?n8(6#87y%^;xRhXf3ajHiM#;3*myJPxoDqT>XZ3Mt?XI0|IpTn{1{hupD%SWN|1Q<0H4
zq_*scfeP*UjwsPuSx&p03V7f!mjv+NMer5|rc{=sUJ)U^
zWWiAZ3|xS{d5`{a2U!VVE|K6+7p6iMOl6#TeHRnQ1Z3IKat5YV1OrK6B~Er#@ZlnO
z9Mc+$4U*thLQD-QTz(YYBEq=IRsZZ{%Kl!|#{kxv4*}$`FaPh3=O6AT_NntC`RU*)8!)>xMtgKc(VkwGy_4U0vc>tvf=dbHpCU$9xM9)*^DwF8YUx31n@Zm;5v#i
zkOGPBz{$O^j3p3!URg#3>PFEr+0bPv;IVMQU;$$zfT2t+T%|yV2#ypXtEs@hMq#rs
zqmBXdWFp^h!}R)LHm#7r%WN2Y6g@`~I?=BvN-%gTtW{dQNs6fxz+_qbOVsw=IE;)$
z+Qcln#Ns!LFz6ds+n8`gI%L2A>V)uB&CQ!km_Mx)tOfEtqw?KMYy&ROW|Dh_B%cu0
zdGthQ;)%|~r3iMXhOD;XxKDt?LD0*U#>|
zUTA;4=*IP&=IaF?ua{|cm&$ja_v$WhzIH>HMlx1C`&)GwydYLi$=-}uv;+3s1-G)n
z2gXhPk+QGqGoFzm%@D@%VCAFeOLR7E6q8I~Z=z#XI(_wU$mj)`4<0$vB)c&nlL_7J
zZn?LT)v|cFXGC)of{lypeUWh?Zn@9FtdB232b!Q`lnDruLDGq?Tict1f
zxiC30;(2Z!}|b5$*EyfpBRC0BZ(_W&9aPw+XBXkF1rW7x~!%;
zDJ?4PHFgp@)*BH11Kt3<^UCr)cvRi6KT6VCYwHJHzn7800*{_{n9^!_f
zLJyK+zuLF1V%*-!YCYi%ye$D&>;YMu``6#Y;$AS2u@^>%2}q#T0!*b;fEd0KvOb59
zT)jEDss*^_$^>7bKo|i)N_*XKn2>kqxKW8^7|o%9yJ^
ziFII*-Z>|(&pT>1VREScm{FTa%#g;h&8ja}L~C7-DNMjsVuMxe@8LK2`0a1m?5Hqi
zy7<^6a;$zU*6&>d|4!VUJa9>i*ltQJ%PPyalUx|O@%kt1GQ0h#I_
zc1$f*Xx)IRdX0Z20A9>dqzJQ~1U4~7n{$P|=DrKbQ
zH{k0Jt)E#w<@>*K?#^`v>Mf^iNuY9{6~(_2cT8PZI}DKCL+N=Ge#kw|pP14WGW1J*xQm<9ecA^5+}(
z2lTd+|NZ^&^<;vKuwRt{XwWIPxZ