Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implemented audio extraction, adding audio streams, displaying audio stream #4

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
Prev Previous commit
Next Next commit
Corrected errors mentioned in prior PR. Initializes SS object with re…
…ference MEG. Implemented dispaly of all pulse channels (could be updated to be more user friendly).
Ashton Doane authored and Ashton Doane committed Sep 11, 2024
commit 32648c6377d25ed2963112c1cd492f97a15d1bbb
5 changes: 3 additions & 2 deletions src/ilabs_streamsync/example_script.py
Original file line number Diff line number Diff line change
@@ -16,8 +16,9 @@
extract_audio_from_video(cam, output_dir)
ss = StreamSync(raw, channel)

# ss.add_stream("/Users/ashtondoane/VideoSync_NonSubject/output/sinclair_alexis_audiosync_240110_CAM3_16bit.wav", channel=1)
# ss.plot_sync_pulses(tmin=0.998,tmax=1)
# TODO: Perhaps the extraction above could return the newly created paths so that this doesn't need to be hard coded.
ss.add_stream("/Users/user/VideoSync_NonSubject/output/sinclair_alexis_audiosync_240110_CAM3_16bit.wav", channel=1)
ss.plot_sync_pulses(tmin=0.5,tmax=50)

# subjects = ["146a", "222b"]

35 changes: 26 additions & 9 deletions src/ilabs_streamsync/streamsync.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@

from __future__ import annotations

import logging
import os
import pathlib
import subprocess

import logger
import matplotlib.pyplot as plt
import mne
import numpy as np
@@ -33,21 +33,21 @@ def __init__(self, reference_object, pulse_channel):
"""
# Check provided reference_object for type and existence.
if not reference_object:
raise TypeError("reference_object is None. Please provide reference_object of type str.")
raise TypeError("reference_object is None. Please provide a path.")
if type(reference_object) is not str:
raise TypeError("reference_object must be a file path of type str.")
ref_path_obj = pathlib.Path(reference_object)
if not ref_path_obj.exists():
raise OSError("reference_object file path does not exist.")
if not ref_path_obj.suffix == ".fif":
raise ValueError("Provided reference object is not of type .fif")
raise ValueError("Provided reference object does not point to a .fif file.")

# Load in raw file if valid
raw = mne.io.read_raw_fif(reference_object, preload=False, allow_maxshield=True)

#Check type and value of pulse_channel, and ensure reference object has such a channel.
if not pulse_channel:
raise TypeError("pulse_channel is None. Please provide pulse_chanel parameter of type int.")
raise TypeError("pulse_channel is None. Please provide a channel name of type str.")
if type(pulse_channel) is not str:
raise TypeError("pulse_chanel parameter must be of type str.")
if raw[pulse_channel] is None:
@@ -56,6 +56,7 @@ def __init__(self, reference_object, pulse_channel):

self.raw = mne.io.read_raw_fif(reference_object, preload=False, allow_maxshield=True)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here you are loading in (a second time) the same file you already loaded into the variable raw. Assuming that's a mistake?

setting that aside: what is the motivation for keeping a reference to the Raw object as part of the StreamSync object? I think all we need is sfreq and a numpy array of the pulse channel data (or am I forgetting something)?

self.ref_stream = raw[pulse_channel]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

raw[pulse_channel] returns a tuple of two arrays: (data, times). do we need the times? If not we can do raw.get_data(picks=[pulse_channel])


self.sfreq = self.raw.info["sfreq"] # Hz

self.streams = [] # of (filename, srate, Pulses, Data)
@@ -75,7 +76,7 @@ def add_stream(self, stream, channel=None, events=None):
self.streams.append((stream, srate, pulses, data))

def _extract_data_from_stream(self, stream, channel):
"""Extract pulses and raw data from stream provided."""
"""Extract pulses and raw data from stream provided. TODO: Implement adding a annotation stream."""
ext = pathlib.Path(stream).suffix
if ext == ".wav":
return self._extract_data_from_wav(stream, channel)
@@ -87,15 +88,30 @@ def _extract_data_from_wav(self, stream, channel):
srate, wav_signal = wavread(stream)
return (srate, wav_signal[:,channel], wav_signal[:,1-channel])
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not sure about the API here. we probably want this func and the _extract_data_from_raw func to return the same thing (e.g., tuple of same length), but for the Raw case there's no analogue to "audio channel". I also don't know what the audio channel is useful for --- e.g., if it's downsampled from 44.1kHz (typical audio) to 1kHz (typical MEG), it will be pretty much useless / unintelligible, so I don't see the point of syncing the audio data itself.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps I don't understand the end goal of this API. I wasn't intending to use the audio data for syncing (the pulses are the goal here), but rather holding onto it to create a file in the future that has been aligned. I'm not sure I understand the point about downsampling. Would you mind clarifying here?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if this question is still relevant, but here goes an answer: as I understand it, the end goal here is to convert researcher-created timestamps (in HH:MM:SS.ssssss format) into an mne.Annotations object, after first figuring out what tranformations (shift and/or stretch) must be done to get the video time domain aligned with the MEG time domain. In that sense, there is no need to write out the camera's audio channel to a WAV file (either before or after it's been warped/synced to the MEG time domain).


def remove_stream(self, stream):
pass

def do_syncing(self):
"""Synchronize all streams with the reference stream."""
# TODO (waves hands) do the hard part.
# TODO spit out a report of correlation/association between all pairs of streams

def plot_sync_pulses(self, tmin=0, tmax=float('inf')):
ashtondoane marked this conversation as resolved.
Show resolved Hide resolved
"""Plot each stream in the class."""
# TODO Plot the raw file on the first plot.
"""Plot each stream in the class.

tmin: int
Minimum timestamp to be graphed.
tmax: int
Maximum timestamp to be graphed.
"""
fig, axset = plt.subplots(len(self.streams)+1, 1, figsize = [8,6]) #show individual channels seperately, and the 0th plot is the combination of these.
# Plot reference_object
trig, tt_trig = self.ref_stream
trig = trig.reshape(tt_trig.shape)
idx = np.where((tt_trig>=tmin) & (tt_trig<tmax))
axset[0].plot(tt_trig[idx], trig[idx]*100, c='r')
axset[0].set_title("Reference MEG")
# Plot all other streams
for i, stream in enumerate(self.streams):
npts = len(stream[2])
ashtondoane marked this conversation as resolved.
Show resolved Hide resolved
tt = np.arange(npts) / stream[1]
@@ -148,7 +164,8 @@ def extract_audio_from_video(path_to_video, output_dir):
output_path]
pipe = subprocess.run(command, timeout=FFMPEG_TIMEOUT_SEC, check=False)

logger = logging.getLogger(__name__)
if pipe.returncode==0:
print(f'Audio extraction was successful for {path_to_video}')
logger.info(f'Audio extraction was successful for {path_to_video}')
else:
print(f"Audio extraction unsuccessful for {path_to_video}")
logger.info(f"Audio extraction unsuccessful for {path_to_video}")