From 2bb029b1f269fedf0a5c3ad16adbfa5cb061f8ae Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 8 Jul 2024 07:28:54 -0700 Subject: [PATCH 01/34] development --- GeneralTools/catSWIFT.m | 23 +- GeneralTools/postprocess_SWIFT.m | 97 +++++++ IMU/reprocess_IMU.m | 448 +++++++++++++++---------------- SBG/reprocess_SBG.m | 405 ++++++++++++---------------- Signature/reprocess_SIG.m | 334 +++++++++++------------ Waves/GPSwaves.m | 2 +- Waves/rawdisplacements.m | 19 +- Winds/reprocess_WXT.m | 120 ++++----- 8 files changed, 733 insertions(+), 715 deletions(-) create mode 100644 GeneralTools/postprocess_SWIFT.m diff --git a/GeneralTools/catSWIFT.m b/GeneralTools/catSWIFT.m index b5a42f3..887499f 100644 --- a/GeneralTools/catSWIFT.m +++ b/GeneralTools/catSWIFT.m @@ -2,13 +2,6 @@ function swift = catSWIFT(SWIFT) % Returns concatenated swift data in structure format -% ID -swift.ID = SWIFT(1).ID; - -% Met Height + CT depth -swift.metheight = SWIFT(1).metheight; -swift.ctdepth = SWIFT(1).CTdepth; - %Time, lat, lon, battery swift.time = [SWIFT.time]; nt = length(swift.time); @@ -178,12 +171,22 @@ swift.wavepower(:,it) = 0; swift.wavefreq(:,it) = NaN; else - swift.wavepower(:,it) = wavepower; - swift.wavefreq(:,it) = wavefreq; + swift.wavepower(1:length(wavepower),it) = wavepower; + swift.wavefreq(1:length(wavepower),it) = wavefreq; end end swift.wavepower(swift.wavepower<0) = 0; -swift.wavefreq = median(swift.wavefreq,2,'omitnan'); +wavefreq = median(swift.wavefreq,2,'omitnan'); +wavepower = NaN(length(wavefreq),nt); +% Interpolate to median frequency +for it = 1:nt + ireal = ~isnan(swift.wavepower(:,it)) & swift.wavepower(:,it)~=0; + if ireal > 3 + wavepower(:,it) = interp1(swift.wavefreq(ireal,it),swift.wavepower(ireal,it),wavefreq); + end +end +swift.wavepower = wavepower; +swift.wavefreq = wavefreq; swift.wavesigH = [SWIFT.sigwaveheight]; swift.wavepeakT = [SWIFT.peakwaveperiod]; swift.wavepeakdir = [SWIFT.peakwavedirT]; diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m new file mode 100644 index 0000000..c61b2b2 --- /dev/null +++ b/GeneralTools/postprocess_SWIFT.m @@ -0,0 +1,97 @@ +% [SWIFT,sinfo] = postprocess_SWIFT(expdir) + +% Master post-processing function, that calls sub-functions to reprocess +% raw SWIFT data. + +% K. Zeiden 07/2024 + +expdir = 'S:\SEAFAC\June2024\SouthMooring'; +cd(expdir) + +if ispc + slash = '\'; +else + slash = '/'; +end + +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +%% Loop through missions and reprocess + +for im = 1:length(missions) + + disp(['Post-processing ' missions(im).name]) + + missiondir = [missions(im).folder slash missions(im).name]; + cd(missiondir) + + % Locate L1 product, skip if does not exist. + % Else create 'sinfo' and modify L1 product. + l1file = dir([missiondir slash '*L1*']); + if isempty(l1file) + disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + else + load([l1file.folder slash l1file.name],'SWIFT'); + if isfield(SWIFT,'ID') + disp('Create information structure ''sinfo''') + sinfo.ID = SWIFT(1).ID; + sinfo.CTdepth = SWIFT(1).CTdepth; + sinfo.metheight = SWIFT(1).metheight; + SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); + disp('Saving new L1 product...') + save([l1file.folder slash l1file.name],'SWIFT','sinfo') + else + load([l1file.folder slash l1file.name],'sinfo') + end + end + + %% Reprocess IMU + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) + disp('Reprocessing IMU data...') + calctype = 'IMUandGPS'; + filtertype = 'RC'; + saveraw = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); + else + disp('No IMU data...') + end + + %% Reprocess SBG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) + disp('Reprocessing SBG data...') + saveraw = false; + useGPS = false; + interpf = false; + [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + end + + %% Reprocess SIG + % if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + % disp('Reprocessing SIG data...') + % plotburst = false; + % readraw = false; + % [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + % end + + %% + % Reprocess WXT + + % Reprocess Y81 + + % Reprocess ACS + + % Reprocess PB2 + + % Reprocess IMU + + % Reprocess AQD + + % Reprocess AQH + + + %clear SWIFT sinfo +end + + + diff --git a/IMU/reprocess_IMU.m b/IMU/reprocess_IMU.m index 9856dcf..ab5a0a6 100644 --- a/IMU/reprocess_IMU.m +++ b/IMU/reprocess_IMU.m @@ -1,3 +1,5 @@ +function [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw) + % reprocess SWIFT v3 wave results using a surface reconstruction % and acounting for listing or capsizing during icing conditions % loop thru raw data for a given SWIFT deployment, then @@ -11,262 +13,254 @@ % use RC prefilter in displacements, Oct 2016 % revert to original directional moments, Oct 2017 % optional prefilters and GPS, GPSandIMU reprocessing, Jun 2022 +% K. Zeiden, July 2024 +% reformatting for use in master postprocessing script, +% 'postprocess_SWIFT'. +% Turned into function with mission directory as input +% User input to determine what type of data to use (IMU, GPS or both), +% whether to save raw displacements, and what filter type to use -%% set up -clear all; close all -parentdir = pwd; % change this to be the parent directory of all raw raw data (CF card offload from SWIFT) -tic -saverawdisplacements = false; % logical flag to increase speed by not saving the raw displacements. +if ispc + slash = '\'; +else + slash = '/'; +end -%% choose a prefilter -%prefilter = {'no'}; %prefilter = str2cell('no') -prefilter = str2cell('RC'); -RC = 3.5; -%prefilter = str2cell('elliptic'), dB = 5; % lower is strong filter?? - %note that dB is set seperately (again) within rawdisplacements.m +%% Make sure 'calctype' specifies 'IMU', 'GPS' or 'IMUandGPS' -%% load existing SWIFT structure created during concatSWIFTv3_processed, replace only the new wave results -% save time by screening the existing structure for bad bursts (out of water, etc) -cd(parentdir); -wd = pwd; -wdi = find(wd == '/',1,'last'); -wd = wd((wdi+1):length(wd)); +if ~strcmp(calctype,'IMU') && ~strcmp(calctype,'GPS') && ~strcmp(calctype,'IMUandGPS') + error('Please specify ''IMU'', ''GPS'' or ''IMUandGPS'' for reprocessing.') +end -telemetryfile = dir('*SWIFT*.mat'); -load(telemetryfile(1).name) % loads the standard structure from onboard processing (named for the workding dir 'wd') -%save([wd '_onboardprocessing.mat']) % saves onboard results (for posteriety) -IMUresults = SWIFT; % make copy to use in populating with GPS results -GPSandIMUresults = SWIFT; % make copy to use in populating with GPS results -GPSresults = SWIFT; % make copy to use in populating with GPS results +%% Filter Type -prune = false(1,length(SWIFT)); % initialize logical array for later pruning of bad data +if strcmp(filtertype,'RC') + prefilter = 'RC'; + RC = 3.5; +elseif strcmp(filtertype,'elliptic') + prefilter = 'elliptic'; + dB = 5; % Lower is a strong filter? + % note that dB is set seperately (again) within rawdisplacements.m +else + error('Filter type must be ''RC'' or ''elliptic''.') +end -%cd('IMU/Raw/') % v3.2 -cd('COM-6/Raw/') % v3.3 +%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); -%% loop thru raw data +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit IMU reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end -dirlist = dir('20*'); +% Flag bad wave data +badwaves = false(1,length(SWIFT)); -for di = 1:length(dirlist), - - cd([dirlist(di).name]) - filelist = dir('*.dat'); +%% Loop through raw burst files and reprocess +bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*IMU*.dat']); + +for iburst = 1:length(bfiles) + + disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - for fi=1:length(filelist), + % Read or load raw IMU data + if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) + disp('Reading raw IMU data...') + [AHRS,GPS] = readSWIFTv3_IMU([bfiles(iburst).folder slash bfiles(iburst).name]); + else + load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'],'AHRS','GPS') + end + + % If data is missing, skip burst + if isempty(GPS) || isempty(AHRS) || length(AHRS.Accel) < 12000 + disp('No data. Skipping...') + continue + end - % read or load raw data - if isempty(dir([filelist(fi).name(1:end-4) '.mat'])), - [ AHRS GPS ] = readSWIFTv3_IMU( filelist(fi).name ); - else - load([filelist(fi).name(1:end-4) '.mat']), - end + % Find matching time index in the existing SWIFT structure + time = median(datenum(GPS.UTC.Yr, GPS.UTC.Mo, GPS.UTC.Da, GPS.UTC.Hr, GPS.UTC.Mn, GPS.UTC.Sec),'omitnan');% First entries are bad (no satellites acquired yet) + [tdiff,tindex] = min(abs([SWIFT.time]-time)); + if tdiff > 1/(5*24) % If no close time index, skip burst + disp('No time match. Skipping...') + continue + end - % make sure there is data to work with - if ~isempty(GPS) & ~isempty(AHRS) & length(AHRS.Accel) > 12000 & ~isempty(SWIFT), - - % find matching time index for the existing SWIFT structure, - % (for replacing onboard processed results) - % use median to get burst time, because first entries are bad (no satellites acquired yet) - time = nanmedian(datenum(GPS.UTC.Yr,GPS.UTC.Mo, GPS.UTC.Da, GPS.UTC.Hr, GPS.UTC.Mn, GPS.UTC.Sec)); - [tdiff tindex] = min(abs([SWIFT.time]-time)); - if tdiff < 1/(5*24) % proceed if time match is close enough - - % sampling rates and frequency bands - dt = median(diff(AHRS.Timestamp_sec)); % time step of raw IMU data - if isnan(dt) - dt = 600 ./ length(AHRS.Accel); - else - end - fs_ahrs = 1/dt; % should be 25 Hz - fs_gps = 1000./median(diff(GPS.UTC.mSec)); % should be 4 Hz - f_original = SWIFT(tindex).wavespectra.freq; % original frequency bands from onboard processing - - - %% reconstruct sea surface by double integrating (and prefiltering) the accelerations, with rotations - [y,x,z, hs ] = rawdisplacements(AHRS, prefilter); % call is [y,x,z] to get output in east, north, up instead of NEU - if strcmp( cellstr( prefilter ), 'RC') % eliptic prefilter option - x=x'; y=y'; z=z'; - end - if saverawdisplacements - save([filelist(fi).name(1:end-4) '.mat'],'z','-APPEND') % option to add heave estimates to each data file - end - - %% reprocess for waves from the IMU (AHRS) displacements - good = ~isnan( x + y + z); - [ Hs, Tp, Dp, E, f, a1, b1, a2, b2, check ] = XYZwaves(x(good),y(good),z(good),fs_ahrs); %wave spectra based on displacements - - % interp to the original freq bands - E = interp1(f,E,f_original); - a1 = interp1(f,a1,f_original); - b1 = interp1(f,b1,f_original); - a2 = interp1(f,a2,f_original); - b2 = interp1(f,b2,f_original); - check = interp1(f,check,f_original); - - % replace scalar values - IMUresults(tindex).sigwaveheight = Hs; - IMUresults(tindex).peakwaveperiod = Tp; - IMUresults(tindex).peakwaveperiod = Tp; - IMUresults(tindex).peakwavedirT = Dp; - IMUresults(tindex).wavespectra.energy = E; - IMUresults(tindex).wavespectra.a1 = a1; - IMUresults(tindex).wavespectra.b1 = b1; - IMUresults(tindex).wavespectra.a2 = a2; - IMUresults(tindex).wavespectra.b2 = b2; - IMUresults(tindex).wavespectra.check = check; - - - %% prepares GPS and IMU data for reprocessing - [UT forinterp] = unique( AHRS.GPS_Time.TimeOfWeek ); - az = interp1( AHRS.GPS_Time.TimeOfWeek(forinterp), AHRS.Accel(forinterp,3), GPS.Time.TimeOfWeek(end-2047:end) ); - indices = 1:2048; - az = interp1( indices(~isnan(az) ) , az(~isnan(az)), indices,'linear',nanmean(az))'; - u = GPS.NED_Vel.Velocity_NED(end-2047:end,2); - v = GPS.NED_Vel.Velocity_NED(end-2047:end,1); - z_gps = GPS.Geodetic_Pos.H_above_MSL(end-2047:end); - - % prefilter - if strcmp( cellstr( prefilter ), 'elliptic') % eliptic prefilter option - [B,A] = ellip(3, .5, dB, 0.05/(fs_gps/2), 'high'); % original is ellip(3, .5, 20, 0.05/(fs/2), 'high'); - u = filtfilt(B, A, double(u)); - v = filtfilt(B, A, double(v)); - z_gps = filtfilt(B, A, double(z_gps)); - az = filtfilt(B, A, double(az)) + nanmean(az); % restore the mean, because GPSandIMUwaves looks for it to be ~1 g - - elseif strcmp( cellstr( prefilter ), 'RC') % RC filter - alpha = RC / (RC + 1./fs_gps); - fu(1) = u(1); fv(1)=v(1); fz(1)=z_gps(1); faz(1)=az(1); - for ui = 2:length(u), - fu(ui) = alpha * fu(ui-1) + alpha * ( u(ui) - u(ui-1) ); - fv(ui) = alpha * fv(ui-1) + alpha * ( v(ui) - v(ui-1) ); - fz(ui) = alpha * fz(ui-1) + alpha * ( z_gps(ui) - z_gps(ui-1) ); - faz(ui) = alpha * faz(ui-1) + alpha * ( az(ui) - az(ui-1) ); - end - u = fu; v=fv; z_gps = fz; az=faz + nanmean(az); % restore the mean, because GPSandIMUwaves looks for it to be ~1 g - end - - %% reprocess with GPS and IMU - good = ~isnan( u + v + az ); - if sum(good)>1024 - [ Hs, Tp, Dp, E, f, a1, b1, a2, b2 ] = GPSandIMUwaves(u(good), v(good), az(good), [], [], fs_gps); - - % interp to the original freq bands - E = interp1(f,E,f_original); - a1 = interp1(f,a1,f_original); - b1 = interp1(f,b1,f_original); - a2 = interp1(f,a2,f_original); - b2 = interp1(f,b2,f_original); - - % replace scalar values, but not directional moments - GPSandIMUresults(tindex).sigwaveheight = Hs; - GPSandIMUresults(tindex).peakwaveperiod = Tp; - GPSandIMUresults(tindex).peakwaveperiod = Tp; - GPSandIMUresults(tindex).peakwavedirT = Dp; - GPSandIMUresults(tindex).wavespectra.energy = E; - GPSandIMUresults(tindex).wavespectra.a1 = a1; - GPSandIMUresults(tindex).wavespectra.b1 = b1; - GPSandIMUresults(tindex).wavespectra.a2 = a2; - GPSandIMUresults(tindex).wavespectra.b2 = b2; - GPSandIMUresults(tindex).wavespectra.check = check; - else - disp(['did not run GPSandIMU for ' datestr(GPSandIMUresults(tindex).time)]) - end - - %% reprocess with GPS only - - [ Hs, Tp, Dp, E_gps, f_gps, a1, b1, a2, b2 ] = GPSwaves(u, v, z_gps, fs_gps ); - - % interp to the original freq bands - E = interp1(f,E,f_original); - a1 = interp1(f,a1,f_original); - b1 = interp1(f,b1,f_original); - a2 = interp1(f,a2,f_original); - b2 = interp1(f,b2,f_original); - - % replace scalar values, but not directional moments - GPSresults(tindex).sigwaveheight = Hs; - GPSresults(tindex).peakwaveperiod = Tp; - GPSresults(tindex).peakwaveperiod = Tp; - GPSresults(tindex).peakwavedirT = Dp; - GPSresults(tindex).wavespectra.energy = E; - GPSresults(tindex).wavespectra.a1 = a1; - GPSresults(tindex).wavespectra.b1 = b1; - GPSresults(tindex).wavespectra.a2 = a2; - GPSresults(tindex).wavespectra.b2 = b2; - GPSresults(tindex).wavespectra.check = check; - - %% add raw displacements to SWIFT structure - - if saverawdisplacements - IMUresults(tindex).x = x; - IMUresults(tindex).y = y; - IMUresults(tindex).z = z; - IMUresults(tindex).u = u; - IMUresults(tindex).v = v; - IMUresults(tindex).rawlat = GPS.Geodetic_Pos.Lat_Lon(end-2047:end,1); - IMUresults(tindex).rawlon = GPS.Geodetic_Pos.Lat_Lon(end-2047:end,2); - end - - %% flag bad data - - if Hs==9999 | isnan(Hs) % only replace valid results - prune(tindex) = true; % set for pruning, b/c invalid wave result - end - - end - end + % Sampling rates and frequency bands + dt = median(diff(AHRS.Timestamp_sec)); % time step of raw IMU data + if isnan(dt) + dt = 600./length(AHRS.Accel); + else end - - cd('../') - -end - -cd(parentdir) + fs_ahrs = 1/dt; % should be 25 Hz + fs_gps = 1000./median(diff(GPS.UTC.mSec)); % should be 4 Hz + f_original = SWIFT(tindex).wavespectra.freq; % original frequency bands from onboard processing + + % Reconstruct sea surface (get raw displacements) + [y,x,z,~] = rawdisplacements(AHRS,prefilter); % call is [y,x,z] to get output in east, north, up instead of NEU + if strcmp(prefilter,'RC') + x=x'; y=y'; z=z'; + end + + % Prepare GPS and IMU data for reprocessing + [~,iinterp] = unique(AHRS.GPS_Time.TimeOfWeek); + az = interp1(AHRS.GPS_Time.TimeOfWeek(iinterp),AHRS.Accel(iinterp,3),GPS.Time.TimeOfWeek(end-2047:end)); + indices = 1:2048; + az = interp1(indices(~isnan(az)),az(~isnan(az)),indices,'linear',mean(az,'omitnan'))'; + u = GPS.NED_Vel.Velocity_NED(end-2047:end,2); + v = GPS.NED_Vel.Velocity_NED(end-2047:end,1); + z_gps = GPS.Geodetic_Pos.H_above_MSL(end-2047:end); + + % Prefilter + if strcmp(prefilter,'elliptic') % Elliptic filter -% Quality control -IMUresults(prune) = []; -GPSandIMUresults(prune)=[]; -GPSresults(prune)=[]; + [B,A] = ellip(3, .5, dB, 0.05/(fs_gps/2), 'high'); % original is ellip(3, .5, 20, 0.05/(fs/2), 'high'); + u = filtfilt(B, A, double(u)); + v = filtfilt(B, A, double(v)); + z_gps = filtfilt(B, A, double(z_gps)); + az = filtfilt(B, A, double(az)) + mean(az,'omitnan'); % restore the mean, because GPSandIMUwaves looks for it to be ~1g + + elseif strcmp(prefilter,'RC') % RC filter + alpha = RC / (RC + 1./fs_gps); + fu = NaN(1,length(u)); + fv = fu; fz = fu; faz = fu; + fu(1) = u(1); fv(1)=v(1); fz(1)=z_gps(1); faz(1)=az(1); + for ui = 2:length(u) + fu(ui) = alpha * fu(ui-1) + alpha * ( u(ui) - u(ui-1) ); + fv(ui) = alpha * fv(ui-1) + alpha * ( v(ui) - v(ui-1) ); + fz(ui) = alpha * fz(ui-1) + alpha * ( z_gps(ui) - z_gps(ui-1) ); + faz(ui) = alpha * faz(ui-1) + alpha * ( az(ui) - az(ui-1) ); + end + u = fu; v = fv; z_gps = fz; az = faz + mean(az,'omitnan'); % restore the mean, because GPSandIMUwaves looks for it to be ~1 g -%% save a big file with raw displacements, then a small file with stats only + end -SWIFT = IMUresults; + %% Recalculate wave spectra + if strcmp(calctype,'IMU') % using IMU displacements only -if saverawdisplacements - save([ wd '_reprocessedIMU_' prefilter{1} 'prefilter_displacements.mat'],'SWIFT') - SWIFT = rmfield(SWIFT,'x'); - SWIFT = rmfield(SWIFT,'y'); - SWIFT = rmfield(SWIFT,'z'); - SWIFT = rmfield(SWIFT,'u'); - SWIFT = rmfield(SWIFT,'v'); - SWIFT = rmfield(SWIFT,'rawlat'); - SWIFT = rmfield(SWIFT,'rawlon'); -end + igood = ~isnan( x + y + z); + [Hs,Tp,Dp,E,f,a1,b1,a2,b2,check] = XYZwaves(x(igood),y(igood),z(igood),fs_ahrs); + + % Interpolate back to the original frequency bands + E = interp1(f,E,f_original); + a1 = interp1(f,a1,f_original); + b1 = interp1(f,b1,f_original); + a2 = interp1(f,a2,f_original); + b2 = interp1(f,b2,f_original); + check = interp1(f,check,f_original); + + % Replace scalar values + SWIFT(tindex).sigwaveheight = Hs; + SWIFT(tindex).peakwaveperiod = Tp; + SWIFT(tindex).peakwavedirT = Dp; + SWIFT(tindex).wavespectra.energy = E; + SWIFT(tindex).wavespectra.a1 = a1; + SWIFT(tindex).wavespectra.b1 = b1; + SWIFT(tindex).wavespectra.a2 = a2; + SWIFT(tindex).wavespectra.b2 = b2; + SWIFT(tindex).wavespectra.check = check; + + elseif strcmp(calctype,'IMUandGPS') % using GPS velocities and IMU acceleration -save([ wd '_reprocessedIMU_' prefilter{1} 'prefitler.mat'],'SWIFT') + igood = ~isnan(u + v + az); + if sum(igood) > 1024 -%% plot results from the IMU reprocessing, as a sanity check + [Hs,Tp,Dp,E,f,a1,b1,a2,b2,check] = GPSandIMUwaves(u(igood),v(igood),az(igood),[],[],fs_gps); + + % Interpolate to the original freq bands + E = interp1(f,E,f_original); + a1 = interp1(f,a1,f_original); + b1 = interp1(f,b1,f_original); + a2 = interp1(f,a2,f_original); + b2 = interp1(f,b2,f_original); + + % Replace scalar values, but not directional moments + SWIFT(tindex).sigwaveheight = Hs; + SWIFT(tindex).peakwaveperiod = Tp; + SWIFT(tindex).peakwavedirT = Dp; + SWIFT(tindex).wavespectra.energy = E; + SWIFT(tindex).wavespectra.a1 = a1; + SWIFT(tindex).wavespectra.b1 = b1; + SWIFT(tindex).wavespectra.a2 = a2; + SWIFT(tindex).wavespectra.b2 = b2; + SWIFT(tindex).wavespectra.check = check; + else + disp(['Bad u,v, az values -- did not reprocess ' bfiles(iburst).name '...']) + end + + elseif strcmp(calctype,'GPS') % using GPS veloctiies and vertical displacement only -plotSWIFT(SWIFT) + [Hs,Tp,Dp,E,f,a1,b1,a2,b2,check] = GPSwaves(u,v,z_gps,fs_gps); + + % interp to the original freq bands + E = interp1(f,E,f_original); + a1 = interp1(f,a1,f_original); + b1 = interp1(f,b1,f_original); + a2 = interp1(f,a2,f_original); + b2 = interp1(f,b2,f_original); + + % replace scalar values, but not directional moments + SWIFT(tindex).sigwaveheight = Hs; + SWIFT(tindex).peakwaveperiod = Tp; + SWIFT(tindex).peakwavedirT = Dp; + SWIFT(tindex).wavespectra.energy = E; + SWIFT(tindex).wavespectra.a1 = a1; + SWIFT(tindex).wavespectra.b1 = b1; + SWIFT(tindex).wavespectra.a2 = a2; + SWIFT(tindex).wavespectra.b2 = b2; + SWIFT(tindex).wavespectra.check = check; -[Etheta theta f dir spread spread2 spread2alt ] = SWIFTdirectionalspectra(SWIFT, 1); + end + + %% Flag bad results + + if Hs == 9999 || isnan(Hs) % invalid wave result + disp('Bad waves. Flagging...') + badwaves(tindex) = true; + end -%% save the GPSandIMU results as their own structure + %% Save raw displacements to SWIFT structure and burst file if specified + if saveraw + save([bfiles(iburst).name(1:end-4) '.mat'],'z','-APPEND') + SWIFT(tindex).x = x; + SWIFT(tindex).y = y; + SWIFT(tindex).z = z; + SWIFT(tindex).u = u; + SWIFT(tindex).v = v; + SWIFT(tindex).rawlat = GPS.Geodetic_Pos.Lat_Lon(end-2047:end,1); + SWIFT(tindex).rawlon = GPS.Geodetic_Pos.Lat_Lon(end-2047:end,2); + end -SWIFT = GPSandIMUresults; +end -save([ wd '_reprocessedGPSandIMU_' prefilter{1} 'prefitler.mat'],'SWIFT') +%% Log reprocessing and flags, then save new L2 file or overwrite existing one -%% save the GPS results as their own structure +params.Data = calctype; +params.Filter = filtertype; -SWIFT = GPSresults; +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'IMU'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags.badwaves = badwaves; +sinfo.postproc(ip).params = params; -save([ wd '_reprocessedGPS_' prefilter{1} 'prefitler.mat'],'SWIFT') -%% close the timer +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') -toc +%% End function +end \ No newline at end of file diff --git a/SBG/reprocess_SBG.m b/SBG/reprocess_SBG.m index 2945bd3..eb0367b 100644 --- a/SBG/reprocess_SBG.m +++ b/SBG/reprocess_SBG.m @@ -1,3 +1,6 @@ +function [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf) + + % Batch Matlab read-in and reprocess of SWIFT v4 SBG wave data % reprocessing is necessary to fix a bug in directional momements % all data prior 11/2017 need this reprocessing @@ -5,279 +8,219 @@ % M. Schwendeman, 01/2017 % J. Thomson, 10/2017 add reprocessing to batch read of raw data, % and replace SWIFT data structure results. -clear all, close all - -saveraw = false; - -plotflag = false; - -parentdir = pwd; % change this to be the parent directory of all raw raw data (CF card offload from SWIFT) -%parentdir = ('/Volumes/Data/Newport/SWIFT19_15-18Oct2016'); % change this to be the parent directory (CF card offload from SWIFT) +% K. Zeiden, July 2024 +% reformatting for use in master postprocessing script, +% 'postprocess_SWIFT'. +% Turned into function with mission directory as input -readfromconcat = 0; % force starting with original onboard results -useGPSpositions = false; % option instead of GPS velocities for alt spectra -secondsofdata = 475; % seconds of raw data to process (from the end of each burst, not beginning), must be more than 1536/5 = 307.2 -interpf = false; % binary flag to interp to original (onboard) frequency bands - -%% load existing SWIFT structure created during concatSWIFT_processed, replace only the new wave results -cd(parentdir); -wd = pwd; if ispc slash = '\'; else slash = '/'; end -wdi = find(wd == slash,1,'last'); -wd = wd((wdi+1):length(wd)); -telemfile = dir('SWIFT*L1.mat'); -if ~isempty(dir([wd '_reprocessedSIG.mat'])) & readfromconcat~=1, - SIGrep = true; - load([wd '_reprocessedSIG.mat']) -else - SIGrep = false; - load(telemfile.name,'SWIFT') -end +% Length of raw burst data to process, from end of burst (must be > 1536/5 = 307.2 s) +tproc = 475;% seconds -cd('SBG/Raw/') % v4.0 +%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); -%% loop thru raw data +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end -dirlist = dir('20*'); +%% Flag bad wave data +badwaves = false(1,length(SWIFT)); -for di = 1:length(dirlist), +%% Loop through raw burst files and reprocess - cd([dirlist(di).name]) - filelist = dir('*.dat'); +bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*SBG*.dat']); - for fi=1:length(filelist), +for iburst = 1:length(bfiles) - % read or load raw data - if isempty(dir([filelist(fi).name(1:end-4) '.mat'])), - sbgData = sbgBinaryToMatlab(filelist(fi).name); - save([filelist(fi).name(1:end-4) '.mat'],'sbgData'), - else - load([filelist(fi).name(1:end-4) '.mat']), - end + disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) + % Read or load raw IMU data + if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) + disp('Reading raw SBG data...') + sbgData = sbgBinaryToMatlab([bfiles(iburst).folder slash bfiles(iburst).name]); + save([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'],'sbgData'), + else + load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'],'sbgData') + end - alltime = datenum(sbgData.UtcTime.year, sbgData.UtcTime.month, sbgData.UtcTime.day, sbgData.UtcTime.hour,... - sbgData.UtcTime.min, sbgData.UtcTime.sec + sbgData.UtcTime.nanosec./1e9); + % SBG time + time = datenum(sbgData.UtcTime.year, sbgData.UtcTime.month, sbgData.UtcTime.day, sbgData.UtcTime.hour,... + sbgData.UtcTime.min, sbgData.UtcTime.sec + sbgData.UtcTime.nanosec./1e9); + % Find matching time index in the existing SWIFT structure + btime = median(time,'omitnan');% First entries are bad (no satellites acquired yet) + [tdiff,tindex] = min(abs([SWIFT.time]-btime)); + if tdiff > 1/(5*24) % If no close time index, skip burst + disp('No time match. Skipping...') + continue + end - %% find matching time index - % match time to SWIFT structure and replace values - % use median to get burst time, because first entries are bad (no satellites acquired yet) - time = nanmedian(alltime); - [tdiff tindex] = min(abs([SWIFT.time]-time)); - if tdiff>1/48, - %disp('time gap too large at '), - %datestr(time) + % If not enough data to work with, skip burst + if isempty(tindex) || length(time)secondsofdata*5 & length(sbgData.ShipMotion.heave)>secondsofdata*5 & ... - length(sbgData.GpsPos.lat)>secondsofdata*5 & length(sbgData.GpsVel.vel_e)>secondsofdata*5 - - %% despike data and make convenience variables - t = alltime(end-secondsofdata*5+1:end); - t = filloutliers(t,'linear'); - z = sbgData.ShipMotion.heave(end-secondsofdata*5+1:end); - z = filloutliers(z,'linear'); - x = sbgData.ShipMotion.surge(end-secondsofdata*5+1:end); - x = filloutliers(x,'linear'); - y = sbgData.ShipMotion.sway(end-secondsofdata*5+1:end); - y = filloutliers(y,'linear'); - lat = sbgData.GpsPos.lat(end-secondsofdata*5+1:end); - lat = filloutliers(lat, 'linear'); - lon = sbgData.GpsPos.long(end-secondsofdata*5+1:end); - lon = filloutliers(lon, "linear"); - u = sbgData.GpsVel.vel_e(end-secondsofdata*5+1:end); - u = filloutliers(u,'linear'); - v = sbgData.GpsVel.vel_n(end-secondsofdata*5+1:end); - v = filloutliers(v,'linear'); - - bad = isnan( z + x + y + u + v + lat + lon); - t(bad)=[]; z(bad)=[]; x(bad)=[]; y(bad)=[]; u(bad)=[]; v(bad)=[]; lat(bad)=[]; lon(bad)=[]; - - %% plot raw heave - if plotflag %& length(alltime)==length(sbgData.ShipMotion.heave) - %plot(alltime,sbgData.ShipMotion.heave,'k'), hold on - %figure(1), clf - subplot(4,1,1) - plot(t,z,'b.') - datetick, grid, ylabel('Sea surface elevation [m]') - subplot(4,1,2) - plot(t,lat,'b.') - datetick, grid, ylabel('lat') - subplot(4,1,3) - plot(t,lon,'b.') - datetick, grid, ylabel('lon') - subplot(4,1,4) - plot(t,u,'r.', t,v,'g.') - datetick, grid, ylabel('[m/s]'), legend('u','v') - print('-dpng',[filelist(fi).name(1:end-4) '_raw_trimmed.png']) - end - - %% wave spectra - - f = SWIFT(tindex).wavespectra.freq; % original frequency bands - - fs = 5; % should be 5 Hz for standard SBG settings - - % reprocess to get proper directional momements (bug fix in 11/2017) - [ newHs, newTp, newDp, newE, newf, newa1, newb1, newa2, newb2, newcheck ] = SBGwaves(u, v, z, fs); - - % reprocess using GPS velocites to get alternate results - [ altHs, altTp, altDp, altE, altf, alta1, altb1, alta2, altb2 ] = GPSwaves(u,v,[],fs); - - % reprocess using GPS positions - [Elat fgps] = pwelch(detrend(deg2km(lat)*1000),[],[],[], fs ); - [Elon fgps] = pwelch(detrend(deg2km(lon,cosd(median(lat))*6371)*1000),[],[],[], fs ); - - - % interp to the original freq bands - if interpf - E = interp1(newf,newE,f); - if length(altE) > 1 , altE = interp1(altf,altE,f); else altE = NaN(size(f)); end - a1 = interp1(newf,newa1,f); - b1 = interp1(newf,newb1,f); - a2 = interp1(newf,newa2,f); - b2 = interp1(newf,newb2,f); - check = interp1(newf,newcheck,f); - else - E = newE; - f = newf; - a1 = newa1; - b1 = newb1; - a2 = newa2; - b2 = newb2; - check = newcheck; - end + end - if useGPSpositions - altE = interp1(fgps, Elat + Elon, f); + % Despike data and make convenience variables + t = time(end-tproc*5+1:end); + t = filloutliers(t,'linear'); + z = sbgData.ShipMotion.heave(end-tproc*5+1:end); + z = filloutliers(z,'linear'); + x = sbgData.ShipMotion.surge(end-tproc*5+1:end); + x = filloutliers(x,'linear'); + y = sbgData.ShipMotion.sway(end-tproc*5+1:end); + y = filloutliers(y,'linear'); + lat = sbgData.GpsPos.lat(end-tproc*5+1:end); + lat = filloutliers(lat, 'linear'); + lon = sbgData.GpsPos.long(end-tproc*5+1:end); + lon = filloutliers(lon, "linear"); + u = sbgData.GpsVel.vel_e(end-tproc*5+1:end); + u = filloutliers(u,'linear'); + v = sbgData.GpsVel.vel_n(end-tproc*5+1:end); + v = filloutliers(v,'linear'); + + % Remove NaNs? + ibad = isnan(z + x + y + u + v + lat + lon); + t(ibad) = []; z(ibad) = []; x(ibad) = []; y(ibad)=[]; u(ibad)=[]; + v(ibad)=[]; lat(ibad)=[]; lon(ibad)=[]; + + % Recalculate wave spectra to get proper directional moments + % (bug fix in 11/2017) + f = SWIFT(tindex).wavespectra.freq; % original frequency bands + fs = 5; % should be 5 Hz for standard SBG settings + [newHs,newTp,newDp,newE,newf,newa1,newb1,newa2,newb2,newcheck] = SBGwaves(u,v,z,fs); + + % Alternative results using GPS velocites + [altHs,altTp,altDp,altE,altf,alta1,altb1,alta2,altb2] = GPSwaves(u,v,[],fs); + + % reprocess using GPS positions + [Elat,~] = pwelch(detrend(deg2km(lat)*1000),[],[],[], fs ); + [Elon,fgps] = pwelch(detrend(deg2km(lon,cosd(median(lat))*6371)*1000),[],[],[],fs); + + + % Interpolate results to L1 frequency bands + if interpf + E = interp1(newf,newE,f); + if length(altE) > 1 + altE = interp1(altf,altE,f); + else + altE = NaN(size(f)); end + a1 = interp1(newf,newa1,f); + b1 = interp1(newf,newb1,f); + a2 = interp1(newf,newa2,f); + b2 = interp1(newf,newb2,f); + check = interp1(newf,newcheck,f); + else + E = newE; + altE = altE; + f = newf; + a1 = newa1; + b1 = newb1; + a2 = newa2; + b2 = newb2; + check = newcheck; + end + % Use spectra computed from GPS as alternative if specified + if useGPS + altE = interp1(fgps, Elat + Elon, f); + end - % take reciprocal of wave directions (to make result direction FROM) - dirto = newDp; - if dirto >=180, - newDp = dirto - 180; - elseif dirto <180, - newDp = dirto + 180; - else - end + % Convert wave directions to degrees FROM + dirto = newDp; + if dirto >=180 + newDp = dirto - 180; + elseif dirto <180 + newDp = dirto + 180; + else + end - % replace scalar values - SWIFT(tindex).sigwaveheight = newHs; - SWIFT(tindex).sigwaveheight_alt = altHs; - SWIFT(tindex).peakwaveperiod = newTp; - SWIFT(tindex).peakwaveperiod_alt = altTp; - SWIFT(tindex).peakwavedirT = newDp; - SWIFT(tindex).wavespectra.energy = E; - SWIFT(tindex).wavespectra.energy_alt = altE; - SWIFT(tindex).wavespectra.freq = f; - SWIFT(tindex).wavespectra.a1 = a1; - SWIFT(tindex).wavespectra.b1 = b1; - SWIFT(tindex).wavespectra.a2 = a2; - SWIFT(tindex).wavespectra.b2 = b2; - SWIFT(tindex).wavespectra.check = check; - - % include raw displacements (5 Hz) + % Replace new wave spectral variables in original SWIFT structure + SWIFT(tindex).sigwaveheight = newHs; + SWIFT(tindex).sigwaveheight_alt = altHs; + SWIFT(tindex).peakwaveperiod = newTp; + SWIFT(tindex).peakwaveperiod_alt = altTp; + SWIFT(tindex).peakwavedirT = newDp; + SWIFT(tindex).wavespectra.energy = E; + SWIFT(tindex).wavespectra.energy_alt = altE; + SWIFT(tindex).wavespectra.freq = f; + SWIFT(tindex).wavespectra.a1 = a1; + SWIFT(tindex).wavespectra.b1 = b1; + SWIFT(tindex).wavespectra.a2 = a2; + SWIFT(tindex).wavespectra.b2 = b2; + SWIFT(tindex).wavespectra.check = check; + + % Save raw displacements (5 Hz) if specified + if saveraw SWIFT(tindex).x = x; SWIFT(tindex).y = y; SWIFT(tindex).z = z; - - % include raw times (5 Hz) SWIFT(tindex).rawtime = t; - - % include raw GPS velocities (5 Hz) SWIFT(tindex).u = u; SWIFT(tindex).v = v; - - - % remove bulk result if wave processing fails (9999 error code) - if newHs == 9999, - disp('wave processing gave 9999') - SWIFT(tindex).sigwaveheight = NaN; - SWIFT(tindex).peakwaveperiod = NaN; - SWIFT(tindex).peakwaveperiod = NaN; - SWIFT(tindex).peakwavedirT = NaN; - end - - if altHs == 9999, - SWIFT(tindex).sigwaveheight_alt = NaN; - SWIFT(tindex).peakwaveperiod_alt = NaN; - end - - if newDp > 9000, % sometimes only the directions fail - SWIFT(tindex).peakwavedirT = NaN; - end - - - else - % ignoreif insufficient raw data - disp('not enough raw data') - SWIFT(tindex).u = []; - SWIFT(tindex).v = []; end - end - cd('../') + % Flag bad bursts when processing fails (9999 error code) + if isempty(u) + badwaves(tindex) = true; + end -end + if newHs == 9999 + disp('wave processing gave 9999') + SWIFT(tindex).sigwaveheight = NaN; + SWIFT(tindex).peakwaveperiod = NaN; + SWIFT(tindex).peakwaveperiod = NaN; + SWIFT(tindex).peakwavedirT = NaN; + badwaves(tindex) = true; + end -cd(parentdir) + if altHs == 9999 + SWIFT(tindex).sigwaveheight_alt = NaN; + SWIFT(tindex).peakwaveperiod_alt = NaN; + end -%% Quality control -bad = false(size(SWIFT)); + if newDp > 9000 % sometimes only the directions fail + SWIFT(tindex).peakwavedirT = NaN; + end -for si=1:length(SWIFT) - if isempty(SWIFT(si).u), - bad(si) = true; - else - end +% End file loop end -SWIFT(bad) = []; - - -%% (re)plotting - -if ~isempty(SWIFT) - - if plotflag==true - plotSWIFT(SWIFT) - - [Etheta theta f dir1 spread1 spread2 spread2alt ] = SWIFTdirectionalspectra(SWIFT, 1, 1); - end - - %% save a big file with raw displacements and dir spectra - if saveraw +%% Log reprocessing and flags, then save new L2 file or overwrite existing one - save([ wd '_reprocessedSBG_displacements.mat'],'SWIFT')%,'Etheta','theta','f') +params.AltGPS = useGPS; - end - - - %% save a small file with stats only - - SWIFT = rmfield(SWIFT,'x'); - SWIFT = rmfield(SWIFT,'y'); - SWIFT = rmfield(SWIFT,'z'); - SWIFT = rmfield(SWIFT,'u'); - SWIFT = rmfield(SWIFT,'v'); - SWIFT = rmfield(SWIFT,'rawtime'); +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'SBG'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags.badwaves = badwaves; +sinfo.postproc(ip).params = params; - if SIGrep, - save([ wd '_reprocessedSIGandSBG.mat'],'SWIFT') - else - save([ wd '_reprocessedSBG.mat'],'SWIFT') - end +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +%% End function end \ No newline at end of file diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index be3edc3..f1048e3 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -1,4 +1,4 @@ -function [SWIFT,SIG] = reprocess_SIG(missiondir,savedir,varargin) +function [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst) % Reprocess SWIFT v4 signature velocities from burst data % Loops through burst MAT or DAT files for a given SWIFT deployment, @@ -80,6 +80,11 @@ % simplify directory usage,convert reprocess_SIG to % function where mission directories are inputs % removed spectral estimate of dissipation rate, as is debunked +% July 2024 (K. Zeiden) +% some reformatting to adhere to postprocess_SIG.m and mirror +% other reprocessing scripts. Moved 'readraw' and 'plotburst' as +% external toggles/function inputs. The others are changed less +% frequently so they are 'internal' % NOTE: Known issue -- sometimes the ADCP 'sputters' and for a few minutes % will record perfectly periodic ping-ping oscillations in correlation, @@ -88,35 +93,19 @@ % traps due to the periodic oscillations which make the mean value % reasonable. So far only known to have happened on SWIFT 22, LC-DRI Exp. -%% Make sure save directory exists - -if ~exist(savedir,'dir') - error('Save directory does not exist.') +if ispc + slash = '\'; +else + slash = '/'; end -%% Load/Save/Plot Toggles - -% Plotting toggles -opt.readraw = false;% read raw binary files -opt.saveSWIFT = false;% save updated SWIFT structure -opt.saveSIG = false; %save detailed sig data in separate SIG structure -opt.plotburst = false; % generate plots for each burst -opt.plotmission = false; % generate summary plot for mission -opt.saveplots = false; % save generated plots - -% Compare with varargin -togvars = fieldnames(opt); -if length(varargin) >= 1 - for ivar = 1:length(varargin) - if any(strcmp(varargin{ivar},togvars)) - opt.(varargin{ivar}) = true; - else - error(['Input toggle ''' varargin{ivar} ''' is not an option']) - end - end -end +%% Load User Input Toggles +opt.readraw = readraw;% read raw binary files +opt.plotburst = plotburst; % generate plots for each burst -%% Other QC params +%% Internal Toggles +opt.saveSIG = true; %save detailed sig data in separate SIG structure +opt.saveplots = true; % save generated plots % QC Options (broadband) opt.QCcorr = false;% (NOT recommended) standard, QC removes any data below 'mincorr' @@ -128,17 +117,13 @@ % Config Parameters opt.xz = 0.2; % depth of transducer [m] -% QC Parameters +% Calculation Parameters opt.mincorr = 40; % burst-avg correlation minimum +opt.pbadmax = 80; opt.maxamp = 150; % burst-avg amplitude maximum opt.maxwvar = 0.2; % burst-avg HR velocity (percent) error maximum -opt.pbadmax = 80; % maximum percent 'bad' amp/corr/err values per bin or ping allowed opt.nsumeof = 3;% Default 3? Number of lowest-mode EOFs to remove from turbulent velocity -if opt.nsumeof~=3 - warning(['EOF filter changed to ' num2str(opt.nsumeof)]) -end - %% Data type to be read in if opt.readraw ftype = '.dat'; @@ -146,73 +131,53 @@ ftype = '.mat'; end -%% Ensure input directories end with slash -if ispc - slash = '\'; -else - slash = '/'; -end - -if ~strcmp(missiondir(end),slash) - missiondir = [missiondir slash]; -end -if ~strcmp(savedir(end),slash) - savedir = [savedir slash]; -end - -dirdelim = strfind(missiondir,slash); -SNprocess = missiondir(dirdelim(end-1)+1:dirdelim(end)-1); -disp(['*** Reprocessing ' SNprocess ' ***']) - %% Load or create SWIFT structure, create SIG structure, list burst files -clear SWIFT SIG -mfiles = dir([missiondir 'SWIFT*.mat']); -if isempty(mfiles) - disp('No SWIFT structure found...') - SWIFT = struct; -else - if length(mfiles) > 1 - if any(contains({mfiles.name},'reprocessedSBG.mat')) - mfile = mfiles(contains({mfiles.name},'reprocessedSBG.mat')); % this might vary - elseif any(contains({mfiles.name},'reprocessedSIGandSBG.mat')) - mfile = mfiles(contains({mfiles.name},'reprocessedSIGandSBG.mat')); % this might vary - else - mfile = mfiles(1); - end - else - mfile = mfiles; - end - load([mfile.folder slash mfile.name],'SWIFT') - burstreplaced = false(length(SWIFT),1); +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); + +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return end +burstreplaced = false(length(SWIFT),1); + SIG = struct; isig = 1; -% Populate list of burst files -bfiles = dir([missiondir 'SIG' slash 'Raw' slash '*' slash '*' ftype]); +%% Flag bad Signature data + +badsig = false(1,length(SWIFT)); + +%% List of burst files + +bfiles = dir([missiondir slash 'SIG' slash 'Raw' slash '*' slash '*' ftype]); if isempty(bfiles) error(' No burst files found ') end bfiles = bfiles(~contains({bfiles.name},'smoothwHR')); -% Deal with 'partial' files (two options) -%bfiles = bfiles(~contains({bfiles.name},'partial')); -ipart = find(contains({bfiles.name},'partial')); -idel = []; -for ip = 1:length(ipart) - pname = bfiles(ipart(ip)).name; - mname = [pname(1:end-12) '.mat']; - pdir = bfiles(ipart(ip)).folder; - if exist([pdir slash mname]) - idel = [idel find(strcmp({bfiles.name}',bfiles(1).name))]; +% Deal with 'partial' files: only use if full file is not available or is smaller +partburst = find(contains({bfiles.name},'partial')); +rmpart = false(1,length(partburst)); +for iburst = 1:length(partburst) + pdir = bfiles(partburst(iburst)).folder; + pname = bfiles(partburst(iburst)).name; + matburst = dir([pdir slash pname(1:end-12) '.mat']); + if ~isempty(matburst) && matburst.bytes > bfiles(partburst(iburst)).bytes + rmpart(iburst) = true; end end -bfiles(idel) = []; +bfiles(partburst(rmpart)) = []; nburst = length(bfiles); -%% Loop through and process burst files - +%% Loop through burst files and reprocess signature data for iburst = 1:nburst % Burst time stamp and name @@ -241,16 +206,22 @@ disp('Bad file (small), skipping burst...') continue end - - %%%%%%% FLAGS %%%%%% - - % Flag if burst time from name is very different from recorded - t0 = datenum(datestr(min(avg.time),'dd-mmm-yyyy HH:MM')); + + % Burst time + t0 = min(avg.time); if abs(btime - t0) > 12/(60*24) - disp(' WARNING: File name time disagrees with recorded time. Using recorded time... ') + disp(' WARNING: File name disagrees with recorded time. Using recorded time... ') + btime = t0; + end + + % Altimeter Distance + if isfield(avg,'AltimeterDistance') + maxz = median(avg.AltimeterDistance); else - t0 = btime; + maxz = inf; end + + %%%%%%% FLAGS %%%%%%= % Flag if coming in/out of the water if any(ischange(burst.Pressure)) && any(ischange(burst.Temperature)) @@ -260,7 +231,7 @@ outofwater = false; end - % Flag out of water based on bursts w/anomalously high amp + % Flag out of water based on bursts w/anomalously high amplitude if mean(burst.AmplitudeData(:),'omitnan') > opt.maxamp disp(' FLAG: Bad Amp (high average amp)...') badamp = true; @@ -268,7 +239,7 @@ badamp = false; end - % Flag out of water based on bursts w/low cor + % Flag out of water based on bursts w/low correlation if mean(burst.CorrelationData(:),'omitnan') < opt.mincorr disp(' FLAG: Bad Corr (low average corr)...') badcorr = true; @@ -276,15 +247,6 @@ badcorr = false; end - % Determine Altimeter Distance - if isfield(avg,'AltimeterDistance') - maxz = median(avg.AltimeterDistance); - else - maxz = inf; - end - - badburst = outofwater | badamp | badcorr;% | badvel; - %%%%%%% Process Broadband velocity data ('avg' structure) %%%%%% [profile,fh] = processSIGavg(avg,opt); @@ -322,12 +284,7 @@ figname = [bfiles(iburst).folder slash get(gcf,'Name')]; print(figname,'-dpng') close gcf - -% figure(fh(2)) -% set(gcf,'Name',[bname '_HR_profiles']) -% figname = [savedir SNprocess slash get(gcf,'Name')]; -% print(figname,'-dpng') -% close gcf + end %%%%%%%% Process Echo data %%%%%%%%%%% @@ -354,7 +311,7 @@ %%%%%%%% Save detailed signature data in SIG structure %%%%%%%% %Time - SIG(isig).time = t0; + SIG(isig).time = btime; % HR data SIG(isig).HRprofile = HRprofile; %Temperaure @@ -373,23 +330,25 @@ SIG(isig).motion.rollvar = var(avg.Roll,'omitnan'); SIG(isig).motion.headvar = var(unwrap(avg.Heading),'omitnan'); % Badburst & flags - SIG(isig).badburst = badburst; - SIG(isig).flag.altimeter = maxz; + SIG(isig).badburst = outofwater | badamp | badcorr; SIG(isig).flag.outofwater = outofwater; SIG(isig).flag.badamp = badamp; SIG(isig).flag.badcorr = badcorr; + SIG(isig).flag.notimematch = false; % Updated below isig = isig+1; %%%%%%%% Match burst time to existing SWIFT fields and replace data %%%%%%%% + badburst = outofwater | badamp | badcorr;% | badvel; + if ~isempty(fieldnames(SWIFT)) && ~isempty(SWIFT) [tdiff,tindex] = min(abs([SWIFT.time]-btime)); - if tdiff > 1/(24*10) % must be within 15 min + if tdiff > 12/(60*24)% must be within 10 min disp(' NO time index match...') timematch = false; - elseif tdiff < 1/(24*10) + elseif tdiff < 12/(60*24) timematch = true; burstreplaced(tindex) = true; elseif isempty(tdiff) @@ -397,7 +356,7 @@ timematch = false; end - if timematch %&& ~badburst % time match + if timematch && ~badburst % time match, good burst % HR data SWIFT(tindex).signature.HRprofile = []; SWIFT(tindex).signature.HRprofile.w = HRprofile.w; @@ -414,64 +373,79 @@ SWIFT(tindex).signature.profile.vvar = profile.vvar; SWIFT(tindex).signature.profile.wvar = profile.wvar; SWIFT(tindex).signature.profile.z = profile.z; + % Echogram data if ~isempty(echo) - SWIFT(tindex).signature.echogram = echogram; + SWIFT(tindex).signature.echo = echogram.echoc; + SWIFT(tindex).signature.echoz = echogram.r + opt.xz; end % Altimeter & Out-of-Water Flag SWIFT(tindex).signature.altimeter = maxz; % Temperaure SWIFT(tindex).watertemp = profile.temp; -% elseif timematch && badburst % Bad burst & time match -% % HR data -% SWIFT(tindex).signature.HRprofile = []; -% SWIFT(tindex).signature.HRprofile.w = NaN(size(HRprofile.w)); -% SWIFT(tindex).signature.HRprofile.wvar = NaN(size(HRprofile.w)); -% SWIFT(tindex).signature.HRprofile.z = HRprofile.z'; -% SWIFT(tindex).signature.HRprofile.tkedissipationrate = ... -% NaN(size(HRprofile.w')); -% % Broadband data -% SWIFT(tindex).signature.profile = []; -% SWIFT(tindex).signature.profile.w = NaN(size(profile.u)); -% SWIFT(tindex).signature.profile.east = NaN(size(profile.u)); -% SWIFT(tindex).signature.profile.north = NaN(size(profile.u)); -% SWIFT(tindex).signature.profile.uvar = NaN(size(profile.u)); -% SWIFT(tindex).signature.profile.vvar = NaN(size(profile.u)); -% SWIFT(tindex).signature.profile.wvar = NaN(size(profile.u)); -% SWIFT(tindex).signature.profile.z = profile.z; - elseif ~timematch && ~badburst % Good burst, no time match - disp(' ALERT: Burst good, but no time match...') - tindex = length(SWIFT)+1; - burstreplaced = [burstreplaced; true]; %#ok - % Copy fields from SWIFT(1); - SWIFT(tindex) = NaNstructR(SWIFT(1)); + elseif timematch && badburst % time match, bad burst % HR data SWIFT(tindex).signature.HRprofile = []; - SWIFT(tindex).signature.HRprofile.w = HRprofile.w; - SWIFT(tindex).signature.HRprofile.wvar = HRprofile.wvar; + SWIFT(tindex).signature.HRprofile.w = NaN(size(HRprofile.w)); + SWIFT(tindex).signature.HRprofile.wvar = NaN(size(HRprofile.w)); SWIFT(tindex).signature.HRprofile.z = HRprofile.z'; SWIFT(tindex).signature.HRprofile.tkedissipationrate = ... - HRprofile.eps; + NaN(size(HRprofile.w')); % Broadband data SWIFT(tindex).signature.profile = []; - SWIFT(tindex).signature.profile.east = profile.u; - SWIFT(tindex).signature.profile.north = profile.v; - SWIFT(tindex).signature.profile.w = profile.w; - SWIFT(tindex).signature.profile.uvar = profile.uvar; - SWIFT(tindex).signature.profile.vvar = profile.vvar; - SWIFT(tindex).signature.profile.wvar = profile.wvar; + SWIFT(tindex).signature.profile.w = NaN(size(profile.u)); + SWIFT(tindex).signature.profile.east = NaN(size(profile.u)); + SWIFT(tindex).signature.profile.north = NaN(size(profile.u)); + SWIFT(tindex).signature.profile.uvar = NaN(size(profile.u)); + SWIFT(tindex).signature.profile.vvar = NaN(size(profile.u)); + SWIFT(tindex).signature.profile.wvar = NaN(size(profile.u)); SWIFT(tindex).signature.profile.z = profile.z; + % Echogram data if ~isempty(echo) - SWIFT(tindex).signature.echogram = echogram; + SWIFT(tindex).signature.echo = NaN(size(echogram.echoc)); + SWIFT(tindex).signature.echo = echogram.r + opt.xz; end - % Altimeter - SWIFT(tindex).signature.altimeter = maxz; - % Temperaure - SWIFT(tindex).watertemp = profile.temp; - % Time - SWIFT(tindex).time = btime; - SWIFT(tindex).date = datestr(btime,'ddmmyyyy'); - disp([' (new) SWIFT time: ' datestr(SWIFT(tindex).time)]) + % Flag + badsig(tindex) = true; + + elseif ~timematch && ~badburst % Good burst, no time match + disp(' ALERT: Burst good, but no time match...') + SIG(isig-1).flag.notimematch = true; + + % tindex = length(SWIFT)+1; + % burstreplaced = [burstreplaced; true]; %#ok + % % Copy fields from SWIFT(1); + % SWIFT(tindex) = NaNstructR(SWIFT(1)); + % % HR data + % SWIFT(tindex).signature.HRprofile = []; + % SWIFT(tindex).signature.HRprofile.w = HRprofile.w; + % SWIFT(tindex).signature.HRprofile.wvar = HRprofile.wvar; + % SWIFT(tindex).signature.HRprofile.z = HRprofile.z'; + % SWIFT(tindex).signature.HRprofile.tkedissipationrate = ... + % HRprofile.eps; + % % Broadband data + % SWIFT(tindex).signature.profile = []; + % SWIFT(tindex).signature.profile.east = profile.u; + % SWIFT(tindex).signature.profile.north = profile.v; + % SWIFT(tindex).signature.profile.w = profile.w; + % SWIFT(tindex).signature.profile.uvar = profile.uvar; + % SWIFT(tindex).signature.profile.vvar = profile.vvar; + % SWIFT(tindex).signature.profile.wvar = profile.wvar; + % SWIFT(tindex).signature.profile.z = profile.z; + % % Echogram data + % if ~isempty(echo) + % SWIFT(tindex).signature.echo = echogram.echoc; + % SWIFT(tindex).signature.echoz = echogram.r + opt.xz; + % end + % % Altimeter + % SWIFT(tindex).signature.altimeter = maxz; + % % Temperaure + % SWIFT(tindex).watertemp = profile.temp; + % % Time + % SWIFT(tindex).time = btime; + % SWIFT(tindex).date = datestr(btime,'ddmmyyyy'); + % disp([' (new) SWIFT time: ' datestr(SWIFT(tindex).time)]) + end end @@ -510,32 +484,38 @@ SWIFT = SWIFT(isort); end -%%%%%% Save SWIFT Structure %%%%%%%% -if opt.saveSWIFT && ~isempty(fieldnames(SWIFT)) && isfield(SWIFT,'time') - if strcmp(mfile.name(end-6:end-4),'SBG') - save([savedir SNprocess '_reprocessedSIGandSBG.mat'],'SWIFT') - else - save([savedir SNprocess '_reprocessedSIG.mat'],'SWIFT') - end +%% Log reprocessing and flags, then save new L2 file or overwrite existing one + +params = opt; + +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; end +sinfo.postproc(ip).type = 'SIG'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags.badsig = badsig; +sinfo.postproc(ip).params = params; + +save([sfile.folder slash sfile.name(1:end-7) '_L2.mat'],'SWIFT','sinfo') -%%%%%% Save SIG Structure %%%%%%%% +%% Save SIG Structure + Plot %%%%%%%% if opt.saveSIG - save([savedir SNprocess '_burstavgSIG.mat'],'SIG') + save([sfile.folder slash sfile.name(1:end-7) '_burstavgSIG.mat'],'SIG') end -%%%%%% Plot burst Averaged SWIFT Signature Data %%%%%% -if opt.plotmission - catSIG(SIG,'plot'); - set(gcf,'Name',SNprocess) - if opt.saveplots - figname = [savedir get(gcf,'Name')]; - print(figname,'-dpng') - close gcf - end +% Plot burst Averaged SWIFT Signature Data +catSIG(SIG,'plot'); +set(gcf,'Name',sfile.name(1:end-7)) +if opt.saveplots + figname = [missiondir slash get(gcf,'Name')]; + print([figname '_SIG'],'-dpng') + close gcf end - -cd(savedir) +cd(missiondir) end \ No newline at end of file diff --git a/Waves/GPSwaves.m b/Waves/GPSwaves.m index b5c6704..dba8c49 100644 --- a/Waves/GPSwaves.m +++ b/Waves/GPSwaves.m @@ -1,4 +1,4 @@ -function [ Hs, Tp, Dp, E, f, a1, b1, a2, b2] = GPSwaves(u,v,z,fs) +function [Hs,Tp,Dp,E,f,a1,b1,a2,b2,check] = GPSwaves(u,v,z,fs) % matlab function to read and process GPS data % to estimate wave height, period, direction, and spectral moments diff --git a/Waves/rawdisplacements.m b/Waves/rawdisplacements.m index 8380bc6..e32e63d 100644 --- a/Waves/rawdisplacements.m +++ b/Waves/rawdisplacements.m @@ -1,4 +1,4 @@ -function [x,y,z, hs ] = rawdisplacements(AHRS,varargin); +function [x,y,z, hs ] = rawdisplacements(AHRS,varargin) % function to estimate raw displacements from SWIFT buoys % also returns significant wave height, as estimated from the displacements % this Hs should be checked against the spectral result @@ -19,13 +19,14 @@ dB = 20; % minimum stopband attenuation in elliptic fiklter dt = median(diff(AHRS.Timestamp_sec)); % should be 0.04 s -if isnan(dt), dt = 600 ./ length(AHRS.Accel); else end +if isnan(dt) + dt = 600 ./ length(AHRS.Accel); +end fs = 1/dt; % should be 25 Hz %% convert to earth reference frame a = quat_rotate_vector( AHRS.Accel , AHRS.Quat ); % very specific to microstrain IMU on SWIFT v3 - %% filter and integrate accelarations to velocities ax = 9.8 * a(:,1); % m/ s^2 %duplicate steps ay = 9.8 * a(:,2); % m/ s^2 @@ -34,16 +35,16 @@ ay = ay - nanmean(ay); az = az - nanmean(az); -if strcmp( cellstr( varargin{1} ), 'elliptic') % eliptic filter +if strcmp(varargin{1},'elliptic') % eliptic filter [B,A] = ellip(3, .5, dB, 0.05/(fs/2), 'high'); % original is ellip(3, .5, 20, 0.05/(fs/2), 'high'); fax = filtfilt(B, A, double(ax(~isnan(ax)))); fay = filtfilt(B, A, double(ay(~isnan(ay)))); faz = filtfilt(B, A, double(az(~isnan(az)))); -elseif strcmp( cellstr( varargin{1} ), 'RC') % RC filter +elseif strcmp(varargin{1},'RC') % RC filter alpha = RC / (RC + 1./fs); fax(1) = ax(1); fay(1)=ay(1); faz(1)=az(1); - for ui = 2:length(ax), + for ui = 2:length(ax) fax(ui) = alpha * fax(ui-1) + alpha * ( ax(ui) - ax(ui-1) ); fay(ui) = alpha * fay(ui-1) + alpha * ( ay(ui) - ay(ui-1) ); faz(ui) = alpha * faz(ui-1) + alpha * ( az(ui) - az(ui-1) ); @@ -63,15 +64,15 @@ %% filter and integrate velocities to displacements -if strcmp( cellstr( varargin{1} ), 'elliptic') % eliptic filter +if strcmp(varargin{1},'elliptic') % eliptic filter fu = filtfilt(B, A, double(u)); fv = filtfilt(B, A, double(v)); fw = filtfilt(B, A, double(w)); -elseif strcmp( cellstr( varargin{1} ), 'RC') % RC filter +elseif strcmp(varargin{1},'RC') % RC filter alpha = RC / (RC + 1./fs); fu(1) = u(1); fv(1)=v(1); fw(1)=w(1); - for ui = 2:length(ax), + for ui = 2:length(ax) fu(ui) = alpha * fu(ui-1) + alpha * ( u(ui) - u(ui-1) ); fv(ui) = alpha * fv(ui-1) + alpha * ( v(ui) - v(ui-1) ); fw(ui) = alpha * fw(ui-1) + alpha * ( w(ui) - w(ui-1) ); diff --git a/Winds/reprocess_WXT.m b/Winds/reprocess_WXT.m index 66619a6..eef0834 100644 --- a/Winds/reprocess_WXT.m +++ b/Winds/reprocess_WXT.m @@ -1,3 +1,5 @@ +function [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw) + % reprocess SWIFT Vaisala WXT files % loop thru raw data for a given SWIFT deployment, then % replace values in the SWIFT data structure of results @@ -6,58 +8,59 @@ % % J. Thomson, 4/2020 (derived from reprocess_AQD.m) -clear all; close all -readraw = false; -parentdir = './'; -parentdir = pwd; - +if ispc + slash = '\'; +else + slash = '/'; +end -%% load existing SWIFT structure created during concatSWIFTv3_processed, replace only the new results -cd(parentdir); -wd = pwd; -wdi = find(wd == '/',1,'last'); -wd = wd((wdi+1):length(wd)); +%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function -load([ wd '.mat']) +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); -if isfield(SWIFT(1),'signature'), - SWIFTversion = 4; -else - SWIFTversion = 3; +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return end -cd('WXT/Raw/') - +%% Loop through raw burst files and reprocess -%% loop thru raw data +bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*WXT*.dat']); -dirlist = dir('20*'); +for iburst = 1:length(bfiles) -for di = 1:length(dirlist), - - cd([dirlist(di).name]) - filelist = dir('*.dat'); - - for fi=1:length(filelist), + disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - if filelist(fi).bytes > 0, + if bfiles(iburst).bytes > 0 + disp('Burst file is empty. Skippping ...') + end - % read or load raw data - if isempty(dir([filelist(fi).name(1:end-4) '.mat'])) | readraw, - [winddirR windspd airtemp relhumidity airpres rainaccum rainint ] = ... - readSWIFT_WXT( filelist(fi).name ); + % Read mat file or load raw data + if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) || readraw + [winddirR,windspd,airtemp,relhumidity,airpres,rainaccum,rainint] = ... + readSWIFT_WXT([bfiles(iburst).folder slash bfiles(iburst).name]); else - load([filelist(fi).name(1:end-4) '.mat']), + load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat']), %#ok end - time = datenum(filelist(fi).name(13:21)) + str2num(filelist(fi).name(23:24))./24 ... - + str2num(filelist(fi).name(26:27))./(24*6); - % match time to SWIFT structure and replace values - [tdiff, tindex] = min(abs([SWIFT.time]-time)); - SWIFT_tindex(di,fi) = tindex; - %tindex + % Find matching time + time = datenum(bfiles(iburst).name(13:21)) + str2double(bfiles(iburst).name(23:24))./24 ... + + str2double(bfiles(iburst).name(26:27))./(24*6); + [tdiff,tindex] = min(abs([SWIFT.time]-time)); + + if tdiff > 12/(60*24) + disp('No time match. Skippping...') + continue + end - SWIFT(tindex).winddirR = nanmean(winddirR); % mean wind direction (deg relative) + SWIFT(tindex).winddirR = nanmean(winddirR); %#ok<*NANMEAN> % mean wind direction (deg relative) SWIFT(tindex).winddirRstddev = nanstd(winddirR); % std dev of wind direction (deg) SWIFT(tindex).windspd = nanmean(windspd); % mean wind speed (m/s) SWIFT(tindex).windspdstddev = nanstd(windspd); % std dev of wind spd (m/s) @@ -69,27 +72,14 @@ SWIFT(tindex).airpresstddev = nanstd(airpres); % millibars SWIFT(tindex).rainaccum = nanmean(rainaccum); % millimeters SWIFT(tindex).rainint = nanmean(rainint); % millimeters_per_hour - - if ~isempty(SWIFTversion) && SWIFTversion==3, - SWIFT(tindex).metheight = 0.84; % height of measurement, meters - elseif ~isempty(SWIFTversion) && SWIFTversion==4, - SWIFT(tindex).metheight = 0.4; % height of measurement, meters - else - SWIFT(tindex).metheight = 0.4; % height of measurement, meters - end - - else - end - end - - cd('../') - + end -%If SWIFT structure elements not replaced, fill variables with NaNs +%% If SWIFT structure elements not replaced, fill variables with NaNs + for i = 1:length(SWIFT) - if ~isfield(SWIFT(i),'windspd') | isempty(SWIFT(i).windspd), + if ~isfield(SWIFT(i),'windspd') || isempty(SWIFT(i).windspd) disp(['bad data at index ' num2str(i)]) SWIFT(i).winddirR = NaN; SWIFT(i).winddirRstddev = NaN; @@ -106,11 +96,21 @@ end end +%% Log reprocessing and flags, then save new L2 file or overwrite existing one -cd(parentdir) - -save([ wd '_reprocessedWXT.mat'],'SWIFT') - -plotSWIFT(SWIFT) +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'WXT'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags = []; +sinfo.postproc(ip).params = []; +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +%% End function +end \ No newline at end of file From 38fcea4efc62b3d37deea568f74f1044d19d4a5d Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:39:29 -0700 Subject: [PATCH 02/34] more changes --- Aquadopp/reprocess_AQD.asv | 147 ++++++++++++++++ Aquadopp/reprocess_AQD.m | 260 +++++++++++++---------------- CT/reprocess_ACS.m | 82 +++++++-- GeneralTools/postprocess_SWIFT.asv | 107 ++++++++++++ GeneralTools/postprocess_SWIFT.m | 60 +++++-- Signature/catSIG.m | 21 ++- Signature/reprocess_SIG.m | 20 ++- Winds/reprocess_WXT.m | 5 +- Winds/reprocess_Y81.m | 140 +++++++--------- 9 files changed, 574 insertions(+), 268 deletions(-) create mode 100644 Aquadopp/reprocess_AQD.asv create mode 100644 GeneralTools/postprocess_SWIFT.asv diff --git a/Aquadopp/reprocess_AQD.asv b/Aquadopp/reprocess_AQD.asv new file mode 100644 index 0000000..46dfa01 --- /dev/null +++ b/Aquadopp/reprocess_AQD.asv @@ -0,0 +1,147 @@ +function [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw) + +% reprocess SWIFT v3 downlooking Aquadopp (AQD) results +% loop thru raw data for a given SWIFT deployment, then +% replace values in the SWIFT data structure of results +% (assuming concatSWIFTv3_processed.m has already been run. +% +% +% M. Smith 10/2015 based on reprocess_AQH.m code +% J. Thomson 12/2015 to include option phase resolved dissipation (commented-out) +% cleaned and revised with AQD read function, Thomson, Jun 2016 +% M. Moulton 3/2017 correct error in Dir calculation +% K. Zeiden 07/2024 reformatted for symmetry with bulk post processing +% postprocess_SWIFT.m + +if ispc + slash = '\'; +else + slash = '/'; +end + +%% Parameters + +% QC parameters +minamp = 30; % amplitude cutoff, usually 30 or 40 +minn = 50; % number of cells averaged for each depth +z = 1.25:0.5:20.75; + +% Raw Doppler velocity precision of 1 MHz Aquadopp +Vert_prec = .074; %m/s +Hori_prec = .224; %m/s + +%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function + +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); + +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end + +burstreplaced = NaN( + +%% Loop through raw burst files and reprocess + +bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*AQD*.dat']); + +for iburst = 1:length(bfiles) + + disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) + + % read or load raw data + if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) || readraw + [~,VelE,VelN,VelU,Amp1,Amp2,~,~,~,~,~] = readSWIFTv3_AQD([bfiles(iburst).folder slash bfiles(iburst).name]); + else + load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat']) %#ok + end + + % Use velocities only when sufficient return + exclude = Amp1 < minamp | Amp2 < minamp; % Used for vel. magnitude + + % Average amplitudes of just velocity measurements used + Amp1(exclude) = NaN; + Amp2(exclude) = NaN; + Amp = (Amp1+Amp2)./2; % Corresponds to those used in velocity + + n = sum(~isnan(Amp),1); + E_error = Hori_prec./sqrt(n); + N_error = E_error; + Hori_error = sqrt((E_error.^2) + (N_error.^2)); + Hori_error(n < minn) = NaN; + Amp = nanmean(Amp,1); + Amp(n < minn) = NaN; + + % Average velocities first for "net" velocity + direction + VelE(Amp1 12/(60*24) + disp('No time match. Skippping...') + else + burstreplaced(tindex) = true; + end + + % Replace Values in SWIFT structure + SWIFT(tindex).downlooking.velocitydirection = Dir'; + SWIFT(tindex).downlooking.amplitude = Amp'; + SWIFT(tindex).downlooking.z = z; + SWIFT(tindex).downlooking.velocityerror = Hori_error; + SWIFT(tindex).downlooking.vertvel = nanmean(VelU,1); + + if nansum(SWIFT(tindex).downlooking.velocityprofile) > 0 && tdiff < 0.04 + SWIFT(tindex).downlooking.velocityprofile = Vel'; + else + SWIFT(tindex).downlooking.velocityprofile = NaN(40,1); + end + +end + + +%% Log reprocessing and flags, then save new L2 file or overwrite existing one + +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'AQD'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags = []; +sinfo.postproc(ip).params = []; + +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') + +%% End function +end + + diff --git a/Aquadopp/reprocess_AQD.m b/Aquadopp/reprocess_AQD.m index 9182ba2..215be27 100644 --- a/Aquadopp/reprocess_AQD.m +++ b/Aquadopp/reprocess_AQD.m @@ -1,3 +1,5 @@ +function [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw) + % reprocess SWIFT v3 downlooking Aquadopp (AQD) results % loop thru raw data for a given SWIFT deployment, then % replace values in the SWIFT data structure of results @@ -8,174 +10,150 @@ % J. Thomson 12/2015 to include option phase resolved dissipation (commented-out) % cleaned and revised with AQD read function, Thomson, Jun 2016 % M. Moulton 3/2017 correct error in Dir calculation +% K. Zeiden 07/2024 reformatted for symmetry with bulk post processing +% postprocess_SWIFT.m +if ispc + slash = '\'; +else + slash = '/'; +end -clear all; close all -%parentdir = ('/Users/jthomson/Desktop/SWIFT15_14Jan2016'); % change this to be the parent directory of all raw raw data (CF card offload from SWIFT) -%parentdir='/Users/msmith/Documents/PIPERS/SWIFT/Polynya_May2017/SWIFT15_05May2017'; -parentdir = './'; -parentdir = pwd; - -plotflag = true; +%% Parameters -%quality control parameters -minamp = 30; %amplitude cutoff, usually 30 or 40 -minn = 50; %number of cells averaged for each depth -z = [1.25:0.5:20.75]; +% QC parameters +minamp = 30; % amplitude cutoff, usually 30 or 40 +minn = 50; % number of cells averaged for each depth +z = 1.25:0.5:20.75; -% raw Doppler velocity precision of 1 MHz Aquadopp +% Raw Doppler velocity precision of 1 MHz Aquadopp Vert_prec = .074; %m/s Hori_prec = .224; %m/s +%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function -%% load existing SWIFT structure created during concatSWIFTv3_processed, replace only the new results -cd(parentdir); -wd = pwd; -wdi = find(wd == '/',1,'last'); -wd = wd((wdi+1):length(wd)); +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); -load([ wd '.mat']) +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end -cd('AQD/Raw/') % v3.2 or V3.3 +burstreplaced = NaN(length(SWIFT),1); +%% Loop through raw burst files and reprocess -%% loop thru raw data +bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*AQD*.dat']); -dirlist = dir('20*'); +for iburst = 1:length(bfiles) -for di = 1:length(dirlist), + disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - cd([dirlist(di).name]) - filelist = dir('*.dat'); + % Read or load raw data + if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) || readraw + [~,VelE,VelN,VelU,Amp1,Amp2,~,~,~,~,~] = readSWIFTv3_AQD([bfiles(iburst).folder slash bfiles(iburst).name]); + else + load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat']) %#ok + end - for fi=1:length(filelist), - - % read or load raw data - if isempty(dir([filelist(fi).name(1:end-4) '.mat'])), - [time VelE VelN VelU Amp1 Amp2 Amp3 Pressure Pitch Roll Heading ] = readSWIFTv3_AQD( filelist(fi).name ); - else - load([filelist(fi).name(1:end-4) '.mat']), - end - - %use velocities only when sufficient return - exclude = Amp1 < minamp | Amp2 < minamp; %used for vel. magnitude - - % Average amplitudes of just velocity measurements used - Amp1(exclude) = NaN; Amp2(exclude) = NaN; - Amp = (Amp1+Amp2)./2;%corresponds to those used in velocity - - if plotflag - pcolor(time, z, Amp') - shading flat - set(gca,'YDir','reverse') - ylabel('Depth [m]') - datetick - title([filelist(fi).name(1:end-4) ' AQD backscatter' ],'interp','none') - print('-dpng',[filelist(fi).name(1:end-4) '_backscatter.png']), - end - - n = sum(~isnan(Amp),1); - E_error = Hori_prec./sqrt(n); N_error = E_error; - Hori_error = sqrt((E_error.^2) + (N_error.^2)); - Hori_error(n < minn) = NaN; - - Amp = nanmean(Amp,1); - Amp(n < minn) = NaN; - - %Average velocities first for "net" velocity + direction - VelE(Amp10 && tdiff<0.04, - SWIFT(tindex).downlooking.velocityprofile = Vel'; - else - SWIFT(tindex).downlooking.velocityprofile = NaN(40,1); - % SWIFT(tindex).downlooking.epsilon = NaN(40,1); - % SWIFT(tindex).phaseresolvedepsilon = NaN(size(phaseresolvedepsilon)); - + % Use velocities only when sufficient return + exclude = Amp1 < minamp | Amp2 < minamp; % Used for vel. magnitude + + % Average amplitudes of just velocity measurements used + Amp1(exclude) = NaN; + Amp2(exclude) = NaN; + Amp = (Amp1+Amp2)./2; % Corresponds to those used in velocity + + n = sum(~isnan(Amp),1); + E_error = Hori_prec./sqrt(n); + N_error = E_error; + Hori_error = sqrt((E_error.^2) + (N_error.^2)); + Hori_error(n < minn) = NaN; + Amp = nanmean(Amp,1); + Amp(n < minn) = NaN; + + % Average velocities first for "net" velocity + direction + VelE(Amp1 12/(60*24) + disp('No time match...') + continue + else + burstreplaced(tindex) = true; end - cd('../') + % Replace Values in SWIFT structure + SWIFT(tindex).downlooking.velocitydirection = Dir'; + SWIFT(tindex).downlooking.amplitude = Amp'; + SWIFT(tindex).downlooking.z = z; + SWIFT(tindex).downlooking.velocityerror = Hori_error; + SWIFT(tindex).downlooking.vertvel = nanmean(VelU,1); + if nansum(SWIFT(tindex).downlooking.velocityprofile) > 0 + SWIFT(tindex).downlooking.velocityprofile = Vel'; + else + SWIFT(tindex).downlooking.velocityprofile = NaN(40,1); + end + end +%% If SWIFT structure elements not replaced, fill variables with NaNs -%If SWIFT structure elements not replaced, fill variables with NaNs -for i = 1:length(SWIFT) - if length(find(SWIFT_tindex ==i)) < 1, - SWIFT(i).downlooking.velocitydirection = NaN(40,1); - SWIFT(i).downlooking.amplitude = NaN(40,1); - SWIFT(i).downlooking.z = 0.25:0.5:19.75; - SWIFT(i).downlooking.velocityerror = NaN(40,1); - SWIFT(i).downlooking.velocityprofile = NaN(40,1); -% SWIFT(tindex).downlooking.epsilon = NaN(40,1); -% SWIFT(tindex).phaseresolvedepsilon = NaN(size(phaseresolvedepsilon)); +for iburst = 1:length(SWIFT) + if ~burstreplaced(iburst) + SWIFT(iburst).downlooking.velocitydirection = NaN(40,1); + SWIFT(iburst).downlooking.amplitude = NaN(40,1); + SWIFT(iburst).downlooking.z = 0.25:0.5:19.75; + SWIFT(iburst).downlooking.velocityerror = NaN(40,1); + SWIFT(iburst).downlooking.velocityprofile = NaN(40,1); end end +%% Log reprocessing and flags, then save new L2 file or overwrite existing one + +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'AQD'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags = []; +sinfo.postproc(ip).params = []; -cd(parentdir) +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') -save([ wd '_reprocessedAQD.mat'],'SWIFT') +%% End function +end diff --git a/CT/reprocess_ACS.m b/CT/reprocess_ACS.m index 84d303f..d48a295 100644 --- a/CT/reprocess_ACS.m +++ b/CT/reprocess_ACS.m @@ -1,26 +1,78 @@ +function [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw) + % reprocess SWIFT v3 ACS results to get raw % M. Smith 08/2016 -clear all; close all -parentdir = ('/Users/msmith/Documents/Sikuliaq2015/SWIFT/Racetrack/SWIFT14_25-27Oct2015/ACS/Raw/20151026'); % change this to be the parent directory of all raw raw data (CF card offload from SWIFT) -cd(parentdir) +% K. Zeiden 07/2024 reformatted for symmetry with bulk post processing +% postprocess_SWIFT.m + +if ispc + slash = '\'; +else + slash = '/'; +end + +%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function + +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); + +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end + +%% Loop through raw burst files and reprocess + +bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*ACS*.dat']); -filelist = dir('*ACS*.dat'); +for iburst = 1:length(bfiles) -for fi=1:length(filelist), + disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - % read or load raw data - if isempty(dir([filelist(fi).name(1:end-4) '.mat'])), - [Conductivity, Temperature, Salinity, Density, Soundspeed] = readSWIFTv3_ACS( filelist(fi).name ); + % Read mat file or load raw data + if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) || readraw + [~,~,Salinity,~,~] = readSWIFTv3_ACS([bfiles(iburst).folder slash bfiles(iburst).name]); else - load([filelist(fi).name(1:end-4) '.mat']), + load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat']), %#ok end - - salinity_std(fi)=std(Salinity); - ACS_time=datenum(2015,10,str2num(filelist(fi).name(13:14)),str2num(filelist(fi).name(23:24)),str2num(filelist(fi).name(26:27))*12,00); - [tdiff, tindex] = min(abs([SWIFT(1).SWIFT.time]-ACS_time)); - SWIFT(1).SWIFT(tindex).salinityvariance = salinity_std(fi); + % Find matching time + time = datenum(bfiles(iburst).name(13:21)) + str2double(bfiles(iburst).name(23:24))./24 ... + + str2double(bfiles(iburst).name(26:27))./(24*6); + [tdiff,tindex] = min(abs([SWIFT.time]-time)); + + if tdiff > 12/(60*24) + disp('No time match. Skippping...') + continue + end + + % Replace Values in SWIFT structure + SWIFT(tindex).salinityvariance = std(Salinity); + end -%save([ wd '.mat'],'SWIFT') \ No newline at end of file +%% Log reprocessing and flags, then save new L2 file or overwrite existing one + +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'ACS'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags = []; +sinfo.postproc(ip).params = []; + +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') + +%% End function +end \ No newline at end of file diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv new file mode 100644 index 0000000..6cc9684 --- /dev/null +++ b/GeneralTools/postprocess_SWIFT.asv @@ -0,0 +1,107 @@ +% [SWIFT,sinfo] = postprocess_SWIFT(expdir) + +% Master post-processing function, that calls sub-functions to reprocess +% raw SWIFT data. + +% K. Zeiden 07/2024 + +expdir = 'S:\SEAFAC\June2024\NorthMooring'; +cd(expdir) + +if ispc + slash = '\'; +else + slash = '/'; +end + +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +%% Loop through missions and reprocess + +for im = 1:length(missions) + + disp(['Post-processing ' missions(im).name]) + + missiondir = [missions(im).folder slash missions(im).name]; + cd(missiondir) + + % Locate L1 product, skip if does not exist. + % Else create 'sinfo' and modify L1 product. + l1file = dir([missiondir slash '*L1*']); + if isempty(l1file) + disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + else + load([l1file.folder slash l1file.name],'SWIFT'); + if isfield(SWIFT,'ID') + disp('Create information structure ''sinfo''') + sinfo.ID = SWIFT(1).ID; + sinfo.CTdepth = SWIFT(1).CTdepth; + sinfo.metheight = SWIFT(1).metheight; + SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); + disp('Saving new L1 product...') + save([l1file.folder slash l1file.name],'SWIFT','sinfo') + else + load([l1file.folder slash l1file.name],'sinfo') + end + end + + %% Reprocess IMU + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) + disp('Reprocessing IMU data...') + calctype = 'IMUandGPS'; + filtertype = 'RC'; + saveraw = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); + else + disp('No IMU data...') + end + + %% Reprocess SBG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) + disp('Reprocessing SBG data...') + saveraw = false; + useGPS = false; + interpf = false; + [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + end + + %% Reprocess SIG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + end + + %% Reprocess WXT + + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) + disp('Reprocessing Vaisala WXT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw); + end + + %% Reprocess Y81 + + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) + disp('Reprocessing Young Sonic Anemommetr data...') + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir); + end + + % Reprocess ACS + + % Reprocess PB2 + + % Reprocess IMU + + % Reprocess AQD + + % Reprocess AQH + + % clear SWIFT sinfo +end + + + diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index c61b2b2..fa60f16 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -5,7 +5,7 @@ % K. Zeiden 07/2024 -expdir = 'S:\SEAFAC\June2024\SouthMooring'; +expdir = 'S:\SEAFAC\June2024'; cd(expdir) if ispc @@ -19,7 +19,8 @@ %% Loop through missions and reprocess -for im = 1:length(missions) +%for im = 1:length(missions) +im = 1; disp(['Post-processing ' missions(im).name]) @@ -67,31 +68,54 @@ end %% Reprocess SIG - % if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - % disp('Reprocessing SIG data...') - % plotburst = false; - % readraw = false; - % [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - % end + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + end - %% - % Reprocess WXT + %% Reprocess WXT + + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) + disp('Reprocessing Vaisala WXT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); + end + + %% Reprocess Y81 + + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) + disp('Reprocessing Y81 Sonic Anemometer data...') + [SWIFT,sinfo] = reprocess_Y81(missiondir); + end + %% Reprocess ACS + + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) + disp('Reprocessing ACS CT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); + end - % Reprocess Y81 + %% Reprocess PB2 - % Reprocess ACS + % does not exist - % Reprocess PB2 + %% - % Reprocess IMU + %% Reprocess AQD - % Reprocess AQD + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) + disp('Reprocessing Aquadopp data...') + readraw = false; + [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); + end - % Reprocess AQH + %% Reprocess AQH + %% Reprocess 536 - %clear SWIFT sinfo -end +%end diff --git a/Signature/catSIG.m b/Signature/catSIG.m index 87de038..d53b89f 100644 --- a/Signature/catSIG.m +++ b/Signature/catSIG.m @@ -45,6 +45,12 @@ sig.wpeofmag = sig.hrcorr; sig.eofs = NaN(nzhr,nzhr,nt); sig.eofsvar = sig.hrcorr; + sig.pitch = NaN(1,nt); + sig.roll = NaN(1,nt); + sig.head = NaN(1,nt); + sig.pitchvar = NaN(1,nt); + sig.rollvar = NaN(1,nt); + sig.headvar = NaN(1,nt); for it = 1:length(sig.time) %Broadband @@ -72,6 +78,14 @@ sig.wpeofmag(1:nz,it) = SIG(it).HRprofile.QC.wpeofmag; sig.eofs(1:nz,1:nz,it) = SIG(it).HRprofile.QC.eofs; sig.eofsvar(1:nz,it) = SIG(it).HRprofile.QC.eofsvar; + + % Motion + sig.pitch(it) = SIG(it).motion.pitch; + sig.roll(it) = SIG(it).motion.roll; + sig.head(it) = SIG(it).motion.head; + sig.pitchvar(it) = SIG(it).motion.pitchvar; + sig.rollvar(it) = SIG(it).motion.rollvar; + sig.headvar(it) = SIG(it).motion.headvar; end %QC @@ -96,10 +110,15 @@ sig.pspike(:,badburst) = []; sig.pbadping(badburst) = []; sig.time(badburst) = []; - sig.wpeofmag(:,badburst) = []; sig.eofs(:,:,badburst) = []; sig.eofsvar(:,badburst) = []; + sig.pitch(badburst) = SIG(badburst).motion.pitch; + sig.roll(badburst) = SIG(badburst).motion.roll; + sig.head(badburst) = SIG(badburst).motion.head; + sig.pitchvar(badburst) = SIG(badburst).motion.pitchvar; + sig.rollvar(badburst) = SIG(badburst).motion.rollvar; + sig.headvar(badburst) = SIG(badburst).motion.headvar; else sig.badburst = badburst; end diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index f1048e3..1821d57 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -201,12 +201,6 @@ continue end - % Skip burst if file is too small... - if bfiles(iburst).bytes < 1e6 % 2e6, - disp('Bad file (small), skipping burst...') - continue - end - % Burst time t0 = min(avg.time); if abs(btime - t0) > 12/(60*24) @@ -231,6 +225,14 @@ outofwater = false; end + % Flag burst if file is too small... + if bfiles(iburst).bytes < 1e6 % 2e6, + disp('FLAG: Small file...') + smallfile = true; + else + smallfile = false; + end + % Flag out of water based on bursts w/anomalously high amplitude if mean(burst.AmplitudeData(:),'omitnan') > opt.maxamp disp(' FLAG: Bad Amp (high average amp)...') @@ -330,17 +332,18 @@ SIG(isig).motion.rollvar = var(avg.Roll,'omitnan'); SIG(isig).motion.headvar = var(unwrap(avg.Heading),'omitnan'); % Badburst & flags - SIG(isig).badburst = outofwater | badamp | badcorr; + SIG(isig).badburst = outofwater | badamp | badcorr | smallfile; SIG(isig).flag.outofwater = outofwater; SIG(isig).flag.badamp = badamp; SIG(isig).flag.badcorr = badcorr; + SIG(isig).flag.smallfile = smallfile; SIG(isig).flag.notimematch = false; % Updated below isig = isig+1; %%%%%%%% Match burst time to existing SWIFT fields and replace data %%%%%%%% - badburst = outofwater | badamp | badcorr;% | badvel; + badburst = outofwater | badamp | badcorr | smallfile; if ~isempty(fieldnames(SWIFT)) && ~isempty(SWIFT) @@ -503,6 +506,7 @@ save([sfile.folder slash sfile.name(1:end-7) '_L2.mat'],'SWIFT','sinfo') %% Save SIG Structure + Plot %%%%%%%% + if opt.saveSIG save([sfile.folder slash sfile.name(1:end-7) '_burstavgSIG.mat'],'SIG') end diff --git a/Winds/reprocess_WXT.m b/Winds/reprocess_WXT.m index eef0834..25c1f53 100644 --- a/Winds/reprocess_WXT.m +++ b/Winds/reprocess_WXT.m @@ -4,9 +4,10 @@ % loop thru raw data for a given SWIFT deployment, then % replace values in the SWIFT data structure of results % (assuming concatSWIFTv3_processed.m has already been run. -% -% + % J. Thomson, 4/2020 (derived from reprocess_AQD.m) +% K. Zeiden 07/2024 reformatted for symmetry with bulk post processing +% postprocess_SWIFT.m if ispc slash = '\'; diff --git a/Winds/reprocess_Y81.m b/Winds/reprocess_Y81.m index b6c1bda..7377bda 100644 --- a/Winds/reprocess_Y81.m +++ b/Winds/reprocess_Y81.m @@ -1,3 +1,5 @@ +function [SWIFT,sinfo] = reprocess_Y81(missiondir) + % SWIFT RM Young and IMU processing % % Loop through files, grabs raw IMU and sonic data, pass to motion @@ -5,117 +7,89 @@ % structure 'sonic' % S Zippel, 2018? % J. Thomson, 12/2019 revs +% K. Zeiden 07/2024 reformatted for symmetry with bulk post processing +% postprocess_SWIFT.m +if ispc + slash = '\'; +else + slash = '/'; +end -clc -clear - -tic - -plots = 1; -parentdir = pwd; - -cd(parentdir) -wd = pwd; -wdi = find(wd == '/',1,'last'); -wd = wd((wdi+1):length(wd)); +%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function -load([wd '.mat']) +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); -cd('./Y81/Raw/') +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end +%% Loop through raw burst files and reprocess -%% loop thru raw data +bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*Y81*.dat']); -dirlist = dir('20*'); +for iburst = 1:length(bfiles) -for di = 1:length(dirlist), - - cd([dirlist(di).name]) - filelist = dir('*.dat'); - - for fi=1:length(filelist), + disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - disp(['file ' num2str(fi) ' of ' num2str(length(filelist)) ]) - - if filelist(fi).bytes > 1e5, - RMYdata = importdata([filelist(fi).name]); + if bfiles(iburst).bytes > 1e5 + RMYdata = importdata([bfiles(iburst).folder slash bfiles(iburst).name]); uvw = RMYdata.data(:,1:3); temp = RMYdata.data(:,4); - errorflag = RMYdata.data(:,5); else uvw = NaN(1000,3); temp = NaN(1000,1); - errorflag = NaN(1000,1); end - windspd = mean((uvw(:,1).^2 + uvw(:,2).^2 + uvw(:,3).^2).^.5); - windspd_alt = (mean(uvw(:,1)).^2 + mean(uvw(:,2).^2)).^.5; - z = 0.71; % SWIFT.metheight + % Recalculate friction velocity + z = sinfo.metheight; fs = 10; + [ustar,~,~,~,~,~,~,~,~,~] = inertialdissipation(uvw(:,1),uvw(:,2),uvw(:,3),temp,z,fs); - [ustar epsilon meanu meanv meanw meantemp anisotropy quality freq tkespectrum ] = inertialdissipation(uvw(:,1), uvw(:,2), uvw(:,3), temp, z, fs); - - - - %% match time to SWIFT structure and replace values - time=datenum(filelist(fi).name(13:21))+datenum(0,0,0,str2num(filelist(fi).name(23:24)),(str2num(filelist(fi).name(26:27))-1)*12,0); - [tdiff tindex] = min(abs([SWIFT.time]-time)); - bad(tindex) = false; - updated(tindex) = false; - if ~isempty(tdiff) && tdiff < 1/(24*5) + + % Find matching time + time = datenum(bfiles(iburst).name(13:21)) + str2double(bfiles(iburst).name(23:24))./24 ... + + str2double(bfiles(iburst).name(26:27))./(24*6); + [tdiff,tindex] = min(abs([SWIFT.time]-time)); + + + % Replace wind speed, NaN out wind direction and replace ustar + if ~isempty(tdiff) && tdiff < 12/(24*60) SWIFT(tindex).windspd = windspd; SWIFT(tindex).winddirR = NaN; if ustar ~= 9999 SWIFT(tindex).windustar = ustar; - updated(tindex) = true; end - else - end - - if plots - figure(1), clf - subplot(3,1,1) - plot(uvw) - ylabel('m/s'), - legend('u','v','w') - subplot(3,1,2) - plot(temp) - ylabel('deg C') - subplot(3,1,3) - plot(errorflag) - ylabel('error flag') - - figure(2), clf - loglog(freq,tkespectrum) - hold on - loglog([.5 2],1e0*[.5 2].^-(5/2)) - xlabel('f [Hz]'),ylabel('TKE [m^2/s^2/Hz]') - end - - - end - - cd('../') - end + -cd(parentdir) - -figure(3), clf -plot([SWIFT.windspd],[SWIFT.windustar],'bx') -hold on -plot([SWIFT(updated).windspd],[SWIFT(updated).windustar],'r.') -legend('onboard','updated') -xlabel('Wind spd [m/s]') -ylabel('Friction Vel [m/s]') -axis([0 20 0 1]), grid -print('-dpng',[wd '_windstress.png']) +%% Log reprocessing and flags, then save new L2 file or overwrite existing one +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'Y81'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags = []; +sinfo.postproc(ip).params = []; -save([ wd '_reprocessedY81.mat'],'SWIFT') +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') -toc \ No newline at end of file +%% End function +end + From 7c876cfcd9da12b9c321980a6358a739b65395c6 Mon Sep 17 00:00:00 2001 From: Michael James Date: Fri, 12 Jul 2024 22:10:11 +0000 Subject: [PATCH 03/34] Adding Percentile Outlier finder and hist analysis to aid SWIFT_QC --- GeneralTools/findoutliersSWIFT.m | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 GeneralTools/findoutliersSWIFT.m diff --git a/GeneralTools/findoutliersSWIFT.m b/GeneralTools/findoutliersSWIFT.m new file mode 100644 index 0000000..8751130 --- /dev/null +++ b/GeneralTools/findoutliersSWIFT.m @@ -0,0 +1,11 @@ +function [SWIFT,outliersSWIFT] = findoutliersSWIFT(flist`) +%findoutliersSWIFT - finds outliers out of % range and plots hist of all +%SWIFT fields +% Find outliers in SWIFT L1 DATA +% Michael James +% 7_12_2024 + + +outputArg1 = inputArg1; +outputArg2 = inputArg2; +end \ No newline at end of file From a9f3bb2205786385b106a2cfec1cf4183d60b183 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 22 Jul 2024 07:49:59 -0700 Subject: [PATCH 04/34] Final updates --- Aquadopp/catAQH.m | 251 +++++++---- Aquadopp/processAQHburst.m | 138 ++++++ Aquadopp/readSWIFTv3_AQH.m | 2 +- Aquadopp/reprocess_AQD.asv | 147 ------ Aquadopp/reprocess_AQD.m | 15 +- Aquadopp/reprocess_AQH.m | 695 +++++++++++------------------ CT/reprocess_ACS.m | 5 + GeneralTools/postprocess_SWIFT.asv | 71 ++- GeneralTools/postprocess_SWIFT.m | 71 +-- Signature/reprocess_SIG.m | 5 +- Winds/inertialdissipation.m | 27 +- Winds/reprocess_Y81.m | 8 +- 12 files changed, 680 insertions(+), 755 deletions(-) create mode 100644 Aquadopp/processAQHburst.m delete mode 100644 Aquadopp/reprocess_AQD.asv diff --git a/Aquadopp/catAQH.m b/Aquadopp/catAQH.m index 63e2873..1f18371 100644 --- a/Aquadopp/catAQH.m +++ b/Aquadopp/catAQH.m @@ -1,104 +1,173 @@ -function aqh = catAQH(AQH) +function aqh = catAQH(AQH,varargin) %Produces summary plot of burst-averaged signature data stored in 'SIG' % also returns concatenated data -plotaqh = true; -aqh.time = [AQH.time]; -aqh.hrz = AQH(round(end/2)).HRprofile.z; -aqh.hrw = NaN(length(aqh.hrz),length(aqh.time)); -aqh.hrwvar = aqh.hrw; -aqh.hrcorr = aqh.hrw; -aqh.hramp = aqh.hrw; -aqh.eps_struct = aqh.hrw; -aqh.struct_slope = aqh.hrw; -aqh.pitchvar = NaN(1,length(aqh.time)); -for it = 1:length(aqh.time) - %HR - aqh.hrw(:,it) = AQH(it).HRprofile.w; - aqh.hrwvar(:,it) = AQH(it).HRprofile.werr; - aqh.hrcorr(:,it) = AQH(it).QC.hrcorr; - aqh.hramp(:,it) = AQH(it).QC.hramp; - aqh.eps_struct(:,it) = AQH(it).HRprofile.eps_structHP; - aqh.struct_slope(:,it) = AQH(it).QC.slopeHP; - aqh.pitchvar(it) = AQH(it).QC.pitchvar; -end +plotaqh = false; +QCaqh = false; -% Plot -if plotaqh && length(aqh.time)>1 - clear b - figure('color','w') - MP = get(0,'monitorposition'); - set(gcf,'outerposition',MP(1,:).*[1 1 1 1]); - h = tight_subplot(6,1); +if nargin > 1 + if any(strcmp(varargin,'plot')) + plotaqh = true; + end + if any(strcmp(varargin,'qc')) + QCaqh = true; + end + if ~(any(strcmp(varargin,'plot') | strcmp(varargin,'qc'))) + error('Optional inputs must be ''plot'' or ''qc''') + end +end - %Pitch + Roll - axes(h(1)) - b(1) = bar(aqh.time,sqrt(aqh.pitchvar)); - b(1).FaceColor = 'r'; - set(b,'EdgeColor',rgb('grey')) - ylabel('\sigma_{\phi} (^{\circ})');title('Pitch Variance') - c = colorbar;c.Visible = 'off'; - xlim([min(aqh.time) max(aqh.time)]) - ylim([0 mean(sqrt(aqh.pitchvar),'omitnan') + 2*std(sqrt(aqh.pitchvar),'omitnan')]) - datetick('x','KeepLimits') +if isfield(AQH,'time') + aqh.time = [AQH.time]; + nt = length(aqh.time); + + aqh.hrz = AQH(round(end/2)).HRprofile.z; + nzhr = length(aqh.hrz); + aqh.hrcorr = NaN(nzhr,nt); + aqh.hramp = aqh.hrcorr; + aqh.hrw = aqh.hrcorr; + aqh.hrwvar = aqh.hrcorr; + aqh.eps = aqh.hrcorr; + aqh.epsnf = aqh.hrcorr; + aqh.mspe = aqh.hrcorr; + aqh.slope = aqh.hrcorr; + aqh.pspike = aqh.hrcorr; + aqh.pbadping = NaN(1,nt); + aqh.wpeofmag = aqh.hrcorr; + aqh.eofs = NaN(nzhr,nzhr,nt); + aqh.eofsvar = aqh.hrcorr; + aqh.pitch = NaN(1,nt); + aqh.roll = NaN(1,nt); + aqh.head = NaN(1,nt); + aqh.pitchvar = NaN(1,nt); + aqh.rollvar = NaN(1,nt); + aqh.headvar = NaN(1,nt); - %Amplitude - axes(h(2)) - pcolor(aqh.time,-aqh.hrz,aqh.hramp);shading flat - caxis([80 150]) - ylabel('Depth (m)');cmocean('thermal');title('Amplitude') - c = colorbar;c.Label.String = 'A [counts]'; - xlim([min(aqh.time) max(aqh.time)]) - datetick('x','KeepLimits') + for it = 1:length(aqh.time) + + %HR + nz = length(AQH(it).HRprofile.w); + aqh.hrcorr(1:nz,it) = AQH(it).HRprofile.QC.hrcorr; + aqh.hramp(1:nz,it) = AQH(it).HRprofile.QC.hramp; + aqh.pspike(1:nz,it) = AQH(it).HRprofile.QC.pspike; + aqh.pbadping(it) = AQH(it).HRprofile.QC.pbadping; + aqh.hrw(1:nz,it) = AQH(it).HRprofile.w; + aqh.hrwvar(1:nz,it) = AQH(it).HRprofile.wvar; + aqh.eps(1:nz,it) = AQH(it).HRprofile.eps; + aqh.epsnf(1:nz,it) = AQH(it).HRprofile.QC.epsNF; + aqh.mspe(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.mspe; + aqh.slope(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.slope; + + aqh.wpeofmag(1:nz,it) = AQH(it).HRprofile.QC.wpeofmag; + aqh.eofs(1:nz,1:nz,it) = AQH(it).HRprofile.QC.eofs; + aqh.eofsvar(1:nz,it) = AQH(it).HRprofile.QC.eofsvar; - %Correlation - axes(h(3)) - pcolor(aqh.time,-aqh.hrz,aqh.hrcorr);shading flat - caxis([0 100]) - ylabel('Depth (m)');cmocean('curl');title('Correlation') - c = colorbar;c.Label.String = 'C [%]'; - xlim([min(aqh.time) max(aqh.time)]) - datetick('x','KeepLimits') + % Motion + aqh.pitch(it) = AQH(it).motion.pitch; + aqh.roll(it) = AQH(it).motion.roll; + aqh.head(it) = AQH(it).motion.head; + aqh.pitchvar(it) = AQH(it).motion.pitchvar; + aqh.rollvar(it) = AQH(it).motion.rollvar; + aqh.headvar(it) = AQH(it).motion.headvar; + end - %Velocity - axes(h(4)) - pcolor(aqh.time,-aqh.hrz,aqh.hrw);shading flat - caxis([-0.025 0.025]) - ylabel('Depth (m)');cmocean('balance');title('W_{HR}') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(aqh.time) max(aqh.time)]) - datetick('x','KeepLimits') + %QC + badburst = [AQH.badburst]; + if QCaqh && sum(badburst) < length(aqh.time) + + aqh.hrcorr(:,badburst) = []; + aqh.hramp(:,badburst) = []; + aqh.hrw(:,badburst) = []; + aqh.hrwvar(:,badburst) = []; + aqh.eps(:,badburst) = []; + aqh.epsnf(:,badburst) = []; + aqh.mspe(:,badburst) = []; + aqh.slope(:,badburst) = []; + aqh.pspike(:,badburst) = []; + aqh.pbadping(badburst) = []; + aqh.time(badburst) = []; + aqh.wpeofmag(:,badburst) = []; + aqh.eofs(:,:,badburst) = []; + aqh.eofsvar(:,badburst) = []; + aqh.pitch(badburst) = AQH(badburst).motion.pitch; + aqh.roll(badburst) = AQH(badburst).motion.roll; + aqh.head(badburst) = AQH(badburst).motion.head; + aqh.pitchvar(badburst) = AQH(badburst).motion.pitchvar; + aqh.rollvar(badburst) = AQH(badburst).motion.rollvar; + aqh.headvar(badburst) = AQH(badburst).motion.headvar; + else + aqh.badburst = badburst; + end - % Velocity Variance - % pcolor(aqh.time,-aqh.hrz,aqh.hrwvar);shading flat - % caxis([0 0.03]); - % ylabel('Depth (m)');title('\sigma_W_{HR}') - % c = colorbar;c.Label.String = 'ms^{-1}'; - % xlim([min(aqh.time) max(aqh.time)]) - % datetick('x','KeepLimits') + % Plot + if plotaqh && length(aqh.time)>1 - %Dissipation Rate - axes(h(5)) - pcolor(aqh.time,-aqh.hrz,log10(aqh.eps_struct));shading flat - caxis([-7.5 -4.5]); - ylabel('Depth (m)');title('SF \epsilon') - c = colorbar;c.Label.String = 'log_{10}(m^3s^{-2})'; - xlim([min(aqh.time) max(aqh.time)]) - datetick('x','KeepLimits') + figure('color','w') + MP = get(0,'monitorposition'); + set(gcf,'outerposition',MP(1,:).*[1 1 0.5 1]); + clear b - % SF Slope - axes(h(6)) - pcolor(aqh.time,-aqh.hrz,aqh.struct_slope);shading flat - caxis([0 2*2/3]); - ylabel('Depth (m)');title('SF Slope') - c = colorbar;c.Label.String = 'D \propto r^n'; - xlim([min(aqh.time) max(aqh.time)]) - datetick('x','KeepLimits') - cmocean('curl') + % HR + h(1) = subplot(4,2,1);% HR Correlation + pcolor(aqh.time,-aqh.hrz,aqh.hrcorr);shading flat + caxis([50 100]); + ylabel('Depth (m)');cmocean('thermal');title('HR Correlation') + c = colorbar;c.Label.String = '[%]'; + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + h(3) = subplot(4,2,3);% Vertical Velocity + pcolor(aqh.time,-aqh.hrz,aqh.hrw);shading flat + caxis([-0.05 0.05]); + ylabel('Depth (m)');cmocean('balance');title('W') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + h(5) = subplot(4,2,5);% Velocity Variance + pcolor(aqh.time,-aqh.hrz,sqrt(aqh.hrwvar));shading flat + caxis([0 0.5]); + ylabel('Depth (m)');title('\sigma_W') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + h(7) = subplot(4,2,7);% Percent Spikes + pcolor(aqh.time,-aqh.hrz,100*aqh.pspike);shading flat + caxis([0 25]);colormap(gca,lansey(25)) + ylabel('Depth (m)');title('Spike Percent') + c = colorbar;c.Label.String = 'P [%]'; + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + h(2) = subplot(4,2,2);% HR Amplitude + pcolor(aqh.time,-aqh.hrz,aqh.hramp);shading flat + caxis([60 160]); + ylabel('Depth (m)');cmocean('haline');title('HR Amplitude') + c = colorbar;c.Label.String = '[counts]'; + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + h(8) = subplot(4,2,4);% Dissipation Rate + pcolor(aqh.time,-aqh.hrz,log10(aqh.eps));shading flat + caxis([-6 -3]);colormap(gca,'jet') + ylabel('Depth (m)');title('Dissipation Rate') + c = colorbar;c.Label.String = 'log_{10}(m^3s^{-2})'; + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + h(6) = subplot(4,2,6);% MSPE of SF r^(2/3) fit + pcolor(aqh.time,-aqh.hrz,100*sqrt(aqh.mspe));shading flat + caxis([0 25]);colormap(gca,lansey(25)) + ylabel('Depth (m)');title('MSPE') + c = colorbar;c.Label.String = '[%]'; + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + h(8) = subplot(4,2,8);% Best-fit slope to SF + pcolor(aqh.time,-aqh.hrz,aqh.slope);shading flat + caxis([0 2*2/3]); + ylabel('Depth (m)');title('SF Slope') + c = colorbar;c.Label.String = 'D \propto r^n';cmocean('curl') + xlim([min(aqh.time) max(aqh.time)]) + datetick('x','KeepLimits') + drawnow + end - h = findobj('Type','axes'); - h = flipud(h); - set(h(1:end-1),'XTickLabel',[]) +else + aqh = []; + warning('AQH empty...') end - -end \ No newline at end of file diff --git a/Aquadopp/processAQHburst.m b/Aquadopp/processAQHburst.m new file mode 100644 index 0000000..6608d7d --- /dev/null +++ b/Aquadopp/processAQHburst.m @@ -0,0 +1,138 @@ +function [HRprofile,fh] = processAQHburst(burst,opt) + +% Data +corr = burst.CorrelationData'; +amp = burst.AmplitudeData'; +w = -burst.VelocityData'; + +% N pings + N z-bins +[nbin,nping] = size(w); +xz = opt.xz; +dz = opt.dz; +bz = opt.bz; +z = xz + bz + dz*(1:nbin)'; + +% Find Spikes +nsm = round(opt.lsm/dz); % 0.25 m +wfilt = w - smooth_mat(w',hann(nsm))'; +ispike = abs(wfilt) > 3*std(wfilt); +winterp = NaN(size(w)); +for iping = 1:nping + igood = find(~ispike(:,iping)); + if length(igood) > 3 + winterp(:,iping) = interp1(igood,w(igood,iping),1:nbin,'linear','extrap'); + end +end + +%%%%%% Velocity Profile %%%%%% +HRprofile.w = mean(winterp,2,'omitnan'); +HRprofile.wvar = var(winterp,[],2,'omitnan'); +HRprofile.z = z; + +%%%%%% Dissipation Estimates %%%%%% + +% 1) Spatial High-pass +nsm = round(opt.lsm/dz); % 1 m +wphp = winterp - smooth_mat(winterp',hann(nsm))'; +% wphp = winterp-mean(winterp,1,'omitnan'); + +% 2) EOF High-pass using interpolated data +badping = sum(ispike)./nbin > 0.5; +% - remove worst pings to get good EOFs, then interpolate back in time +% - alternatively, could use *only* despiked data. This does not +% work well when data are very noisy/poor quality +eofamp = NaN(size(w')); +[eofs,eofamp(~badping,:),eofvar,~] = eof(winterp(:,~badping)'); +if sum(~badping) > 2 + for ieof = 1:nbin + eofamp(:,ieof) = interp1(find(~badping),eofamp(~badping,ieof),1:nping); + end + wpeof = eofs(:,opt.nsumeof+1:end)*(eofamp(:,opt.nsumeof+1:end)'); +else + wpeof = NaN(size(w)); +end + +%%%%%%%%% 3) Estimate dissipation rate from velocity structure functions +ibad = ispike | repmat(badping,nbin,1); +rmin = dz; +rmax = 4*dz; +nzfit = 1; +w(ibad) = NaN; +wpeof(ibad) = NaN; +wphp(ibad) = NaN; + +warning('off','all') +% No filter, no analytic wave fit (D ~ r^{-2/3}) +[epsNF,qualNF] = SFdissipation(w,z,rmin,2*rmax,nzfit,'linear','mean'); +% Analytic wave fit (D ~ Ar^{-2/3} + Br^2) +[epsWV,qualWV] = SFdissipation(w,z,rmin,2*rmax,nzfit,'cubic','mean'); +% EOF filter (D ~ r^{-2/3}) +[epsEOF,qualEOF] = SFdissipation(wpeof,z,rmin,rmax,nzfit,'linear','mean'); +% High-pass filter (D ~ r^{-2/3}) +[epsHP,qualHP] = SFdissipation(wphp,z,rmin,rmax,nzfit,'linear','mean'); +warning('on','all') + +% Save Dissipation Results +HRprofile.eps = epsHP';% For AQH, simple high pass seems like the better estimate. Too few bins for good EOFs. + +% Additional information for quality control +HRprofile.QC.epsNF = epsNF'; +HRprofile.QC.epsWV = epsWV'; +HRprofile.QC.epsHP = epsHP'; +HRprofile.QC.epsEOF = epsEOF'; +HRprofile.QC.qualNF = qualNF; +HRprofile.QC.qualWV = qualWV; +HRprofile.QC.qualHP = qualHP; +HRprofile.QC.qualEOF = qualEOF; +HRprofile.QC.eofs = eofs; +HRprofile.QC.eofsvar = eofvar; +HRprofile.QC.wpeofmag = std(wpeof,[],2,'omitnan')'; +HRprofile.QC.hrcorr = mean(corr,2,'omitnan'); +HRprofile.QC.hramp = mean(amp,2,'omitnan'); +HRprofile.QC.pspike = (sum(ispike,2,'omitnan')./nping); +HRprofile.QC.pbadping = sum(badping)./nping; + +% Plot Burst +if opt.plotburst + + % Visualize Data + fh(1) = figure('color','w'); + clear c + MP = get(0,'monitorposition'); + set(gcf,'outerposition',MP(1,:).*[1 1 1 1]); + subplot(5,1,1) + imagesc(amp) + caxis([50 160]); cmocean('amp') + title('HR Data'); + ylabel('Bin #') + c = colorbar;c.Label.String = 'A (dB)'; + subplot(5,1,2) + imagesc(corr) + caxis([35 100]);cmocean('amp') + ylabel('Bin #') + c = colorbar;c.Label.String = 'C (%)'; + subplot(5,1,3) + imagesc(winterp) + caxis([-0.5 0.5]);cmocean('balance'); + ylabel('Bin #') + c = colorbar;c.Label.String = 'W_r (m/s)'; + subplot(5,1,4) + imagesc(ispike) + caxis([0 2]);colormap(gca,[rgb('white'); rgb('black')]) + ylabel('Bin #') + c = colorbar;c.Ticks = [0.5 1.5]; + c.TickLabels = {'Good','Spike'}; + subplot(5,1,5) + imagesc(wphp) + ylabel('Bin #') + caxis([-0.05 0.05]);cmocean('balance') + c = colorbar;c.Label.String = 'W_hp (m/s)'; + + xlabel('Ping #') + drawnow +else + fh = []; +end + +end + diff --git a/Aquadopp/readSWIFTv3_AQH.m b/Aquadopp/readSWIFTv3_AQH.m index f505335..aed0e0a 100644 --- a/Aquadopp/readSWIFTv3_AQH.m +++ b/Aquadopp/readSWIFTv3_AQH.m @@ -139,4 +139,4 @@ save([filename(1:end-4) '.mat']) -batteryVoltage = mean(battery) \ No newline at end of file +%batteryVoltage = mean(battery) \ No newline at end of file diff --git a/Aquadopp/reprocess_AQD.asv b/Aquadopp/reprocess_AQD.asv deleted file mode 100644 index 46dfa01..0000000 --- a/Aquadopp/reprocess_AQD.asv +++ /dev/null @@ -1,147 +0,0 @@ -function [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw) - -% reprocess SWIFT v3 downlooking Aquadopp (AQD) results -% loop thru raw data for a given SWIFT deployment, then -% replace values in the SWIFT data structure of results -% (assuming concatSWIFTv3_processed.m has already been run. -% -% -% M. Smith 10/2015 based on reprocess_AQH.m code -% J. Thomson 12/2015 to include option phase resolved dissipation (commented-out) -% cleaned and revised with AQD read function, Thomson, Jun 2016 -% M. Moulton 3/2017 correct error in Dir calculation -% K. Zeiden 07/2024 reformatted for symmetry with bulk post processing -% postprocess_SWIFT.m - -if ispc - slash = '\'; -else - slash = '/'; -end - -%% Parameters - -% QC parameters -minamp = 30; % amplitude cutoff, usually 30 or 40 -minn = 50; % number of cells averaged for each depth -z = 1.25:0.5:20.75; - -% Raw Doppler velocity precision of 1 MHz Aquadopp -Vert_prec = .074; %m/s -Hori_prec = .224; %m/s - -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function - -l1file = dir([missiondir slash '*SWIFT*L1.mat']); -l2file = dir([missiondir slash '*SWIFT*L2.mat']); - -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) - return -end - -burstreplaced = NaN( - -%% Loop through raw burst files and reprocess - -bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*AQD*.dat']); - -for iburst = 1:length(bfiles) - - disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - - % read or load raw data - if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) || readraw - [~,VelE,VelN,VelU,Amp1,Amp2,~,~,~,~,~] = readSWIFTv3_AQD([bfiles(iburst).folder slash bfiles(iburst).name]); - else - load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat']) %#ok - end - - % Use velocities only when sufficient return - exclude = Amp1 < minamp | Amp2 < minamp; % Used for vel. magnitude - - % Average amplitudes of just velocity measurements used - Amp1(exclude) = NaN; - Amp2(exclude) = NaN; - Amp = (Amp1+Amp2)./2; % Corresponds to those used in velocity - - n = sum(~isnan(Amp),1); - E_error = Hori_prec./sqrt(n); - N_error = E_error; - Hori_error = sqrt((E_error.^2) + (N_error.^2)); - Hori_error(n < minn) = NaN; - Amp = nanmean(Amp,1); - Amp(n < minn) = NaN; - - % Average velocities first for "net" velocity + direction - VelE(Amp1 12/(60*24) - disp('No time match. Skippping...') - else - burstreplaced(tindex) = true; - end - - % Replace Values in SWIFT structure - SWIFT(tindex).downlooking.velocitydirection = Dir'; - SWIFT(tindex).downlooking.amplitude = Amp'; - SWIFT(tindex).downlooking.z = z; - SWIFT(tindex).downlooking.velocityerror = Hori_error; - SWIFT(tindex).downlooking.vertvel = nanmean(VelU,1); - - if nansum(SWIFT(tindex).downlooking.velocityprofile) > 0 && tdiff < 0.04 - SWIFT(tindex).downlooking.velocityprofile = Vel'; - else - SWIFT(tindex).downlooking.velocityprofile = NaN(40,1); - end - -end - - -%% Log reprocessing and flags, then save new L2 file or overwrite existing one - -if isfield(sinfo,'postproc') -ip = length(sinfo.postproc)+1; -else - sinfo.postproc = struct; - ip = 1; -end -sinfo.postproc(ip).type = 'AQD'; -sinfo.postproc(ip).usr = getenv('username'); -sinfo.postproc(ip).time = string(datetime('now')); -sinfo.postproc(ip).flags = []; -sinfo.postproc(ip).params = []; - -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') - -%% End function -end - - diff --git a/Aquadopp/reprocess_AQD.m b/Aquadopp/reprocess_AQD.m index 215be27..6d9e3d2 100644 --- a/Aquadopp/reprocess_AQD.m +++ b/Aquadopp/reprocess_AQD.m @@ -21,14 +21,13 @@ %% Parameters -% QC parameters minamp = 30; % amplitude cutoff, usually 30 or 40 minn = 50; % number of cells averaged for each depth z = 1.25:0.5:20.75; % Raw Doppler velocity precision of 1 MHz Aquadopp -Vert_prec = .074; %m/s -Hori_prec = .224; %m/s +Vert_prec = .074; % m/s +Hori_prec = .224; % m/s %% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function @@ -76,7 +75,7 @@ N_error = E_error; Hori_error = sqrt((E_error.^2) + (N_error.^2)); Hori_error(n < minn) = NaN; - Amp = nanmean(Amp,1); + Amp = mean(Amp,1,'omitnan'); Amp(n < minn) = NaN; % Average velocities first for "net" velocity + direction @@ -84,8 +83,8 @@ VelN(Amp2 0 + if sum(SWIFT(tindex).downlooking.velocityprofile,'omitnan') > 0 SWIFT(tindex).downlooking.velocityprofile = Vel'; else SWIFT(tindex).downlooking.velocityprofile = NaN(40,1); diff --git a/Aquadopp/reprocess_AQH.m b/Aquadopp/reprocess_AQH.m index 83f11c6..ae60932 100644 --- a/Aquadopp/reprocess_AQH.m +++ b/Aquadopp/reprocess_AQH.m @@ -1,4 +1,6 @@ -%% Reprocess SWIFT v4 Aquadopp velocities from burst data +function [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst) + +% Reprocess SWIFT v4 Aquadopp velocities from burst data % Loops through burst MAT or DAT files for a given SWIFT deployment, % reprocessing AQH data: 1) quality control the data, 2) compute % mean profiles of velocity, 3) compute dissipation from the HR beam 4) @@ -12,470 +14,287 @@ % the EOF method isn't working very well due to the low N (16 bins % vs. 128 bins for Sig1000). Low-pass does well b/c the shear is % moderate? +% K. Zeiden 07/2024 - + +if ispc + slash = '\'; +else + slash = '/'; +end -%% User Defined Inputs - -% JIM - MAC -% % Directory with existing SWIFT structures (e.g. from telemetry) -% swiftdir = './'; -% % Directory with signature burst files -% burstdir = './';% -% % Directory to save updated/new SWIFT/SIG structures (see toggle 'saveSWIFT') -% saveswiftdir = [ swiftstructdir ]; -% savesigdir = [ swiftstructdir ]; -% % Directory to save figures (will create folder for each mission if doesn't already exist) -% savefigdir = './'; - -% KRISTIN - PC -% Directory with existing SWIFT structures (e.g. from telemetry) -swiftdir = 'S:\LC-DRI\'; -% Directory with signature burst files -burstdir = 'S:\LC-DRI\'; -% Directory to save updated/new SWIFT/SIG structures (see toggle 'saveSWIFT') -saveswiftdir = 'C:\Users\kfitz\Dropbox\MATLAB\LC-DRI\Data\SWIFT\L2\V3\neof5\reprocessAQH\'; -savesigdir = 'C:\Users\kfitz\Dropbox\MATLAB\LC-DRI\Data\SWIFT\L2\V3\neof5\reprocessAQH\AQH\'; -% Directory to save figures (will create folder for each mission if doesn't already exist) -savefigdir = 'C:\Users\kfitz\Dropbox\MATLAB\LC-DRI\Figures\AQH'; - -%Data Load/Save Toggles -readraw = true;% read raw binary files -saveSWIFT = true;% save updated SWIFT structure -saveAQH = true; %save detailed sig data in separate SIG structure - -% Plotting Toggles -plotburst = false; % generate plots for each burst -plotmission = true; % generate summary plot for mission -saveplots = false; % save generated plots -MP = get(0,'monitorposition'); +%% Internal Toggles + +% Data Load/Save Toggles +opt.saveAQH = true; %save detailed sig data in separate SIG structure +opt.saveplots = true; % save generated plots +opt.plotburst = plotburst; +opt.readraw = readraw; % AQH Config -xcdrdepth = 0.8; % depth of transducer [m] -dz = 0.04; % cell size -bz = 0.1; % blanking distance -dt = 1/4; % seconds - -% User defined QC parameters -nsumeof = 3;% Number of lowest-mode EOFs to remove from turbulent velocity - -%Populate list of SWIFT missions to re-process -cd(burstdir) -swifts = dir('SWIFT1*'); -swifts = {swifts.name}; -nswift = length(swifts); - -clear SWIFT AQH -badburst = false; - -%% Loop through SWIFT missions -% For each mission, loop through burst files and process the data - -for iswift = [1:3 6:nswift] - - SNprocess = swifts{iswift}; - disp(['********** Reprocessing ' SNprocess ' **********']) - - % Create SIG structure for detailed signature data results - AQH = struct; - isig = 1; - - % Load pre-existing mission mat file with SWIFT structure - structfile = dir([swiftdir SNprocess '\' SNprocess(1:6) '*.mat']); - if length(structfile) > 1 - structfile = structfile(contains({structfile.name},'reprocessedIMU.mat')); - end - if isempty(structfile) - disp('No SWIFT structure found...') - SWIFT = struct; +opt.xz = 0.8; % depth of transducer [m] +opt.dz = 0.04; % cell size +opt.bz = 0.1; % blanking distance +opt.dt = 1/4; % seconds + +% QC +opt.mincorr = 50; +opt.minamp = 80; +opt.nsumeof = 3; +opt.lsm = 0.5; + +%% Data type to be read in +if opt.readraw + ftype = '.dat'; +else + ftype = '.mat'; +end + +%% Load or create SWIFT structure, create SIG structure, list burst files + +l1file = dir([missiondir slash '*SWIFT*L1.mat']); +l2file = dir([missiondir slash '*SWIFT*L2.mat']); + +if ~isempty(l2file) % First check to see if there is an existing L2 file to load + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file + sfile = l1file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L1 or L2 product exists + warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end +burstreplaced = false(length(SWIFT),1); +badaqh = false(length(SWIFT),1); + +AQH = struct; +iaqh = 1; + +%% List of burst files + +bfiles = dir([missiondir slash 'AQH' slash 'Raw' slash '*' slash '*' ftype]); +if isempty(bfiles) + error(' No burst files found ') +end +nburst = length(bfiles); + +%% Loop through burst files and reprocess signature data +for iburst = 1:nburst + + % Burst time stamp and name + day = bfiles(iburst).name(13:21); + hour = bfiles(iburst).name(23:24); + mint = bfiles(iburst).name(26:27); + btime = datenum(day) + datenum(0,0,0,str2double(hour),(str2double(mint)-1)*12,0); + bname = bfiles(iburst).name(1:end-4); + disp(['Burst ' num2str(iburst) ' : ' bname]) + + % Load burst file + if readraw + disp('Reading raw AQH file...') + [time,vel,amp,cor,press,pitch,roll,heading] = ... + readSWIFTv3_AQH([bfiles(iburst).folder '\' bfiles(iburst).name]); else - load([structfile.folder '\' structfile.name]) - % Prepare flag vector for replaced burst data - burstreplaced = false(length(SWIFT),1); + load([bfiles(iburst).folder '\' bfiles(iburst).name],... + 'Vel','Cor','Amp','Pressure','pitch','roll','heading','time') + vel = Vel; + cor = Cor; + amp = Amp; + press = Pressure'; end - % Populate list of burst files - if readraw - ftype = '.dat'; + % Skip burst if empty + if isempty(vel) + disp('Failed to read, skipping burst...') + continue + end + + % Create burst structure + burst.time = time; + burst.CorrelationData = cor; + burst.AmplitudeData = amp; + burst.VelocityData = vel; + burst.Heading = heading; + burst.Pitch = pitch; + burst.Roll = roll; + burst.Pressure = press; + clear cor amp vel time heading pitch roll press + + % Burst time + t0 = min(burst.time); + if abs(btime - t0) > 12/(60*24) + disp(' WARNING: File name disagrees with recorded time. Using recorded time... ') + btime = t0; + end + + %%%%%%% FLAGS %%%%%%= + + % Flag if coming in/out of the water + if any(ischange(burst.Pressure)) + disp(' FLAG: Out-of-Water (temp/pressure change)...') + outofwater = true; else - ftype = '.mat'; + outofwater = false; end - if ispc - fpath = '\AQH\Raw\*\*'; + + % Flag out of water based on bursts w/anomalously high amplitude + if mean(burst.AmplitudeData(:),'omitnan') < opt.minamp + disp(' FLAG: Low Amp (low average amp)...') + badamp = true; else - fpath = '/AQH/Raw/*/*'; + badamp = false; end - bfiles = dir([burstdir SNprocess fpath ftype]); - if isempty(bfiles) - disp(' No burst files found, skipping SWIFT...') - continue + + % Flag out of water based on bursts w/low correlation + if mean(burst.CorrelationData(:),'omitnan') < opt.mincorr + disp(' FLAG: Bad Corr (low average corr)...') + badcorr = true; + else + badcorr = false; end - nburst = length(bfiles); + + badburst = outofwater | badamp | badcorr; + + %%%%%%% Process HR velocity data ('burst' structure) %%%%%% + + [HRprofile,fh] = processAQHburst(burst,opt); - % Loop through and process burst files - for iburst = 1:nburst + if isempty(HRprofile) + HRprofile = NaNstructR(AQH(iaqh-1).HRprofile); + end + + if opt.saveplots && ~isempty(fh) + figure(fh(1)) + set(gcf,'Name',[bname '_HR_data']) + figname = [bfiles(iburst).folder slash get(gcf,'Name')]; + print(figname,'-dpng') + close gcf + + end - % Burst time stamp - day = bfiles(iburst).name(13:21); - hour = bfiles(iburst).name(23:24); - mint = bfiles(iburst).name(26:27); - btime = datenum(day)+datenum(0,0,0,str2double(hour),(str2double(mint)-1)*12,0); - bname = bfiles(iburst).name(1:end-4); - disp(['Burst ' num2str(iburst) ' : ' bname]) - - % Load burst file - if readraw - disp('Reading raw AQH file...') - [~,vel,amp,corr,press,pitch,roll,heading] = ... - readSWIFTv3_AQH([bfiles(iburst).folder '\' bfiles(iburst).name]); - else - load([bfiles(iburst).folder '\' bfiles(iburst).name],... - 'Vel','Cor','Amp','Pressure','pitch','roll','heading') - vel = Vel; - corr = Cor; - amp = Amp; - press = Pressure'; + %%%%%%%% Save detailed signature data in SIG structure %%%%%%%% + + %Time + AQH(iaqh).time = btime; + % HR data + AQH(iaqh).HRprofile = HRprofile; + % Motion + AQH(iaqh).motion.pitch = mean(burst.Pitch,'omitnan'); + AQH(iaqh).motion.roll = mean(burst.Roll,'omitnan'); + AQH(iaqh).motion.head = mean(burst.Heading,'omitnan'); + AQH(iaqh).motion.pitchvar = var(burst.Pitch,'omitnan'); + AQH(iaqh).motion.rollvar = var(burst.Roll,'omitnan'); + AQH(iaqh).motion.headvar = var(unwrap(burst.Heading),'omitnan'); + % Badburst & flags + AQH(iaqh).badburst = outofwater | badamp | badcorr; + AQH(iaqh).flag.outofwater = outofwater; + AQH(iaqh).flag.badamp = badamp; + AQH(iaqh).flag.badcorr = badcorr; + AQH(iaqh).flag.notimematch = false; % Updated below + + iaqh = iaqh+1; + + %%%%%%%% Match burst time to existing SWIFT fields and replace data %%%%%%%% + + if ~isempty(fieldnames(SWIFT)) && ~isempty(SWIFT) + + [tdiff,tindex] = min(abs([SWIFT.time]-btime)); + if tdiff > 12/(60*24)% must be within 10 min + disp(' NO time index match...') + timematch = false; + elseif tdiff < 12/(60*24) + timematch = true; + burstreplaced(tindex) = true; + elseif isempty(tdiff) + disp(' NO time index match...') + timematch = false; end - % Skip burst if empty - if isempty(vel) - disp('Failed to read, skipping burst...') - continue - end + if timematch && ~badburst % time match, good burst + % HR data + SWIFT(tindex).uplooking = []; + SWIFT(tindex).uplooking.w = HRprofile.w; + SWIFT(tindex).uplooking.wvar = HRprofile.wvar; + SWIFT(tindex).uplooking.z = HRprofile.z'; + SWIFT(tindex).uplooking.tkedissipationrate = HRprofile.eps; - % HR Data - hrcorr = corr'; - hramp = amp'; - hrvel = -vel'; - clear corr amp vel + elseif timematch && badburst % time match, bad burst + % HR data + SWIFT(tindex).uplooking = []; + SWIFT(tindex).uplooking.w = NaN(size(HRprofile.w)); + SWIFT(tindex).uplooking.wvar = NaN(size(HRprofile.w)); + SWIFT(tindex).uplooking.z = HRprofile.z'; + SWIFT(tindex).uplooking.tkedissipationrate = ... + NaN(size(HRprofile.w')); - % N pings + N z-bins - [nbin,nping] = size(hrvel); - hrz = xcdrdepth - bz - dz*(1:nbin); - hrtime = (0:nping)*dt; - - %%%%% QC %%%%% - - % Low correlation - corrmin = 50; - ilowcorr = hrcorr < corrmin; - - % Too high velocity - nsm = round(0.5/dz); % 0.25 m - wlp = smooth_mat(hrvel',hann(nsm))'; - wphp = hrvel - wlp; - wstd = std(wphp(~ilowcorr)); - ispike = abs(wphp) > 3*wstd; - - % Bad pts - ibad = ispike | ilowcorr; - badping = sum(ibad)/nbin > 0.5; - - % EOF High-pass - nsumeof = 3; - [eofs,eof_amp,~,~] = eof(hrvel'); - weof = eofs(:,1:nsumeof)*(eof_amp(:,1:nsumeof)'); - wpeof = eofs(:,nsumeof+1:end)*(eof_amp(:,nsumeof+1:end)'); - - % Plot beam data and QC info - if plotburst - clear c - figure('color','w','Name',[bname '_hr_data']) - set(gcf,'outerposition',MP(1,:).*[1 1 1 1]); - subplot(5,1,1) - imagesc(hramp) - caxis([60 160]); cmocean('thermal') - title('HR Data'); - ylabel('Bin #') - c = colorbar;c.Label.String = 'A (dB)'; - subplot(5,1,2) - imagesc(hrcorr) - caxis([0 100]);colormap(gca,'jet') - ylabel('Bin #') - c = colorbar;c.Label.String = 'C (%)'; - subplot(5,1,3) - plot(press,'k','LineWidth',1.5) - axis tight - ylabel('P [dB]') - c = colorbar;c.Visible = 'off'; - subplot(5,1,4) - imagesc(hrvel) - caxis([-0.5 0.5]);cmocean('balance'); - ylabel('Bin #') - c = colorbar;c.Label.String = 'U_r(m/s)'; - subplot(5,1,5) - imagesc(ibad) - caxis([0 2]);colormap(gca,[rgb('white'); rgb('black')]) - ylabel('Bin #') - c = colorbar;c.Ticks = [0.5 1.5]; - c.TickLabels = {'Good','Bad'}; - xlabel('Ping #') - - drawnow - set(get(gcf,'Children'),'YDir','Normal') - if saveplots - figname = [savefigdir SNprocess '\' get(gcf,'Name')]; - print(figname,'-dpng') - close gcf - end - end - - %%%%%% Mean velocity profile + standard error %%%%% - - hrw = nanmean(wlp,2); - hrwerr = nanstd(wlp,[],2)./sqrt(nping); + badaqh(tindex) = true; - %%%%%% Dissipation Estimates %%%%%% + elseif ~timematch && ~badburst % Good burst, no time match + disp(' ALERT: Burst good, but no time match...') + AQH(iaqh-1).flag.notimematch = true; - % Sampling rate and window size - fs = 1/dt; nwin = 64; - if nwin > nping - nwin = nping; end + end - % Skip dissipation estimates if bad-burst - if sum(badping)/nping > 0.90 - disp(' Skipping dissipation...') - eps_struct0 = NaN(size(hrw)); - eps_structHP = NaN(size(hrw)); - eps_structEOF = NaN(size(hrw)); - eps_spectral = NaN(size(hrw)); - mspe0 = NaN(size(hrw)); - mspeHP = NaN(size(hrw)); - mspeEOF = NaN(size(hrw)); - slope0 = NaN(size(hrw)); - slopeHP = NaN(size(hrw)); - slopeEOF = NaN(size(hrw)); - wpsd = NaN(nbin,2*nwin+1); - bobpsd = NaN(1,2*nwin+1); - f = NaN(1,2*nwin+1); - else - - %Structure Function Dissipation - rmin = dz; - rmax = 4*dz; - nzfit = 1; - w = wlp; - wp1 = wpeof; - wp2 = wphp; - w(ibad) = NaN; - wp1(ibad) = NaN; - wp2(ibad) = NaN; - warning('off','all') - % z = -hrz, for some reason. too tired - [eps_struct0,~,fitcoeff0,qual0] = SFdissipation(w,-hrz,rmin,2*rmax,nzfit,'cubic','mean'); - [eps_structEOF,~,fitcoeffEOF,qualEOF] = SFdissipation(wp1,-hrz,rmin,rmax,nzfit,'linear','mean'); - [eps_structHP,~,fitcoefHP,qualHP] = SFdissipation(wp2,-hrz,rmin,rmax,nzfit,'linear','mean'); - warning('on','all') - mspe0 = qual0.mspe; - mspeHP = qualHP.mspe; - mspeEOF = qualEOF.mspe; - slope0 = qual0.slope; - slopeHP = qualHP.slope; - slopeEOF = qualEOF.slope; - - % Motion spectra (bobbing) - [bobpsd,f] = pwelch(detrend(gradient(press,dt)),nwin,[],[],fs); - for ibin = 1:nbin - [wpsd(ibin,:),f] = pwelch(detrend(hrvel(ibin,:)),nwin,[],[],fs); - end - - if plotburst - clear b s - figure('color','w','Name',[bname '_wspectra_eps']) - set(gcf,'outerposition',MP(1,:).*[1 1 1 1]); - subplot(1,4,[1 2]) - cmap = colormap; - for ibin = 1:nbin - cind = round(size(cmap,1)*ibin/nbin); - l1 = loglog(f,wpsd(ibin,:),'color',cmap(cind,:),'LineWidth',1.5); - hold on - end - l2 = loglog(f,bobpsd,'LineWidth',2,'color',rgb('black')); - xlabel('Frequency [Hz]') - ylabel('TKE [m^2/s^2/Hz]') - title('HR Spectra') - c = colorbar; - c.Label.String = 'Bin #'; - c.TickLabels = num2str(round(c.Ticks'.*nbin)); - legend([l1 l2],'S_{w}','S_{bob}','Location','northwest') - ylim(10.^[-6 0.8]) - xlim([10^-0.5 max(f)]) - subplot(1,4,3) - b(1) = errorbar(hrw,hrz,hrwerr,'horizontal'); - hold on - b(2) = errorbar(mean(hrvel,2,'omitnan'),hrz,std(hrvel,[],2,'omitnan')/nping,'horizontal'); - set(b,'LineWidth',2) - plot([0 0],[0 1],'k--') - xlabel('w [m/s]'); - title('Velocity') - set(gca,'Ydir','reverse') - legend(b,'HR','HR (no QC)','Location','southeast') - ylim([0 1]) - xlim([-0.075 0.075]) - set(gca,'YAxisLocation','right') - subplot(1,4,4) - s(1) = semilogx(eps_structEOF,hrz,'r','LineWidth',2); - hold on - s(2) = semilogx(eps_structHP,hrz,':r','LineWidth',2); - s(3) = semilogx(eps_struct0,hrz,'color',rgb('grey'),'LineWidth',2); - xlim(10.^([-9 -3])) - ylim([0 1]) - legend(s,'SF (EOF)','SF (high-pass)','SF (modified)','Location','southeast') - title('Dissipation') - xlabel('\epsilon [W/Kg]'),ylabel('z [m]') - set(gca,'Ydir','reverse') - set(gca,'YAxisLocation','right') - drawnow - if saveplots - figname = [savefigdir SNprocess '\' get(gcf,'Name')]; - print(figname,'-dpng') - close gcf - end - end - end - - %%%%%%%% Save processed signature data in seperate structure %%%%%%%% - - % HR data - AQH(isig).HRprofile.w = hrw; - AQH(isig).HRprofile.werr = hrwerr; - AQH(isig).HRprofile.z = hrz'; - AQH(isig).HRprofile.eps_struct0 = eps_struct0'; - AQH(isig).HRprofile.eps_structHP = eps_structHP'; - AQH(isig).HRprofile.eps_structEOF = eps_structEOF'; - %Time - AQH(isig).time = btime; - %QC Info - AQH(isig).QC.hrcorr = mean(hrcorr,2,'omitnan')'; - AQH(isig).QC.hramp = mean(hramp,2,'omitnan')'; - AQH(isig).QC.pitch = mean(pitch,'omitnan'); - AQH(isig).QC.roll = mean(roll,'omitnan'); - AQH(isig).QC.head = mean(heading,'omitnan'); - AQH(isig).QC.pitchvar = var(pitch,'omitnan'); - AQH(isig).QC.rollvar = var(roll,'omitnan'); - AQH(isig).QC.headvar = var(unwrap(heading),'omitnan'); - AQH(isig).QC.wpsd = wpsd; - AQH(isig).QC.bobpsd = bobpsd; - AQH(isig).QC.f = f; - AQH(isig).QC.mspe0 = mspe0; - AQH(isig).QC.mspeHP = mspeHP; - AQH(isig).QC.mspeEOF = mspeEOF; - AQH(isig).QC.slope0 = slope0; - AQH(isig).QC.slopeHP = slopeHP; - AQH(isig).QC.slopeEOF = slopeEOF; - isig = isig+1; - - %%%%%%%% Match burst time to existing SWIFT fields and replace data %%%%%%%% - - if ~isempty(fieldnames(SWIFT)) && ~isempty(SWIFT) - - [tdiff,tindex] = min(abs([SWIFT.time]-btime)); - if tdiff > 1/(24*10) % must be within 15 min - disp(' NO time index match...') - timematch = false; - elseif tdiff < 1/(24*10) - timematch = true; - burstreplaced(tindex) = true; - elseif isempty(tdiff) - disp(' NO time index match...') - timematch = false; - end - - if timematch && ~badburst % Good burst & time match - % HR data - SWIFT(tindex).uplooking= []; - SWIFT(tindex).uplooking.w = hrw; - SWIFT(tindex).uplooking.werr = hrwerr; - SWIFT(tindex).uplooking.z = hrz'; - SWIFT(tindex).uplooking.tkedissipationrate = eps_structHP'; - - elseif timematch && badburst % Bad burst & time match - % HR data - SWIFT(tindex).uplooking = []; - SWIFT(tindex).uplooking.w = NaN(size(hrw)); - SWIFT(tindex).uplooking.werr = NaN(size(hrw)); - SWIFT(tindex).uplooking.z = hrz; - SWIFT(tindex).uplooking.tkedissipationrate = NaN(size(eps_structHP')); - elseif ~timematch && ~badburst % Good burst, no time match - disp(' ALERT: Burst good, but no time match...') - tindex = length(SWIFT)+1; - burstreplaced = [burstreplaced; true]; - varcopy = fieldnames(SWIFT); - varcopy = varcopy(~strcmp(varcopy,'signature')); - for icopy = 1:length(varcopy) - if isa(SWIFT(1).(varcopy{icopy}),'double') - SWIFT(tindex).(varcopy{icopy}) = NaN; - elseif isa(SWIFT(1).(varcopy{icopy}),'struct') - varcopy2 = fieldnames(SWIFT(1).(varcopy{icopy})); - for icopy2 = 1:length(varcopy2) - varsize = size(SWIFT(1).(varcopy{icopy}).(varcopy2{icopy2})); - SWIFT(tindex).(varcopy{icopy}).(varcopy2{icopy2}) = NaN(varsize); - end - end - end - % HR data - SWIFT(tindex).uplooking= []; - SWIFT(tindex).uplooking.w = hrw; - SWIFT(tindex).uplooking.werr = hrwerr; - SWIFT(tindex).uplooking.z = hrz'; - SWIFT(tindex).uplooking.tkedissipationrate = eps_structHP'; - % Time - SWIFT(tindex).time = btime; - disp([' Burst time: ' datestr(btime)]) - disp([' (new) SWIFT time: ' datestr(SWIFT(tindex).time)]) - end - end +% End burst loop +end - % End burst loop - end - - % NaN out SWIFT sig fields which were not matched to bursts +%% Clean up and save + +% NaN out SWIFT aqh fields which were not matched to bursts +if ~isempty(fieldnames(SWIFT)) inan = find(~burstreplaced); if ~isempty(inan) for it = inan' % HR data - SWIFT(tindex).uplooking = []; - SWIFT(tindex).uplooking.w = NaN(size(hrw)); - SWIFT(tindex).uplooking.werr = NaN(size(hrw)); - SWIFT(tindex).uplooking.z = hrz; - SWIFT(tindex).uplooking.tkedissipationrate = NaN(size(eps_structHP')); + SWIFT(it).uplooking = []; + SWIFT(it).uplooking.w = NaN(size(HRprofile.w)); + SWIFT(it).uplooking.wvar = NaN(size(HRprofile.w)); + SWIFT(it).uplooking.z = HRprofile.z; + SWIFT(it).uplooking.tkedissipationrate = NaN(size(HRprofile.w')); end end +end - % Sort by time - if ~isempty(fieldnames(SWIFT)) - [~,isort] = sort([SWIFT.time]); - SWIFT = SWIFT(isort); - end - - %%%%%% Plot burst Averaged SWIFT Signature Data %%%%%% - if plotmission - catAQH(AQH); - set(gcf,'Name',SNprocess) - if saveplots - %Create mission folder if doesn't already exist - if ~isfolder([savefigdir SNprocess]) - mkdir([savefigdir SNprocess]) - end - figname = [savefigdir '\' get(gcf,'Name')]; - print(figname,'-dpng') - close gcf - end - end - - %%%%%% Save SWIFT Structure %%%%%%%% - if saveSWIFT && ~isempty(fieldnames(SWIFT)) - if strcmp(structfile.name(end-6:end-4),'IMU') - save([saveswiftdir SNprocess '_reprocessedAQHandIMU.mat'],'SWIFT') - else - save([saveswiftdir SNprocess '_reprocessedAQH.mat'],'SWIFT') - end - end - - %%%%%% Save SIG Structure %%%%%%%% - if saveAQH - save([savesigdir SNprocess '_burstavgAQH.mat'],'AQH') - end - -% End mission loop +% Sort by time +if ~isempty(fieldnames(SWIFT)) && isfield(SWIFT,'time') +[~,isort] = sort([SWIFT.time]); +SWIFT = SWIFT(isort); +end + +%% Log reprocessing and flags, then save new L2 file or overwrite existing one + +params = opt; + +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; end -% clear all \ No newline at end of file +sinfo.postproc(ip).type = 'AQH'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags.badaqh = badaqh; +sinfo.postproc(ip).params = params; + +save([sfile.folder slash sfile.name(1:end-7) '_L2.mat'],'SWIFT','sinfo') + +%% Save SIG Structure + Plot %%%%%%%% + +if opt.saveAQH + save([sfile.folder slash sfile.name(1:end-7) '_burstavgAQH.mat'],'AQH') +end + +% Plot burst Averaged SWIFT Signature Data +catAQH(AQH,'plot'); +set(gcf,'Name',sfile.name(1:end-7)) +if opt.saveplots + figname = [missiondir slash get(gcf,'Name')]; + print([figname '_AQH'],'-dpng') + close gcf +end + +cd(missiondir) + +end \ No newline at end of file diff --git a/CT/reprocess_ACS.m b/CT/reprocess_ACS.m index d48a295..ceb6502 100644 --- a/CT/reprocess_ACS.m +++ b/CT/reprocess_ACS.m @@ -38,7 +38,12 @@ % Read mat file or load raw data if isempty(dir([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat'])) || readraw + try [~,~,Salinity,~,~] = readSWIFTv3_ACS([bfiles(iburst).folder slash bfiles(iburst).name]); + catch + disp(['Cannot read ' bfiles(iburst).name '. Skipping...']) + continue + end else load([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '.mat']), %#ok end diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv index 6cc9684..b7f2e46 100644 --- a/GeneralTools/postprocess_SWIFT.asv +++ b/GeneralTools/postprocess_SWIFT.asv @@ -5,7 +5,7 @@ % K. Zeiden 07/2024 -expdir = 'S:\SEAFAC\June2024\NorthMooring'; +expdir = 'S:\SEAFAC\June2024'; cd(expdir) if ispc @@ -46,7 +46,7 @@ for im = 1:length(missions) end end - %% Reprocess IMU + % Reprocess IMU if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) disp('Reprocessing IMU data...') calctype = 'IMUandGPS'; @@ -57,50 +57,79 @@ for im = 1:length(missions) disp('No IMU data...') end - %% Reprocess SBG + % Reprocess SBG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) disp('Reprocessing SBG data...') saveraw = false; useGPS = false; interpf = false; [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + else + disp('No SBG data...') end - %% Reprocess SIG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - disp('Reprocessing Signature1000 data...') - plotburst = false; - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - end - - %% Reprocess WXT - + % Reprocess WXT if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) disp('Reprocessing Vaisala WXT data...') readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw); + [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); + else + disp('No WXT data...') end - %% Reprocess Y81 - + % Reprocess Y81 if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) - disp('Reprocessing Young Sonic Anemommetr data...') - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir); + disp('Reprocessing Y81 Sonic Anemometer data...') + [SWIFT,sinfo] = reprocess_Y81(missiondir); + else + disp('No Y81 data...') end % Reprocess ACS + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) + disp('Reprocessing ACS CT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); + else + disp('No ACS data...') + end % Reprocess PB2 + % does not exist - % Reprocess IMU + % Reprocess SIG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + else + disp('No SIG data...') + end % Reprocess AQD + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) + disp('Reprocessing Aquadopp (AQD) data...') + readraw = true; + [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); + else + disp('No AQD data...') + end % Reprocess AQH + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) + disp('Reprocessing Aquadopp (AQH) data...') + readraw = true; + plotburst = true; + [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); + else + disp('No AQH data...') + end + + % Reprocess 536 + % Heitronics CT15 ? + % Does not exist - % clear SWIFT sinfo end diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index fa60f16..dab7833 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,8 +19,7 @@ %% Loop through missions and reprocess -%for im = 1:length(missions) -im = 1; +for im = 5:length(missions) disp(['Post-processing ' missions(im).name]) @@ -39,7 +38,7 @@ sinfo.ID = SWIFT(1).ID; sinfo.CTdepth = SWIFT(1).CTdepth; sinfo.metheight = SWIFT(1).metheight; - SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); + SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); disp('Saving new L1 product...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') else @@ -47,7 +46,7 @@ end end - %% Reprocess IMU + % Reprocess IMU if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) disp('Reprocessing IMU data...') calctype = 'IMUandGPS'; @@ -58,64 +57,80 @@ disp('No IMU data...') end - %% Reprocess SBG + % Reprocess SBG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) disp('Reprocessing SBG data...') saveraw = false; useGPS = false; interpf = false; [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + else + disp('No SBG data...') end - %% Reprocess SIG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - disp('Reprocessing Signature1000 data...') - plotburst = false; - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - end - - %% Reprocess WXT - + % Reprocess WXT if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) disp('Reprocessing Vaisala WXT data...') readraw = false; [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); + else + disp('No WXT data...') end - %% Reprocess Y81 - + % Reprocess Y81 if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) disp('Reprocessing Y81 Sonic Anemometer data...') [SWIFT,sinfo] = reprocess_Y81(missiondir); + else + disp('No Y81 data...') end - %% Reprocess ACS + % Reprocess ACS if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) disp('Reprocessing ACS CT data...') readraw = false; [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); + else + disp('No ACS data...') end - %% Reprocess PB2 - + % Reprocess PB2 % does not exist - %% - - %% Reprocess AQD + % Reprocess SIG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + else + disp('No SIG data...') + end + % Reprocess AQD if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) - disp('Reprocessing Aquadopp data...') - readraw = false; + disp('Reprocessing Aquadopp (AQD) data...') + readraw = true; [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); + else + disp('No AQD data...') end - %% Reprocess AQH + % Reprocess AQH + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) + disp('Reprocessing Aquadopp (AQH) data...') + readraw = true; + plotburst = true; + [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); + else + disp('No AQH data...') + end - %% Reprocess 536 + % Reprocess 536 + % Heitronics CT15 ? + % Does not exist -%end +end diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index 1821d57..4d5190f 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -147,14 +147,11 @@ return end burstreplaced = false(length(SWIFT),1); +badsig = false(1,length(SWIFT)); SIG = struct; isig = 1; -%% Flag bad Signature data - -badsig = false(1,length(SWIFT)); - %% List of burst files bfiles = dir([missiondir slash 'SIG' slash 'Raw' slash '*' slash '*' ftype]); diff --git a/Winds/inertialdissipation.m b/Winds/inertialdissipation.m index eddbcf7..7ecde11 100644 --- a/Winds/inertialdissipation.m +++ b/Winds/inertialdissipation.m @@ -40,10 +40,7 @@ v( bad ) = []; w( bad ) = []; temp( bad ) = []; -%u = u( ~bad ); -%v = v( ~bad ); -%w = w( ~bad ); -%temp = temp( ~bad ); + %% means meanu = mean(u); @@ -54,18 +51,20 @@ %% begin processing, if data sufficient pts = length(u); % record length in data points -if pts >= 2*wsecs & fs>1 & sum(bad) < 0.1*pts, % minimum length and quality for processing +if pts >= 2*wsecs && fs>1 && sum(bad) < 0.1*pts % minimum length and quality for processing %% break into windows (use 75 percent overlap) -if rem(windowlength,2)~=0, windowlength = windowlength-1; else end % make w an even number +if rem(windowlength,2)~=0 + windowlength = windowlength-1; +end % make w an even number windows = floor( 4*(pts/windowlength - 1)+1 ); % number of windows, the 4 comes from a 75% overlap dof = 2*windows*merge; % degrees of freedom % loop to create a matrix of time series, where COLUMN = WINDOW uwindow = zeros(windowlength,windows); vwindow = zeros(windowlength,windows); wwindow = zeros(windowlength,windows); -for q=1:windows, +for q=1:windows uwindow(:,q) = u( (q-1)*(.25*windowlength)+1 : (q-1)*(.25*windowlength)+windowlength ); vwindow(:,q) = v( (q-1)*(.25*windowlength)+1 : (q-1)*(.25*windowlength)+windowlength ); wwindow(:,q) = w( (q-1)*(.25*windowlength)+1 : (q-1)*(.25*windowlength)+windowlength ); @@ -184,17 +183,17 @@ %% ensemble average windows together % take the average of all windows at each freq-band -UU = mean( UUwindowmerged.' ) ; -VV = mean( VVwindowmerged.' ) ; -WW = mean( WWwindowmerged.' ) ; -UV = mean( UVwindowmerged.' ) ; -UW = mean( UWwindowmerged.' ) ; -VW = mean( VWwindowmerged.' ) ; +UU = mean( UUwindowmerged',1,'omitnan') ; +VV = mean( VVwindowmerged',1,'omitnan') ; +WW = mean( WWwindowmerged',1,'omitnan') ; +UV = mean( UVwindowmerged',1,'omitnan') ; +UW = mean( UWwindowmerged',1,'omitnan') ; +VW = mean( VWwindowmerged',1,'omitnan') ; % find the windows with stable mean in the streamwise direction, use only those for ustar and epsilon ensembles good = abs(mean(uwindow)) > std(uwindow); %disp('stable windows'), sum(good) % debug -if sum(good) >= 2, +if sum(good) >= 2 epsilon = mean(epsilonwindow(good)); ustar = mean(ustarwindow(good)); quality = mean(qualitywindow(good)); diff --git a/Winds/reprocess_Y81.m b/Winds/reprocess_Y81.m index 7377bda..a4cbe23 100644 --- a/Winds/reprocess_Y81.m +++ b/Winds/reprocess_Y81.m @@ -49,19 +49,21 @@ temp = NaN(1000,1); end windspd = mean((uvw(:,1).^2 + uvw(:,2).^2 + uvw(:,3).^2).^.5); + u = uvw(:,1); + v = uvw(:,2); + w = uvw(:,3); + % Recalculate friction velocity z = sinfo.metheight; fs = 10; - [ustar,~,~,~,~,~,~,~,~,~] = inertialdissipation(uvw(:,1),uvw(:,2),uvw(:,3),temp,z,fs); + [ustar,~,~,~,~,~,~,~,~,~] = inertialdissipation(u,v,w,temp,z,fs); - % Find matching time time = datenum(bfiles(iburst).name(13:21)) + str2double(bfiles(iburst).name(23:24))./24 ... + str2double(bfiles(iburst).name(26:27))./(24*6); [tdiff,tindex] = min(abs([SWIFT.time]-time)); - % Replace wind speed, NaN out wind direction and replace ustar if ~isempty(tdiff) && tdiff < 12/(24*60) SWIFT(tindex).windspd = windspd; From 19b79614d1ea32af98f056fa6324d3e7425d7f90 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:26:20 -0700 Subject: [PATCH 05/34] updates --- Aquadopp/catAQH.m | 3 +++ Aquadopp/processAQHburst.m | 8 ++++---- GeneralTools/catSWIFT.m | 14 ++++++++++---- GeneralTools/postprocess_SWIFT.m | 31 ++++++++++++++++++++----------- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/Aquadopp/catAQH.m b/Aquadopp/catAQH.m index 1f18371..2d57f49 100644 --- a/Aquadopp/catAQH.m +++ b/Aquadopp/catAQH.m @@ -28,6 +28,7 @@ aqh.hrw = aqh.hrcorr; aqh.hrwvar = aqh.hrcorr; aqh.eps = aqh.hrcorr; + aqh.epseof = aqh.hrcorr; aqh.epsnf = aqh.hrcorr; aqh.mspe = aqh.hrcorr; aqh.slope = aqh.hrcorr; @@ -54,6 +55,7 @@ aqh.hrw(1:nz,it) = AQH(it).HRprofile.w; aqh.hrwvar(1:nz,it) = AQH(it).HRprofile.wvar; aqh.eps(1:nz,it) = AQH(it).HRprofile.eps; + aqh.epseof(1:nz,it) = AQH(it).HRprofile.QC.epsEOF; aqh.epsnf(1:nz,it) = AQH(it).HRprofile.QC.epsNF; aqh.mspe(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.mspe; aqh.slope(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.slope; @@ -80,6 +82,7 @@ aqh.hrw(:,badburst) = []; aqh.hrwvar(:,badburst) = []; aqh.eps(:,badburst) = []; + aqh.epseof(:,badburst) = []; aqh.epsnf(:,badburst) = []; aqh.mspe(:,badburst) = []; aqh.slope(:,badburst) = []; diff --git a/Aquadopp/processAQHburst.m b/Aquadopp/processAQHburst.m index 6608d7d..4f60a87 100644 --- a/Aquadopp/processAQHburst.m +++ b/Aquadopp/processAQHburst.m @@ -53,13 +53,13 @@ end %%%%%%%%% 3) Estimate dissipation rate from velocity structure functions -ibad = ispike | repmat(badping,nbin,1); rmin = dz; rmax = 4*dz; nzfit = 1; -w(ibad) = NaN; -wpeof(ibad) = NaN; -wphp(ibad) = NaN; +% ibad = ispike | repmat(badping,nbin,1); +% w(ibad) = NaN; +% wpeof(ibad) = NaN; +% wphp(ibad) = NaN; warning('off','all') % No filter, no analytic wave fit (D ~ r^{-2/3}) diff --git a/GeneralTools/catSWIFT.m b/GeneralTools/catSWIFT.m index 887499f..27d9c13 100644 --- a/GeneralTools/catSWIFT.m +++ b/GeneralTools/catSWIFT.m @@ -160,7 +160,7 @@ swift.subu = swift.relu + swift.driftu; swift.subv = swift.relv + swift.driftv; -%Waves +% Wave Spectra for it = 1:nt wavepower = SWIFT(it).wavespectra.energy; wavefreq = SWIFT(it).wavespectra.freq; @@ -171,8 +171,8 @@ swift.wavepower(:,it) = 0; swift.wavefreq(:,it) = NaN; else - swift.wavepower(1:length(wavepower),it) = wavepower; - swift.wavefreq(1:length(wavepower),it) = wavefreq; + swift.wavepower(1:length(wavepower),it) = wavepower; + swift.wavefreq(1:length(wavepower),it) = wavefreq; end end swift.wavepower(swift.wavepower<0) = 0; @@ -181,15 +181,18 @@ % Interpolate to median frequency for it = 1:nt ireal = ~isnan(swift.wavepower(:,it)) & swift.wavepower(:,it)~=0; - if ireal > 3 + if sum(ireal)>3 wavepower(:,it) = interp1(swift.wavefreq(ireal,it),swift.wavepower(ireal,it),wavefreq); end end swift.wavepower = wavepower; swift.wavefreq = wavefreq; + +% Wave Bulk Variables swift.wavesigH = [SWIFT.sigwaveheight]; swift.wavepeakT = [SWIFT.peakwaveperiod]; swift.wavepeakdir = [SWIFT.peakwavedirT]; + % Calculate new Stokes drift (Us = omega*k*(Hs/4)^2) om = 2*pi./swift.wavepeakT; k = om.^2./9.81; @@ -269,6 +272,9 @@ swift.surftke(:,it) = SWIFT(it).uplooking.tkedissipationrate; end swift.surftke(1:4,:) = NaN;% Deepest three bins are bad +else + swift.surfz = (0.1+0.2+0.04*(0:127)); + swift.surftke = NaN(128,nt); end % Echograms diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index dab7833..124bdcc 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -14,12 +14,21 @@ slash = '/'; end -missions = dir([expdir slash 'SWIFT*']); +missions = dir([expdir slash 'SWIFT1*']); missions = missions([missions.isdir]); +rpIMU = false; % Waves +rpSBG = false; % Waves +rpWXT = false; % MET +rpY81 = false; % MET +rpACS = false; % CT +rpSIG = false; % TKE +rpAQH = true; % TKE +rpAQD = false; % TKE + %% Loop through missions and reprocess -for im = 5:length(missions) +for im = 1:length(missions) disp(['Post-processing ' missions(im).name]) @@ -47,7 +56,7 @@ end % Reprocess IMU - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) + if rpIMU && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) disp('Reprocessing IMU data...') calctype = 'IMUandGPS'; filtertype = 'RC'; @@ -58,7 +67,7 @@ end % Reprocess SBG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) + if rpSBG && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) disp('Reprocessing SBG data...') saveraw = false; useGPS = false; @@ -69,7 +78,7 @@ end % Reprocess WXT - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) + if rpSBG && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) disp('Reprocessing Vaisala WXT data...') readraw = false; [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); @@ -78,7 +87,7 @@ end % Reprocess Y81 - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) + if rpY81 && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) disp('Reprocessing Y81 Sonic Anemometer data...') [SWIFT,sinfo] = reprocess_Y81(missiondir); else @@ -86,7 +95,7 @@ end % Reprocess ACS - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) + if rpACS && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) disp('Reprocessing ACS CT data...') readraw = false; [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); @@ -98,7 +107,7 @@ % does not exist % Reprocess SIG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + if rpSIG && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) disp('Reprocessing Signature1000 data...') plotburst = false; readraw = false; @@ -108,7 +117,7 @@ end % Reprocess AQD - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) + if rpAQD && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) disp('Reprocessing Aquadopp (AQD) data...') readraw = true; [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); @@ -117,10 +126,10 @@ end % Reprocess AQH - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) + if rpAQH && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) disp('Reprocessing Aquadopp (AQH) data...') readraw = true; - plotburst = true; + plotburst = false; [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); else disp('No AQH data...') From 24be0d5c52aa14ac18fb5ba168ec82a48a828373 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:07:31 -0700 Subject: [PATCH 06/34] Update catAQH --- Aquadopp/catAQH.m | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Aquadopp/catAQH.m b/Aquadopp/catAQH.m index 2d57f49..86621d4 100644 --- a/Aquadopp/catAQH.m +++ b/Aquadopp/catAQH.m @@ -27,9 +27,13 @@ aqh.hramp = aqh.hrcorr; aqh.hrw = aqh.hrcorr; aqh.hrwvar = aqh.hrcorr; - aqh.eps = aqh.hrcorr; + aqh.epshp = aqh.hrcorr; aqh.epseof = aqh.hrcorr; aqh.epsnf = aqh.hrcorr; + aqh.epswv = aqh.hrcorr; + aqh.A = aqh.hrcorr; + aqh.B = aqh.hrcorr; + aqh.N = aqh.hrcorr; aqh.mspe = aqh.hrcorr; aqh.slope = aqh.hrcorr; aqh.pspike = aqh.hrcorr; @@ -54,9 +58,13 @@ aqh.pbadping(it) = AQH(it).HRprofile.QC.pbadping; aqh.hrw(1:nz,it) = AQH(it).HRprofile.w; aqh.hrwvar(1:nz,it) = AQH(it).HRprofile.wvar; - aqh.eps(1:nz,it) = AQH(it).HRprofile.eps; + aqh.epshp(1:nz,it) = AQH(it).HRprofile.QC.epsHP; aqh.epseof(1:nz,it) = AQH(it).HRprofile.QC.epsEOF; aqh.epsnf(1:nz,it) = AQH(it).HRprofile.QC.epsNF; + aqh.epswv(1:nz,it) = AQH(it).HRprofile.QC.epsWV; + aqh.A(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.A; + aqh.B(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.B; + aqh.N(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.N; aqh.mspe(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.mspe; aqh.slope(1:nz,it) = AQH(it).HRprofile.QC.qualEOF.slope; @@ -81,9 +89,13 @@ aqh.hramp(:,badburst) = []; aqh.hrw(:,badburst) = []; aqh.hrwvar(:,badburst) = []; - aqh.eps(:,badburst) = []; + aqh.epshp(:,badburst) = []; aqh.epseof(:,badburst) = []; aqh.epsnf(:,badburst) = []; + aqh.epswv(:,badburst) = []; + aqh.A(:,badburst) = []; + aqh.B(:,badburst) = []; + aqh.N(:,badburst) = []; aqh.mspe(:,badburst) = []; aqh.slope(:,badburst) = []; aqh.pspike(:,badburst) = []; @@ -107,7 +119,7 @@ figure('color','w') MP = get(0,'monitorposition'); - set(gcf,'outerposition',MP(1,:).*[1 1 0.5 1]); + set(gcf,'outerposition',MP(1,:).*[1 1 1 1]); clear b % HR @@ -147,7 +159,7 @@ xlim([min(aqh.time) max(aqh.time)]) datetick('x','KeepLimits') h(8) = subplot(4,2,4);% Dissipation Rate - pcolor(aqh.time,-aqh.hrz,log10(aqh.eps));shading flat + pcolor(aqh.time,-aqh.hrz,log10(aqh.epseof));shading flat caxis([-6 -3]);colormap(gca,'jet') ylabel('Depth (m)');title('Dissipation Rate') c = colorbar;c.Label.String = 'log_{10}(m^3s^{-2})'; From 04680883370e33406a253476f5507371d1f7d448 Mon Sep 17 00:00:00 2001 From: Michael James Date: Tue, 23 Jul 2024 23:19:52 +0000 Subject: [PATCH 07/34] Finished rev1 product for findoutliersSWIFT.m ready for testing, next to add plot feature --- GeneralTools/findoutliersSWIFT.m | 88 ++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 5 deletions(-) diff --git a/GeneralTools/findoutliersSWIFT.m b/GeneralTools/findoutliersSWIFT.m index 8751130..93a2b55 100644 --- a/GeneralTools/findoutliersSWIFT.m +++ b/GeneralTools/findoutliersSWIFT.m @@ -1,11 +1,89 @@ -function [SWIFT,outliersSWIFT] = findoutliersSWIFT(flist`) -%findoutliersSWIFT - finds outliers out of % range and plots hist of all -%SWIFT fields +function [SWIFT,outliersSWIFT,booloutSWIFT] = findoutliersSWIFT(SWIFT, method, pct,varargin) +% findoutliersSWIFT - finds outliers out of % range and plots hist of all +% SWIFT fields % Find outliers in SWIFT L1 DATA % Michael James % 7_12_2024 +% findoutliersSWIFT(SWIFT, method, pctdatathresh,varargin) +% method == +% defined as "percentile" or "movmedian" (moving percentile) +% pct == +% defined as percent used in percentile method, write as "~" or leave blank for +% movmedian +% varargin defined as string / value dictionary pairs. List below: +% 'plot_results' ; true/false bool +% 'window' ; float (this is a window for "movmedian") +% 7_23_2024 +% Scoped down to per SWIFT variable so can be run in created loop +% determined by user +% Added boolean output for ease of indexing -outputArg1 = inputArg1; -outputArg2 = inputArg2; +% Preset binary flags +vararginlist = ["plot_results" ; "window"]; +plot_results = true; %default create figures +window = 40; % Window size of 100 values as default + +% Check if there are any optional arguments +if ~isempty(varargin) + if mod(length(varargin),2) == 0 %check for pairs + for i= 1:2:length(varargin) + if mean(contains(string(vararginlist),char(varargin{i}))) ~=0 + eval([char(vararginlist(contains(string(vararginlist),varargin{i}))) ' = varargin{i+1};']); + % taking the name and setting the variable to be the user + % input + else + error('Check naming of the variable arguements') + end + end; + else + error('incorrect variable arguements; must be in pairs') + end; +elseif isempty(varargin) + % Do nothing +else + error('Input error'); help findoutliersSWIFT; +end + + +if mean(contains(["percentile" ; "movmedian"],char(method))) ==0 + error('use "percentile" or "movmedian" to define the method'); +end + + +if string(method) == "percentile" + if 0 Date: Tue, 23 Jul 2024 21:26:20 -0700 Subject: [PATCH 08/34] updates --- GeneralTools/concatSWIFT_offloadedSDcard.m | 2 +- GeneralTools/postprocess_SWIFT.asv | 148 ++++++++++++-------- GeneralTools/postprocess_SWIFT.m | 152 +++++++++++++-------- Signature/reprocess_SIG.m | 43 +++--- 4 files changed, 206 insertions(+), 139 deletions(-) diff --git a/GeneralTools/concatSWIFT_offloadedSDcard.m b/GeneralTools/concatSWIFT_offloadedSDcard.m index c956062..7a2f0b4 100644 --- a/GeneralTools/concatSWIFT_offloadedSDcard.m +++ b/GeneralTools/concatSWIFT_offloadedSDcard.m @@ -115,7 +115,7 @@ if isfield(SWIFT(1),'ID') load(['SWIFT' SWIFT(1).ID '_telemetry.mat']) - save(['SWIFT' SWIFT(1).ID '_' datestr(min([SWIFT.time]),'ddmmmyyyy') '-' datestr(max([SWIFT.time]),'ddmmmyyyy') '_L1.mat' ], 'SWIFT') + save(['SWIFT' SWIFT(1).ID '_' datestr(min([SWIFT.time]),'ddmmmyyyy') '_L1.mat' ], 'SWIFT') eval(['!rm *telemetry.mat']) else end diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv index b7f2e46..827b99e 100644 --- a/GeneralTools/postprocess_SWIFT.asv +++ b/GeneralTools/postprocess_SWIFT.asv @@ -2,6 +2,8 @@ % Master post-processing function, that calls sub-functions to reprocess % raw SWIFT data. +% L1 product must have been created prior to running this script, by +% calling % K. Zeiden 07/2024 @@ -17,9 +19,18 @@ end missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); +rpIMU = false; % Waves +rpSBG = false; % Waves +rpWXT = false; % MET +rpY81 = false; % MET +rpACS = false; % CT +rpSIG = false; % TKE +rpAQH = false; % TKE +rpAQD = false; % TKE + %% Loop through missions and reprocess -for im = 1:length(missions) +for im = 1%:length(missions) disp(['Post-processing ' missions(im).name]) @@ -28,7 +39,7 @@ for im = 1:length(missions) % Locate L1 product, skip if does not exist. % Else create 'sinfo' and modify L1 product. - l1file = dir([missiondir slash '*L1*']); + l1file = dir([missiondir slash '*L1.mat']); if isempty(l1file) disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) else @@ -38,98 +49,121 @@ for im = 1:length(missions) sinfo.ID = SWIFT(1).ID; sinfo.CTdepth = SWIFT(1).CTdepth; sinfo.metheight = SWIFT(1).metheight; + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + else + sinfo.type = 'V3'; + end SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); disp('Saving new L1 product...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') - else - load([l1file.folder slash l1file.name],'sinfo') end end + SWIFTL1 = SWIFT; % Reprocess IMU - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) - disp('Reprocessing IMU data...') - calctype = 'IMUandGPS'; - filtertype = 'RC'; - saveraw = false; - [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); - else - disp('No IMU data...') + if rpIMU + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) + disp('Reprocessing IMU data...') + calctype = 'IMUandGPS'; + filtertype = 'RC'; + saveraw = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); + else + disp('No IMU data...') + end end % Reprocess SBG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) - disp('Reprocessing SBG data...') - saveraw = false; - useGPS = false; - interpf = false; - [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); - else - disp('No SBG data...') + if rpSBG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) + disp('Reprocessing SBG data...') + saveraw = false; + useGPS = false; + interpf = false; + [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + else + disp('No SBG data...') + end end % Reprocess WXT - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) - disp('Reprocessing Vaisala WXT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); - else - disp('No WXT data...') + if rpWXT + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) + disp('Reprocessing Vaisala WXT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); + else + disp('No WXT data...') + end end % Reprocess Y81 - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) - disp('Reprocessing Y81 Sonic Anemometer data...') - [SWIFT,sinfo] = reprocess_Y81(missiondir); - else - disp('No Y81 data...') + if rpY81 + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) + disp('Reprocessing Y81 Sonic Anemometer data...') + [SWIFT,sinfo] = reprocess_Y81(missiondir); + else + disp('No Y81 data...') + end end % Reprocess ACS - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) - disp('Reprocessing ACS CT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); - else - disp('No ACS data...') + if rpACS + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) + disp('Reprocessing ACS CT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); + else + disp('No ACS data...') + end end % Reprocess PB2 % does not exist % Reprocess SIG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - disp('Reprocessing Signature1000 data...') - plotburst = false; - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - else - disp('No SIG data...') + if rpSIG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + else + disp('No SIG data...') + end end % Reprocess AQD - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) - disp('Reprocessing Aquadopp (AQD) data...') - readraw = true; - [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); - else - disp('No AQD data...') + if rpAQD + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) + disp('Reprocessing Aquadopp (AQD) data...') + readraw = true; + [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); + else + disp('No AQD data...') + end end % Reprocess AQH - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) - disp('Reprocessing Aquadopp (AQH) data...') - readraw = true; - plotburst = true; - [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); - else - disp('No AQH data...') + if rpAQH + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) + disp('Reprocessing Aquadopp (AQH) data...') + readraw = true; + plotburst = false; + [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); + else + disp('No AQH data...') + end end % Reprocess 536 % Heitronics CT15 ? % Does not exist + l1file = dir([missiondir slash '*L1.mat']); + + end diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index 124bdcc..b4110b0 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -2,6 +2,8 @@ % Master post-processing function, that calls sub-functions to reprocess % raw SWIFT data. +% L1 product must have been created prior to running this script, by +% calling % K. Zeiden 07/2024 @@ -14,7 +16,7 @@ slash = '/'; end -missions = dir([expdir slash 'SWIFT1*']); +missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); rpIMU = false; % Waves @@ -23,12 +25,12 @@ rpY81 = false; % MET rpACS = false; % CT rpSIG = false; % TKE -rpAQH = true; % TKE +rpAQH = false; % TKE rpAQD = false; % TKE %% Loop through missions and reprocess -for im = 1:length(missions) +for im = 1%:length(missions) disp(['Post-processing ' missions(im).name]) @@ -37,7 +39,7 @@ % Locate L1 product, skip if does not exist. % Else create 'sinfo' and modify L1 product. - l1file = dir([missiondir slash '*L1*']); + l1file = dir([missiondir slash '*L1.mat']); if isempty(l1file) disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) else @@ -47,98 +49,128 @@ sinfo.ID = SWIFT(1).ID; sinfo.CTdepth = SWIFT(1).CTdepth; sinfo.metheight = SWIFT(1).metheight; - SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + else + sinfo.type = 'V3'; + end + SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); disp('Saving new L1 product...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') - else - load([l1file.folder slash l1file.name],'sinfo') end end + SWIFTL1 = SWIFT; % Reprocess IMU - if rpIMU && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) - disp('Reprocessing IMU data...') - calctype = 'IMUandGPS'; - filtertype = 'RC'; - saveraw = false; - [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); - else - disp('No IMU data...') + if rpIMU + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) + disp('Reprocessing IMU data...') + calctype = 'IMUandGPS'; + filtertype = 'RC'; + saveraw = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); + else + disp('No IMU data...') + end end % Reprocess SBG - if rpSBG && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) - disp('Reprocessing SBG data...') - saveraw = false; - useGPS = false; - interpf = false; - [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); - else - disp('No SBG data...') + if rpSBG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) + disp('Reprocessing SBG data...') + saveraw = false; + useGPS = false; + interpf = false; + [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + else + disp('No SBG data...') + end end % Reprocess WXT - if rpSBG && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) - disp('Reprocessing Vaisala WXT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); - else - disp('No WXT data...') + if rpWXT + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) + disp('Reprocessing Vaisala WXT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); + else + disp('No WXT data...') + end end % Reprocess Y81 - if rpY81 && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) - disp('Reprocessing Y81 Sonic Anemometer data...') - [SWIFT,sinfo] = reprocess_Y81(missiondir); - else - disp('No Y81 data...') + if rpY81 + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) + disp('Reprocessing Y81 Sonic Anemometer data...') + [SWIFT,sinfo] = reprocess_Y81(missiondir); + else + disp('No Y81 data...') + end end % Reprocess ACS - if rpACS && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) - disp('Reprocessing ACS CT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); - else - disp('No ACS data...') + if rpACS + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) + disp('Reprocessing ACS CT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); + else + disp('No ACS data...') + end end % Reprocess PB2 % does not exist % Reprocess SIG - if rpSIG && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - disp('Reprocessing Signature1000 data...') - plotburst = false; - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - else - disp('No SIG data...') + if rpSIG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + else + disp('No SIG data...') + end end % Reprocess AQD - if rpAQD && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) - disp('Reprocessing Aquadopp (AQD) data...') - readraw = true; - [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); - else - disp('No AQD data...') + if rpAQD + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) + disp('Reprocessing Aquadopp (AQD) data...') + readraw = true; + [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); + else + disp('No AQD data...') + end end % Reprocess AQH - if rpAQH && ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) - disp('Reprocessing Aquadopp (AQH) data...') - readraw = true; - plotburst = false; - [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); - else - disp('No AQH data...') + if rpAQH + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) + disp('Reprocessing Aquadopp (AQH) data...') + readraw = true; + plotburst = false; + [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); + else + disp('No AQH data...') + end end % Reprocess 536 % Heitronics CT15 ? % Does not exist + l2file = dir([missiondir slash '*L2.mat']); + load([l1file.folder slash l1file.name],'SWIFT'); + SWIFTL2 = SWIFT; + + % Plot Results + plotSWIFTV3(SWIFTL1) + set(gcf,'Name',l1file.name(1:end-4)) + plotSWIFTV3(SWIFTL2) + set(gcf,'Name',l1file.name(1:end-4)) + end diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index 4d5190f..b22f0ae 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -200,11 +200,24 @@ % Burst time t0 = min(avg.time); - if abs(btime - t0) > 12/(60*24) + if abs(btime - t0) > 15/(60*24) disp(' WARNING: File name disagrees with recorded time. Using recorded time... ') btime = t0; end + % Time match + [tdiff,tindex] = min(abs([SWIFT.time]-btime)); + if tdiff > 12/(60*24)% must be within 15 min + disp(' NO time index match...') + timematch = false; + elseif tdiff < 12/(60*24) + timematch = true; + burstreplaced(tindex) = true; + elseif isempty(tdiff) + disp(' NO time index match...') + timematch = false; + end + % Altimeter Distance if isfield(avg,'AltimeterDistance') maxz = median(avg.AltimeterDistance); @@ -267,6 +280,9 @@ print(figname,'-dpng') close gcf end + + % Flag bad bursts + badburst = badamp | badcorr | smallfile; %%%%%%% Process HR velocity data ('burst' structure) %%%%%% @@ -329,33 +345,19 @@ SIG(isig).motion.rollvar = var(avg.Roll,'omitnan'); SIG(isig).motion.headvar = var(unwrap(avg.Heading),'omitnan'); % Badburst & flags - SIG(isig).badburst = outofwater | badamp | badcorr | smallfile; - SIG(isig).flag.outofwater = outofwater; + SIG(isig).badburst = badburst; + SIG(isig).timematch = timematch; + SIG(isig).outofwater = outofwater; SIG(isig).flag.badamp = badamp; SIG(isig).flag.badcorr = badcorr; SIG(isig).flag.smallfile = smallfile; - SIG(isig).flag.notimematch = false; % Updated below isig = isig+1; %%%%%%%% Match burst time to existing SWIFT fields and replace data %%%%%%%% - badburst = outofwater | badamp | badcorr | smallfile; - if ~isempty(fieldnames(SWIFT)) && ~isempty(SWIFT) - [tdiff,tindex] = min(abs([SWIFT.time]-btime)); - if tdiff > 12/(60*24)% must be within 10 min - disp(' NO time index match...') - timematch = false; - elseif tdiff < 12/(60*24) - timematch = true; - burstreplaced(tindex) = true; - elseif isempty(tdiff) - disp(' NO time index match...') - timematch = false; - end - if timematch && ~badburst % time match, good burst % HR data SWIFT(tindex).signature.HRprofile = []; @@ -378,7 +380,7 @@ SWIFT(tindex).signature.echo = echogram.echoc; SWIFT(tindex).signature.echoz = echogram.r + opt.xz; end - % Altimeter & Out-of-Water Flag + % Altimeter SWIFT(tindex).signature.altimeter = maxz; % Temperaure SWIFT(tindex).watertemp = profile.temp; @@ -410,8 +412,7 @@ elseif ~timematch && ~badburst % Good burst, no time match disp(' ALERT: Burst good, but no time match...') - SIG(isig-1).flag.notimematch = true; - + % tindex = length(SWIFT)+1; % burstreplaced = [burstreplaced; true]; %#ok % % Copy fields from SWIFT(1); From 1d5c057e065ce7d672f13e24ad992942b143a320 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Wed, 24 Jul 2024 11:55:48 -0700 Subject: [PATCH 09/34] updates --- GeneralTools/postprocess_SWIFT.asv | 47 ++++++++++++++++++---------- GeneralTools/postprocess_SWIFT.m | 50 +++++++++++++++++------------- 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv index 827b99e..7b3a5eb 100644 --- a/GeneralTools/postprocess_SWIFT.asv +++ b/GeneralTools/postprocess_SWIFT.asv @@ -1,24 +1,27 @@ % [SWIFT,sinfo] = postprocess_SWIFT(expdir) % Master post-processing function, that calls sub-functions to reprocess -% raw SWIFT data. -% L1 product must have been created prior to running this script, by -% calling +% each different type of raw SWIFT data. +% L1 product must have been created prior to running this script, +% by running 'concatSWIFT_offloadedSDcard.m' in the mission directory +% Note: concatSWIFT in turn runs 'compileSWIFT_SBDservertelemetry.m', +% which in turn calls the function 'readSWIFT_SBD.m' +% Currently no reprocessing option for Airmar (PB2) +% nor Heitronics CTs (536?) % K. Zeiden 07/2024 -expdir = 'S:\SEAFAC\June2024'; -cd(expdir) - if ispc slash = '\'; else slash = '/'; end -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); +%% User defined experiment directory (to be later converted to function inputs) +expdir = 'S:\SEAFAC\June2024'; + +% Processing toggles rpIMU = false; % Waves rpSBG = false; % Waves rpWXT = false; % MET @@ -29,6 +32,9 @@ rpAQH = false; % TKE rpAQD = false; % TKE %% Loop through missions and reprocess +cd(expdir) +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); for im = 1%:length(missions) @@ -59,7 +65,6 @@ for im = 1%:length(missions) save([l1file.folder slash l1file.name],'SWIFT','sinfo') end end - SWIFTL1 = SWIFT; % Reprocess IMU if rpIMU @@ -119,9 +124,6 @@ for im = 1%:length(missions) end end - % Reprocess PB2 - % does not exist - % Reprocess SIG if rpSIG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) @@ -157,14 +159,25 @@ for im = 1%:length(missions) end end - % Reprocess 536 - % Heitronics CT15 ? - % Does not exist - - l1file = dir([missiondir slash '*L1.mat']); + % Re-load L1 and L2 product and plot each + load([l1file.folder slash l1file.name],'SWIFT'); + SWIFTL1 = SWIFT; + l2file = dir([missiondir slash '*L2.mat']); + load([l2file.folder slash l2file.name],'SWIFT','sinfo'); + SWIFTL2 = SWIFT; + if strcmp(sinfo.type,'V3') + fh1 = plotSWIFTV3(SWIFTL1); + fh2 = plotSWIFTV3(SWIFTL2); + else + fh1 = plotSWIFTV4(SWIFTL1); + fh2 = plotSWIFTV4(SWIFTL2); + end + print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') + print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end + diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index b4110b0..726b8d7 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -1,24 +1,27 @@ % [SWIFT,sinfo] = postprocess_SWIFT(expdir) % Master post-processing function, that calls sub-functions to reprocess -% raw SWIFT data. -% L1 product must have been created prior to running this script, by -% calling +% each different type of raw SWIFT data. +% L1 product must have been created prior to running this script, +% by running 'concatSWIFT_offloadedSDcard.m' in the mission directory +% Note: concatSWIFT in turn runs 'compileSWIFT_SBDservertelemetry.m', +% which in turn calls the function 'readSWIFT_SBD.m' +% Currently no reprocessing option for Airmar (PB2) +% nor Heitronics CTs (536?) % K. Zeiden 07/2024 -expdir = 'S:\SEAFAC\June2024'; -cd(expdir) - if ispc slash = '\'; else slash = '/'; end -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); +%% User defined experiment directory (to be later converted to function inputs) +expdir = 'S:\SEAFAC\June2024'; + +% Processing toggles rpIMU = false; % Waves rpSBG = false; % Waves rpWXT = false; % MET @@ -29,6 +32,9 @@ rpAQD = false; % TKE %% Loop through missions and reprocess +cd(expdir) +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); for im = 1%:length(missions) @@ -59,7 +65,6 @@ save([l1file.folder slash l1file.name],'SWIFT','sinfo') end end - SWIFTL1 = SWIFT; % Reprocess IMU if rpIMU @@ -119,9 +124,6 @@ end end - % Reprocess PB2 - % does not exist - % Reprocess SIG if rpSIG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) @@ -157,21 +159,25 @@ end end - % Reprocess 536 - % Heitronics CT15 ? - % Does not exist - - l2file = dir([missiondir slash '*L2.mat']); + % Re-load L1 and L2 product and plot each for comparison load([l1file.folder slash l1file.name],'SWIFT'); + SWIFTL1 = SWIFT; + l2file = dir([missiondir slash '*L2.mat']); + load([l2file.folder slash l2file.name],'SWIFT','sinfo'); SWIFTL2 = SWIFT; - % Plot Results - plotSWIFTV3(SWIFTL1) - set(gcf,'Name',l1file.name(1:end-4)) - plotSWIFTV3(SWIFTL2) - set(gcf,'Name',l1file.name(1:end-4)) + if strcmp(sinfo.type,'V3') + fh1 = plotSWIFTV3(SWIFTL1); + fh2 = plotSWIFTV3(SWIFTL2); + else + fh1 = plotSWIFTV4(SWIFTL1); + fh2 = plotSWIFTV4(SWIFTL2); + end + print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') + print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end + From dcfe607da318f06647badbe49158d0fc0cff3711 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 25 Jul 2024 07:14:43 -0700 Subject: [PATCH 10/34] updates --- GeneralTools/postprocess_SWIFT.asv | 32 ++++++++++++++++++++---------- GeneralTools/postprocess_SWIFT.m | 30 ++++++++++++++++++---------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv index 7b3a5eb..e88ce5e 100644 --- a/GeneralTools/postprocess_SWIFT.asv +++ b/GeneralTools/postprocess_SWIFT.asv @@ -56,14 +56,22 @@ for im = 1%:length(missions) sinfo.CTdepth = SWIFT(1).CTdepth; sinfo.metheight = SWIFT(1).metheight; if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - else + sinfo.type = 'V4'; + else sinfo.type = 'V3'; end SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); disp('Saving new L1 product...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') end + % One time, will remove later + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + save([l1file.folder slash l1file.name],'sinfo','-append') + else + sinfo.type = 'V3'; + save([l1file.folder slash l1file.name],'sinfo','-append') + end end % Reprocess IMU @@ -159,22 +167,24 @@ for im = 1%:length(missions) end end - % Re-load L1 and L2 product and plot each + % Re-load L1 and L2 product and plot each for comparison load([l1file.folder slash l1file.name],'SWIFT'); SWIFTL1 = SWIFT; l2file = dir([missiondir slash '*L2.mat']); load([l2file.folder slash l2file.name],'SWIFT','sinfo'); SWIFTL2 = SWIFT; - if strcmp(sinfo.type,'V3') - fh1 = plotSWIFTV3(SWIFTL1); - fh2 = plotSWIFTV3(SWIFTL2); - else - fh1 = plotSWIFTV4(SWIFTL1); - fh2 = plotSWIFTV4(SWIFTL2); + if plotL1L2 + if strcmp(sinfo.type,'V3') + fh1 = plotSWIFTV3(SWIFTL1); + fh2 = plotSWIFTV3(SWIFTL2); + else + fh1 = plotSWIFTV4(SWIFTL1); + fh2 = plotSWIFTV4(SWIFTL2); + end + print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') + print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end - print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') - print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index 726b8d7..e88ce5e 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -56,14 +56,22 @@ sinfo.CTdepth = SWIFT(1).CTdepth; sinfo.metheight = SWIFT(1).metheight; if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - else + sinfo.type = 'V4'; + else sinfo.type = 'V3'; end SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); disp('Saving new L1 product...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') end + % One time, will remove later + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + save([l1file.folder slash l1file.name],'sinfo','-append') + else + sinfo.type = 'V3'; + save([l1file.folder slash l1file.name],'sinfo','-append') + end end % Reprocess IMU @@ -166,15 +174,17 @@ load([l2file.folder slash l2file.name],'SWIFT','sinfo'); SWIFTL2 = SWIFT; - if strcmp(sinfo.type,'V3') - fh1 = plotSWIFTV3(SWIFTL1); - fh2 = plotSWIFTV3(SWIFTL2); - else - fh1 = plotSWIFTV4(SWIFTL1); - fh2 = plotSWIFTV4(SWIFTL2); + if plotL1L2 + if strcmp(sinfo.type,'V3') + fh1 = plotSWIFTV3(SWIFTL1); + fh2 = plotSWIFTV3(SWIFTL2); + else + fh1 = plotSWIFTV4(SWIFTL1); + fh2 = plotSWIFTV4(SWIFTL2); + end + print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') + print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end - print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') - print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end From 41655365d16fb775591bbc13df4b1e95640e0d40 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 25 Jul 2024 07:15:00 -0700 Subject: [PATCH 11/34] Delete postprocess_SWIFT.asv --- GeneralTools/postprocess_SWIFT.asv | 193 ----------------------------- 1 file changed, 193 deletions(-) delete mode 100644 GeneralTools/postprocess_SWIFT.asv diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv deleted file mode 100644 index e88ce5e..0000000 --- a/GeneralTools/postprocess_SWIFT.asv +++ /dev/null @@ -1,193 +0,0 @@ -% [SWIFT,sinfo] = postprocess_SWIFT(expdir) - -% Master post-processing function, that calls sub-functions to reprocess -% each different type of raw SWIFT data. -% L1 product must have been created prior to running this script, -% by running 'concatSWIFT_offloadedSDcard.m' in the mission directory -% Note: concatSWIFT in turn runs 'compileSWIFT_SBDservertelemetry.m', -% which in turn calls the function 'readSWIFT_SBD.m' -% Currently no reprocessing option for Airmar (PB2) -% nor Heitronics CTs (536?) - -% K. Zeiden 07/2024 - -if ispc - slash = '\'; -else - slash = '/'; -end - -%% User defined experiment directory (to be later converted to function inputs) - -expdir = 'S:\SEAFAC\June2024'; - -% Processing toggles -rpIMU = false; % Waves -rpSBG = false; % Waves -rpWXT = false; % MET -rpY81 = false; % MET -rpACS = false; % CT -rpSIG = false; % TKE -rpAQH = false; % TKE -rpAQD = false; % TKE - -%% Loop through missions and reprocess -cd(expdir) -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); - -for im = 1%:length(missions) - - disp(['Post-processing ' missions(im).name]) - - missiondir = [missions(im).folder slash missions(im).name]; - cd(missiondir) - - % Locate L1 product, skip if does not exist. - % Else create 'sinfo' and modify L1 product. - l1file = dir([missiondir slash '*L1.mat']); - if isempty(l1file) - disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) - else - load([l1file.folder slash l1file.name],'SWIFT'); - if isfield(SWIFT,'ID') - disp('Create information structure ''sinfo''') - sinfo.ID = SWIFT(1).ID; - sinfo.CTdepth = SWIFT(1).CTdepth; - sinfo.metheight = SWIFT(1).metheight; - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - else - sinfo.type = 'V3'; - end - SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); - disp('Saving new L1 product...') - save([l1file.folder slash l1file.name],'SWIFT','sinfo') - end - % One time, will remove later - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - save([l1file.folder slash l1file.name],'sinfo','-append') - else - sinfo.type = 'V3'; - save([l1file.folder slash l1file.name],'sinfo','-append') - end - end - - % Reprocess IMU - if rpIMU - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) - disp('Reprocessing IMU data...') - calctype = 'IMUandGPS'; - filtertype = 'RC'; - saveraw = false; - [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); - else - disp('No IMU data...') - end - end - - % Reprocess SBG - if rpSBG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) - disp('Reprocessing SBG data...') - saveraw = false; - useGPS = false; - interpf = false; - [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); - else - disp('No SBG data...') - end - end - - % Reprocess WXT - if rpWXT - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) - disp('Reprocessing Vaisala WXT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); - else - disp('No WXT data...') - end - end - - % Reprocess Y81 - if rpY81 - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) - disp('Reprocessing Y81 Sonic Anemometer data...') - [SWIFT,sinfo] = reprocess_Y81(missiondir); - else - disp('No Y81 data...') - end - end - - % Reprocess ACS - if rpACS - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) - disp('Reprocessing ACS CT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); - else - disp('No ACS data...') - end - end - - % Reprocess SIG - if rpSIG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - disp('Reprocessing Signature1000 data...') - plotburst = false; - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - else - disp('No SIG data...') - end - end - - % Reprocess AQD - if rpAQD - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) - disp('Reprocessing Aquadopp (AQD) data...') - readraw = true; - [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); - else - disp('No AQD data...') - end - end - - % Reprocess AQH - if rpAQH - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) - disp('Reprocessing Aquadopp (AQH) data...') - readraw = true; - plotburst = false; - [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); - else - disp('No AQH data...') - end - end - - % Re-load L1 and L2 product and plot each for comparison - load([l1file.folder slash l1file.name],'SWIFT'); - SWIFTL1 = SWIFT; - l2file = dir([missiondir slash '*L2.mat']); - load([l2file.folder slash l2file.name],'SWIFT','sinfo'); - SWIFTL2 = SWIFT; - - if plotL1L2 - if strcmp(sinfo.type,'V3') - fh1 = plotSWIFTV3(SWIFTL1); - fh2 = plotSWIFTV3(SWIFTL2); - else - fh1 = plotSWIFTV4(SWIFTL1); - fh2 = plotSWIFTV4(SWIFTL2); - end - print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') - print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') - end - -end - - - - From 978c3f6e642f4470603ca373ad2f73513d1087aa Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:03:28 -0700 Subject: [PATCH 12/34] updates --- GeneralTools/postprocess_SWIFT.m | 39 +++++++++++--------- IMU/reprocess_IMU.m | 62 ++++++++++++++++++++++---------- Signature/reprocess_SIG.m | 12 ++++++- Winds/reprocess_Y81.m | 2 +- 4 files changed, 79 insertions(+), 36 deletions(-) diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index e88ce5e..997f05d 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,7 +19,7 @@ %% User defined experiment directory (to be later converted to function inputs) -expdir = 'S:\SEAFAC\June2024'; +expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'NorthMooring']; % Processing toggles rpIMU = false; % Waves @@ -27,10 +27,12 @@ rpWXT = false; % MET rpY81 = false; % MET rpACS = false; % CT -rpSIG = false; % TKE +rpSIG = true; % TKE rpAQH = false; % TKE rpAQD = false; % TKE +plotL1L2 = true; + %% Loop through missions and reprocess cd(expdir) missions = dir([expdir slash 'SWIFT*']); @@ -48,6 +50,7 @@ l1file = dir([missiondir slash '*L1.mat']); if isempty(l1file) disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + continue else load([l1file.folder slash l1file.name],'SWIFT'); if isfield(SWIFT,'ID') @@ -63,16 +66,18 @@ SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); disp('Saving new L1 product...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') + else + load([l1file.folder slash l1file.name],'sinfo'); end - % One time, will remove later - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - save([l1file.folder slash l1file.name],'sinfo','-append') - else - sinfo.type = 'V3'; - save([l1file.folder slash l1file.name],'sinfo','-append') - end end + + % Save SWIFT type (one time, remove later) + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + else + sinfo.type = 'V3'; + end + save([l1file.folder slash l1file.name],'SWIFT','sinfo') % Reprocess IMU if rpIMU @@ -81,7 +86,8 @@ calctype = 'IMUandGPS'; filtertype = 'RC'; saveraw = false; - [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); + interpf = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw,interpf); else disp('No IMU data...') end @@ -92,7 +98,7 @@ if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) disp('Reprocessing SBG data...') saveraw = false; - useGPS = false; + useGPS = true; interpf = false; [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); else @@ -168,10 +174,10 @@ end % Re-load L1 and L2 product and plot each for comparison - load([l1file.folder slash l1file.name],'SWIFT'); + load([l1file.folder slash l1file.name],'SWIFT','sinfo'); SWIFTL1 = SWIFT; l2file = dir([missiondir slash '*L2.mat']); - load([l2file.folder slash l2file.name],'SWIFT','sinfo'); + load([l2file.folder slash l2file.name],'SWIFT'); SWIFTL2 = SWIFT; if plotL1L2 @@ -182,6 +188,8 @@ fh1 = plotSWIFTV4(SWIFTL1); fh2 = plotSWIFTV4(SWIFTL2); end + set(fh1,'Name',l1file.name(1:end-4)) + set(fh2,'Name',l2file.name(1:end-4)) print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end @@ -189,5 +197,4 @@ end - - +% end diff --git a/IMU/reprocess_IMU.m b/IMU/reprocess_IMU.m index ab5a0a6..8f373dd 100644 --- a/IMU/reprocess_IMU.m +++ b/IMU/reprocess_IMU.m @@ -1,4 +1,4 @@ -function [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw) +function [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw,interpf) % reprocess SWIFT v3 wave results using a surface reconstruction % and acounting for listing or capsizing during icing conditions @@ -150,18 +150,22 @@ [Hs,Tp,Dp,E,f,a1,b1,a2,b2,check] = XYZwaves(x(igood),y(igood),z(igood),fs_ahrs); % Interpolate back to the original frequency bands - E = interp1(f,E,f_original); - a1 = interp1(f,a1,f_original); - b1 = interp1(f,b1,f_original); - a2 = interp1(f,a2,f_original); - b2 = interp1(f,b2,f_original); - check = interp1(f,check,f_original); + if interpf + E = interp1(f,E,f_original); + a1 = interp1(f,a1,f_original); + b1 = interp1(f,b1,f_original); + a2 = interp1(f,a2,f_original); + b2 = interp1(f,b2,f_original); + check = interp1(f,check,f_original); + f = f_original; + end % Replace scalar values SWIFT(tindex).sigwaveheight = Hs; SWIFT(tindex).peakwaveperiod = Tp; SWIFT(tindex).peakwavedirT = Dp; SWIFT(tindex).wavespectra.energy = E; + SWIFT(tindex).wavespectra.freq = f; SWIFT(tindex).wavespectra.a1 = a1; SWIFT(tindex).wavespectra.b1 = b1; SWIFT(tindex).wavespectra.a2 = a2; @@ -176,18 +180,22 @@ [Hs,Tp,Dp,E,f,a1,b1,a2,b2,check] = GPSandIMUwaves(u(igood),v(igood),az(igood),[],[],fs_gps); + if interpf % Interpolate to the original freq bands - E = interp1(f,E,f_original); - a1 = interp1(f,a1,f_original); - b1 = interp1(f,b1,f_original); - a2 = interp1(f,a2,f_original); - b2 = interp1(f,b2,f_original); + E = interp1(f,E,f_original); + a1 = interp1(f,a1,f_original); + b1 = interp1(f,b1,f_original); + a2 = interp1(f,a2,f_original); + b2 = interp1(f,b2,f_original); + f = f_original; + end % Replace scalar values, but not directional moments SWIFT(tindex).sigwaveheight = Hs; SWIFT(tindex).peakwaveperiod = Tp; SWIFT(tindex).peakwavedirT = Dp; SWIFT(tindex).wavespectra.energy = E; + SWIFT(tindex).wavespectra.freq = f; SWIFT(tindex).wavespectra.a1 = a1; SWIFT(tindex).wavespectra.b1 = b1; SWIFT(tindex).wavespectra.a2 = a2; @@ -202,30 +210,48 @@ [Hs,Tp,Dp,E,f,a1,b1,a2,b2,check] = GPSwaves(u,v,z_gps,fs_gps); % interp to the original freq bands - E = interp1(f,E,f_original); - a1 = interp1(f,a1,f_original); - b1 = interp1(f,b1,f_original); - a2 = interp1(f,a2,f_original); - b2 = interp1(f,b2,f_original); + if interpf + E = interp1(f,E,f_original); + a1 = interp1(f,a1,f_original); + b1 = interp1(f,b1,f_original); + a2 = interp1(f,a2,f_original); + b2 = interp1(f,b2,f_original); + f = f_original; + end % replace scalar values, but not directional moments SWIFT(tindex).sigwaveheight = Hs; SWIFT(tindex).peakwaveperiod = Tp; SWIFT(tindex).peakwavedirT = Dp; SWIFT(tindex).wavespectra.energy = E; + SWIFT(tindex).wavespectra.freq = f; SWIFT(tindex).wavespectra.a1 = a1; SWIFT(tindex).wavespectra.b1 = b1; SWIFT(tindex).wavespectra.a2 = a2; SWIFT(tindex).wavespectra.b2 = b2; SWIFT(tindex).wavespectra.check = check; + end %% Flag bad results if Hs == 9999 || isnan(Hs) % invalid wave result - disp('Bad waves. Flagging...') + disp('Bad wave height. Flagging...') + badwaves(tindex) = true; + SWIFT(tindex).sigwaveheight = NaN; + end + + if Tp == 9999 || isnan(Tp) % invalid wave result + disp('Bad wave period. Flagging...') + badwaves(tindex) = true; + SWIFT(tindex).peakwaveperiod = NaN; + end + + if Dp == 9999 || isnan(Dp) % invalid wave result + disp('Bad wave direction. Flagging...') badwaves(tindex) = true; + SWIFT(tindex).peakwavedirT = NaN; end %% Save raw displacements to SWIFT structure and burst file if specified diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index b22f0ae..9471d5d 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -198,6 +198,12 @@ continue end + % Skip burst if way too small + if length(burst.time) < 500 + disp('Not enough data, skipping burst...') + continue + end + % Burst time t0 = min(avg.time); if abs(btime - t0) > 15/(60*24) @@ -383,7 +389,7 @@ % Altimeter SWIFT(tindex).signature.altimeter = maxz; % Temperaure - SWIFT(tindex).watertemp = profile.temp; + SWIFT(tindex).watertemp2 = profile.temp; elseif timematch && badburst % time match, bad burst % HR data @@ -407,6 +413,10 @@ SWIFT(tindex).signature.echo = NaN(size(echogram.echoc)); SWIFT(tindex).signature.echo = echogram.r + opt.xz; end + % Altimeter + SWIFT(tindex).signature.altimeter = NaN; + % Temperaure + SWIFT(tindex).watertemp2 = NaN; % Flag badsig(tindex) = true; diff --git a/Winds/reprocess_Y81.m b/Winds/reprocess_Y81.m index a4cbe23..ce9f3c3 100644 --- a/Winds/reprocess_Y81.m +++ b/Winds/reprocess_Y81.m @@ -48,7 +48,7 @@ uvw = NaN(1000,3); temp = NaN(1000,1); end - windspd = mean((uvw(:,1).^2 + uvw(:,2).^2 + uvw(:,3).^2).^.5); + windspd = mean((uvw(:,1).^2 + uvw(:,2).^2 + uvw(:,3).^2).^.5,'omitnan'); u = uvw(:,1); v = uvw(:,2); w = uvw(:,3); From 755fe60db7ea9c1a36e64492dc301f0309de426a Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 29 Jul 2024 06:24:00 -0700 Subject: [PATCH 13/34] updates --- GeneralTools/catSWIFT.m | 4 + GeneralTools/postprocess_SWIFT.asv | 193 +++++++++++++++++++++++++++++ GeneralTools/postprocess_SWIFT.m | 38 +++--- Signature/reprocess_SIG.m | 6 + 4 files changed, 222 insertions(+), 19 deletions(-) create mode 100644 GeneralTools/postprocess_SWIFT.asv diff --git a/GeneralTools/catSWIFT.m b/GeneralTools/catSWIFT.m index 27d9c13..4bcbddd 100644 --- a/GeneralTools/catSWIFT.m +++ b/GeneralTools/catSWIFT.m @@ -212,6 +212,7 @@ % [swift.dirwavepower(:,:,it),~,~,~,~,~,~,~] = SWIFTdirectionalspectra(SWIFT(it),0); % end + % Wind if isfield(SWIFT,'windspd') swift.windu = [SWIFT.windspd]; @@ -246,6 +247,9 @@ swift.windpower = NaN(116,nt); end swift.windfreq = median(swift.windfreq,2,'omitnan'); + if isfield(SWIFT,'windustar') + swift.windustar = [SWIFT.windustar]; + end % TKE Dissipation Rate and HR vertical velocity if isfield(SWIFT,'signature') diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv new file mode 100644 index 0000000..4777e8a --- /dev/null +++ b/GeneralTools/postprocess_SWIFT.asv @@ -0,0 +1,193 @@ +% [SWIFT,sinfo] = postprocess_SWIFT(expdir) + +% Master post-processing function, that calls sub-functions to reprocess +% each different type of raw SWIFT data. +% L1 product must have been created prior to running this script, +% by running 'concatSWIFT_offloadedSDcard.m' in the mission directory +% Note: concatSWIFT in turn runs 'compileSWIFT_SBDservertelemetry.m', +% which in turn calls the function 'readSWIFT_SBD.m' +% Currently no reprocessing option for Airmar (PB2) +% nor Heitronics CTs (536?) + +% K. Zeiden 07/2024 + +if ispc + slash = '\'; +else + slash = '/'; +end + +%% User defined experiment directory (to be later converted to function inputs) + +expdir = 'S:\SEAFAC\June2024'; + +% Processing toggles +rpIMU = true; % Waves +rpSBG = true; % Waves +rpWXT = false; % MET +rpY81 = false; % MET +rpACS = false; % CT +rpSIG = false; % TKE +rpAQH = false; % TKE +rpAQD = false; % TKE + +%% Loop through missions and reprocess +cd(expdir) +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +for im = 1:length(missions) + + disp(['Post-processing ' missions(im).name]) + + missiondir = [missions(im).folder slash missions(im).name]; + cd(missiondir) + + %% Locate L1 product, skip if does not exist. + % Else create 'sinfo' and modify L1 product. + l1file = dir([missiondir slash '*L1.mat']); + if isempty(l1file) + disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + else + load([l1file.folder slash l1file.name],'SWIFT'); + if isfield(SWIFT,'ID') + disp('Create information structure ''sinfo''') + sinfo.ID = SWIFT(1).ID; + sinfo.CTdepth = SWIFT(1).CTdepth; + sinfo.metheight = SWIFT(1).metheight; + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + else + sinfo.type = 'V3'; + end + SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); + disp('Saving new L1 product...') + save([l1file.folder slash l1file.name],'SWIFT','sinfo') + end + % One time, will remove later + % if isfield(SWIFT,'signature') + % sinfo.type = 'V4'; + % save([l1file.folder slash l1file.name],'sinfo','-append') + % else + % sinfo.type = 'V3'; + % save([l1file.folder slash l1file.name],'sinfo','-append') + % end + end + + %% Reprocess IMU + if rpIMU + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) + disp('Reprocessing IMU data...') + calctype = 'IMUandGPS'; + filtertype = 'RC'; + saveraw = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); + else + disp('No IMU data...') + end + end + + %% Reprocess SBG + if rpSBG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) + disp('Reprocessing SBG data...') + saveraw = false; + useGPS = false; + interpf = false; + [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + else + disp('No SBG data...') + end + end + + %% Reprocess WXT + if rpWXT + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) + disp('Reprocessing Vaisala WXT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); + else + disp('No WXT data...') + end + end + + %% Reprocess Y81 + if rpY81 + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) + disp('Reprocessing Y81 Sonic Anemometer data...') + [SWIFT,sinfo] = reprocess_Y81(missiondir); + else + disp('No Y81 data...') + end + end + + %% Reprocess ACS + if rpACS + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) + disp('Reprocessing ACS CT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); + else + disp('No ACS data...') + end + end + + %% Reprocess SIG + if rpSIG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + else + disp('No SIG data...') + end + end + + %% Reprocess AQD + if rpAQD + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) + disp('Reprocessing Aquadopp (AQD) data...') + readraw = true; + [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); + else + disp('No AQD data...') + end + end + + %% Reprocess AQH + if rpAQH + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) + disp('Reprocessing Aquadopp (AQH) data...') + readraw = true; + plotburst = false; + [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); + else + disp('No AQH data...') + end + end + + % Re-load L1 and L2 product and plot each for comparison + load([l1file.folder slash l1file.name],'SWIFT'); + SWIFTL1 = SWIFT; + l2file = dir([missiondir slash '*L2.mat']); + load([l2file.folder slash l2file.name],'SWIFT','sinfo'); + SWIFTL2 = SWIFT; + + if plotL1L2 + if strcmp(sinfo.type,'V3') + fh1 = plotSWIFTV3(SWIFTL1); + fh2 = plotSWIFTV3(SWIFTL2); + else + fh1 = plotSWIFTV4(SWIFTL1); + fh2 = plotSWIFTV4(SWIFTL2); + end + print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') + print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') + end + +end + + + + diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index e88ce5e..da02243 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,7 +19,7 @@ %% User defined experiment directory (to be later converted to function inputs) -expdir = 'S:\SEAFAC\June2024'; +expdir = 'S:\SEAFAC\June2024\NorthMooring'; % Processing toggles rpIMU = false; % Waves @@ -27,7 +27,7 @@ rpWXT = false; % MET rpY81 = false; % MET rpACS = false; % CT -rpSIG = false; % TKE +rpSIG = true; % TKE rpAQH = false; % TKE rpAQD = false; % TKE @@ -36,14 +36,14 @@ missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); -for im = 1%:length(missions) +for im = 1:length(missions) disp(['Post-processing ' missions(im).name]) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) - % Locate L1 product, skip if does not exist. + %% Locate L1 product, skip if does not exist. % Else create 'sinfo' and modify L1 product. l1file = dir([missiondir slash '*L1.mat']); if isempty(l1file) @@ -65,16 +65,16 @@ save([l1file.folder slash l1file.name],'SWIFT','sinfo') end % One time, will remove later - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - save([l1file.folder slash l1file.name],'sinfo','-append') - else - sinfo.type = 'V3'; - save([l1file.folder slash l1file.name],'sinfo','-append') - end + % if isfield(SWIFT,'signature') + % sinfo.type = 'V4'; + % save([l1file.folder slash l1file.name],'sinfo','-append') + % else + % sinfo.type = 'V3'; + % save([l1file.folder slash l1file.name],'sinfo','-append') + % end end - % Reprocess IMU + %% Reprocess IMU if rpIMU if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) disp('Reprocessing IMU data...') @@ -87,7 +87,7 @@ end end - % Reprocess SBG + %% Reprocess SBG if rpSBG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) disp('Reprocessing SBG data...') @@ -100,7 +100,7 @@ end end - % Reprocess WXT + %% Reprocess WXT if rpWXT if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) disp('Reprocessing Vaisala WXT data...') @@ -111,7 +111,7 @@ end end - % Reprocess Y81 + %% Reprocess Y81 if rpY81 if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) disp('Reprocessing Y81 Sonic Anemometer data...') @@ -121,7 +121,7 @@ end end - % Reprocess ACS + %% Reprocess ACS if rpACS if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) disp('Reprocessing ACS CT data...') @@ -132,7 +132,7 @@ end end - % Reprocess SIG + %% Reprocess SIG if rpSIG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) disp('Reprocessing Signature1000 data...') @@ -144,7 +144,7 @@ end end - % Reprocess AQD + %% Reprocess AQD if rpAQD if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) disp('Reprocessing Aquadopp (AQD) data...') @@ -155,7 +155,7 @@ end end - % Reprocess AQH + %% Reprocess AQH if rpAQH if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) disp('Reprocessing Aquadopp (AQH) data...') diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index b22f0ae..f275e99 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -198,6 +198,12 @@ continue end + % Skip if any of the burst fields are 3D + if ndims(burst.VelocityData)>2 || ndims(burst.CorrelationData)>2 || ndims(burst.AmplitudeData)>2 + disp('Burst fields have more than 2 dimensions, skipping burst...') + continue + end + % Burst time t0 = min(avg.time); if abs(btime - t0) > 15/(60*24) From e9c0976178d71f516deeb12993c3aad92a77216a Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 29 Jul 2024 07:23:54 -0700 Subject: [PATCH 14/34] Update postprocess_SWIFT.m --- GeneralTools/postprocess_SWIFT.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index da02243..7c28d06 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,7 +19,7 @@ %% User defined experiment directory (to be later converted to function inputs) -expdir = 'S:\SEAFAC\June2024\NorthMooring'; +expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'NorthMooring']; % Processing toggles rpIMU = false; % Waves From 65acbea1464c1aebeace0475c24b2017a53867b3 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Tue, 30 Jul 2024 07:29:54 -0700 Subject: [PATCH 15/34] updates --- GeneralTools/catSWIFT.m | 8 +- GeneralTools/postprocess_SWIFT.asv | 193 ----------------------------- GeneralTools/postprocess_SWIFT.m | 40 +++--- Winds/reprocess_WXT.m | 21 ++-- 4 files changed, 40 insertions(+), 222 deletions(-) delete mode 100644 GeneralTools/postprocess_SWIFT.asv diff --git a/GeneralTools/catSWIFT.m b/GeneralTools/catSWIFT.m index 4bcbddd..f8f0b8f 100644 --- a/GeneralTools/catSWIFT.m +++ b/GeneralTools/catSWIFT.m @@ -51,21 +51,21 @@ end % Radiometer -if isfield(SWIFT,'radiometertemp1mean') +if isfield(SWIFT,'radiometertemp1mean')% This is the brightness (target) temperature nrad = length(SWIFT(1).radiometertemp1mean); swift.radtemp1 = reshape([SWIFT.radiometertemp1mean],nrad,length(SWIFT)); if size(swift.radtemp1,2)~= nt swift.radtemp1 = swift.radtemp1'; end end -if isfield(SWIFT,'radiometerrad1') +if isfield(SWIFT,'radiometerrad1')% This is derived from brightness temp using stefan boltzman nrad = length(SWIFT(1).radiometerrad1); swift.rad1 = reshape([SWIFT.radiometerrad1],nrad,length(SWIFT)); if size(swift.rad1,2)~= nt swift.rad1 = swift.rad1'; end end -if isfield(SWIFT,'radiometertemp2mean') +if isfield(SWIFT,'radiometertemp2mean')% This is the jacket temperature nrad = length(SWIFT(1).radiometertemp2mean); swift.radtemp2 = reshape([SWIFT.radiometertemp2mean],nrad,length(SWIFT)); if size(swift.radtemp2,2)~= nt @@ -73,7 +73,7 @@ end end if isfield(SWIFT,'radiometerrad1') - nrad = length(SWIFT(1).radiometerrad2); + nrad = length(SWIFT(1).radiometerrad2);% This is derived from jacket temp using stefan boltzman swift.rad2 = reshape([SWIFT.radiometerrad2],nrad,length(SWIFT)); if size(swift.rad2,2)~= nt swift.rad2 = swift.rad2'; diff --git a/GeneralTools/postprocess_SWIFT.asv b/GeneralTools/postprocess_SWIFT.asv deleted file mode 100644 index 4777e8a..0000000 --- a/GeneralTools/postprocess_SWIFT.asv +++ /dev/null @@ -1,193 +0,0 @@ -% [SWIFT,sinfo] = postprocess_SWIFT(expdir) - -% Master post-processing function, that calls sub-functions to reprocess -% each different type of raw SWIFT data. -% L1 product must have been created prior to running this script, -% by running 'concatSWIFT_offloadedSDcard.m' in the mission directory -% Note: concatSWIFT in turn runs 'compileSWIFT_SBDservertelemetry.m', -% which in turn calls the function 'readSWIFT_SBD.m' -% Currently no reprocessing option for Airmar (PB2) -% nor Heitronics CTs (536?) - -% K. Zeiden 07/2024 - -if ispc - slash = '\'; -else - slash = '/'; -end - -%% User defined experiment directory (to be later converted to function inputs) - -expdir = 'S:\SEAFAC\June2024'; - -% Processing toggles -rpIMU = true; % Waves -rpSBG = true; % Waves -rpWXT = false; % MET -rpY81 = false; % MET -rpACS = false; % CT -rpSIG = false; % TKE -rpAQH = false; % TKE -rpAQD = false; % TKE - -%% Loop through missions and reprocess -cd(expdir) -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); - -for im = 1:length(missions) - - disp(['Post-processing ' missions(im).name]) - - missiondir = [missions(im).folder slash missions(im).name]; - cd(missiondir) - - %% Locate L1 product, skip if does not exist. - % Else create 'sinfo' and modify L1 product. - l1file = dir([missiondir slash '*L1.mat']); - if isempty(l1file) - disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) - else - load([l1file.folder slash l1file.name],'SWIFT'); - if isfield(SWIFT,'ID') - disp('Create information structure ''sinfo''') - sinfo.ID = SWIFT(1).ID; - sinfo.CTdepth = SWIFT(1).CTdepth; - sinfo.metheight = SWIFT(1).metheight; - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - else - sinfo.type = 'V3'; - end - SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); - disp('Saving new L1 product...') - save([l1file.folder slash l1file.name],'SWIFT','sinfo') - end - % One time, will remove later - % if isfield(SWIFT,'signature') - % sinfo.type = 'V4'; - % save([l1file.folder slash l1file.name],'sinfo','-append') - % else - % sinfo.type = 'V3'; - % save([l1file.folder slash l1file.name],'sinfo','-append') - % end - end - - %% Reprocess IMU - if rpIMU - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) - disp('Reprocessing IMU data...') - calctype = 'IMUandGPS'; - filtertype = 'RC'; - saveraw = false; - [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); - else - disp('No IMU data...') - end - end - - %% Reprocess SBG - if rpSBG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) - disp('Reprocessing SBG data...') - saveraw = false; - useGPS = false; - interpf = false; - [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); - else - disp('No SBG data...') - end - end - - %% Reprocess WXT - if rpWXT - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) - disp('Reprocessing Vaisala WXT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); - else - disp('No WXT data...') - end - end - - %% Reprocess Y81 - if rpY81 - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) - disp('Reprocessing Y81 Sonic Anemometer data...') - [SWIFT,sinfo] = reprocess_Y81(missiondir); - else - disp('No Y81 data...') - end - end - - %% Reprocess ACS - if rpACS - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) - disp('Reprocessing ACS CT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); - else - disp('No ACS data...') - end - end - - %% Reprocess SIG - if rpSIG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - disp('Reprocessing Signature1000 data...') - plotburst = false; - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - else - disp('No SIG data...') - end - end - - %% Reprocess AQD - if rpAQD - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) - disp('Reprocessing Aquadopp (AQD) data...') - readraw = true; - [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); - else - disp('No AQD data...') - end - end - - %% Reprocess AQH - if rpAQH - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) - disp('Reprocessing Aquadopp (AQH) data...') - readraw = true; - plotburst = false; - [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); - else - disp('No AQH data...') - end - end - - % Re-load L1 and L2 product and plot each for comparison - load([l1file.folder slash l1file.name],'SWIFT'); - SWIFTL1 = SWIFT; - l2file = dir([missiondir slash '*L2.mat']); - load([l2file.folder slash l2file.name],'SWIFT','sinfo'); - SWIFTL2 = SWIFT; - - if plotL1L2 - if strcmp(sinfo.type,'V3') - fh1 = plotSWIFTV3(SWIFTL1); - fh2 = plotSWIFTV3(SWIFTL2); - else - fh1 = plotSWIFTV4(SWIFTL1); - fh2 = plotSWIFTV4(SWIFTL2); - end - print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') - print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') - end - -end - - - - diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index 7c28d06..c4c0859 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,18 +19,21 @@ %% User defined experiment directory (to be later converted to function inputs) -expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'NorthMooring']; +expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'SouthMooring']; % Processing toggles -rpIMU = false; % Waves +rpIMU = true; % Waves rpSBG = false; % Waves -rpWXT = false; % MET +rpWXT = true; % MET rpY81 = false; % MET rpACS = false; % CT -rpSIG = true; % TKE +rpSIG = false; % TKE rpAQH = false; % TKE rpAQD = false; % TKE +% Plotting toggle +plotL1L2 = true; + %% Loop through missions and reprocess cd(expdir) missions = dir([expdir slash 'SWIFT*']); @@ -65,13 +68,13 @@ save([l1file.folder slash l1file.name],'SWIFT','sinfo') end % One time, will remove later - % if isfield(SWIFT,'signature') - % sinfo.type = 'V4'; - % save([l1file.folder slash l1file.name],'sinfo','-append') - % else - % sinfo.type = 'V3'; - % save([l1file.folder slash l1file.name],'sinfo','-append') - % end + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + save([l1file.folder slash l1file.name],'sinfo','-append') + else + sinfo.type = 'V3'; + save([l1file.folder slash l1file.name],'sinfo','-append') + end end %% Reprocess IMU @@ -81,7 +84,8 @@ calctype = 'IMUandGPS'; filtertype = 'RC'; saveraw = false; - [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw); + interpf = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw,interpf); else disp('No IMU data...') end @@ -100,12 +104,14 @@ end end - %% Reprocess WXT + %% Reprocess WXT (536!!!) if rpWXT - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_WXT_*.dat'])) + if ~isempty(dir([missiondir slash 'WXT' slash 'Raw' slash '*' slash '*_536_*.dat'])) disp('Reprocessing Vaisala WXT data...') readraw = false; - [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw); + usewind = false; + + [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw,usewind); else disp('No WXT data...') end @@ -168,10 +174,10 @@ end % Re-load L1 and L2 product and plot each for comparison - load([l1file.folder slash l1file.name],'SWIFT'); + load([l1file.folder slash l1file.name],'SWIFT','sinfo'); SWIFTL1 = SWIFT; l2file = dir([missiondir slash '*L2.mat']); - load([l2file.folder slash l2file.name],'SWIFT','sinfo'); + load([l2file.folder slash l2file.name],'SWIFT'); SWIFTL2 = SWIFT; if plotL1L2 diff --git a/Winds/reprocess_WXT.m b/Winds/reprocess_WXT.m index 25c1f53..eda6fed 100644 --- a/Winds/reprocess_WXT.m +++ b/Winds/reprocess_WXT.m @@ -1,4 +1,4 @@ -function [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw) +function [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw,usewind) % reprocess SWIFT Vaisala WXT files % loop thru raw data for a given SWIFT deployment, then @@ -32,14 +32,16 @@ end %% Loop through raw burst files and reprocess +burstreplaced = false(length(SWIFT),1); -bfiles = dir([missiondir slash '*' slash 'Raw' slash '*' slash '*WXT*.dat']); +bfiles = dir([missiondir slash 'WXT' slash 'Raw' slash '*' slash '*536*.dat']); +disp(['Found ' num2str(length(bfiles)) ' burst files...']) for iburst = 1:length(bfiles) disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - if bfiles(iburst).bytes > 0 + if bfiles(iburst).bytes == 0 disp('Burst file is empty. Skippping ...') end @@ -61,10 +63,12 @@ continue end + if usewind SWIFT(tindex).winddirR = nanmean(winddirR); %#ok<*NANMEAN> % mean wind direction (deg relative) - SWIFT(tindex).winddirRstddev = nanstd(winddirR); % std dev of wind direction (deg) SWIFT(tindex).windspd = nanmean(windspd); % mean wind speed (m/s) + SWIFT(tindex).winddirRstddev = nanstd(winddirR); % std dev of wind direction (deg) SWIFT(tindex).windspdstddev = nanstd(windspd); % std dev of wind spd (m/s) + end SWIFT(tindex).airtemp = nanmean(airtemp); % deg C SWIFT(tindex).airtempstddev = nanstd(airtemp); % deg C SWIFT(tindex).relhumidity = nanmean(relhumidity); % percent @@ -73,6 +77,7 @@ SWIFT(tindex).airpresstddev = nanstd(airpres); % millibars SWIFT(tindex).rainaccum = nanmean(rainaccum); % millimeters SWIFT(tindex).rainint = nanmean(rainint); % millimeters_per_hour + burstreplaced(tindex) = true; end @@ -80,11 +85,11 @@ %% If SWIFT structure elements not replaced, fill variables with NaNs for i = 1:length(SWIFT) - if ~isfield(SWIFT(i),'windspd') || isempty(SWIFT(i).windspd) - disp(['bad data at index ' num2str(i)]) - SWIFT(i).winddirR = NaN; + if ~isfield(SWIFT(i),'relhumidity') || isempty(SWIFT(i).relhumidity) + disp(['No data at index ' num2str(i)]) + % SWIFT(i).winddirR = NaN; SWIFT(i).winddirRstddev = NaN; - SWIFT(i).windspd = NaN; + % SWIFT(i).windspd = NaN; SWIFT(i).windspdstddev = NaN; SWIFT(i).airtemp = NaN; SWIFT(i).airtempstddev = NaN; From af34a61948434c4155c4b4c1f24a3f6e332c78a4 Mon Sep 17 00:00:00 2001 From: Michael James Date: Thu, 22 Aug 2024 10:52:12 -0700 Subject: [PATCH 16/34] working changes to SWIFT outliers --- GeneralTools/findoutliersSWIFT.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GeneralTools/findoutliersSWIFT.m b/GeneralTools/findoutliersSWIFT.m index 93a2b55..fc3dbd4 100644 --- a/GeneralTools/findoutliersSWIFT.m +++ b/GeneralTools/findoutliersSWIFT.m @@ -11,7 +11,7 @@ % defined as percent used in percentile method, write as "~" or leave blank for % movmedian % varargin defined as string / value dictionary pairs. List below: -% 'plot_results' ; true/false bool +% 'plot_results' ; true/false bool (IN DEVELOPMENT) % 'window' ; float (this is a window for "movmedian") % 7_23_2024 % Scoped down to per SWIFT variable so can be run in created loop @@ -53,6 +53,7 @@ if string(method) == "percentile" if 0 Date: Thu, 22 Aug 2024 11:06:31 -0700 Subject: [PATCH 17/34] added in Non oulier as inverse to outlier structure; support for NaN array of multiple sizes --- GeneralTools/findoutliersSWIFT.asv | 94 ++++++++++++++++++++++++++++++ GeneralTools/findoutliersSWIFT.m | 9 ++- 2 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 GeneralTools/findoutliersSWIFT.asv diff --git a/GeneralTools/findoutliersSWIFT.asv b/GeneralTools/findoutliersSWIFT.asv new file mode 100644 index 0000000..9fec0fc --- /dev/null +++ b/GeneralTools/findoutliersSWIFT.asv @@ -0,0 +1,94 @@ +function [nonoutliersSWIFT,outliersSWIFT,booloutSWIFT] = findoutliersSWIFT(SWIFT, method, pct,varargin) +% findoutliersSWIFT - finds outliers out of % range and plots hist of all +% SWIFT fields +% Find outliers in SWIFT L1 DATA +% Michael James +% 7_12_2024 +% findoutliersSWIFT(SWIFT, method, pctdatathresh,varargin) +% method == +% defined as "percentile" or "movmedian" (moving percentile) +% pct == +% defined as percent used in percentile method, write as "~" or leave blank for +% movmedian +% varargin defined as string / value dictionary pairs. List below: +% 'plot_results' ; true/false bool (IN DEVELOPMENT) +% 'window' ; float (this is a window for "movmedian") +% 7_23_2024 +% Scoped down to per SWIFT variable so can be run in created loop +% determined by user +% Added boolean output for ease of indexing + + +% Preset binary flags +vararginlist = ["plot_results" ; "window"]; +plot_results = true; %default create figures +window = 40; % Window size of 100 values as default + +% Check if there are any optional arguments +if ~isempty(varargin) + if mod(length(varargin),2) == 0 %check for pairs + for i= 1:2:length(varargin) + if mean(contains(string(vararginlist),char(varargin{i}))) ~=0 + eval([char(vararginlist(contains(string(vararginlist),varargin{i}))) ' = varargin{i+1};']); + % taking the name and setting the variable to be the user + % input + else + error('Check naming of the variable arguements') + end + end; + else + error('incorrect variable arguements; must be in pairs') + end; +elseif isempty(varargin) + % Do nothing +else + error('Input error'); help findoutliersSWIFT; +end + + +if mean(contains(["percentile" ; "movmedian"],char(method))) ==0 + error('use "percentile" or "movmedian" to define the method'); +end + + +if string(method) == "percentile" + if 0 Date: Wed, 4 Sep 2024 07:08:01 -0700 Subject: [PATCH 18/34] updates --- GeneralTools/postprocess_SWIFT.m | 2 +- Signature/hannwinPSD2.m | 99 +++++++++++++++++++------------- Waves/XYZwaves.m | 1 - 3 files changed, 59 insertions(+), 43 deletions(-) diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index c4c0859..67efc7f 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,7 +19,7 @@ %% User defined experiment directory (to be later converted to function inputs) -expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'SouthMooring']; +expdir = ['S:' slash 'SEAFAC' slash 'June2024']; % Processing toggles rpIMU = true; % Waves diff --git a/Signature/hannwinPSD2.m b/Signature/hannwinPSD2.m index dd48c2c..0e84255 100644 --- a/Signature/hannwinPSD2.m +++ b/Signature/hannwinPSD2.m @@ -1,56 +1,73 @@ -function [PS,F,PH,err] = hannwinPSD2(data,M,fs,norm) +function [PX,F,Ph,err] = hannwinPSD2(datax,nwin,fs,norm) % hannwinPSD2 Computes spectra via windowing % PS is either unitless (if normalized to integrate to 1) or units of % variance of data (e.g. m^2/s^2 for velocity (m/s)). -data = data(:)'; %force row - -N = length(data); %Total number of data points (whole timeseries) -nwin = 2*(floor(N/M))-1; %Number of windows (%50 overlap) -h = sqrt(8/3)*hann(M);% Hanning window variance preservation - -PS = NaN(nwin,floor(M/2)); -PH = NaN(nwin,floor(M/2)); -for i=1:nwin - k1 = (i-1)*M/2 + 1; - k2 = (i+1)*M/2; - datai = data(k1:k2); - ts = datai; - ts(isnan(ts)) = nanmean(ts); % Replace NaN w/mean - ts = ts - nanmean(ts); % Remove mean - ts = detrend(ts); % Remove trend - %(IMPORTANT TO REMOVE MEAN BEFORE WINDOW!) - ts = h.*ts'; % Window using normalized hanning window - ph = angle(fft(ts)); - ps = (abs(fft(ts)).^2); +datax = datax(:)'; %force row + +% Create taper (window) +N = length(datax); %Total number of data points (whole timeseries) +M = 2*(floor(N/nwin))-1; % Number of windows (%50 overlap) +h = sqrt(8/3)*hanning(nwin);% Hanning window variance preservation + +% Frequencies +T = nwin/fs; +df = 1/T; +F = (0:nwin-1)*df; +F = F(1:floor(end/2)); + +PX = NaN(M,floor(nwin/2)); +Ph = NaN(M,floor(nwin/2)); +DX = NaN(M,nwin); +for i = 1:M + + % Select window + k1 = (i-1)*nwin/2 + 1; + k2 = (i+1)*nwin/2; + dataxi = datax(k1:k2); + + % Remove mean, trend and apply hanning window + dataxi(isnan(dataxi)) = mean(dataxi,'omitnan'); % Replace NaN w/mean + dataxi = dataxi - mean(dataxi,'omitnan'); % Remove mean + dataxi = detrend(dataxi); % Remove trend + dataxi = h.*dataxi'; % Window using normalized hanning window + + % Take fft and square + ph = angle(fft(dataxi));% phase + px = abs(fft(dataxi)).^2;% power (magnitude) + + % Select for positive frequencies + double magnitude of PS ph = ph(1:floor(end/2)); - ps = ps(1:floor(end/2)); - ps(2:end) = 2*ps(2:end); - PS(i,:) = ps; - PH(i,:) = ph; + px = px(1:floor(end/2)); + px(2:end) = 2*px(2:end); + + % Save in matrix for averaging + PX(i,:) = px; + Ph(i,:) = ph; + DX(i,:) = dataxi; end -%Average -PS = nanmean(PS,1); -PH = nanmean(PH,1); - -%Frequencies Computed -T = M/fs; -ff = 1/T; -F = (0:M-1)*ff; -F = F(1:floor(end/2)); -%Take Positive Frequencies only +% Average +PX = mean(PX,'omitnan'); +Ph = mean(Ph,'omitnan'); -%Normalize to satisfy Parsevals Theorem or produce PSD +% Normalize to satisfy Parsevals Theorem or produce PSD if strcmp(norm,'psd') - PS = PS/sum(PS); + PX = PX/sum(PX); elseif strcmp(norm,'par') - PS = PS/(ff*M^2); + PX = PX/(df*nwin^2); + vard = mean(var(DX,[],2,'omitnan'),'omitnan'); + varPS = sum(PX,'omitnan').*df; + pvardiff = 100*(varPS-vard)/vard; + if abs(pvardiff) > 5 + warning('Parseval''s Theorom not satisifed to within 5%') + else + end end -%Confidence Intervals -err_low = (2*nwin)/chi2inv(.05/2,2*nwin); -err_high = (2*nwin)/chi2inv(1-.05/2,2*nwin); +% Confidence Intervals +err_low = (2*M)/chi2inv(.05/2,2*M); +err_high = (2*M)/chi2inv(1-.05/2,2*M); err = [err_low err_high]; end diff --git a/Waves/XYZwaves.m b/Waves/XYZwaves.m index ba2c736..56cfb70 100644 --- a/Waves/XYZwaves.m +++ b/Waves/XYZwaves.m @@ -34,7 +34,6 @@ merge = 3; % freq bands to merge, must be odd? maxf = .5; % frequency cutoff for telemetry Hz - %% begin processing, if data sufficient pts = length(z); % record length in data points From 02ffafdc2d9d6e8a39492b92c74ec9db551e7dc5 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:50:56 -0700 Subject: [PATCH 19/34] Update hannwinPSD2.m --- Signature/hannwinPSD2.m | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Signature/hannwinPSD2.m b/Signature/hannwinPSD2.m index 0e84255..61f2f32 100644 --- a/Signature/hannwinPSD2.m +++ b/Signature/hannwinPSD2.m @@ -1,4 +1,4 @@ -function [PX,F,Ph,err] = hannwinPSD2(datax,nwin,fs,norm) +function [PX,F,Ph,err,PXstd] = hannwinPSD2(datax,nwin,fs,norm) % hannwinPSD2 Computes spectra via windowing % PS is either unitless (if normalized to integrate to 1) or units of % variance of data (e.g. m^2/s^2 for velocity (m/s)). @@ -48,14 +48,17 @@ end % Average +PXstd = std(PX,[],1,'omitnan'); PX = mean(PX,'omitnan'); Ph = mean(Ph,'omitnan'); % Normalize to satisfy Parsevals Theorem or produce PSD if strcmp(norm,'psd') + PXstd = PXstd./sum(PX); PX = PX/sum(PX); elseif strcmp(norm,'par') PX = PX/(df*nwin^2); + PXstd = PXstd/(df*nwin^2); vard = mean(var(DX,[],2,'omitnan'),'omitnan'); varPS = sum(PX,'omitnan').*df; pvardiff = 100*(varPS-vard)/vard; From 3d25dfb1058d553b8035246c2e1293ec93bd3cf0 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 9 Sep 2024 06:29:11 -0700 Subject: [PATCH 20/34] Create sonicdissipation.m --- Winds/sonicdissipation.m | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 Winds/sonicdissipation.m diff --git a/Winds/sonicdissipation.m b/Winds/sonicdissipation.m new file mode 100644 index 0000000..8bd58e1 --- /dev/null +++ b/Winds/sonicdissipation.m @@ -0,0 +1,72 @@ +function [eps,ustar,ps,cs,f,uadv,theta,iso,power] = sonicdissipation(u,v,w,temp,fs,twin,z) + +%%% Rotate to get streamwise + cross-stream velocities +theta = atan2d(mean(v,'omitnan'),mean(u,'omitnan')); +R = [cosd(theta), -sind(theta);... + sind(theta), cosd(theta)]; +uv_rot = R'*([u(:)'; v(:)']); +u_rot = uv_rot(1,:); +v_rot = uv_rot(2,:); +uadv = mean(u_rot,'omitnan'); + +%%% Convert direction to oceanographic convention +theta = -theta + 90; +if theta < 0 + theta = theta + 360; +end + +% Spectra +nwin = floor(twin.*fs); +norm = 'par'; +[UU,f,~,~,~] = hannwinPSD2(u_rot,nwin,fs,norm); +[VV,~,~,~,~] = hannwinPSD2(v_rot,nwin,fs,norm); +[WW,~,~,~,~] = hannwinPSD2(w,nwin,fs,norm); +[TT,~,~,~,~] = hannwinPSD2(temp,nwin,fs,norm); + +% Cospectra +[UW,~,~] = hannwinCOPSD2(u,w,nwin,fs,norm); +[VW,~,~] = hannwinCOPSD2(v,w,nwin,fs,norm); +[TW,~,~] = hannwinCOPSD2(temp,w,nwin,fs,norm); + +% Dissipation Rate and Friction Velocity +K = 0.55; +k = 0.41; +isr = f>= 1.5 & f<=4; + +EUisr = mean(UU(isr).*f(isr).^(5/3),2,'omitnan'); +EVisr = mean(VV(:,isr).*f(isr).^(5/3),2,'omitnan'); +EWisr = mean(WW(:,isr).*f(isr).^(5/3),2,'omitnan'); + +iso.v = EVisr./EUisr; +iso.w = EWisr./EUisr; + +eps.u = (EUisr./K).^(3/2).*(2*pi./uadv); +eps.v = ((3/4)*EVisr./K).^(3/2).*(2*pi./uadv); +eps.w = ((3/4)*EWisr./K).^(3/2).*(2*pi./uadv); + +ustar.u = (k.*z.*eps.u).^(1/3); +ustar.v = (k.*z.*eps.v).^(1/3); +ustar.w = (k.*z.*eps.w).^(1/3); + +% Best-fit power-law to inertial subrange +[~,mu,bu,~] = fitline(log10(f(isr)),log10(UU(isr))); +[~,mv,bv,~] = fitline(log10(f(isr)),log10(VV(isr))); +[~,mw,bw,~] = fitline(log10(f(isr)),log10(WW(isr))); +power.u = mu; +power.v = mv; +power.w = mw; +power.uint = bu; +power.vint = bv; +power.wint = bw; + +% Save Spectra +ps.UU = UU; +ps.VV = VV; +ps.WW = WW; +ps.TT = TT; + +cs.UW = UW; +cs.VW = VW; +cs.TW = TW; + +end \ No newline at end of file From 028914b53f0e880fe2b19fe976092e706c258807 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:33:21 -0700 Subject: [PATCH 21/34] updates to postprocess_SWIFT --- Aquadopp/reprocess_AQH.m | 4 ++++ GeneralTools/concatSWIFT_offloadedSDcard.m | 10 +++++++-- GeneralTools/postprocess_SWIFT.m | 23 ++++++++++--------- Signature/SFdissipation.m | 26 +++++++++++----------- 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/Aquadopp/reprocess_AQH.m b/Aquadopp/reprocess_AQH.m index ae60932..f368b7d 100644 --- a/Aquadopp/reprocess_AQH.m +++ b/Aquadopp/reprocess_AQH.m @@ -97,10 +97,14 @@ else load([bfiles(iburst).folder '\' bfiles(iburst).name],... 'Vel','Cor','Amp','Pressure','pitch','roll','heading','time') + if exist('Vel','var') vel = Vel; cor = Cor; amp = Amp; press = Pressure'; + else + continue + end end % Skip burst if empty diff --git a/GeneralTools/concatSWIFT_offloadedSDcard.m b/GeneralTools/concatSWIFT_offloadedSDcard.m index 7a2f0b4..125ce0a 100644 --- a/GeneralTools/concatSWIFT_offloadedSDcard.m +++ b/GeneralTools/concatSWIFT_offloadedSDcard.m @@ -14,12 +14,18 @@ clear all, close all, clc +if ispc + slash = '\'; +else + slash = '/'; +end + plotflag = 1; % binary flag for plotting burstinterval = 12; % minutes between bursts burstlength = 512/60; % minutes of sampling during each burst -dirpath = './'%'~/Desktop/'; '~/Dropbox/SWIFT_v3.x/TestData/'; -sourcedir = ''%'SWIFT11_02Oct2014_SDcard';%'SWIFT15_Test_19Jun2014'%'SWIFTdata_27Apr2014' %'SWIFT15_LakeWA_Stereo_09May2014'; +dirpath = ['.' slash];%'~/Desktop/'; '~/Dropbox/SWIFT_v3.x/TestData/'; +sourcedir = '';%'SWIFT11_02Oct2014_SDcard';%'SWIFT15_Test_19Jun2014'%'SWIFTdata_27Apr2014' %'SWIFT15_LakeWA_Stereo_09May2014'; %% glob all processed files (per instrument) % and put these all in one directory (skip if already in one directory) diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index 67efc7f..49506d1 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -11,23 +11,23 @@ % K. Zeiden 07/2024 +%% User defined experiment directory (to be later converted to function inputs) + if ispc slash = '\'; else slash = '/'; end -%% User defined experiment directory (to be later converted to function inputs) - -expdir = ['S:' slash 'SEAFAC' slash 'June2024']; +expdir = ['S:' slash 'NORSE' slash '2023']; % Processing toggles -rpIMU = true; % Waves +rpIMU = false; % Waves rpSBG = false; % Waves -rpWXT = true; % MET +rpWXT = false; % MET rpY81 = false; % MET rpACS = false; % CT -rpSIG = false; % TKE +rpSIG = true; % TKE rpAQH = false; % TKE rpAQD = false; % TKE @@ -39,7 +39,7 @@ missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); -for im = 1:length(missions) +for im = 17 disp(['Post-processing ' missions(im).name]) @@ -57,13 +57,14 @@ disp('Create information structure ''sinfo''') sinfo.ID = SWIFT(1).ID; sinfo.CTdepth = SWIFT(1).CTdepth; + if isfield(SWIFT,'metheight') sinfo.metheight = SWIFT(1).metheight; + end if isfield(SWIFT,'signature') sinfo.type = 'V4'; else sinfo.type = 'V3'; end - SWIFT = rmfield(SWIFT,{'ID','CTdepth','metheight'}); disp('Saving new L1 product...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') end @@ -142,7 +143,7 @@ if rpSIG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) disp('Reprocessing Signature1000 data...') - plotburst = false; + plotburst = true; readraw = false; [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); else @@ -165,7 +166,7 @@ if rpAQH if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) disp('Reprocessing Aquadopp (AQH) data...') - readraw = true; + readraw = false; plotburst = false; [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); else @@ -173,7 +174,7 @@ end end - % Re-load L1 and L2 product and plot each for comparison + %% Re-load L1 and L2 product and plot each for comparison load([l1file.folder slash l1file.name],'SWIFT','sinfo'); SWIFTL1 = SWIFT; l2file = dir([missiondir slash '*L2.mat']); diff --git a/Signature/SFdissipation.m b/Signature/SFdissipation.m index 790eb84..b7702bb 100644 --- a/Signature/SFdissipation.m +++ b/Signature/SFdissipation.m @@ -125,14 +125,14 @@ end xN = ones(nfit,1); x = Ri(ifit); - x1 = Ri(ifit).^(2/3); - x3 = x1.^3; + x23 = Ri(ifit).^(2/3); + x3 = x23.^3; d = Di(ifit); derr = mean(Dierr(ifit),'omitnan'); %Best-fit power-law to the structure function % THIS IS WRONG!!!! - ilog = x1 > 0 & d > 0;% log(0) = -Inf - x1log = log10(x1(ilog)); + ilog = x23 > 0 & d > 0;% log(0) = -Inf + x1log = log10(x23(ilog)); dlog = log10(d(ilog)); xNlog = xN(ilog); % xlog = log10(x(ilog)); @@ -145,7 +145,7 @@ if strcmp(fittype,'cubic') % Fit structure function to D(z,r) = Br^2 + Ar^(2/3) + N - G = [x3(:) x1(:) xN(:)]; + G = [x3(:) x23(:) xN(:)]; Gg = (G'*G)\G'; m = Gg*d(:); B(ibin) = m(1); @@ -153,7 +153,7 @@ %Remove model shear term & fit Ar^(2/3) to residual (to get mspe) dmod = d-B(ibin)*x3; - G = [x1(:) xN(:)]; + G = [x23(:) xN(:)]; Gg = (G'*G)\G'; m = Gg*dmod(:); dm = G*m; @@ -168,8 +168,8 @@ Aerr(ibin) = merr(1); % update w/slope of residual structure function - ilog = x1 > 0 & dmod > 0;% log(0) = -Inf - x1log = log10(x1(ilog)); + ilog = x23 > 0 & dmod > 0;% log(0) = -Inf + x1log = log10(x23(ilog)); dlog = log10(dmod(ilog)); xNlog = xN(ilog); G = [x1log(:) xNlog(:)]; @@ -180,7 +180,7 @@ elseif strcmp(fittype,'linear') % Fit structure function to D(z,r) = Ar^(2/3) + N - G = [x1(:) xN(:)]; + G = [x23(:) xN(:)]; Gg = (G'*G)\G'; m = Gg*d(:); dm = G*m; @@ -194,10 +194,10 @@ elseif strcmp(fittype,'log') % Don't presume a slope - d = d(x1>0); - xN = xN(x1>0); - x1 = x1(x1>0); - x1log = log10(x1); + d = d(x23>0); + xN = xN(x23>0); + x23 = x23(x23>0); + x1log = log10(x23); dlog = log10(d); G = [x1log(:) xN(:)]; Gg = (G'*G)\G'; From 47fa215bcb8b1a21b0b26f91a731cb1401ba556f Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:19:37 -0700 Subject: [PATCH 22/34] updates --- GeneralTools/catSWIFT.m | 2 +- GeneralTools/postprocess_SWIFT.m | 2 +- Signature/SFdissipation.asv | 244 +++++++++++++++++++++++++++++++ Signature/SFdissipation.m | 19 ++- Signature/hannwinPSD2.m | 14 +- Signature/processSIGburst.m | 7 +- 6 files changed, 266 insertions(+), 22 deletions(-) create mode 100644 Signature/SFdissipation.asv diff --git a/GeneralTools/catSWIFT.m b/GeneralTools/catSWIFT.m index f8f0b8f..63487b8 100644 --- a/GeneralTools/catSWIFT.m +++ b/GeneralTools/catSWIFT.m @@ -283,7 +283,7 @@ % Echograms if isfield(SWIFT,'signature') - if isfield(SWIFT(1).signature,'echogram') + if isfield(SWIFT(floor(end/2)).signature,'echogram') swift.echoz = SWIFT(1).signature.echogram.z; nz = length(swift.echoz); swift.echo = NaN(nz,nt); diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index 49506d1..35326de 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -39,7 +39,7 @@ missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); -for im = 17 +for im = 14 disp(['Post-processing ' missions(im).name]) diff --git a/Signature/SFdissipation.asv b/Signature/SFdissipation.asv new file mode 100644 index 0000000..6bc6b06 --- /dev/null +++ b/Signature/SFdissipation.asv @@ -0,0 +1,244 @@ +function [eps,qual] = SFdissipation(w,z,rmin,rmax,nzfit,fittype,avgtype) +% This function applies Taylor cascade theory to estimate dissipation from +% the second order velocity structure function computed from vertical profiles +% of turbulent velocity (see Wiles et al. 2006). SFdissipation was +% formulated with data from the Nortek Signature 1000 ADCP operating in pulse-coherent +% (HR) mode, but can be applied to any ensemble of velocity profiles. +% +% in: w (or dW) nbin x nping ensemble of velocity profiles. Ensemble averaging +% occurs across the 'ping' dimension. Can alternatively +% input the velocity difference matrix (dW). +% z 1 x nbin +% rmin minimum separation distance allowed in the fit +% rmax maximum separation distance, assumed to be within the +% inerital subrange +% nzfit number of vertical bins to include in fit at each +% depth, e.g. for nzfit = 1 , fit all pairs with +% mean pair depth <= z0 +\- dz/2, i.e. vertical +% smoothing +% fittype either 'linear' or 'cubic', determines whether the +% structure function is fit to a theoretical curve which is +% linear or cubic in R^(2/3). The latter should be used if +% there is likely significant profile-scale shear in the profiles, +% such as surface waves (Scannell et al. 2017) +% 12/2022: Added 'log', which does the linear fit +% in log space instead. Assumes noise term is +% very low. +% avgtype either 'mean','logmean' or 'median', determines whether the +% mean of squares, mean of the log of squares, or median of squares +% is taken to determine the expected value of the squared velocity difference +% +% out: eps 1 x nbin profile of dissipation +% qual structure with metrics for evaluating quality of eps including: +% - mean square percent error of the fit (mspe), +% - propagated error of the fit (epserr), +% - ADCP error inferred from the SF intercept (N), +% - slope of the SF (slope), +% - wave term coefficient (B, if modified r^2 fit used). + +% K.Zeiden Summer/Fall 2022 + +% Return control to calling function/script if all NaN data +nz = length(z); +if ~any(~isnan(w(:))) + eps = NaN(1,nz); + qual.mspe = NaN(1,nz); + qual.slope = NaN(1,nz); + qual.epserr = NaN(1,nz); + qual.A = NaN(1,nz); + qual.B = NaN(1,nz); + qual.N = NaN(1,nz); + return +end + +% Matrices of all possible data pair separation distances (R), and +% corresponding mean vertical position (Z0) +z = z(:)'; +dz = median(diff(z)); +R = z-z'; +R = round(R,2); +[Z1,Z2] = meshgrid(z); +Z0 = (Z1+Z2)/2; + +% Matrices of all possible data pair velocity differences for each ping. +% Points greater than +/- 5 standard deviation are removed from each dist. +if ismatrix(w) + [nbin,~] = size(w); + if nbin ~= length(z) + w = w'; + [nbin,~] = size(w); + end + dW = repmat(w-mean(w,2,'omitnan'),1,1,nbin); + dW = permute(dW,[1 3 2])-permute(dW,[3 1 2]); + dW(abs(dW) > 5*std(dW,[],3,'omitnan')) = NaN; + elseif ndims(w) == 3 + dW = w; + [nbin,nbin2,~] = size(dW); + if nbin ~= nbin2 || nbin ~= length(z) + error('Check dimensions of ''dW''') + end + else + error('Check dimensions of ''w''') +end + +% Take mean (or median, or mean-of-the-logs) squared velocity difference to get D(z,r) +if strcmp(avgtype,'mean') + D = mean(dW.^2,3,'omitnan'); + elseif strcmp(avgtype,'logmean') + D = 10.^(mean(log10(dW.^2),3,'omitnan')); + elseif strcmp(avgtype,'median') + D = median(dW.^2,3,'omitnan'); + else + error('Average estimator must be ''mean'', ''logmean'' or ''median''.') +end + +%Standard Error on the mean +Derr = sqrt(var(dW.^2,[],3,'omitnan')./sum(~isnan(dW),3)); + +%Fit structure function to theoretical curve +Cv2 = 2.1; +eps = NaN(size(z)); +epserr = eps; +A = NaN(size(z)); +B = NaN(size(z)); +Aerr = NaN(size(z)); +N = NaN(size(z)); +mspe = NaN(size(z)); +slope = NaN(size(z)); +for ibin = 1:length(z) + + %Find points in z0 bin + iplot = Z0 >= z(ibin)- nzfit*dz/2 & Z0 <= z(ibin)+ nzfit*dz/2; + Di = D(iplot); + Dierr = Derr(iplot); + Ri = R(iplot); + [Ri,isort] = sort(Ri); + Di = Di(isort); + Dierr = Dierr(isort); + + %Select points within specified separation scale range + ifit = Ri <= rmax & Ri >= rmin; + nfit = sum(ifit); + if nfit < 3 % Must contain more than 3 points + continue + + end + xN = ones(nfit,1); + x1 = Ri(ifit); + x23 = Ri(ifit).^(2/3); + x2 = x23.^3; + d = Di(ifit); + derr = mean(Dierr(ifit),'omitnan'); + + %Best-fit power-law to the structure function % THIS IS WRONG!!!! + ilog = x23 > 0 & d > 0;% log(0) = -Inf + x1log = log10(x23(ilog)); + dlog = log10(d(ilog)); + xNlog = xN(ilog); + G = [x1log(:) xNlog(:)]; + Gg = (G'*G)\G'; + m = Gg*dlog(:); + slope(ibin) = m(1); + + %Fit Structure function to theoretical curves + if strcmp(fittype,'cubic') + + % Fit structure function to D(z,r) = Br^2 + Ar^(2/3) + N +<<<<<<< Updated upstream + G = [x3(:) x23(:) xN(:)]; +======= + G = [x2(:) x23(:) xN(:)]; +>>>>>>> Stashed changes + Gg = (G'*G)\G'; + m = Gg*d(:); + B(ibin) = m(1); + A(ibin) = m(2); + +<<<<<<< Updated upstream + %Remove model shear term & fit Ar^(2/3) to residual (to get mspe) + dmod = d-B(ibin)*x3; +======= + % Remove model shear term & fit Ar^(2/3) to residual (to get mspe) + dmod = d-B(ibin)*x2; +>>>>>>> Stashed changes + G = [x23(:) xN(:)]; + Gg = (G'*G)\G'; + m = Gg*dmod(:); + dm = G*m; + imse = abs(dm) > 10^(-8); + mspe(ibin) = mean(((dm(imse)-dmod(imse))./dm(imse)).^2); + if A(ibin) ~= m(1) + warning('new value for eps') + end +% A(ibin) = m(1); + N(ibin) = m(2); + merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); + Aerr(ibin) = merr(1); + + % update w/slope of residual structure function + ilog = x23 > 0 & dmod > 0;% log(0) = -Inf + x1log = log10(x23(ilog)); + dlog = log10(dmod(ilog)); + xNlog = xN(ilog); + G = [x1log(:) xNlog(:)]; + Gg = (G'*G)\G'; + m = Gg*dlog(:); + slope(ibin) = m(1); + + elseif strcmp(fittype,'linear') + + % Fit structure function to D(z,r) = Ar^(2/3) + N + G = [x23(:) xN(:)]; + Gg = (G'*G)\G'; + m = Gg*d(:); + dm = G*m; + imse = abs(dm) > 10^(-8); + mspe(ibin) = mean(((dm(imse)-d(imse))./dm(imse)).^2); + A(ibin) = m(1); + N(ibin) = m(2); + merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); + Aerr(ibin) = merr(1); + + elseif strcmp(fittype,'log') + + % Don't presume a slope + d = d(x23>0); + xN = xN(x23>0); + x23 = x23(x23>0); + x1log = log10(x23); + dlog = log10(d); + G = [x1log(:) xN(:)]; + Gg = (G'*G)\G'; + m = Gg*dlog(:); + dm = G*m; + imse = abs(dm) > 10^(-8); + mspe(ibin) = mean(((dm(imse)-d(imse))./dm(imse)).^2); + slope(ibin) = m(1); + A(ibin) = 10.^(m(2)); + merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); + Aerr(ibin) = merr(2); + + else + error('Fit type must be ''linear'', ''cubic'' or ''log''') + end + eps(ibin) = (A(ibin)./Cv2).^(3/2); + epserr(ibin) = Aerr(ibin)*(3/2)*eps(ibin)./A(ibin); + +end + +% Remove unphysical values +eps(A<0) = NaN; +epserr(A<0) = NaN; + +% Save quality metrics +qual.mspe = mspe; +qual.slope = slope; +qual.epserr = epserr; +qual.A = A; +qual.B = B; +qual.N = N; + +%%%%% End function + +end + diff --git a/Signature/SFdissipation.m b/Signature/SFdissipation.m index b7702bb..7fba9f7 100644 --- a/Signature/SFdissipation.m +++ b/Signature/SFdissipation.m @@ -124,18 +124,17 @@ end xN = ones(nfit,1); - x = Ri(ifit); + x1 = Ri(ifit); x23 = Ri(ifit).^(2/3); - x3 = x23.^3; + x2 = x23.^3; d = Di(ifit); derr = mean(Dierr(ifit),'omitnan'); %Best-fit power-law to the structure function % THIS IS WRONG!!!! - ilog = x23 > 0 & d > 0;% log(0) = -Inf - x1log = log10(x23(ilog)); + ilog = x1 > 0 & d > 0;% log(0) = -Inf + x1log = log10(x1(ilog)); dlog = log10(d(ilog)); xNlog = xN(ilog); -% xlog = log10(x(ilog)); G = [x1log(:) xNlog(:)]; Gg = (G'*G)\G'; m = Gg*dlog(:); @@ -145,14 +144,14 @@ if strcmp(fittype,'cubic') % Fit structure function to D(z,r) = Br^2 + Ar^(2/3) + N - G = [x3(:) x23(:) xN(:)]; + G = [x2(:) x23(:) xN(:)]; Gg = (G'*G)\G'; m = Gg*d(:); B(ibin) = m(1); A(ibin) = m(2); - %Remove model shear term & fit Ar^(2/3) to residual (to get mspe) - dmod = d-B(ibin)*x3; + % Remove model shear term & fit Ar^(2/3) to residual (to get mspe) + dmod = d-B(ibin)*x2; G = [x23(:) xN(:)]; Gg = (G'*G)\G'; m = Gg*dmod(:); @@ -168,8 +167,8 @@ Aerr(ibin) = merr(1); % update w/slope of residual structure function - ilog = x23 > 0 & dmod > 0;% log(0) = -Inf - x1log = log10(x23(ilog)); + ilog = x1 > 0 & dmod > 0;% log(0) = -Inf + x1log = log10(x1(ilog)); dlog = log10(dmod(ilog)); xNlog = xN(ilog); G = [x1log(:) xNlog(:)]; diff --git a/Signature/hannwinPSD2.m b/Signature/hannwinPSD2.m index 61f2f32..ae0cf02 100644 --- a/Signature/hannwinPSD2.m +++ b/Signature/hannwinPSD2.m @@ -59,13 +59,13 @@ elseif strcmp(norm,'par') PX = PX/(df*nwin^2); PXstd = PXstd/(df*nwin^2); - vard = mean(var(DX,[],2,'omitnan'),'omitnan'); - varPS = sum(PX,'omitnan').*df; - pvardiff = 100*(varPS-vard)/vard; - if abs(pvardiff) > 5 - warning('Parseval''s Theorom not satisifed to within 5%') - else - end + % vard = mean(var(DX,[],2,'omitnan'),'omitnan'); + % varPS = sum(PX,'omitnan').*df; + % pvardiff = 100*(varPS-vard)/vard; + % if abs(pvardiff) > 5 + % warning('Parseval''s Theorom not satisifed to within 5%') + % else + % end end % Confidence Intervals diff --git a/Signature/processSIGburst.m b/Signature/processSIGburst.m index 0fc828c..0619bde 100644 --- a/Signature/processSIGburst.m +++ b/Signature/processSIGburst.m @@ -55,19 +55,20 @@ end % 3) Estimate dissipation rate from velocity structure functions +wnf = w; ibad = ispike | repmat(badping,nbin,1); rmin = dz; rmax = 4*dz; nzfit = 1; -w(ibad) = NaN; +wnf(ibad) = NaN; wpeof(ibad) = NaN; wphp(ibad) = NaN; warning('off','all') % No filter, no analytic wave fit (D ~ r^{-2/3}) -[epsNF,qualNF] = SFdissipation(w,z,rmin,2*rmax,nzfit,'linear','mean'); +[epsNF,qualNF] = SFdissipation(wnf,z,rmin,2*rmax,nzfit,'linear','mean'); % Analytic wave fit (D ~ Ar^{-2/3} + Br^2) -[epsWV,qualWV] = SFdissipation(w,z,rmin,2*rmax,nzfit,'cubic','mean'); +[epsWV,qualWV] = SFdissipation(wnf,z,rmin,2*rmax,nzfit,'cubic','mean'); % EOF filter (D ~ r^{-2/3}) [epsEOF,qualEOF] = SFdissipation(wpeof,z,rmin,rmax,nzfit,'linear','mean'); % High-pass filter (D ~ r^{-2/3}) From 8c5f5d22046e52e98d55fac3e3dac4e8a43247d1 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:58:27 -0700 Subject: [PATCH 23/34] Update hannwinPSD2.m --- Signature/hannwinPSD2.m | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Signature/hannwinPSD2.m b/Signature/hannwinPSD2.m index ae0cf02..478123a 100644 --- a/Signature/hannwinPSD2.m +++ b/Signature/hannwinPSD2.m @@ -27,6 +27,10 @@ dataxi = datax(k1:k2); % Remove mean, trend and apply hanning window + inan = isnan(dataxi); + if sum(inan)/length(dataxi) > 0.2 + dataxi = NaN(size(dataxi));% NaN out entire window if too many NaNs + end dataxi(isnan(dataxi)) = mean(dataxi,'omitnan'); % Replace NaN w/mean dataxi = dataxi - mean(dataxi,'omitnan'); % Remove mean dataxi = detrend(dataxi); % Remove trend From cb9295513498e79e46b8c2f3ec58d3af46e05aa3 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:11:43 -0700 Subject: [PATCH 24/34] updates --- GeneralTools/postprocess_SWIFT.m | 10 +++++---- Signature/SFdissipation.m | 37 +++++++------------------------- Signature/catSIG.m | 30 +++++++++++++------------- Signature/hannwinPSD2.m | 8 +++---- Signature/processSIGburst.m | 34 ++--------------------------- Signature/reprocess_SIG.m | 2 +- 6 files changed, 36 insertions(+), 85 deletions(-) diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index 35326de..a956435 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,7 +19,7 @@ slash = '/'; end -expdir = ['S:' slash 'NORSE' slash '2023']; +expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'NorthMooring']; % Processing toggles rpIMU = false; % Waves @@ -34,12 +34,14 @@ % Plotting toggle plotL1L2 = true; -%% Loop through missions and reprocess +%% List of missions + cd(expdir) missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); -for im = 14 +%% Loop through missions and reprocess +for im = 1 disp(['Post-processing ' missions(im).name]) @@ -143,7 +145,7 @@ if rpSIG if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) disp('Reprocessing Signature1000 data...') - plotburst = true; + plotburst = false; readraw = false; [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); else diff --git a/Signature/SFdissipation.m b/Signature/SFdissipation.m index 7fba9f7..7954481 100644 --- a/Signature/SFdissipation.m +++ b/Signature/SFdissipation.m @@ -24,6 +24,8 @@ % 12/2022: Added 'log', which does the linear fit % in log space instead. Assumes noise term is % very low. +% 9/2024: removed 'log', and just compute the +% slope to know what it is % avgtype either 'mean','logmean' or 'median', determines whether the % mean of squares, mean of the log of squares, or median of squares % is taken to determine the expected value of the squared velocity difference @@ -92,10 +94,10 @@ error('Average estimator must be ''mean'', ''logmean'' or ''median''.') end -%Standard Error on the mean +% Standard Error on the mean Derr = sqrt(var(dW.^2,[],3,'omitnan')./sum(~isnan(dW),3)); -%Fit structure function to theoretical curve +% Fit structure function to theoretical curve Cv2 = 2.1; eps = NaN(size(z)); epserr = eps; @@ -107,7 +109,7 @@ slope = NaN(size(z)); for ibin = 1:length(z) - %Find points in z0 bin + % Find points in z0 bin iplot = Z0 >= z(ibin)- nzfit*dz/2 & Z0 <= z(ibin)+ nzfit*dz/2; Di = D(iplot); Dierr = Derr(iplot); @@ -116,7 +118,7 @@ Di = Di(isort); Dierr = Dierr(isort); - %Select points within specified separation scale range + % Select points within specified separation scale range ifit = Ri <= rmax & Ri >= rmin; nfit = sum(ifit); if nfit < 3 % Must contain more than 3 points @@ -130,7 +132,7 @@ d = Di(ifit); derr = mean(Dierr(ifit),'omitnan'); - %Best-fit power-law to the structure function % THIS IS WRONG!!!! + %Best-fit power-law to the structure function ilog = x1 > 0 & d > 0;% log(0) = -Inf x1log = log10(x1(ilog)); dlog = log10(d(ilog)); @@ -158,10 +160,6 @@ dm = G*m; imse = abs(dm) > 10^(-8); mspe(ibin) = mean(((dm(imse)-dmod(imse))./dm(imse)).^2); - if A(ibin) ~= m(1) - warning('new value for eps') - end -% A(ibin) = m(1); N(ibin) = m(2); merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); Aerr(ibin) = merr(1); @@ -190,27 +188,8 @@ merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); Aerr(ibin) = merr(1); - elseif strcmp(fittype,'log') - - % Don't presume a slope - d = d(x23>0); - xN = xN(x23>0); - x23 = x23(x23>0); - x1log = log10(x23); - dlog = log10(d); - G = [x1log(:) xN(:)]; - Gg = (G'*G)\G'; - m = Gg*dlog(:); - dm = G*m; - imse = abs(dm) > 10^(-8); - mspe(ibin) = mean(((dm(imse)-d(imse))./dm(imse)).^2); - slope(ibin) = m(1); - A(ibin) = 10.^(m(2)); - merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); - Aerr(ibin) = merr(2); - else - error('Fit type must be ''linear'', ''cubic'' or ''log''') + error('Fit type must be ''linear'' or ''cubic''') end eps(ibin) = (A(ibin)./Cv2).^(3/2); epserr(ibin) = Aerr(ibin)*(3/2)*eps(ibin)./A(ibin); diff --git a/Signature/catSIG.m b/Signature/catSIG.m index d53b89f..38416c5 100644 --- a/Signature/catSIG.m +++ b/Signature/catSIG.m @@ -37,7 +37,7 @@ sig.hrw = sig.hrcorr; sig.hrwvar = sig.hrcorr; sig.eps = sig.hrcorr; - sig.epsnf = sig.hrcorr; + sig.N = sig.hrcorr; sig.mspe = sig.hrcorr; sig.slope = sig.hrcorr; sig.pspike = sig.hrcorr; @@ -63,21 +63,22 @@ sig.avgvvar(:,it) = SIG(it).profile.vvar; sig.avgwvar(:,it) = SIG(it).profile.wvar; %HR - nz = length(SIG(it).HRprofile.w); - sig.hrcorr(1:nz,it) = SIG(it).HRprofile.QC.hrcorr; - sig.hramp(1:nz,it) = SIG(it).HRprofile.QC.hramp; - sig.pspike(1:nz,it) = SIG(it).HRprofile.QC.pspike; + + sig.hrcorr(1:nzhr,it) = SIG(it).HRprofile.QC.hrcorr(1:nzhr); + sig.hramp(1:nzhr,it) = SIG(it).HRprofile.QC.hramp(1:nzhr); + sig.pspike(1:nzhr,it) = SIG(it).HRprofile.QC.pspike(1:nzhr); sig.pbadping(it) = SIG(it).HRprofile.QC.pbadping; - sig.hrw(1:nz,it) = SIG(it).HRprofile.w; - sig.hrwvar(1:nz,it) = SIG(it).HRprofile.wvar; - sig.eps(1:nz,it) = SIG(it).HRprofile.eps; - sig.epsnf(1:nz,it) = SIG(it).HRprofile.QC.epsNF; - sig.mspe(1:nz,it) = SIG(it).HRprofile.QC.qualEOF.mspe; - sig.slope(1:nz,it) = SIG(it).HRprofile.QC.qualEOF.slope; + sig.hrw(1:nzhr,it) = SIG(it).HRprofile.w(1:nzhr); + sig.hrwvar(1:nzhr,it) = SIG(it).HRprofile.wvar(1:nzhr); + sig.eps(1:nzhr,it) = SIG(it).HRprofile.eps(1:nzhr); + sig.N(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.N(1:nzhr); + sig.mspe(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.mspe(1:nzhr); + sig.slope(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.slope(1:nzhr); - sig.wpeofmag(1:nz,it) = SIG(it).HRprofile.QC.wpeofmag; - sig.eofs(1:nz,1:nz,it) = SIG(it).HRprofile.QC.eofs; - sig.eofsvar(1:nz,it) = SIG(it).HRprofile.QC.eofsvar; + sig.wpeofmag(1:nzhr,it) = SIG(it).HRprofile.QC.wpeofmag(1:nzhr); + sig.eofs(1:nzhr,1:nzhr,it) = SIG(it).HRprofile.QC.eofs(1:nzhr,1:nzhr,:); + sig.eofvar(1:nzhr,it) = SIG(it).HRprofile.QC.eofsvar(1:nzhr); + sig.eofamp() % Motion sig.pitch(it) = SIG(it).motion.pitch; @@ -104,7 +105,6 @@ sig.hrw(:,badburst) = []; sig.hrwvar(:,badburst) = []; sig.eps(:,badburst) = []; - sig.epsnf(:,badburst) = []; sig.mspe(:,badburst) = []; sig.slope(:,badburst) = []; sig.pspike(:,badburst) = []; diff --git a/Signature/hannwinPSD2.m b/Signature/hannwinPSD2.m index 478123a..b11eee3 100644 --- a/Signature/hannwinPSD2.m +++ b/Signature/hannwinPSD2.m @@ -28,10 +28,10 @@ % Remove mean, trend and apply hanning window inan = isnan(dataxi); - if sum(inan)/length(dataxi) > 0.2 - dataxi = NaN(size(dataxi));% NaN out entire window if too many NaNs - end - dataxi(isnan(dataxi)) = mean(dataxi,'omitnan'); % Replace NaN w/mean + % if sum(inan)/length(dataxi) > 0.2 + % dataxi = NaN(size(dataxi));% NaN out entire window if too many NaNs + % end + dataxi(inan) = mean(dataxi,'omitnan'); % Replace NaN w/mean dataxi = dataxi - mean(dataxi,'omitnan'); % Remove mean dataxi = detrend(dataxi); % Remove trend dataxi = h.*dataxi'; % Window using normalized hanning window diff --git a/Signature/processSIGburst.m b/Signature/processSIGburst.m index 0619bde..314ea63 100644 --- a/Signature/processSIGburst.m +++ b/Signature/processSIGburst.m @@ -88,7 +88,8 @@ HRprofile.QC.qualHP = qualHP; HRprofile.QC.qualEOF = qualEOF; HRprofile.QC.eofs = eofs; -HRprofile.QC.eofsvar = eofvar; +HRprofile.QC.eofvar = eofvar; +HRprofile.QC.eofamp = eofamp'; HRprofile.QC.wpeofmag = std(wpeof,[],2,'omitnan')'; HRprofile.QC.hrcorr = mean(corr,2,'omitnan'); HRprofile.QC.hramp = mean(amp,2,'omitnan'); @@ -133,37 +134,6 @@ xlabel('Ping #') drawnow - - % Velocity and Dissipation Profiles -% fh(2) = figure('color','w'); -% clear b s -% MP = get(0,'monitorposition'); -% set(gcf,'outerposition',MP(1,:).*[1 1 0.5 1]); -% subplot(1,2,1) -% b(1) = errorbar(HRprofile.w,HRprofile.z,sqrt(HRprofile.wvar)./nping,'horizontal'); -% hold on -% set(b,'LineWidth',2) -% plot([0 0],[0 20],'k--') -% xlabel('w [m/s]'); -% title('Velocity') -% set(gca,'Ydir','reverse') -% ylim([0 6]) -% xlim([-0.1 0.1]) -% set(gca,'YAxisLocation','right') -% subplot(1,2,2) -% s(1) = semilogx(epsNF,z,'k','LineWidth',2); -% hold on -% s(2) = semilogx(epsEOF,z,'r','LineWidth',2); -% s(3) = semilogx(epsHP,z,'m','LineWidth',2); -% s(4) = semilogx(epsWV,z,'c','LineWidth',2); -% xlim(10.^([-9 -3])) -% ylim([0 6]) -% legend(s,'No-filter','EOF Filter','HP Filter','Analytic Filter','Location','southeast') -% title('Dissipation') -% xlabel('\epsilon [W/Kg]'),ylabel('z [m]') -% set(gca,'Ydir','reverse') -% set(gca,'YAxisLocation','right') -% drawnow else fh = []; end diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index f275e99..25051a9 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -459,7 +459,7 @@ % End burst loop end -%% Clean up and save +%% Clean up % NaN out SWIFT sig fields which were not matched to bursts if ~isempty(fieldnames(SWIFT)) From 5fb5e8c086fa1c0fc0f15176c9dc8c013006146d Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Fri, 13 Sep 2024 05:56:13 -0700 Subject: [PATCH 25/34] Update catSIG.m --- Signature/catSIG.m | 498 +++++++++++++++++++++++---------------------- 1 file changed, 251 insertions(+), 247 deletions(-) diff --git a/Signature/catSIG.m b/Signature/catSIG.m index 38416c5..d90c1b8 100644 --- a/Signature/catSIG.m +++ b/Signature/catSIG.m @@ -17,259 +17,263 @@ end end -if isfield(SIG,'time') - sig.time = [SIG.time]; - nt = length(sig.time); - sig.avgz = SIG(round(end/2)).profile.z'; - nzavg = length(sig.avgz); - sig.avgcorr = NaN(nzavg,nt); - sig.avgamp = sig.avgcorr; - sig.avgu = sig.avgcorr; - sig.avgv = sig.avgcorr; - sig.avgw = sig.avgcorr; - sig.avguvar = sig.avgcorr; - sig.avgvvar = sig.avgcorr; - sig.avgwvar = sig.avgcorr; - sig.hrz = SIG(round(end/2)).HRprofile.z; - nzhr = length(sig.hrz); - sig.hrcorr = NaN(nzhr,nt); - sig.hramp = sig.hrcorr; - sig.hrw = sig.hrcorr; - sig.hrwvar = sig.hrcorr; - sig.eps = sig.hrcorr; - sig.N = sig.hrcorr; - sig.mspe = sig.hrcorr; - sig.slope = sig.hrcorr; - sig.pspike = sig.hrcorr; - sig.pbadping = NaN(1,nt); - sig.wpeofmag = sig.hrcorr; - sig.eofs = NaN(nzhr,nzhr,nt); - sig.eofsvar = sig.hrcorr; - sig.pitch = NaN(1,nt); - sig.roll = NaN(1,nt); - sig.head = NaN(1,nt); - sig.pitchvar = NaN(1,nt); - sig.rollvar = NaN(1,nt); - sig.headvar = NaN(1,nt); +% Check to make sure isn't empty +if ~isfield(SIG,'time') + sig = []; + warning('SIG empty...') + return +end - for it = 1:length(sig.time) - %Broadband - sig.avgamp(:,it) = SIG(it).profile.QC.uamp; - sig.avgcorr(:,it) = SIG(it).profile.QC.ucorr; - sig.avgu(:,it) = SIG(it).profile.u; - sig.avgv(:,it) = SIG(it).profile.v; - sig.avgw(:,it) = SIG(it).profile.w; - sig.avguvar(:,it) = SIG(it).profile.uvar; - sig.avgvvar(:,it) = SIG(it).profile.vvar; - sig.avgwvar(:,it) = SIG(it).profile.wvar; - %HR - - sig.hrcorr(1:nzhr,it) = SIG(it).HRprofile.QC.hrcorr(1:nzhr); - sig.hramp(1:nzhr,it) = SIG(it).HRprofile.QC.hramp(1:nzhr); - sig.pspike(1:nzhr,it) = SIG(it).HRprofile.QC.pspike(1:nzhr); - sig.pbadping(it) = SIG(it).HRprofile.QC.pbadping; - sig.hrw(1:nzhr,it) = SIG(it).HRprofile.w(1:nzhr); - sig.hrwvar(1:nzhr,it) = SIG(it).HRprofile.wvar(1:nzhr); - sig.eps(1:nzhr,it) = SIG(it).HRprofile.eps(1:nzhr); - sig.N(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.N(1:nzhr); - sig.mspe(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.mspe(1:nzhr); - sig.slope(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.slope(1:nzhr); - - sig.wpeofmag(1:nzhr,it) = SIG(it).HRprofile.QC.wpeofmag(1:nzhr); - sig.eofs(1:nzhr,1:nzhr,it) = SIG(it).HRprofile.QC.eofs(1:nzhr,1:nzhr,:); - sig.eofvar(1:nzhr,it) = SIG(it).HRprofile.QC.eofsvar(1:nzhr); - sig.eofamp() +% Preallocate +sig.time = [SIG.time]; +nt = length(sig.time); +sig.avgz = SIG(round(end/2)).profile.z'; +nzavg = length(sig.avgz); +sig.avgcorr = NaN(nzavg,nt); +sig.avgamp = sig.avgcorr; +sig.avgu = sig.avgcorr; +sig.avgv = sig.avgcorr; +sig.avgw = sig.avgcorr; +sig.avguvar = sig.avgcorr; +sig.avgvvar = sig.avgcorr; +sig.avgwvar = sig.avgcorr; +sig.hrz = SIG(round(end/2)).HRprofile.z; +nzhr = length(sig.hrz); +sig.hrcorr = NaN(nzhr,nt); +sig.hramp = sig.hrcorr; +sig.hrw = sig.hrcorr; +sig.hrwvar = sig.hrcorr; +sig.eps = sig.hrcorr; +sig.N = sig.hrcorr; +sig.mspe = sig.hrcorr; +sig.slope = sig.hrcorr; +sig.pspike = sig.hrcorr; +sig.pbadping = NaN(1,nt); +sig.wpeofmag = sig.hrcorr; +sig.eofs = NaN(nzhr,nzhr,nt); +sig.eofvar = sig.hrcorr; +sig.pitch = NaN(1,nt); +sig.roll = NaN(1,nt); +sig.head = NaN(1,nt); +sig.pitchvar = NaN(1,nt); +sig.rollvar = NaN(1,nt); +sig.headvar = NaN(1,nt); - % Motion - sig.pitch(it) = SIG(it).motion.pitch; - sig.roll(it) = SIG(it).motion.roll; - sig.head(it) = SIG(it).motion.head; - sig.pitchvar(it) = SIG(it).motion.pitchvar; - sig.rollvar(it) = SIG(it).motion.rollvar; - sig.headvar(it) = SIG(it).motion.headvar; - end +% Loop through bursts and structurize +for it = 1:length(sig.time) - %QC - badburst = [SIG.badburst]; - if QCsig && sum(badburst) < length(sig.time) - sig.avgcorr(:,badburst) = []; - sig.avgamp(:,badburst) = []; - sig.avgu(:,badburst) = []; - sig.avgv(:,badburst) = []; - sig.avgw(:,badburst) = []; - sig.avguvar(:,badburst) = []; - sig.avgvvar(:,badburst) = []; - sig.avgwvar(:,badburst) = []; - sig.hrcorr(:,badburst) = []; - sig.hramp(:,badburst) = []; - sig.hrw(:,badburst) = []; - sig.hrwvar(:,badburst) = []; - sig.eps(:,badburst) = []; - sig.mspe(:,badburst) = []; - sig.slope(:,badburst) = []; - sig.pspike(:,badburst) = []; - sig.pbadping(badburst) = []; - sig.time(badburst) = []; - sig.wpeofmag(:,badburst) = []; - sig.eofs(:,:,badburst) = []; - sig.eofsvar(:,badburst) = []; - sig.pitch(badburst) = SIG(badburst).motion.pitch; - sig.roll(badburst) = SIG(badburst).motion.roll; - sig.head(badburst) = SIG(badburst).motion.head; - sig.pitchvar(badburst) = SIG(badburst).motion.pitchvar; - sig.rollvar(badburst) = SIG(badburst).motion.rollvar; - sig.headvar(badburst) = SIG(badburst).motion.headvar; - else - sig.badburst = badburst; - end + % Broadband + sig.avgamp(:,it) = SIG(it).profile.QC.uamp; + sig.avgcorr(:,it) = SIG(it).profile.QC.ucorr; + sig.avgu(:,it) = SIG(it).profile.u; + sig.avgv(:,it) = SIG(it).profile.v; + sig.avgw(:,it) = SIG(it).profile.w; + sig.avguvar(:,it) = SIG(it).profile.uvar; + sig.avgvvar(:,it) = SIG(it).profile.vvar; + sig.avgwvar(:,it) = SIG(it).profile.wvar; - % Plot - if plotsig && length(sig.time)>1 + % HR + sig.hrcorr(1:nzhr,it) = SIG(it).HRprofile.QC.hrcorr(1:nzhr); + sig.hramp(1:nzhr,it) = SIG(it).HRprofile.QC.hramp(1:nzhr); + sig.pspike(1:nzhr,it) = SIG(it).HRprofile.QC.pspike(1:nzhr); + sig.pbadping(it) = SIG(it).HRprofile.QC.pbadping; + sig.hrw(1:nzhr,it) = SIG(it).HRprofile.w(1:nzhr); + sig.hrwvar(1:nzhr,it) = SIG(it).HRprofile.wvar(1:nzhr); + sig.eps(1:nzhr,it) = SIG(it).HRprofile.eps(1:nzhr); + sig.N(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.N(1:nzhr); + sig.mspe(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.mspe(1:nzhr); + sig.slope(1:nzhr,it) = SIG(it).HRprofile.QC.qualEOF.slope(1:nzhr); + sig.wpeofmag(1:nzhr,it) = SIG(it).HRprofile.QC.wpeofmag(1:nzhr); + sig.eofs(1:nzhr,1:nzhr,it) = SIG(it).HRprofile.QC.eofs(1:nzhr,1:nzhr,:); + sig.eofvar(1:nzhr,it) = SIG(it).HRprofile.QC.eofvar(1:nzhr); - figure('color','w') - MP = get(0,'monitorposition'); - set(gcf,'outerposition',MP(1,:).*[1 1 1 1]); - clear b + % Motion + sig.pitch(it) = SIG(it).motion.pitch; + sig.roll(it) = SIG(it).motion.roll; + sig.head(it) = SIG(it).motion.head; + sig.pitchvar(it) = SIG(it).motion.pitchvar; + sig.rollvar(it) = SIG(it).motion.rollvar; + sig.headvar(it) = SIG(it).motion.headvar; +end + +% QC +badburst = [SIG.badburst]; +if QCsig && sum(badburst) < length(sig.time) + sig.avgcorr(:,badburst) = []; + sig.avgamp(:,badburst) = []; + sig.avgu(:,badburst) = []; + sig.avgv(:,badburst) = []; + sig.avgw(:,badburst) = []; + sig.avguvar(:,badburst) = []; + sig.avgvvar(:,badburst) = []; + sig.avgwvar(:,badburst) = []; + sig.hrcorr(:,badburst) = []; + sig.hramp(:,badburst) = []; + sig.hrw(:,badburst) = []; + sig.hrwvar(:,badburst) = []; + sig.eps(:,badburst) = []; + sig.mspe(:,badburst) = []; + sig.slope(:,badburst) = []; + sig.pspike(:,badburst) = []; + sig.pbadping(badburst) = []; + sig.time(badburst) = []; + sig.wpeofmag(:,badburst) = []; + sig.eofs(:,:,badburst) = []; + sig.eofvar(:,badburst) = []; + sig.pitch(badburst) = SIG(badburst).motion.pitch; + sig.roll(badburst) = SIG(badburst).motion.roll; + sig.head(badburst) = SIG(badburst).motion.head; + sig.pitchvar(badburst) = SIG(badburst).motion.pitchvar; + sig.rollvar(badburst) = SIG(badburst).motion.rollvar; + sig.headvar(badburst) = SIG(badburst).motion.headvar; +else + sig.badburst = badburst; +end - % Broadband - h(1) = subplot(4,4,1); - pcolor(sig.time,-sig.avgz,sig.avgcorr);shading flat - caxis([50 100]); - ylabel('Depth (m)');cmocean('thermal');title('Correlation') - c = colorbar;c.Label.String = '[%]'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - h(2) = subplot(4,4,2); - pcolor(sig.time,-sig.avgz,sig.avgamp);shading flat - caxis([60 160]); - ylabel('Depth (m)');cmocean('haline');title('Amplitude') - c = colorbar;c.Label.String = '[counts]'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - h(5) = subplot(4,4,5); - pcolor(sig.time,-sig.avgz,sig.avgu);shading flat - caxis([-0.25 0.25]); - ylabel('Depth (m)');cmocean('balance');title('U') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - h(9) = subplot(4,4,9); - pcolor(sig.time,-sig.avgz,sig.avgv);shading flat - caxis([-0.25 0.25]); - ylabel('Depth (m)');cmocean('balance');title('V') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - h(13) = subplot(4,4,13); - pcolor(sig.time,-sig.avgz,sig.avgw);shading flat - caxis([-0.25 0.25]); - ylabel('Depth (m)');cmocean('balance');title('W') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - h(6) = subplot(4,4,6); - pcolor(sig.time,-sig.avgz,sqrt(sig.avguvar));shading flat - caxis([0 0.25]); - ylabel('Depth (m)');title('\sigma_U') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - h(10) = subplot(4,4,10); - pcolor(sig.time,-sig.avgz,sqrt(sig.avgvvar));shading flat - caxis([0 0.25]); - ylabel('Depth (m)');title('\sigma_V') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - h(14) = subplot(4,4,14); - pcolor(sig.time,-sig.avgz,sqrt(sig.avgwvar));shading flat - caxis([0 0.25]); - ylabel('Depth (m)');title('\sigma_W') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') - % HR - h(3) = subplot(4,4,3);% HR Correlation - pcolor(sig.time,-sig.hrz,sig.hrcorr);shading flat - caxis([50 100]); - ylabel('Depth (m)');cmocean('thermal');title('HR Correlation') - c = colorbar;c.Label.String = '[%]'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - h(7) = subplot(4,4,7);% Vertical Velocity - pcolor(sig.time,-sig.hrz,sig.hrw);shading flat - caxis([-0.05 0.05]); - ylabel('Depth (m)');cmocean('balance');title('W') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - h(11) = subplot(4,4,11);% Velocity Variance - pcolor(sig.time,-sig.hrz,sqrt(sig.hrwvar));shading flat - caxis([0 0.5]); - ylabel('Depth (m)');title('\sigma_W') - c = colorbar;c.Label.String = 'ms^{-1}'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - h(15) = subplot(4,4,15);% Percent Spikes - pcolor(sig.time,-sig.hrz,100*sig.pspike);shading flat - caxis([0 25]);colormap(gca,lansey(25)) - ylabel('Depth (m)');title('Spike Percent') - c = colorbar;c.Label.String = 'P [%]'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - h(4) = subplot(4,4,4);% HR Amplitude - pcolor(sig.time,-sig.hrz,sig.hramp);shading flat - caxis([60 160]); - ylabel('Depth (m)');cmocean('haline');title('HR Amplitude') - c = colorbar;c.Label.String = '[counts]'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - h(8) = subplot(4,4,8);% Dissipation Rate - pcolor(sig.time,-sig.hrz,log10(sig.eps));shading flat - caxis([-8 -5]);colormap(gca,'jet') - ylabel('Depth (m)');title('Dissipation Rate') - c = colorbar;c.Label.String = 'log_{10}(m^3s^{-2})'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - h(12) = subplot(4,4,12);% MSPE of SF r^(2/3) fit - pcolor(sig.time,-sig.hrz,100*sqrt(sig.mspe));shading flat - caxis([0 25]);colormap(gca,lansey(25)) - ylabel('Depth (m)');title('MSPE') - c = colorbar;c.Label.String = '[%]'; - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - h(16) = subplot(4,4,16);% Best-fit slope to SF - pcolor(sig.time,-sig.hrz,sig.slope);shading flat - caxis([0 2*2/3]); - ylabel('Depth (m)');title('SF Slope') - c = colorbar;c.Label.String = 'D \propto r^n';cmocean('curl') - xlim([min(sig.time) max(sig.time)]) - datetick('x','KeepLimits') - drawnow - lw = h(1).Position([3 4]); - ax = [2 4] + 4*(0:3)'; - for iax = 1:numel(ax) - set(h(ax(iax)),'YTickLabel',[],'YLabel',[]) - h(ax(iax)).Position = h(ax(iax)).Position + [-0.02 0 0 0]; - end - ax = [3 4] + 4*(0:3)'; - for iax = 1:numel(ax) - h(ax(iax)).Position = h(ax(iax)).Position + [0.025 0 0 0]; - end - for iax = 1:16 - h(iax).Position([3 4]) = lw; - end +% Plot +if plotsig && length(sig.time)>1 + + figure('color','w') + MP = get(0,'monitorposition'); + set(gcf,'outerposition',MP(1,:).*[1 1 1 1]); + clear b + + % Broadband + h(1) = subplot(4,4,1); + pcolor(sig.time,-sig.avgz,sig.avgcorr);shading flat + caxis([50 100]); + ylabel('Depth (m)');cmocean('thermal');title('Correlation') + c = colorbar;c.Label.String = '[%]'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + h(2) = subplot(4,4,2); + pcolor(sig.time,-sig.avgz,sig.avgamp);shading flat + caxis([60 160]); + ylabel('Depth (m)');cmocean('haline');title('Amplitude') + c = colorbar;c.Label.String = '[counts]'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + h(5) = subplot(4,4,5); + pcolor(sig.time,-sig.avgz,sig.avgu);shading flat + caxis([-0.25 0.25]); + ylabel('Depth (m)');cmocean('balance');title('U') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + h(9) = subplot(4,4,9); + pcolor(sig.time,-sig.avgz,sig.avgv);shading flat + caxis([-0.25 0.25]); + ylabel('Depth (m)');cmocean('balance');title('V') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + h(13) = subplot(4,4,13); + pcolor(sig.time,-sig.avgz,sig.avgw);shading flat + caxis([-0.25 0.25]); + ylabel('Depth (m)');cmocean('balance');title('W') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + h(6) = subplot(4,4,6); + pcolor(sig.time,-sig.avgz,sqrt(sig.avguvar));shading flat + caxis([0 0.25]); + ylabel('Depth (m)');title('\sigma_U') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + h(10) = subplot(4,4,10); + pcolor(sig.time,-sig.avgz,sqrt(sig.avgvvar));shading flat + caxis([0 0.25]); + ylabel('Depth (m)');title('\sigma_V') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + h(14) = subplot(4,4,14); + pcolor(sig.time,-sig.avgz,sqrt(sig.avgwvar));shading flat + caxis([0 0.25]); + ylabel('Depth (m)');title('\sigma_W') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + hold on; plot(xlim,-max(sig.hrz)*[1 1],'--k') + % HR + h(3) = subplot(4,4,3);% HR Correlation + pcolor(sig.time,-sig.hrz,sig.hrcorr);shading flat + caxis([50 100]); + ylabel('Depth (m)');cmocean('thermal');title('HR Correlation') + c = colorbar;c.Label.String = '[%]'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + h(7) = subplot(4,4,7);% Vertical Velocity + pcolor(sig.time,-sig.hrz,sig.hrw);shading flat + caxis([-0.05 0.05]); + ylabel('Depth (m)');cmocean('balance');title('W') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + h(11) = subplot(4,4,11);% Velocity Variance + pcolor(sig.time,-sig.hrz,sqrt(sig.hrwvar));shading flat + caxis([0 0.5]); + ylabel('Depth (m)');title('\sigma_W') + c = colorbar;c.Label.String = 'ms^{-1}'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + h(15) = subplot(4,4,15);% Percent Spikes + pcolor(sig.time,-sig.hrz,100*sig.pspike);shading flat + caxis([0 25]);colormap(gca,lansey(25)) + ylabel('Depth (m)');title('Spike Percent') + c = colorbar;c.Label.String = 'P [%]'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + h(4) = subplot(4,4,4);% HR Amplitude + pcolor(sig.time,-sig.hrz,sig.hramp);shading flat + caxis([60 160]); + ylabel('Depth (m)');cmocean('haline');title('HR Amplitude') + c = colorbar;c.Label.String = '[counts]'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + h(8) = subplot(4,4,8);% Dissipation Rate + pcolor(sig.time,-sig.hrz,log10(sig.eps));shading flat + caxis([-8 -5]);colormap(gca,'jet') + ylabel('Depth (m)');title('Dissipation Rate') + c = colorbar;c.Label.String = 'log_{10}(m^3s^{-2})'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + h(12) = subplot(4,4,12);% MSPE of SF r^(2/3) fit + pcolor(sig.time,-sig.hrz,100*sqrt(sig.mspe));shading flat + caxis([0 25]);colormap(gca,lansey(25)) + ylabel('Depth (m)');title('MSPE') + c = colorbar;c.Label.String = '[%]'; + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + h(16) = subplot(4,4,16);% Best-fit slope to SF + pcolor(sig.time,-sig.hrz,sig.slope);shading flat + caxis([0 2*2/3]); + ylabel('Depth (m)');title('SF Slope') + c = colorbar;c.Label.String = 'D \propto r^n';cmocean('curl') + xlim([min(sig.time) max(sig.time)]) + datetick('x','KeepLimits') + drawnow + lw = h(1).Position([3 4]); + ax = [2 4] + 4*(0:3)'; + for iax = 1:numel(ax) + set(h(ax(iax)),'YTickLabel',[],'YLabel',[]) + h(ax(iax)).Position = h(ax(iax)).Position + [-0.02 0 0 0]; + end + ax = [3 4] + 4*(0:3)'; + for iax = 1:numel(ax) + h(ax(iax)).Position = h(ax(iax)).Position + [0.025 0 0 0]; + end + for iax = 1:16 + h(iax).Position([3 4]) = lw; end -else - sig = []; - warning('SIG empty...') +end end From 34e96d09357df9aa38fd1d10898f95820d52d6a1 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Wed, 18 Sep 2024 08:26:41 -0700 Subject: [PATCH 26/34] Update catSWIFT.m --- GeneralTools/catSWIFT.m | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GeneralTools/catSWIFT.m b/GeneralTools/catSWIFT.m index 63487b8..eee87d8 100644 --- a/GeneralTools/catSWIFT.m +++ b/GeneralTools/catSWIFT.m @@ -212,8 +212,10 @@ % [swift.dirwavepower(:,:,it),~,~,~,~,~,~,~] = SWIFTdirectionalspectra(SWIFT(it),0); % end - % Wind + if isfield(SWIFT,'windustar') + swift.windustar = [SWIFT.windustar]; + end if isfield(SWIFT,'windspd') swift.windu = [SWIFT.windspd]; else From 9041ec90011b4c659ab3c9f3ca76cf8ca1539d33 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Mon, 30 Sep 2024 07:13:48 -0700 Subject: [PATCH 27/34] updates --- GeneralTools/postprocess_SWIFT.m | 24 ++- Signature/SFdissipation.asv | 244 ------------------------------- Signature/readSWIFTv4_SIG.m | 2 +- Waves/GPSandIMUwaves.m | 18 +-- Waves/GPSwaves.m | 24 +-- Waves/XYZwaves.m | 18 +-- Winds/reprocess_Y81.m | 40 +++-- 7 files changed, 74 insertions(+), 296 deletions(-) delete mode 100644 Signature/SFdissipation.asv diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/postprocess_SWIFT.m index a956435..857a94e 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/postprocess_SWIFT.m @@ -19,15 +19,15 @@ slash = '/'; end -expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'NorthMooring']; +expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'SouthMooring']; % Processing toggles -rpIMU = false; % Waves +rpIMU = true; % Waves rpSBG = false; % Waves rpWXT = false; % MET rpY81 = false; % MET rpACS = false; % CT -rpSIG = true; % TKE +rpSIG = false; % TKE rpAQH = false; % TKE rpAQD = false; % TKE @@ -41,13 +41,19 @@ missions = missions([missions.isdir]); %% Loop through missions and reprocess -for im = 1 - - disp(['Post-processing ' missions(im).name]) +for im = 1:length(missions) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) + diaryfile = [missions(im).name '_postprocess.txt']; + if exist(diaryfile,'file') + delete(diaryfile); + end + diary(diaryfile) + + disp(['Post-processing ' missions(im).name]) + %% Locate L1 product, skip if does not exist. % Else create 'sinfo' and modify L1 product. l1file = dir([missiondir slash '*L1.mat']); @@ -67,7 +73,7 @@ else sinfo.type = 'V3'; end - disp('Saving new L1 product...') + disp('Updating L1 product sinfo...') save([l1file.folder slash l1file.name],'SWIFT','sinfo') end % One time, will remove later @@ -84,7 +90,7 @@ if rpIMU if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) disp('Reprocessing IMU data...') - calctype = 'IMUandGPS'; + calctype = 'GPS'; filtertype = 'RC'; saveraw = false; interpf = false; @@ -195,6 +201,8 @@ print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end + +diary off end diff --git a/Signature/SFdissipation.asv b/Signature/SFdissipation.asv deleted file mode 100644 index 6bc6b06..0000000 --- a/Signature/SFdissipation.asv +++ /dev/null @@ -1,244 +0,0 @@ -function [eps,qual] = SFdissipation(w,z,rmin,rmax,nzfit,fittype,avgtype) -% This function applies Taylor cascade theory to estimate dissipation from -% the second order velocity structure function computed from vertical profiles -% of turbulent velocity (see Wiles et al. 2006). SFdissipation was -% formulated with data from the Nortek Signature 1000 ADCP operating in pulse-coherent -% (HR) mode, but can be applied to any ensemble of velocity profiles. -% -% in: w (or dW) nbin x nping ensemble of velocity profiles. Ensemble averaging -% occurs across the 'ping' dimension. Can alternatively -% input the velocity difference matrix (dW). -% z 1 x nbin -% rmin minimum separation distance allowed in the fit -% rmax maximum separation distance, assumed to be within the -% inerital subrange -% nzfit number of vertical bins to include in fit at each -% depth, e.g. for nzfit = 1 , fit all pairs with -% mean pair depth <= z0 +\- dz/2, i.e. vertical -% smoothing -% fittype either 'linear' or 'cubic', determines whether the -% structure function is fit to a theoretical curve which is -% linear or cubic in R^(2/3). The latter should be used if -% there is likely significant profile-scale shear in the profiles, -% such as surface waves (Scannell et al. 2017) -% 12/2022: Added 'log', which does the linear fit -% in log space instead. Assumes noise term is -% very low. -% avgtype either 'mean','logmean' or 'median', determines whether the -% mean of squares, mean of the log of squares, or median of squares -% is taken to determine the expected value of the squared velocity difference -% -% out: eps 1 x nbin profile of dissipation -% qual structure with metrics for evaluating quality of eps including: -% - mean square percent error of the fit (mspe), -% - propagated error of the fit (epserr), -% - ADCP error inferred from the SF intercept (N), -% - slope of the SF (slope), -% - wave term coefficient (B, if modified r^2 fit used). - -% K.Zeiden Summer/Fall 2022 - -% Return control to calling function/script if all NaN data -nz = length(z); -if ~any(~isnan(w(:))) - eps = NaN(1,nz); - qual.mspe = NaN(1,nz); - qual.slope = NaN(1,nz); - qual.epserr = NaN(1,nz); - qual.A = NaN(1,nz); - qual.B = NaN(1,nz); - qual.N = NaN(1,nz); - return -end - -% Matrices of all possible data pair separation distances (R), and -% corresponding mean vertical position (Z0) -z = z(:)'; -dz = median(diff(z)); -R = z-z'; -R = round(R,2); -[Z1,Z2] = meshgrid(z); -Z0 = (Z1+Z2)/2; - -% Matrices of all possible data pair velocity differences for each ping. -% Points greater than +/- 5 standard deviation are removed from each dist. -if ismatrix(w) - [nbin,~] = size(w); - if nbin ~= length(z) - w = w'; - [nbin,~] = size(w); - end - dW = repmat(w-mean(w,2,'omitnan'),1,1,nbin); - dW = permute(dW,[1 3 2])-permute(dW,[3 1 2]); - dW(abs(dW) > 5*std(dW,[],3,'omitnan')) = NaN; - elseif ndims(w) == 3 - dW = w; - [nbin,nbin2,~] = size(dW); - if nbin ~= nbin2 || nbin ~= length(z) - error('Check dimensions of ''dW''') - end - else - error('Check dimensions of ''w''') -end - -% Take mean (or median, or mean-of-the-logs) squared velocity difference to get D(z,r) -if strcmp(avgtype,'mean') - D = mean(dW.^2,3,'omitnan'); - elseif strcmp(avgtype,'logmean') - D = 10.^(mean(log10(dW.^2),3,'omitnan')); - elseif strcmp(avgtype,'median') - D = median(dW.^2,3,'omitnan'); - else - error('Average estimator must be ''mean'', ''logmean'' or ''median''.') -end - -%Standard Error on the mean -Derr = sqrt(var(dW.^2,[],3,'omitnan')./sum(~isnan(dW),3)); - -%Fit structure function to theoretical curve -Cv2 = 2.1; -eps = NaN(size(z)); -epserr = eps; -A = NaN(size(z)); -B = NaN(size(z)); -Aerr = NaN(size(z)); -N = NaN(size(z)); -mspe = NaN(size(z)); -slope = NaN(size(z)); -for ibin = 1:length(z) - - %Find points in z0 bin - iplot = Z0 >= z(ibin)- nzfit*dz/2 & Z0 <= z(ibin)+ nzfit*dz/2; - Di = D(iplot); - Dierr = Derr(iplot); - Ri = R(iplot); - [Ri,isort] = sort(Ri); - Di = Di(isort); - Dierr = Dierr(isort); - - %Select points within specified separation scale range - ifit = Ri <= rmax & Ri >= rmin; - nfit = sum(ifit); - if nfit < 3 % Must contain more than 3 points - continue - - end - xN = ones(nfit,1); - x1 = Ri(ifit); - x23 = Ri(ifit).^(2/3); - x2 = x23.^3; - d = Di(ifit); - derr = mean(Dierr(ifit),'omitnan'); - - %Best-fit power-law to the structure function % THIS IS WRONG!!!! - ilog = x23 > 0 & d > 0;% log(0) = -Inf - x1log = log10(x23(ilog)); - dlog = log10(d(ilog)); - xNlog = xN(ilog); - G = [x1log(:) xNlog(:)]; - Gg = (G'*G)\G'; - m = Gg*dlog(:); - slope(ibin) = m(1); - - %Fit Structure function to theoretical curves - if strcmp(fittype,'cubic') - - % Fit structure function to D(z,r) = Br^2 + Ar^(2/3) + N -<<<<<<< Updated upstream - G = [x3(:) x23(:) xN(:)]; -======= - G = [x2(:) x23(:) xN(:)]; ->>>>>>> Stashed changes - Gg = (G'*G)\G'; - m = Gg*d(:); - B(ibin) = m(1); - A(ibin) = m(2); - -<<<<<<< Updated upstream - %Remove model shear term & fit Ar^(2/3) to residual (to get mspe) - dmod = d-B(ibin)*x3; -======= - % Remove model shear term & fit Ar^(2/3) to residual (to get mspe) - dmod = d-B(ibin)*x2; ->>>>>>> Stashed changes - G = [x23(:) xN(:)]; - Gg = (G'*G)\G'; - m = Gg*dmod(:); - dm = G*m; - imse = abs(dm) > 10^(-8); - mspe(ibin) = mean(((dm(imse)-dmod(imse))./dm(imse)).^2); - if A(ibin) ~= m(1) - warning('new value for eps') - end -% A(ibin) = m(1); - N(ibin) = m(2); - merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); - Aerr(ibin) = merr(1); - - % update w/slope of residual structure function - ilog = x23 > 0 & dmod > 0;% log(0) = -Inf - x1log = log10(x23(ilog)); - dlog = log10(dmod(ilog)); - xNlog = xN(ilog); - G = [x1log(:) xNlog(:)]; - Gg = (G'*G)\G'; - m = Gg*dlog(:); - slope(ibin) = m(1); - - elseif strcmp(fittype,'linear') - - % Fit structure function to D(z,r) = Ar^(2/3) + N - G = [x23(:) xN(:)]; - Gg = (G'*G)\G'; - m = Gg*d(:); - dm = G*m; - imse = abs(dm) > 10^(-8); - mspe(ibin) = mean(((dm(imse)-d(imse))./dm(imse)).^2); - A(ibin) = m(1); - N(ibin) = m(2); - merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); - Aerr(ibin) = merr(1); - - elseif strcmp(fittype,'log') - - % Don't presume a slope - d = d(x23>0); - xN = xN(x23>0); - x23 = x23(x23>0); - x1log = log10(x23); - dlog = log10(d); - G = [x1log(:) xN(:)]; - Gg = (G'*G)\G'; - m = Gg*dlog(:); - dm = G*m; - imse = abs(dm) > 10^(-8); - mspe(ibin) = mean(((dm(imse)-d(imse))./dm(imse)).^2); - slope(ibin) = m(1); - A(ibin) = 10.^(m(2)); - merr = sqrt(diag(derr.^2*((G'*G)^(-1)))); - Aerr(ibin) = merr(2); - - else - error('Fit type must be ''linear'', ''cubic'' or ''log''') - end - eps(ibin) = (A(ibin)./Cv2).^(3/2); - epserr(ibin) = Aerr(ibin)*(3/2)*eps(ibin)./A(ibin); - -end - -% Remove unphysical values -eps(A<0) = NaN; -epserr(A<0) = NaN; - -% Save quality metrics -qual.mspe = mspe; -qual.slope = slope; -qual.epserr = epserr; -qual.A = A; -qual.B = B; -qual.N = N; - -%%%%% End function - -end - diff --git a/Signature/readSWIFTv4_SIG.m b/Signature/readSWIFTv4_SIG.m index 20dbc23..abde126 100644 --- a/Signature/readSWIFTv4_SIG.m +++ b/Signature/readSWIFTv4_SIG.m @@ -121,7 +121,7 @@ burst.Accelerometer(ensemblecount_burst,1) = fread(fid,1,'int16','ieee-le'); burst.Accelerometer(ensemblecount_burst,2) = fread(fid,1,'int16','ieee-le'); burst.Accelerometer(ensemblecount_burst,3) = fread(fid,1,'int16','ieee-le'); - AmbiquityVelocity(ensemblecount_burst) = fread(fid,1,'uint16','ieee-le'); + AmbiuityVelocity(ensemblecount_burst) = fread(fid,1,'uint16','ieee-le'); DataSetDescription(ensemblecount_burst) = fread(fid,1,'uint16','ieee-le'); TransmitEnergy(ensemblecount_burst) = fread(fid,1,'uint16','ieee-le'); VelocityScaling_burst(ensemblecount_burst) = fread(fid,1,'int8','ieee-le'); diff --git a/Waves/GPSandIMUwaves.m b/Waves/GPSandIMUwaves.m index b20bd55..6b11a06 100644 --- a/Waves/GPSandIMUwaves.m +++ b/Waves/GPSandIMUwaves.m @@ -369,15 +369,15 @@ %% prune high frequency results -E( f > maxf ) = []; -dir( f > maxf ) = []; -spread( f > maxf ) = []; -a1( f > maxf ) = []; -b1( f > maxf ) = []; -a2( f > maxf ) = []; -b2( f > maxf ) = []; -check( f > maxf ) = []; -f( f > maxf ) = []; +% E( f > maxf ) = []; +% dir( f > maxf ) = []; +% spread( f > maxf ) = []; +% a1( f > maxf ) = []; +% b1( f > maxf ) = []; +% a2( f > maxf ) = []; +% b2( f > maxf ) = []; +% check( f > maxf ) = []; +% f( f > maxf ) = []; else % if not enough points or sufficent sampling rate or data, give 9999 diff --git a/Waves/GPSwaves.m b/Waves/GPSwaves.m index dba8c49..b602de9 100644 --- a/Waves/GPSwaves.m +++ b/Waves/GPSwaves.m @@ -364,18 +364,18 @@ %% prune high frequency results -E( f > maxf ) = []; -Exx( f > maxf ) = []; -Eyy( f > maxf ) = []; -Ezz( f > maxf ) = []; -dir( f > maxf ) = []; -spread( f > maxf ) = []; -a1( f > maxf ) = []; -b1( f > maxf ) = []; -a2( f > maxf ) = []; -b2( f > maxf ) = []; -check( f > maxf ) = []; -f( f > maxf ) = []; +% E( f > maxf ) = []; +% Exx( f > maxf ) = []; +% Eyy( f > maxf ) = []; +% Ezz( f > maxf ) = []; +% dir( f > maxf ) = []; +% spread( f > maxf ) = []; +% a1( f > maxf ) = []; +% b1( f > maxf ) = []; +% a2( f > maxf ) = []; +% b2( f > maxf ) = []; +% check( f > maxf ) = []; +% f( f > maxf ) = []; else % if not enough points or sufficent sampling rate or data, give 9999 diff --git a/Waves/XYZwaves.m b/Waves/XYZwaves.m index 56cfb70..20d8a05 100644 --- a/Waves/XYZwaves.m +++ b/Waves/XYZwaves.m @@ -231,15 +231,15 @@ %% prune high frequency results -E( f > maxf ) = []; -dir( f > maxf ) = []; -spread( f > maxf ) = []; -a1( f > maxf ) = []; -b1( f > maxf ) = []; -a2( f > maxf ) = []; -b2( f > maxf ) = []; -check( f > maxf ) = []; -f( f > maxf ) = []; +% E( f > maxf ) = []; +% dir( f > maxf ) = []; +% spread( f > maxf ) = []; +% a1( f > maxf ) = []; +% b1( f > maxf ) = []; +% a2( f > maxf ) = []; +% b2( f > maxf ) = []; +% check( f > maxf ) = []; +% f( f > maxf ) = []; else % if not enough points or sufficent sampling rate or data, give 9999 diff --git a/Winds/reprocess_Y81.m b/Winds/reprocess_Y81.m index ce9f3c3..1ba3baf 100644 --- a/Winds/reprocess_Y81.m +++ b/Winds/reprocess_Y81.m @@ -40,28 +40,40 @@ disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) + % if bfiles(iburst).bytes > 1e5 + % RMYdata = importdata([bfiles(iburst).folder slash bfiles(iburst).name]); + % uvw = RMYdata.data(:,1:3); + % temp = RMYdata.data(:,4); + % else + % uvw = NaN(1000,3); + % temp = NaN(1000,1); + % end + % windspd = mean((uvw(:,1).^2 + uvw(:,2).^2 + uvw(:,3).^2).^.5); + % u = uvw(:,1); + % v = uvw(:,2); + % w = uvw(:,3); + + % Read raw data if bfiles(iburst).bytes > 1e5 - RMYdata = importdata([bfiles(iburst).folder slash bfiles(iburst).name]); - uvw = RMYdata.data(:,1:3); - temp = RMYdata.data(:,4); - else - uvw = NaN(1000,3); - temp = NaN(1000,1); + [u,v,w,temp,~] = readSWIFT_Y81([bfiles(iburst).folder slash bfiles(iburst).name]); + windspd = mean( (u.^2 + v.^2 + w.^2).^.5 ); + else + u = NaN(1000,1); + v = u;w = u;temp = u; end - windspd = mean((uvw(:,1).^2 + uvw(:,2).^2 + uvw(:,3).^2).^.5,'omitnan'); - u = uvw(:,1); - v = uvw(:,2); - w = uvw(:,3); - % Recalculate friction velocity z = sinfo.metheight; fs = 10; [ustar,~,~,~,~,~,~,~,~,~] = inertialdissipation(u,v,w,temp,z,fs); % Find matching time - time = datenum(bfiles(iburst).name(13:21)) + str2double(bfiles(iburst).name(23:24))./24 ... - + str2double(bfiles(iburst).name(26:27))./(24*6); + day = bfiles(iburst).name(13:21); + hour = bfiles(iburst).name(23:24); + mint = bfiles(iburst).name(26:27); + % time = datenum(bfiles(iburst).name(13:21)) + str2double(bfiles(iburst).name(23:24))./24 ... + % + str2double(bfiles(iburst).name(26:27))./(24*6); + time = datenum(day)+datenum(0,0,0,str2double(hour),(str2double(mint)-1)*12,0); [tdiff,tindex] = min(abs([SWIFT.time]-time)); % Replace wind speed, NaN out wind direction and replace ustar @@ -71,6 +83,8 @@ if ustar ~= 9999 SWIFT(tindex).windustar = ustar; end + else + disp(['No time match, dt = ' num2str(tdiff*24*60) ' min...']) end end From 08ec11ea5155d44d8a5374186d34c2b7b1dd5f02 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:59:15 -0700 Subject: [PATCH 28/34] Updates to post processing --- .../Archive/compileSWIFT_SBD_test2.m | 336 ++++++++++++ GeneralTools/{ => KZ_Processing}/NaNstruct.m | 0 GeneralTools/{ => KZ_Processing}/NaNstructR.m | 0 GeneralTools/{ => KZ_Processing}/catSWIFT.m | 41 +- GeneralTools/KZ_Processing/compileSWIFT.asv | 491 ++++++++++++++++++ GeneralTools/KZ_Processing/compileSWIFT.m | 491 ++++++++++++++++++ GeneralTools/KZ_Processing/createSBD.asv | 200 +++++++ GeneralTools/KZ_Processing/createSBD.m | 126 +++++ .../postprocessSWIFT.asv} | 33 +- GeneralTools/KZ_Processing/postprocessSWIFT.m | 206 ++++++++ .../compileSWIFT_SBDservertelemetry.m | 13 +- GeneralTools/concatSWIFT_offloadedSDcard.m | 15 +- GeneralTools/findoutliersSWIFT.asv | 94 ---- GeneralTools/plotSWIFT.m | 34 +- GeneralTools/pruneSWIFT.m | 58 +++ GeneralTools/qcSWIFT.m | 61 +++ GeneralTools/readSWIFT_SBD.m | 47 +- SBG/reprocess_SBG.m | 10 +- Winds/inertialdissipation.m | 423 ++++++++------- Winds/readSWIFT_Y81.m | 2 + Winds/reprocess_Y81.m | 24 +- Winds/sonicdissipation.m | 35 +- 22 files changed, 2303 insertions(+), 437 deletions(-) create mode 100644 GeneralTools/KZ_Processing/Archive/compileSWIFT_SBD_test2.m rename GeneralTools/{ => KZ_Processing}/NaNstruct.m (100%) rename GeneralTools/{ => KZ_Processing}/NaNstructR.m (100%) rename GeneralTools/{ => KZ_Processing}/catSWIFT.m (86%) create mode 100644 GeneralTools/KZ_Processing/compileSWIFT.asv create mode 100644 GeneralTools/KZ_Processing/compileSWIFT.m create mode 100644 GeneralTools/KZ_Processing/createSBD.asv create mode 100644 GeneralTools/KZ_Processing/createSBD.m rename GeneralTools/{postprocess_SWIFT.m => KZ_Processing/postprocessSWIFT.asv} (90%) create mode 100644 GeneralTools/KZ_Processing/postprocessSWIFT.m delete mode 100644 GeneralTools/findoutliersSWIFT.asv create mode 100644 GeneralTools/pruneSWIFT.m create mode 100644 GeneralTools/qcSWIFT.m diff --git a/GeneralTools/KZ_Processing/Archive/compileSWIFT_SBD_test2.m b/GeneralTools/KZ_Processing/Archive/compileSWIFT_SBD_test2.m new file mode 100644 index 0000000..cca5cd0 --- /dev/null +++ b/GeneralTools/KZ_Processing/Archive/compileSWIFT_SBD_test2.m @@ -0,0 +1,336 @@ +% Aggregate and read all SWIFT sbd data after concatenating the +% offload from SD card or the sbd email attachments. SBD files should be in the +% 'ConcatProcessed' subfolder of any given SWIFT mission folder. +% Based on compileSWIFT_SBDservertelemetry.m +% K. Zeiden 10/01/2024 + +%% Experiment directory +expdir = 'S:\SEAFAC\June2024'; + +%% Set Parameters +if ispc + slash = '\'; +else + slash = '/'; +end + +% Processing parameters +plotflag = false; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) +fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights +fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. + +% QC Parameters +minwaveheight = 0;% minimum wave height in data screening +minsalinity = 0;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) +maxdriftspd = 5;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that +maxwindspd = 30;% m/s for malfunctioning Airmars +minairtemp = -20;% min airtemp +disp('-------------------------------------') +disp('QC settings:') +disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) +disp(['Minimum salinity: ' num2str(minsalinity) ' m']) +disp(['Maximum drift speed: ' num2str(maxdriftspd) ' m']) +disp(['Maximum wind speed: ' num2str(maxwindspd) ' m']) +disp(['Minimum air temp: ' num2str(minairtemp) ' m']) +disp('-------------------------------------') + +%% List missions + +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +% for im = 1:length(missions) +im = 1; + +missiondir = [missions(im).folder slash missions(im).name]; +sname = missions(im).name; + +disp(['Compiling ' sname ' SWIFT structure:']) + +% Compile list of sbd burst files +if exist([missiondir slash 'ConcatProcessed'],'dir') + blist = dir([missiondir slash 'ConcatProcessed' slash '*.sbd']); +else + disp('Processed files have not been concatenated...') + blist = []; +end +nburst = length(blist); + +% Initialize badburst flag +badburst = false(1,nburst); + +% Keep track of the number of fields in each burst +nfields = NaN(1,nburst); + +%% Loop through all SBD burst files, load, QC, and save in SWIFT structure + +for iburst = 1:nburst + + disp(['Burst file ''' blist(iburst).name '''']) + + [oneSWIFT,voltage]= readSWIFT_SBD([blist(iburst).folder slash blist(iburst).name],0); + + if voltage == 9999 % error flag from SBD message + badburst(iburst) = true; + end + + if ~isempty(voltage) + oneSWIFT.battery = voltage; + else + oneSWIFT.battery = NaN; + end + + if isempty(oneSWIFT.lat) || isempty(oneSWIFT.lon) + oneSWIFT.lat = NaN; + oneSWIFT.lon = NaN; + end + + %%% Time stamp %%% + % Take the time from the filename, even when there is time from + % the airmar (because of parsing errors). For telemetry, this + % is the telemtry time (at the end of the burst). For offloaded data, + % this the concat file name (from the start of the burst). + if blist(iburst).name(6) == 'S' % SWIFT v3 and v4 + nameoffset = 14; + day = blist(iburst).name(nameoffset + (1:2)); + month = blist(iburst).name(nameoffset + (3:5)); + year = blist(iburst).name(nameoffset + (6:9)); + hr = blist(iburst).name(nameoffset + (11:12)); + minute = blist(iburst).name(nameoffset + (13:14)); + sec = blist(iburst).name(nameoffset + (15:16)); + oneSWIFT.time = datenum([day ' ' month ' ' year ' ' hr ':' minute ':' sec]); + micro = false; + elseif blist(iburst).name(6) == 'm' % microSWIFT + % Use the time embedded within the payload 50 or 51 or 52 of the + % SBD file, which is the time at the end of the burst of raw data. + nameoffset = 20; + micro = true; + else + nameoffset = 0; + oneSWIFT.time = NaN; + micro = false; + end + + %%% Remove bad Airmar data %%% + if isfield(oneSWIFT,'airtemp') + if oneSWIFT.airtemp == 0.0 || oneSWIFT.airtemp < minairtemp || oneSWIFT.airtemp > 50 + oneSWIFT.airtemp = NaN; + oneSWIFT.windspd = NaN; + end + end + + %%% Extrapolate missing low frequencies of wave energy spectra %% + % while requiring energy at lowest frequenc to be zero + % not neccessary if post-processing for raw displacements + % less necessary after Oct 2017 rev of onboard processing with improved RC filter + if fixspectra && isfield(oneSWIFT,'wavespectra') + notzero = find(oneSWIFT.wavespectra.energy ~= 0 & oneSWIFT.wavespectra.freq > 0.04); + tobereplaced = find(oneSWIFT.wavespectra.energy == 0 & oneSWIFT.wavespectra.freq > 0.04); + if length(notzero) > 10 + E = interp1([0.04; oneSWIFT.wavespectra.freq(notzero)],[0; oneSWIFT.wavespectra.energy(notzero)],oneSWIFT.wavespectra.freq); + oneSWIFT.wavespectra.energy(tobereplaced) = E(tobereplaced); + df = median(diff(oneSWIFT.wavespectra.freq)); + oneSWIFT.sigwaveheight = 4*sqrt(sum(oneSWIFT.wavespectra.energy,'omitnan')*df); + end + end + + %%% Remove wave histograms %%% + if isfield(oneSWIFT,'wavehistogram') + oneSWIFT = rmfield(oneSWIFT,'wavehistogram'); + end + + %%% Screen the bad data (usually out of the water) %%% + disp('=================================') + % No data + if isempty(oneSWIFT.lon) || isempty(oneSWIFT.lat) || isempty(oneSWIFT.time) + badburst(iburst) = true; + disp('No position or timestamp.') + end + % No position + if oneSWIFT.lon == 0 || ~isnumeric(oneSWIFT.lon) + badburst(iburst) = true; + disp('No position.') + end + % Waves too small + if isfield(oneSWIFT,'sigwaveheight') + if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 + badburst(iburst) = true; + disp('Waves too small, removing burst.') + end + end + % Salinity too small + if isfield(oneSWIFT,'salinity') %&& ~micro + if all(oneSWIFT.salinity < minsalinity) % & all(~isnan(oneSWIFT.salinity)), + badburst(iburst) = true; + disp('Salinity too low, removing burst.') + end + end + % Drift speed limit + if isfield(oneSWIFT,'driftspd') + if oneSWIFT.driftspd > maxdriftspd + badburst(iburst) = true; + disp('Speed too fast, removing burst.') + end + end + disp('=================================') + + %%% Increment main structure %%% + onefields = fieldnames(oneSWIFT); + nfields(iburst) = length(onefields); + for ifield = 1:nfields(iburst) + SWIFT(iburst).(onefields{ifield}) = oneSWIFT.(onefields{ifield}); + end + +% End burst loop +end + +%% Apply QC + +SWIFT(badburst) = []; + +%% Sort final structure + +[~,tinds] = sort([SWIFT.time]); +SWIFT = SWIFT(tinds); + +%% Look for outliers of position + +if fixpositions + [cleanlon,cloni] = filloutliers([SWIFT.lon],'linear'); + [cleanlat,clati] = filloutliers([SWIFT.lat],'linear'); + if cloni == clati + for ci = find(cloni) + SWIFT(ci).lon = cleanlon(ci); + SWIFT(ci).lat = cleanlat(ci); + end + disp([num2str(sum(cloni)) ' positions filled that were outliers']) + end +end + +%% Recalculate drift (note that wind slip, which is 1%, is not removed) +% Drift speed is included from the Airmar results, +% but that sensor is not always available or included +% (so simpler to just calculate it from differencing positions). + +if length(SWIFT) > 3 + + time = [SWIFT.time];%[time tinds ] = sort(time); + lat = [SWIFT.lat]; %lat = lat(tinds); + lon = [SWIFT.lon]; %lon = lon(tinds); + dlondt = gradient(lon,time); % deg per day + dxdt = deg2km(dlondt,6371*cosd(mean(lat,'omitnan'))) .* 1000 ./ ( 24*3600 ); % m/s + dlatdt = gradient(lat,time); % deg per day + dydt = deg2km(dlatdt) .* 1000 ./ ( 24*3600 ); % m/s + dxdt(isinf(dxdt)) = NaN; + dydt(isinf(dydt)) = NaN; + speed = sqrt(dxdt.^2 + dydt.^2); % m/s + direction = -180 ./ 3.14 .* atan2(dydt,dxdt); % cartesian direction [deg] + direction = direction + 90; % rotate from eastward = 0 to northward = 0 + direction( direction<0) = direction( direction<0 ) + 360; % make quadrant II 270->360 instead of -90->0 + + for si = 1:length(SWIFT) + if si == 1 || si == length(SWIFT) + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + else + SWIFT(si).driftspd = speed(si); + SWIFT(si).driftdirT = direction(si); + end + end + + % % remove last burst, if big change in direction (suggests recovery by ship) + % dirchange = abs( SWIFT( length(SWIFT) - 2).driftdirT - SWIFT( length(SWIFT) - 1).driftdirT ); + % if dirchange > 90, + % disp('removing last burst, suspect includes ship recovery') + % SWIFT( length(SWIFT) - 1).driftdirT = NaN; + % SWIFT( length(SWIFT) - 1).driftspd = NaN; + % SWIFT( length(SWIFT) ) = []; + % battery( length(SWIFT) ) = []; + % end + +elseif length(SWIFT) <= 3 + for si = 1:length(SWIFT) + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + end +end + +% Quality control by removing drift results associated with large time gaps +if length([SWIFT.time]) > 1 + dt = gradient([SWIFT.time]); + for si = 1:length(SWIFT) + if dt(si) > 1/12 % 1/12 of day is two hours + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + end + end +end + +% Quality control drift speeds too fast (prob on deck) with new drift spd +if length([SWIFT.time]) > 1 && isfield(SWIFT(1),'driftspd') + toofast = [SWIFT.driftspd] > maxdriftspd; + SWIFT( toofast ) =[]; +end + +%% Quality control wind speeds +if length([SWIFT.time]) > 1 && isfield(SWIFT(1),'windspd') + for si = 1:length(SWIFT) + if SWIFT(si).windspd > maxwindspd + SWIFT(si).windspd = NaN; + SWIFT(si).winddirT = NaN; + SWIFT(si).winddirR = NaN; + end + end +end + +%% Sort the microSWIFT onboard processing, using the battery voltage as a flag +% only applies to v1 microSWIFTs from 2022 + +IMU = find([SWIFT.battery]==0); +GPS = find([SWIFT.battery]==1); + +if ~isempty(IMU), SWIFT_IMU = SWIFT(IMU); end +if ~isempty(GPS), SWIFT_GPS = SWIFT(GPS); end + +%% Fill in the ID field from mission directory name if NaN + +idnan = isnan([SWIFT.ID]); +ID = sname(strfind(sname,'SWIFT')+ (5:6)); + +for si = find(idnan) + SWIFT(si).ID = ID; +end + +%% Save L1 file + +if micro + save([missiondir slash 'micro' sname '_L1.mat'],'SWIFT*') +elseif length([SWIFT.time]) > 1 + save([missiondir slash sname '_L1.mat'],'SWIFT') +end + +%% Plot + +if plotflag + + plotSWIFT(SWIFT) + + wd = pwd; + wdi = find(wd == slash,1,'last'); + wd = wd((wdi+1):length(wd)); + + % battery plot + if any(~isnan([SWIFT.battery])) + figure(7), clf, + plot([SWIFT.time],[SWIFT.battery],'kx','linewidth',3) + datetick, grid + ylabel('Voltage') + print('-dpng',[wd '_battery.png']) + end + +end + +close all + +% End mission loop +% end diff --git a/GeneralTools/NaNstruct.m b/GeneralTools/KZ_Processing/NaNstruct.m similarity index 100% rename from GeneralTools/NaNstruct.m rename to GeneralTools/KZ_Processing/NaNstruct.m diff --git a/GeneralTools/NaNstructR.m b/GeneralTools/KZ_Processing/NaNstructR.m similarity index 100% rename from GeneralTools/NaNstructR.m rename to GeneralTools/KZ_Processing/NaNstructR.m diff --git a/GeneralTools/catSWIFT.m b/GeneralTools/KZ_Processing/catSWIFT.m similarity index 86% rename from GeneralTools/catSWIFT.m rename to GeneralTools/KZ_Processing/catSWIFT.m index eee87d8..1e6791a 100644 --- a/GeneralTools/catSWIFT.m +++ b/GeneralTools/KZ_Processing/catSWIFT.m @@ -51,32 +51,18 @@ end % Radiometer -if isfield(SWIFT,'radiometertemp1mean')% This is the brightness (target) temperature - nrad = length(SWIFT(1).radiometertemp1mean); - swift.radtemp1 = reshape([SWIFT.radiometertemp1mean],nrad,length(SWIFT)); - if size(swift.radtemp1,2)~= nt - swift.radtemp1 = swift.radtemp1'; +if isfield(SWIFT,'infraredtempmean')% This is the brightness (target) temperature + nrad = length(SWIFT(1).infraredtempmean); + swift.IRtemp = reshape([SWIFT.infraredtempmean],nrad,length(SWIFT)); + if size(swift.IRtemp,2)~= nt + swift.IRtemp = swift.IRtemp'; end end -if isfield(SWIFT,'radiometerrad1')% This is derived from brightness temp using stefan boltzman - nrad = length(SWIFT(1).radiometerrad1); - swift.rad1 = reshape([SWIFT.radiometerrad1],nrad,length(SWIFT)); - if size(swift.rad1,2)~= nt - swift.rad1 = swift.rad1'; - end -end -if isfield(SWIFT,'radiometertemp2mean')% This is the jacket temperature - nrad = length(SWIFT(1).radiometertemp2mean); - swift.radtemp2 = reshape([SWIFT.radiometertemp2mean],nrad,length(SWIFT)); - if size(swift.radtemp2,2)~= nt - swift.radtemp2 = swift.radtemp2'; - end -end -if isfield(SWIFT,'radiometerrad1') - nrad = length(SWIFT(1).radiometerrad2);% This is derived from jacket temp using stefan boltzman - swift.rad2 = reshape([SWIFT.radiometerrad2],nrad,length(SWIFT)); - if size(swift.rad2,2)~= nt - swift.rad2 = swift.rad2'; +if isfield(SWIFT,'ambienttempmean')% This is the jacket temperature + nrad = length(SWIFT(1).ambienttempmean); + swift.AMBtemp = reshape([SWIFT.ambienttempmean],nrad,length(SWIFT)); + if size(swift.AMBtemp,2)~= nt + swift.AMBtemp = swift.AMBtemp'; end end @@ -97,7 +83,7 @@ % Relative Velocity if isfield(SWIFT,'signature') && isstruct(SWIFT(1).signature.profile) - swift.depth = SWIFT(end).signature.profile.z'; + swift.depth = SWIFT(round(end/2)).signature.profile.z'; nz = length(swift.depth); swift.relu = NaN(nz,nt); swift.relv = NaN(nz,nt); @@ -178,6 +164,7 @@ swift.wavepower(swift.wavepower<0) = 0; wavefreq = median(swift.wavefreq,2,'omitnan'); wavepower = NaN(length(wavefreq),nt); + % Interpolate to median frequency for it = 1:nt ireal = ~isnan(swift.wavepower(:,it)) & swift.wavepower(:,it)~=0; @@ -197,12 +184,14 @@ om = 2*pi./swift.wavepeakT; k = om.^2./9.81; swift.waveustokes = (swift.wavesigH./4).^2.*om.*k; + % Re-calculate peak wave period (via centroid method) wavepower = swift.wavepower; wavefreq = swift.wavefreq; wavevar = sum(wavepower,1,'omitnan'); waveweight = sum(wavepower.*repmat(wavefreq,1,size(wavepower,2)),1,'omitnan'); swift.wavepeakT = 1./(waveweight./wavevar); + % Directional Wave Spectra % [~,swift.wavedir,~,~,~,~,~,~] = SWIFTdirectionalspectra(SWIFT(1),0); % ndir = length(swift.wavedir); @@ -255,7 +244,7 @@ % TKE Dissipation Rate and HR vertical velocity if isfield(SWIFT,'signature') - swift.surfz = SWIFT(end).signature.HRprofile.z'; + swift.surfz = SWIFT(round(end/2)).signature.HRprofile.z'; nz = length(swift.surfz); swift.surftke = NaN(nz,nt); for it = 1:nt diff --git a/GeneralTools/KZ_Processing/compileSWIFT.asv b/GeneralTools/KZ_Processing/compileSWIFT.asv new file mode 100644 index 0000000..093748a --- /dev/null +++ b/GeneralTools/KZ_Processing/compileSWIFT.asv @@ -0,0 +1,491 @@ +% Aggregate and read all SWIFT sbd data after concatenating the +% offload from SD card or the sbd email attachments. SBD files should be in the +% subfolder specified by 'SBDfold' of any given SWIFT mission folder. +% Based on compileSWIFT_SBDservertelemetry.m by J. Thomson. +% QC has been removed. The intention is to create a pure L1 product which +% has all data recorded. Basic QC has been allocated to another parallel +% script, 'pruneSWIFT.m'. + +% Time: SWIFT - Take the time from the filename, even when there is time from + % the airmar (because of parsing errors). For telemetry, this + % is the telemtry time (at the end of the burst). For offloaded data, + % this the concat file name (from the start of the burst). + % microSWIFT - Use the time embedded within the payload 50 or 51 or 52 of the + % SBD file, which is the time at the end of the burst of raw data. + +% K. Zeiden 10/01/2024 + +%% Experiment directory and QC parameters (user defined) + +if ispc + slash = '\'; +else + slash = '/'; +end + +% Experiment Directory +expdir = 'S:\SEAFAC\June2024'; + +% SBD folder +SBDfold = 'ProcessedSBD'; + +% Processing parameters +plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) +fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights +fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. + +% % QC Parameters +% minwaveheight = 0;% minimum wave height in data screening +% minsalinity = 0;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) +% maxdriftspd = 5;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that +% maxwindspd = 30;% m/s for malfunctioning Airmars +% minairtemp = -20;% min airtemp +% maxairtemp = 50;% max airtemp +% +% disp('-------------------------------------') +% disp('QC settings:') +% disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) +% disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) +% disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) +% disp(['Maximum wind speed: ' num2str(maxwindspd) ' ms^{-1}']) +% disp(['Minimum air temp: ' num2str(minairtemp) ' C']) +% disp(['Maximum air temp: ' num2str(maxairtemp) ' C']) +% disp('-------------------------------------') + +% List missions +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +%% Loop through missions + +for im = 1:length(missions) + +missiondir = [missions(im).folder slash missions(im).name]; +cd(missiondir) +sname = missions(im).name; + + diaryfile = [missions(im).name '_compileSWIFT.txt']; + if exist(diaryfile,'file') + delete(diaryfile); + end + diary(diaryfile) + disp(['Compiling ' sname]) + +%%% Compile list of sbd burst files %%% +if exist([missiondir slash SBDfold],'dir') + blist = dir([missiondir slash SBDfold slash '*.sbd']); +else + disp('Processed files have not been concatenated...') + blist = []; +end +nburst = length(blist); + +%%% Initialize badburst flag %%% +badburst = false(1,nburst); + +%%% Initialize vectors %%% +battery = NaN(1,nburst); +npayloads = NaN(1,nburst); + +%%% Loop through all SBD burst files, load, QC, and save in SWIFT structure +for iburst = 1:nburst + + disp('=================================') + disp(['Burst ' num2str(iburst) ' : ' blist(iburst).name ]) + + [oneSWIFT,voltage]= readSWIFT_SBD([blist(iburst).folder slash blist(iburst).name],0); + + if voltage == 9999 % error flag from SBD message + badburst(iburst) = true; + end + + if ~isempty(voltage) + battery(iburst) = voltage; + oneSWIFT.battery = voltage; + else + oneSWIFT.battery = NaN; + end + + if isempty(oneSWIFT.lat) || isempty(oneSWIFT.lon) + oneSWIFT.lat = NaN; + oneSWIFT.lon = NaN; + end + + %%% SWIFT type and Time stamp %%% + if blist(iburst).name(6) == 'S' % SWIFT v3 and v4 + nameoffset = 14; + day = blist(iburst).name(nameoffset + (1:2)); + month = blist(iburst).name(nameoffset + (3:5)); + year = blist(iburst).name(nameoffset + (6:9)); + hr = blist(iburst).name(nameoffset + (11:12)); + minute = blist(iburst).name(nameoffset + (13:14)); + sec = blist(iburst).name(nameoffset + (15:16)); + oneSWIFT.time = datenum([day ' ' month ' ' year ' ' hr ':' minute ':' sec]); + micro = false; + elseif blist(iburst).name(6) == 'm' % microSWIFT + micro = true; + else + oneSWIFT.time = NaN; + micro = false; + end + + %%% Remove wave histograms %%% + if isfield(oneSWIFT,'wavehistogram') + oneSWIFT = rmfield(oneSWIFT,'wavehistogram'); + end + + %%% Increment main structure %%% + burstpayloads = string(fieldnames(oneSWIFT)); + npayloads(iburst) = length(burstpayloads); + + if iburst == 1 + allpayloads = burstpayloads; + disp('Initial payloads:') + disp(allpayloads) + elseif iburst ~= 1 && npayloads(iburst) > length(allpayloads) + + isnew = false(npayloads(iburst),1); + for ip = 1:npayloads(iburst) + isnew(ip) = ~any(strcmp(burstpayloads(ip),allpayloads)); + end + newpayloads = burstpayloads(isnew); + disp(['New payloads in burst file ' num2str(iburst) ':']) + disp(newpayloads) + allpayloads = burstpayloads; + + elseif iburst ~= 1 && npayloads(iburst) < length(allpayloads) + + ismissing = false(length(allpayloads),1); + for ip = 1:length(allpayloads) + ismissing(ip) = ~any(strcmp(allpayloads(ip),burstpayloads)); + end + missingpayloads = allpayloads(ismissing); + disp(['Missing payloads in burst file ' num2str(iburst) ':']) + disp(missingpayloads) + end + + for ipay = 1:npayloads(iburst) + SWIFT(iburst).(burstpayloads{ipay}) = oneSWIFT.(burstpayloads{ipay}); + end + + %%% Original Increment Method -- reinitialize if new payloads, skip if + %%% fewer payloads... misses a lot of good data + % % First SBD: set the structure fields as the standard + % if iburst == 1 && voltage ~= 9999 + % SWIFT(iburst) = oneSWIFT; + % initpayloads = burstpayloads; + % disp(['Initial payloads in burst file ' num2str(iburst) ':']) + % disp(initpayloads) + % + % elseif iburst == 1 && voltage == 9999 + % + % badburst(iburst) = true; + % initpayloads = []; + % + % % Payloads match: increment + % elseif iburst > 1 && npayloads(iburst) == length(initpayloads) + % SWIFT(iburst) = oneSWIFT; + % + % % Additional payloads: favor that new structure (removing other) + % elseif iburst > 1 && npayloads(iburst) > length(initpayloads) + % + % isnew = false(npayloads(iburst),1); + % for ip = 1:npayloads(iburst) + % isnew(ip) = ~any(strcmp(burstpayloads(ip),initpayloads)); + % end + % newpayloads = burstpayloads(isnew); + % + % clear SWIFT + % badburst(iburst-1) = true; + % SWIFT(iburst) = oneSWIFT; + % disp(['Found extra payloads in burst file ' num2str(iburst) ':']) + % disp(newpayloads) + % disp('...re-initializing SWIFT structure...') + % initpayloads = burstpayloads; % reset the prefer field names + % + % % Fewer paylaods: skip that burst + % elseif iburst > 1 && npayloads(iburst) < length(initpayloads) || voltage==9999 + % + % ismissing = false(length(initpayloads),1); + % for ip = 1:length(initpayloads) + % ismissing(ip) = ~any(strcmp(initpayloads(ip),burstpayloads)); + % end + % missingpayloads = initpayloads(ismissing); + % + % disp(['Missing payloads in burst file ' num2str(iburst) ':']) + % disp(missingpayloads) + % SWIFT(iburst) = SWIFT(iburst-1); % placeholder, which will be removed when badburst applied + % badburst(iburst) = true; + % disp('...skipping burst...') + % end + + %%% Screen the bad data (usually out of the water) %%% + % No data + if isempty(oneSWIFT.lon) || isempty(oneSWIFT.lat) || isempty(oneSWIFT.time) + badburst(iburst) = true; + disp('No position or timestamp!') + end + % No position + if oneSWIFT.lon == 0 || ~isnumeric(oneSWIFT.lon) || isnan(oneSWIFT.lon) + badburst(iburst) = true; + disp('No position!') + end + % Waves too small + if isfield(oneSWIFT,'sigwaveheight') + if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 + badburst(iburst) = true; + disp('Waves too small, removing burst.') + end + end + % Salinity too small + if isfield(oneSWIFT,'salinity') %&& ~micro + if all(oneSWIFT.salinity < minsalinity) % & all(~isnan(oneSWIFT.salinity)), + badburst(iburst) = true; + disp('Salinity too low, removing burst.') + end + end + % Drift speed limit + if isfield(oneSWIFT,'driftspd') + if oneSWIFT.driftspd > maxdriftspd + badburst(iburst) = true; + disp('Speed too fast, removing burst.') + end + end + + disp('=================================') + +% End burst loop +end +diary off + +% Remove bursts which had less payloads +% badburst(npayloads < length(initpayloads)) = true; + +%% Apply QC + +% SWIFT(badburst) = []; +% battery(badburst) = []; + +%% Fill empty SWIFT fields +payloads = fieldnames(SWIFT); +npay = length(payloads); +nburst = length(SWIFT); + +for ipay = 1:npay + + var = [SWIFT.(payloads{ipay})]; + + % If some burst values are empty + if length(var) ~= nburst + + % If variable is a scalar (e.g. 'watertemp') fill with NaN + if isa(var,'double') + for iburst = 1:nburst + if isempty(SWIFT(iburst).(payloads{ipay})) + SWIFT(iburst).(payloads{ipay}) = NaN; + end + end + elseif isa(var,'char') + for iburst = 1:nburst + if isempty(SWIFT(iburst).(payloads{ipay})) + SWIFT(iburst).(payloads{ipay}) = var(1); + end + end + else % If variable is a structure array (e.g. 'wavespectra') fill w/NaN structure + for iburst = 1:nburst + if isempty(SWIFT(iburst).(payloads{ipay})) + SWIFT(iburst).(payloads{ipay}) = NaNstructR(var(1)); + end + end + end + + end + +end + +%% Sort final structure +[~,tinds] = sort([SWIFT.time]); +SWIFT = SWIFT(tinds); +battery = battery(tinds); + +%% Enforce a single value for CT and MET sensor heights + +if isfield(SWIFT,'CTdepth') + for si = 1:length(SWIFT) + SWIFT(si).CTdepth = median([SWIFT.CTdepth],'omitnan'); + end +end + +if isfield(SWIFT,'metheight') + for si = 1:length(SWIFT) + SWIFT(si).metheight = median([SWIFT.metheight],'omitnan'); + end +end + + +%% Fill position outliers + +if fixpositions + [cleanlon,cloni] = filloutliers([SWIFT.lon],'linear'); + [cleanlat,clati] = filloutliers([SWIFT.lat],'linear'); + if cloni == clati + for ci = find(cloni) + SWIFT(ci).lon = cleanlon(ci); + SWIFT(ci).lat = cleanlat(ci); + end + disp(['Filled ' num2str(sum(cloni)) ' position outliers.']) + end +end + +%% Recalculate drift (note that wind slip, which is 1%, is not removed) +% Drift speed is included from the Airmar results, +% but that sensor is not always available or included +% (so simpler to just calculate it from differencing positions). + +if length(SWIFT) > 3 + + time = [SWIFT.time];%[time tinds ] = sort(time); + lat = [SWIFT.lat]; %lat = lat(tinds); + lon = [SWIFT.lon]; %lon = lon(tinds); + dlondt = gradient(lon,time); % deg per day + dxdt = deg2km(dlondt,6371*cosd(mean(lat,'omitnan'))) .* 1000 ./ ( 24*3600 ); % m/s + dlatdt = gradient(lat,time); % deg per day + dydt = deg2km(dlatdt) .* 1000 ./ ( 24*3600 ); % m/s + dxdt(isinf(dxdt)) = NaN; + dydt(isinf(dydt)) = NaN; + speed = sqrt(dxdt.^2 + dydt.^2); % m/s + direction = -180 ./ 3.14 .* atan2(dydt,dxdt); % cartesian direction [deg] + direction = direction + 90; % rotate from eastward = 0 to northward = 0 + direction( direction<0) = direction( direction<0 ) + 360; % make quadrant II 270->360 instead of -90->0 + + for si = 1:length(SWIFT) + if si == 1 || si == length(SWIFT) + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + else + SWIFT(si).driftspd = speed(si); + SWIFT(si).driftdirT = direction(si); + end + end + + % % remove last burst, if big change in direction (suggests recovery by ship) + % dirchange = abs( SWIFT( length(SWIFT) - 2).driftdirT - SWIFT( length(SWIFT) - 1).driftdirT ); + % if dirchange > 90, + % disp('removing last burst, suspect includes ship recovery') + % SWIFT( length(SWIFT) - 1).driftdirT = NaN; + % SWIFT( length(SWIFT) - 1).driftspd = NaN; + % SWIFT( length(SWIFT) ) = []; + % battery( length(SWIFT) ) = []; + % end + +elseif length(SWIFT) <= 3 + for si = 1:length(SWIFT) + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + end +end + +% Quality control by removing drift results associated with large time gaps +if length([SWIFT.time]) > 1 + dt = gradient([SWIFT.time]); + for si = 1:length(SWIFT) + if dt(si) > 1/12 % 1/12 of day is two hours + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + end + end +end + +% Quality control drift speeds too fast (prob on deck) with new drift spd +if length([SWIFT.time]) > 1 && isfield(SWIFT(1),'driftspd') + toofast = [SWIFT.driftspd] > maxdriftspd; + SWIFT( toofast ) =[]; + battery( toofast ) = []; +end + +%% Extrapolate missing low frequencies of wave energy spectra %% +% Require energy at lowest frequenc to be zero. +% Not neccessary if post-processing for raw displacements. +% Less necessary after Oct 2017 rev of onboard processing with improved RC filter. + +if fixspectra && isfield(SWIFT,'wavespectra') + for si = 1:length(SWIFT) + notzero = find(SWIFT(si).wavespectra.energy ~= 0 & SWIFT(si).wavespectra.freq > 0.04); + tobereplaced = find(SWIFT(si).wavespectra.energy == 0 & SWIFT(si).wavespectra.freq > 0.04); + if length(notzero) > 10 + E = interp1([0.04; SWIFT(si).wavespectra.freq(notzero)],[0; SWIFT(si).wavespectra.energy(notzero)],SWIFT(si).wavespectra.freq); + SWIFT(si).wavespectra.energy(tobereplaced) = E(tobereplaced); + df = median(diff(SWIFT(si).wavespectra.freq)); + SWIFT(si).sigwaveheight = 4*sqrt(sum(SWIFT(si).wavespectra.energy,'omitnan')*df); + end + end +end + +%% Quality control wind speeds +if length([SWIFT.time]) > 1 && isfield(SWIFT,'windspd') + for si = 1:length(SWIFT) + if SWIFT(si).windspd > maxwindspd + SWIFT(si).windspd = NaN; + SWIFT(si).winddirT = NaN; + SWIFT(si).winddirR = NaN; + end + end +end + +%% Quality control airmar temperature +if isfield(SWIFT,'airtemp') + for si = 1:length(SWIFT) + if SWIFT(si).airtemp == 0.0 || SWIFT(si).airtemp < minairtemp || SWIFT(si).airtemp > maxairtemp + SWIFT(si).airtemp = NaN; + SWIFT(si).windspd = NaN; + end + end +end + +%% Sort the microSWIFT onboard processing, using the battery voltage as a flag +% only applies to v1 microSWIFTs from 2022 + +IMU = find(battery==0); +GPS = find(battery==1); + +if ~isempty(IMU), SWIFT_IMU = SWIFT(IMU); end +if ~isempty(GPS), SWIFT_GPS = SWIFT(GPS); end + +%% Fill in the ID field from mission directory name if NaN + +idnan = isnan([SWIFT.ID]); +ID = sname(strfind(sname,'SWIFT')+ (5:6)); + +for si = find(idnan) + SWIFT(si).ID = ID; +end + +%% Save L1 file + +if micro + save([missiondir slash 'micro' sname '_L1.mat'],'SWIFT*') +elseif length([SWIFT.time]) > 1 + save([missiondir slash sname '_L1.mat'],'SWIFT') +end + +%% Plot + +if plotflag + + plotSWIFT(SWIFT) + + % battery plot + if any(~isnan(battery)) + figure(7), clf, + plot([SWIFT.time],battery,'kx','linewidth',3) + datetick, grid + ylabel('Voltage') + print('-dpng',[sname(1:7) '_battery.png']) + end + +end + +close all +clear SWIFT + +end % End mission loop diff --git a/GeneralTools/KZ_Processing/compileSWIFT.m b/GeneralTools/KZ_Processing/compileSWIFT.m new file mode 100644 index 0000000..093748a --- /dev/null +++ b/GeneralTools/KZ_Processing/compileSWIFT.m @@ -0,0 +1,491 @@ +% Aggregate and read all SWIFT sbd data after concatenating the +% offload from SD card or the sbd email attachments. SBD files should be in the +% subfolder specified by 'SBDfold' of any given SWIFT mission folder. +% Based on compileSWIFT_SBDservertelemetry.m by J. Thomson. +% QC has been removed. The intention is to create a pure L1 product which +% has all data recorded. Basic QC has been allocated to another parallel +% script, 'pruneSWIFT.m'. + +% Time: SWIFT - Take the time from the filename, even when there is time from + % the airmar (because of parsing errors). For telemetry, this + % is the telemtry time (at the end of the burst). For offloaded data, + % this the concat file name (from the start of the burst). + % microSWIFT - Use the time embedded within the payload 50 or 51 or 52 of the + % SBD file, which is the time at the end of the burst of raw data. + +% K. Zeiden 10/01/2024 + +%% Experiment directory and QC parameters (user defined) + +if ispc + slash = '\'; +else + slash = '/'; +end + +% Experiment Directory +expdir = 'S:\SEAFAC\June2024'; + +% SBD folder +SBDfold = 'ProcessedSBD'; + +% Processing parameters +plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) +fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights +fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. + +% % QC Parameters +% minwaveheight = 0;% minimum wave height in data screening +% minsalinity = 0;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) +% maxdriftspd = 5;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that +% maxwindspd = 30;% m/s for malfunctioning Airmars +% minairtemp = -20;% min airtemp +% maxairtemp = 50;% max airtemp +% +% disp('-------------------------------------') +% disp('QC settings:') +% disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) +% disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) +% disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) +% disp(['Maximum wind speed: ' num2str(maxwindspd) ' ms^{-1}']) +% disp(['Minimum air temp: ' num2str(minairtemp) ' C']) +% disp(['Maximum air temp: ' num2str(maxairtemp) ' C']) +% disp('-------------------------------------') + +% List missions +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +%% Loop through missions + +for im = 1:length(missions) + +missiondir = [missions(im).folder slash missions(im).name]; +cd(missiondir) +sname = missions(im).name; + + diaryfile = [missions(im).name '_compileSWIFT.txt']; + if exist(diaryfile,'file') + delete(diaryfile); + end + diary(diaryfile) + disp(['Compiling ' sname]) + +%%% Compile list of sbd burst files %%% +if exist([missiondir slash SBDfold],'dir') + blist = dir([missiondir slash SBDfold slash '*.sbd']); +else + disp('Processed files have not been concatenated...') + blist = []; +end +nburst = length(blist); + +%%% Initialize badburst flag %%% +badburst = false(1,nburst); + +%%% Initialize vectors %%% +battery = NaN(1,nburst); +npayloads = NaN(1,nburst); + +%%% Loop through all SBD burst files, load, QC, and save in SWIFT structure +for iburst = 1:nburst + + disp('=================================') + disp(['Burst ' num2str(iburst) ' : ' blist(iburst).name ]) + + [oneSWIFT,voltage]= readSWIFT_SBD([blist(iburst).folder slash blist(iburst).name],0); + + if voltage == 9999 % error flag from SBD message + badburst(iburst) = true; + end + + if ~isempty(voltage) + battery(iburst) = voltage; + oneSWIFT.battery = voltage; + else + oneSWIFT.battery = NaN; + end + + if isempty(oneSWIFT.lat) || isempty(oneSWIFT.lon) + oneSWIFT.lat = NaN; + oneSWIFT.lon = NaN; + end + + %%% SWIFT type and Time stamp %%% + if blist(iburst).name(6) == 'S' % SWIFT v3 and v4 + nameoffset = 14; + day = blist(iburst).name(nameoffset + (1:2)); + month = blist(iburst).name(nameoffset + (3:5)); + year = blist(iburst).name(nameoffset + (6:9)); + hr = blist(iburst).name(nameoffset + (11:12)); + minute = blist(iburst).name(nameoffset + (13:14)); + sec = blist(iburst).name(nameoffset + (15:16)); + oneSWIFT.time = datenum([day ' ' month ' ' year ' ' hr ':' minute ':' sec]); + micro = false; + elseif blist(iburst).name(6) == 'm' % microSWIFT + micro = true; + else + oneSWIFT.time = NaN; + micro = false; + end + + %%% Remove wave histograms %%% + if isfield(oneSWIFT,'wavehistogram') + oneSWIFT = rmfield(oneSWIFT,'wavehistogram'); + end + + %%% Increment main structure %%% + burstpayloads = string(fieldnames(oneSWIFT)); + npayloads(iburst) = length(burstpayloads); + + if iburst == 1 + allpayloads = burstpayloads; + disp('Initial payloads:') + disp(allpayloads) + elseif iburst ~= 1 && npayloads(iburst) > length(allpayloads) + + isnew = false(npayloads(iburst),1); + for ip = 1:npayloads(iburst) + isnew(ip) = ~any(strcmp(burstpayloads(ip),allpayloads)); + end + newpayloads = burstpayloads(isnew); + disp(['New payloads in burst file ' num2str(iburst) ':']) + disp(newpayloads) + allpayloads = burstpayloads; + + elseif iburst ~= 1 && npayloads(iburst) < length(allpayloads) + + ismissing = false(length(allpayloads),1); + for ip = 1:length(allpayloads) + ismissing(ip) = ~any(strcmp(allpayloads(ip),burstpayloads)); + end + missingpayloads = allpayloads(ismissing); + disp(['Missing payloads in burst file ' num2str(iburst) ':']) + disp(missingpayloads) + end + + for ipay = 1:npayloads(iburst) + SWIFT(iburst).(burstpayloads{ipay}) = oneSWIFT.(burstpayloads{ipay}); + end + + %%% Original Increment Method -- reinitialize if new payloads, skip if + %%% fewer payloads... misses a lot of good data + % % First SBD: set the structure fields as the standard + % if iburst == 1 && voltage ~= 9999 + % SWIFT(iburst) = oneSWIFT; + % initpayloads = burstpayloads; + % disp(['Initial payloads in burst file ' num2str(iburst) ':']) + % disp(initpayloads) + % + % elseif iburst == 1 && voltage == 9999 + % + % badburst(iburst) = true; + % initpayloads = []; + % + % % Payloads match: increment + % elseif iburst > 1 && npayloads(iburst) == length(initpayloads) + % SWIFT(iburst) = oneSWIFT; + % + % % Additional payloads: favor that new structure (removing other) + % elseif iburst > 1 && npayloads(iburst) > length(initpayloads) + % + % isnew = false(npayloads(iburst),1); + % for ip = 1:npayloads(iburst) + % isnew(ip) = ~any(strcmp(burstpayloads(ip),initpayloads)); + % end + % newpayloads = burstpayloads(isnew); + % + % clear SWIFT + % badburst(iburst-1) = true; + % SWIFT(iburst) = oneSWIFT; + % disp(['Found extra payloads in burst file ' num2str(iburst) ':']) + % disp(newpayloads) + % disp('...re-initializing SWIFT structure...') + % initpayloads = burstpayloads; % reset the prefer field names + % + % % Fewer paylaods: skip that burst + % elseif iburst > 1 && npayloads(iburst) < length(initpayloads) || voltage==9999 + % + % ismissing = false(length(initpayloads),1); + % for ip = 1:length(initpayloads) + % ismissing(ip) = ~any(strcmp(initpayloads(ip),burstpayloads)); + % end + % missingpayloads = initpayloads(ismissing); + % + % disp(['Missing payloads in burst file ' num2str(iburst) ':']) + % disp(missingpayloads) + % SWIFT(iburst) = SWIFT(iburst-1); % placeholder, which will be removed when badburst applied + % badburst(iburst) = true; + % disp('...skipping burst...') + % end + + %%% Screen the bad data (usually out of the water) %%% + % No data + if isempty(oneSWIFT.lon) || isempty(oneSWIFT.lat) || isempty(oneSWIFT.time) + badburst(iburst) = true; + disp('No position or timestamp!') + end + % No position + if oneSWIFT.lon == 0 || ~isnumeric(oneSWIFT.lon) || isnan(oneSWIFT.lon) + badburst(iburst) = true; + disp('No position!') + end + % Waves too small + if isfield(oneSWIFT,'sigwaveheight') + if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 + badburst(iburst) = true; + disp('Waves too small, removing burst.') + end + end + % Salinity too small + if isfield(oneSWIFT,'salinity') %&& ~micro + if all(oneSWIFT.salinity < minsalinity) % & all(~isnan(oneSWIFT.salinity)), + badburst(iburst) = true; + disp('Salinity too low, removing burst.') + end + end + % Drift speed limit + if isfield(oneSWIFT,'driftspd') + if oneSWIFT.driftspd > maxdriftspd + badburst(iburst) = true; + disp('Speed too fast, removing burst.') + end + end + + disp('=================================') + +% End burst loop +end +diary off + +% Remove bursts which had less payloads +% badburst(npayloads < length(initpayloads)) = true; + +%% Apply QC + +% SWIFT(badburst) = []; +% battery(badburst) = []; + +%% Fill empty SWIFT fields +payloads = fieldnames(SWIFT); +npay = length(payloads); +nburst = length(SWIFT); + +for ipay = 1:npay + + var = [SWIFT.(payloads{ipay})]; + + % If some burst values are empty + if length(var) ~= nburst + + % If variable is a scalar (e.g. 'watertemp') fill with NaN + if isa(var,'double') + for iburst = 1:nburst + if isempty(SWIFT(iburst).(payloads{ipay})) + SWIFT(iburst).(payloads{ipay}) = NaN; + end + end + elseif isa(var,'char') + for iburst = 1:nburst + if isempty(SWIFT(iburst).(payloads{ipay})) + SWIFT(iburst).(payloads{ipay}) = var(1); + end + end + else % If variable is a structure array (e.g. 'wavespectra') fill w/NaN structure + for iburst = 1:nburst + if isempty(SWIFT(iburst).(payloads{ipay})) + SWIFT(iburst).(payloads{ipay}) = NaNstructR(var(1)); + end + end + end + + end + +end + +%% Sort final structure +[~,tinds] = sort([SWIFT.time]); +SWIFT = SWIFT(tinds); +battery = battery(tinds); + +%% Enforce a single value for CT and MET sensor heights + +if isfield(SWIFT,'CTdepth') + for si = 1:length(SWIFT) + SWIFT(si).CTdepth = median([SWIFT.CTdepth],'omitnan'); + end +end + +if isfield(SWIFT,'metheight') + for si = 1:length(SWIFT) + SWIFT(si).metheight = median([SWIFT.metheight],'omitnan'); + end +end + + +%% Fill position outliers + +if fixpositions + [cleanlon,cloni] = filloutliers([SWIFT.lon],'linear'); + [cleanlat,clati] = filloutliers([SWIFT.lat],'linear'); + if cloni == clati + for ci = find(cloni) + SWIFT(ci).lon = cleanlon(ci); + SWIFT(ci).lat = cleanlat(ci); + end + disp(['Filled ' num2str(sum(cloni)) ' position outliers.']) + end +end + +%% Recalculate drift (note that wind slip, which is 1%, is not removed) +% Drift speed is included from the Airmar results, +% but that sensor is not always available or included +% (so simpler to just calculate it from differencing positions). + +if length(SWIFT) > 3 + + time = [SWIFT.time];%[time tinds ] = sort(time); + lat = [SWIFT.lat]; %lat = lat(tinds); + lon = [SWIFT.lon]; %lon = lon(tinds); + dlondt = gradient(lon,time); % deg per day + dxdt = deg2km(dlondt,6371*cosd(mean(lat,'omitnan'))) .* 1000 ./ ( 24*3600 ); % m/s + dlatdt = gradient(lat,time); % deg per day + dydt = deg2km(dlatdt) .* 1000 ./ ( 24*3600 ); % m/s + dxdt(isinf(dxdt)) = NaN; + dydt(isinf(dydt)) = NaN; + speed = sqrt(dxdt.^2 + dydt.^2); % m/s + direction = -180 ./ 3.14 .* atan2(dydt,dxdt); % cartesian direction [deg] + direction = direction + 90; % rotate from eastward = 0 to northward = 0 + direction( direction<0) = direction( direction<0 ) + 360; % make quadrant II 270->360 instead of -90->0 + + for si = 1:length(SWIFT) + if si == 1 || si == length(SWIFT) + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + else + SWIFT(si).driftspd = speed(si); + SWIFT(si).driftdirT = direction(si); + end + end + + % % remove last burst, if big change in direction (suggests recovery by ship) + % dirchange = abs( SWIFT( length(SWIFT) - 2).driftdirT - SWIFT( length(SWIFT) - 1).driftdirT ); + % if dirchange > 90, + % disp('removing last burst, suspect includes ship recovery') + % SWIFT( length(SWIFT) - 1).driftdirT = NaN; + % SWIFT( length(SWIFT) - 1).driftspd = NaN; + % SWIFT( length(SWIFT) ) = []; + % battery( length(SWIFT) ) = []; + % end + +elseif length(SWIFT) <= 3 + for si = 1:length(SWIFT) + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + end +end + +% Quality control by removing drift results associated with large time gaps +if length([SWIFT.time]) > 1 + dt = gradient([SWIFT.time]); + for si = 1:length(SWIFT) + if dt(si) > 1/12 % 1/12 of day is two hours + SWIFT(si).driftspd = NaN; + SWIFT(si).driftdirT = NaN; + end + end +end + +% Quality control drift speeds too fast (prob on deck) with new drift spd +if length([SWIFT.time]) > 1 && isfield(SWIFT(1),'driftspd') + toofast = [SWIFT.driftspd] > maxdriftspd; + SWIFT( toofast ) =[]; + battery( toofast ) = []; +end + +%% Extrapolate missing low frequencies of wave energy spectra %% +% Require energy at lowest frequenc to be zero. +% Not neccessary if post-processing for raw displacements. +% Less necessary after Oct 2017 rev of onboard processing with improved RC filter. + +if fixspectra && isfield(SWIFT,'wavespectra') + for si = 1:length(SWIFT) + notzero = find(SWIFT(si).wavespectra.energy ~= 0 & SWIFT(si).wavespectra.freq > 0.04); + tobereplaced = find(SWIFT(si).wavespectra.energy == 0 & SWIFT(si).wavespectra.freq > 0.04); + if length(notzero) > 10 + E = interp1([0.04; SWIFT(si).wavespectra.freq(notzero)],[0; SWIFT(si).wavespectra.energy(notzero)],SWIFT(si).wavespectra.freq); + SWIFT(si).wavespectra.energy(tobereplaced) = E(tobereplaced); + df = median(diff(SWIFT(si).wavespectra.freq)); + SWIFT(si).sigwaveheight = 4*sqrt(sum(SWIFT(si).wavespectra.energy,'omitnan')*df); + end + end +end + +%% Quality control wind speeds +if length([SWIFT.time]) > 1 && isfield(SWIFT,'windspd') + for si = 1:length(SWIFT) + if SWIFT(si).windspd > maxwindspd + SWIFT(si).windspd = NaN; + SWIFT(si).winddirT = NaN; + SWIFT(si).winddirR = NaN; + end + end +end + +%% Quality control airmar temperature +if isfield(SWIFT,'airtemp') + for si = 1:length(SWIFT) + if SWIFT(si).airtemp == 0.0 || SWIFT(si).airtemp < minairtemp || SWIFT(si).airtemp > maxairtemp + SWIFT(si).airtemp = NaN; + SWIFT(si).windspd = NaN; + end + end +end + +%% Sort the microSWIFT onboard processing, using the battery voltage as a flag +% only applies to v1 microSWIFTs from 2022 + +IMU = find(battery==0); +GPS = find(battery==1); + +if ~isempty(IMU), SWIFT_IMU = SWIFT(IMU); end +if ~isempty(GPS), SWIFT_GPS = SWIFT(GPS); end + +%% Fill in the ID field from mission directory name if NaN + +idnan = isnan([SWIFT.ID]); +ID = sname(strfind(sname,'SWIFT')+ (5:6)); + +for si = find(idnan) + SWIFT(si).ID = ID; +end + +%% Save L1 file + +if micro + save([missiondir slash 'micro' sname '_L1.mat'],'SWIFT*') +elseif length([SWIFT.time]) > 1 + save([missiondir slash sname '_L1.mat'],'SWIFT') +end + +%% Plot + +if plotflag + + plotSWIFT(SWIFT) + + % battery plot + if any(~isnan(battery)) + figure(7), clf, + plot([SWIFT.time],battery,'kx','linewidth',3) + datetick, grid + ylabel('Voltage') + print('-dpng',[sname(1:7) '_battery.png']) + end + +end + +close all +clear SWIFT + +end % End mission loop diff --git a/GeneralTools/KZ_Processing/createSBD.asv b/GeneralTools/KZ_Processing/createSBD.asv new file mode 100644 index 0000000..1b21d61 --- /dev/null +++ b/GeneralTools/KZ_Processing/createSBD.asv @@ -0,0 +1,200 @@ +% aggregate, concat, and read all onboard processed SWIFT data (once offloaded from SD card) +% run this in a dedicated directory for the results +% +% (this fills in the results not sent by Iridium when running more than 1 burst per hour) +% +% J. Thomson, 4/2014 +% 9/2015 v3.3, use com names but currently only pulls one ACS file (out of three), because they are not uniquely named. +% 1/2016 v3.3, so up to 3 ACS files can be included +% 6/2016 v 3.4, includ sdi coms (Vasaila 536) +% 1/2017 v4.0, include SBG, Nortek Signature and RM Young 8100 sonic, plus index on SBG, not AQ +% 9/2017 add oxygen optode and sea owl fluorometer +% 9/2019 changed time convention to use start of the burst,rather than end +% + +if ispc + slash = '\'; +else + slash = '/'; +end + +% Experiment directory and sampling parameters (user defined) +expdir = 'S:\SEAFAC\June2024'; +SBDfold = 'ProcessedSBD'; + +% Sampling Parameters +burstinterval = 12; % minutes between bursts +burstlength = 512/60; % minutes of sampling during each burst +payloadtype = '7'; % v3.3 (2015) and up + +% Identify missions +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +%% Loop through missions + +for im = 2%:length(missions) + +missiondir = [missions(im).folder slash missions(im).name]; +cd(missiondir) +sname = missions(im).name; + +diaryfile = [missions(im).name '_createSBD.txt']; +if exist(diaryfile,'file') +delete(diaryfile); +end +diary(diaryfile) +disp(['Compiling ' sname]) + + +%%% Create folder for new SBD files %%% +if ~exist([missiondir slash SBDfold],'dir') + mkdir([missiondir slash SBDfold]) +end + +%%% Create payload type temporary file %%% + +payloadfile = [missiondir slash 'payload']; +fid = fopen(payloadfile,'wb'); +fwrite(fid,payloadtype,'uint8'); +fclose(fid); + +%%% Find all processed (PRC) files in all port folders + +PRCfiles = dir([missiondir slash '*' slash 'Processed' slash '*' slash '*PRC*.dat']); + +%%% Glob all processed files (per instrument) %%% +% and put these all in one directory (skip if already in one directory) + +% if ~exist([missiondir slash 'PRC'],'dir') +% +% mkdir([missiondir slash 'PRC']) +% +% progressbar(['Globbing PRC files for ' sname]) +% for iprc = 1:length(PRCfiles) +% progressbar(iprc/length(PRCfiles)) +% copyfile([PRCfiles(iprc).folder slash PRCfiles(iprc).name],[missiondir slash 'PRC']) +% end +% +% else +% disp(['*** PRC files for ' sname ' already globbed ***']) +% +% end + +%%% Use IMU or SBG to identify all bursts, as reference files %%% +% (b/c either IMU or SBG is always present) + +% refbfiles = dir([missiondir slash 'PRC' slash '*IMU*_PRC*.dat']); +% if isempty(refbfiles) +% refbfiles = dir([missiondir slash 'PRC' slash '*SBG*_PRC*.dat']); +% else +% disp('NO IMU (or SBG) files found'); +% end + +% refbfiles = PRCfiles(contains({PRCfiles.name},'IMU')); +% if isempty(refbfiles) +% refbfiles = PRCfiles(contains({PRCfiles.name},'SBG')); +% if isempty(refbfiles) +% disp('No IMU or SBG files found') +% end +% end + +% Get reference burst +burstfiletimes = NaN(length(PRCfiles),1); +for iburst = 1:length(burstfiletimes) + date = datenum(PRCfiles(iburst).name(13:21)); + hour = str2double(PRCfiles(iburst).name(23:24)); + min = (str2double(PRCfiles(iburst).name(26:27))-1)*12; + burstfiletimes(iburst) = date + datenum([0 0 0 hour min 0]); +end +[~,b] = unique(burstfiletimes); +refbfiles = PRCfiles(b); + +%%% Loop through reference burst files (IMU or SBG) %%% + +for iburst = 1:length(refbfiles) + + disp(['Creating SBD file for burst ' num2str(iburst) ' (' num2str(length(refbfiles)) ' bursts)']) + + ID = refbfiles(iburst).name(1:7); + date = refbfiles(iburst).name(13:21); + hour = refbfiles(iburst).name(23:24); + burst = refbfiles(iburst).name(26:27); + bfilename = [date '_' hour '_' burst '_PRC.dat']; + + % Name concat file same as if pulled from swiftserver + minute = num2str((str2double(burst(2))-1) * burstinterval); + if length(minute) == 1 + minute = ['0' minute]; %#ok + end + sbdfile = [missiondir slash SBDfold slash 'buoy-SWIFT_' ID(6:7) '-' date '_' hour minute '000.sbd']; + + % Find all PRC files for this burst (instead of globbing files) + PRCburstfiles = PRCfiles(contains({PRCfiles.name},bfilename)); + + % % Look for, rename and copy multiple ACS files (on different com ports) + % ACStopfile = dir([missiondir slash 'COM-7' slash 'Processed' slash '*' slash ID '_ACS_' bfilename]); + % if ~empty(ACStopfile) + % copyfile([ACStopfile.folder slash ACStopfile.name],... + % [missiondir slash 'PRC' slash ID '_ACStop_' bfilename]) + % end + % ACSmidfile = dir([missiondir slash 'COM-8' slash 'Processed' slash '*' slash ID '_ACS_' bfilename]); + % if ~isempty(ACSmidfile) + % copyfile([ACSmidfile.folder slash ACSmidfile.name],... + % [missiondir slash 'PRC' slash ID '_ACSmid_' bfilename]) + % end + % ACSbottomfile = dir([missiondir slash 'COM-9' slash 'Processed' slash '*' slash ID '_ACS_' bfilename]); + % if ~isempty(ACSbottomfile) + % copyfile([ACSbottomfile.folder slash ACSbottomfile.name],... + % [missiondir slash 'PRC' slash ID '_ACSbottom_' bfilename]) + % end + % + % % List of files to concatenate (to make SBD files) + % AQHfile = [missiondir slash 'PRC' slash ID '_AQH_' bfilename]; + % AQDfile = [missiondir slash 'PRC' slash ID '_AQD_' bfilename]; + % PB2file = [missiondir slash 'PRC' slash ID '_PB2_' bfilename]; + % RADfile = [missiondir slash 'PRC' slash ID '_536_' bfilename]; + % IMUfile = [missiondir slash 'PRC' slash ID '_IMU_' bfilename]; + % SBGfile = [missiondir slash 'PRC' slash ID '_SBG_' bfilename]; + % ACSfiletop = [missiondir slash 'PRC' slash ID '_ACStop_' bfilename]; + % ACSfilemid = [missiondir slash 'PRC' slash ID '_ACSmid_' bfilename]; + % ACSfilebottom = [missiondir slash 'PRC' slash ID '_ACSbottom_' bfilename]; + % Y81file = [missiondir slash 'PRC' slash ID '_Y81_' bfilename]; + % SIGfile = [missiondir slash 'PRC' slash ID '_SIG_' bfilename]; + % ACOfile = [missiondir slash 'PRC' slash ID '_ACO_' bfilename]; + % SWLfile = [missiondir slash 'PRC' slash ID '_SWL_' bfilename]; + % + % % Create system command to concatenate files + % if ispc + % syscommand = ['copy /b payload+' AQHfile '+' AQDfile '+' PB2file '+' RADfile '+' IMUfile '+' SBGfile '+' ACSfiletop '+' ACSfilemid '+' ACSfilebottom '+' Y81file '+' SIGfile '+' ACOfile '+' SWLfile '+' ' ' sbdfile]; + % else + % syscommand = ['!cat payload ' AQHfile ' ' AQDfile ' ' PB2file ' ' RADfile ' ' IMUfile ' ' SBGfile ' ' Y81file ' ' SIGfile ' ' ACSfiletop ' ' ACSfilemid ' ' ACSfilebottom ' ' ACOfile ' ' SWLfile ' > ' sbdfile]; + % end + + % Create system commmand to concatenate files + if ispc + syscommand = ['copy /b ' payloadfile]; + for iprc = 1:length(PRCburstfiles) + syscommand = [syscommand '+' PRCburstfiles(iprc).folder slash PRCburstfiles(iprc).name]; %#ok<*AGROW> + end + syscommand = [syscommand ' ' sbdfile]; + else + syscommand = ['!cat ' payloadfile]; + for iprc = 1:length(PRCburstfiles) + syscommand = [syscommand ' ' PRCburstfiles(iprc).folder slash PRCburstfiles(iprc).name]; + end + syscommand = [syscommand ' > ' sbdfile]; + end + + % Execute system command to concatenate files + status = system(syscommand); + if status~=0 + warning('SBD file creation (PRC concatenation) failed...') + end + +end % End burst loop + +delete(payloadfile) +diary off + +end % End mission loop \ No newline at end of file diff --git a/GeneralTools/KZ_Processing/createSBD.m b/GeneralTools/KZ_Processing/createSBD.m new file mode 100644 index 0000000..eed3189 --- /dev/null +++ b/GeneralTools/KZ_Processing/createSBD.m @@ -0,0 +1,126 @@ +% aggregate, concat, and read all onboard processed SWIFT data (once offloaded from SD card) +% run this in a dedicated directory for the results +% +% (this fills in the results not sent by Iridium when running more than 1 burst per hour) +% +% J. Thomson, 4/2014 +% 9/2015 v3.3, use com names but currently only pulls one ACS file (out of three), because they are not uniquely named. +% 1/2016 v3.3, so up to 3 ACS files can be included +% 6/2016 v 3.4, includ sdi coms (Vasaila 536) +% 1/2017 v4.0, include SBG, Nortek Signature and RM Young 8100 sonic, plus index on SBG, not AQ +% 9/2017 add oxygen optode and sea owl fluorometer +% 9/2019 changed time convention to use start of the burst,rather than end +% + +if ispc + slash = '\'; +else + slash = '/'; +end + +% Experiment directory and sampling parameters (user defined) +expdir = 'S:\SEAFAC\June2024'; +SBDfold = 'ProcessedSBD'; + +% Sampling Parameters +burstinterval = 12; % minutes between bursts +burstlength = 512/60; % minutes of sampling during each burst +payloadtype = '7'; % v3.3 (2015) and up + +% Identify missions +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +%% Loop through missions + +for im = 1:length(missions) + +missiondir = [missions(im).folder slash missions(im).name]; +cd(missiondir) +sname = missions(im).name; + +diaryfile = [missions(im).name '_createSBD.txt']; +if exist(diaryfile,'file') +delete(diaryfile); +end +diary(diaryfile) +disp(['Compiling ' sname]) + + +%%% Create folder for new SBD files %%% +if ~exist([missiondir slash SBDfold],'dir') + mkdir([missiondir slash SBDfold]) +end + +%%% Create payload type temporary file %%% + +payloadfile = [missiondir slash 'payload']; +fid = fopen(payloadfile,'wb'); +fwrite(fid,payloadtype,'uint8'); +fclose(fid); + +%%% Find all processed (PRC) files in all port folders + +PRCfiles = dir([missiondir slash '*' slash 'Processed' slash '*' slash '*PRC*.dat']); + +% Get reference burst files by identifying unique burst times from +% all payloads +burstfiletimes = NaN(length(PRCfiles),1); +for iburst = 1:length(burstfiletimes) + date = datenum(PRCfiles(iburst).name(13:21)); + hour = str2double(PRCfiles(iburst).name(23:24)); + min = (str2double(PRCfiles(iburst).name(26:27))-1)*12; + burstfiletimes(iburst) = date + datenum([0 0 0 hour min 0]); +end +[~,b] = unique(burstfiletimes); +refbfiles = PRCfiles(b); + +%%% Loop through reference burst files (IMU or SBG) %%% + +for iburst = 1:length(refbfiles) + + ID = refbfiles(iburst).name(1:7); + date = refbfiles(iburst).name(13:21); + hour = refbfiles(iburst).name(23:24); + burst = refbfiles(iburst).name(26:27); + bfilename = [date '_' hour '_' burst '_PRC.dat']; + + disp(['Creating SBD file for burst ' num2str(iburst) ' : ' bfilename(1:end-8)]) + + % Name concat file same as if pulled from swiftserver + minute = num2str((str2double(burst(2))-1) * burstinterval); + if length(minute) == 1 + minute = ['0' minute]; %#ok + end + sbdfile = [missiondir slash SBDfold slash 'buoy-SWIFT_' ID(6:7) '-' date '_' hour minute '000.sbd']; + + % Find all PRC files for this burst (instead of globbing files) + PRCburstfiles = PRCfiles(contains({PRCfiles.name},bfilename)); + + % Create system commmand to concatenate files + if ispc + syscommand = ['copy /b ' payloadfile]; + for iprc = 1:length(PRCburstfiles) + syscommand = [syscommand '+' PRCburstfiles(iprc).folder slash PRCburstfiles(iprc).name]; %#ok<*AGROW> + end + syscommand = [syscommand ' ' sbdfile]; + else + syscommand = ['!cat ' payloadfile]; + for iprc = 1:length(PRCburstfiles) + syscommand = [syscommand ' ' PRCburstfiles(iprc).folder slash PRCburstfiles(iprc).name]; + end + syscommand = [syscommand ' > ' sbdfile]; + end + + % Execute system command to concatenate files + status = system(syscommand); + if status~=0 + warning('SBD file creation (PRC concatenation) failed...') + end + +end % End burst loop + +delete(payloadfile) +diary off + +end % End mission loop \ No newline at end of file diff --git a/GeneralTools/postprocess_SWIFT.m b/GeneralTools/KZ_Processing/postprocessSWIFT.asv similarity index 90% rename from GeneralTools/postprocess_SWIFT.m rename to GeneralTools/KZ_Processing/postprocessSWIFT.asv index 857a94e..3e733d7 100644 --- a/GeneralTools/postprocess_SWIFT.m +++ b/GeneralTools/KZ_Processing/postprocessSWIFT.asv @@ -3,11 +3,7 @@ % Master post-processing function, that calls sub-functions to reprocess % each different type of raw SWIFT data. % L1 product must have been created prior to running this script, -% by running 'concatSWIFT_offloadedSDcard.m' in the mission directory -% Note: concatSWIFT in turn runs 'compileSWIFT_SBDservertelemetry.m', -% which in turn calls the function 'readSWIFT_SBD.m' -% Currently no reprocessing option for Airmar (PB2) -% nor Heitronics CTs (536?) +% by running 'compileSWIFT.m' % K. Zeiden 07/2024 @@ -19,24 +15,22 @@ slash = '/'; end -expdir = ['S:' slash 'SEAFAC' slash 'June2024' slash 'SouthMooring']; +expdir = ['S:' slash 'SEAFAC' slash 'June2024']; % Processing toggles rpIMU = true; % Waves -rpSBG = false; % Waves -rpWXT = false; % MET -rpY81 = false; % MET -rpACS = false; % CT -rpSIG = false; % TKE -rpAQH = false; % TKE -rpAQD = false; % TKE +rpSBG = true; % Waves +rpWXT = true; % MET +rpY81 = true; % MET +rpACS = true; % CT +rpSIG = true; % TKE +rpAQH = true; % TKE +rpAQD = true; % TKE % Plotting toggle plotL1L2 = true; -%% List of missions - -cd(expdir) +% List of missions missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); @@ -46,7 +40,7 @@ missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) - diaryfile = [missions(im).name '_postprocess.txt']; + diaryfile = [missions(im).name '_postprocessSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end @@ -113,7 +107,7 @@ end end - %% Reprocess WXT (536!!!) + %% Reprocess WXT (536, confusing b/c CT15 can also be 536) if rpWXT if ~isempty(dir([missiondir slash 'WXT' slash 'Raw' slash '*' slash '*_536_*.dat'])) disp('Reprocessing Vaisala WXT data...') @@ -183,6 +177,7 @@ end %% Re-load L1 and L2 product and plot each for comparison + load([l1file.folder slash l1file.name],'SWIFT','sinfo'); SWIFTL1 = SWIFT; l2file = dir([missiondir slash '*L2.mat']); @@ -201,6 +196,8 @@ print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') end + close all + diary off end diff --git a/GeneralTools/KZ_Processing/postprocessSWIFT.m b/GeneralTools/KZ_Processing/postprocessSWIFT.m new file mode 100644 index 0000000..eb489fb --- /dev/null +++ b/GeneralTools/KZ_Processing/postprocessSWIFT.m @@ -0,0 +1,206 @@ +% [SWIFT,sinfo] = postprocess_SWIFT(expdir) + +% Master post-processing function, that calls sub-functions to reprocess +% each different type of raw SWIFT data and create an L3 product. +% L1 product must have been created prior to running this script, +% by running 'compileSWIFT.m', and L2 created by running 'qcSWIFT.m'; + +% K. Zeiden 07/2024 + +%% User defined experiment directory (to be later converted to function inputs) + +if ispc + slash = '\'; +else + slash = '/'; +end + +expdir = ['S:' slash 'SEAFAC' slash 'June2024']; + +% Processing toggles +rpIMU = false; % Waves +rpSBG = false; % Waves +rpWXT = false; % MET +rpY81 = true; % MET +rpACS = false; % CT +rpSIG = false; % TKE +rpAQH = false; % TKE +rpAQD = false; % TKE + +% Plotting toggle +plotL1L2 = true; + +% List of missions +missions = dir([expdir slash 'SWIFT1*']); +missions = missions([missions.isdir]); + +%% Loop through missions and reprocess +for im = 1:length(missions) + + missiondir = [missions(im).folder slash missions(im).name]; + cd(missiondir) + + diaryfile = [missions(im).name '_postprocessSWIFT.txt']; + if exist(diaryfile,'file') + delete(diaryfile); + end + diary(diaryfile) + + disp(['Post-processing ' missions(im).name]) + + %% Locate L1 product, skip if does not exist. + % Else create 'sinfo' and modify L1 product. + l1file = dir([missiondir slash '*L1.mat']); + if isempty(l1file) + disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + else + load([l1file.folder slash l1file.name],'SWIFT'); + if isfield(SWIFT,'ID') + disp('Create information structure ''sinfo''') + sinfo.ID = SWIFT(1).ID; + sinfo.CTdepth = SWIFT(1).CTdepth; + if isfield(SWIFT,'metheight') + sinfo.metheight = SWIFT(1).metheight; + end + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + else + sinfo.type = 'V3'; + end + disp('Updating L1 product sinfo...') + save([l1file.folder slash l1file.name],'SWIFT','sinfo') + end + % One time, will remove later + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + save([l1file.folder slash l1file.name],'sinfo','-append') + else + sinfo.type = 'V3'; + save([l1file.folder slash l1file.name],'sinfo','-append') + end + end + + %% Reprocess IMU + if rpIMU + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) + disp('Reprocessing IMU data...') + calctype = 'IMU'; + filtertype = 'RC'; + saveraw = false; + interpf = false; + [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw,interpf); + else + disp('No IMU data...') + end + end + + %% Reprocess SBG + if rpSBG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) + disp('Reprocessing SBG data...') + saveraw = false; + useGPS = false; + interpf = false; + [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); + else + disp('No SBG data...') + end + end + + %% Reprocess WXT (536, confusing b/c CT15 can also be 536) + if rpWXT + if ~isempty(dir([missiondir slash 'WXT' slash 'Raw' slash '*' slash '*_536_*.dat'])) + disp('Reprocessing Vaisala WXT data...') + readraw = false; + usewind = false; + + [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw,usewind); + else + disp('No WXT data...') + end + end + + %% Reprocess Y81 + if rpY81 + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) + disp('Reprocessing Y81 Sonic Anemometer data...') + [SWIFT,sinfo] = reprocess_Y81(missiondir); + else + disp('No Y81 data...') + end + end + + %% Reprocess ACS + if rpACS + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) + disp('Reprocessing ACS CT data...') + readraw = false; + [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); + else + disp('No ACS data...') + end + end + + %% Reprocess SIG + if rpSIG + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) + disp('Reprocessing Signature1000 data...') + plotburst = false; + readraw = false; + [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); + else + disp('No SIG data...') + end + end + + %% Reprocess AQD + if rpAQD + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) + disp('Reprocessing Aquadopp (AQD) data...') + readraw = true; + [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); + else + disp('No AQD data...') + end + end + + %% Reprocess AQH + if rpAQH + if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) + disp('Reprocessing Aquadopp (AQH) data...') + readraw = false; + plotburst = false; + [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); + else + disp('No AQH data...') + end + end + + %% Re-load L1 and L2 product and plot each for comparison + + load([l1file.folder slash l1file.name],'SWIFT','sinfo'); + SWIFTL1 = SWIFT; + l2file = dir([missiondir slash '*L2.mat']); + load([l2file.folder slash l2file.name],'SWIFT'); + SWIFTL2 = SWIFT; + + if plotL1L2 + if strcmp(sinfo.type,'V3') + fh1 = plotSWIFTV3(SWIFTL1); + fh2 = plotSWIFTV3(SWIFTL2); + else + fh1 = plotSWIFTV4(SWIFTL1); + fh2 = plotSWIFTV4(SWIFTL2); + end + set(fh1,'Name',l1file.name(1:end-4)) + set(fh2,'Name',l2file.name(1:end-4)) + print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') + print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') + end + +diary off +end + + + + diff --git a/GeneralTools/compileSWIFT_SBDservertelemetry.m b/GeneralTools/compileSWIFT_SBDservertelemetry.m index 6f6454a..6ba333b 100644 --- a/GeneralTools/compileSWIFT_SBDservertelemetry.m +++ b/GeneralTools/compileSWIFT_SBDservertelemetry.m @@ -18,7 +18,7 @@ % and give messages for burst screening % 9/2019 force timestamp from filename always, rather than Airmar % 7/2022 allow microSWIFT timestamps (not from filename) -clear all, +clear all plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights @@ -27,15 +27,15 @@ disp('-------------------------------------') disp('Check QC settings... currently using:') -minwaveheight = 0 % minimum wave height in data screening +minwaveheight = 0; % minimum wave height in data screening -minsalinity = 0 % PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) +minsalinity = 0; % PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) -maxdriftspd = 5 % m/s, this is applied to telemetry drift speed, but reported drift is calculated after that +maxdriftspd = 5; % m/s, this is applied to telemetry drift speed, but reported drift is calculated after that -maxwindspd = 30 % m/s for malfunctioning Airmars +maxwindspd = 30; % m/s for malfunctioning Airmars -minairtemp = -20 % min airtemp +minairtemp = -20; % min airtemp disp('-------------------------------------') wd = pwd; @@ -61,6 +61,7 @@ else battery(ai) = voltage; end + oneSWIFT.battery = battery; if isempty(oneSWIFT.lat) | isempty(oneSWIFT.lon), oneSWIFT.lat = NaN; diff --git a/GeneralTools/concatSWIFT_offloadedSDcard.m b/GeneralTools/concatSWIFT_offloadedSDcard.m index 125ce0a..333d286 100644 --- a/GeneralTools/concatSWIFT_offloadedSDcard.m +++ b/GeneralTools/concatSWIFT_offloadedSDcard.m @@ -14,18 +14,12 @@ clear all, close all, clc -if ispc - slash = '\'; -else - slash = '/'; -end - plotflag = 1; % binary flag for plotting burstinterval = 12; % minutes between bursts burstlength = 512/60; % minutes of sampling during each burst -dirpath = ['.' slash];%'~/Desktop/'; '~/Dropbox/SWIFT_v3.x/TestData/'; -sourcedir = '';%'SWIFT11_02Oct2014_SDcard';%'SWIFT15_Test_19Jun2014'%'SWIFTdata_27Apr2014' %'SWIFT15_LakeWA_Stereo_09May2014'; +dirpath = './'%'~/Desktop/'; '~/Dropbox/SWIFT_v3.x/TestData/'; +sourcedir = ''%'SWIFT11_02Oct2014_SDcard';%'SWIFT15_Test_19Jun2014'%'SWIFTdata_27Apr2014' %'SWIFT15_LakeWA_Stereo_09May2014'; %% glob all processed files (per instrument) % and put these all in one directory (skip if already in one directory) @@ -121,10 +115,9 @@ if isfield(SWIFT(1),'ID') load(['SWIFT' SWIFT(1).ID '_telemetry.mat']) - save(['SWIFT' SWIFT(1).ID '_' datestr(min([SWIFT.time]),'ddmmmyyyy') '_L1.mat' ], 'SWIFT') + save(['SWIFT' SWIFT(1).ID '_' datestr(min([SWIFT.time]),'ddmmmyyyy') '-' datestr(max([SWIFT.time]),'ddmmmyyyy') '_L1.mat' ], 'SWIFT') eval(['!rm *telemetry.mat']) else end -plotSWIFT(SWIFT) - +plotSWIFT(SWIFT) \ No newline at end of file diff --git a/GeneralTools/findoutliersSWIFT.asv b/GeneralTools/findoutliersSWIFT.asv deleted file mode 100644 index 9fec0fc..0000000 --- a/GeneralTools/findoutliersSWIFT.asv +++ /dev/null @@ -1,94 +0,0 @@ -function [nonoutliersSWIFT,outliersSWIFT,booloutSWIFT] = findoutliersSWIFT(SWIFT, method, pct,varargin) -% findoutliersSWIFT - finds outliers out of % range and plots hist of all -% SWIFT fields -% Find outliers in SWIFT L1 DATA -% Michael James -% 7_12_2024 -% findoutliersSWIFT(SWIFT, method, pctdatathresh,varargin) -% method == -% defined as "percentile" or "movmedian" (moving percentile) -% pct == -% defined as percent used in percentile method, write as "~" or leave blank for -% movmedian -% varargin defined as string / value dictionary pairs. List below: -% 'plot_results' ; true/false bool (IN DEVELOPMENT) -% 'window' ; float (this is a window for "movmedian") -% 7_23_2024 -% Scoped down to per SWIFT variable so can be run in created loop -% determined by user -% Added boolean output for ease of indexing - - -% Preset binary flags -vararginlist = ["plot_results" ; "window"]; -plot_results = true; %default create figures -window = 40; % Window size of 100 values as default - -% Check if there are any optional arguments -if ~isempty(varargin) - if mod(length(varargin),2) == 0 %check for pairs - for i= 1:2:length(varargin) - if mean(contains(string(vararginlist),char(varargin{i}))) ~=0 - eval([char(vararginlist(contains(string(vararginlist),varargin{i}))) ' = varargin{i+1};']); - % taking the name and setting the variable to be the user - % input - else - error('Check naming of the variable arguements') - end - end; - else - error('incorrect variable arguements; must be in pairs') - end; -elseif isempty(varargin) - % Do nothing -else - error('Input error'); help findoutliersSWIFT; -end - - -if mean(contains(["percentile" ; "movmedian"],char(method))) ==0 - error('use "percentile" or "movmedian" to define the method'); -end - - -if string(method) == "percentile" - if 0 maxdrift; + SWIFT(baddrift) = []; + end + +%% Log QC and flags, then save new L2 file or overwrite existing one + +if isfield(sinfo,'postproc') +ip = length(sinfo.postproc)+1; +else + sinfo.postproc = struct; + ip = 1; +end +sinfo.postproc(ip).type = 'Y81'; +sinfo.postproc(ip).usr = getenv('username'); +sinfo.postproc(ip).time = string(datetime('now')); +sinfo.postproc(ip).flags = []; +sinfo.postproc(ip).params = []; + +save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') + + + +end \ No newline at end of file diff --git a/GeneralTools/readSWIFT_SBD.m b/GeneralTools/readSWIFT_SBD.m index 5168d5b..a12fe98 100644 --- a/GeneralTools/readSWIFT_SBD.m +++ b/GeneralTools/readSWIFT_SBD.m @@ -1,4 +1,4 @@ -function [SWIFT, BatteryVoltage ] = readSWIFT_SBD( fname , plotflag ); +function [SWIFT, BatteryVoltage ] = readSWIFT_SBD( fname , plotflag ) % Matlab readin of SWIFT Iridum SBD (short burst data) messages % which are binary files with onboard processed results % see Sutron documentation for message format @@ -83,16 +83,16 @@ %% SWIFT id flag from file name % note that all telemetry files from server start with 5 char 'buoy-' % this will fail for any other prefix of file naming convention +ibuoy = strfind(fname,'buoy'); -if fname(6)=='S', % SWIFT v3 and v4 - SWIFT.ID = fname(12:13); -elseif fname(6)=='m', % microSWIFT - SWIFT.ID = fname(17:19); +if fname(ibuoy + 5)=='S' % SWIFT v3 and v4 + SWIFT.ID = fname(ibuoy + (11:12)); +elseif fname(ibuoy + 5)=='m' % microSWIFT + SWIFT.ID = fname(ibuoy + (16:18)); else SWIFT.ID = NaN; end - %% begin reading file % Note that the actual email attachment from the Iridium system has an % ASCII header " ,,,: " @@ -112,7 +112,7 @@ while 1 - disp('-----------------') + % disp('-----------------') type = fread(fid,1,'uint8'); port = fread(fid,1,'uint8'); size = fread(fid,1,'uint16'); @@ -375,20 +375,20 @@ elseif type == 14 & size == 24, % CT15 radiometer with radiance (newer) disp(['reading CT15 Radiometer results, size = ' num2str(size)]) - SWIFT.radiometertemp1mean(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometertemp1std(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometertemp2mean(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometertemp2std(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometerrad1(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometerrad2(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.infraredtempmean(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.infraredtempstd(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.ambienttempmean(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.ambienttempstd(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.radiancemean(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.radiancestd(Radcounter + 1) = fread(fid,1,'float'); Radcounter = Radcounter + 1; elseif type == 14 & size == 16, % CT15 radiometer without radiance (older) disp(['reading CT15 Radiometer results, size = ' num2str(size)]) - SWIFT.radiometertemp1mean(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometertemp1std(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometertemp2mean(Radcounter + 1) = fread(fid,1,'float'); - SWIFT.radiometertemp2std(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.infraredtempmean(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.infraredtempstd(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.ambienttempmean(Radcounter + 1) = fread(fid,1,'float'); + SWIFT.ambienttempstd(Radcounter + 1) = fread(fid,1,'float'); Radcounter = Radcounter + 1; elseif type == 50 & size ==1228, % microSWIFT, size should be 1228 bytes @@ -606,7 +606,20 @@ SWIFT.peakwavedirT = NaN; end +%% change indicator for no conductivity - temperature + +% use NaN instead of 9999 for no data +if isfield(SWIFT,'salinity') + if all(SWIFT.salinity >= 9999), + SWIFT.salinity = NaN; + end +end +if isfield(SWIFT,'watertemp') + if all(SWIFT.watertemp >= 9999), + SWIFT.watertemp = NaN; + end +end %% fill in time if none read (time is a required field) diff --git a/SBG/reprocess_SBG.m b/SBG/reprocess_SBG.m index eb0367b..f2c740b 100644 --- a/SBG/reprocess_SBG.m +++ b/SBG/reprocess_SBG.m @@ -59,26 +59,26 @@ end % SBG time - time = datenum(sbgData.UtcTime.year, sbgData.UtcTime.month, sbgData.UtcTime.day, sbgData.UtcTime.hour,... + btime = datenum(sbgData.UtcTime.year, sbgData.UtcTime.month, sbgData.UtcTime.day, sbgData.UtcTime.hour,... sbgData.UtcTime.min, sbgData.UtcTime.sec + sbgData.UtcTime.nanosec./1e9); % Find matching time index in the existing SWIFT structure - btime = median(time,'omitnan');% First entries are bad (no satellites acquired yet) - [tdiff,tindex] = min(abs([SWIFT.time]-btime)); + % First entries are bad (no satellites acquired yet) + [tdiff,tindex] = min(abs([SWIFT.time]-median(btime,'omitnan'))); if tdiff > 1/(5*24) % If no close time index, skip burst disp('No time match. Skipping...') continue end % If not enough data to work with, skip burst - if isempty(tindex) || length(time)= 2*wsecs && fs>1 && sum(bad) < 0.1*pts % minimum length and quality for processing - - -%% break into windows (use 75 percent overlap) -if rem(windowlength,2)~=0 - windowlength = windowlength-1; -end % make w an even number -windows = floor( 4*(pts/windowlength - 1)+1 ); % number of windows, the 4 comes from a 75% overlap -dof = 2*windows*merge; % degrees of freedom -% loop to create a matrix of time series, where COLUMN = WINDOW -uwindow = zeros(windowlength,windows); -vwindow = zeros(windowlength,windows); -wwindow = zeros(windowlength,windows); -for q=1:windows - uwindow(:,q) = u( (q-1)*(.25*windowlength)+1 : (q-1)*(.25*windowlength)+windowlength ); - vwindow(:,q) = v( (q-1)*(.25*windowlength)+1 : (q-1)*(.25*windowlength)+windowlength ); - wwindow(:,q) = w( (q-1)*(.25*windowlength)+1 : (q-1)*(.25*windowlength)+windowlength ); +%% Ensure sufficient data +% (this should be taken care of outside of function actually) + +% if N <= 2*nwin +% ustar = NaN; +% epsilon = NaN; +% freq = NaN(1,116); +% tkespectrum = NaN(1,116); +% anisotropy = NaN; +% quality = 0; +% disp('Not enough points to compute spectra...') +% return +% end + +%% Break into windows with 75% overlap. Windows are matrix columns. + +U = NaN(nwin,M); +V = NaN(nwin,M); +W = NaN(nwin,M); + +for iwin = 1:M + j2win = (iwin-1)*(nwin/4)+1 : (iwin-1)*(nwin/4)+nwin; + U(:,iwin) = u(j2win); + V(:,iwin) = v(j2win); + W(:,iwin) = w(j2win); end -%% detrend individual windows (full series already detrended) -udetrend = zeros(windowlength,windows); -vdetrend = zeros(windowlength,windows); -wdetrend = zeros(windowlength,windows); -for q=1:windows - udetrend(:,q) = detrend(uwindow(:,q)); - vdetrend(:,q) = detrend(vwindow(:,q)); - wdetrend(:,q) = detrend(wwindow(:,q)); +%% Rotate into down-stream and cross-stream coordinates +% Ur = downstream, Vr = horizontal cross stream, +% Wr = vertical cross stream + +Ur = NaN(size(U)); +Vr = NaN(size(V)); +Wr = NaN(size(W)); + +% Rotate horizontally into downstream direction +thetaH = atan2d(mean(V),mean(U)); +for iwin = 1:M +R1 = [cosd(thetaH(iwin)), -sind(thetaH(iwin));... + sind(thetaH(iwin)), cosd(thetaH(iwin))]; + +UV_rot = R1'*([U(:,iwin)'; V(:,iwin)']); +Ur(:,iwin) = UV_rot(1,:); +Vr(:,iwin) = UV_rot(2,:); end -%% taper and rescale (to preserve variance) -% form taper matrix (columns of taper coef) -taper = sin ( (1:windowlength) * pi/windowlength )' * ones(1,windows); -% taper each window -xwindowtaper = udetrend .* taper; -ywindowtaper = vdetrend .* taper; -zwindowtaper = wdetrend .* taper; -% now find the correction factor (comparing old/new variance) -factx = sqrt( var(udetrend) ./ var(xwindowtaper) ); -facty = sqrt( var(vdetrend) ./ var(ywindowtaper) ); -factz = sqrt( var(wdetrend) ./ var(zwindowtaper) ); -% and correct for the change in variance -% (mult each window by it's variance ratio factor) -xwindowready = (ones(windowlength,1)*factx).* xwindowtaper; -ywindowready = (ones(windowlength,1)*facty).* ywindowtaper; -zwindowready = (ones(windowlength,1)*factz).* zwindowtaper; - - -%% FFT -% calculate Fourier coefs -Uwindow = fft(xwindowready); -Vwindow = fft(ywindowready); -Wwindow = fft(zwindowready); -% second half of fft is redundant, so throw it out -Uwindow( (windowlength/2+1):windowlength, : ) = []; -Vwindow( (windowlength/2+1):windowlength, : ) = []; -Wwindow( (windowlength/2+1):windowlength, : ) = []; -% throw out the mean (first coef) and add a zero (to make it the right length) -%Uwindow(1,:)=[]; Vwindow(1,:)=[]; Wwindow(1,:)=[]; -Uwindow(1:(windowlength/2-1),:) = Uwindow(2:(windowlength/2),:); -Vwindow(1:(windowlength/2-1),:) = Vwindow(2:(windowlength/2),:); -Wwindow(1:(windowlength/2-1),:) = Wwindow(2:(windowlength/2),:); -Uwindow(windowlength/2,:)=0; Vwindow(windowlength/2,:)=0; Wwindow(windowlength/2,:)=0; +% Rotate vertically into tilt direction +thetaV = atan2d(mean(W),mean(Ur)); +for iwin = 1:M +R1 = [cosd(thetaV(iwin)), -sind(thetaV(iwin));... + sind(thetaV(iwin)), cosd(thetaV(iwin))]; + +UrW_rot = R1'*([Ur(:,iwin)'; W(:,iwin)']); +Ur(:,iwin) = UrW_rot(1,:); +Wr(:,iwin) = UrW_rot(2,:); +end + +%% Detrend windows +Ud = detrend(Ur); +Vd = detrend(Vr); +Wd = detrend(Wr); + +%% Taper windows to prevent edge effects in spectra +taper = sin((1:nwin)*pi/nwin)'; + +% Apply taper +Ut = Ud.*repmat(taper,1,M); +Vt = Vd.*repmat(taper,1,M); +Wt = Wd.*repmat(taper,1,M); + +%% Rescale to preserve variance + +% Variance correction factor (preserve original variance) +usca = sqrt(var(Ud)./var(Ut)); +vsca = sqrt(var(Vd)./var(Vt)); +wsca = sqrt(var(Wd)./var(Wt)); + +% Apply scaling +Us = Ut.*repmat(usca,nwin,1); +Vs = Ut.*repmat(vsca,nwin,1); +Ws = Ut.*repmat(wsca,nwin,1); + +%% Compute power-spectra (auto) and cross-spectra + +% Fourier Coefficients +FU = fft(Us); +FV = fft(Vs); +FW = fft(Ws); + +% FFT is symmetric, so keep only positive frequencies +% (need to double the power later) +FU = FU(1:nwin/2,:); +FV = FV(1:nwin/2,:); +FW = FW(1:nwin/2,:); + +% Throw out the mean (first coef) and add a zero (to make it the right length) +FU(1:(nwin/2-1),:) = FU(2:(nwin/2),:); +FV(1:(nwin/2-1),:) = FV(2:(nwin/2),:); +FW(1:(nwin/2-1),:) = FW(2:(nwin/2),:); +FU(nwin/2,:) = 0; FV(nwin/2,:) = 0; FW(nwin/2,:) = 0; + % POWER SPECTRA (auto-spectra) -UUwindow = real ( Uwindow .* conj(Uwindow) ); -VVwindow = real ( Vwindow .* conj(Vwindow) ); -WWwindow = real ( Wwindow .* conj(Wwindow) ); +PU = 2*real(FU.*conj(FU)); +PV = 2*real(FV.*conj(FV)); +PW = 2*real(FW.*conj(FW)); + % CROSS-SPECTRA -UVwindow = ( Uwindow .* conj(Vwindow) ); -UWwindow = ( Uwindow .* conj(Wwindow) ); -VWwindow = ( Vwindow .* conj(Wwindow) ); - - -%% merge neighboring freq bands (number of bands to merge is a fixed parameter) -% initialize -UUwindowmerged = zeros(floor(windowlength/(2*merge)),windows); -VVwindowmerged = zeros(floor(windowlength/(2*merge)),windows); -WWwindowmerged = zeros(floor(windowlength/(2*merge)),windows); -UVwindowmerged = 1i*ones(floor(windowlength/(2*merge)),windows); -UWwindowmerged = 1i*ones(floor(windowlength/(2*merge)),windows); -VWwindowmerged = 1i*ones(floor(windowlength/(2*merge)),windows); - -for mi = merge:merge:(windowlength/2) - UUwindowmerged(mi/merge,:) = mean( UUwindow((mi-merge+1):mi , : ) ); - VVwindowmerged(mi/merge,:) = mean( VVwindow((mi-merge+1):mi , : ) ); - WWwindowmerged(mi/merge,:) = mean( WWwindow((mi-merge+1):mi , : ) ); - UVwindowmerged(mi/merge,:) = mean( UVwindow((mi-merge+1):mi , : ) ); - UWwindowmerged(mi/merge,:) = mean( UWwindow((mi-merge+1):mi , : ) ); - VWwindowmerged(mi/merge,:) = mean( VWwindow((mi-merge+1):mi , : ) ); -end -% freq range and bandwidth -n = (windowlength/2) / merge; % number of f bands -Nyquist = .5 * fs; % highest spectral frequency -bandwidth = Nyquist/n ; % freq (Hz) bandwitdh -% find middle of each freq band, ONLY WORKS WHEN MERGING ODD NUMBER OF BANDS! -f = 1/(wsecs) + bandwidth/2 + bandwidth.*(0:(n-1)) ; -freq = f; - - -%% normalize (to get spectral density)... divide by N*samplerate to get power spectral density -% the two is b/c Matlab's fft output is the symmetric FFT, and we did not use the redundant half (so need to multiply the psd by 2) - -UUwindowmerged = ( UUwindowmerged ) / (windowlength/2 * fs ); -VVwindowmerged = ( VVwindowmerged ) / (windowlength/2 * fs ); -WWwindowmerged = ( WWwindowmerged ) / (windowlength/2 * fs ); -UVwindowmerged = ( UVwindowmerged ) / (windowlength/2 * fs ); -UWwindowmerged = ( UWwindowmerged ) / (windowlength/2 * fs ); -VWwindowmerged = ( VWwindowmerged ) / (windowlength/2 * fs ); - - -%% find interial sub range (hard wired or dynamic), -% then get dissipation rate and ustar using the vertical component -% do this for each window, rather than ensemble spectra, -% because the advected velocity might change between windows (if vehicle turns, etc) - -fmin = 2; % inertial sub-range, min freq -fmax = 4.9; % inertial sub-range, max freq -inertialfreqs = f > fmin & f < fmax ; -inertiallevel = mean( (f(inertialfreqs)'.^(5/3)*ones(1,windows)) .* WWwindowmerged(inertialfreqs,:) ); % average value of compensated spectra -inertialstd = std( (f(inertialfreqs)'.^(5/3)*ones(1,windows)) .* WWwindowmerged(inertialfreqs,:) ); % average value of compensated spectra -advectionspeed = ( mean(uwindow).^2 + mean(vwindow).^2 ) .^ 0.5; % speed at which frozen field turbulence is advected past sensor -epsilonwindow = ( inertiallevel ./ ( ( advectionspeed ./ (2*pi) ).^(2/3) .* K ) ).^(3/2); -ustarwindow = (kv * epsilonwindow * z ).^(1/3); % assumes neutral - -%% quality metrics -qualitywindow = (inertiallevel-inertialstd)./ inertiallevel; % quality of fit in ISR -%qualitywindow = WWwindowmerged(1,:)./WWwindowmerged(end,:); % low frequency contamination -%qualitywindow = ( std(uwindow) + std(vwindow) )./ advectionspeed; % advective speed variations - - - -%% ensemble average windows together -% take the average of all windows at each freq-band -UU = mean( UUwindowmerged',1,'omitnan') ; -VV = mean( VVwindowmerged',1,'omitnan') ; -WW = mean( WWwindowmerged',1,'omitnan') ; -UV = mean( UVwindowmerged',1,'omitnan') ; -UW = mean( UWwindowmerged',1,'omitnan') ; -VW = mean( VWwindowmerged',1,'omitnan') ; - -% find the windows with stable mean in the streamwise direction, use only those for ustar and epsilon ensembles -good = abs(mean(uwindow)) > std(uwindow); %disp('stable windows'), sum(good) % debug - -if sum(good) >= 2 - epsilon = mean(epsilonwindow(good)); - ustar = mean(ustarwindow(good)); - quality = mean(qualitywindow(good)); -else - ustar = 9999; - epsilon = 9999; - quality = mean(qualitywindow); +CUV = 2*(FU.*conj(FV)); +CUW = 2*(FU.*conj(FW)); +CVW = 2*(FV.*conj(FW)); + +%% Average within designated frequency bands + +PUm = NaN(nf,M); +PVm = NaN(nf,M); +PWm = NaN(nf,M); +CUVm = 1i*ones(nf,M); +CUWm = 1i*ones(nf,M); +CVWm = 1i*ones(nf,M); + +for ifreq = 1:nf + iband = (ifreq-1)*nfmerge + (1:nfmerge); + PUm(ifreq,:) = mean(PU(iband,:)); + PVm(ifreq,:) = mean(PV(iband,:)); + PWm(ifreq,:) = mean(PW(iband,:)); + CUVm(ifreq,:) = mean(CUV(iband,:)); + CUWm(ifreq,:) = mean(CUW(iband,:)); + CVWm(ifreq,:) = mean(CVW(iband,:)); end +%% Normalize by N*fs to get power spectral density -%% sum component spectra to get proxy for TKE spectra +PUm = PUm/(fs*nwin); +PVm = PVm/(fs*nwin); +PWm = PWm/(fs*nwin); +CUVm = CUVm/(fs*nwin); +CUWm = CUWm/(fs*nwin); +CVWm = CVWm/(fs*nwin); -tkespectrum = ( UU + VV + WW ); +%% Calculate dissipation rate and friction velocity for each window +% Advective velocity may be different for each window -% !!! Mike S: Fix output to exactly length 116 - Assumes: wsecs = 256, -% merge = 11, fs = 10 !!! -tkespectrum = tkespectrum(1:116); -freq = freq(1:116); +% Frequencies in ISR +isrf = f > fmin & f < fmax ; +% ISR mean energy level (compensated spectra) +Eisr = mean(repmat(f(isrf)'.^(5/3),1,M).* PUm(isrf,:)); +% Standard deviation of compensated spectra +Eisrstd = std((f(isrf)'.^(5/3)*ones(1,M)).*PUm(isrf,:)); -%% anisotropy -anisotropy = ( mean(UU(inertialfreqs))./mean(WW(inertialfreqs)) + mean(VV(inertialfreqs))./mean(WW(inertialfreqs)) )./2; ; +% Downstream advective speed (frozen field) +Uadv = mean(Ur); -%% Quality control (check ustar against drag law) -dragcoef = ustar.^2 ./ (advectionspeed).^2 ; +% Dissipation rate +Eps = (Eisr./((Uadv./(2*pi)).^(2/3).*K)).^(3/2); -dragoutofrange = [ dragcoef > 1e-2 dragcoef < 1e-5 ]; -if all(dragoutofrange), - QCflag = true; -else - QCflag = false; -end +% Friction velocity (assumes neutral stability) +Ustar = (kv*Eps*z).^(1/3); -%% housekeeping - -else % if not enough points or sufficent sampling rate or data, give 9999 - - ustar = 9999; - epsilon = 9999; - % !!! Mike S: Fix output to exactly length 116 - Assumes: wsecs = 256, - % merge = 11, fs = 10 !!! - freq = 9999*ones(1,116); - tkespectrum = 9999*ones(1,116); - anisotropy = 9999; - QCflag = true; - quality = 0; - disp('not enough pts') +%% Ensemble average across windows -end +upsd = mean(PUm,2); +vpsd = mean(PVm,2); +wpsd = mean(PWm,2); +uvcopsd = mean(CUVm,2); +uwcopsd = mean(CUWm,2); +vwcopsd = mean(CVWm,2); +% uadv = mean(Uadv); +epsilon = mean(Eps); +ustar = mean(Ustar); -% quality control -if QCflag == true, - ustar = 9999; - epsilon = 9999; -else -end +%% Sum component spectra to get proxy for TKE spectra +TKEpsd = (upsd + vpsd + wpsd); + +%% Quality Checks + +% ISR Fit quality +quality = mean((Eisr-Eisrstd)./Eisr); +% Anisotropy +anisotropy = (mean(upsd(isrf))./mean(wpsd(isrf)) + mean(vpsd(isrf))./mean(wpsd(isrf)) )./2; + +% Drag coefficient +% Cd = ustar.^2 ./ (uadv).^2; + +end diff --git a/Winds/readSWIFT_Y81.m b/Winds/readSWIFT_Y81.m index 42c6858..12ef8f8 100644 --- a/Winds/readSWIFT_Y81.m +++ b/Winds/readSWIFT_Y81.m @@ -18,4 +18,6 @@ % [ustar,epsilon,meanu,meanv,meanw,meantemp,anisotropy,quality,freq,tkespectrum] = ... % inertialdissipation(uvw(:,1), uvw(:,2), uvw(:,3), temp, z, fs); +save([filename(1:end-4) '.mat'],'u','v','w','temp','errorflag'); + end \ No newline at end of file diff --git a/Winds/reprocess_Y81.m b/Winds/reprocess_Y81.m index 1ba3baf..544a9c5 100644 --- a/Winds/reprocess_Y81.m +++ b/Winds/reprocess_Y81.m @@ -40,39 +40,25 @@ disp(['Burst ' num2str(iburst) ' : ' bfiles(iburst).name(1:end-4)]) - % if bfiles(iburst).bytes > 1e5 - % RMYdata = importdata([bfiles(iburst).folder slash bfiles(iburst).name]); - % uvw = RMYdata.data(:,1:3); - % temp = RMYdata.data(:,4); - % else - % uvw = NaN(1000,3); - % temp = NaN(1000,1); - % end - % windspd = mean((uvw(:,1).^2 + uvw(:,2).^2 + uvw(:,3).^2).^.5); - % u = uvw(:,1); - % v = uvw(:,2); - % w = uvw(:,3); - % Read raw data if bfiles(iburst).bytes > 1e5 [u,v,w,temp,~] = readSWIFT_Y81([bfiles(iburst).folder slash bfiles(iburst).name]); - windspd = mean( (u.^2 + v.^2 + w.^2).^.5 ); else + disp('File size too small, skipping ...') u = NaN(1000,1); v = u;w = u;temp = u; end + windspd = mean( (u.^2 + v.^2 + w.^2).^.5 ); % Recalculate friction velocity z = sinfo.metheight; fs = 10; - [ustar,~,~,~,~,~,~,~,~,~] = inertialdissipation(u,v,w,temp,z,fs); + [ustar,~,~,~,~,~,~,~,windfreq,windpower] = inertialdissipation(u,v,w,temp,z,fs); % Find matching time day = bfiles(iburst).name(13:21); hour = bfiles(iburst).name(23:24); mint = bfiles(iburst).name(26:27); - % time = datenum(bfiles(iburst).name(13:21)) + str2double(bfiles(iburst).name(23:24))./24 ... - % + str2double(bfiles(iburst).name(26:27))./(24*6); time = datenum(day)+datenum(0,0,0,str2double(hour),(str2double(mint)-1)*12,0); [tdiff,tindex] = min(abs([SWIFT.time]-time)); @@ -82,9 +68,11 @@ SWIFT(tindex).winddirR = NaN; if ustar ~= 9999 SWIFT(tindex).windustar = ustar; + SWIFT(tindex).windspectra.freq = windfreq(:); + SWIFT(tindex).windspectra.energy = windpower(:); end else - disp(['No time match, dt = ' num2str(tdiff*24*60) ' min...']) + disp(['No time match, dt = ' num2str(tdiff*24*60) ' min. Skipping...']) end end diff --git a/Winds/sonicdissipation.m b/Winds/sonicdissipation.m index 8bd58e1..925aeee 100644 --- a/Winds/sonicdissipation.m +++ b/Winds/sonicdissipation.m @@ -1,18 +1,29 @@ -function [eps,ustar,ps,cs,f,uadv,theta,iso,power] = sonicdissipation(u,v,w,temp,fs,twin,z) +function [eps,ustar,ps,cs,f,uadv,thetaH,iso,power] = sonicdissipation(u,v,w,temp,fs,twin,z) %%% Rotate to get streamwise + cross-stream velocities -theta = atan2d(mean(v,'omitnan'),mean(u,'omitnan')); -R = [cosd(theta), -sind(theta);... - sind(theta), cosd(theta)]; -uv_rot = R'*([u(:)'; v(:)']); + +% Rotate horizontally into downstream direction +thetaH = atan2d(mean(v,'omitnan'),mean(u,'omitnan')); +RH = [cosd(thetaH), -sind(thetaH);... + sind(thetaH), cosd(thetaH)]; +uv_rot = RH'*([u(:)'; v(:)']); u_rot = uv_rot(1,:); v_rot = uv_rot(2,:); + +% Rotate vertically into tilt direction +thetaV = atan2d(mean(w,'omitnan'),mean(u_rot,'omitnan')); +RV = [cosd(thetaV), -sind(thetaV);... + sind(thetaV), cosd(thetaV)]; +uw_rot = RV'*([u_rot(:)'; w(:)']); +u_rot = uw_rot(1,:); +w_rot = uw_rot(2,:); + uadv = mean(u_rot,'omitnan'); %%% Convert direction to oceanographic convention -theta = -theta + 90; -if theta < 0 - theta = theta + 360; +thetaH = -thetaH + 90; +if thetaH < 0 + thetaH = thetaH + 360; end % Spectra @@ -20,13 +31,13 @@ norm = 'par'; [UU,f,~,~,~] = hannwinPSD2(u_rot,nwin,fs,norm); [VV,~,~,~,~] = hannwinPSD2(v_rot,nwin,fs,norm); -[WW,~,~,~,~] = hannwinPSD2(w,nwin,fs,norm); +[WW,~,~,~,~] = hannwinPSD2(w_rot,nwin,fs,norm); [TT,~,~,~,~] = hannwinPSD2(temp,nwin,fs,norm); % Cospectra -[UW,~,~] = hannwinCOPSD2(u,w,nwin,fs,norm); -[VW,~,~] = hannwinCOPSD2(v,w,nwin,fs,norm); -[TW,~,~] = hannwinCOPSD2(temp,w,nwin,fs,norm); +[UW,~,~] = hannwinCOPSD2(u_rot,w_rot,nwin,fs,norm); +[VW,~,~] = hannwinCOPSD2(v_rot,w_rot,nwin,fs,norm); +[TW,~,~] = hannwinCOPSD2(temp,w_rot,nwin,fs,norm); % Dissipation Rate and Friction Velocity K = 0.55; From b9e7234264cd4cba308cfac25d979f65ca4e915c Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 10 Oct 2024 12:45:11 -0700 Subject: [PATCH 29/34] updates --- GeneralTools/KZ_Processing/compileSWIFT.asv | 132 +-------- GeneralTools/KZ_Processing/compileSWIFT.m | 274 +++--------------- GeneralTools/KZ_Processing/createSBD.m | 11 +- GeneralTools/KZ_Processing/postprocessSWIFT.m | 13 +- GeneralTools/pruneSWIFT.asv | 106 +++++++ GeneralTools/pruneSWIFT.m | 92 ++++-- GeneralTools/qcSWIFT.m | 61 ---- 7 files changed, 248 insertions(+), 441 deletions(-) create mode 100644 GeneralTools/pruneSWIFT.asv delete mode 100644 GeneralTools/qcSWIFT.m diff --git a/GeneralTools/KZ_Processing/compileSWIFT.asv b/GeneralTools/KZ_Processing/compileSWIFT.asv index 093748a..33e336f 100644 --- a/GeneralTools/KZ_Processing/compileSWIFT.asv +++ b/GeneralTools/KZ_Processing/compileSWIFT.asv @@ -34,24 +34,6 @@ plotflag = true; % binary flag for plotting (compiled plots, not individual plo fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. -% % QC Parameters -% minwaveheight = 0;% minimum wave height in data screening -% minsalinity = 0;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) -% maxdriftspd = 5;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that -% maxwindspd = 30;% m/s for malfunctioning Airmars -% minairtemp = -20;% min airtemp -% maxairtemp = 50;% max airtemp -% -% disp('-------------------------------------') -% disp('QC settings:') -% disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) -% disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) -% disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) -% disp(['Maximum wind speed: ' num2str(maxwindspd) ' ms^{-1}']) -% disp(['Minimum air temp: ' num2str(minairtemp) ' C']) -% disp(['Maximum air temp: ' num2str(maxairtemp) ' C']) -% disp('-------------------------------------') - % List missions missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); @@ -134,16 +116,21 @@ for iburst = 1:nburst oneSWIFT = rmfield(oneSWIFT,'wavehistogram'); end - %%% Increment main structure %%% + %%% Add burst to SWIFT structure %%% burstpayloads = string(fieldnames(oneSWIFT)); npayloads(iburst) = length(burstpayloads); - if iburst == 1 + % Loop through paylaods on current burst, add to SWIFT structure + for ipay = 1:npayloads(iburst) + SWIFT(iburst).(burstpayloads{ipay}) = oneSWIFT.(burstpayloads{ipay}); + end + + % List initial payloads, as well as any missing or additional payloads + if iburst == 1 % Initial payloads allpayloads = burstpayloads; disp('Initial payloads:') disp(allpayloads) - elseif iburst ~= 1 && npayloads(iburst) > length(allpayloads) - + elseif iburst ~= 1 && npayloads(iburst) > length(allpayloads) % Additional payloads isnew = false(npayloads(iburst),1); for ip = 1:npayloads(iburst) isnew(ip) = ~any(strcmp(burstpayloads(ip),allpayloads)); @@ -152,9 +139,7 @@ for iburst = 1:nburst disp(['New payloads in burst file ' num2str(iburst) ':']) disp(newpayloads) allpayloads = burstpayloads; - - elseif iburst ~= 1 && npayloads(iburst) < length(allpayloads) - + elseif iburst ~= 1 && npayloads(iburst) < length(allpayloads)% Missing payloads ismissing = false(length(allpayloads),1); for ip = 1:length(allpayloads) ismissing(ip) = ~any(strcmp(allpayloads(ip),burstpayloads)); @@ -164,109 +149,14 @@ for iburst = 1:nburst disp(missingpayloads) end - for ipay = 1:npayloads(iburst) - SWIFT(iburst).(burstpayloads{ipay}) = oneSWIFT.(burstpayloads{ipay}); - end - - %%% Original Increment Method -- reinitialize if new payloads, skip if - %%% fewer payloads... misses a lot of good data - % % First SBD: set the structure fields as the standard - % if iburst == 1 && voltage ~= 9999 - % SWIFT(iburst) = oneSWIFT; - % initpayloads = burstpayloads; - % disp(['Initial payloads in burst file ' num2str(iburst) ':']) - % disp(initpayloads) - % - % elseif iburst == 1 && voltage == 9999 - % - % badburst(iburst) = true; - % initpayloads = []; - % - % % Payloads match: increment - % elseif iburst > 1 && npayloads(iburst) == length(initpayloads) - % SWIFT(iburst) = oneSWIFT; - % - % % Additional payloads: favor that new structure (removing other) - % elseif iburst > 1 && npayloads(iburst) > length(initpayloads) - % - % isnew = false(npayloads(iburst),1); - % for ip = 1:npayloads(iburst) - % isnew(ip) = ~any(strcmp(burstpayloads(ip),initpayloads)); - % end - % newpayloads = burstpayloads(isnew); - % - % clear SWIFT - % badburst(iburst-1) = true; - % SWIFT(iburst) = oneSWIFT; - % disp(['Found extra payloads in burst file ' num2str(iburst) ':']) - % disp(newpayloads) - % disp('...re-initializing SWIFT structure...') - % initpayloads = burstpayloads; % reset the prefer field names - % - % % Fewer paylaods: skip that burst - % elseif iburst > 1 && npayloads(iburst) < length(initpayloads) || voltage==9999 - % - % ismissing = false(length(initpayloads),1); - % for ip = 1:length(initpayloads) - % ismissing(ip) = ~any(strcmp(initpayloads(ip),burstpayloads)); - % end - % missingpayloads = initpayloads(ismissing); - % - % disp(['Missing payloads in burst file ' num2str(iburst) ':']) - % disp(missingpayloads) - % SWIFT(iburst) = SWIFT(iburst-1); % placeholder, which will be removed when badburst applied - % badburst(iburst) = true; - % disp('...skipping burst...') - % end - - %%% Screen the bad data (usually out of the water) %%% - % No data - if isempty(oneSWIFT.lon) || isempty(oneSWIFT.lat) || isempty(oneSWIFT.time) - badburst(iburst) = true; - disp('No position or timestamp!') - end - % No position - if oneSWIFT.lon == 0 || ~isnumeric(oneSWIFT.lon) || isnan(oneSWIFT.lon) - badburst(iburst) = true; - disp('No position!') - end - % Waves too small - if isfield(oneSWIFT,'sigwaveheight') - if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 - badburst(iburst) = true; - disp('Waves too small, removing burst.') - end - end - % Salinity too small - if isfield(oneSWIFT,'salinity') %&& ~micro - if all(oneSWIFT.salinity < minsalinity) % & all(~isnan(oneSWIFT.salinity)), - badburst(iburst) = true; - disp('Salinity too low, removing burst.') - end - end - % Drift speed limit - if isfield(oneSWIFT,'driftspd') - if oneSWIFT.driftspd > maxdriftspd - badburst(iburst) = true; - disp('Speed too fast, removing burst.') - end - end - disp('=================================') % End burst loop end diary off -% Remove bursts which had less payloads -% badburst(npayloads < length(initpayloads)) = true; - -%% Apply QC - -% SWIFT(badburst) = []; -% battery(badburst) = []; +%% Fill empty SWIFT fields due to missing payloads in a burst -%% Fill empty SWIFT fields payloads = fieldnames(SWIFT); npay = length(payloads); nburst = length(SWIFT); diff --git a/GeneralTools/KZ_Processing/compileSWIFT.m b/GeneralTools/KZ_Processing/compileSWIFT.m index 093748a..69a20e9 100644 --- a/GeneralTools/KZ_Processing/compileSWIFT.m +++ b/GeneralTools/KZ_Processing/compileSWIFT.m @@ -15,7 +15,13 @@ % K. Zeiden 10/01/2024 -%% Experiment directory and QC parameters (user defined) +%% Experiment directory +expdir = 'S:\SEAFAC\June2024'; + +% SBD folder +SBDfold = 'ProcessedSBD'; + +%% QC parameters (user defined) if ispc slash = '\'; @@ -23,42 +29,18 @@ slash = '/'; end -% Experiment Directory -expdir = 'S:\SEAFAC\June2024'; - -% SBD folder -SBDfold = 'ProcessedSBD'; - % Processing parameters plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. -% % QC Parameters -% minwaveheight = 0;% minimum wave height in data screening -% minsalinity = 0;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) -% maxdriftspd = 5;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that -% maxwindspd = 30;% m/s for malfunctioning Airmars -% minairtemp = -20;% min airtemp -% maxairtemp = 50;% max airtemp -% -% disp('-------------------------------------') -% disp('QC settings:') -% disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) -% disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) -% disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) -% disp(['Maximum wind speed: ' num2str(maxwindspd) ' ms^{-1}']) -% disp(['Minimum air temp: ' num2str(minairtemp) ' C']) -% disp(['Maximum air temp: ' num2str(maxairtemp) ' C']) -% disp('-------------------------------------') - % List missions missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); %% Loop through missions -for im = 1:length(missions) +for im = 1%:length(missions) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) @@ -134,16 +116,21 @@ oneSWIFT = rmfield(oneSWIFT,'wavehistogram'); end - %%% Increment main structure %%% + %%% Add burst to SWIFT structure %%% burstpayloads = string(fieldnames(oneSWIFT)); npayloads(iburst) = length(burstpayloads); - if iburst == 1 + % Loop through paylaods on current burst, add to SWIFT structure + for ipay = 1:npayloads(iburst) + SWIFT(iburst).(burstpayloads{ipay}) = oneSWIFT.(burstpayloads{ipay}); + end + + % List initial payloads, as well as any missing or additional payloads + if iburst == 1 % Initial payloads allpayloads = burstpayloads; disp('Initial payloads:') disp(allpayloads) - elseif iburst ~= 1 && npayloads(iburst) > length(allpayloads) - + elseif iburst ~= 1 && npayloads(iburst) > length(allpayloads) % Additional payloads isnew = false(npayloads(iburst),1); for ip = 1:npayloads(iburst) isnew(ip) = ~any(strcmp(burstpayloads(ip),allpayloads)); @@ -152,9 +139,7 @@ disp(['New payloads in burst file ' num2str(iburst) ':']) disp(newpayloads) allpayloads = burstpayloads; - - elseif iburst ~= 1 && npayloads(iburst) < length(allpayloads) - + elseif iburst ~= 1 && npayloads(iburst) < length(allpayloads)% Missing payloads ismissing = false(length(allpayloads),1); for ip = 1:length(allpayloads) ismissing(ip) = ~any(strcmp(allpayloads(ip),burstpayloads)); @@ -164,109 +149,14 @@ disp(missingpayloads) end - for ipay = 1:npayloads(iburst) - SWIFT(iburst).(burstpayloads{ipay}) = oneSWIFT.(burstpayloads{ipay}); - end - - %%% Original Increment Method -- reinitialize if new payloads, skip if - %%% fewer payloads... misses a lot of good data - % % First SBD: set the structure fields as the standard - % if iburst == 1 && voltage ~= 9999 - % SWIFT(iburst) = oneSWIFT; - % initpayloads = burstpayloads; - % disp(['Initial payloads in burst file ' num2str(iburst) ':']) - % disp(initpayloads) - % - % elseif iburst == 1 && voltage == 9999 - % - % badburst(iburst) = true; - % initpayloads = []; - % - % % Payloads match: increment - % elseif iburst > 1 && npayloads(iburst) == length(initpayloads) - % SWIFT(iburst) = oneSWIFT; - % - % % Additional payloads: favor that new structure (removing other) - % elseif iburst > 1 && npayloads(iburst) > length(initpayloads) - % - % isnew = false(npayloads(iburst),1); - % for ip = 1:npayloads(iburst) - % isnew(ip) = ~any(strcmp(burstpayloads(ip),initpayloads)); - % end - % newpayloads = burstpayloads(isnew); - % - % clear SWIFT - % badburst(iburst-1) = true; - % SWIFT(iburst) = oneSWIFT; - % disp(['Found extra payloads in burst file ' num2str(iburst) ':']) - % disp(newpayloads) - % disp('...re-initializing SWIFT structure...') - % initpayloads = burstpayloads; % reset the prefer field names - % - % % Fewer paylaods: skip that burst - % elseif iburst > 1 && npayloads(iburst) < length(initpayloads) || voltage==9999 - % - % ismissing = false(length(initpayloads),1); - % for ip = 1:length(initpayloads) - % ismissing(ip) = ~any(strcmp(initpayloads(ip),burstpayloads)); - % end - % missingpayloads = initpayloads(ismissing); - % - % disp(['Missing payloads in burst file ' num2str(iburst) ':']) - % disp(missingpayloads) - % SWIFT(iburst) = SWIFT(iburst-1); % placeholder, which will be removed when badburst applied - % badburst(iburst) = true; - % disp('...skipping burst...') - % end - - %%% Screen the bad data (usually out of the water) %%% - % No data - if isempty(oneSWIFT.lon) || isempty(oneSWIFT.lat) || isempty(oneSWIFT.time) - badburst(iburst) = true; - disp('No position or timestamp!') - end - % No position - if oneSWIFT.lon == 0 || ~isnumeric(oneSWIFT.lon) || isnan(oneSWIFT.lon) - badburst(iburst) = true; - disp('No position!') - end - % Waves too small - if isfield(oneSWIFT,'sigwaveheight') - if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 - badburst(iburst) = true; - disp('Waves too small, removing burst.') - end - end - % Salinity too small - if isfield(oneSWIFT,'salinity') %&& ~micro - if all(oneSWIFT.salinity < minsalinity) % & all(~isnan(oneSWIFT.salinity)), - badburst(iburst) = true; - disp('Salinity too low, removing burst.') - end - end - % Drift speed limit - if isfield(oneSWIFT,'driftspd') - if oneSWIFT.driftspd > maxdriftspd - badburst(iburst) = true; - disp('Speed too fast, removing burst.') - end - end - disp('=================================') % End burst loop end diary off -% Remove bursts which had less payloads -% badburst(npayloads < length(initpayloads)) = true; - -%% Apply QC +%% Fill empty SWIFT fields due to missing payloads in a burst -% SWIFT(badburst) = []; -% battery(badburst) = []; - -%% Fill empty SWIFT fields payloads = fieldnames(SWIFT); npay = length(payloads); nburst = length(SWIFT); @@ -303,51 +193,24 @@ end -%% Sort final structure +%% Sort final structure by time [~,tinds] = sort([SWIFT.time]); SWIFT = SWIFT(tinds); battery = battery(tinds); -%% Enforce a single value for CT and MET sensor heights - -if isfield(SWIFT,'CTdepth') - for si = 1:length(SWIFT) - SWIFT(si).CTdepth = median([SWIFT.CTdepth],'omitnan'); - end -end - -if isfield(SWIFT,'metheight') - for si = 1:length(SWIFT) - SWIFT(si).metheight = median([SWIFT.metheight],'omitnan'); - end -end - - -%% Fill position outliers - -if fixpositions - [cleanlon,cloni] = filloutliers([SWIFT.lon],'linear'); - [cleanlat,clati] = filloutliers([SWIFT.lat],'linear'); - if cloni == clati - for ci = find(cloni) - SWIFT(ci).lon = cleanlon(ci); - SWIFT(ci).lat = cleanlat(ci); - end - disp(['Filled ' num2str(sum(cloni)) ' position outliers.']) - end -end - -%% Recalculate drift (note that wind slip, which is 1%, is not removed) -% Drift speed is included from the Airmar results, -% but that sensor is not always available or included -% (so simpler to just calculate it from differencing positions). +%% Calculate drift speed +% Drift speed is included from the Airmar results, but not always avail. +% Compute drift speed by differencing position. +% Note that wind slip, which is 1%, is not removed. +% NaN out values associated with large time gaps. if length(SWIFT) > 3 - time = [SWIFT.time];%[time tinds ] = sort(time); - lat = [SWIFT.lat]; %lat = lat(tinds); - lon = [SWIFT.lon]; %lon = lon(tinds); - dlondt = gradient(lon,time); % deg per day + time = [SWIFT.time]; + lat = [SWIFT.lat]; + lon = [SWIFT.lon]; + dt = gradient(time); + dlondt = gradient(lon,time); dxdt = deg2km(dlondt,6371*cosd(mean(lat,'omitnan'))) .* 1000 ./ ( 24*3600 ); % m/s dlatdt = gradient(lat,time); % deg per day dydt = deg2km(dlatdt) .* 1000 ./ ( 24*3600 ); % m/s @@ -356,10 +219,10 @@ speed = sqrt(dxdt.^2 + dydt.^2); % m/s direction = -180 ./ 3.14 .* atan2(dydt,dxdt); % cartesian direction [deg] direction = direction + 90; % rotate from eastward = 0 to northward = 0 - direction( direction<0) = direction( direction<0 ) + 360; % make quadrant II 270->360 instead of -90->0 + direction( direction<0) = direction( direction<0 ) + 360; % make quadrant II 270->360 instead of -90 -> 0 for si = 1:length(SWIFT) - if si == 1 || si == length(SWIFT) + if si == 1 || si == length(SWIFT) || dt(si) > 1/12 SWIFT(si).driftspd = NaN; SWIFT(si).driftdirT = NaN; else @@ -368,88 +231,41 @@ end end - % % remove last burst, if big change in direction (suggests recovery by ship) - % dirchange = abs( SWIFT( length(SWIFT) - 2).driftdirT - SWIFT( length(SWIFT) - 1).driftdirT ); - % if dirchange > 90, - % disp('removing last burst, suspect includes ship recovery') - % SWIFT( length(SWIFT) - 1).driftdirT = NaN; - % SWIFT( length(SWIFT) - 1).driftspd = NaN; - % SWIFT( length(SWIFT) ) = []; - % battery( length(SWIFT) ) = []; - % end -elseif length(SWIFT) <= 3 +else + for si = 1:length(SWIFT) SWIFT(si).driftspd = NaN; SWIFT(si).driftdirT = NaN; end -end -% Quality control by removing drift results associated with large time gaps -if length([SWIFT.time]) > 1 - dt = gradient([SWIFT.time]); - for si = 1:length(SWIFT) - if dt(si) > 1/12 % 1/12 of day is two hours - SWIFT(si).driftspd = NaN; - SWIFT(si).driftdirT = NaN; - end - end -end - -% Quality control drift speeds too fast (prob on deck) with new drift spd -if length([SWIFT.time]) > 1 && isfield(SWIFT(1),'driftspd') - toofast = [SWIFT.driftspd] > maxdriftspd; - SWIFT( toofast ) =[]; - battery( toofast ) = []; end -%% Extrapolate missing low frequencies of wave energy spectra %% -% Require energy at lowest frequenc to be zero. -% Not neccessary if post-processing for raw displacements. -% Less necessary after Oct 2017 rev of onboard processing with improved RC filter. +%% Enforce a single value for CT and MET sensor heights -if fixspectra && isfield(SWIFT,'wavespectra') +if isfield(SWIFT,'CTdepth') for si = 1:length(SWIFT) - notzero = find(SWIFT(si).wavespectra.energy ~= 0 & SWIFT(si).wavespectra.freq > 0.04); - tobereplaced = find(SWIFT(si).wavespectra.energy == 0 & SWIFT(si).wavespectra.freq > 0.04); - if length(notzero) > 10 - E = interp1([0.04; SWIFT(si).wavespectra.freq(notzero)],[0; SWIFT(si).wavespectra.energy(notzero)],SWIFT(si).wavespectra.freq); - SWIFT(si).wavespectra.energy(tobereplaced) = E(tobereplaced); - df = median(diff(SWIFT(si).wavespectra.freq)); - SWIFT(si).sigwaveheight = 4*sqrt(sum(SWIFT(si).wavespectra.energy,'omitnan')*df); - end + SWIFT(si).CTdepth = median([SWIFT.CTdepth],'omitnan'); end end -%% Quality control wind speeds -if length([SWIFT.time]) > 1 && isfield(SWIFT,'windspd') +if isfield(SWIFT,'metheight') for si = 1:length(SWIFT) - if SWIFT(si).windspd > maxwindspd - SWIFT(si).windspd = NaN; - SWIFT(si).winddirT = NaN; - SWIFT(si).winddirR = NaN; - end + SWIFT(si).metheight = median([SWIFT.metheight],'omitnan'); end end -%% Quality control airmar temperature -if isfield(SWIFT,'airtemp') - for si = 1:length(SWIFT) - if SWIFT(si).airtemp == 0.0 || SWIFT(si).airtemp < minairtemp || SWIFT(si).airtemp > maxairtemp - SWIFT(si).airtemp = NaN; - SWIFT(si).windspd = NaN; - end - end -end - %% Sort the microSWIFT onboard processing, using the battery voltage as a flag % only applies to v1 microSWIFTs from 2022 IMU = find(battery==0); GPS = find(battery==1); - -if ~isempty(IMU), SWIFT_IMU = SWIFT(IMU); end -if ~isempty(GPS), SWIFT_GPS = SWIFT(GPS); end +if ~isempty(IMU) + SWIFT_IMU = SWIFT(IMU); +end +if ~isempty(GPS) + SWIFT_GPS = SWIFT(GPS); +end %% Fill in the ID field from mission directory name if NaN diff --git a/GeneralTools/KZ_Processing/createSBD.m b/GeneralTools/KZ_Processing/createSBD.m index eed3189..6050edc 100644 --- a/GeneralTools/KZ_Processing/createSBD.m +++ b/GeneralTools/KZ_Processing/createSBD.m @@ -12,16 +12,19 @@ % 9/2019 changed time convention to use start of the burst,rather than end % +%% Experiment directory + +expdir = 'S:\SEAFAC\June2024'; +SBDfold = 'ProcessedSBD'; + +%% Sampling Parameters + if ispc slash = '\'; else slash = '/'; end -% Experiment directory and sampling parameters (user defined) -expdir = 'S:\SEAFAC\June2024'; -SBDfold = 'ProcessedSBD'; - % Sampling Parameters burstinterval = 12; % minutes between bursts burstlength = 512/60; % minutes of sampling during each burst diff --git a/GeneralTools/KZ_Processing/postprocessSWIFT.m b/GeneralTools/KZ_Processing/postprocessSWIFT.m index eb489fb..af00144 100644 --- a/GeneralTools/KZ_Processing/postprocessSWIFT.m +++ b/GeneralTools/KZ_Processing/postprocessSWIFT.m @@ -7,7 +7,14 @@ % K. Zeiden 07/2024 -%% User defined experiment directory (to be later converted to function inputs) +% Need to consider additional QC steps after processing... +% Airmar temp, NaN out if below/above -20/50 deg C +% Wind speed, NaN out above 30 m/s + +%% Experiment Directory +expdir = ['S:' slash 'SEAFAC' slash 'June2024']; + +%% Processing toggles if ispc slash = '\'; @@ -15,8 +22,6 @@ slash = '/'; end -expdir = ['S:' slash 'SEAFAC' slash 'June2024']; - % Processing toggles rpIMU = false; % Waves rpSBG = false; % Waves @@ -31,7 +36,7 @@ plotL1L2 = true; % List of missions -missions = dir([expdir slash 'SWIFT1*']); +missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); %% Loop through missions and reprocess diff --git a/GeneralTools/pruneSWIFT.asv b/GeneralTools/pruneSWIFT.asv new file mode 100644 index 0000000..68b3d70 --- /dev/null +++ b/GeneralTools/pruneSWIFT.asv @@ -0,0 +1,106 @@ +% Find out-of-water bursts and prune them from the SWIFT structure + +% K. Zeiden 10/10/2024 + +%% Experiment Directory +expdir = 'S:\SEAFAC\June2024'; + +%% Parameters for QC/out-of-water identification + +if ispc + slash = '\'; +else + slash = '/'; +end + +% Processing parameters +plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) + +% QC Parameters +minwaveheight = 0;% minimum wave height in data screening +minsalinity = 1;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) +maxdriftspd = 3;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that + +disp('-------------------------------------') +disp('Out-of-water parameters:') +disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) +disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) +disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) +disp('-------------------------------------') + +% List missions +missions = dir([expdir slash 'SWIFT*']); +missions = missions([missions.isdir]); + +%% Loop through missions and remove burst identified as out-of-water +for im = 1%:length(missions) + + missiondir = [missions(im).folder slash missions(im).name]; + cd(missiondir) + sname = missions(im).name; + + % Load L1 file + sfile = dir([missiondir slash '*SWIFT*L1.mat']); + if ~isempty(sfile) + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); + else % Exit reprocessing if no L1 product exists + warning(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + return + end + + % Create diary file + diaryfile = [missions(im).name '_pruneSWIFT.txt']; + if exist(diaryfile,'file') + delete(diaryfile); + end + diary(diaryfile) + disp(['Pruning ' sname]) + + % Loop through and identify out-of-water bursts + nburst = length(SWIFT); + outofwater = false(1,nburst); + + for iburst = 1:nburst + oneSWIFT = SWIFT(iburst); + + % Waves too small (probably out of water) + if isfield(oneSWIFT,'sigwaveheight') + if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 + outofwater(iburst) = true; + disp('Waves too small, removing burst.') + end + end + + % Salinity too small (probably out of water) + if isfield(oneSWIFT,'salinity') + if all(oneSWIFT.salinity < minsalinity) + outofwater(iburst) = true; + disp('Salinity too low, removing burst.') + end + end + + % Drift speed limit (if too fast, probably out of water) + if isfield(oneSWIFT,'driftspd') + if oneSWIFT.driftspd > maxdriftspd + outofwater(iburst) = true; + disp('Speed too fast, removing burst.') + end + end + + end + + % Remove out-of-water bursts + SWIFT(outofwater) = []; + + % Plot + if plotflag + plotSWIFT(SWIFT) + end + + % Save L2 file + save([missiondir slash sname '_L2.mat'],'SWIFT') + + % Turn off diary + diary off + +end % End mission loop \ No newline at end of file diff --git a/GeneralTools/pruneSWIFT.m b/GeneralTools/pruneSWIFT.m index a6ab133..68b3d70 100644 --- a/GeneralTools/pruneSWIFT.m +++ b/GeneralTools/pruneSWIFT.m @@ -1,3 +1,11 @@ +% Find out-of-water bursts and prune them from the SWIFT structure + +% K. Zeiden 10/10/2024 + +%% Experiment Directory +expdir = 'S:\SEAFAC\June2024'; + +%% Parameters for QC/out-of-water identification if ispc slash = '\'; @@ -5,49 +13,42 @@ slash = '/'; end -% Experiment Directory -expdir = 'S:\SEAFAC\June2024'; - -% SBD folder -SBDfold = 'ProcessedSBD'; - % Processing parameters plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) -fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights -fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. % QC Parameters minwaveheight = 0;% minimum wave height in data screening -minsalinity = 0;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) -maxdriftspd = 5;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that -maxwindspd = 30;% m/s for malfunctioning Airmars -minairtemp = -20;% min airtemp -maxairtemp = 50;% max airtemp +minsalinity = 1;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) +maxdriftspd = 3;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that disp('-------------------------------------') -disp('QC settings:') +disp('Out-of-water parameters:') disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) -disp(['Maximum wind speed: ' num2str(maxwindspd) ' ms^{-1}']) -disp(['Minimum air temp: ' num2str(minairtemp) ' C']) -disp(['Maximum air temp: ' num2str(maxairtemp) ' C']) disp('-------------------------------------') % List missions missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); -%% Loop through missions - -for im = 1:length(missions) +%% Loop through missions and remove burst identified as out-of-water +for im = 1%:length(missions) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) sname = missions(im).name; - l1file + % Load L1 file + sfile = dir([missiondir slash '*SWIFT*L1.mat']); + if ~isempty(sfile) + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); + else % Exit reprocessing if no L1 product exists + warning(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + return + end + % Create diary file diaryfile = [missions(im).name '_pruneSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); @@ -55,4 +56,51 @@ diary(diaryfile) disp(['Pruning ' sname]) -end + % Loop through and identify out-of-water bursts + nburst = length(SWIFT); + outofwater = false(1,nburst); + + for iburst = 1:nburst + oneSWIFT = SWIFT(iburst); + + % Waves too small (probably out of water) + if isfield(oneSWIFT,'sigwaveheight') + if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 + outofwater(iburst) = true; + disp('Waves too small, removing burst.') + end + end + + % Salinity too small (probably out of water) + if isfield(oneSWIFT,'salinity') + if all(oneSWIFT.salinity < minsalinity) + outofwater(iburst) = true; + disp('Salinity too low, removing burst.') + end + end + + % Drift speed limit (if too fast, probably out of water) + if isfield(oneSWIFT,'driftspd') + if oneSWIFT.driftspd > maxdriftspd + outofwater(iburst) = true; + disp('Speed too fast, removing burst.') + end + end + + end + + % Remove out-of-water bursts + SWIFT(outofwater) = []; + + % Plot + if plotflag + plotSWIFT(SWIFT) + end + + % Save L2 file + save([missiondir slash sname '_L2.mat'],'SWIFT') + + % Turn off diary + diary off + +end % End mission loop \ No newline at end of file diff --git a/GeneralTools/qcSWIFT.m b/GeneralTools/qcSWIFT.m deleted file mode 100644 index 55c36d9..0000000 --- a/GeneralTools/qcSWIFT.m +++ /dev/null @@ -1,61 +0,0 @@ -function [SWIFT,sinfo] = qcSWIFT(missiondir,minsalinity,minwave,maxdrift) - -% Based on SWIFT_QC by J. Thomson - -% Basic QC; goal is to remove bursts when SWIFT is out of water. - -if ispc - slash = '\'; -else - slash = '/'; -end - -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function - -l1file = dir([missiondir slash '*SWIFT*L1.mat']); - -if ~isempty(l1file) - load([l1file.folder slash l1file.name],'SWIFT','sinfo'); -else - warning(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) - return -end - - -%% QC - - if isfield(SWIFT,'salinity') - badsal = [SWIFT.salinity] < minsalinity; % this will break if multiple salinity values (salty SWIFTs) - SWIFT(badsal) = []; - end - - - if isfield(SWIFT,'sigwaveheight') - badwave = [SWIFT.sigwaveheight] < minwave; - SWIFT(badwave) = []; - end - - if isfield(SWIFT,'driftspd') - baddrift = [SWIFT.driftspd] > maxdrift; - SWIFT(baddrift) = []; - end - -%% Log QC and flags, then save new L2 file or overwrite existing one - -if isfield(sinfo,'postproc') -ip = length(sinfo.postproc)+1; -else - sinfo.postproc = struct; - ip = 1; -end -sinfo.postproc(ip).type = 'Y81'; -sinfo.postproc(ip).usr = getenv('username'); -sinfo.postproc(ip).time = string(datetime('now')); -sinfo.postproc(ip).flags = []; -sinfo.postproc(ip).params = []; - -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') - - - -end \ No newline at end of file From 0d52249d43b898abd8f498f23cb4ce7a2f75e15a Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:50:40 -0700 Subject: [PATCH 30/34] updates --- Aquadopp/reprocess_AQD.m | 36 +- Aquadopp/reprocess_AQH.m | 65 +-- CT/reprocess_ACS.m | 20 +- .../Archive/compileSWIFT_SBD_test2.m | 0 .../L0_createSBD.m} | 2 +- .../L1_compileSWIFT.m} | 40 +- .../L2_pruneSWIFT.asv} | 24 +- .../L2_pruneSWIFT.m} | 27 +- .../L3_postprocessSWIFT.m} | 63 +-- GeneralTools/KZ_Processing/compileSWIFT.asv | 381 ------------------ GeneralTools/KZ_Processing/createSBD.asv | 200 --------- .../KZ_Processing/postprocessSWIFT.asv | 207 ---------- GeneralTools/{KZ_Processing => }/NaNstruct.m | 0 GeneralTools/{KZ_Processing => }/NaNstructR.m | 0 GeneralTools/{KZ_Processing => }/catSWIFT.m | 0 GeneralTools/plotSWIFTV3.m | 249 ++++++++++++ GeneralTools/plotSWIFTV4.m | 179 ++++++++ IMU/reprocess_IMU.m | 36 +- SBG/reprocess_SBG.m | 26 +- Signature/reprocess_SIG.m | 66 +-- Winds/reprocess_WXT.m | 20 +- Winds/reprocess_Y81.m | 20 +- 22 files changed, 654 insertions(+), 1007 deletions(-) rename GeneralTools/{KZ_Processing => KZ_PostProcessing}/Archive/compileSWIFT_SBD_test2.m (100%) rename GeneralTools/{KZ_Processing/createSBD.m => KZ_PostProcessing/L0_createSBD.m} (98%) rename GeneralTools/{KZ_Processing/compileSWIFT.m => KZ_PostProcessing/L1_compileSWIFT.m} (92%) rename GeneralTools/{pruneSWIFT.asv => KZ_PostProcessing/L2_pruneSWIFT.asv} (84%) rename GeneralTools/{pruneSWIFT.m => KZ_PostProcessing/L2_pruneSWIFT.m} (83%) rename GeneralTools/{KZ_Processing/postprocessSWIFT.m => KZ_PostProcessing/L3_postprocessSWIFT.m} (70%) delete mode 100644 GeneralTools/KZ_Processing/compileSWIFT.asv delete mode 100644 GeneralTools/KZ_Processing/createSBD.asv delete mode 100644 GeneralTools/KZ_Processing/postprocessSWIFT.asv rename GeneralTools/{KZ_Processing => }/NaNstruct.m (100%) rename GeneralTools/{KZ_Processing => }/NaNstructR.m (100%) rename GeneralTools/{KZ_Processing => }/catSWIFT.m (100%) create mode 100644 GeneralTools/plotSWIFTV3.m create mode 100644 GeneralTools/plotSWIFTV4.m diff --git a/Aquadopp/reprocess_AQD.m b/Aquadopp/reprocess_AQD.m index 6d9e3d2..72463a9 100644 --- a/Aquadopp/reprocess_AQD.m +++ b/Aquadopp/reprocess_AQD.m @@ -19,6 +19,22 @@ slash = '/'; end +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function + +l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); + +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end + %% Parameters minamp = 30; % amplitude cutoff, usually 30 or 40 @@ -29,22 +45,6 @@ Vert_prec = .074; % m/s Hori_prec = .224; % m/s -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function - -l1file = dir([missiondir slash '*SWIFT*L1.mat']); -l2file = dir([missiondir slash '*SWIFT*L2.mat']); - -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) - return -end - burstreplaced = NaN(length(SWIFT),1); %% Loop through raw burst files and reprocess @@ -136,7 +136,7 @@ end end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Log reprocessing and flags, then save new L3 file or overwrite existing one if isfield(sinfo,'postproc') ip = length(sinfo.postproc)+1; @@ -150,7 +150,7 @@ sinfo.postproc(ip).flags = []; sinfo.postproc(ip).params = []; -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +save([sfile.folder slash sfile.name(1:end-6) 'L3.mat'],'SWIFT','sinfo') %% End function end diff --git a/Aquadopp/reprocess_AQH.m b/Aquadopp/reprocess_AQH.m index f368b7d..be0652e 100644 --- a/Aquadopp/reprocess_AQH.m +++ b/Aquadopp/reprocess_AQH.m @@ -22,6 +22,22 @@ slash = '/'; end +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function + +l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); + +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end + %% Internal Toggles % Data Load/Save Toggles @@ -49,21 +65,6 @@ ftype = '.mat'; end -%% Load or create SWIFT structure, create SIG structure, list burst files - -l1file = dir([missiondir slash '*SWIFT*L1.mat']); -l2file = dir([missiondir slash '*SWIFT*L2.mat']); - -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) - return -end burstreplaced = false(length(SWIFT),1); badaqh = false(length(SWIFT),1); @@ -266,7 +267,22 @@ SWIFT = SWIFT(isort); end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Save AQH Structure + Plot %%%%%%%% + +if opt.saveAQH + save([sfile.folder slash sfile.name(1:end-7) '_burstavgAQH.mat'],'AQH') +end + +% Plot burst Averaged SWIFT Signature Data +catAQH(AQH,'plot'); +set(gcf,'Name',sfile.name(1:end-7)) +if opt.saveplots + figname = [missiondir slash get(gcf,'Name')]; + print([figname '_AQH'],'-dpng') + close gcf +end + +%% Log reprocessing and flags, then save new L3 file or overwrite existing one params = opt; @@ -282,23 +298,10 @@ sinfo.postproc(ip).flags.badaqh = badaqh; sinfo.postproc(ip).params = params; -save([sfile.folder slash sfile.name(1:end-7) '_L2.mat'],'SWIFT','sinfo') - -%% Save SIG Structure + Plot %%%%%%%% - -if opt.saveAQH - save([sfile.folder slash sfile.name(1:end-7) '_burstavgAQH.mat'],'AQH') -end +save([sfile.folder slash sfile.name(1:end-7) '_L3.mat'],'SWIFT','sinfo') -% Plot burst Averaged SWIFT Signature Data -catAQH(AQH,'plot'); -set(gcf,'Name',sfile.name(1:end-7)) -if opt.saveplots - figname = [missiondir slash get(gcf,'Name')]; - print([figname '_AQH'],'-dpng') - close gcf -end +%% Return to mission directory cd(missiondir) end \ No newline at end of file diff --git a/CT/reprocess_ACS.m b/CT/reprocess_ACS.m index ceb6502..a74180c 100644 --- a/CT/reprocess_ACS.m +++ b/CT/reprocess_ACS.m @@ -12,19 +12,19 @@ slash = '/'; end -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function -l1file = dir([missiondir slash '*SWIFT*L1.mat']); l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) return end @@ -63,7 +63,7 @@ end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Log reprocessing and flags, then save new L3 file or overwrite existing one if isfield(sinfo,'postproc') ip = length(sinfo.postproc)+1; @@ -77,7 +77,7 @@ sinfo.postproc(ip).flags = []; sinfo.postproc(ip).params = []; -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +save([sfile.folder slash sfile.name(1:end-6) 'L3.mat'],'SWIFT','sinfo') %% End function end \ No newline at end of file diff --git a/GeneralTools/KZ_Processing/Archive/compileSWIFT_SBD_test2.m b/GeneralTools/KZ_PostProcessing/Archive/compileSWIFT_SBD_test2.m similarity index 100% rename from GeneralTools/KZ_Processing/Archive/compileSWIFT_SBD_test2.m rename to GeneralTools/KZ_PostProcessing/Archive/compileSWIFT_SBD_test2.m diff --git a/GeneralTools/KZ_Processing/createSBD.m b/GeneralTools/KZ_PostProcessing/L0_createSBD.m similarity index 98% rename from GeneralTools/KZ_Processing/createSBD.m rename to GeneralTools/KZ_PostProcessing/L0_createSBD.m index 6050edc..8c660ff 100644 --- a/GeneralTools/KZ_Processing/createSBD.m +++ b/GeneralTools/KZ_PostProcessing/L0_createSBD.m @@ -42,7 +42,7 @@ cd(missiondir) sname = missions(im).name; -diaryfile = [missions(im).name '_createSBD.txt']; +diaryfile = ['L0_' missions(im).name '_createSBD.txt']; if exist(diaryfile,'file') delete(diaryfile); end diff --git a/GeneralTools/KZ_Processing/compileSWIFT.m b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m similarity index 92% rename from GeneralTools/KZ_Processing/compileSWIFT.m rename to GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m index 69a20e9..102e46c 100644 --- a/GeneralTools/KZ_Processing/compileSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m @@ -46,7 +46,8 @@ cd(missiondir) sname = missions(im).name; - diaryfile = [missions(im).name '_compileSWIFT.txt']; +% Create diary file + diaryfile = ['L1_' missions(im).name '_compileSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end @@ -276,28 +277,43 @@ SWIFT(si).ID = ID; end +%% Create sinfo structure + +if ~micro + disp('Create information structure ''sinfo''') + sinfo.ID = SWIFT(1).ID; + sinfo.CTdepth = SWIFT(1).CTdepth; + if isfield(SWIFT,'metheight') + sinfo.metheight = SWIFT(1).metheight; + end + if isfield(SWIFT,'signature') + sinfo.type = 'V4'; + else + sinfo.type = 'V3'; + end +end + + %% Save L1 file if micro save([missiondir slash 'micro' sname '_L1.mat'],'SWIFT*') -elseif length([SWIFT.time]) > 1 - save([missiondir slash sname '_L1.mat'],'SWIFT') +else + save([missiondir slash sname '_L1.mat'],'SWIFT',sinfo) end %% Plot if plotflag - plotSWIFT(SWIFT) - - % battery plot - if any(~isnan(battery)) - figure(7), clf, - plot([SWIFT.time],battery,'kx','linewidth',3) - datetick, grid - ylabel('Voltage') - print('-dpng',[sname(1:7) '_battery.png']) + l1file = dir([missiondir slash '*L2.mat']); + if strcmp(sinfo.type,'V3') + fh = plotSWIFTV3(SWIFT); + else + fh = plotSWIFTV4(SWIFT); end + set(fh,'Name',l1file.name(1:end-4)) + print(fh,[l1file.folder slash l1file.name(1:end-4)],'-dpng') end diff --git a/GeneralTools/pruneSWIFT.asv b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.asv similarity index 84% rename from GeneralTools/pruneSWIFT.asv rename to GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.asv index 68b3d70..1f1b1a4 100644 --- a/GeneralTools/pruneSWIFT.asv +++ b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.asv @@ -28,7 +28,7 @@ disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) disp('-------------------------------------') -% List missions +%% List missions missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); @@ -40,16 +40,16 @@ for im = 1%:length(missions) sname = missions(im).name; % Load L1 file - sfile = dir([missiondir slash '*SWIFT*L1.mat']); - if ~isempty(sfile) - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); + l1file = dir([missiondir slash '*SWIFT*L1.mat']); + if ~isempty(l1file) + load([l1file.folder slash l1file.name],'SWIFT','sinfo'); else % Exit reprocessing if no L1 product exists warning(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) return end % Create diary file - diaryfile = [missions(im).name '_pruneSWIFT.txt']; + diaryfile = ['L2_' missions(im).name '_pruneSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end @@ -93,12 +93,18 @@ for im = 1%:length(missions) SWIFT(outofwater) = []; % Plot - if plotflag - plotSWIFT(SWIFT) - end + if plotflag + if strcmp(sinfo.type,'V3') + fh = plotSWIFTV3(SWIFT); + else + fh = plotSWIFTV4(SWIFT); + end + set(fh,'Name',l2file.name(1:end-4)) + print(fh,[l3file.folder slash l3file.name(1:end-4)],'-dpng') + end % Save L2 file - save([missiondir slash sname '_L2.mat'],'SWIFT') + save([missiondir slash sname '_L2.mat'],'SWIFT','sinfo') % Turn off diary diary off diff --git a/GeneralTools/pruneSWIFT.m b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m similarity index 83% rename from GeneralTools/pruneSWIFT.m rename to GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m index 68b3d70..5e8087d 100644 --- a/GeneralTools/pruneSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m @@ -28,7 +28,7 @@ disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) disp('-------------------------------------') -% List missions +%% List missions missions = dir([expdir slash 'SWIFT*']); missions = missions([missions.isdir]); @@ -40,16 +40,16 @@ sname = missions(im).name; % Load L1 file - sfile = dir([missiondir slash '*SWIFT*L1.mat']); - if ~isempty(sfile) - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); + l1file = dir([missiondir slash '*SWIFT*L1.mat']); + if ~isempty(l1file) + load([l1file.folder slash l1file.name],'SWIFT','sinfo'); else % Exit reprocessing if no L1 product exists warning(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) return end % Create diary file - diaryfile = [missions(im).name '_pruneSWIFT.txt']; + diaryfile = ['L2_' missions(im).name '_pruneSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end @@ -92,13 +92,20 @@ % Remove out-of-water bursts SWIFT(outofwater) = []; + % Save L2 file + save([missiondir slash sname '_L2.mat'],'SWIFT','sinfo') + % Plot if plotflag - plotSWIFT(SWIFT) - end - - % Save L2 file - save([missiondir slash sname '_L2.mat'],'SWIFT') + l2file = dir([missiondir slash '*L2.mat']); + if strcmp(sinfo.type,'V3') + fh = plotSWIFTV3(SWIFT); + else + fh = plotSWIFTV4(SWIFT); + end + set(fh,'Name',l2file.name(1:end-4)) + print(fh,[l2file.folder slash l2file.name(1:end-4)],'-dpng') + end % Turn off diary diary off diff --git a/GeneralTools/KZ_Processing/postprocessSWIFT.m b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m similarity index 70% rename from GeneralTools/KZ_Processing/postprocessSWIFT.m rename to GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m index af00144..f399265 100644 --- a/GeneralTools/KZ_Processing/postprocessSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m @@ -33,7 +33,7 @@ rpAQD = false; % TKE % Plotting toggle -plotL1L2 = true; +plotflag = true; % List of missions missions = dir([expdir slash 'SWIFT*']); @@ -45,7 +45,7 @@ missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) - diaryfile = [missions(im).name '_postprocessSWIFT.txt']; + diaryfile = ['L3_' missions(im).name '_postprocessSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end @@ -53,36 +53,15 @@ disp(['Post-processing ' missions(im).name]) - %% Locate L1 product, skip if does not exist. - % Else create 'sinfo' and modify L1 product. - l1file = dir([missiondir slash '*L1.mat']); - if isempty(l1file) - disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) + %% Locate L2 product, skip if does not exist. + + l2file = dir([missiondir slash '*L2.mat']); + + if isempty(l2file) + disp(['No L2 product found for ' missiondir(end-16:end) '. Skipping...']) + return else - load([l1file.folder slash l1file.name],'SWIFT'); - if isfield(SWIFT,'ID') - disp('Create information structure ''sinfo''') - sinfo.ID = SWIFT(1).ID; - sinfo.CTdepth = SWIFT(1).CTdepth; - if isfield(SWIFT,'metheight') - sinfo.metheight = SWIFT(1).metheight; - end - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - else - sinfo.type = 'V3'; - end - disp('Updating L1 product sinfo...') - save([l1file.folder slash l1file.name],'SWIFT','sinfo') - end - % One time, will remove later - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - save([l1file.folder slash l1file.name],'sinfo','-append') - else - sinfo.type = 'V3'; - save([l1file.folder slash l1file.name],'sinfo','-append') - end + load([l2file.folder slash l2file.name],'SWIFT','sinfo'); end %% Reprocess IMU @@ -181,26 +160,18 @@ end end - %% Re-load L1 and L2 product and plot each for comparison + %% Plot - load([l1file.folder slash l1file.name],'SWIFT','sinfo'); - SWIFTL1 = SWIFT; - l2file = dir([missiondir slash '*L2.mat']); - load([l2file.folder slash l2file.name],'SWIFT'); - SWIFTL2 = SWIFT; + l3file = dir([missiondir slash '*L3.mat']); - if plotL1L2 + if plotflag if strcmp(sinfo.type,'V3') - fh1 = plotSWIFTV3(SWIFTL1); - fh2 = plotSWIFTV3(SWIFTL2); + fh = plotSWIFTV3(SWIFT); else - fh1 = plotSWIFTV4(SWIFTL1); - fh2 = plotSWIFTV4(SWIFTL2); + fh = plotSWIFTV4(SWIFT); end - set(fh1,'Name',l1file.name(1:end-4)) - set(fh2,'Name',l2file.name(1:end-4)) - print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') - print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') + set(fh,'Name',l3file.name(1:end-4)) + print(fh,[l3file.folder slash l3file.name(1:end-4)],'-dpng') end diary off diff --git a/GeneralTools/KZ_Processing/compileSWIFT.asv b/GeneralTools/KZ_Processing/compileSWIFT.asv deleted file mode 100644 index 33e336f..0000000 --- a/GeneralTools/KZ_Processing/compileSWIFT.asv +++ /dev/null @@ -1,381 +0,0 @@ -% Aggregate and read all SWIFT sbd data after concatenating the -% offload from SD card or the sbd email attachments. SBD files should be in the -% subfolder specified by 'SBDfold' of any given SWIFT mission folder. -% Based on compileSWIFT_SBDservertelemetry.m by J. Thomson. -% QC has been removed. The intention is to create a pure L1 product which -% has all data recorded. Basic QC has been allocated to another parallel -% script, 'pruneSWIFT.m'. - -% Time: SWIFT - Take the time from the filename, even when there is time from - % the airmar (because of parsing errors). For telemetry, this - % is the telemtry time (at the end of the burst). For offloaded data, - % this the concat file name (from the start of the burst). - % microSWIFT - Use the time embedded within the payload 50 or 51 or 52 of the - % SBD file, which is the time at the end of the burst of raw data. - -% K. Zeiden 10/01/2024 - -%% Experiment directory and QC parameters (user defined) - -if ispc - slash = '\'; -else - slash = '/'; -end - -% Experiment Directory -expdir = 'S:\SEAFAC\June2024'; - -% SBD folder -SBDfold = 'ProcessedSBD'; - -% Processing parameters -plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) -fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights -fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. - -% List missions -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); - -%% Loop through missions - -for im = 1:length(missions) - -missiondir = [missions(im).folder slash missions(im).name]; -cd(missiondir) -sname = missions(im).name; - - diaryfile = [missions(im).name '_compileSWIFT.txt']; - if exist(diaryfile,'file') - delete(diaryfile); - end - diary(diaryfile) - disp(['Compiling ' sname]) - -%%% Compile list of sbd burst files %%% -if exist([missiondir slash SBDfold],'dir') - blist = dir([missiondir slash SBDfold slash '*.sbd']); -else - disp('Processed files have not been concatenated...') - blist = []; -end -nburst = length(blist); - -%%% Initialize badburst flag %%% -badburst = false(1,nburst); - -%%% Initialize vectors %%% -battery = NaN(1,nburst); -npayloads = NaN(1,nburst); - -%%% Loop through all SBD burst files, load, QC, and save in SWIFT structure -for iburst = 1:nburst - - disp('=================================') - disp(['Burst ' num2str(iburst) ' : ' blist(iburst).name ]) - - [oneSWIFT,voltage]= readSWIFT_SBD([blist(iburst).folder slash blist(iburst).name],0); - - if voltage == 9999 % error flag from SBD message - badburst(iburst) = true; - end - - if ~isempty(voltage) - battery(iburst) = voltage; - oneSWIFT.battery = voltage; - else - oneSWIFT.battery = NaN; - end - - if isempty(oneSWIFT.lat) || isempty(oneSWIFT.lon) - oneSWIFT.lat = NaN; - oneSWIFT.lon = NaN; - end - - %%% SWIFT type and Time stamp %%% - if blist(iburst).name(6) == 'S' % SWIFT v3 and v4 - nameoffset = 14; - day = blist(iburst).name(nameoffset + (1:2)); - month = blist(iburst).name(nameoffset + (3:5)); - year = blist(iburst).name(nameoffset + (6:9)); - hr = blist(iburst).name(nameoffset + (11:12)); - minute = blist(iburst).name(nameoffset + (13:14)); - sec = blist(iburst).name(nameoffset + (15:16)); - oneSWIFT.time = datenum([day ' ' month ' ' year ' ' hr ':' minute ':' sec]); - micro = false; - elseif blist(iburst).name(6) == 'm' % microSWIFT - micro = true; - else - oneSWIFT.time = NaN; - micro = false; - end - - %%% Remove wave histograms %%% - if isfield(oneSWIFT,'wavehistogram') - oneSWIFT = rmfield(oneSWIFT,'wavehistogram'); - end - - %%% Add burst to SWIFT structure %%% - burstpayloads = string(fieldnames(oneSWIFT)); - npayloads(iburst) = length(burstpayloads); - - % Loop through paylaods on current burst, add to SWIFT structure - for ipay = 1:npayloads(iburst) - SWIFT(iburst).(burstpayloads{ipay}) = oneSWIFT.(burstpayloads{ipay}); - end - - % List initial payloads, as well as any missing or additional payloads - if iburst == 1 % Initial payloads - allpayloads = burstpayloads; - disp('Initial payloads:') - disp(allpayloads) - elseif iburst ~= 1 && npayloads(iburst) > length(allpayloads) % Additional payloads - isnew = false(npayloads(iburst),1); - for ip = 1:npayloads(iburst) - isnew(ip) = ~any(strcmp(burstpayloads(ip),allpayloads)); - end - newpayloads = burstpayloads(isnew); - disp(['New payloads in burst file ' num2str(iburst) ':']) - disp(newpayloads) - allpayloads = burstpayloads; - elseif iburst ~= 1 && npayloads(iburst) < length(allpayloads)% Missing payloads - ismissing = false(length(allpayloads),1); - for ip = 1:length(allpayloads) - ismissing(ip) = ~any(strcmp(allpayloads(ip),burstpayloads)); - end - missingpayloads = allpayloads(ismissing); - disp(['Missing payloads in burst file ' num2str(iburst) ':']) - disp(missingpayloads) - end - - disp('=================================') - -% End burst loop -end -diary off - -%% Fill empty SWIFT fields due to missing payloads in a burst - -payloads = fieldnames(SWIFT); -npay = length(payloads); -nburst = length(SWIFT); - -for ipay = 1:npay - - var = [SWIFT.(payloads{ipay})]; - - % If some burst values are empty - if length(var) ~= nburst - - % If variable is a scalar (e.g. 'watertemp') fill with NaN - if isa(var,'double') - for iburst = 1:nburst - if isempty(SWIFT(iburst).(payloads{ipay})) - SWIFT(iburst).(payloads{ipay}) = NaN; - end - end - elseif isa(var,'char') - for iburst = 1:nburst - if isempty(SWIFT(iburst).(payloads{ipay})) - SWIFT(iburst).(payloads{ipay}) = var(1); - end - end - else % If variable is a structure array (e.g. 'wavespectra') fill w/NaN structure - for iburst = 1:nburst - if isempty(SWIFT(iburst).(payloads{ipay})) - SWIFT(iburst).(payloads{ipay}) = NaNstructR(var(1)); - end - end - end - - end - -end - -%% Sort final structure -[~,tinds] = sort([SWIFT.time]); -SWIFT = SWIFT(tinds); -battery = battery(tinds); - -%% Enforce a single value for CT and MET sensor heights - -if isfield(SWIFT,'CTdepth') - for si = 1:length(SWIFT) - SWIFT(si).CTdepth = median([SWIFT.CTdepth],'omitnan'); - end -end - -if isfield(SWIFT,'metheight') - for si = 1:length(SWIFT) - SWIFT(si).metheight = median([SWIFT.metheight],'omitnan'); - end -end - - -%% Fill position outliers - -if fixpositions - [cleanlon,cloni] = filloutliers([SWIFT.lon],'linear'); - [cleanlat,clati] = filloutliers([SWIFT.lat],'linear'); - if cloni == clati - for ci = find(cloni) - SWIFT(ci).lon = cleanlon(ci); - SWIFT(ci).lat = cleanlat(ci); - end - disp(['Filled ' num2str(sum(cloni)) ' position outliers.']) - end -end - -%% Recalculate drift (note that wind slip, which is 1%, is not removed) -% Drift speed is included from the Airmar results, -% but that sensor is not always available or included -% (so simpler to just calculate it from differencing positions). - -if length(SWIFT) > 3 - - time = [SWIFT.time];%[time tinds ] = sort(time); - lat = [SWIFT.lat]; %lat = lat(tinds); - lon = [SWIFT.lon]; %lon = lon(tinds); - dlondt = gradient(lon,time); % deg per day - dxdt = deg2km(dlondt,6371*cosd(mean(lat,'omitnan'))) .* 1000 ./ ( 24*3600 ); % m/s - dlatdt = gradient(lat,time); % deg per day - dydt = deg2km(dlatdt) .* 1000 ./ ( 24*3600 ); % m/s - dxdt(isinf(dxdt)) = NaN; - dydt(isinf(dydt)) = NaN; - speed = sqrt(dxdt.^2 + dydt.^2); % m/s - direction = -180 ./ 3.14 .* atan2(dydt,dxdt); % cartesian direction [deg] - direction = direction + 90; % rotate from eastward = 0 to northward = 0 - direction( direction<0) = direction( direction<0 ) + 360; % make quadrant II 270->360 instead of -90->0 - - for si = 1:length(SWIFT) - if si == 1 || si == length(SWIFT) - SWIFT(si).driftspd = NaN; - SWIFT(si).driftdirT = NaN; - else - SWIFT(si).driftspd = speed(si); - SWIFT(si).driftdirT = direction(si); - end - end - - % % remove last burst, if big change in direction (suggests recovery by ship) - % dirchange = abs( SWIFT( length(SWIFT) - 2).driftdirT - SWIFT( length(SWIFT) - 1).driftdirT ); - % if dirchange > 90, - % disp('removing last burst, suspect includes ship recovery') - % SWIFT( length(SWIFT) - 1).driftdirT = NaN; - % SWIFT( length(SWIFT) - 1).driftspd = NaN; - % SWIFT( length(SWIFT) ) = []; - % battery( length(SWIFT) ) = []; - % end - -elseif length(SWIFT) <= 3 - for si = 1:length(SWIFT) - SWIFT(si).driftspd = NaN; - SWIFT(si).driftdirT = NaN; - end -end - -% Quality control by removing drift results associated with large time gaps -if length([SWIFT.time]) > 1 - dt = gradient([SWIFT.time]); - for si = 1:length(SWIFT) - if dt(si) > 1/12 % 1/12 of day is two hours - SWIFT(si).driftspd = NaN; - SWIFT(si).driftdirT = NaN; - end - end -end - -% Quality control drift speeds too fast (prob on deck) with new drift spd -if length([SWIFT.time]) > 1 && isfield(SWIFT(1),'driftspd') - toofast = [SWIFT.driftspd] > maxdriftspd; - SWIFT( toofast ) =[]; - battery( toofast ) = []; -end - -%% Extrapolate missing low frequencies of wave energy spectra %% -% Require energy at lowest frequenc to be zero. -% Not neccessary if post-processing for raw displacements. -% Less necessary after Oct 2017 rev of onboard processing with improved RC filter. - -if fixspectra && isfield(SWIFT,'wavespectra') - for si = 1:length(SWIFT) - notzero = find(SWIFT(si).wavespectra.energy ~= 0 & SWIFT(si).wavespectra.freq > 0.04); - tobereplaced = find(SWIFT(si).wavespectra.energy == 0 & SWIFT(si).wavespectra.freq > 0.04); - if length(notzero) > 10 - E = interp1([0.04; SWIFT(si).wavespectra.freq(notzero)],[0; SWIFT(si).wavespectra.energy(notzero)],SWIFT(si).wavespectra.freq); - SWIFT(si).wavespectra.energy(tobereplaced) = E(tobereplaced); - df = median(diff(SWIFT(si).wavespectra.freq)); - SWIFT(si).sigwaveheight = 4*sqrt(sum(SWIFT(si).wavespectra.energy,'omitnan')*df); - end - end -end - -%% Quality control wind speeds -if length([SWIFT.time]) > 1 && isfield(SWIFT,'windspd') - for si = 1:length(SWIFT) - if SWIFT(si).windspd > maxwindspd - SWIFT(si).windspd = NaN; - SWIFT(si).winddirT = NaN; - SWIFT(si).winddirR = NaN; - end - end -end - -%% Quality control airmar temperature -if isfield(SWIFT,'airtemp') - for si = 1:length(SWIFT) - if SWIFT(si).airtemp == 0.0 || SWIFT(si).airtemp < minairtemp || SWIFT(si).airtemp > maxairtemp - SWIFT(si).airtemp = NaN; - SWIFT(si).windspd = NaN; - end - end -end - -%% Sort the microSWIFT onboard processing, using the battery voltage as a flag -% only applies to v1 microSWIFTs from 2022 - -IMU = find(battery==0); -GPS = find(battery==1); - -if ~isempty(IMU), SWIFT_IMU = SWIFT(IMU); end -if ~isempty(GPS), SWIFT_GPS = SWIFT(GPS); end - -%% Fill in the ID field from mission directory name if NaN - -idnan = isnan([SWIFT.ID]); -ID = sname(strfind(sname,'SWIFT')+ (5:6)); - -for si = find(idnan) - SWIFT(si).ID = ID; -end - -%% Save L1 file - -if micro - save([missiondir slash 'micro' sname '_L1.mat'],'SWIFT*') -elseif length([SWIFT.time]) > 1 - save([missiondir slash sname '_L1.mat'],'SWIFT') -end - -%% Plot - -if plotflag - - plotSWIFT(SWIFT) - - % battery plot - if any(~isnan(battery)) - figure(7), clf, - plot([SWIFT.time],battery,'kx','linewidth',3) - datetick, grid - ylabel('Voltage') - print('-dpng',[sname(1:7) '_battery.png']) - end - -end - -close all -clear SWIFT - -end % End mission loop diff --git a/GeneralTools/KZ_Processing/createSBD.asv b/GeneralTools/KZ_Processing/createSBD.asv deleted file mode 100644 index 1b21d61..0000000 --- a/GeneralTools/KZ_Processing/createSBD.asv +++ /dev/null @@ -1,200 +0,0 @@ -% aggregate, concat, and read all onboard processed SWIFT data (once offloaded from SD card) -% run this in a dedicated directory for the results -% -% (this fills in the results not sent by Iridium when running more than 1 burst per hour) -% -% J. Thomson, 4/2014 -% 9/2015 v3.3, use com names but currently only pulls one ACS file (out of three), because they are not uniquely named. -% 1/2016 v3.3, so up to 3 ACS files can be included -% 6/2016 v 3.4, includ sdi coms (Vasaila 536) -% 1/2017 v4.0, include SBG, Nortek Signature and RM Young 8100 sonic, plus index on SBG, not AQ -% 9/2017 add oxygen optode and sea owl fluorometer -% 9/2019 changed time convention to use start of the burst,rather than end -% - -if ispc - slash = '\'; -else - slash = '/'; -end - -% Experiment directory and sampling parameters (user defined) -expdir = 'S:\SEAFAC\June2024'; -SBDfold = 'ProcessedSBD'; - -% Sampling Parameters -burstinterval = 12; % minutes between bursts -burstlength = 512/60; % minutes of sampling during each burst -payloadtype = '7'; % v3.3 (2015) and up - -% Identify missions -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); - -%% Loop through missions - -for im = 2%:length(missions) - -missiondir = [missions(im).folder slash missions(im).name]; -cd(missiondir) -sname = missions(im).name; - -diaryfile = [missions(im).name '_createSBD.txt']; -if exist(diaryfile,'file') -delete(diaryfile); -end -diary(diaryfile) -disp(['Compiling ' sname]) - - -%%% Create folder for new SBD files %%% -if ~exist([missiondir slash SBDfold],'dir') - mkdir([missiondir slash SBDfold]) -end - -%%% Create payload type temporary file %%% - -payloadfile = [missiondir slash 'payload']; -fid = fopen(payloadfile,'wb'); -fwrite(fid,payloadtype,'uint8'); -fclose(fid); - -%%% Find all processed (PRC) files in all port folders - -PRCfiles = dir([missiondir slash '*' slash 'Processed' slash '*' slash '*PRC*.dat']); - -%%% Glob all processed files (per instrument) %%% -% and put these all in one directory (skip if already in one directory) - -% if ~exist([missiondir slash 'PRC'],'dir') -% -% mkdir([missiondir slash 'PRC']) -% -% progressbar(['Globbing PRC files for ' sname]) -% for iprc = 1:length(PRCfiles) -% progressbar(iprc/length(PRCfiles)) -% copyfile([PRCfiles(iprc).folder slash PRCfiles(iprc).name],[missiondir slash 'PRC']) -% end -% -% else -% disp(['*** PRC files for ' sname ' already globbed ***']) -% -% end - -%%% Use IMU or SBG to identify all bursts, as reference files %%% -% (b/c either IMU or SBG is always present) - -% refbfiles = dir([missiondir slash 'PRC' slash '*IMU*_PRC*.dat']); -% if isempty(refbfiles) -% refbfiles = dir([missiondir slash 'PRC' slash '*SBG*_PRC*.dat']); -% else -% disp('NO IMU (or SBG) files found'); -% end - -% refbfiles = PRCfiles(contains({PRCfiles.name},'IMU')); -% if isempty(refbfiles) -% refbfiles = PRCfiles(contains({PRCfiles.name},'SBG')); -% if isempty(refbfiles) -% disp('No IMU or SBG files found') -% end -% end - -% Get reference burst -burstfiletimes = NaN(length(PRCfiles),1); -for iburst = 1:length(burstfiletimes) - date = datenum(PRCfiles(iburst).name(13:21)); - hour = str2double(PRCfiles(iburst).name(23:24)); - min = (str2double(PRCfiles(iburst).name(26:27))-1)*12; - burstfiletimes(iburst) = date + datenum([0 0 0 hour min 0]); -end -[~,b] = unique(burstfiletimes); -refbfiles = PRCfiles(b); - -%%% Loop through reference burst files (IMU or SBG) %%% - -for iburst = 1:length(refbfiles) - - disp(['Creating SBD file for burst ' num2str(iburst) ' (' num2str(length(refbfiles)) ' bursts)']) - - ID = refbfiles(iburst).name(1:7); - date = refbfiles(iburst).name(13:21); - hour = refbfiles(iburst).name(23:24); - burst = refbfiles(iburst).name(26:27); - bfilename = [date '_' hour '_' burst '_PRC.dat']; - - % Name concat file same as if pulled from swiftserver - minute = num2str((str2double(burst(2))-1) * burstinterval); - if length(minute) == 1 - minute = ['0' minute]; %#ok - end - sbdfile = [missiondir slash SBDfold slash 'buoy-SWIFT_' ID(6:7) '-' date '_' hour minute '000.sbd']; - - % Find all PRC files for this burst (instead of globbing files) - PRCburstfiles = PRCfiles(contains({PRCfiles.name},bfilename)); - - % % Look for, rename and copy multiple ACS files (on different com ports) - % ACStopfile = dir([missiondir slash 'COM-7' slash 'Processed' slash '*' slash ID '_ACS_' bfilename]); - % if ~empty(ACStopfile) - % copyfile([ACStopfile.folder slash ACStopfile.name],... - % [missiondir slash 'PRC' slash ID '_ACStop_' bfilename]) - % end - % ACSmidfile = dir([missiondir slash 'COM-8' slash 'Processed' slash '*' slash ID '_ACS_' bfilename]); - % if ~isempty(ACSmidfile) - % copyfile([ACSmidfile.folder slash ACSmidfile.name],... - % [missiondir slash 'PRC' slash ID '_ACSmid_' bfilename]) - % end - % ACSbottomfile = dir([missiondir slash 'COM-9' slash 'Processed' slash '*' slash ID '_ACS_' bfilename]); - % if ~isempty(ACSbottomfile) - % copyfile([ACSbottomfile.folder slash ACSbottomfile.name],... - % [missiondir slash 'PRC' slash ID '_ACSbottom_' bfilename]) - % end - % - % % List of files to concatenate (to make SBD files) - % AQHfile = [missiondir slash 'PRC' slash ID '_AQH_' bfilename]; - % AQDfile = [missiondir slash 'PRC' slash ID '_AQD_' bfilename]; - % PB2file = [missiondir slash 'PRC' slash ID '_PB2_' bfilename]; - % RADfile = [missiondir slash 'PRC' slash ID '_536_' bfilename]; - % IMUfile = [missiondir slash 'PRC' slash ID '_IMU_' bfilename]; - % SBGfile = [missiondir slash 'PRC' slash ID '_SBG_' bfilename]; - % ACSfiletop = [missiondir slash 'PRC' slash ID '_ACStop_' bfilename]; - % ACSfilemid = [missiondir slash 'PRC' slash ID '_ACSmid_' bfilename]; - % ACSfilebottom = [missiondir slash 'PRC' slash ID '_ACSbottom_' bfilename]; - % Y81file = [missiondir slash 'PRC' slash ID '_Y81_' bfilename]; - % SIGfile = [missiondir slash 'PRC' slash ID '_SIG_' bfilename]; - % ACOfile = [missiondir slash 'PRC' slash ID '_ACO_' bfilename]; - % SWLfile = [missiondir slash 'PRC' slash ID '_SWL_' bfilename]; - % - % % Create system command to concatenate files - % if ispc - % syscommand = ['copy /b payload+' AQHfile '+' AQDfile '+' PB2file '+' RADfile '+' IMUfile '+' SBGfile '+' ACSfiletop '+' ACSfilemid '+' ACSfilebottom '+' Y81file '+' SIGfile '+' ACOfile '+' SWLfile '+' ' ' sbdfile]; - % else - % syscommand = ['!cat payload ' AQHfile ' ' AQDfile ' ' PB2file ' ' RADfile ' ' IMUfile ' ' SBGfile ' ' Y81file ' ' SIGfile ' ' ACSfiletop ' ' ACSfilemid ' ' ACSfilebottom ' ' ACOfile ' ' SWLfile ' > ' sbdfile]; - % end - - % Create system commmand to concatenate files - if ispc - syscommand = ['copy /b ' payloadfile]; - for iprc = 1:length(PRCburstfiles) - syscommand = [syscommand '+' PRCburstfiles(iprc).folder slash PRCburstfiles(iprc).name]; %#ok<*AGROW> - end - syscommand = [syscommand ' ' sbdfile]; - else - syscommand = ['!cat ' payloadfile]; - for iprc = 1:length(PRCburstfiles) - syscommand = [syscommand ' ' PRCburstfiles(iprc).folder slash PRCburstfiles(iprc).name]; - end - syscommand = [syscommand ' > ' sbdfile]; - end - - % Execute system command to concatenate files - status = system(syscommand); - if status~=0 - warning('SBD file creation (PRC concatenation) failed...') - end - -end % End burst loop - -delete(payloadfile) -diary off - -end % End mission loop \ No newline at end of file diff --git a/GeneralTools/KZ_Processing/postprocessSWIFT.asv b/GeneralTools/KZ_Processing/postprocessSWIFT.asv deleted file mode 100644 index 3e733d7..0000000 --- a/GeneralTools/KZ_Processing/postprocessSWIFT.asv +++ /dev/null @@ -1,207 +0,0 @@ -% [SWIFT,sinfo] = postprocess_SWIFT(expdir) - -% Master post-processing function, that calls sub-functions to reprocess -% each different type of raw SWIFT data. -% L1 product must have been created prior to running this script, -% by running 'compileSWIFT.m' - -% K. Zeiden 07/2024 - -%% User defined experiment directory (to be later converted to function inputs) - -if ispc - slash = '\'; -else - slash = '/'; -end - -expdir = ['S:' slash 'SEAFAC' slash 'June2024']; - -% Processing toggles -rpIMU = true; % Waves -rpSBG = true; % Waves -rpWXT = true; % MET -rpY81 = true; % MET -rpACS = true; % CT -rpSIG = true; % TKE -rpAQH = true; % TKE -rpAQD = true; % TKE - -% Plotting toggle -plotL1L2 = true; - -% List of missions -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); - -%% Loop through missions and reprocess -for im = 1:length(missions) - - missiondir = [missions(im).folder slash missions(im).name]; - cd(missiondir) - - diaryfile = [missions(im).name '_postprocessSWIFT.txt']; - if exist(diaryfile,'file') - delete(diaryfile); - end - diary(diaryfile) - - disp(['Post-processing ' missions(im).name]) - - %% Locate L1 product, skip if does not exist. - % Else create 'sinfo' and modify L1 product. - l1file = dir([missiondir slash '*L1.mat']); - if isempty(l1file) - disp(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) - else - load([l1file.folder slash l1file.name],'SWIFT'); - if isfield(SWIFT,'ID') - disp('Create information structure ''sinfo''') - sinfo.ID = SWIFT(1).ID; - sinfo.CTdepth = SWIFT(1).CTdepth; - if isfield(SWIFT,'metheight') - sinfo.metheight = SWIFT(1).metheight; - end - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - else - sinfo.type = 'V3'; - end - disp('Updating L1 product sinfo...') - save([l1file.folder slash l1file.name],'SWIFT','sinfo') - end - % One time, will remove later - if isfield(SWIFT,'signature') - sinfo.type = 'V4'; - save([l1file.folder slash l1file.name],'sinfo','-append') - else - sinfo.type = 'V3'; - save([l1file.folder slash l1file.name],'sinfo','-append') - end - end - - %% Reprocess IMU - if rpIMU - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) - disp('Reprocessing IMU data...') - calctype = 'GPS'; - filtertype = 'RC'; - saveraw = false; - interpf = false; - [SWIFT,sinfo] = reprocess_IMU(missiondir,calctype,filtertype,saveraw,interpf); - else - disp('No IMU data...') - end - end - - %% Reprocess SBG - if rpSBG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SBG_*.dat'])) - disp('Reprocessing SBG data...') - saveraw = false; - useGPS = false; - interpf = false; - [SWIFT,sinfo] = reprocess_SBG(missiondir,saveraw,useGPS,interpf); - else - disp('No SBG data...') - end - end - - %% Reprocess WXT (536, confusing b/c CT15 can also be 536) - if rpWXT - if ~isempty(dir([missiondir slash 'WXT' slash 'Raw' slash '*' slash '*_536_*.dat'])) - disp('Reprocessing Vaisala WXT data...') - readraw = false; - usewind = false; - - [SWIFT,sinfo] = reprocess_WXT(missiondir,readraw,usewind); - else - disp('No WXT data...') - end - end - - %% Reprocess Y81 - if rpY81 - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_Y81_*.dat'])) - disp('Reprocessing Y81 Sonic Anemometer data...') - [SWIFT,sinfo] = reprocess_Y81(missiondir); - else - disp('No Y81 data...') - end - end - - %% Reprocess ACS - if rpACS - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_ACS_*.dat'])) - disp('Reprocessing ACS CT data...') - readraw = false; - [SWIFT,sinfo] = reprocess_ACS(missiondir,readraw); - else - disp('No ACS data...') - end - end - - %% Reprocess SIG - if rpSIG - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_SIG_*.dat'])) - disp('Reprocessing Signature1000 data...') - plotburst = false; - readraw = false; - [SWIFT,sinfo] = reprocess_SIG(missiondir,readraw,plotburst); - else - disp('No SIG data...') - end - end - - %% Reprocess AQD - if rpAQD - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQD_*.dat'])) - disp('Reprocessing Aquadopp (AQD) data...') - readraw = true; - [SWIFT,sinfo] = reprocess_AQD(missiondir,readraw); - else - disp('No AQD data...') - end - end - - %% Reprocess AQH - if rpAQH - if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_AQH_*.dat'])) - disp('Reprocessing Aquadopp (AQH) data...') - readraw = false; - plotburst = false; - [SWIFT,sinfo] = reprocess_AQH(missiondir,readraw,plotburst); - else - disp('No AQH data...') - end - end - - %% Re-load L1 and L2 product and plot each for comparison - - load([l1file.folder slash l1file.name],'SWIFT','sinfo'); - SWIFTL1 = SWIFT; - l2file = dir([missiondir slash '*L2.mat']); - load([l2file.folder slash l2file.name],'SWIFT'); - SWIFTL2 = SWIFT; - - if plotL1L2 - if strcmp(sinfo.type,'V3') - fh1 = plotSWIFTV3(SWIFTL1); - fh2 = plotSWIFTV3(SWIFTL2); - else - fh1 = plotSWIFTV4(SWIFTL1); - fh2 = plotSWIFTV4(SWIFTL2); - end - print(fh1,[l1file.folder slash l1file.name(1:end-4)],'-dpng') - print(fh2,[l2file.folder slash l2file.name(1:end-4)],'-dpng') - end - - close all - - -diary off -end - - - - diff --git a/GeneralTools/KZ_Processing/NaNstruct.m b/GeneralTools/NaNstruct.m similarity index 100% rename from GeneralTools/KZ_Processing/NaNstruct.m rename to GeneralTools/NaNstruct.m diff --git a/GeneralTools/KZ_Processing/NaNstructR.m b/GeneralTools/NaNstructR.m similarity index 100% rename from GeneralTools/KZ_Processing/NaNstructR.m rename to GeneralTools/NaNstructR.m diff --git a/GeneralTools/KZ_Processing/catSWIFT.m b/GeneralTools/catSWIFT.m similarity index 100% rename from GeneralTools/KZ_Processing/catSWIFT.m rename to GeneralTools/catSWIFT.m diff --git a/GeneralTools/plotSWIFTV3.m b/GeneralTools/plotSWIFTV3.m new file mode 100644 index 0000000..2169a37 --- /dev/null +++ b/GeneralTools/plotSWIFTV3.m @@ -0,0 +1,249 @@ +function fh = plotSWIFTV3(swift) + +if size(swift,2)>1 + swift = catSWIFT(swift); +end + +lw = 2; +nt = length(swift.time); + +fh = figure('color','w'); +fullscreen(2) +h = tight_subplot(8,3,[0.035 0.05],[0.1 0.075],0.075); +% tight_subplot(Nh, Nw, [gap_h gap_w], [lower upper], [left right] ) + +% Anemometer +axes(h(1)) +plot(swift.time,swift.windu,'-rx','LineWidth',lw) +ylabel('U [ms^{-1}]') +title('Wind Speed') +ylim([0 15]) +set(gca,'Clipping','off') +axes(h(4)) +plot(swift.time,swift.tair,'-rx','LineWidth',lw) +ylabel('T_{air} [^{\circ}C]') +title('Air Temperature') + +% IMU/GPS +axes(h(7)) +plot(swift.time,swift.wavesigH,'-bx','LineWidth',lw) +ylabel('H_s [m]') +title('Significant Wave Height') +ylim([0 5]) +set(gca,'Clipping','off') +axes(h(10)) +plot(swift.time,swift.wavepeakT,'-bx','LineWidth',lw) +ylabel('T_p [s]') +title('Peak Wave Period') +ylim([0 10]) +set(gca,'Clipping','off') +axes(h(13)) +plot(swift.time,swift.wavepeakdir,'-bx','LineWidth',lw) +ylabel('\Theta [^{\circ}]') +title('Peak Wave Direction') +ylim([0 360]) +axes(h(16)) +plot(swift.time,swift.tsea,'-bx','LineWidth',lw) +ylabel('T_{sea} [^{\circ}C]') +title('Sea Temperature') +axes(h(19)) +plot(swift.time,swift.sal,'-bx','LineWidth',lw) +ylabel('S [psu]') +title('Salinity') + +% Battery +axes(h(22)) +if isfield(swift,'battery') +plot(swift.time,swift.battery,'-kx','linewidth',lw) +else + plot(swift.time,NaN(1,nt),'-kx','linewidth',lw) +end +ylabel('[V]') +title('Battery Level') + +% Additional Met (humidity, rain, raddiance, skin temp) +axes(h(2)) +if isfield(swift,'rain') +plot(swift.time,swift.rain,'-x','color',rgb('grey'),'LineWidth',lw) +else + plot(swift.time,NaN(1,nt),'-x','color',rgb('grey'),'LineWidth',lw) +end +ylabel('Rain [mm]') +title('Rainfall') + +axes(h(5)) +if isfield(swift,'humid') +plot(swift.time,swift.humid,'-rx','LineWidth',lw) +else + plot(swift.time,NaN(1,nt),'-rx','LineWidth',lw) +end +ylabel('RH [%]') +ylim([0 100]) +title('Relative Humidity') + +% Radiometers +axes(h(8)) +if isfield(swift,'IRtemp') + plot(swift.time,swift.IRtemp,'-bx','LineWidth',lw); + if size(swift.IRtemp,2)==2 + hold on + plot(swift.time,swift.IRtemp(:,2),'-rx','LineWidth',lw) + end +else + plot(swift.time,NaN(1,nt),'-bx','LineWidth',lw); +end +ylabel('T [^{\circ}C]') +title('Brightness Temperature') +axis tight +datetick('x','KeepLimits') + +axes(h(11)) +if isfield(swift,'AMBtemp') + plot(swift.time,swift.AMBtemp,'-bx','LineWidth',lw); + if size(swift.AMBtemp,2)==2 + hold on + plot(swift.time,swift.AMBtemp(:,2),'-rx','LineWidth',lw) + end +else + plot(swift.time,NaN(1,nt),'-bx','LineWidth',lw); +end +ylabel('T [^{\circ}C]') +title('Jacket Temperature') +axis tight +datetick('x','KeepLimits') + +% Location +axes(h(12)) +ax = gca;ax.Position = ax.Position.*[1 1 1 5]; +lonscale = mean(cos(swift.lat*pi/180),'omitnan'); +scatter(swift.lon,swift.lat,[],swift.time,'filled'); +set(gca,'YAxisLocation','right') +hold on +quiver(swift.lon,swift.lat,swift.driftu./lonscale,swift.driftv,'k') +c = slimcolorbar; +c.Location = 'South'; +c.TickLabels = datestr(c.Ticks,'mmm-dd'); +ylabel('Lat [^{\circ}N]') +xlabel('Lon [^{\circ}E]') +set(gca,'XAxisLocation','top') +axis equal square + +% Wind Spectra +axes(h(17)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2]; +cwind = cmocean('matter',18); +ubin = [0 2:18 50]; +if isfield(swift,'windpower') + for it = 1:length(swift.time) + if ~isnan(swift.windu(it)) + [~,~,ibin] = histcounts(swift.windu(it),ubin); + if ibin ~= 0 + plot(swift.windfreq,swift.windpower(:,it),'color',cwind(ibin,:),'LineWidth',2) + hold on + else + plot(swift.windfreq,swift.windpower(:,it),'k','LineWidth',2) + end + else + plot(swift.windfreq,swift.windpower(:,it),'k','LineWidth',2) + end + end +end +xlim([0.05 5]) +ylim(10.^([-4 2])) +set(gca,'YScale','log','XScale','log') +colormap(gca,cwind) +xlabel('F [Hz]') +ylabel('P [m^2s^{-2}Hz^{-1}]') +title('Wind Spectra') +set(gca,'YTick',10.^(-5:2:2)) +set(gca,'XTick',10.^(-2:2)) +set(gca,'XTickLabel',{'10^{-2','10^0','10^2'}) + +axes(h(23)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2]; +if isfield(swift,'windpower') && any(~isnan(swift.windpower(:))) +pcolor(swift.time,swift.windfreq,log10(swift.windpower)) +shading flat +end +ylim([0.05 5]) +set(gca,'YScale','log') +c = slimcolorbar; +set(gca,'YDir','Reverse') +ylabel('F [Hz]') +cmocean('thermal') +clim([-3 2]) +c.Ticks = -3:2:2; +c.TickLabels = {'10^{-3}','10^{-1}','10^1'}; + c.Position = c.Position+[-0.01 0 0 0]; + +% Waves Spectra +axes(h(18)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2]; +cwind = cmocean('matter',18); +ubin = [0 2:18 50]; +if isfield(swift,'wavepower') + for it = 1:length(swift.time) + if ~isnan(swift.windu(it)) + [~,~,ibin] = histcounts(swift.windu(it),ubin); + if ibin ~= 0 + plot(swift.wavefreq,swift.wavepower(:,it),'color',cwind(ibin,:),'LineWidth',2) + hold on + else + plot(swift.wavefreq,swift.wavepower(:,it),'k','LineWidth',2) + end + else + plot(swift.wavefreq,swift.wavepower(:,it),'k','LineWidth',2) + end + end +end +xlim([0.05 1]) +ylim(10.^([-5 0])) +set(gca,'YScale','log','XScale','log') +colormap(gca,cwind) +c = slimcolorbar; +c.Label.String = 'U [ms^{-1}]'; +c.Ticks = 0:0.25:1; +c.TickLabels = num2str((c.Ticks*16 + 2)'); +xlabel('F [Hz]') +ylabel('P [m^2Hz^{-1}]') +title('Wave Spectra') +set(gca,'YTick',10.^(-4:2:1)) + +axes(h(24)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2]; +if isfield(swift,'wavepower') +pcolor(swift.time,swift.wavefreq,log10(swift.wavepower)) +end +shading flat +set(gca,'YScale','log') +c = slimcolorbar; +c.Label.String = 'P [m^2Hz^{-1}]'; +set(gca,'YDir','Reverse') +c.Ticks = -3:1:0; +c.TickLabels = {'10^{-3}','10^{-2}','10^{-1}','10^0'}; +ylim([0.05 1]) +set(gca,'YTick',[0.1 0.5 1]) +cmocean('thermal') + c.Position = c.Position+[-0.01 0 0 0]; + +linkaxes(h([1:3:end 2:3:11 23 24]),'x') +set(h([1:3:end-3 2:3:8]),'XTickLabel',[]) +axes(h(1)) +axis tight +axes(h(11)) +datetick('x','KeepLimits') +xlabel('Time') +axes(h(22)) +datetick('x','KeepLimits') +xlabel('Time') +axes(h(23)) +datetick('x','KeepLimits') +xlabel('Time') +axes(h(24)) +datetick('x','KeepLimits') +xlabel('Time') + +set(h,'FontSize',12) +rmemptysub + +end \ No newline at end of file diff --git a/GeneralTools/plotSWIFTV4.m b/GeneralTools/plotSWIFTV4.m new file mode 100644 index 0000000..5a030f9 --- /dev/null +++ b/GeneralTools/plotSWIFTV4.m @@ -0,0 +1,179 @@ +function fh = plotSWIFTV4(swift) + +if size(swift,2)>1 + swift = catSWIFT(swift); +end + +lw = 2; +nt = length(swift.time); + +fh = figure('color','w'); +fullscreen(2) +h = tight_subplot(8,3,[0.035 0.05],[0.1 0.075],0.075); +% tight_subplot(Nh, Nw, [gap_h gap_w], [lower upper], [left right] ) + +% Anemometer +axes(h(1)) +plot(swift.time,swift.windu,'-rx','LineWidth',lw) +ylabel('U [ms^{-1}]') +title('Wind Speed') +ylim([0 15]) +set(gca,'Clipping','off') +axes(h(4)) +plot(swift.time,swift.tair,'-rx','LineWidth',lw) +ylim([10 20]) +ylabel('T_{air} [^{\circ}C]') +title('Air Temperature') + +%IMU/GPS +axes(h(7)) +plot(swift.time,swift.wavesigH,'-bx','LineWidth',lw) +ylabel('H_s [m]') +title('Significant Wave Height') +ylim([0 5]) +axes(h(10)) +plot(swift.time,swift.wavepeakT,'-bx','LineWidth',lw) +ylabel('T_p [s]') +title('Peak Wave Period') +ylim([0 10]) +axes(h(13)) +plot(swift.time,swift.wavepeakdir,'-bx','LineWidth',lw) +ylabel('\Theta [^{\circ}]') +title('Peak Wave Direction') +ylim([0 360]) +axes(h(16)) +plot(swift.time,swift.tsea,'-bx','LineWidth',lw) +ylabel('T_{sea} [^{\circ}C]') +title('Sea Temperature') +axes(h(19)) +plot(swift.time,swift.sal,'-bx','LineWidth',lw) +ylabel('S [psu]') +title('Salinity') + +axes(h(22)) +if isfield(swift,'battery') +plot(swift.time,swift.battery,'-kx','linewidth',lw) +else + plot(swift.time,NaN(1,nt),'-kx','linewidth',lw) +end +ylabel('[V]') +title('Battery Level') + +% ADCP +axes(h(5)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2.1]+[-0.01 0.025 0 0]; +pcolor(swift.time,-swift.depth,swift.relu);shading flat +c = slimcolorbar; +c.Label.String = 'U_r [ms^{-1}]'; +cmocean('balance') +clim([-0.25 0.25]) +ylabel('Z [m]') +title('Zonal Velocity (Relative)') + +axes(h(11)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2.1]+[-0.01 0.03 0 0]; +pcolor(swift.time,-swift.depth,swift.relv);shading flat +c = slimcolorbar; +c.Label.String = 'V_r [ms^{-1}]'; +cmocean('balance') +clim([-0.25 0.25]) +ylabel('Z [m]') +title('Merid. Velocity (Relative)') + +axes(h(23)) +ax = gca;ax.Position = ax.Position.*[1 1 1 4.2]+[-0.01 0 0 0]; +if isfield(swift,'surftke') +pcolor(swift.time,-swift.surfz,log10(swift.surftke));shading flat + if median(log10(swift.surftke(:)),'omitnan')<-5.5 + clim([-8 -5]) + else + clim([-6 -3]) + end +end +colormap(gca,'jet') +c = slimcolorbar; +c.Label.String = '\epsilon [m^2s^{-3}]'; +c.Location = 'NorthOutside'; +ylabel('Z [m]') +title('Dissipation Rate (0-5 m, Wave Biased)') + +% Trajectory +axes(h(12)) +ax = gca;ax.Position = ax.Position.*[1 1 1 5]; +lonscale = mean(cos(swift.lat*pi/180),'omitnan'); +scatter(swift.lon,swift.lat,[],swift.time,'filled'); +set(gca,'YAxisLocation','right') +hold on +quiver(swift.lon,swift.lat,swift.driftu./lonscale,swift.driftv,'k') +c = slimcolorbar; +c.Location = 'South'; +c.TickLabels = datestr(c.Ticks,'mmm-dd'); +ylabel('Lat [^{\circ}N]') +xlabel('Lon [^{\circ}E]') +set(gca,'XAxisLocation','top') +axis equal square + +% Waves Spectra +axes(h(18)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2]; +cwind = cmocean('matter',18); +ubin = [0 2:18 50]; +for it = 1:length(swift.time) + if ~isnan(swift.windu(it)) + [~,~,ibin] = histcounts(swift.windu(it),ubin); + if ibin ~= 0 + plot(swift.wavefreq,swift.wavepower(:,it),'color',cwind(ibin,:),'LineWidth',2) + hold on + else + plot(swift.wavefreq,swift.wavepower(:,it),'k','LineWidth',2) + end + else + plot(swift.wavefreq,swift.wavepower(:,it),'k','LineWidth',2) + end +end +ylim(10.^([-5 0])) +xlim([0.1 1]) +set(gca,'YScale','log','XScale','log') +colormap(gca,cwind) +c = slimcolorbar; +c.Label.String = 'U [ms^{-1}]'; +c.Ticks = 0:0.25:1; +c.TickLabels = num2str((c.Ticks*16 + 2)'); +xlabel('F [Hz]') +ylabel('P [m^2Hz^{-1}]') +title('Wave Spectra') +set(gca,'YTick',10.^(-4:2:1)) + +axes(h(24)) +ax = gca;ax.Position = ax.Position.*[1 1 1 2]; +pcolor(swift.time,swift.wavefreq,log10(swift.wavepower)) +shading flat +set(gca,'YScale','log') +c = slimcolorbar; +c.Label.String = 'P [m^2Hz^{-1}]'; +set(gca,'YDir','Reverse') +clim([-5 0]) +c.Ticks = -5:1:0; +c.TickLabels = {'10^{-3}','10^{-2}','10^{-1}','10^0'}; +ylim([0.05 1]) +set(gca,'YTick',[0.1 0.5 1]) +cmocean('thermal') + +linkaxes(h([1:3:end 2:3:end 24]),'x') +set(h([1:3:end-3 2:3:end-3]),'XTickLabel',[]) +axes(h(1)) +xlim([min(swift.time) max(swift.time)]) +axes(h(22)) +datetick('x','KeepLimits') +xlabel('Time') +axes(h(23)) +datetick('x','KeepLimits') +xlabel('Time') +axes(h(24)) +datetick('x','KeepLimits') +xlabel('Time') + +set(h,'FontSize',12) +rmemptysub + +end \ No newline at end of file diff --git a/IMU/reprocess_IMU.m b/IMU/reprocess_IMU.m index 8f373dd..4bea586 100644 --- a/IMU/reprocess_IMU.m +++ b/IMU/reprocess_IMU.m @@ -26,6 +26,22 @@ slash = '/'; end +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function + +l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); + +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end + %% Make sure 'calctype' specifies 'IMU', 'GPS' or 'IMUandGPS' if ~strcmp(calctype,'IMU') && ~strcmp(calctype,'GPS') && ~strcmp(calctype,'IMUandGPS') @@ -45,22 +61,6 @@ error('Filter type must be ''RC'' or ''elliptic''.') end -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function - -l1file = dir([missiondir slash '*SWIFT*L1.mat']); -l2file = dir([missiondir slash '*SWIFT*L2.mat']); - -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit IMU reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) - return -end - % Flag bad wave data badwaves = false(1,length(SWIFT)); @@ -268,7 +268,7 @@ end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Log reprocessing and flags, then save new L3 file or overwrite existing one params.Data = calctype; params.Filter = filtertype; @@ -286,7 +286,7 @@ sinfo.postproc(ip).params = params; -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +save([sfile.folder slash sfile.name(1:end-6) 'L3.mat'],'SWIFT','sinfo') %% End function end \ No newline at end of file diff --git a/SBG/reprocess_SBG.m b/SBG/reprocess_SBG.m index f2c740b..2d41263 100644 --- a/SBG/reprocess_SBG.m +++ b/SBG/reprocess_SBG.m @@ -19,25 +19,25 @@ slash = '/'; end -% Length of raw burst data to process, from end of burst (must be > 1536/5 = 307.2 s) -tproc = 475;% seconds - -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function -l1file = dir([missiondir slash '*SWIFT*L1.mat']); l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) return end +%% Length of raw burst data to process, from end of burst (must be > 1536/5 = 307.2 s) +tproc = 475;% seconds + %% Flag bad wave data badwaves = false(1,length(SWIFT)); @@ -204,7 +204,7 @@ end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Log reprocessing and flags, then save new L3 file or overwrite existing one params.AltGPS = useGPS; @@ -220,7 +220,7 @@ sinfo.postproc(ip).flags.badwaves = badwaves; sinfo.postproc(ip).params = params; -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +save([sfile.folder slash sfile.name(1:end-6) 'L3.mat'],'SWIFT','sinfo') %% End function end \ No newline at end of file diff --git a/Signature/reprocess_SIG.m b/Signature/reprocess_SIG.m index 25051a9..4d17d41 100644 --- a/Signature/reprocess_SIG.m +++ b/Signature/reprocess_SIG.m @@ -99,6 +99,22 @@ slash = '/'; end +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function + +l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); + +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; + load([sfile.folder slash sfile.name],'SWIFT','sinfo'); +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) + return +end + %% Load User Input Toggles opt.readraw = readraw;% read raw binary files opt.plotburst = plotburst; % generate plots for each burst @@ -131,21 +147,8 @@ ftype = '.mat'; end -%% Load or create SWIFT structure, create SIG structure, list burst files +%% Create SIG structure, list burst files -l1file = dir([missiondir slash '*SWIFT*L1.mat']); -l2file = dir([missiondir slash '*SWIFT*L2.mat']); - -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; - load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) - return -end burstreplaced = false(length(SWIFT),1); badsig = false(1,length(SWIFT)); @@ -491,8 +494,23 @@ SWIFT = SWIFT(isort); end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Save SIG Structure + Plot %%%%%%%% + +if opt.saveSIG + save([sfile.folder slash sfile.name(1:end-7) '_burstavgSIG.mat'],'SIG') +end + +% Plot burst Averaged SWIFT Signature Data +catSIG(SIG,'plot'); +set(gcf,'Name',sfile.name(1:end-7)) +if opt.saveplots + figname = [missiondir slash get(gcf,'Name')]; + print([figname '_SIG'],'-dpng') + close gcf +end + +%% Log reprocessing and flags, then save new L3 file or overwrite existing one params = opt; if isfield(sinfo,'postproc') @@ -507,23 +525,9 @@ sinfo.postproc(ip).flags.badsig = badsig; sinfo.postproc(ip).params = params; -save([sfile.folder slash sfile.name(1:end-7) '_L2.mat'],'SWIFT','sinfo') - -%% Save SIG Structure + Plot %%%%%%%% - -if opt.saveSIG - save([sfile.folder slash sfile.name(1:end-7) '_burstavgSIG.mat'],'SIG') -end - -% Plot burst Averaged SWIFT Signature Data -catSIG(SIG,'plot'); -set(gcf,'Name',sfile.name(1:end-7)) -if opt.saveplots - figname = [missiondir slash get(gcf,'Name')]; - print([figname '_SIG'],'-dpng') - close gcf -end +save([sfile.folder slash sfile.name(1:end-7) '_L3.mat'],'SWIFT','sinfo') +%% Return to mission directory cd(missiondir) end \ No newline at end of file diff --git a/Winds/reprocess_WXT.m b/Winds/reprocess_WXT.m index eda6fed..1a2bb4a 100644 --- a/Winds/reprocess_WXT.m +++ b/Winds/reprocess_WXT.m @@ -15,19 +15,19 @@ slash = '/'; end -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function -l1file = dir([missiondir slash '*SWIFT*L1.mat']); l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) return end @@ -102,7 +102,7 @@ end end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Log reprocessing and flags, then save new L3 file or overwrite existing one if isfield(sinfo,'postproc') ip = length(sinfo.postproc)+1; @@ -116,7 +116,7 @@ sinfo.postproc(ip).flags = []; sinfo.postproc(ip).params = []; -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +save([sfile.folder slash sfile.name(1:end-6) 'L3.mat'],'SWIFT','sinfo') %% End function end \ No newline at end of file diff --git a/Winds/reprocess_Y81.m b/Winds/reprocess_Y81.m index 544a9c5..7996efe 100644 --- a/Winds/reprocess_Y81.m +++ b/Winds/reprocess_Y81.m @@ -16,19 +16,19 @@ slash = '/'; end -%% Load existing L2 product, or L1 product if does not exist. If no L1 product, return to function +%% Load existing L3 product, or L2 product if does not exist. If no L3 product, return to function -l1file = dir([missiondir slash '*SWIFT*L1.mat']); l2file = dir([missiondir slash '*SWIFT*L2.mat']); +l3file = dir([missiondir slash '*SWIFT*L3.mat']); -if ~isempty(l2file) % First check to see if there is an existing L2 file to load - sfile = l2file; +if ~isempty(l3file) % First check to see if there is an existing L3 file to load + sfile = l3file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -elseif isempty(l2file) && ~isempty(l1file)% If not, load L1 file - sfile = l1file; +elseif isempty(l3file) && ~isempty(l2file)% If not, load L1 file + sfile = l2file; load([sfile.folder slash sfile.name],'SWIFT','sinfo'); -else % Exit reprocessing if no L1 or L2 product exists - warning(['No L1 or L2 product found for ' missiondir(end-16:end) '. Skipping...']) +else % Exit reprocessing if no L2 or L3 product exists + warning(['No L2 or L3 product found for ' missiondir(end-16:end) '. Skipping...']) return end @@ -78,7 +78,7 @@ end -%% Log reprocessing and flags, then save new L2 file or overwrite existing one +%% Log reprocessing and flags, then save new L3 file or overwrite existing one if isfield(sinfo,'postproc') ip = length(sinfo.postproc)+1; @@ -92,7 +92,7 @@ sinfo.postproc(ip).flags = []; sinfo.postproc(ip).params = []; -save([sfile.folder slash sfile.name(1:end-6) 'L2.mat'],'SWIFT','sinfo') +save([sfile.folder slash sfile.name(1:end-6) 'L3.mat'],'SWIFT','sinfo') %% End function end From e7e0a095bf2ef3ad58e28d0e21106648c1f6de3f Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:36:08 -0700 Subject: [PATCH 31/34] updates --- GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m | 1 - .../KZ_PostProcessing/L3_postprocessSWIFT.m | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m index 102e46c..1084514 100644 --- a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m @@ -317,7 +317,6 @@ end -close all clear SWIFT end % End mission loop diff --git a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m index f399265..48524de 100644 --- a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m @@ -23,14 +23,14 @@ end % Processing toggles -rpIMU = false; % Waves -rpSBG = false; % Waves -rpWXT = false; % MET +rpIMU = true; % Waves +rpSBG = true; % Waves +rpWXT = true; % MET rpY81 = true; % MET -rpACS = false; % CT -rpSIG = false; % TKE -rpAQH = false; % TKE -rpAQD = false; % TKE +rpACS = true; % CT +rpSIG = true; % TKE +rpAQH = true; % TKE +rpAQD = true; % TKE % Plotting toggle plotflag = true; @@ -68,7 +68,7 @@ if rpIMU if ~isempty(dir([missiondir slash '*' slash 'Raw' slash '*' slash '*_IMU_*.dat'])) disp('Reprocessing IMU data...') - calctype = 'IMU'; + calctype = 'GPS'; filtertype = 'RC'; saveraw = false; interpf = false; From 7f5d05a8e2ca1bcd01ceee3efdeabf864f0fd5e8 Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:47:52 -0700 Subject: [PATCH 32/34] update --- GeneralTools/KZ_PostProcessing/L0_createSBD.m | 4 ++-- GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m | 11 +++++------ GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m | 2 +- GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m | 4 ++-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/GeneralTools/KZ_PostProcessing/L0_createSBD.m b/GeneralTools/KZ_PostProcessing/L0_createSBD.m index 8c660ff..5aa1b18 100644 --- a/GeneralTools/KZ_PostProcessing/L0_createSBD.m +++ b/GeneralTools/KZ_PostProcessing/L0_createSBD.m @@ -42,7 +42,7 @@ cd(missiondir) sname = missions(im).name; -diaryfile = ['L0_' missions(im).name '_createSBD.txt']; +diaryfile = [missions(im).name '_L0_createSBD.txt']; if exist(diaryfile,'file') delete(diaryfile); end @@ -126,4 +126,4 @@ delete(payloadfile) diary off -end % End mission loop \ No newline at end of file +end % End mission loop diff --git a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m index 1084514..f153aca 100644 --- a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m @@ -31,8 +31,6 @@ % Processing parameters plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) -fixspectra = false; % binary flag to redact low freq wave spectra, note this also recalcs wave heights -fixpositions = false; % binary flag to use "filloutliers" to fix spurious positions. Use with care. % List missions missions = dir([expdir slash 'SWIFT*']); @@ -47,7 +45,7 @@ sname = missions(im).name; % Create diary file - diaryfile = ['L1_' missions(im).name '_compileSWIFT.txt']; + diaryfile = [missions(im).name '_L1_compileSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end @@ -154,7 +152,6 @@ % End burst loop end -diary off %% Fill empty SWIFT fields due to missing payloads in a burst @@ -299,14 +296,14 @@ if micro save([missiondir slash 'micro' sname '_L1.mat'],'SWIFT*') else - save([missiondir slash sname '_L1.mat'],'SWIFT',sinfo) + save([missiondir slash sname '_L1.mat'],'SWIFT','sinfo') end %% Plot if plotflag - l1file = dir([missiondir slash '*L2.mat']); + l1file = dir([missiondir slash '*L1.mat']); if strcmp(sinfo.type,'V3') fh = plotSWIFTV3(SWIFT); else @@ -319,4 +316,6 @@ clear SWIFT +diary off + end % End mission loop diff --git a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m index 5e8087d..e52b3d5 100644 --- a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m @@ -49,7 +49,7 @@ end % Create diary file - diaryfile = ['L2_' missions(im).name '_pruneSWIFT.txt']; + diaryfile = [missions(im).name '_L2_pruneSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end diff --git a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m index 48524de..dba2652 100644 --- a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m @@ -40,12 +40,12 @@ missions = missions([missions.isdir]); %% Loop through missions and reprocess -for im = 1:length(missions) +for im = 1%:length(missions) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) - diaryfile = ['L3_' missions(im).name '_postprocessSWIFT.txt']; + diaryfile = [missions(im).name '_L3_postprocessSWIFT.txt']; if exist(diaryfile,'file') delete(diaryfile); end From b15446d33b7807a7212cc41153e7c92049b70acf Mon Sep 17 00:00:00 2001 From: kzeiden <97851010+kzeiden@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:04:51 -0700 Subject: [PATCH 33/34] updates --- GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m | 2 +- GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m | 2 +- GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m index f153aca..8b575a4 100644 --- a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m @@ -38,7 +38,7 @@ %% Loop through missions -for im = 1%:length(missions) +for im = 1:length(missions) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) diff --git a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m index e52b3d5..2cc9314 100644 --- a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m @@ -33,7 +33,7 @@ missions = missions([missions.isdir]); %% Loop through missions and remove burst identified as out-of-water -for im = 1%:length(missions) +for im = 1:length(missions) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) diff --git a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m index dba2652..72e069c 100644 --- a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m @@ -40,7 +40,7 @@ missions = missions([missions.isdir]); %% Loop through missions and reprocess -for im = 1%:length(missions) +for im = 1:length(missions) missiondir = [missions(im).folder slash missions(im).name]; cd(missiondir) From d4b2d0340d0f0f27ca07c1d135934ad0ec5fe505 Mon Sep 17 00:00:00 2001 From: Jim Thomson Date: Fri, 11 Oct 2024 13:48:50 -0700 Subject: [PATCH 34/34] testing on Mac --- .DS_Store | Bin 16388 -> 18436 bytes Aquadopp/reprocess_AQH.m | 2 +- GeneralTools/.DS_Store | Bin 8196 -> 8196 bytes GeneralTools/KZ_PostProcessing/L0_createSBD.m | 5 +- .../KZ_PostProcessing/L1_compileSWIFT.m | 6 +- .../KZ_PostProcessing/L2_pruneSWIFT.asv | 112 ------------------ .../KZ_PostProcessing/L2_pruneSWIFT.m | 6 +- .../KZ_PostProcessing/L3_postprocessSWIFT.m | 8 +- GeneralTools/plotSWIFTV3.m | 3 +- Signature/.DS_Store | Bin 6148 -> 6148 bytes Signature/absorption.m | 45 +++++++ Winds/inertialdissipation.m | 2 +- 12 files changed, 62 insertions(+), 127 deletions(-) delete mode 100644 GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.asv create mode 100644 Signature/absorption.m diff --git a/.DS_Store b/.DS_Store index ed4cc254047661111133b96ed7022ac2c7d52d80..33d99bb20a80c808d1ffb1b4d62fd834fd5bc7b8 100644 GIT binary patch delta 3024 zcmchZdvFui6~^zklFdrHvQ`+`mS4!o7(XN|*|KCC6KsSZ6aoeWwh1lRmJCX4$+m0= zIL4-Nl0ImI=T3N})24-%c9Ln+jG=A%2&B!lNm|l|xDBCc8EBblrcH(>O`D<9Y0s`? zutN#`tD2GS-h211?tbSxzjG(XF>wfmj4`Yn4tW@>r%RE(YOBcf^9W;hYTfc=VVSj= z2J|$+SSQ=b!mO1Ive@K+gS+eWDS|r5|FE zoOZ9@H|n&Hc-(HU)9$OPz&h(mpQ?ZeMlGR6!FNRK>L#H!g^7Gyp2 zHV4>2_5?e{e#D+*XV@$3b#{gQiM`9NvwuT?4tki71_c&m!+}B+VF|ogj=Qh|YY@a{ zw4ogzM-O%)f_o9e{n(2UJb+K)vp9&)<53*Jm+=)G_u(m=!fAXL-^0)FE4+&HcoSD~ z4e#PT{0;vS%z`3VglxejEEURyYN0_`C#>g9^AkAnp>Jz%f35^#Lqc*@w2vj>wGSgtUxqO+p=?VYnREgqF3Su!*p(Tj!n*&8w>`8 zX>);9CX*s6FN&fgDJ@HAdk1CDv){7I>@D^_yN-E~Nl!aUP>w3nu^Fqe7I$MEY1x5$ zuuao*7pZw4dNGVWr071Gn-DdxH8q4^HZ|#MjaF`+bi=#*1!hXFbN$Zcl5LVu08Rj)a*t6 zihH~`b7aKrwENwa3DZ{9IPFz#-{`18%v|hQzIsRB-hCqvjP5_mW&gs0WAs0_JaSGd z)Y+fVQo-l1cG_Jv$)|cKFRFcBN(DbZTezSskc{?QGts^T-rZwxQ)69xxzQl!;(lOD33Bz=nVzy*vNmBc8S7cx~GPGB5 zh@v){nkCJR z;ubiQ!*{Kaq)(`Sy|yzxw7x$arEQTdwQU7xTOI0LLnx$e%A%Olq!#I8qHsOdzqalkX#_kzsat$ci1)dS3=4^A;JiW%x^^j77=!0BsX^i2VO3zLcN|!Q?c)LoiXz{u8`_U*ZDZ zz(xE8@8j?IhhP-ag)G7Lp#*4~n*c%%|3RVO)YNR8g^F1Q%AaMRH%yvk(id1W7OFn- z*ER50m7p%yn9plmMPr#+wrr`u=nyHKO3Jj>Eix916i6jC(Jhl)B6&v{#->(mz$T4N+pz;( z8k_oN@ji1u9?|fmLjMSk;&FTdU&KjnTXF&~spu%l8X3{j-0x1XBbnv5u;ZA@4m{4U zmF4ouVt4*W6~Z=|ca~VG2L6JdDanJ(e^=sOE8k2$O7e84OO|gYpE}~HkWGo?({fK8 z^7&=coZM3fmsw@rQ93!hjym|E>|Fj@X2zJAC_W$Q$R(1Ok(piaky-&wuZ>!5+=A^0 zX|-_|24-?rv5&WAPw$G54U&3pI?O)^fxP;5Ng11#K;{(mj3k0iBEO>-k zp;>5UjJM3o!v+#*N&fAMU{iBb%jy|{9_CwZJvvjq%&T3lF?n&)r)tGB70abbpK4u~ zsXXaZwJTScyh)#GU6)BB25FbIa$VA;S~p*=zfmbo0n>~{XL9a_SGR@u%a$w>vSO^1 z>{g3!AwlT-muX;^qx ztAH2oQ~?EC(QNkmQuE}H6ArHEOQy{3-=tbA)q>=ysGeh0DoBTUy)R>`RiB~LE*2@9 zb@G2JlAiyyNRA!5NQbU>*n40|Awt@SjsH(otZH~}IT+kjr8qA&5x{zh+Qtu`rRlW$ wXB{`!dBAv`oRdpOXEj|~)HQVMAfsyS4z=a|bq`J6pLXJ^SNLdsd8@wqKd-c_Gynhq delta 2130 zcmai03v5$W7(U-^H}36ew_CP#?KWVO0o|jcD;w=Lz-=8HxWRaA3`R2PXeZM>*g6nF zvjHPw5Co6$1230!d+pEk+ex+c4r|*ba`z9C%iU;STrF`?3u8Z(*iV2hs^1XWaGd%^S1`r^#^|~ zt~V&b@Zz@S&dxR_FvTu1RUy7qOYy3h@98qwR;>~rdor+IT=I;pO~CpWeRLmfS>Ev=z$MX{p4t|QPMide;dpUY^CSL4K4pRHHB$mAJ(VKC4W z@^^>m9F*mV7IyM%Q(d@qWk^;7iXL&9#+&NyXzd6p1h^jYmyILS1H&tKfx-cCCT(`Is`;j$X;=S|wuHGyzX?b1*OtxkkRtjTsq zWd`R9w7IfiP~Vgz*>2_4e1@!^ksHHH0^On7J3B(%k-_+Eaj3{1qdieCzAPFUqvtF3 zP8ws~&i1ea>`iu<9cL%lN9>fCR6Mu-VWMvb_EQzVOtig?BlrMEaT;fcwoh@Es`)(C z^Pf_zG>qywRmzmIB+n2shZgVzK`e=rV=@d=E31f@72=_i;V2a@TdlZK(x01RcO(-Q zCY5Q7C=p6Cg>!#}Oj)Z`rh}5bw)J&MeN=Cn_%`7I65oS(h-z(@NOtZP4W4CUoTExK zIUQn0mRt9PM1@=8>guT|7Kqut1hG1)g^YoV%ie@kMUnT#+eh3qHpg99JjGvCGb^H6 zhbJT^aV~adSA&a-+3p&yGp+_mUfOJ~i*ucEU13@z-@`3Y#?_!A>km!za?3Tw)#|}u zNfH;WIa?A-S1i&oyPjLrj?#Knkk^i2ZdT6bu_m^Z-NiPrhuL;^kiEiQWv{U#>?Hew zon`0Q&+IDujs4F4Kn&ub!iwQYg%jDxK`shWM)LKehTLneF5V^*ZU`;7jfA@#D{vQ9 zl4LjNl6?S=VkdT?VK-jDTX+YD@GjAQ2A>e^Uy+HOCC&bbUvLFi$;5_}i6xSWxuqhh zT&k7k|4#uliiOTJp}M&4Vmcq1k5&7I@&QHIZA%Q6kz7p9bE$lcOn9m>x^Y>YT&&DX zZQveDfkIrU20cu+=Tl@A(VG%JS(Bk^jNXpqW!H_*QemmoWTaKpjo|+u@Q<-i*yrpV zyGY>w4I{M;J2H@k@hHO0QY-(U1D56SJv9dpot{3kDHiksqVAr9M$skw86O a@7da)0sRoSnKz2@1&)D&!q0Hywf+IlC+gJz diff --git a/Aquadopp/reprocess_AQH.m b/Aquadopp/reprocess_AQH.m index be0652e..c29caa8 100644 --- a/Aquadopp/reprocess_AQH.m +++ b/Aquadopp/reprocess_AQH.m @@ -96,7 +96,7 @@ [time,vel,amp,cor,press,pitch,roll,heading] = ... readSWIFTv3_AQH([bfiles(iburst).folder '\' bfiles(iburst).name]); else - load([bfiles(iburst).folder '\' bfiles(iburst).name],... + load([bfiles(iburst).folder slash bfiles(iburst).name],... 'Vel','Cor','Amp','Pressure','pitch','roll','heading','time') if exist('Vel','var') vel = Vel; diff --git a/GeneralTools/.DS_Store b/GeneralTools/.DS_Store index 97336dc5a31f3f479d7593a69dfc1c3c68fb2176..94c8f56a6bf2c86dd9cb81398a4dd08564926a14 100644 GIT binary patch delta 85 zcmZp1XmOa}&nUhzU^hRb_+}mfcP1S{25*KahIobmhJ1!%h7urM1Y{*Mqyl-x44Djh l4CyJw$vH{+`8kY}1BCuDGHmu2?qu1_F7b_J^EnZAW&oWe7SjL# delta 46 zcmZp1XmOa}&nUJrU^hRb*k&F9cc#frLcbXqH`fVwvP^8y-^?!Yjb*cr=xwHn4LSg0 CTMvi; diff --git a/GeneralTools/KZ_PostProcessing/L0_createSBD.m b/GeneralTools/KZ_PostProcessing/L0_createSBD.m index 5aa1b18..6129b80 100644 --- a/GeneralTools/KZ_PostProcessing/L0_createSBD.m +++ b/GeneralTools/KZ_PostProcessing/L0_createSBD.m @@ -11,10 +11,11 @@ % 9/2017 add oxygen optode and sea owl fluorometer % 9/2019 changed time convention to use start of the burst,rather than end % +% K. Zeiden, 10/2024 overhaul to make PC compatible and catch all sensor payloads %% Experiment directory -expdir = 'S:\SEAFAC\June2024'; +expdir = '/Volumes/Data/SEAFAC/June2024'; SBDfold = 'ProcessedSBD'; %% Sampling Parameters @@ -108,7 +109,7 @@ end syscommand = [syscommand ' ' sbdfile]; else - syscommand = ['!cat ' payloadfile]; + syscommand = ['cat ' payloadfile]; for iprc = 1:length(PRCburstfiles) syscommand = [syscommand ' ' PRCburstfiles(iprc).folder slash PRCburstfiles(iprc).name]; end diff --git a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m index 8b575a4..dca34e1 100644 --- a/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L1_compileSWIFT.m @@ -13,10 +13,10 @@ % microSWIFT - Use the time embedded within the payload 50 or 51 or 52 of the % SBD file, which is the time at the end of the burst of raw data. -% K. Zeiden 10/01/2024 +% K. Zeiden 10/01/2024, based on orginal compileSWIFT_SBDservertelemetry.m %% Experiment directory -expdir = 'S:\SEAFAC\June2024'; +expdir = '/Volumes/Data/SEAFAC/June2024'; % SBD folder SBDfold = 'ProcessedSBD'; @@ -30,7 +30,7 @@ end % Processing parameters -plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) +plotflag = false; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) % List missions missions = dir([expdir slash 'SWIFT*']); diff --git a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.asv b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.asv deleted file mode 100644 index 1f1b1a4..0000000 --- a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.asv +++ /dev/null @@ -1,112 +0,0 @@ -% Find out-of-water bursts and prune them from the SWIFT structure - -% K. Zeiden 10/10/2024 - -%% Experiment Directory -expdir = 'S:\SEAFAC\June2024'; - -%% Parameters for QC/out-of-water identification - -if ispc - slash = '\'; -else - slash = '/'; -end - -% Processing parameters -plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) - -% QC Parameters -minwaveheight = 0;% minimum wave height in data screening -minsalinity = 1;% PSU, for use in screen points when buoy is out of the water (unless testing on Lake WA) -maxdriftspd = 3;% m/s, this is applied to telemetry drift speed, but reported drift is calculated after that - -disp('-------------------------------------') -disp('Out-of-water parameters:') -disp(['Minimum wave height: ' num2str(minwaveheight) ' m']) -disp(['Minimum salinity: ' num2str(minsalinity) ' PSU']) -disp(['Maximum drift speed: ' num2str(maxdriftspd) ' ms^{-1}']) -disp('-------------------------------------') - -%% List missions -missions = dir([expdir slash 'SWIFT*']); -missions = missions([missions.isdir]); - -%% Loop through missions and remove burst identified as out-of-water -for im = 1%:length(missions) - - missiondir = [missions(im).folder slash missions(im).name]; - cd(missiondir) - sname = missions(im).name; - - % Load L1 file - l1file = dir([missiondir slash '*SWIFT*L1.mat']); - if ~isempty(l1file) - load([l1file.folder slash l1file.name],'SWIFT','sinfo'); - else % Exit reprocessing if no L1 product exists - warning(['No L1 product found for ' missiondir(end-16:end) '. Skipping...']) - return - end - - % Create diary file - diaryfile = ['L2_' missions(im).name '_pruneSWIFT.txt']; - if exist(diaryfile,'file') - delete(diaryfile); - end - diary(diaryfile) - disp(['Pruning ' sname]) - - % Loop through and identify out-of-water bursts - nburst = length(SWIFT); - outofwater = false(1,nburst); - - for iburst = 1:nburst - oneSWIFT = SWIFT(iburst); - - % Waves too small (probably out of water) - if isfield(oneSWIFT,'sigwaveheight') - if oneSWIFT.sigwaveheight < minwaveheight || oneSWIFT.sigwaveheight >= 999 - outofwater(iburst) = true; - disp('Waves too small, removing burst.') - end - end - - % Salinity too small (probably out of water) - if isfield(oneSWIFT,'salinity') - if all(oneSWIFT.salinity < minsalinity) - outofwater(iburst) = true; - disp('Salinity too low, removing burst.') - end - end - - % Drift speed limit (if too fast, probably out of water) - if isfield(oneSWIFT,'driftspd') - if oneSWIFT.driftspd > maxdriftspd - outofwater(iburst) = true; - disp('Speed too fast, removing burst.') - end - end - - end - - % Remove out-of-water bursts - SWIFT(outofwater) = []; - - % Plot - if plotflag - if strcmp(sinfo.type,'V3') - fh = plotSWIFTV3(SWIFT); - else - fh = plotSWIFTV4(SWIFT); - end - set(fh,'Name',l2file.name(1:end-4)) - print(fh,[l3file.folder slash l3file.name(1:end-4)],'-dpng') - end - - % Save L2 file - save([missiondir slash sname '_L2.mat'],'SWIFT','sinfo') - - % Turn off diary - diary off - -end % End mission loop \ No newline at end of file diff --git a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m index 2cc9314..8dc770c 100644 --- a/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L2_pruneSWIFT.m @@ -1,9 +1,9 @@ % Find out-of-water bursts and prune them from the SWIFT structure -% K. Zeiden 10/10/2024 +% K. Zeiden 10/10/2024... similar to SWIFT_QC.m %% Experiment Directory -expdir = 'S:\SEAFAC\June2024'; +expdir = '/Volumes/Data/SEAFAC/June2024'; %% Parameters for QC/out-of-water identification @@ -14,7 +14,7 @@ end % Processing parameters -plotflag = true; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) +plotflag = false; % binary flag for plotting (compiled plots, not individual plots... that flag is in the readSWIFT_SBD call) % QC Parameters minwaveheight = 0;% minimum wave height in data screening diff --git a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m index 72e069c..6851b52 100644 --- a/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m +++ b/GeneralTools/KZ_PostProcessing/L3_postprocessSWIFT.m @@ -3,16 +3,16 @@ % Master post-processing function, that calls sub-functions to reprocess % each different type of raw SWIFT data and create an L3 product. % L1 product must have been created prior to running this script, -% by running 'compileSWIFT.m', and L2 created by running 'qcSWIFT.m'; +% by running 'compileSWIFT.m', and L2 created by running 'pruneSWIFT.m'; -% K. Zeiden 07/2024 +% K. Zeiden 07/2024, based on existing sensor-specific processing codes w % Need to consider additional QC steps after processing... % Airmar temp, NaN out if below/above -20/50 deg C % Wind speed, NaN out above 30 m/s %% Experiment Directory -expdir = ['S:' slash 'SEAFAC' slash 'June2024']; +expdir = ['/Volumes/Data/SEAFAC/June2024']; %% Processing toggles @@ -33,7 +33,7 @@ rpAQD = true; % TKE % Plotting toggle -plotflag = true; +plotflag = false; % List of missions missions = dir([expdir slash 'SWIFT*']); diff --git a/GeneralTools/plotSWIFTV3.m b/GeneralTools/plotSWIFTV3.m index 2169a37..3a72e95 100644 --- a/GeneralTools/plotSWIFTV3.m +++ b/GeneralTools/plotSWIFTV3.m @@ -8,7 +8,8 @@ nt = length(swift.time); fh = figure('color','w'); -fullscreen(2) +%fullscreen(2) +h = tight_subplot(8,3,[0.035 0.05],[0.1 0.075],0.075); h = tight_subplot(8,3,[0.035 0.05],[0.1 0.075],0.075); % tight_subplot(Nh, Nw, [gap_h gap_w], [lower upper], [left right] ) diff --git a/Signature/.DS_Store b/Signature/.DS_Store index 0030a9b020c495b88566c005d5926597f423b116..305d62b84e54279ebe15c97fb4129045b1d7c043 100644 GIT binary patch delta 46 zcmZoMXfc@J&&a