%% CC1611SR - Underway Data Processing
% Goericke, Feb 2017
%% Process Underway data from cruise CC1611SR.  The steps are: read the raw data files into Matlab as tables.  Generate one table from those data with standardized variable names,  QC the data (remove outliers, filter the data and remove sections of the data compromized by bubbles or stopped flow), generate calibrations for SSTemp, SSSal and ChlFluor using CalCOFI bottle data, use the calibrations to generate corrected data, assemble a final data structure and calculate a reduced datafile that has 10 min resolution.
%
% OUTPUT from the script:
% CC<cruise>UW_RawFiles.mat - The individual raw data files converted to matlab tables. Includeds all original variables and their original names
% CC<cruise>UW_RawData.mat -  A table with all the relevant data from the *.elg files properly formatted. Includeds all original variables but using standardized variable names.
% CC<cruise>UW_FilteredData.mat - The quality controlled data.
% CC<cruise>UW_FinalData.mat -
% CC<cruise>UW_10MinData.mat -
%
% SCRIPT IS USING THE FOLLOWING FUNCTIONS:
% sw-dist.m - calculate distance given two positions. Part of the SW Toolbox.  Should switch to the new version
% UWData_Filter.m -
% daylight.m - function that results in a 1 if the PST is with sunrise + 1 hour and sunset - 1 hour.
%
% SCRIPT IS USING THE FOLLOWING DATA:
% D:\Documents\MATLAB\CalCOFI\Underway\CalCOFI_BottleData.mat
%
%% Read the Data Files
% The data are located in 'F:\2016 Raw\1604SH'.  The raw data files are the files *.MET. The files come in two versions.  Files 1-3 have 72 columns and files 4 to 17 have 62 columns.  The data are read and written to two files D1 and D2.

Cruise = 'CC1611';
CruiseDir = 'F:\2016 Raw\1611SR\met\data';
cd(CruiseDir);

%
% Either load the 17 files into the workspace using ImportData or, if the file '<cruise>UW_Raw.mat' exists, load this file.

Filename = [Cruise 'UW_RawFiles.mat'];
load(Filename)

%
% Read the 17 files individually, concatenate these to get tables D1 and D2 and then combine these into one table DRaw.
% Read Files

FileList = dir('*.MET');
NoFiles = length(FileList);
N = 1;

clear D1 D2 T

% files 1-3 have 72 columns
for N = 1 : 3
    FileList(N).name
    % generate a Data from the file name
    Date = datetime(FileList(N).name(1:6), 'InputFormat', 'yyMMdd');
    
    D1Format = ['%s ' repmat('%f', 1, 72)] ;
    T =  readtable(FileList(N).name, 'FileType', 'text', 'HeaderLines', 3, 'Format', D1Format);
    L = height(T);
    % add a date variable to the table T
    T.Date = repmat(Date, L, 1);
    
    if ~exist('D1', 'var')
        D1 = T;
    else
        D1 = [D1;T];
    end
end

% files 4 - 17 have 62 columns
for N = 4 : NoFiles
    FileList(N).name
    Date = datetime(FileList(N).name(1:6), 'InputFormat', 'yyMMdd');
    
    D2Format = ['%s ' repmat('%f', 1, 61)] ;
    T =  readtable(FileList(N).name, 'FileType', 'text', 'HeaderLines', 3, 'Format', D2Format);
    L = height(T);
    T.Date = repmat(Date, L, 1);
    
    if ~exist('D2', 'var')
        D2 = T;
    else
        D2 = [D2;T];
    end
end

clear D1Format D2Format FileList L N NoFiles 

%
% Combine D1 and D2 into one file

% generate a table DRaw starting out with the Data
clear DRaw
DRaw = [ D1(:,{'Date'}); D2(:,{'Date'})];
DRaw.Time = [D1.x_Time; D2.x_Time];
DRaw.Time = timeofday(datetime(DRaw.Time, 'Format', 'HHmmss'));
DRaw.DateTimeUTC = DRaw.Date + DRaw.Time;

% preliminary table T consisting of the first 38 variables of both D1 and D2
T = [D1(:,2:38); D2(:,2:38)];
DRaw = [DRaw, T];
NoRecD1 = height(D1);
NoRecD2 = height(D2);

%

% assign the remaining variables one by one
var = 'TT_2';
DRaw{:,var}  = [D1{:, var}; D2{:,var}];

var = 'SA_2';
DRaw{:,var}  = [D1{:, var}; D2{:,var}];

% The following variables only exist in D1
var = 'SD_2';
DRaw{:,var}  = [D1{:, var}; NaN(NoRecD2,1) ];

var = 'SV_2';
DRaw{:,var}  = [D1{:, var}; NaN(NoRecD2,1) ];

var = 'FI';
DRaw{:,var}  = [D1{:, var}; NaN(NoRecD2,1) ];

var = 'OT';
DRaw{:,var}  = [D1{:, var}; NaN(NoRecD2,1) ];

var = 'OX';
DRaw{:,var}  = [D1{:, var}; NaN(NoRecD2,1) ];

var = 'OS';
DRaw{:,var}  = [D1{:, var}; NaN(NoRecD2,1) ];

var = 'FL';
DRaw{:,var}  = [D1{:, var}; NaN(NoRecD2,1) ];

% The following variables only exist in D2
var = 'PH';
DRaw{:,var}  = [NaN(NoRecD1,1); D2{:, var};  ];


%
% Save the file and do housekeeping

Filename = [Cruise, 'UW_RawFiles.mat'];
save(Filename, 'DRaw', 'D1', 'D2');
clear D1 D2 NoRecD1 NoRecD2 FileList NoFiles N T L var  ans junk

%
%% Generate one data structure for all data with standardized variable names
% CAUTION: Read the UW data documentation.  Make sure that the assumptions about the variable names and identities is correct!

NoRecords = height(DRaw);
clear D

D.Cruise = repmat({Cruise}, NoRecords, 1);
D = struct2table(D);

D.Date = DRaw.Date;
D.Time = DRaw.Time;
D.DateTime = DRaw.Date + DRaw.Time;

D.GPS_DateTime_UTC_Sec = DRaw.ZD;
D.Lat = DRaw.LA;
D.Lon = DRaw.LO;

D.COG = DRaw.CR;
D.SOG = DRaw.SP;
D.BottomDepth = DRaw.BT;
D.BottomDepth_MB = DRaw.MB;

D.USWFlow = DRaw.FI;

D.PARSurf = DRaw.PA;
D.LongWaveRad = DRaw.LW;
D.ShortWaveRad = DRaw.SW;

D.WindSpeed = DRaw.TW;
D.WindDir = DRaw.TI;
D.AirTemp = DRaw.AT;
D.AtmPress = DRaw.BP;
D.AtmPress_SLC = DRaw.BS;
D.RelHum = DRaw.RH;

D.TSG_Temp = DRaw.TT;
D.TSG_Cond = DRaw.TC;
D.TSG_Sal = DRaw.SA;
D.TSG_Dens = DRaw.SD;
D.SoundVel = DRaw.SV;

D.TSG_Temp_2 = DRaw.TT_2;
D.TSG_Sal_2 = DRaw.SA_2;
D.TSG_Dens_2 = DRaw.SD_2;
D.SoundVel_2 = DRaw.SV_2;

D.SSTemp = DRaw.ST;
D.SW_pH = DRaw.PH;
D.Oxygen = DRaw.OX;
D.OxygenSat = DRaw.OS;
D.OxygenTemp = DRaw.OT;

D.ChlFluor = DRaw.FL;

%
% Save preliminary data,  do housekeeping and generate a table for the filtered data, DF

clear DF
NoRecords = height(D);
DF.ID = (1:NoRecords)';
DF = struct2table(DF);
DF = [DF, D]; % DF is used below and is used to hold the QC'ed data at original temporal resolution
Filename = [Cruise 'UW_RawData.mat'];
save(Filename, 'D');
clear DRaw D1 D2 D3 D4 D5 D6 

%
%% Quality Control
% 1. Look at the progression of DataTime

figure
scatter(DF.ID, datenum(DF.DateTime), '.k')

% Looks Good
%
% 2. Look at the cruise track to determine if the Lat Lon have errors

figure
scatter(DF.Lon, DF.Lat, '.b');

plot(DF.Lon, DF.Lat, '-b')

% Cruise track looks good .
%
% 3. Check the COG, SOB, AirTemp, WetBulbAirTemp and RelHum.  Disable ylim's the first time the plot is generated

figure
subplot(4,1,1)
plot(DF.DateTimeUTC, DF.COG, '.k');
title('COG');
subplot(4,1,2)
plot(DF.DateTimeUTC, DF.SOG, '.k');
title('SOG');
subplot(4,1,3)
plot(DF.DateTimeUTC, DF.AirTemp, '.k');
title('AirTemp');
subplot(4,1,4)
plot(DF.DateTimeUTC, DF.RelHum, '.k');
%ylim ([0 120])
title('RelHum');

% Looks like there are bad data in AirTemp and RelHum.  (What is shown above may well be the plot of the corrected data - see below)
%
% Based on the first version of the plots above AirTemp and RelHum need to be filtered using cutoffs

DF.AirTemp(DF.AirTemp > 30 | DF.AirTemp < 0) = NaN;
DF.RelHum(DF.RelHum > 120 | DF.RelHum < 10) = NaN;

%
% 4. Check the WindSpeed, WindDir, and AtmPres. Disable ylim's the first time the plot is generated

figure
subplot(3,1,1)
plot(DF.DateTimeUTC, DF.WindSpeed, '.k');
title('WindSpeed');
subplot(3,1,2)
plot(DF.DateTimeUTC, DF.WindDir, '.k');
title('WindDir');
subplot(3,1,3)
plot(DF.DateTimeUTC, DF.AtmPress, '.k');
title('AtmPres');

% Looks like there are bad data in AtmPress.
%
%

DF.AtmPress(DF.AtmPress > 1100 | DF.AtmPress < 900) = NaN;

%
% 5. Check SST,  SSSal and ChlFluor

figure
subplot(4,1,1)
plot(DF.DateTimeUTC, DF.TSG_Temp, '.k');
title('SSTemp');
subplot(4,1,2)
plot(DF.DateTimeUTC, DF.TSG_Sal, '.k');
title('SSSal');
subplot(4,1,3)
plot(DF.DateTimeUTC, DF.ChlFluor, '.k');
title('ChlFluor');
subplot(4,1,4)
plot(DF.DateTimeUTC, DF.Oxygen, '.k');
title('Oxygen');


%

DF.TSG_Temp(DF.TSG_Temp > 30 | DF.TSG_Temp < 5) = NaN;
DF.SSTemp(DF.SSTemp > 30 | DF.SSTemp < 5) = NaN;
DF.TSG_Sal(DF.TSG_Sal > 40 | DF.TSG_Sal < 25) = NaN;

%
%
% Cut out sections of the data that are affected by stopped flow (does not look like there are any sections for CC1601
% Temp is a good variable to check for stopped flow.  Plot Temp vs time to identify the sections of the data that need to be exised from the TSG datastream.  Place a DataPick marker at the beginning and end of a section to be exised and export the datapick data as EX1.  Delete the existing DataPick markers, select the next section to be removed, place DataPick marker at the beginning and end of a section to be exised and export the datapick data as EX2.  Delete the existing DataPick markers, ...
%
% USWFlow only exists for the early part of the cruise.  Values suggest that there is an early period of stopped flow.  Use TSG_Temp.  Check cruise records if there are notes

figure
 plot(DF.ID, DF.SSTemp, '-b') 
 hold on
 plot(DF.ID, DF.TSG_Temp, '-r')
 

%
%
%
% 7. Remove the sections idientified by EX1, EX2, ...

 % the following is not very efficient.  This needs to be rewritten with loops
I1 = sort([EX1(1).Position(1), EX1(2).Position(1)]); 
I2 = sort([EX2(1).Position(1), EX2(2).Position(1)]);
I3 = sort([EX3(1).Position(1), EX3(2).Position(1)]);
I4 = sort([EX4(1).Position(1), EX4(2).Position(1)]);
I5 = sort([EX5(1).Position(1), EX5(2).Position(1)]);
I6 = sort([EX6(1).Position(1), EX6(2).Position(1)]);

I = [I1(1):I1(2), I2(1):I2(2), I3(1):I3(2), I4(1):I4(2), I5(1):I5(2), I6(1):I6(2)]; 

%Set all USWFlow dependent variables tagged by the EXi to NaN
DF(I, 23:37) = num2cell(NaN(length(I), 15));

%


Filename = [Cruise, 'UW_Exised.mat'];
save(Filename, 'EX1', 'EX2', 'EX3', 'EX4', 'EX5', 'EX6');

clear I1 I2 I3 I4 I5 I6 EX1 EX2 EX3 EX4 EX5 EX6 

%
% 8. Filter the Temp and SSSal Data.  Filtering takes a lot of time so only process variables that need it.

DF.TSG_Temp(DF.TSG_Temp > 25 | DF.TSG_Temp <  8) = NaN;
DF.SSTemp(DF.SSTemp > 25 | DF.SSTemp <  8) = NaN;
DF.TSG_Sal(DF.TSG_Sal < 30) = NaN;
DF.ChlFluor(DF.ChlFluor > 20 | DF.ChlFluor == 0) = NaN;
max(DF.ChlFluor)

tic
DF.TSG_Temp = UWData_Filter (DF.TSG_Temp, 100, 2.5);
DF.TSG_Sal = UWData_Filter (DF.TSG_Sal, 100, 2.5);
DF.ChlFluor = UWData_Filter (DF.ChlFluor, 100, 2.5);
toc

% After filtering the data replot the data in section 5.
%
% Save the filtered data as CC1601_FinalData.mat

Filename = [Cruise, 'UW_FilteredData.mat'];
save (Filename, 'DF');

%
%
%% Generate SSTemp, SSSal and Chl a data using CalCOFI bottle data for calibration
% The CalCOFI bottle data are generated in the CalCOFI database (querry 'CCE Cast CTD_Chl_Nuts_10m') and are stored as:  'D:\Documents\MATLAB\CalCOFI\Underway\CalCOFI_BottleData.xlsx' and CalCOFI_BottleData.mat.  If the mat file does not exist load the *xlsx file manually using ImportData and name 'D_CalCOFI'

CruiseNo = 201611;  % make sure that this is the cruise number used in D_CalCOFI

load('D:\Documents\MATLAB\CalCOFI\Underway\CalCOFI_BottleData.mat');

%

% if DF does not exist in the workspace
load('CC1604UW_FilteredData.mat');

%
% Assemble the table that will be used for the calibrations

   
% Generate a list of stations for that cruise
I_Cruise = D_CalCOFI.Cruise == CruiseNo;
StationList = unique(D_CalCOFI.Sta_ID(I_Cruise));
NoStations = length(StationList);
clear D_UWCal
D_UWCal.Cruise = repmat({Cruise}, NoStations, 1);


% Assemble a calibration data table D_UWCal
D_UWCal = struct2table(D_UWCal);
D_UWCal.Station = StationList;

N = 1;

for N = 1 : NoStations
    % select the indices for the top 12m samples of each station
    I_Sta = strcmp(D_CalCOFI.Sta_ID, D_UWCal.Station(N)) & D_CalCOFI.Cruise == CruiseNo;
    
    % pull out the data from DCalCOFI and assign to D_UWCal
    D_UWCal.CC_DateTimeUTC(N) = mean(D_CalCOFI.DateTime(I_Sta));
    D_UWCal.CC_DateTimePST(N) = mean(D_CalCOFI.DateTime(I_Sta)- 8/24);
    D_UWCal.CC_TimeOfDayPST(N) = timeofday(D_UWCal.CC_DateTimePST(N));
    D_UWCal.DayLight_I(N) = daylight (D_UWCal.CC_DateTimePST(N), D_CalCOFI.Lat_Dec(N));
    
    D_UWCal.CC_Lat(N) = nanmean(D_CalCOFI.Lat_Dec(I_Sta));
    D_UWCal.CC_Lon(N) = nanmean(D_CalCOFI.Lon_Dec(I_Sta));    
    D_UWCal.Distance(N) = nanmean(-D_CalCOFI.Distance(I_Sta));

    D_UWCal.Cast(N) = nanmean(D_CalCOFI.Order_Occ(I_Sta));
    D_UWCal.NoCastBottle(N) = max(D_CalCOFI.BtlNum(I_Sta));
    
    D_UWCal.CC_Temp(N) = nanmean(D_CalCOFI.T_degC(I_Sta));
    D_UWCal.CC_Sal(N) = nanmean(D_CalCOFI.Salnty(I_Sta));
    D_UWCal.CC_Chl(N) = nanmean(D_CalCOFI.ChlorA(I_Sta));
    D_UWCal.CC_NO3(N) = nanmean(D_CalCOFI.NO3ug(I_Sta));
    D_UWCal.CC_SiO3(N) = nanmean(D_CalCOFI.SiO3ug(I_Sta));
    D_UWCal.CC_PO4(N) = nanmean(D_CalCOFI.PO4ug(I_Sta));
end

%
% Pull out the SSTemp, SSSal and ChlFluor data from the UW Data:  The best time for a possible match between the CalCOFI bottle samples is estimated based on the time of the CTD cast at depth AND the number of bottles used during the cast since this affects the time between Time-at-Depth and surface bottle closing.  So, the code below is a bit dense but achieves this.  The match between the two times is independently checked by comparing the position associated with the CTD cast and the position recored with the UW samples.  A summary of that is plotted below.
%

    N = 1;

for N = 1 : NoStations
    
    TimeInt = (D_UWCal.NoCastBottle(N) * .02) / 24;
    BeginT  = D_UWCal.CC_DateTimeUTC(N) +TimeInt  ;
    EndT	 = D_UWCal.CC_DateTimeUTC(N) +1.5 * TimeInt;
    I_UW = DF.DateTimeUTC > BeginT & DF.DateTimeUTC < EndT;
    
    if sum(I_UW) > 0
        
        D_UWCal.UW_DateTimeUTC(N,1) = mean(DF.DateTimeUTC(I_UW));
        D_UWCal.UW_Lat(N,1) = mean(DF.Lat(I_UW));
        D_UWCal.UW_Lon(N,1) = mean(DF.Lon(I_UW));
        
        
        
        D_UWCal.DelTimeNum(N,1) = D_UWCal.UW_DateTimeUTC(N)  - D_UWCal.CC_DateTimeUTC(N) ;
        
        %
        D_UWCal.UW_TSGTemp(N,1) = nanmean(DF.TSG_Temp(I_UW));
        D_UWCal.UW_TSGSal(N,1) = nanmean(DF.TSG_Sal(I_UW));
        %D_UWCal.UW_Fluor(N,1) = nanmean(DF.ChlFluor(I_UW));
        
        D_UWCal.DelDistance(N,1) = sw_dist([D_UWCal.UW_Lat(N); D_UWCal.CC_Lat(N)], ...
            [D_UWCal.UW_Lon(N); D_UWCal.CC_Lon(N)], 'km');
        
    end
    
end

%
% Plot for each sample point, i.e. CalCOFI station, the distance between the CalCOFI cast position and the average position of the UW data chose for comparison.

figure
plot(D_UWCal.Cast, D_UWCal.DelDistance, '.r')

%

Filename = [Cruise, 'UW_CalData'];
save (Filename, 'D_UWCal');

%
%% Generate TSG_Temp, TSG_Sal and Chl a calibrations and calculate calibrated variables
%
% Temperature:
% Develop calibrations using:  cftool(D_UWCal.UW_Temp, D_UWCal.CC_Temp);  It is important that the data do not have any NaN or Inf. If they do a match no longer exists between Data(I) and the data in fitout_X.

I = ~isnan( D_UWCal.UW_TSGTemp) & ~isnan( D_UWCal.CC_Temp);
sum(I)

[xData, yData] = prepareCurveData( D_UWCal.UW_TSGTemp(I), D_UWCal.CC_Temp(I) );

ft = fittype( 'poly2' );
opts = fitoptions( 'Method', 'LinearLeastSquares' );
opts.Robust = 'Bisquare';

% Fit model to data.
[fitresult_T, gof_T, fitout_T] = fit( xData, yData, ft, opts );

% Create a figure for the plots.
figure( 'Name', 'untitled fit 1' );

% Plot fit with data.
subplot( 2, 1, 1 );
h = plot( fitresult_T, xData, yData );
legend( h, 'UW vs CalCOFI Temp', 'poly2 fit', 'Location', 'NorthWest' );
% Label axes
xlabel 'UW Temp'
ylabel 'CC Temp'
grid on

% Plot residuals.
subplot( 2, 1, 2 );
h = plot( fitresult_T, xData, yData, 'residuals' );
legend( h, 'Resisuals vs CalCOFI Temp', 'Zero Line', 'Location', 'SouthWest' );
% Label axes
xlabel 'CC Temp'
ylabel Residuals
grid on

% The large number of negative residuals at high CC Temp suggests problems with heating in the UW system


fitresult_T
gof_T

% Note that the non-linarity is not signifcant
%
%% Salinity Calibration
%

I = ~isnan( D_UWCal.UW_TSGSal) & ~isnan(D_UWCal.CC_Sal);
sum(I)

[xData, yData] = prepareCurveData( D_UWCal.UW_TSGSal(I), D_UWCal.CC_Sal(I) );

% Set up fittype and options.
ft = fittype( 'poly1' );
opts = fitoptions( 'Method', 'LinearLeastSquares' );
opts.Robust = 'Bisquare';

% Fit model to data.
[fitresult_S, gof_S, fitout_S] = fit( xData, yData, ft, opts );

% Create a figure for the plots.
figure( 'Name', 'Salinity Calibration' );

% Plot fit with data.
subplot( 2, 1, 1 );
h = plot( fitresult_S, xData, yData );
legend( h, 'CalCOFI vs. xData', 'Linar Fit', 'Location', 'NorthWest' );
% Label axes
xlabel xData
ylabel yData
grid on

% Plot residuals.
subplot( 2, 1, 2 );
h = plot( fitresult_S, xData, yData, 'residuals' );
legend( h, 'untitled fit 2 - residuals', 'Zero Line', 'Location', 'NorthWest' );
% Label axes
xlabel xData
ylabel yData
grid on

fitresult_S
gof_S

% OK correspondence
%
%
%% Chl a - Fluor Calibration
% no data for 1611.

I = ~isnan( D_UWCal.UW_Fluor) & ~isnan(D_UWCal.CC_Chl);
sum(I)

[xData, yData] = prepareCurveData( D_UWCal.UW_Fluor(I), D_UWCal.CC_Chl(I) );

% Set up fittype and options.
ft = fittype( 'poly2' );
opts = fitoptions( 'Method', 'LinearLeastSquares' );
%opts.Robust = 'Bisquare';

% Fit model to data.
[fitresult_C, gof_C, fitout_C] = fit( xData, yData, ft, opts );

% Create a figure for the plots.
figure

% Plot fit with data.
subplot( 2, 1, 1 );
h = plot( fitresult_C, xData, yData );
legend( h, 'UW Fluor vs. CC Chl a', 'Linar Fit', 'Location', 'NorthWest' );
% Label axes
xlabel 'UW Fluor'
ylabel 'CC Chl a'
grid on

% Plot residuals.
subplot( 2, 1, 2 );
h = plot( fitresult_C, xData, yData, 'Residuals vs. CalCOFI Chl a' );
legend( h, 'Residuals', 'Zero Line', 'Location', 'NorthEast' );
% Label axes
xlabel 'CalCOFI Chl a'
ylabel  Residuals
grid on

fitresult_C
gof_C

% No results for CC1611. Plots below were produced for other cruises

figure
plot(D_UWCal.Cast(I), fitout_C.residuals, 'ok')

%

figure
plot(D_UWCal.CC_TimeOfDayPST(I), fitout_C.residuals, 'ok')

%



%
%% Generate calibrated values of Temp, Sal and Chl
% Check the models and predictions by predicting CalCOFI data from UW_Fluor data and comparing those to CalCOFI data.  The comparison is made to check that calculations are correct.

D_UWCal.Pred_Temp = feval(fitresult_T,  D_UWCal.UW_TSGTemp);
figure
scatter(D_UWCal.CC_Temp, D_UWCal.Pred_Temp)
xlabel 'CalCOFI Temp'
ylabel 'Predicted Temp'

D_UWCal.Pred_Sal = feval(fitresult_S,  D_UWCal.UW_TSGSal);
scatter(D_UWCal.CC_Sal, D_UWCal.Pred_Sal)
xlabel 'CalCOFI Sal'
ylabel 'Predicted Sal'

% D_UWCal.Pred_Chl = feval(fitresult_C,  D_UWCal.UW_Fluor);
% figure
% scatter(D_UWCal.CC_Chl, D_UWCal.Pred_Chl)
% 
% xlabel 'CalCOFI Chl'
% ylabel 'Predicted Chl'

%
%
% Calculate corrected Temp, Sal and Chl a from data in DF and for DF and save that part of the DF data structure corresponding to FileNo 1 and 3.

DF.Pred_TSGTemp = feval(fitresult_T,  DF.TSG_Temp);
DF.Pred_TSGSal = feval(fitresult_S,  DF.TSG_Sal);
% for CC1611 Pred_Chl was not calculated 
% DF.Pred_Chl = feval(fitresult_C,  DF.ChlFluor); 

Filename = [Cruise, 'UW_FinalData.mat'];
save(Filename, 'DF');

clear xData yData opts fitresult_T fitresult_S fitresult_C gof_T gof_S gof_C fitout_T fitout_S fitout_C   

%
%% Generate the final data structures: Original resolution and 10 min resolution
% Generate an UW time series based on 10 min averages.  You may want to close Matlab to clean out memory, reopen Matlab and just load CC1601UW_FinalData.mat for calculations below. Below specify the  length of the time intervals as a fraction of the day.  10 min is chosen for the CalCOFI UW data.

DelTime = 10/(24*60);

% A time based is generated below and averages of DF are taken

clear D10Min

TMin = min(DF.DateTimeUTC);
TMin = dateshift(TMin, 'start', 'hour');

TMax = max(DF.DateTimeUTC);
TMax = dateshift(TMax, 'end', 'hour');

TimeBase = ([TMin : DelTime : TMax])';
NoTimes = length(TimeBase);

% Build the table for the 10 min data
D10Min.ID = (1:NoTimes)';
D10Min.Cruise = repmat(cellstr(Cruise), NoTimes, 1);
D10Min = struct2table(D10Min);

D10Min.DateTimeUTC = TimeBase;

% fill in the missing columns of D10Min using the data section of DF
NoVariables = width(DF) -6;
TempTable = DF(1:NoTimes,7:end);
TempTable(:,:) = {NaN};
D10Min = [D10Min, TempTable];

for N = 1 : NoTimes
    
    % for each time point in D10
    Time_B = TimeBase(N) - DelTime/2;
    Time_E = TimeBase(N) + DelTime/2;
    
    % generate the data index over which means need to be calculated 
    I = DF.DateTimeUTC > Time_B  & DF.DateTimeUTC < Time_E;
    
    % assign those  data to X
    X = DF{I,7:end};
    % calculate the mean for these data, i.e. Y
    Y = trimmean(X, 10, 1);
    % enter the values of Y into D10Min
    D10Min(N, 4:end) = num2cell(Y);
    
end

clear TMin TMax TimeBase NoTimes TempTable NoVariables N Time_B Time_E I X Y

%
%
%% Plot the D10Min data as a final check
% Check the cruise track

figure
scatter(D10Min.Lon, D10Min.Lat, '.k');

% Check the COG, SOB, AirTemp and RelHum


figure
subplot(4,1,1)
plot(D10Min.DateTimeUTC, D10Min.COG, '.k');
title('COG');
subplot(4,1,2)
plot(D10Min.DateTimeUTC, D10Min.SOG, '.k');
title('SOG');
subplot(4,1,3)
plot(D10Min.DateTimeUTC, D10Min.AirTemp, '.k');
title('AirTemp');
subplot(4,1,4)
plot(D10Min.DateTimeUTC, D10Min.RelHum, '.k');
ylim ([0 120])
title('RelHum');

% Check the WindSpeed, WindDir, AtmPres and ChlFluor

figure
subplot(4,1,1)
plot(D10Min.DateTimeUTC, D10Min.WindSpeed, '.k');
title('WindSpeed');
subplot(4,1,2)
plot(D10Min.DateTimeUTC, D10Min.WindDir, '.k');
title('WindDir');
subplot(4,1,3)
plot(D10Min.DateTimeUTC, D10Min.AtmPress, '.k');
title('AtmPress');
subplot(4,1,4)
plot(D10Min.DateTimeUTC, D10Min.ChlFluor, '.k');
ylim ([0 10])
title('ChlFluor');

% Check PredTemp and PredSal

figure
subplot(2,1,1)
plot(D10Min.DateTimeUTC, D10Min.Pred_TSGTemp, '.k');
title('Pred_TSGTemp');
subplot(2,1,2)
plot(D10Min.DateTimeUTC, D10Min.Pred_TSGSal, '.k');
title('Pred_TSGSal');

%
% Save the 10-Min Data

Filename = [Cruise, 'UW_10MinData.mat'];
save(Filename, 'D10Min');