function sio2univ2(directory)
%using this function to change the SIO Raw data files to those of the
%universal format.  this works on a rather large amount of data, so it
%might actually crash.
%
%     Written by Robert Thombley
%     On and around 21 Dec 2005
%
%     Requires the use of cell2csv.m

%mapping of sio heading label to column position in universal output
sio_mapping = struct('LA',4, ...
    'LO',5, ...
    'FI',6, ...
    'TT',7, ...
    'SA',8, ...
    'TC',9, ...
    'PA',10, ...
    'Press',11, ...
    'FL',12, ...
    'ST',13, ...
    'ST2',14, ...
    'SD',16, ...
    'AT',18, ...
    'BP',19, ...
    'RH',20, ...
    'RT',21, ...
    'WD',22, ...
    'WD2',23, ...
    'WS',24, ...
    'WS2',25, ...
    'TI',26, ...
    'TI2',27, ...
    'TW',28, ...
    'TW2',29, ...
    'BT',30, ...
    'GA',32, ...
    'CR',33, ...
    'CR2',34, ...
    'ZD',35, ...
    'ZD2',36, ...
    'GS',37, ...
    'SP',38, ...
    'GT',39, ...
    'GT2',40, ...
    'GY',41, ...
    'LA2',42, ...
    'LO2',43, ...
    'SL',44, ...
    'SY',46, ...
    'TSG68',47, ...
    'TB',48, ...
    'IA',49, ...
    'IP',50, ...
    'IS',51, ...
    'IT',52);

%this dictates how the output looks
formatstring = ['%06.0f,%06.0f,%1.0f,%8.6f,%9.6f,%2.1f,%4.3f,%4.3f,', ...
'%4.3f,%5.2f,%1.1f,%4.3f,%4.3f,%4.3f,%4.3f,%4.3f,%6.3f,%3.2f,', ...
'%5.2f,%3.2f,%3.2f,%2.1f,%2.1f,%2.1f,%2.1f,%2.1f,%2.1f,%2.1f,', ...
'%2.1f,%2.1f,%4.3f,%2.1f,%2.1f,%2.1f,%1.0f,%1.0f,%1.0f,%1.1f,', ...
'%06.0f,%06.0f,%2.1f,%8.6f,%9.6f,%3.2f,%2.1f,%1.0f,%4.3f,%4.3f', ...
',%2.1f,%2.1f,%4.3f,%4.3f'];


%Start by moving to the right directory
cd (fullfile(directory,'Raw'));
%now we iterate over the COR files in the directory

corfiles = ls('*.COR');
file_cnt = 0;
for filenum = 1:size(corfiles,1)
    file_cnt = file_cnt + 1;
    cordate = corfiles(filenum,1:6);

    doutfile = fullfile(directory,'Universal',cordate,'univ_data.csv');
    foutfile = fullfile(directory,'Universal',cordate,'univ_flags.csv');
    fid = fopen(doutfile,'wt');
    
    
    %set the file name
    file_in = corfiles(filenum,:);
    %create a GIANT vector (~375,000 values) of all data in the day's file,
    %such that an element's index in the vector is row# * column#.
    data = textread(file_in,'%s','delimiter',',');

    %find the length of the data vector
    dlen = size(data,1);

    %find the indices for the starts of each line (ie- column == 1)
    ind1 = find(strcmp(data,'$WICOR'));
    %find the number of rows (size of ind1)
    rownum = size(ind1,1);
    %now find the indices for the ends of each line (ie- column == last
    %col)
    indLast = [ind1(2:end)-1;size(data,1)] ; %skips the first line, because there is no 0 row

    %these are the checksums
    checksums = char(data(indLast));
    cslabels = checksums(:,1:3);
    checksums = checksums(:,end-1:end);

    %pre-allocate the arrays
    colnum = uint16(dlen/rownum);
    flags = 9*ones(rownum,colnum + 2);
    indata = NaN*ones(rownum,colnum + 2);
    outdata = NaN*ones(rownum,52);
    outflags = 9*ones(rownum,52);

    %put the labels back without the checksums
    data(indLast) = cellstr(cslabels);

    %find the end points of the lines
    dends = [ind1(:) indLast(:)];

    %grab first date
    if (filenum == 1)
        initDate = ['20' file_in(1:6)];
        initTime = data{3};
        initETime = datenum([initDate initTime],'yyyymmddHHMMSS');
    end
    %put time in mmddyyHHMMSS format in a char array
    eTime = char(data(ind1+1));
    eTime = strcat('20',[eTime(:,end-1:end) eTime(:,1:end-2) char(data(ind1+2))]);
    outdata(:,3) = floor((datenum(eTime,'yyyymmddHHMMSS') - initETime)*86400);

    %create date and time fields
    outdata(:,1) = str2num(['20' file_in(1:6)]);%*ones(rownum,1);
    outdata(:,2) = str2num(char(data(ind1+2)));
    
    %

    for i=1:rownum
        line = data(dends(i,1):dends(i,2));
        %start checksumming
        csum = 0;
        vals = double(char(line));
        for j=1:length(line)
            csum = bitxor(csum,vals(j));
            csum = uint16(csum);
        end
        if bitxor(hex2dec(checksums(i,:)),csum)
            flags(j,:) = 4;
        else
            flags(j,:) = 0;
        end
        %end checksumming

        %start data collection
        head = line(5:2:end);
        chead = char(head);
        qcVal = [1 1 1 str2num(chead(:,3)')];
        qcVal(qcVal == 0) = 9;
        qcInd = (qcVal >= 7);
        %assign qcValues to flags
        flags(j,qcInd) = qcVal(qcInd)';

        head = cellstr(chead(:,1:2));
        %the only secondary data we care about is
        %ST2,WD2,WS2,TI2,TW2,CR2,ZD2,GT2,LA2,LO2
        okSec = {'ST','WD','WS','TI','TW','CR','ZD','GT','LA','LO'};
        for m=1:10
            if (sum(strcmp(head,okSec{m})) == 2)
                head(max(find(strcmp(head,okSec{m})))) = {[okSec{m} '2']};
            end
        end



        %start data transform
        ldata = line([2 3 4:2:end]);

        %artificially sets pressure to 0.  It will be zero for every
        %cruise ever - since it is surface data.
        outdata(i,sio_mapping.('Press')) = 0;
        %currently this is set to emulated.  This may or may not need
        %to change
        outflags(i,sio_mapping.('Press')) = 8;

        for j=3:colnum
            if (isfield(sio_mapping,head(j)))
                switch found{j}
                    case 'GS'
                        if (ldata{j} == '-99')
                            len = 3;
                        else
                            len = 1;
                        end
                        outdata(i,sio_mapping.('GS')) = ldata{j}(1:len);
                        flags(i,sio_mapping.('GS')) = flags(i,j);
                    case 'PA'
                        if (ldata(j) < 0)
                            outdata(i,sio_mapping.('PA')) = 0;
                            outflags(i,sio_mapping.('PA')) = flags(i,j);
                        else
                            outdata(i,sio_mapping.('PA')) = ldata{j};
                            outflags(i,sio_mapping.('PA')) = flags(i,j);
                        end
                    case 'TT'
                        if (ldata(j) == -99)
                            outdata(i,sio_mapping.('TSG68')) = NaN;
                            outflags(i,sio_mapping.('TSG68')) = 9;
                        else
                            outdata(i,sio_mapping.('TSG68')) = ldata{j}*1.00024;
                            outflags(i,sio_mapping.('TSG68')) = flags(i,j);
                        end
                        outdata(i,sio_mapping.('TT')) = ldata{j};
                        outflags(i,sio_mapping.('TT')) = flags(i,j);
                    otherwise
                        outdata(i,sio_mapping.(found{j})) = ldata{j};
                        outflags(i,sio_mapping.(found{j})) = flags(i,j);
                end
            end
        end
        fprintf(fid,formatstring,outdata);
        fprintf(fid,'\n');
    end
    csvwrite(foutfile,outflags);
    disp(['Converted file: ' corfiles(filenum,:)]);
    clear outdata outflags data eTime;
end