"""parseDatalogUserFiles.py Parses data from Picarro Datalog_User raw files into hourly files. Datalog_User files have 22 tab-delimited columns: 1 DATE 2 TIME 3 FRAC_DAYS_SINCE_JAN1 4 FRAC_HRS_SINCE_JAN1 5 JULIAN_DAYS 6 EPOCH_TIME 7 ALARM_STATUS 8 INST_STATUS 9 CavityPressure 10 CavityTemp 11 DasTemp 12 EtalonTemp 13 WarmBoxTemp 14 species 15 MPVPosition 16 OutletValve 17 solenoid_valves 18 CH4 19 CH4_dry 20 CO2 21 CO2_dry 22 H2O raw file names are: CFHADS2043-YYYYMMDD-hhmmssZ-DataLog_User.dat where the time indicated is approximately the start time, usually a few seconds after the first data point in the file. the script assumes current directory contains the raw files, and output files are written to the same directory output file names follow PSD style: ghg0YYDDDHH.txt Byron Blomquist, NOAA/ESRL/PSD, Nov 2019 """ import os import string # assume current directory contains raw DataLog_User files, # dump list of data file names into a list variable files = [f for f in os.listdir('.') if f.startswith('CFHADS')] files.sort() # parse data files and save into txt files HH_prior = ''; for file in files: print(file) fin = open(file) line1 = fin.readline() # read header line for line in fin: # read data lines sequentially cleanLine = line.strip() fields = cleanLine.split() ddd = fields[4] idx = ddd.find('.') DDD = ddd[0:idx] # day of year string YY = fields[0][2:4] # 2 digit year string HH = fields[1][0:2] # 2 digit hour string if HH != HH_prior: # new outp if hour changes or first file HH_prior = fields[1][0:2] # remember current hour of day # open output file, new name begins with ghg0 outp = 'ghg0'+YY+DDD+HH+'_raw.txt' fghg = open(outp, 'w') # first, write header line to outp cleanLine = line1.strip() vars = cleanLine.split() header = '' for item in vars: header += item + ',' headerLine = header[0:-1] + '\n' fghg.write(headerLine) # then write data line to outp lineOut = '' for item in fields: # concatenate numeric fields to output lineOut += item + ',' printLine = lineOut[0:-1] + '\n' # add return character fghg.write(printLine) # write comma delimited line to output file else: # just write data line to outp lineOut = '' for item in fields: lineOut += item + ',' printLine = lineOut[0:-1] + '\n' fghg.write(printLine) fghg.close() # end