#! /usr/bin/python # USAGE: MosaicLogFileTxFromTowerToDaq.py # PURPOSE: Performs FTP data-acquisition file transfers from MOSAiC tower computer to the ship computer # AUTHOR: Dan Gottas (NOAA/PSD) import ftplib, time, datetime, os, sys, glob #### begin user defined variables #### LogFilePath = 'C:\\Noaa\\Logs\\MosaicLogFileTxFromTowerToDaq' FtpServer = '192.168.202.4' FtpPort = 21 FtpUserName = 'mosaic_tower' FtpPassword = 'mosaic_tower' # FTP transfer parameter dictionary # SYNTAX: { SessionName : [ FtpInputPath, OutputPath, BeginHourOffset, EndHourOffset, [ FileTxList ] ] } # DESCRIPTION: SessionName = name of the parameter set (no impact on the file transfers) # FtpPath = source path on the remote FTP server where files to be transferred exist # OutputPath = destination path on the local computer where files will be transferred to # BeginHourOffset = hour offset from the execution time that specifies the begin time of the files to transfer # EndHourOffset = hour offset from the execution time that specifies the end time of the files to transfer # FileTxList = list of files to transfer; dynamic time elements in file name are defined by strftime # FtpTxParmDict = { 'Data Acquisition Log' : [ '/Logs/DataAcquisition', 'd:/tower/Logs/DataAcquisition', 1, 1, ['%y%j.log']], 'SystemAdministration Log' : [ '/Logs/SystemAdministration', 'd:/tower/Logs/SystemAdministration', 1, 1, ['%y%j.log']], 'ClockSynchronization Log' : [ '/Logs/ClockSynchronization', 'd:/tower/Logs/ClockSynchronization', 1, 1, ['%y%j.log']] } ParseInputLogPath = 'd:/tower/Logs/DataAcquisition' ParseFilterHour = 24 # Log file parser dictionary # SYNTAX: { SessionName : [ ParserOutputFile, [ ParsingStrings ] ] } # DESCRIPTION: SessionName = name of the parameter set (no impact on the parsing) # ParserOutputFile = output file to output parsed log entries # ParsingStrings = substrings used to parse desired log entries # LogParserDict = { 'Lagged sample' : [ 'd:/tower/Logs/DataAcquisitionParse/TowerLags.log' , ['Lag']], 'Error' : [ 'd:/tower/Logs/DataAcquisitionParse/TowerErrors.log' , ['Fail', 'fail', 'Error', 'error', 'Invalid', 'invalid', 'exceed', 'unreachable', ' not', ' full']], } #### end user defined variables #### def WriteMessage(LogFilePath, Message): LogTime = datetime.datetime.utcnow() LogFile = LogFilePath + LogTime.strftime('\\MosaicLogFileTxFromTowerToDaq_%y%j.log') # output to stdout print(Message) # output to log file with open(LogFile, 'a') as fout: fout.write(Message + '\n') ### main #### InputPathIndex = 0 OutputPathIndex = 1 FileTxBeginHourOffsetIndex = 2 FileTxEndHourOffsetIndex = 3 FileListIndex = 4 WriteMessage(LogFilePath, '\nRunning MosaicLogFileTxFromTowerToDaq.py on ' + time.ctime() + '\n') #### FTP log file transfers #### # create ftp object try: f = ftplib.FTP() except: WriteMessage(LogFilePath, 'Error creating FTP object',) sys.exit() # connect to ftp server try: f.connect(FtpServer, FtpPort) except: WriteMessage(LogFilePath, 'Error connecting to FTP server ' + FtpServer + ' on port ' + str(FtpPort)) sys.exit() # ftp authentication try: f.login(FtpUserName, FtpPassword) except: WriteMessage(LogFilePath, 'Error logging into FTP server as ' + FtpUserName) sys.exit() WriteMessage(LogFilePath, 'FTP connection connection established with ' + FtpServer + ' on port ' + str(FtpPort) + ' as user ' + FtpUserName) # cycle through each ftp file transfer session for FtpTxSession in sorted(FtpTxParmDict): WriteMessage(LogFilePath, '\nLoading FTP transfer parameters for session \'' + FtpTxSession + '\':\n') # assign dictionary elements FtpBasePath = FtpTxParmDict[FtpTxSession][InputPathIndex] OutputBasePath = FtpTxParmDict[FtpTxSession][OutputPathIndex] FtpTxFileList = FtpTxParmDict[FtpTxSession][FileListIndex] FileTxBeginHourOffset = FtpTxParmDict[FtpTxSession][FileTxBeginHourOffsetIndex] FileTxEndHourOffset = FtpTxParmDict[FtpTxSession][FileTxEndHourOffsetIndex] # create time objects for the begin and end time of the file transfer CurrentTime = datetime.datetime.utcnow() BeginTxTime = CurrentTime - datetime.timedelta(hours=FileTxBeginHourOffset) EndTxTime = CurrentTime - datetime.timedelta(hours=FileTxEndHourOffset) FileTxTime = EndTxTime try: # change directory on the FTP server f.cwd(FtpBasePath) except: WriteMessage(LogFilePath, 'FTP directory ' + FtpBasePath + ' does not exist; skipping') continue # cycle through all file transfer hours while(FileTxTime >= BeginTxTime): # cycle through each file transfer expression for FtpTxFile in FtpTxFileList: File = FileTxTime.strftime(FtpTxFile) InputFile = FtpBasePath + '/' + File OutputFile = OutputBasePath + '/' + File try: # get remote file size try: InputFileSize = f.size(InputFile) except: InputFileSize = 0 # get local file size try: statinfo = os.stat(OutputFile) OutputFileSize = statinfo.st_size except: OutputFileSize = 0 # transfer the file if(InputFileSize > 0): if(InputFileSize != OutputFileSize): # create output path if it doesn't exist if(os.path.isdir(os.path.dirname(OutputFile)) == False): os.makedirs(os.path.dirname(OutputFile)) WriteMessage(LogFilePath, 'Transferring ' + InputFile + ' to ' + OutputFile) # transfer file in binary mode f.retrbinary('RETR ' + InputFile, open(OutputFile,'wb').write) else: WriteMessage(LogFilePath, 'Skipping ' + InputFile + '; local and remote files are the same size') else: WriteMessage(LogFilePath, 'Input file ' + InputFile + ' does not exist; skipping') except: WriteMessage(LogFilePath, 'Error transferring ' + InputFile) # decrease the time by one hour FileTxTime -= datetime.timedelta(hours=1) #### log file message parsing #### ParseOutputFileIndex = 0 ParseSubstringsIndex = 1 LastLogFile = "" LogEntryList = [] LogEndTime = datetime.datetime.utcnow() LogBeginTime = LogEndTime - datetime.timedelta(hours=ParseFilterHour) LogTime = LogEndTime # cycle through all hours within the filter period while(LogTime >= LogBeginTime): LogFile = ParseInputLogPath + LogTime.strftime('/%y%j.log') # prevent redundant file reads if(LogFile != LastLogFile): if(os.path.isfile(LogFile)): print('Reading log records in ' + LogFile) with open(LogFile, 'r') as fout: LineList = fout.readlines() # insert date in front of time stamp for Index,Line in enumerate(LineList): LineList[Index] = LogTime.strftime('%y%j ') + Line # most recent records first LineList.reverse() LogEntryList += LineList LastLogFile = LogFile # decrease the time by one hour LogTime -= datetime.timedelta(hours=1) # write headers to new output files for LogParser in sorted(LogParserDict): WriteMessage(LogFilePath, '\nParsing log file entries for session \'' + LogParser + '\'') ParseOutputFile = LogParserDict[LogParser][ParseOutputFileIndex] with open(ParseOutputFile, 'w') as fout: fout.write('Tower data acquisition log filter report generated on ' + LogEndTime.strftime('%m/%d/%Y (%y%j) %H:%M:%S UTC') + '\n' + LogParser + ' entries over the past ' + str(ParseFilterHour) + ' hours:\n\n') fout.write('YYJJJ HH:MM:SS:mmm Code Message\n') # cycle through all log entries for LogEntry in LogEntryList: # search for all parsing definitions for LogParser in sorted(LogParserDict): ParseOutputFile = LogParserDict[LogParser][ParseOutputFileIndex] ParseSubstringList = LogParserDict[LogParser][ParseSubstringsIndex] if(LogEntry[15:18].isdigit()): LogEntryTime = datetime.datetime.strptime(LogEntry[0:18],'%y%j %H:%M:%S:%f') if(LogEntryTime >= LogBeginTime): if(any(ParseSubstring in LogEntry for ParseSubstring in ParseSubstringList)): with open(ParseOutputFile, 'a') as fout: fout.write(LogEntry) WriteMessage(LogFilePath, '\nMosaicLogFileTxFromTowerToDaq.py finished on ' + time.ctime())