madrigal.ui.web module
web is the module that interfaces to cgi madrigal web pages.
The web module contains general functions to produce html, along with producing html relating to specific user data or madrigal data.
"""web is the module that interfaces to cgi madrigal web pages. The web module contains general functions to produce html, along with producing html relating to specific user data or madrigal data. """ # $Id: web.py 6565 2018-08-23 14:37:22Z brideout $ import time, os import os.path import calendar import fnmatch import datetime import types, traceback import Cookie import urllib import glob import calendar import urlparse import math import random import tempfile import subprocess import tarfile import shutil import re # third party imports import numpy import django.core.urlresolvers import madrigal.metadata import madrigal.derivation import madrigal.data import madrigal.isprint import madrigal.ui.userData import madrigal.admin class MadrigalWeb: """MadrigalWeb is the class that produces output for the web. All text written to the web is produced in this class. Non-standard Python modules used: None Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Dec. 17, 2001 """ _MaxSleep = 10 def __init__(self, madDB = None): """__init__ initializes MadrigalWeb by reading from MadridalDB.. Inputs: Existing MadrigalDB object, by default = None. Returns: void Affects: Initializes self._metaDir, self._logFile. Exceptions: None. """ if madDB == None: self._madDB = madrigal.metadata.MadrigalDB() else: self._madDB = madDB self._binDir = self._madDB.getBinDir() self._instObj = madrigal.metadata.MadrigalInstrument(self._madDB) self._madKindatObj = madrigal.metadata.MadrigalKindat(self._madDB) metaDir = self._madDB.getMetadataDir() # get todays year now = datetime.datetime.now() thisYear = '%04i' % (now.year) self._logFile = os.path.join(metaDir, 'userdata', 'access_%s.log' % (thisYear)) # be sure it exists if not os.access(self._logFile, os.R_OK): f=open(self._logFile, 'w') f.close() # keep track of whether user trusted self._isTrusted_ = None # cache Madrigal objects as needed to imprive performance self._madExpObjExpID = None # will be set to a MadrigalExperiment object sorted by expId when first needed self._madExpObjDate = None # will be set to a MadrigalExperiment object sorted by date when first needed def getRulesOfTheRoad(self, PI=None, PIEmail=None): """ getRulesOfTheRoad returns a string giving the rules in html formal for using madrigal data. Inputs: PI - contact name. Default is site name. PIEmail - email link. Default is site admin. Returns: a string giving the rules in html formal for using madrigal data Affects: None. Exceptions: None. """ if not PI or not PIEmail: # get the site name siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteID = self._madDB.getSiteID() contactName = str(siteObj.getSiteName(siteID)) contactEmail = str(siteObj.getSiteEmail(siteID)) else: contactName = str(PI) contactEmail = str(PIEmail) returnStr = 'Please contact %s at ' % (contactName) returnStr = returnStr + '<a href="mailto:' + contactEmail + '">' + \ contactEmail + '</a> before using this data in a report or publication.' return returnStr def generateLogout(self, fileName, expName): """ generateLogout generates a java script which sends a user to the madLogin page to logout automatically. Inputs: fileName: the madrigal file to return to expName: the experiment name of the file to return to Returns: a java script which sends a user to the madLogin page to logout automatically Affects: None. Exceptions: None. """ print '<script language = "JavaScript">' print '\twindow.location = "madLogin?fileName=' + \ fileName + '&expName=' + \ expName + '&state=autoLogout"' print '</script>' def isTrusted(self): """ isTrusted returns 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise. Inputs: None Returns: 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise. Also returns 0 if no browser ip available or trustedIPs.txt cannot be opened. Affects: None. Exceptions: None. """ if self._isTrusted_ != None: return(self._isTrusted_) try: trustFile = open(self._madDB.getMadroot() + '/trustedIPs.txt', 'r') except: return 0 # try to read env var REMOTE_ADDR and HTTP_X_FORWARDED_FOR userIPList = [] if os.environ.get('REMOTE_ADDR')!= None: userIPList.append(os.environ.get('REMOTE_ADDR')) if os.environ.get('HTTP_X_FORWARDED_FOR') != None: ips = os.environ.get('HTTP_X_FORWARDED_FOR').split(',') for ip in ips: userIPList.append(ip.strip()) if len(userIPList) == 0: self._isTrusted_ = 0 return 0 if len(userIPList[0]) < 7: # ip address too short self._isTrusted_ = 0 return 0 # loop through trustedIPs.txt to find a match ipList = trustFile.readlines() for userIP in userIPList: for ipItem in ipList: # match using filename matching with * if fnmatch.fnmatch(userIP, ipItem.strip()): self._isTrusted_ = 1 return 1 # out of loop, no match found self._isTrusted_ = 0 return 0 def logDataAccess(self, fullFilenameList, user_fullname=None, user_email=None, user_affiliation=None): """ logDataAccess logs queries that access low-level data. Records user name, email, affiliation, datetime, and full path the file(s) accessed. Inputs: fullFilenameList either a list of full filenames, or a string with one filename user_fullname - if None, try to read from cookie. Also, any commas replaced by spaces. user_email - if None, try to read from cookie. Also, any commas replaced by spaces. user_affiliation - if None, try to read from cookie. Also, any commas replaced by spaces. Outputs: None Affects: Write line to log file with 5 or more comma-delimited columns. Example: Bill Rideout,brideout@haystack.mit.edu,MIT Haystack,2002-12-25 00:00:00, \ /opt/madrigal/experiments/2005/mlh/01sep05/mlh050901g.001,/opt/madrigal/experiments/2005/mlh/02sep05/mlh050902g.001 Uses _getLock and _dropLock to ensure single users access to log file """ if user_fullname == None or user_email == None or user_affiliation == None: # try to get name, email, affiliation from cookie cookie = Cookie.SimpleCookie() if os.environ.has_key('HTTP_COOKIE'): cookie.load(os.environ['HTTP_COOKIE']) try: user_fullname = cookie["user_fullname"].value user_email = cookie["user_email"].value user_affiliation = cookie["user_affiliation"].value except: # no way to write log return if user_fullname == None or user_email == None or user_affiliation == None: return # strip out any commas user_fullname = user_fullname.replace(',', ' ') user_email = user_email.replace(',', ' ') user_affiliation = user_affiliation.replace(',', ' ') if type(fullFilenameList) in (types.ListType, types.TupleType): delimiter = ',' fileStr = delimiter.join(fullFilenameList) else: fileStr = str(fullFilenameList) now = datetime.datetime.now() nowStr = now.strftime('%Y-%m-%d %H-%M-%S') # lock out any method that writes to log file self._getLock(self._logFile) f = open(self._logFile, 'a') f.write('%s,%s,%s,%s,%s\n' % (user_fullname.encode('utf8'), user_email.encode('utf8'), user_affiliation.encode('utf8'), nowStr, fileStr)) f.close() # done with log file - allow access to other writing calls self._dropLock(self._logFile) def filterLog(self, tmpFile, kinstList=None, accessStartDate=None, accessEndDate=None): """filterLog writes a subsection of the access log to a temporary file Inputs: tmpFile - temporary file to write subsection of log to kinstList - list of kinsts to accept. If None (the default), accept all instruments accessStartDate - if not None (the default), reject all access dates before datetime accessStartDate accessEndDate - if not None (the default), reject all access dates after datetime accessEndDate """ f = open(tmpFile, 'w') accessLogs = glob.glob(os.path.join(self._madDB.getMadroot(), 'metadata/userdata/access_*.log')) accessLogs.sort() # addition for cedar only """accessLogs2 = glob.glob('/opt/cedar/metadata/userdata/access_*[0-9].log') accessLogs += accessLogs2 accessLogs.sort()""" if kinstList: # create a dictionary of key = 3 letter inst mnem, value = kinstList instDict = {} instList = self._instObj.getInstrumentList() for inst in instList: if instDict.has_key(inst[1]): instDict[inst[1]].append(inst[2]) else: instDict[inst[1]] = [inst[2]] for accessLog in accessLogs: # see if we can skip this year basename = os.path.basename(accessLog) year = int(basename[7:-4]) startYear = datetime.datetime(year,1,1,0,0,0) endYear = datetime.datetime(year,12,31,23,59,59) if accessStartDate: if accessStartDate > endYear: continue if accessEndDate: if accessEndDate < startYear: continue # this file can be huge, so read one line at a time fl = open(accessLog) while True: line = fl.readline() if len(line) == 0: break items = line.strip().split(',') if len(items) != 5: continue # walk through filters # kinst if kinstList: # get the instrument mnemonic dirs = items[-1].split('/') found = False try: for kinst in instDict[dirs[-3]]: if kinst in kinstList: found = True break except KeyError: continue if not found: continue # access time if accessStartDate or accessEndDate: thisDT = datetime.datetime.strptime(items[-2], '%Y-%m-%d %H-%M-%S') if accessStartDate: if accessStartDate > thisDT: continue if accessEndDate: if accessEndDate < thisDT: continue # all filters passed f.write(line) fl.close() f.close() def createGlobalIsprintCmd(self, language, madrigalUrl, parmList, output, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, filterList, kindatList, expName, fileDesc, seasonalStartDate, seasonalEndDate, format=None): """createGlobalIsprintCmd returns a string representing a global isprint command to run in a particular language. Inputs: language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is parmList - ordered list of parameters requested. output - output file name user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) filterList - a list of strings in form "mnem,lower,upper" where lower and/or upper may be empty kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description. seasonalStartDate - a string in form 'MM/DD'. Dates before then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal start date. seasonalEndDate - a string in form 'MM/DD'. Dates after then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal end date. format - 'hdf5', 'ascii', or 'netCDF4'. If None, not specified (Madrigal 2 does not support this) """ if language not in ('python', 'Matlab', 'IDL'): raise ValueError, 'language %s not supported' % (str(language)) # url if language == 'python': cmd = 'globalIsprint.py --verbose --url=%s ' % (madrigalUrl) elif language == 'Matlab': cmd = "globalIsprint('%s', ...\n " % (madrigalUrl) elif language == 'IDL': cmd = "madglobalprint, '%s', $\n " % (madrigalUrl) # parms if len(parmList) == 0: raise ValueError, 'parmList cannot be empty' parmStr = '' for parm in parmList: parmStr += str(parm) if parm != parmList[-1]: parmStr += ',' if language == 'python': cmd += '--parms=%s ' % (parmStr) elif language == 'Matlab': cmd += "'%s', ...\n " % (parmStr) elif language == 'IDL': cmd += "'%s', $\n " % (parmStr) # output if language == 'python': cmd += '--output=%s ' % (output) elif language == 'Matlab': cmd += "'%s', ...\n " % (output) elif language == 'IDL': cmd += "'%s', $\n " % (output) # user_fullname if language == 'python': cmd += '--user_fullname="%s" ' % (user_fullname) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_fullname) elif language == 'IDL': cmd += "'%s', $\n " % (user_fullname) # user_email if language == 'python': cmd += '--user_email=%s ' % (user_email) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_email) elif language == 'IDL': cmd += "'%s', $\n " % (user_email) # user_affiliation if language == 'python': cmd += '--user_affiliation="%s" ' % (user_affiliation) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_affiliation) elif language == 'IDL': cmd += "'%s', $\n " % (user_affiliation) # start_datetime if language == 'python': cmd += '--startDate="%s" ' % (start_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (start_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (start_datetime.month, start_datetime.day, start_datetime.year, start_datetime.hour, start_datetime.minute, start_datetime.second) # end_datetime if language == 'python': cmd += '--endDate="%s" ' % (end_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (end_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (end_datetime.month, end_datetime.day, end_datetime.year, end_datetime.hour, end_datetime.minute, end_datetime.second) # instrument if language == 'python': cmd += '--inst=%i ' % (instCode) elif language == 'Matlab': cmd += "%i, ...\n " % (instCode) elif language == 'IDL': cmd += "%i, $\n " % (instCode) # format is here for Matlab or python if language in ('Matlab', 'python'): if language == 'python': if not format is None: if format.lower() == 'hdf5': cmd += '--format=%s ' % ('Hdf5') elif format in ('netCDF4', 'ascii'): cmd += '--format=%s ' % (format) elif language == 'Matlab': if format is None: cmd += "'%s', ...\n " % ('ascii') elif format.lower() == 'hdf5': cmd += "'%s', ...\n " % ('Hdf5') elif format in ('netCDF4', 'ascii'): cmd += "'%s', ...\n " % (format) # filterList # add seasonal filters if needed if len(seasonalStartDate) or len(seasonalEndDate): daynoFilterStr = self._getDaynoFilter(seasonalStartDate, seasonalEndDate) filterList.append(daynoFilterStr) if language == 'python': for filterItem in filterList: cmd += '--filter=%s ' % (filterItem) elif language == 'Matlab': filterStr = '' for filterItem in filterList: filterStr += 'filter=%s ' % (filterItem) cmd += "'%s', ...\n " % (filterStr) elif language == 'IDL': filterStr = '' for filterItem in filterList: filterStr += 'filter=%s ' % (filterItem) cmd += "'%s', $\n " % (filterStr) # kindatList if language == 'python': if len(kindatList) == 0: pass else: kindatStr = '--kindat=' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' cmd += '%s ' % (kindatStr) elif language == 'Matlab': kindatStr = '[' for kindat in kindatList: if kindat == 0: continue kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, ...\n " % (kindatStr) elif language == 'IDL': # make sure 0 not in list try: kindatList.remove(0) except ValueError: pass if len(kindatList) == 0: kindatStr = 'PTR_NEW()' else: kindatStr = '[' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, $\n " % (kindatStr) # expName if language == 'python': if len(expName) > 0: cmd += '--expName="%s" ' % (expName) elif language == 'Matlab': expName = expName.replace('*', '.*') expName = expName.replace('?', '.?') cmd += "'%s', ...\n " % (expName) elif language == 'IDL': cmd += "'%s', $\n " % (expName) # fileDesc if language == 'python': if len(fileDesc) > 0: cmd += '--fileDesc="%s" ' % (fileDesc) elif language == 'Matlab': fileDesc = fileDesc.replace('*', '.*') fileDesc = fileDesc.replace('?', '.?') cmd += "'%s') " % (fileDesc) elif language == 'IDL': cmd += "'%s', $\n " % (fileDesc) # format is here for idl if language == 'IDL': if format is None: cmd += "'ascii', $\n " elif format.lower() == 'hdf5': cmd += "'hdf5', $\n " elif format in ('netCDF4', 'ascii'): cmd += "'%s', $\n " % (format) return(cmd) def createGlobalDownloadCmd(self, language, madrigalUrl, output, format, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, kindatList, expName, fileDesc): """createGlobalDownloadCmd returns a string representing a global download as is command to run in a particular language. Inputs: language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is output - output directory name format - 'hdf5', 'ascii', or 'netCDF4' user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description. """ if language not in ('python', 'Matlab', 'IDL'): raise ValueError, 'language %s not supported' % (str(language)) # url if language == 'python': cmd = 'globalDownload.py --verbose --url=%s ' % (madrigalUrl) elif language == 'Matlab': cmd = "globalDownload('%s', ...\n " % (madrigalUrl) elif language == 'IDL': cmd = "madglobaldownload, '%s', $\n " % (madrigalUrl) # output if language == 'python': cmd += '--outputDir=%s ' % (output) elif language == 'Matlab': cmd += "'%s', ...\n " % (output) elif language == 'IDL': cmd += "'%s', $\n " % (output) # user_fullname if language == 'python': cmd += '--user_fullname="%s" ' % (user_fullname) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_fullname) elif language == 'IDL': cmd += "'%s', $\n " % (user_fullname) # user_email if language == 'python': cmd += '--user_email=%s ' % (user_email) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_email) elif language == 'IDL': cmd += "'%s', $\n " % (user_email) # user_affiliation if language == 'python': cmd += '--user_affiliation="%s" ' % (user_affiliation) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_affiliation) elif language == 'IDL': cmd += "'%s', $\n " % (user_affiliation) # format part 1 (format is not in same order in Matlab and IDL) if format not in ('hdf5', 'ascii', 'netCDF4'): raise ValueError, 'format not in hdf5, ascii or netCDF4' if language == 'python': cmd += '--format="%s" ' % (format) elif language == 'Matlab': cmd += "'%s', ...\n " % (format) # start_datetime if language == 'python': cmd += '--startDate="%s" ' % (start_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (start_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (start_datetime.month, start_datetime.day, start_datetime.year, start_datetime.hour, start_datetime.minute, start_datetime.second) # end_datetime if language == 'python': cmd += '--endDate="%s" ' % (end_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (end_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (end_datetime.month, end_datetime.day, end_datetime.year, end_datetime.hour, end_datetime.minute, end_datetime.second) # instrument if language == 'python': cmd += '--inst=%i ' % (instCode) elif language == 'Matlab': cmd += "%i, ...\n " % (instCode) elif language == 'IDL': cmd += "%i, $\n " % (instCode) # kindatList if language == 'python': if len(kindatList) == 0: pass else: kindatStr = '--kindat=' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' cmd += '%s ' % (kindatStr) elif language == 'Matlab': kindatStr = '[' for kindat in kindatList: if kindat == 0: continue kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, ...\n " % (kindatStr) elif language == 'IDL': # make sure 0 not in list try: kindatList.remove(0) except ValueError: pass if len(kindatList) == 0: kindatStr = 'PTR_NEW()' else: kindatStr = '[' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, $\n " % (kindatStr) # now deal with IDL format if needed if language == 'IDL': cmd += "'%s', $\n " % (format) # expName if language == 'python': if len(expName) > 0: cmd += '--expName="%s" ' % (expName) elif language == 'Matlab': expName = expName.replace('*', '.*') expName = expName.replace('?', '.?') cmd += "'%s', ...\n " % (expName) elif language == 'IDL': cmd += "'%s', $\n " % (expName) # fileDesc if language == 'python': if len(fileDesc) > 0: cmd += '--fileDesc="%s" ' % (fileDesc) elif language == 'Matlab': fileDesc = fileDesc.replace('*', '.*') fileDesc = fileDesc.replace('?', '.?') cmd += "'%s') " % (fileDesc) elif language == 'IDL': cmd += "'%s' " % (fileDesc) return(cmd) def generateGlobalIsprintScriptFromForm(self, form1, form2, form3, user_fullname, user_email, user_affiliation): """generateGlobalIsprintScriptFromForm converts the three Django forms into arguments so that if can then call createGlobalIsprintCmd. Separate forms used because some parts are created by Ajax. form1 is a dict with keys: instruments, start_date, end_date, format_select, directory_select, language_select, kindat_select, expName, fileDesc form2 is a dict with keys parameters form3 is a dict with keys parm_#, parm_#_lower, parm_#_upper, where # is 1, 2, and 3 user_fullname, user_email, user_affiliation - strings """ instCode = int(form1['instruments']) start_datetime = datetime.datetime(form1['start_date'].year, form1['start_date'].month, form1['start_date'].day) end_datetime = datetime.datetime(form1['end_date'].year, form1['end_date'].month, form1['end_date'].day) format = form1['format_select'] if format == 'ascii' and form1['directory_select'] == 'File': output = 'example.txt' else: output = '/tmp' language = form1['language_select'] kindatList = [int(kindat) for kindat in form1['kindat_select']] expName = form1['expName'].strip() fileDesc = form1['fileDesc'].strip() seasonalStartDay = int(form1['seasonalStartDay']) seasonalStartMonth = int(form1['seasonalStartMonth']) seasonalEndDay = int(form1['seasonalEndDay']) seasonalEndMonth = int(form1['seasonalEndMonth']) if seasonalStartDay == 1 and seasonalStartMonth == 1 and \ seasonalEndDay == 31 and seasonalEndMonth == 12: seasonalStartDate = '' seasonalEndDate = '' else: seasonalStartDate = '%02i/%02i' % (seasonalStartMonth, seasonalStartDay) seasonalEndDate = '%02i/%02i' % (seasonalEndMonth, seasonalStartDay) madrigalUrl = self._madDB.getTopLevelUrl() parmList = form2['parameters'] filterList = [] for i in (1,2,3): try: parm = form3['parm_%i' % (i)] except KeyError: continue if len(parm) == 0: continue filterStr = '%s,' % (parm) try: parm_lower = form3['parm_%i_lower' % (i)] filterStr += '%s,' % (str(parm_lower)) except KeyError: filterStr += ',' try: parm_upper = form3['parm_%i_upper' % (i)] filterStr += '%s' % (str(parm_upper)) except KeyError: pass filterList.append(filterStr) return(self.createGlobalIsprintCmd(language, madrigalUrl, parmList, output, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, filterList, kindatList, expName, fileDesc, seasonalStartDate, seasonalEndDate, format)) def generateDownloadFileScriptFromForm(self, form, user_fullname, user_email, user_affiliation): """generateDownloadFileScriptFromForm converts the Django form into arguments so that if can then call createGlobalDownloadCmd. form is a dict with keys: instruments, start_date, end_date, format_select, language_select, kindat_select, expName, fileDesc user_fullname, user_email, user_affiliation - strings """ instCode = int(form['instruments']) start_datetime = datetime.datetime(form['start_date'].year, form['start_date'].month, form['start_date'].day) end_datetime = datetime.datetime(form['end_date'].year, form['end_date'].month, form['end_date'].day) format = form['format_select'] language = form['language_select'] kindatList = [int(kindat) for kindat in form['kindat_select']] expName = form['expName'].strip() fileDesc = form['fileDesc'].strip() madrigalUrl = self._madDB.getTopLevelUrl() return(self.createGlobalDownloadCmd(language, madrigalUrl, '/tmp', format, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, kindatList, expName, fileDesc)) def getSingleRedirectList(self): """getSingleRedirectList returns a list with tuples (kinst, url) where url is url to redirect single UI to if instrument not local. If no redirect needed because instrument local, url is empty string """ madInstData = madrigal.metadata.MadrigalInstrumentData(self._madDB, True) siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteID = self._madDB.getSiteID() # create a dict with key = siteID, value = redirect url for speed getStr = '?isGlobal=True&categories=%i&instruments=%i' addUrl = django.core.urlresolvers.reverse('view_single') cedarUrl = 'http://cedar.openmadrigal.org/' + addUrl + getStr siteDict = {} for thisSiteID, siteDesc in siteObj.getSiteList(): if thisSiteID == siteID: siteDict[thisSiteID] = '' # local case elif siteObj.getSiteVersion(thisSiteID) == '2.6': # redirect to cedar because other site below Madrigal 3 siteDict[thisSiteID] = cedarUrl else: siteDict[thisSiteID] = os.path.join(siteObj.getSiteServer(thisSiteID), siteObj.getSiteDocRoot(thisSiteID), addUrl + getStr) retList = [] for kinst, desc, thisSiteID in madInstData.getInstruments(): if len(siteDict[thisSiteID]) > 0: url = siteDict[thisSiteID] % (self._instObj.getCategoryId(kinst), kinst) else: url = '' retList.append((kinst, url)) return(retList) def getMonths(self, kinst, year, optimize=True): """getMonths returns a list of tuples of (monthNumber, monthName) where monthNumber is 1-12, and monthName is the form January, Febuary, etc. for the the months where there is local data for kinst & year combination Inputs: kinst - instrument id (int) year - year (int) optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning """ tempDict = {} # dict with key = month number, value = month name sDT = datetime.datetime(year,1,1) eDT = datetime.datetime(year,12,31,23,59,59) madroot = self._madDB.getMadroot() if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue if thisSDT.year == year: startMonth = thisSDT.month else: startMonth = 1 if thisEDT.year == year: endMonth = thisEDT.month else: endMonth = 12 monthList = range(startMonth, endMonth + 1) for thisMonth in monthList: if thisMonth not in tempDict.keys(): tempDict[thisMonth] = calendar.month_name[thisMonth] monthKeys = tempDict.keys() monthKeys.sort() retList = [(monthKey, tempDict[monthKey]) for monthKey in monthKeys] return(retList) def getDays(self, kinst, year, month=None, optimize=True): """getDays returns a sorted list of datetime.date objects where there is local data for kinst & year & possibly month combination Inputs: kinst - instrument id (int) year - year (int) month (1-12) if None, include all months optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning """ retList = [] sDT = datetime.datetime(year,1,1) eDT = datetime.datetime(year,12,31,23,59,59) if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue # loop over all days delta = datetime.timedelta(days=1) loopDT = max(sDT, datetime.datetime(thisSDT.year, thisSDT.month, thisSDT.day)) while (loopDT <= thisEDT): if loopDT > eDT: break if not month is None: if month > loopDT.month: loopDT += delta continue elif month < loopDT.month: break thisDate = loopDT.date() if thisDate not in retList: retList.append(thisDate) loopDT += delta retList.sort() return(retList) def getExperimentList(self, kinstList, startDT, endDT, localOnly): """getExperimentList returns a sorted list of tuples of (expId, expUrl, expName, instName, kinst, expStartDT, expEndDT, siteId, siteName) Inputs: kinstList - a list of instrument id (int) - may include 0 startDT - start datetime to search endDT - end datetime to search localOnly - if True, only search locally. If False, search globally """ retList = [] madroot = self._madDB.getMadroot() siteId = self._madDB.getSiteID() siteObj = madrigal.metadata.MadrigalSite(self._madDB) if localOnly: expTabFile = os.path.join(madroot, 'metadata/expTab.txt') else: expTabFile = os.path.join(madroot, 'metadata/expTabAll.txt') madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB, expTabFile) startIndex = madExpObjDate.getStartPosition(startDT) for i in range(startIndex, madExpObjDate.getExpCount()): thisKinst = madExpObjDate.getKinstByPosition(i) if thisKinst not in kinstList and 0 not in kinstList: continue thisSDTList = madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < startDT: continue if thisSDT > endDT: break # check for security security = madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue thisSiteId = madExpObjDate.getExpSiteIdByPosition(i) if siteId == thisSiteId: isLocal = True else: isLocal = False thisExpId = madExpObjDate.getExpIdByPosition(i) thisExpPath = madExpObjDate.getExpPathByPosition(i) thisExpName = madExpObjDate.getExpNameByPosition(i) if isLocal: thisExpUrl = django.core.urlresolvers.reverse('show_experiment') + \ '?experiment_list=%i' % (thisExpId) elif siteObj.getSiteVersion(thisSiteId) == '2.6': thisExpUrl = 'http://' + os.path.join(siteObj.getSiteServer(thisSiteId), siteObj.getSiteRelativeCGI(thisSiteId), 'madExperiment.cgi?exp=%s' % (thisExpPath)) thisExpUrl += '&displayLevel=0&expTitle=%s' % (django.utils.http.urlquote(thisExpName)) else: # remote Madrigal 3.0 site baseUrl = os.path.basename(django.core.urlresolvers.reverse('show_experiment')[:-1]) thisExpUrl = baseUrl + '?experiment_list=%s' % (thisExpPath) thisSiteUrl = 'http://%s' % (siteObj.getSiteServer(thisSiteId)) relativeUrl = siteObj.getSiteDocRoot(thisSiteId) if relativeUrl not in ('', None): thisSiteUrl = os.path.join(thisSiteUrl, relativeUrl) if thisSiteUrl[-1] != '/' and thisExpUrl[0] != '/': thisSiteUrl += '/' thisExpUrl = thisSiteUrl + thisExpUrl thisSiteName = siteObj.getSiteName(thisSiteId) instName = self._instObj.getInstrumentName(thisKinst) retList.append((thisExpId, thisExpUrl, thisExpName, instName, thisKinst, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S'), thisSiteId, thisSiteName)) return(retList) def getExpsOnDate(self, kinst, year, month, day, optimize=True): """getExpsOnDate returns a sorted list of tuples of (expId, expDesc, expDir, pi_name, pi_email) Inputs: kinst - instrument id (int) year - year (int) month - month (int) day - day (int) optimize - if True, only start search at beginning of day. But may miss long experiments, so if optimization if False, starts at beginning """ retList = [] sDT = datetime.datetime(year,month,day) eDT = datetime.datetime(year,month,day,23,59,59) if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue thisExpId = self._madExpObjDate.getExpIdByPosition(i) thisExpName = self._madExpObjDate.getExpNameByPosition(i) thisExpDesc = '%s: %s-%s' % (thisExpName, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S')) thisExpDir = self._madExpObjDate.getExpDirByPosition(i) thisExpPI = self._madExpObjDate.getPIByPosition(i) thisExpPIEmail = self._madExpObjDate.getPIEmailByPosition(i) if thisExpPI in (None, ''): thisExpPI = self._instObj.getContactName(kinst) thisExpPIEmail = self._instObj.getContactEmail(kinst) retList.append((thisExpId, thisExpDesc, thisExpDir)) return(retList) def getFileFromExpDir(self, expDir, kinst, includeNonDefault=False): """getFileFromExpDir returns a list of tuples of (basename, fileDesc) Inputs: expDir - full path to exp directory kinst - instrument id (used to look up kindat descriptions) includeNonDefault - if True, include variant and history files. If False (the default), do not """ retList = [] realTimeList = [] # in case no default files if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK): # no files in this experiment return(retList) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) for i in range(madFileObj.getFileCount()): if madFileObj.getAccessByPosition(i) == 1 and not self.isTrusted(): continue category = madFileObj.getCategoryByPosition(i) if category in (2,3) and not includeNonDefault: continue basename = madFileObj.getFilenameByPosition(i) kindat = madFileObj.getKindatByPosition(i) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) status = madFileObj.getStatusByPosition(i) fileDesc = '%s: %s - %s' % (basename, kindatDesc, status) if category != 4: retList.append((basename, fileDesc)) else: realTimeList.append((basename, fileDesc)) if len(retList) > 0: return(retList) else: return(realTimeList) def getExpInfoFromExpID(self, expID): """getExpInfoFromExpID returns a tuple of (pi_name, pi_email, expUrl, kinst, expDesc, kinstDesc) for given expID expUrl is url from getRealExpUrlByExpId Inputs: expID - experimentID (int) """ expID = int(expID) if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) kinst = self._madExpObjExpID.getKinstByExpId(expID) kinstDesc = self._instObj.getInstrumentName(kinst) expPI = self._madExpObjExpID.getPIByExpId(expID) expPIEmail = self._madExpObjExpID.getPIEmailByExpId(expID) expUrl = self._madExpObjExpID.getRealExpUrlByExpId(expID) if expPI in (None, ''): expPI = self._instObj.getContactName(kinst) expPIEmail = self._instObj.getContactEmail(kinst) thisSDTList = self._madExpObjExpID.getExpStartDateTimeByExpId(expID) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjExpID.getExpEndDateTimeByExpId(expID) thisEDT = datetime.datetime(*thisEDTList[0:6]) thisExpName = self._madExpObjExpID.getExpNameByExpId(expID) thisExpDesc = '%s: %s-%s' % (thisExpName, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S')) return((expPI, expPIEmail, expUrl, kinst, thisExpDesc, kinstDesc)) def getExpIDFromExpPath(self, expPath, matchAnyExpNum=False): """getExpIDFromExpPath returns the expId for given expPath (starts with 'experiments') If matchAnyExpNum is False, it will only match the right experiments* directory (default). It True, matches via re to experiments[0-9]*/<remaining exp path> Returns None if not found Inputs: expPath - experiment path (starts with 'experiments') """ madExpObj = madrigal.metadata.MadrigalExperiment(self._madDB) if matchAnyExpNum: expPathRE = 'experiments[0-9]*/' + expPath[expPath.find('/')+1:] for i in range(madExpObj.getExpCount()): if not matchAnyExpNum: if madExpObj.getExpPathByPosition(i) == expPath: return(madExpObj.getExpIdByPosition(i)) else: if len(re.findall(expPathRE, madExpObj.getExpPathByPosition(i))) > 0: return(madExpObj.getExpIdByPosition(i)) return(None) def getInfoFromFile(self, filePath): """getInfoFromFile returns a tuple of (expName, kindatDesc) for a given input file """ expDir = os.path.dirname(filePath) basename = os.path.basename(filePath) madExpObj = madrigal.metadata.MadrigalExperiment(self._madDB, os.path.join(expDir, 'expTab.txt')) expName = madExpObj.getExpNameByPosition(0) kinst = madExpObj.getKinstByPosition(0) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) kindat = madFileObj.getKindatByFilename(basename) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) return((expName, kindatDesc)) def getFileFromExpID(self, expID, includeNonDefault=False): """getFileFromExpDir returns a list of tuples of (basename, fileDesc) Inputs: expID - experimentID (int) includeNonDefault - if True, include variant and history files. If False (the default), do not """ retList = [] realTimeList = [] # in case no default files if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) expDir = self._madExpObjExpID.getExpDirByExpId(expID) kinst = self._madExpObjExpID.getKinstByExpId(expID) if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK): # no files in this experiment return(retList) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) for i in range(madFileObj.getFileCount()): if madFileObj.getAccessByPosition(i) == 1 and not self.isTrusted(): continue category = madFileObj.getCategoryByPosition(i) if category in (2,3) and not includeNonDefault: continue if category == 2: categoryStr = '<variant file> ' elif category == 3: categoryStr = '<history file> ' else: categoryStr = '' basename = madFileObj.getFilenameByPosition(i) kindat = madFileObj.getKindatByPosition(i) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) status = madFileObj.getStatusByPosition(i) fileDesc = '%s: %s%s - %s' % (basename, categoryStr, kindatDesc, status) if category != 4: retList.append((basename, fileDesc)) else: realTimeList.append((basename, fileDesc)) if len(retList) > 0: return(retList) else: return(realTimeList) def getSiteInfo(self): """getSiteInfo returns a tuple of two items: 1. local site name 2. list of tuples of (siteName, url) of non-local sites """ siteID = self._madDB.getSiteID() siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteName = siteObj.getSiteName(siteID) retList = [] siteList = siteObj.getSiteList() for thisSiteID, thisSiteName in siteList: if thisSiteID == siteID: continue thisSiteServer = siteObj.getSiteServer(thisSiteID) thisSiteDocRoot = siteObj.getSiteDocRoot(thisSiteID) thisSiteUrl = urlparse.urlunparse(('http', thisSiteServer, thisSiteDocRoot, '','','')) retList.append((thisSiteName, thisSiteUrl)) return((siteName, retList)) def downloadFileAsIs(self, expId, basename, user_fullname, user_email, user_affiliation): """downloadFileAsIs returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters) Inputs: expId - experiment id of experiment basename - basename of file. May have .txt or .nc extension, in which case Hdf5 file is converted user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() hdf5Extensions = ('.hdf5', '.h5', '.hdf') fileName, fileExtension = os.path.splitext(basename) if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) expDir = self._madExpObjExpID.getExpDirByExpId(int(expId)) if expDir is None: raise ValueError, 'No expDir found for exp_id %i' % (int(expId)) if fileExtension in hdf5Extensions: baseHdf5 = basename else: # we need to search for the hdf5 basename madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) baseHdf5 = None for i in range(madFileObj.getFileCount()): thisFileName, thisFileExt = os.path.splitext(madFileObj.getFilenameByPosition(i)) if thisFileName == fileName and thisFileExt in hdf5Extensions: baseHdf5 = madFileObj.getFilenameByPosition(i) break if baseHdf5 is None: raise ValueError, 'No valid file for %s found in %s' % (basename, expDir) if basename == baseHdf5: fullFilename = os.path.join(expDir, basename) fullHdf5Filename = fullFilename tmpDir = None else: fullHdf5Filename = os.path.join(expDir, baseHdf5) # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass fullFilename = os.path.join(tmpDir, basename) if os.access(fullFilename, os.R_OK): try: os.remove(fullFilename) except: pass if fileExtension == '.txt': cachedFile = os.path.join(expDir, 'overview', baseHdf5 + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(fullHdf5Filename, fullFilename) elif fileExtension == '.nc': cachedFile = os.path.join(expDir, 'overview', baseHdf5 + '.nc') if os.access(cachedFile, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(fullHdf5Filename, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(fullHdf5Filename) cedarObj.write('netCDF4', fullFilename) # log access self.logDataAccess(fullHdf5Filename, user_fullname, user_email, user_affiliation) return(fullFilename) def downloadFullFileAsIs(self, fullHdf5Filename, format, user_fullname, user_email, user_affiliation): """downloadFullFileAsIs is similar to downloadFileAsIs with fullFilename input instead of expId . Returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters) Inputs: fullHdf5Filename - full path to Madrigal Hdf5 file format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() if format not in ('hdf5', 'netCDF4', 'ascii'): raise ValueError, 'Illegal format %s' % (str(format)) if not os.access(fullHdf5Filename, os.R_OK): raise IOError, 'Unable to find Hdf5 file %s' % (str(fullHdf5Filename)) if format == 'hdf5': fullFilename = fullHdf5Filename tmpDir = None else: # dynamically create file if format == 'netCDF4': thisExt = '.nc' elif format == 'ascii': thisExt = '.txt' # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass base, file_extension = os.path.splitext(fullHdf5Filename) basename = os.path.basename(base + thisExt) fullFilename = os.path.join(tmpDir, basename) if os.access(fullFilename, os.R_OK): try: os.remove(fullFilename) except: pass if format == 'ascii': cachedFile = os.path.join(os.path.dirname(fullHdf5Filename), 'overview', os.path.basename(fullHdf5Filename) + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(fullHdf5Filename, fullFilename) elif format == 'netCDF4': cachedFile = os.path.join(os.path.dirname(fullHdf5Filename), 'overview', os.path.basename(fullHdf5Filename) + '.nc') if os.access(cachedFile, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(fullHdf5Filename, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(fullHdf5Filename) cedarObj.write('netCDF4', fullFilename) # log access self.logDataAccess(fullHdf5Filename, user_fullname, user_email, user_affiliation) return(fullFilename) def downloadMultipleFiles(self, fileList, format, user_fullname, user_email, user_affiliation): """downloadMultipleFiles downloads multiple files in tarred format Returns a path to a Madrigal tarred file to download in given format as is (that is, with parms in file, and no filters) Inputs: fileList - list of full paths to Madrigal Hdf5 files format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() if format not in ('hdf5', 'netCDF4', 'ascii'): raise ValueError, 'Illegal format %s' % (str(format)) # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass # be sure no duplicate file names basenameList = [] if format == 'hdf5': finalFileList = [] for thisFile in fileList: basename = os.path.basename(thisFile) dirname = os.path.dirname(thisFile) if basename in basenameList: basename = self.modifyBasename(basename) fullFilename = os.path.join(tmpDir, basename) shutil.copy(thisFile, fullFilename) else: fullFilename = os.path.join(dirname, basename) basenameList.append(basename) finalFileList.append(fullFilename) else: # dynamically create files if format == 'netCDF4': thisExt = '.nc' elif format == 'ascii': thisExt = '.txt' finalFileList = [] for thisFile in fileList: if not os.access(thisFile, os.R_OK): raise IOError, 'Unable to find Hdf5 file %s' % (str(thisFile)) base, file_extension = os.path.splitext(thisFile) basename = os.path.basename(base + thisExt) if basename in basenameList: basename = self.modifyBasename(basename) basenameList.append(basename) fullFilename = os.path.join(tmpDir, basename) if format == 'ascii': cachedFile = os.path.join(os.path.dirname(thisFile), 'overview', os.path.basename(thisFile) + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' if not os.access(fullFilename, os.R_OK): shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(thisFile, fullFilename) elif format == 'netCDF4': cachedFile = os.path.join(os.path.dirname(thisFile), 'overview', os.path.basename(thisFile) + '.nc') if os.access(cachedFile, os.R_OK): if not os.access(fullFilename, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(thisFile, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(thisFile) cedarObj.write('netCDF4', fullFilename) finalFileList.append(fullFilename) # create tar file now = datetime.datetime.now() tar_filename = os.path.join(tmpDir, 'madrigalFiles_%s.tar' % (now.strftime('%Y%m%dT%H%M%S'))) tar = tarfile.open(tar_filename, "w") for thisFile in finalFileList: # log access self.logDataAccess(thisFile, user_fullname, user_email, user_affiliation) tar.add(thisFile) # let clean stage do this - safer return(tar_filename) def printFileAsIs(self, fullFilename, user_fullname, user_email, user_affiliation, html=True): """printFileAsIs returns the full path to a temp file representing file as plain text or html to print as is (that is, with parms rom file, and no filters) Inputs: fullFilename - full path to Madrigal Hdf5 file to convert to string user_fullname, user_email, user_affiliation - user identification strings html - if True (the default) return as Html with popup parm names. If False, pure text """ self.cleanStage() # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass fileName, fileExtension = os.path.splitext(fullFilename) fullTmpFilename = os.path.join(tmpDir, os.path.basename(fileName + '.txt')) if os.access(fullTmpFilename, os.R_OK): try: os.remove(fullTmpFilename) except: pass if html: summary = 'html' else: summary = 'plain' madrigal.cedar.convertToText(fullFilename, fullTmpFilename, summary=summary) # log access self.logDataAccess(fullFilename, user_fullname, user_email, user_affiliation) return(fullTmpFilename) def listRecords(self, fullFilename): """listRecords returns the list records html for fullFilename """ # check if record plots exist basename = os.path.basename(fullFilename) thisDir = os.path.dirname(fullFilename) pngFiles = glob.glob(os.path.join(thisDir, 'plots', basename, 'records/*.png')) if len(pngFiles) > 0: url = '<a href="javascript:plotRecno(%i)">View record plot</a>' else: url = None output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.cedar.listRecords(fullFilename, output, url) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip()) def downloadIsprintFileFromIsprintForm(self, isprintForm, user_fullname, user_email, user_affiliation): """downloadIsprintFileFromIsprintForm returns a full path to the temp file is experiments/stage created by isprint to download. Inputs: isprintForm - the django form that encapsulates all information from get_advanced web page. user_fullname, user_email, user_affiliation - user identification strings """ # defaults arguments showHeaders=False missing=None assumed=None knownbad=None orgFilename = isprintForm['fullFilename'] requestedParms = isprintForm['parameters'] # make unique uniqueRequestedParms = [] for s in requestedParms: asciiParm = s.encode('ascii').lower().strip() if asciiParm not in uniqueRequestedParms: uniqueRequestedParms.append(asciiParm) start_date = isprintForm['start_date'] end_date = isprintForm['end_date'] format = isprintForm['formats'] # create name of temp file basename_noext, ext = os.path.splitext(os.path.basename(orgFilename)) randint = random.randint(0,999999) if format in ('ascii', 'netCDF4'): # need new basename if format == 'ascii': basename = basename_noext + '_%06i.txt' % (randint) else: basename = basename_noext + '_%06i.nc' % (randint) else: basename = basename_noext + '_%06i.hdf5' % (randint) tmpFile = os.path.join(self._madDB.getMadroot(), 'experiments/stage', basename) if format == 'ascii': # reset defaults showHeaders = isprintForm['showHeaders'] missing = isprintForm['missing'] assumed = isprintForm['missing'] knownbad = isprintForm['missing'] # next task - create a list of filters, but only if actually modified madFilters = [] madFileObj = madrigal.data.MadrigalFile(orgFilename, self._madDB) # used to determine default values # check if we need a time filer earliestTime = madFileObj.getEarliestTime() latestTime = madFileObj.getLatestTime() earliestDT = datetime.datetime(*earliestTime) latestDT = datetime.datetime(*latestTime) earliest_unix = calendar.timegm(earliestDT.timetuple()) latest_unix = calendar.timegm(latestDT.timetuple()) start_unix = calendar.timegm(start_date.timetuple()) end_unix = calendar.timegm(end_date.timetuple()) if earliest_unix < start_unix or latest_unix > end_unix: # we need a time filter madFilters.append(madrigal.derivation.MadrigalFilter('ut1_unix', [(start_unix, end_unix)])) # altitude filter if isprintForm.has_key('min_alt'): file_min_alt = madFileObj.getMinValidAltitude() file_max_alt = madFileObj.getMaxValidAltitude() try: min_alt = float(isprintForm['min_alt']) except ValueError: min_alt = float('nan') try: max_alt = float(isprintForm['max_alt']) except ValueError: max_alt = float('nan') is_needed = False if not math.isnan(min_alt): if min_alt > file_min_alt + 1.0E-6: is_needed = True if not math.isnan(max_alt): if max_alt < file_max_alt - 1.0E-6: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('gdalt', [(min_alt, max_alt)])) # azimuth filter if isprintForm.has_key('min_az'): try: min_az = float(isprintForm['min_az']) except ValueError: min_az = float('nan') try: max_az = float(isprintForm['max_az']) except ValueError: max_az = float('nan') try: min_az2 = float(isprintForm['min_az2']) except ValueError: min_az2 = float('nan') try: max_az2 = float(isprintForm['max_az2']) except ValueError: max_az2 = float('nan') is_needed = False if not math.isnan(min_az): if min_az > -180.0: is_needed = True if not math.isnan(max_az): if max_az < 180.0: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('azm', [(min_az, max_az), (min_az2, max_az2)])) # elevation filter if isprintForm.has_key('min_el'): try: min_el = float(isprintForm['min_el']) except ValueError: min_el = float('nan') try: max_el = float(isprintForm['max_el']) except ValueError: max_el = float('nan') try: min_el2 = float(isprintForm['min_el2']) except ValueError: min_el2 = float('nan') try: max_el2 = float(isprintForm['max_el2']) except ValueError: max_el2 = float('nan') is_needed = False if not math.isnan(min_el): if min_el > 0.0: is_needed = True if not math.isnan(max_el): if max_el < 90.0: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('azm', [(min_el, max_el), (min_el2, max_el2)])) # pulse length filter if isprintForm.has_key('min_pl'): file_min_pl = madFileObj.getMinPulseLength() file_max_pl = madFileObj.getMaxPulseLength() try: min_pl = float(isprintForm['min_pl']) except ValueError: min_pl = float('nan') try: max_pl = float(isprintForm['max_pl']) except ValueError: max_pl = float('nan') is_needed = False if not math.isnan(min_pl): if min_pl > file_min_pl + 1.0E-9: is_needed = True if not math.isnan(max_pl): if max_pl < file_max_pl - 1.0E-9: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('pl', [(min_pl, max_pl)])) # free parameters for i in range(1, 4): parm_name = isprintForm['parm_%i' % (i)] parm_lower = isprintForm['parm_%i_lower' % (i)] parm_upper = isprintForm['parm_%i_upper' % (i)] if parm_name == 'None': continue if len(parm_lower) == 0 and len(parm_upper) == 0: continue # filter needed try: min_value = float(parm_lower) except ValueError: min_value = float('nan') try: max_value = float(parm_upper) except ValueError: max_value = float('nan') madFilters.append(madrigal.derivation.MadrigalFilter(parm_name, [(min_value, max_value)])) # create new temp file madrigal.isprint.Isprint(orgFilename, tmpFile, uniqueRequestedParms, madFilters, showHeaders=showHeaders, missing=missing, assumed=assumed, knownbad=knownbad) # log access self.logDataAccess(orgFilename, user_fullname, user_email, user_affiliation) return(tmpFile) def runMadrigalCalculatorFromForm(self, madCalculatorForm): """runMadrigalCalculatorFromForm returns the text output of madCalculator from the MadCalulatorForm Inputs: madCalculatorForm - the django form that encapsulates all information from madrigal_calculator web page. """ requestedParms = madCalculatorForm['parameters'] requestedParms = ['gdlat', 'glon', 'gdalt'] + [str(parm) for parm in requestedParms] thisDT = madCalculatorForm['datetime'] min_latitude = madCalculatorForm['min_latitude'] max_latitude = madCalculatorForm['max_latitude'] delta_latitude = madCalculatorForm['delta_latitude'] min_longitude = madCalculatorForm['min_longitude'] max_longitude = madCalculatorForm['max_longitude'] delta_longitude = madCalculatorForm['delta_longitude'] min_altitude = madCalculatorForm['min_altitude'] max_altitude = madCalculatorForm['max_altitude'] delta_altitude = madCalculatorForm['delta_altitude'] latList = numpy.arange(min_latitude, max_latitude+0.001*delta_latitude, delta_latitude).tolist() lonList = numpy.arange(min_longitude, max_longitude+0.001*delta_longitude, delta_longitude).tolist() altList = numpy.arange(min_altitude, max_altitude+0.001*delta_altitude, delta_altitude).tolist() if 0 in (len(latList), len(lonList), len(altList)): raise ValueError, 'Got 0 length spatial range' output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.isprint.MadCalculatorGrid(output, requestedParms, [thisDT], latList, lonList, altList) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip()) def runLookerFromForm(self, form): """runLookerFromForm returns the text output of looker from one of the looker forms Inputs: form - the django form that encapsulates all information from looker web page. """ looker_cmd = os.path.join(self._madDB.getMadroot(), 'bin/looker1') looker_options = int(form['looker_options']) if looker_options in (1,2): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, float(form['start_lat']), float(form['stop_lat']), float(form['step_lat']), float(form['start_lon']), float(form['stop_lon']), float(form['step_lon']), float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) looker_cmd += argStr try: text = subprocess.check_output(looker_cmd.split()) except: raise IOError, 'Unable to run cmd <%s>' % (looker_cmd) return(text) elif looker_options in (3,): try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, 0.0, 0.0, 0.0, float(form['start_lat']), float(form['stop_lat']), float(form['step_lat']), float(form['start_lon']), float(form['stop_lon']), float(form['step_lon']), float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (4,): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, float(form['start_az']), float(form['stop_az']), float(form['step_az']), float(form['start_el']), float(form['stop_el']), float(form['step_el']), float(form['start_range']), float(form['stop_range']), float(form['step_range'])) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (5,6,7): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 if looker_options == 5: p1 = float(form['fl_az']) p2 = float(form['fl_el']) p3 = float(form['fl_range']) elif looker_options == 6: p1 = float(form['fl_lat']) p2 = float(form['fl_lon']) p3 = float(form['fl_alt']) elif looker_options == 7: p1 = float(form['fl_apex_lat']) p2 = float(form['fl_apex_lon']) p3 = 0.0 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, p1, p2, p3, float(form['start_alt']), float(form['stop_alt']), float(form['step_alt']), 0.0, 0.0, 0.0) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (8,): latList = numpy.arange(float(form['start_lat']), float(form['stop_lat']), float(form['step_lat'])) latList = latList.tolist() # check for null list if len(latList) == 0 and abs(float(form['start_lat']) - float(form['stop_lat'])) < 1.0E-6: latList = [float(form['start_lat'])] lonList = numpy.arange(float(form['start_lon']), float(form['stop_lon']), float(form['step_lon'])) lonList = lonList.tolist() # check for null list if len(lonList) == 0 and abs(float(form['start_lon']) - float(form['stop_lon'])) < 1.0E-6: lonList = [float(form['start_lon'])] altList = numpy.arange(float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) altList = altList.tolist() # check for null list if len(altList) == 0 and abs(float(form['start_alt']) - float(form['stop_alt'])) < 1.0E-6: altList = [float(form['start_alt'])] requestedParms = ['gdlat', 'glon', 'gdalt'] + [str(parm.lower()) for parm in form['pList']] dtList = [form['datetime']] output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.isprint.MadCalculatorGrid(output, requestedParms, dtList, latList, lonList, altList) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip()) else: raise ValueError, 'Unknown looker_options %s' % (str(looker_options)) def cleanStage(self): """cleanStage removes all temp files more than 2 hours old from experiments/stage """ stageDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') filesToTest = glob.glob(os.path.join(stageDir, '*')) now = datetime.datetime.now() cutoff = datetime.timedelta(hours=2) for fileToTest in filesToTest: try: mDT = datetime.datetime.fromtimestamp(os.path.getmtime(fileToTest)) except: continue if now - mDT > cutoff: try: os.remove(fileToTest) except: pass # ignore problems def modifyBasename(self, basename): """modifyBasename adds _<num> to make sure basename unique """ base, file_extension = os.path.splitext(basename) index = base.rfind('_') if index != -1: try: version = int(base[index+1:]) return('%s%i%s' % (base[:index], version+1, file_extension)) except: pass return('%s_%i%s' % (base, 1, file_extension)) def _getDaynoFilter(self, seasonalStartDate, seasonalEndDate): """_getDaynoFilter returns a filter str in the form dayno,<lower>,<upper> Inputs: seasonalStartDate - a string in form 'MM/DD'. Assumes non-leap year. Empty string means no filtering by seasonal start date. seasonalStartDate - a string in form 'MM/DD'. Assumes non-leap year. Empty string means no filtering by seasonal end date. """ startDayno = '' endDayno = '' if len(seasonalStartDate): startItems = seasonalStartDate.split('/') startDT = datetime.datetime(1958, int(startItems[0]), int(startItems[1])) startDayno = int(startDT.strftime('%j')) if len(seasonalEndDate): endItems = seasonalEndDate.split('/') endDT = datetime.datetime(1958, int(endItems[0]), int(endItems[1])) endDayno = int(endDT.strftime('%j')) return('dayno,%i,%i' % (startDayno, endDayno)) def _getLock(self, filename): """_getLock is a private helper function that provides exclusive access to filename via a locking file. Inputs: filename = the file that exclusive access is required to. Returns: None Affects: Writes file filename + .LCK as a lock mechanism Exceptions: MadrigalError thrown if unable to write lock file Notes: Will sleep for 1 second at a time, for a maximum of _MaxSleep seconds (presently 10) if the file is not modified. After each second, it will check for the lock file to be removed or modified. If it was modified, it resets the count to 0 sec and starts counting again. After _MaxSleep counts it then assumes lock file is orphaned and returns. Orphaned file will be removed when dropLock is called. """ gotLock = 0 numTries = 0 modificationTime = 0 while (not gotLock): try: file = os.open(filename + '.LCK', os.O_RDWR | os.O_CREAT | os.O_EXCL) os.close(file) gotLock = 1 except OSError, (errno, strerror): # error 17 is "File exists" if errno != 17: raise madrigal.admin.MadrigalError("Unable to open " + filename + ".LCK as locking file ", None) # get modification time - may throw an error if file has disappearred try: newModTime = (os.stat(filename + '.LCK')).st_mtime except: #file has disappeared, no need to sleep continue # if the lock file has been modified (or if this is the first time through) set numTries = 0 if newModTime > modificationTime: modificationTime = newModTime numTries = 0 time.sleep(1) numTries = numTries + 1 if numTries > self._MaxSleep: return def _dropLock(self, filename): """_dropLock is a private helper function that drops exclusive access to filename via a locking file. Inputs: filename = the file that exclusive access is required to. Returns: None Affects: Removes file filename + .LCK as a lock mechanism Exceptions: None. """ try: os.remove(filename + '.LCK') except IOError: return class MadrigalWebFormat: """MadrigalWebFormat defines the format of an web interface. Information about how a web page is formatted is stored in this class. In particular, the possible derived parameters to display for a given format (such as Short or Comprehensive) are set in this class. Edit this class to create new formats or modify existing ones. Non-standard Python modules used: None No exceptions thrown Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Oct. 29, 2001 """ # constants # Edit this data to change which parameters to display # or to add new formats # # Format Parameters # ------ ---------- _privateDict = {'Comprehensive': [ 'year', 'month', 'day', 'bmonth', 'bday', 'hour', 'min', 'sec', 'md', 'dayno', 'bhm', 'bhhmmss', 'ehhmmss', 'uth', 'b_uth', 'ut', 'ut1_unix', 'ut2_unix', 'beg_ut', 'slt', 'sltc', 'fyear', 'sunrise_hour', 'sunset_hour', 'conj_sunrise_h', 'conj_sunset_h', 'aplt', 'julian_date', 'gdalt', 'range', 'resl', 'azm', 'az1', 'az2', 'elm', 'el1', 'el2', 'gdlat', 'glon', 'szen', 'szenc', 'sdwht', 'beamid', 'bn', 'be', 'bd', 'magh', 'magd', 'magzu', 'bmag', 'bdec', 'binc', 'lshell', 'diplat', 'invlat', 'aplat', 'aplon', 'e_reg_s_lat', 'e_reg_s_lon', 'e_reg_s_sdwht', 'e_reg_n_lat', 'e_reg_n_lon', 'e_reg_n_sdwht', 'magconjlat', 'magconjlon', 'magconjsdwht', 'mlt', 'tsyg_eq_xgsm', 'tsyg_eq_ygsm', 'tsyg_eq_xgse', 'tsyg_eq_ygse', 'aacgm_lat', 'aacgm_long', 'aspect', 'cxr', 'cyr', 'czr', 'pl', 'snp3', 'chisq', 'gfit', 'mhdqc1', 'systmp', 'systmi', 'power', 'tfreq', 'popl', 'ne', 'nel', 'ti', 'te', 'tr', 'vo', 'ph+', 'pm', 'co', 'vdopp', 'dvdopp', 'dco', 'dpm', 'dph+', 'dvo', 'dtr', 'dte', 'dti', 'dpopl', 'dne', 'ne_model', 'nel_model', 'te_model', 'ti_model', 'vo_model', 'hmax_model', 'nmax_model', 'ne_modeldiff', 'nel_modeldiff', 'te_modeldiff', 'ti_modeldiff', 'vo_modeldiff', 'tn', 'tnm', 'tinfm', 'mol', 'nn2l', 'no2l', 'nol', 'narl', 'nhel', 'nhl', 'nn4sl', 'fa', 'pnrmd', 'pnrmdi', 'ut1', 'ut2', 'dut21', 'kinst', 'recno', 'kindat', 'fof2', 'dfa', 'dst', 'kp', 'ap', 'ap3', 'f10.7', 'fbar', 'pdcon', 'dpdcon', 'hlcon', 'dhlcon', 'ne_iri', 'nel_iri', 'tn_iri', 'te_iri', 'ti_iri', 'po+_iri', 'pno+_iri', 'po2+_iri', 'phe+_iri', 'ph+_iri', 'pn+_iri', 'bxgsm', 'bygsm', 'bzgsm', 'bimf', 'bxgse', 'bygse', 'bzgse', 'swden', 'swspd', 'swq'], 'Short': [ 'year', 'md', 'dayno', 'uth', 'b_uth', 'ut', 'beg_ut', 'lt', 'aplt', 'jdayno', 'gdalt', 'range', 'azm', 'az1', 'az2', 'elm', 'el1', 'el2', 'gdlat', 'glon', 'popl', 'nel', 'ti', 'te', 'tr', 'vo', 'ph+', 'pm', 'co', 'vdopp', 'dvdopp', 'dco', 'dpm', 'dph+', 'dvo', 'dtr', 'dte', 'dti', 'dpopl', 'dne', 'kp', 'ap', 'ap3', 'f10.7', 'fbar']} def getFormat(self, formatName): return self._privateDict[formatName]
Classes
class MadrigalWeb
MadrigalWeb is the class that produces output for the web.
All text written to the web is produced in this class.
Non-standard Python modules used: None
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Dec. 17, 2001
class MadrigalWeb: """MadrigalWeb is the class that produces output for the web. All text written to the web is produced in this class. Non-standard Python modules used: None Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Dec. 17, 2001 """ _MaxSleep = 10 def __init__(self, madDB = None): """__init__ initializes MadrigalWeb by reading from MadridalDB.. Inputs: Existing MadrigalDB object, by default = None. Returns: void Affects: Initializes self._metaDir, self._logFile. Exceptions: None. """ if madDB == None: self._madDB = madrigal.metadata.MadrigalDB() else: self._madDB = madDB self._binDir = self._madDB.getBinDir() self._instObj = madrigal.metadata.MadrigalInstrument(self._madDB) self._madKindatObj = madrigal.metadata.MadrigalKindat(self._madDB) metaDir = self._madDB.getMetadataDir() # get todays year now = datetime.datetime.now() thisYear = '%04i' % (now.year) self._logFile = os.path.join(metaDir, 'userdata', 'access_%s.log' % (thisYear)) # be sure it exists if not os.access(self._logFile, os.R_OK): f=open(self._logFile, 'w') f.close() # keep track of whether user trusted self._isTrusted_ = None # cache Madrigal objects as needed to imprive performance self._madExpObjExpID = None # will be set to a MadrigalExperiment object sorted by expId when first needed self._madExpObjDate = None # will be set to a MadrigalExperiment object sorted by date when first needed def getRulesOfTheRoad(self, PI=None, PIEmail=None): """ getRulesOfTheRoad returns a string giving the rules in html formal for using madrigal data. Inputs: PI - contact name. Default is site name. PIEmail - email link. Default is site admin. Returns: a string giving the rules in html formal for using madrigal data Affects: None. Exceptions: None. """ if not PI or not PIEmail: # get the site name siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteID = self._madDB.getSiteID() contactName = str(siteObj.getSiteName(siteID)) contactEmail = str(siteObj.getSiteEmail(siteID)) else: contactName = str(PI) contactEmail = str(PIEmail) returnStr = 'Please contact %s at ' % (contactName) returnStr = returnStr + '<a href="mailto:' + contactEmail + '">' + \ contactEmail + '</a> before using this data in a report or publication.' return returnStr def generateLogout(self, fileName, expName): """ generateLogout generates a java script which sends a user to the madLogin page to logout automatically. Inputs: fileName: the madrigal file to return to expName: the experiment name of the file to return to Returns: a java script which sends a user to the madLogin page to logout automatically Affects: None. Exceptions: None. """ print '<script language = "JavaScript">' print '\twindow.location = "madLogin?fileName=' + \ fileName + '&expName=' + \ expName + '&state=autoLogout"' print '</script>' def isTrusted(self): """ isTrusted returns 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise. Inputs: None Returns: 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise. Also returns 0 if no browser ip available or trustedIPs.txt cannot be opened. Affects: None. Exceptions: None. """ if self._isTrusted_ != None: return(self._isTrusted_) try: trustFile = open(self._madDB.getMadroot() + '/trustedIPs.txt', 'r') except: return 0 # try to read env var REMOTE_ADDR and HTTP_X_FORWARDED_FOR userIPList = [] if os.environ.get('REMOTE_ADDR')!= None: userIPList.append(os.environ.get('REMOTE_ADDR')) if os.environ.get('HTTP_X_FORWARDED_FOR') != None: ips = os.environ.get('HTTP_X_FORWARDED_FOR').split(',') for ip in ips: userIPList.append(ip.strip()) if len(userIPList) == 0: self._isTrusted_ = 0 return 0 if len(userIPList[0]) < 7: # ip address too short self._isTrusted_ = 0 return 0 # loop through trustedIPs.txt to find a match ipList = trustFile.readlines() for userIP in userIPList: for ipItem in ipList: # match using filename matching with * if fnmatch.fnmatch(userIP, ipItem.strip()): self._isTrusted_ = 1 return 1 # out of loop, no match found self._isTrusted_ = 0 return 0 def logDataAccess(self, fullFilenameList, user_fullname=None, user_email=None, user_affiliation=None): """ logDataAccess logs queries that access low-level data. Records user name, email, affiliation, datetime, and full path the file(s) accessed. Inputs: fullFilenameList either a list of full filenames, or a string with one filename user_fullname - if None, try to read from cookie. Also, any commas replaced by spaces. user_email - if None, try to read from cookie. Also, any commas replaced by spaces. user_affiliation - if None, try to read from cookie. Also, any commas replaced by spaces. Outputs: None Affects: Write line to log file with 5 or more comma-delimited columns. Example: Bill Rideout,brideout@haystack.mit.edu,MIT Haystack,2002-12-25 00:00:00, \ /opt/madrigal/experiments/2005/mlh/01sep05/mlh050901g.001,/opt/madrigal/experiments/2005/mlh/02sep05/mlh050902g.001 Uses _getLock and _dropLock to ensure single users access to log file """ if user_fullname == None or user_email == None or user_affiliation == None: # try to get name, email, affiliation from cookie cookie = Cookie.SimpleCookie() if os.environ.has_key('HTTP_COOKIE'): cookie.load(os.environ['HTTP_COOKIE']) try: user_fullname = cookie["user_fullname"].value user_email = cookie["user_email"].value user_affiliation = cookie["user_affiliation"].value except: # no way to write log return if user_fullname == None or user_email == None or user_affiliation == None: return # strip out any commas user_fullname = user_fullname.replace(',', ' ') user_email = user_email.replace(',', ' ') user_affiliation = user_affiliation.replace(',', ' ') if type(fullFilenameList) in (types.ListType, types.TupleType): delimiter = ',' fileStr = delimiter.join(fullFilenameList) else: fileStr = str(fullFilenameList) now = datetime.datetime.now() nowStr = now.strftime('%Y-%m-%d %H-%M-%S') # lock out any method that writes to log file self._getLock(self._logFile) f = open(self._logFile, 'a') f.write('%s,%s,%s,%s,%s\n' % (user_fullname.encode('utf8'), user_email.encode('utf8'), user_affiliation.encode('utf8'), nowStr, fileStr)) f.close() # done with log file - allow access to other writing calls self._dropLock(self._logFile) def filterLog(self, tmpFile, kinstList=None, accessStartDate=None, accessEndDate=None): """filterLog writes a subsection of the access log to a temporary file Inputs: tmpFile - temporary file to write subsection of log to kinstList - list of kinsts to accept. If None (the default), accept all instruments accessStartDate - if not None (the default), reject all access dates before datetime accessStartDate accessEndDate - if not None (the default), reject all access dates after datetime accessEndDate """ f = open(tmpFile, 'w') accessLogs = glob.glob(os.path.join(self._madDB.getMadroot(), 'metadata/userdata/access_*.log')) accessLogs.sort() # addition for cedar only """accessLogs2 = glob.glob('/opt/cedar/metadata/userdata/access_*[0-9].log') accessLogs += accessLogs2 accessLogs.sort()""" if kinstList: # create a dictionary of key = 3 letter inst mnem, value = kinstList instDict = {} instList = self._instObj.getInstrumentList() for inst in instList: if instDict.has_key(inst[1]): instDict[inst[1]].append(inst[2]) else: instDict[inst[1]] = [inst[2]] for accessLog in accessLogs: # see if we can skip this year basename = os.path.basename(accessLog) year = int(basename[7:-4]) startYear = datetime.datetime(year,1,1,0,0,0) endYear = datetime.datetime(year,12,31,23,59,59) if accessStartDate: if accessStartDate > endYear: continue if accessEndDate: if accessEndDate < startYear: continue # this file can be huge, so read one line at a time fl = open(accessLog) while True: line = fl.readline() if len(line) == 0: break items = line.strip().split(',') if len(items) != 5: continue # walk through filters # kinst if kinstList: # get the instrument mnemonic dirs = items[-1].split('/') found = False try: for kinst in instDict[dirs[-3]]: if kinst in kinstList: found = True break except KeyError: continue if not found: continue # access time if accessStartDate or accessEndDate: thisDT = datetime.datetime.strptime(items[-2], '%Y-%m-%d %H-%M-%S') if accessStartDate: if accessStartDate > thisDT: continue if accessEndDate: if accessEndDate < thisDT: continue # all filters passed f.write(line) fl.close() f.close() def createGlobalIsprintCmd(self, language, madrigalUrl, parmList, output, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, filterList, kindatList, expName, fileDesc, seasonalStartDate, seasonalEndDate, format=None): """createGlobalIsprintCmd returns a string representing a global isprint command to run in a particular language. Inputs: language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is parmList - ordered list of parameters requested. output - output file name user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) filterList - a list of strings in form "mnem,lower,upper" where lower and/or upper may be empty kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description. seasonalStartDate - a string in form 'MM/DD'. Dates before then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal start date. seasonalEndDate - a string in form 'MM/DD'. Dates after then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal end date. format - 'hdf5', 'ascii', or 'netCDF4'. If None, not specified (Madrigal 2 does not support this) """ if language not in ('python', 'Matlab', 'IDL'): raise ValueError, 'language %s not supported' % (str(language)) # url if language == 'python': cmd = 'globalIsprint.py --verbose --url=%s ' % (madrigalUrl) elif language == 'Matlab': cmd = "globalIsprint('%s', ...\n " % (madrigalUrl) elif language == 'IDL': cmd = "madglobalprint, '%s', $\n " % (madrigalUrl) # parms if len(parmList) == 0: raise ValueError, 'parmList cannot be empty' parmStr = '' for parm in parmList: parmStr += str(parm) if parm != parmList[-1]: parmStr += ',' if language == 'python': cmd += '--parms=%s ' % (parmStr) elif language == 'Matlab': cmd += "'%s', ...\n " % (parmStr) elif language == 'IDL': cmd += "'%s', $\n " % (parmStr) # output if language == 'python': cmd += '--output=%s ' % (output) elif language == 'Matlab': cmd += "'%s', ...\n " % (output) elif language == 'IDL': cmd += "'%s', $\n " % (output) # user_fullname if language == 'python': cmd += '--user_fullname="%s" ' % (user_fullname) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_fullname) elif language == 'IDL': cmd += "'%s', $\n " % (user_fullname) # user_email if language == 'python': cmd += '--user_email=%s ' % (user_email) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_email) elif language == 'IDL': cmd += "'%s', $\n " % (user_email) # user_affiliation if language == 'python': cmd += '--user_affiliation="%s" ' % (user_affiliation) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_affiliation) elif language == 'IDL': cmd += "'%s', $\n " % (user_affiliation) # start_datetime if language == 'python': cmd += '--startDate="%s" ' % (start_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (start_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (start_datetime.month, start_datetime.day, start_datetime.year, start_datetime.hour, start_datetime.minute, start_datetime.second) # end_datetime if language == 'python': cmd += '--endDate="%s" ' % (end_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (end_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (end_datetime.month, end_datetime.day, end_datetime.year, end_datetime.hour, end_datetime.minute, end_datetime.second) # instrument if language == 'python': cmd += '--inst=%i ' % (instCode) elif language == 'Matlab': cmd += "%i, ...\n " % (instCode) elif language == 'IDL': cmd += "%i, $\n " % (instCode) # format is here for Matlab or python if language in ('Matlab', 'python'): if language == 'python': if not format is None: if format.lower() == 'hdf5': cmd += '--format=%s ' % ('Hdf5') elif format in ('netCDF4', 'ascii'): cmd += '--format=%s ' % (format) elif language == 'Matlab': if format is None: cmd += "'%s', ...\n " % ('ascii') elif format.lower() == 'hdf5': cmd += "'%s', ...\n " % ('Hdf5') elif format in ('netCDF4', 'ascii'): cmd += "'%s', ...\n " % (format) # filterList # add seasonal filters if needed if len(seasonalStartDate) or len(seasonalEndDate): daynoFilterStr = self._getDaynoFilter(seasonalStartDate, seasonalEndDate) filterList.append(daynoFilterStr) if language == 'python': for filterItem in filterList: cmd += '--filter=%s ' % (filterItem) elif language == 'Matlab': filterStr = '' for filterItem in filterList: filterStr += 'filter=%s ' % (filterItem) cmd += "'%s', ...\n " % (filterStr) elif language == 'IDL': filterStr = '' for filterItem in filterList: filterStr += 'filter=%s ' % (filterItem) cmd += "'%s', $\n " % (filterStr) # kindatList if language == 'python': if len(kindatList) == 0: pass else: kindatStr = '--kindat=' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' cmd += '%s ' % (kindatStr) elif language == 'Matlab': kindatStr = '[' for kindat in kindatList: if kindat == 0: continue kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, ...\n " % (kindatStr) elif language == 'IDL': # make sure 0 not in list try: kindatList.remove(0) except ValueError: pass if len(kindatList) == 0: kindatStr = 'PTR_NEW()' else: kindatStr = '[' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, $\n " % (kindatStr) # expName if language == 'python': if len(expName) > 0: cmd += '--expName="%s" ' % (expName) elif language == 'Matlab': expName = expName.replace('*', '.*') expName = expName.replace('?', '.?') cmd += "'%s', ...\n " % (expName) elif language == 'IDL': cmd += "'%s', $\n " % (expName) # fileDesc if language == 'python': if len(fileDesc) > 0: cmd += '--fileDesc="%s" ' % (fileDesc) elif language == 'Matlab': fileDesc = fileDesc.replace('*', '.*') fileDesc = fileDesc.replace('?', '.?') cmd += "'%s') " % (fileDesc) elif language == 'IDL': cmd += "'%s', $\n " % (fileDesc) # format is here for idl if language == 'IDL': if format is None: cmd += "'ascii', $\n " elif format.lower() == 'hdf5': cmd += "'hdf5', $\n " elif format in ('netCDF4', 'ascii'): cmd += "'%s', $\n " % (format) return(cmd) def createGlobalDownloadCmd(self, language, madrigalUrl, output, format, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, kindatList, expName, fileDesc): """createGlobalDownloadCmd returns a string representing a global download as is command to run in a particular language. Inputs: language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is output - output directory name format - 'hdf5', 'ascii', or 'netCDF4' user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description. """ if language not in ('python', 'Matlab', 'IDL'): raise ValueError, 'language %s not supported' % (str(language)) # url if language == 'python': cmd = 'globalDownload.py --verbose --url=%s ' % (madrigalUrl) elif language == 'Matlab': cmd = "globalDownload('%s', ...\n " % (madrigalUrl) elif language == 'IDL': cmd = "madglobaldownload, '%s', $\n " % (madrigalUrl) # output if language == 'python': cmd += '--outputDir=%s ' % (output) elif language == 'Matlab': cmd += "'%s', ...\n " % (output) elif language == 'IDL': cmd += "'%s', $\n " % (output) # user_fullname if language == 'python': cmd += '--user_fullname="%s" ' % (user_fullname) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_fullname) elif language == 'IDL': cmd += "'%s', $\n " % (user_fullname) # user_email if language == 'python': cmd += '--user_email=%s ' % (user_email) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_email) elif language == 'IDL': cmd += "'%s', $\n " % (user_email) # user_affiliation if language == 'python': cmd += '--user_affiliation="%s" ' % (user_affiliation) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_affiliation) elif language == 'IDL': cmd += "'%s', $\n " % (user_affiliation) # format part 1 (format is not in same order in Matlab and IDL) if format not in ('hdf5', 'ascii', 'netCDF4'): raise ValueError, 'format not in hdf5, ascii or netCDF4' if language == 'python': cmd += '--format="%s" ' % (format) elif language == 'Matlab': cmd += "'%s', ...\n " % (format) # start_datetime if language == 'python': cmd += '--startDate="%s" ' % (start_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (start_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (start_datetime.month, start_datetime.day, start_datetime.year, start_datetime.hour, start_datetime.minute, start_datetime.second) # end_datetime if language == 'python': cmd += '--endDate="%s" ' % (end_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (end_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (end_datetime.month, end_datetime.day, end_datetime.year, end_datetime.hour, end_datetime.minute, end_datetime.second) # instrument if language == 'python': cmd += '--inst=%i ' % (instCode) elif language == 'Matlab': cmd += "%i, ...\n " % (instCode) elif language == 'IDL': cmd += "%i, $\n " % (instCode) # kindatList if language == 'python': if len(kindatList) == 0: pass else: kindatStr = '--kindat=' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' cmd += '%s ' % (kindatStr) elif language == 'Matlab': kindatStr = '[' for kindat in kindatList: if kindat == 0: continue kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, ...\n " % (kindatStr) elif language == 'IDL': # make sure 0 not in list try: kindatList.remove(0) except ValueError: pass if len(kindatList) == 0: kindatStr = 'PTR_NEW()' else: kindatStr = '[' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, $\n " % (kindatStr) # now deal with IDL format if needed if language == 'IDL': cmd += "'%s', $\n " % (format) # expName if language == 'python': if len(expName) > 0: cmd += '--expName="%s" ' % (expName) elif language == 'Matlab': expName = expName.replace('*', '.*') expName = expName.replace('?', '.?') cmd += "'%s', ...\n " % (expName) elif language == 'IDL': cmd += "'%s', $\n " % (expName) # fileDesc if language == 'python': if len(fileDesc) > 0: cmd += '--fileDesc="%s" ' % (fileDesc) elif language == 'Matlab': fileDesc = fileDesc.replace('*', '.*') fileDesc = fileDesc.replace('?', '.?') cmd += "'%s') " % (fileDesc) elif language == 'IDL': cmd += "'%s' " % (fileDesc) return(cmd) def generateGlobalIsprintScriptFromForm(self, form1, form2, form3, user_fullname, user_email, user_affiliation): """generateGlobalIsprintScriptFromForm converts the three Django forms into arguments so that if can then call createGlobalIsprintCmd. Separate forms used because some parts are created by Ajax. form1 is a dict with keys: instruments, start_date, end_date, format_select, directory_select, language_select, kindat_select, expName, fileDesc form2 is a dict with keys parameters form3 is a dict with keys parm_#, parm_#_lower, parm_#_upper, where # is 1, 2, and 3 user_fullname, user_email, user_affiliation - strings """ instCode = int(form1['instruments']) start_datetime = datetime.datetime(form1['start_date'].year, form1['start_date'].month, form1['start_date'].day) end_datetime = datetime.datetime(form1['end_date'].year, form1['end_date'].month, form1['end_date'].day) format = form1['format_select'] if format == 'ascii' and form1['directory_select'] == 'File': output = 'example.txt' else: output = '/tmp' language = form1['language_select'] kindatList = [int(kindat) for kindat in form1['kindat_select']] expName = form1['expName'].strip() fileDesc = form1['fileDesc'].strip() seasonalStartDay = int(form1['seasonalStartDay']) seasonalStartMonth = int(form1['seasonalStartMonth']) seasonalEndDay = int(form1['seasonalEndDay']) seasonalEndMonth = int(form1['seasonalEndMonth']) if seasonalStartDay == 1 and seasonalStartMonth == 1 and \ seasonalEndDay == 31 and seasonalEndMonth == 12: seasonalStartDate = '' seasonalEndDate = '' else: seasonalStartDate = '%02i/%02i' % (seasonalStartMonth, seasonalStartDay) seasonalEndDate = '%02i/%02i' % (seasonalEndMonth, seasonalStartDay) madrigalUrl = self._madDB.getTopLevelUrl() parmList = form2['parameters'] filterList = [] for i in (1,2,3): try: parm = form3['parm_%i' % (i)] except KeyError: continue if len(parm) == 0: continue filterStr = '%s,' % (parm) try: parm_lower = form3['parm_%i_lower' % (i)] filterStr += '%s,' % (str(parm_lower)) except KeyError: filterStr += ',' try: parm_upper = form3['parm_%i_upper' % (i)] filterStr += '%s' % (str(parm_upper)) except KeyError: pass filterList.append(filterStr) return(self.createGlobalIsprintCmd(language, madrigalUrl, parmList, output, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, filterList, kindatList, expName, fileDesc, seasonalStartDate, seasonalEndDate, format)) def generateDownloadFileScriptFromForm(self, form, user_fullname, user_email, user_affiliation): """generateDownloadFileScriptFromForm converts the Django form into arguments so that if can then call createGlobalDownloadCmd. form is a dict with keys: instruments, start_date, end_date, format_select, language_select, kindat_select, expName, fileDesc user_fullname, user_email, user_affiliation - strings """ instCode = int(form['instruments']) start_datetime = datetime.datetime(form['start_date'].year, form['start_date'].month, form['start_date'].day) end_datetime = datetime.datetime(form['end_date'].year, form['end_date'].month, form['end_date'].day) format = form['format_select'] language = form['language_select'] kindatList = [int(kindat) for kindat in form['kindat_select']] expName = form['expName'].strip() fileDesc = form['fileDesc'].strip() madrigalUrl = self._madDB.getTopLevelUrl() return(self.createGlobalDownloadCmd(language, madrigalUrl, '/tmp', format, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, kindatList, expName, fileDesc)) def getSingleRedirectList(self): """getSingleRedirectList returns a list with tuples (kinst, url) where url is url to redirect single UI to if instrument not local. If no redirect needed because instrument local, url is empty string """ madInstData = madrigal.metadata.MadrigalInstrumentData(self._madDB, True) siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteID = self._madDB.getSiteID() # create a dict with key = siteID, value = redirect url for speed getStr = '?isGlobal=True&categories=%i&instruments=%i' addUrl = django.core.urlresolvers.reverse('view_single') cedarUrl = 'http://cedar.openmadrigal.org/' + addUrl + getStr siteDict = {} for thisSiteID, siteDesc in siteObj.getSiteList(): if thisSiteID == siteID: siteDict[thisSiteID] = '' # local case elif siteObj.getSiteVersion(thisSiteID) == '2.6': # redirect to cedar because other site below Madrigal 3 siteDict[thisSiteID] = cedarUrl else: siteDict[thisSiteID] = os.path.join(siteObj.getSiteServer(thisSiteID), siteObj.getSiteDocRoot(thisSiteID), addUrl + getStr) retList = [] for kinst, desc, thisSiteID in madInstData.getInstruments(): if len(siteDict[thisSiteID]) > 0: url = siteDict[thisSiteID] % (self._instObj.getCategoryId(kinst), kinst) else: url = '' retList.append((kinst, url)) return(retList) def getMonths(self, kinst, year, optimize=True): """getMonths returns a list of tuples of (monthNumber, monthName) where monthNumber is 1-12, and monthName is the form January, Febuary, etc. for the the months where there is local data for kinst & year combination Inputs: kinst - instrument id (int) year - year (int) optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning """ tempDict = {} # dict with key = month number, value = month name sDT = datetime.datetime(year,1,1) eDT = datetime.datetime(year,12,31,23,59,59) madroot = self._madDB.getMadroot() if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue if thisSDT.year == year: startMonth = thisSDT.month else: startMonth = 1 if thisEDT.year == year: endMonth = thisEDT.month else: endMonth = 12 monthList = range(startMonth, endMonth + 1) for thisMonth in monthList: if thisMonth not in tempDict.keys(): tempDict[thisMonth] = calendar.month_name[thisMonth] monthKeys = tempDict.keys() monthKeys.sort() retList = [(monthKey, tempDict[monthKey]) for monthKey in monthKeys] return(retList) def getDays(self, kinst, year, month=None, optimize=True): """getDays returns a sorted list of datetime.date objects where there is local data for kinst & year & possibly month combination Inputs: kinst - instrument id (int) year - year (int) month (1-12) if None, include all months optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning """ retList = [] sDT = datetime.datetime(year,1,1) eDT = datetime.datetime(year,12,31,23,59,59) if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue # loop over all days delta = datetime.timedelta(days=1) loopDT = max(sDT, datetime.datetime(thisSDT.year, thisSDT.month, thisSDT.day)) while (loopDT <= thisEDT): if loopDT > eDT: break if not month is None: if month > loopDT.month: loopDT += delta continue elif month < loopDT.month: break thisDate = loopDT.date() if thisDate not in retList: retList.append(thisDate) loopDT += delta retList.sort() return(retList) def getExperimentList(self, kinstList, startDT, endDT, localOnly): """getExperimentList returns a sorted list of tuples of (expId, expUrl, expName, instName, kinst, expStartDT, expEndDT, siteId, siteName) Inputs: kinstList - a list of instrument id (int) - may include 0 startDT - start datetime to search endDT - end datetime to search localOnly - if True, only search locally. If False, search globally """ retList = [] madroot = self._madDB.getMadroot() siteId = self._madDB.getSiteID() siteObj = madrigal.metadata.MadrigalSite(self._madDB) if localOnly: expTabFile = os.path.join(madroot, 'metadata/expTab.txt') else: expTabFile = os.path.join(madroot, 'metadata/expTabAll.txt') madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB, expTabFile) startIndex = madExpObjDate.getStartPosition(startDT) for i in range(startIndex, madExpObjDate.getExpCount()): thisKinst = madExpObjDate.getKinstByPosition(i) if thisKinst not in kinstList and 0 not in kinstList: continue thisSDTList = madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < startDT: continue if thisSDT > endDT: break # check for security security = madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue thisSiteId = madExpObjDate.getExpSiteIdByPosition(i) if siteId == thisSiteId: isLocal = True else: isLocal = False thisExpId = madExpObjDate.getExpIdByPosition(i) thisExpPath = madExpObjDate.getExpPathByPosition(i) thisExpName = madExpObjDate.getExpNameByPosition(i) if isLocal: thisExpUrl = django.core.urlresolvers.reverse('show_experiment') + \ '?experiment_list=%i' % (thisExpId) elif siteObj.getSiteVersion(thisSiteId) == '2.6': thisExpUrl = 'http://' + os.path.join(siteObj.getSiteServer(thisSiteId), siteObj.getSiteRelativeCGI(thisSiteId), 'madExperiment.cgi?exp=%s' % (thisExpPath)) thisExpUrl += '&displayLevel=0&expTitle=%s' % (django.utils.http.urlquote(thisExpName)) else: # remote Madrigal 3.0 site baseUrl = os.path.basename(django.core.urlresolvers.reverse('show_experiment')[:-1]) thisExpUrl = baseUrl + '?experiment_list=%s' % (thisExpPath) thisSiteUrl = 'http://%s' % (siteObj.getSiteServer(thisSiteId)) relativeUrl = siteObj.getSiteDocRoot(thisSiteId) if relativeUrl not in ('', None): thisSiteUrl = os.path.join(thisSiteUrl, relativeUrl) if thisSiteUrl[-1] != '/' and thisExpUrl[0] != '/': thisSiteUrl += '/' thisExpUrl = thisSiteUrl + thisExpUrl thisSiteName = siteObj.getSiteName(thisSiteId) instName = self._instObj.getInstrumentName(thisKinst) retList.append((thisExpId, thisExpUrl, thisExpName, instName, thisKinst, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S'), thisSiteId, thisSiteName)) return(retList) def getExpsOnDate(self, kinst, year, month, day, optimize=True): """getExpsOnDate returns a sorted list of tuples of (expId, expDesc, expDir, pi_name, pi_email) Inputs: kinst - instrument id (int) year - year (int) month - month (int) day - day (int) optimize - if True, only start search at beginning of day. But may miss long experiments, so if optimization if False, starts at beginning """ retList = [] sDT = datetime.datetime(year,month,day) eDT = datetime.datetime(year,month,day,23,59,59) if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue thisExpId = self._madExpObjDate.getExpIdByPosition(i) thisExpName = self._madExpObjDate.getExpNameByPosition(i) thisExpDesc = '%s: %s-%s' % (thisExpName, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S')) thisExpDir = self._madExpObjDate.getExpDirByPosition(i) thisExpPI = self._madExpObjDate.getPIByPosition(i) thisExpPIEmail = self._madExpObjDate.getPIEmailByPosition(i) if thisExpPI in (None, ''): thisExpPI = self._instObj.getContactName(kinst) thisExpPIEmail = self._instObj.getContactEmail(kinst) retList.append((thisExpId, thisExpDesc, thisExpDir)) return(retList) def getFileFromExpDir(self, expDir, kinst, includeNonDefault=False): """getFileFromExpDir returns a list of tuples of (basename, fileDesc) Inputs: expDir - full path to exp directory kinst - instrument id (used to look up kindat descriptions) includeNonDefault - if True, include variant and history files. If False (the default), do not """ retList = [] realTimeList = [] # in case no default files if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK): # no files in this experiment return(retList) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) for i in range(madFileObj.getFileCount()): if madFileObj.getAccessByPosition(i) == 1 and not self.isTrusted(): continue category = madFileObj.getCategoryByPosition(i) if category in (2,3) and not includeNonDefault: continue basename = madFileObj.getFilenameByPosition(i) kindat = madFileObj.getKindatByPosition(i) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) status = madFileObj.getStatusByPosition(i) fileDesc = '%s: %s - %s' % (basename, kindatDesc, status) if category != 4: retList.append((basename, fileDesc)) else: realTimeList.append((basename, fileDesc)) if len(retList) > 0: return(retList) else: return(realTimeList) def getExpInfoFromExpID(self, expID): """getExpInfoFromExpID returns a tuple of (pi_name, pi_email, expUrl, kinst, expDesc, kinstDesc) for given expID expUrl is url from getRealExpUrlByExpId Inputs: expID - experimentID (int) """ expID = int(expID) if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) kinst = self._madExpObjExpID.getKinstByExpId(expID) kinstDesc = self._instObj.getInstrumentName(kinst) expPI = self._madExpObjExpID.getPIByExpId(expID) expPIEmail = self._madExpObjExpID.getPIEmailByExpId(expID) expUrl = self._madExpObjExpID.getRealExpUrlByExpId(expID) if expPI in (None, ''): expPI = self._instObj.getContactName(kinst) expPIEmail = self._instObj.getContactEmail(kinst) thisSDTList = self._madExpObjExpID.getExpStartDateTimeByExpId(expID) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjExpID.getExpEndDateTimeByExpId(expID) thisEDT = datetime.datetime(*thisEDTList[0:6]) thisExpName = self._madExpObjExpID.getExpNameByExpId(expID) thisExpDesc = '%s: %s-%s' % (thisExpName, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S')) return((expPI, expPIEmail, expUrl, kinst, thisExpDesc, kinstDesc)) def getExpIDFromExpPath(self, expPath, matchAnyExpNum=False): """getExpIDFromExpPath returns the expId for given expPath (starts with 'experiments') If matchAnyExpNum is False, it will only match the right experiments* directory (default). It True, matches via re to experiments[0-9]*/<remaining exp path> Returns None if not found Inputs: expPath - experiment path (starts with 'experiments') """ madExpObj = madrigal.metadata.MadrigalExperiment(self._madDB) if matchAnyExpNum: expPathRE = 'experiments[0-9]*/' + expPath[expPath.find('/')+1:] for i in range(madExpObj.getExpCount()): if not matchAnyExpNum: if madExpObj.getExpPathByPosition(i) == expPath: return(madExpObj.getExpIdByPosition(i)) else: if len(re.findall(expPathRE, madExpObj.getExpPathByPosition(i))) > 0: return(madExpObj.getExpIdByPosition(i)) return(None) def getInfoFromFile(self, filePath): """getInfoFromFile returns a tuple of (expName, kindatDesc) for a given input file """ expDir = os.path.dirname(filePath) basename = os.path.basename(filePath) madExpObj = madrigal.metadata.MadrigalExperiment(self._madDB, os.path.join(expDir, 'expTab.txt')) expName = madExpObj.getExpNameByPosition(0) kinst = madExpObj.getKinstByPosition(0) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) kindat = madFileObj.getKindatByFilename(basename) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) return((expName, kindatDesc)) def getFileFromExpID(self, expID, includeNonDefault=False): """getFileFromExpDir returns a list of tuples of (basename, fileDesc) Inputs: expID - experimentID (int) includeNonDefault - if True, include variant and history files. If False (the default), do not """ retList = [] realTimeList = [] # in case no default files if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) expDir = self._madExpObjExpID.getExpDirByExpId(expID) kinst = self._madExpObjExpID.getKinstByExpId(expID) if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK): # no files in this experiment return(retList) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) for i in range(madFileObj.getFileCount()): if madFileObj.getAccessByPosition(i) == 1 and not self.isTrusted(): continue category = madFileObj.getCategoryByPosition(i) if category in (2,3) and not includeNonDefault: continue if category == 2: categoryStr = '<variant file> ' elif category == 3: categoryStr = '<history file> ' else: categoryStr = '' basename = madFileObj.getFilenameByPosition(i) kindat = madFileObj.getKindatByPosition(i) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) status = madFileObj.getStatusByPosition(i) fileDesc = '%s: %s%s - %s' % (basename, categoryStr, kindatDesc, status) if category != 4: retList.append((basename, fileDesc)) else: realTimeList.append((basename, fileDesc)) if len(retList) > 0: return(retList) else: return(realTimeList) def getSiteInfo(self): """getSiteInfo returns a tuple of two items: 1. local site name 2. list of tuples of (siteName, url) of non-local sites """ siteID = self._madDB.getSiteID() siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteName = siteObj.getSiteName(siteID) retList = [] siteList = siteObj.getSiteList() for thisSiteID, thisSiteName in siteList: if thisSiteID == siteID: continue thisSiteServer = siteObj.getSiteServer(thisSiteID) thisSiteDocRoot = siteObj.getSiteDocRoot(thisSiteID) thisSiteUrl = urlparse.urlunparse(('http', thisSiteServer, thisSiteDocRoot, '','','')) retList.append((thisSiteName, thisSiteUrl)) return((siteName, retList)) def downloadFileAsIs(self, expId, basename, user_fullname, user_email, user_affiliation): """downloadFileAsIs returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters) Inputs: expId - experiment id of experiment basename - basename of file. May have .txt or .nc extension, in which case Hdf5 file is converted user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() hdf5Extensions = ('.hdf5', '.h5', '.hdf') fileName, fileExtension = os.path.splitext(basename) if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) expDir = self._madExpObjExpID.getExpDirByExpId(int(expId)) if expDir is None: raise ValueError, 'No expDir found for exp_id %i' % (int(expId)) if fileExtension in hdf5Extensions: baseHdf5 = basename else: # we need to search for the hdf5 basename madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) baseHdf5 = None for i in range(madFileObj.getFileCount()): thisFileName, thisFileExt = os.path.splitext(madFileObj.getFilenameByPosition(i)) if thisFileName == fileName and thisFileExt in hdf5Extensions: baseHdf5 = madFileObj.getFilenameByPosition(i) break if baseHdf5 is None: raise ValueError, 'No valid file for %s found in %s' % (basename, expDir) if basename == baseHdf5: fullFilename = os.path.join(expDir, basename) fullHdf5Filename = fullFilename tmpDir = None else: fullHdf5Filename = os.path.join(expDir, baseHdf5) # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass fullFilename = os.path.join(tmpDir, basename) if os.access(fullFilename, os.R_OK): try: os.remove(fullFilename) except: pass if fileExtension == '.txt': cachedFile = os.path.join(expDir, 'overview', baseHdf5 + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(fullHdf5Filename, fullFilename) elif fileExtension == '.nc': cachedFile = os.path.join(expDir, 'overview', baseHdf5 + '.nc') if os.access(cachedFile, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(fullHdf5Filename, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(fullHdf5Filename) cedarObj.write('netCDF4', fullFilename) # log access self.logDataAccess(fullHdf5Filename, user_fullname, user_email, user_affiliation) return(fullFilename) def downloadFullFileAsIs(self, fullHdf5Filename, format, user_fullname, user_email, user_affiliation): """downloadFullFileAsIs is similar to downloadFileAsIs with fullFilename input instead of expId . Returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters) Inputs: fullHdf5Filename - full path to Madrigal Hdf5 file format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() if format not in ('hdf5', 'netCDF4', 'ascii'): raise ValueError, 'Illegal format %s' % (str(format)) if not os.access(fullHdf5Filename, os.R_OK): raise IOError, 'Unable to find Hdf5 file %s' % (str(fullHdf5Filename)) if format == 'hdf5': fullFilename = fullHdf5Filename tmpDir = None else: # dynamically create file if format == 'netCDF4': thisExt = '.nc' elif format == 'ascii': thisExt = '.txt' # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass base, file_extension = os.path.splitext(fullHdf5Filename) basename = os.path.basename(base + thisExt) fullFilename = os.path.join(tmpDir, basename) if os.access(fullFilename, os.R_OK): try: os.remove(fullFilename) except: pass if format == 'ascii': cachedFile = os.path.join(os.path.dirname(fullHdf5Filename), 'overview', os.path.basename(fullHdf5Filename) + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(fullHdf5Filename, fullFilename) elif format == 'netCDF4': cachedFile = os.path.join(os.path.dirname(fullHdf5Filename), 'overview', os.path.basename(fullHdf5Filename) + '.nc') if os.access(cachedFile, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(fullHdf5Filename, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(fullHdf5Filename) cedarObj.write('netCDF4', fullFilename) # log access self.logDataAccess(fullHdf5Filename, user_fullname, user_email, user_affiliation) return(fullFilename) def downloadMultipleFiles(self, fileList, format, user_fullname, user_email, user_affiliation): """downloadMultipleFiles downloads multiple files in tarred format Returns a path to a Madrigal tarred file to download in given format as is (that is, with parms in file, and no filters) Inputs: fileList - list of full paths to Madrigal Hdf5 files format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() if format not in ('hdf5', 'netCDF4', 'ascii'): raise ValueError, 'Illegal format %s' % (str(format)) # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass # be sure no duplicate file names basenameList = [] if format == 'hdf5': finalFileList = [] for thisFile in fileList: basename = os.path.basename(thisFile) dirname = os.path.dirname(thisFile) if basename in basenameList: basename = self.modifyBasename(basename) fullFilename = os.path.join(tmpDir, basename) shutil.copy(thisFile, fullFilename) else: fullFilename = os.path.join(dirname, basename) basenameList.append(basename) finalFileList.append(fullFilename) else: # dynamically create files if format == 'netCDF4': thisExt = '.nc' elif format == 'ascii': thisExt = '.txt' finalFileList = [] for thisFile in fileList: if not os.access(thisFile, os.R_OK): raise IOError, 'Unable to find Hdf5 file %s' % (str(thisFile)) base, file_extension = os.path.splitext(thisFile) basename = os.path.basename(base + thisExt) if basename in basenameList: basename = self.modifyBasename(basename) basenameList.append(basename) fullFilename = os.path.join(tmpDir, basename) if format == 'ascii': cachedFile = os.path.join(os.path.dirname(thisFile), 'overview', os.path.basename(thisFile) + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' if not os.access(fullFilename, os.R_OK): shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(thisFile, fullFilename) elif format == 'netCDF4': cachedFile = os.path.join(os.path.dirname(thisFile), 'overview', os.path.basename(thisFile) + '.nc') if os.access(cachedFile, os.R_OK): if not os.access(fullFilename, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(thisFile, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(thisFile) cedarObj.write('netCDF4', fullFilename) finalFileList.append(fullFilename) # create tar file now = datetime.datetime.now() tar_filename = os.path.join(tmpDir, 'madrigalFiles_%s.tar' % (now.strftime('%Y%m%dT%H%M%S'))) tar = tarfile.open(tar_filename, "w") for thisFile in finalFileList: # log access self.logDataAccess(thisFile, user_fullname, user_email, user_affiliation) tar.add(thisFile) # let clean stage do this - safer return(tar_filename) def printFileAsIs(self, fullFilename, user_fullname, user_email, user_affiliation, html=True): """printFileAsIs returns the full path to a temp file representing file as plain text or html to print as is (that is, with parms rom file, and no filters) Inputs: fullFilename - full path to Madrigal Hdf5 file to convert to string user_fullname, user_email, user_affiliation - user identification strings html - if True (the default) return as Html with popup parm names. If False, pure text """ self.cleanStage() # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass fileName, fileExtension = os.path.splitext(fullFilename) fullTmpFilename = os.path.join(tmpDir, os.path.basename(fileName + '.txt')) if os.access(fullTmpFilename, os.R_OK): try: os.remove(fullTmpFilename) except: pass if html: summary = 'html' else: summary = 'plain' madrigal.cedar.convertToText(fullFilename, fullTmpFilename, summary=summary) # log access self.logDataAccess(fullFilename, user_fullname, user_email, user_affiliation) return(fullTmpFilename) def listRecords(self, fullFilename): """listRecords returns the list records html for fullFilename """ # check if record plots exist basename = os.path.basename(fullFilename) thisDir = os.path.dirname(fullFilename) pngFiles = glob.glob(os.path.join(thisDir, 'plots', basename, 'records/*.png')) if len(pngFiles) > 0: url = '<a href="javascript:plotRecno(%i)">View record plot</a>' else: url = None output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.cedar.listRecords(fullFilename, output, url) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip()) def downloadIsprintFileFromIsprintForm(self, isprintForm, user_fullname, user_email, user_affiliation): """downloadIsprintFileFromIsprintForm returns a full path to the temp file is experiments/stage created by isprint to download. Inputs: isprintForm - the django form that encapsulates all information from get_advanced web page. user_fullname, user_email, user_affiliation - user identification strings """ # defaults arguments showHeaders=False missing=None assumed=None knownbad=None orgFilename = isprintForm['fullFilename'] requestedParms = isprintForm['parameters'] # make unique uniqueRequestedParms = [] for s in requestedParms: asciiParm = s.encode('ascii').lower().strip() if asciiParm not in uniqueRequestedParms: uniqueRequestedParms.append(asciiParm) start_date = isprintForm['start_date'] end_date = isprintForm['end_date'] format = isprintForm['formats'] # create name of temp file basename_noext, ext = os.path.splitext(os.path.basename(orgFilename)) randint = random.randint(0,999999) if format in ('ascii', 'netCDF4'): # need new basename if format == 'ascii': basename = basename_noext + '_%06i.txt' % (randint) else: basename = basename_noext + '_%06i.nc' % (randint) else: basename = basename_noext + '_%06i.hdf5' % (randint) tmpFile = os.path.join(self._madDB.getMadroot(), 'experiments/stage', basename) if format == 'ascii': # reset defaults showHeaders = isprintForm['showHeaders'] missing = isprintForm['missing'] assumed = isprintForm['missing'] knownbad = isprintForm['missing'] # next task - create a list of filters, but only if actually modified madFilters = [] madFileObj = madrigal.data.MadrigalFile(orgFilename, self._madDB) # used to determine default values # check if we need a time filer earliestTime = madFileObj.getEarliestTime() latestTime = madFileObj.getLatestTime() earliestDT = datetime.datetime(*earliestTime) latestDT = datetime.datetime(*latestTime) earliest_unix = calendar.timegm(earliestDT.timetuple()) latest_unix = calendar.timegm(latestDT.timetuple()) start_unix = calendar.timegm(start_date.timetuple()) end_unix = calendar.timegm(end_date.timetuple()) if earliest_unix < start_unix or latest_unix > end_unix: # we need a time filter madFilters.append(madrigal.derivation.MadrigalFilter('ut1_unix', [(start_unix, end_unix)])) # altitude filter if isprintForm.has_key('min_alt'): file_min_alt = madFileObj.getMinValidAltitude() file_max_alt = madFileObj.getMaxValidAltitude() try: min_alt = float(isprintForm['min_alt']) except ValueError: min_alt = float('nan') try: max_alt = float(isprintForm['max_alt']) except ValueError: max_alt = float('nan') is_needed = False if not math.isnan(min_alt): if min_alt > file_min_alt + 1.0E-6: is_needed = True if not math.isnan(max_alt): if max_alt < file_max_alt - 1.0E-6: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('gdalt', [(min_alt, max_alt)])) # azimuth filter if isprintForm.has_key('min_az'): try: min_az = float(isprintForm['min_az']) except ValueError: min_az = float('nan') try: max_az = float(isprintForm['max_az']) except ValueError: max_az = float('nan') try: min_az2 = float(isprintForm['min_az2']) except ValueError: min_az2 = float('nan') try: max_az2 = float(isprintForm['max_az2']) except ValueError: max_az2 = float('nan') is_needed = False if not math.isnan(min_az): if min_az > -180.0: is_needed = True if not math.isnan(max_az): if max_az < 180.0: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('azm', [(min_az, max_az), (min_az2, max_az2)])) # elevation filter if isprintForm.has_key('min_el'): try: min_el = float(isprintForm['min_el']) except ValueError: min_el = float('nan') try: max_el = float(isprintForm['max_el']) except ValueError: max_el = float('nan') try: min_el2 = float(isprintForm['min_el2']) except ValueError: min_el2 = float('nan') try: max_el2 = float(isprintForm['max_el2']) except ValueError: max_el2 = float('nan') is_needed = False if not math.isnan(min_el): if min_el > 0.0: is_needed = True if not math.isnan(max_el): if max_el < 90.0: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('azm', [(min_el, max_el), (min_el2, max_el2)])) # pulse length filter if isprintForm.has_key('min_pl'): file_min_pl = madFileObj.getMinPulseLength() file_max_pl = madFileObj.getMaxPulseLength() try: min_pl = float(isprintForm['min_pl']) except ValueError: min_pl = float('nan') try: max_pl = float(isprintForm['max_pl']) except ValueError: max_pl = float('nan') is_needed = False if not math.isnan(min_pl): if min_pl > file_min_pl + 1.0E-9: is_needed = True if not math.isnan(max_pl): if max_pl < file_max_pl - 1.0E-9: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('pl', [(min_pl, max_pl)])) # free parameters for i in range(1, 4): parm_name = isprintForm['parm_%i' % (i)] parm_lower = isprintForm['parm_%i_lower' % (i)] parm_upper = isprintForm['parm_%i_upper' % (i)] if parm_name == 'None': continue if len(parm_lower) == 0 and len(parm_upper) == 0: continue # filter needed try: min_value = float(parm_lower) except ValueError: min_value = float('nan') try: max_value = float(parm_upper) except ValueError: max_value = float('nan') madFilters.append(madrigal.derivation.MadrigalFilter(parm_name, [(min_value, max_value)])) # create new temp file madrigal.isprint.Isprint(orgFilename, tmpFile, uniqueRequestedParms, madFilters, showHeaders=showHeaders, missing=missing, assumed=assumed, knownbad=knownbad) # log access self.logDataAccess(orgFilename, user_fullname, user_email, user_affiliation) return(tmpFile) def runMadrigalCalculatorFromForm(self, madCalculatorForm): """runMadrigalCalculatorFromForm returns the text output of madCalculator from the MadCalulatorForm Inputs: madCalculatorForm - the django form that encapsulates all information from madrigal_calculator web page. """ requestedParms = madCalculatorForm['parameters'] requestedParms = ['gdlat', 'glon', 'gdalt'] + [str(parm) for parm in requestedParms] thisDT = madCalculatorForm['datetime'] min_latitude = madCalculatorForm['min_latitude'] max_latitude = madCalculatorForm['max_latitude'] delta_latitude = madCalculatorForm['delta_latitude'] min_longitude = madCalculatorForm['min_longitude'] max_longitude = madCalculatorForm['max_longitude'] delta_longitude = madCalculatorForm['delta_longitude'] min_altitude = madCalculatorForm['min_altitude'] max_altitude = madCalculatorForm['max_altitude'] delta_altitude = madCalculatorForm['delta_altitude'] latList = numpy.arange(min_latitude, max_latitude+0.001*delta_latitude, delta_latitude).tolist() lonList = numpy.arange(min_longitude, max_longitude+0.001*delta_longitude, delta_longitude).tolist() altList = numpy.arange(min_altitude, max_altitude+0.001*delta_altitude, delta_altitude).tolist() if 0 in (len(latList), len(lonList), len(altList)): raise ValueError, 'Got 0 length spatial range' output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.isprint.MadCalculatorGrid(output, requestedParms, [thisDT], latList, lonList, altList) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip()) def runLookerFromForm(self, form): """runLookerFromForm returns the text output of looker from one of the looker forms Inputs: form - the django form that encapsulates all information from looker web page. """ looker_cmd = os.path.join(self._madDB.getMadroot(), 'bin/looker1') looker_options = int(form['looker_options']) if looker_options in (1,2): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, float(form['start_lat']), float(form['stop_lat']), float(form['step_lat']), float(form['start_lon']), float(form['stop_lon']), float(form['step_lon']), float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) looker_cmd += argStr try: text = subprocess.check_output(looker_cmd.split()) except: raise IOError, 'Unable to run cmd <%s>' % (looker_cmd) return(text) elif looker_options in (3,): try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, 0.0, 0.0, 0.0, float(form['start_lat']), float(form['stop_lat']), float(form['step_lat']), float(form['start_lon']), float(form['stop_lon']), float(form['step_lon']), float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (4,): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, float(form['start_az']), float(form['stop_az']), float(form['step_az']), float(form['start_el']), float(form['stop_el']), float(form['step_el']), float(form['start_range']), float(form['stop_range']), float(form['step_range'])) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (5,6,7): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 if looker_options == 5: p1 = float(form['fl_az']) p2 = float(form['fl_el']) p3 = float(form['fl_range']) elif looker_options == 6: p1 = float(form['fl_lat']) p2 = float(form['fl_lon']) p3 = float(form['fl_alt']) elif looker_options == 7: p1 = float(form['fl_apex_lat']) p2 = float(form['fl_apex_lon']) p3 = 0.0 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, p1, p2, p3, float(form['start_alt']), float(form['stop_alt']), float(form['step_alt']), 0.0, 0.0, 0.0) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (8,): latList = numpy.arange(float(form['start_lat']), float(form['stop_lat']), float(form['step_lat'])) latList = latList.tolist() # check for null list if len(latList) == 0 and abs(float(form['start_lat']) - float(form['stop_lat'])) < 1.0E-6: latList = [float(form['start_lat'])] lonList = numpy.arange(float(form['start_lon']), float(form['stop_lon']), float(form['step_lon'])) lonList = lonList.tolist() # check for null list if len(lonList) == 0 and abs(float(form['start_lon']) - float(form['stop_lon'])) < 1.0E-6: lonList = [float(form['start_lon'])] altList = numpy.arange(float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) altList = altList.tolist() # check for null list if len(altList) == 0 and abs(float(form['start_alt']) - float(form['stop_alt'])) < 1.0E-6: altList = [float(form['start_alt'])] requestedParms = ['gdlat', 'glon', 'gdalt'] + [str(parm.lower()) for parm in form['pList']] dtList = [form['datetime']] output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.isprint.MadCalculatorGrid(output, requestedParms, dtList, latList, lonList, altList) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip()) else: raise ValueError, 'Unknown looker_options %s' % (str(looker_options)) def cleanStage(self): """cleanStage removes all temp files more than 2 hours old from experiments/stage """ stageDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') filesToTest = glob.glob(os.path.join(stageDir, '*')) now = datetime.datetime.now() cutoff = datetime.timedelta(hours=2) for fileToTest in filesToTest: try: mDT = datetime.datetime.fromtimestamp(os.path.getmtime(fileToTest)) except: continue if now - mDT > cutoff: try: os.remove(fileToTest) except: pass # ignore problems def modifyBasename(self, basename): """modifyBasename adds _<num> to make sure basename unique """ base, file_extension = os.path.splitext(basename) index = base.rfind('_') if index != -1: try: version = int(base[index+1:]) return('%s%i%s' % (base[:index], version+1, file_extension)) except: pass return('%s_%i%s' % (base, 1, file_extension)) def _getDaynoFilter(self, seasonalStartDate, seasonalEndDate): """_getDaynoFilter returns a filter str in the form dayno,<lower>,<upper> Inputs: seasonalStartDate - a string in form 'MM/DD'. Assumes non-leap year. Empty string means no filtering by seasonal start date. seasonalStartDate - a string in form 'MM/DD'. Assumes non-leap year. Empty string means no filtering by seasonal end date. """ startDayno = '' endDayno = '' if len(seasonalStartDate): startItems = seasonalStartDate.split('/') startDT = datetime.datetime(1958, int(startItems[0]), int(startItems[1])) startDayno = int(startDT.strftime('%j')) if len(seasonalEndDate): endItems = seasonalEndDate.split('/') endDT = datetime.datetime(1958, int(endItems[0]), int(endItems[1])) endDayno = int(endDT.strftime('%j')) return('dayno,%i,%i' % (startDayno, endDayno)) def _getLock(self, filename): """_getLock is a private helper function that provides exclusive access to filename via a locking file. Inputs: filename = the file that exclusive access is required to. Returns: None Affects: Writes file filename + .LCK as a lock mechanism Exceptions: MadrigalError thrown if unable to write lock file Notes: Will sleep for 1 second at a time, for a maximum of _MaxSleep seconds (presently 10) if the file is not modified. After each second, it will check for the lock file to be removed or modified. If it was modified, it resets the count to 0 sec and starts counting again. After _MaxSleep counts it then assumes lock file is orphaned and returns. Orphaned file will be removed when dropLock is called. """ gotLock = 0 numTries = 0 modificationTime = 0 while (not gotLock): try: file = os.open(filename + '.LCK', os.O_RDWR | os.O_CREAT | os.O_EXCL) os.close(file) gotLock = 1 except OSError, (errno, strerror): # error 17 is "File exists" if errno != 17: raise madrigal.admin.MadrigalError("Unable to open " + filename + ".LCK as locking file ", None) # get modification time - may throw an error if file has disappearred try: newModTime = (os.stat(filename + '.LCK')).st_mtime except: #file has disappeared, no need to sleep continue # if the lock file has been modified (or if this is the first time through) set numTries = 0 if newModTime > modificationTime: modificationTime = newModTime numTries = 0 time.sleep(1) numTries = numTries + 1 if numTries > self._MaxSleep: return def _dropLock(self, filename): """_dropLock is a private helper function that drops exclusive access to filename via a locking file. Inputs: filename = the file that exclusive access is required to. Returns: None Affects: Removes file filename + .LCK as a lock mechanism Exceptions: None. """ try: os.remove(filename + '.LCK') except IOError: return
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None)
init initializes MadrigalWeb by reading from MadridalDB..
Inputs: Existing MadrigalDB object, by default = None.
Returns: void
Affects: Initializes self._metaDir, self._logFile.
Exceptions: None.
def __init__(self, madDB = None): """__init__ initializes MadrigalWeb by reading from MadridalDB.. Inputs: Existing MadrigalDB object, by default = None. Returns: void Affects: Initializes self._metaDir, self._logFile. Exceptions: None. """ if madDB == None: self._madDB = madrigal.metadata.MadrigalDB() else: self._madDB = madDB self._binDir = self._madDB.getBinDir() self._instObj = madrigal.metadata.MadrigalInstrument(self._madDB) self._madKindatObj = madrigal.metadata.MadrigalKindat(self._madDB) metaDir = self._madDB.getMetadataDir() # get todays year now = datetime.datetime.now() thisYear = '%04i' % (now.year) self._logFile = os.path.join(metaDir, 'userdata', 'access_%s.log' % (thisYear)) # be sure it exists if not os.access(self._logFile, os.R_OK): f=open(self._logFile, 'w') f.close() # keep track of whether user trusted self._isTrusted_ = None # cache Madrigal objects as needed to imprive performance self._madExpObjExpID = None # will be set to a MadrigalExperiment object sorted by expId when first needed self._madExpObjDate = None # will be set to a MadrigalExperiment object sorted by date when first needed
def cleanStage(
self)
cleanStage removes all temp files more than 2 hours old from experiments/stage
def cleanStage(self): """cleanStage removes all temp files more than 2 hours old from experiments/stage """ stageDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') filesToTest = glob.glob(os.path.join(stageDir, '*')) now = datetime.datetime.now() cutoff = datetime.timedelta(hours=2) for fileToTest in filesToTest: try: mDT = datetime.datetime.fromtimestamp(os.path.getmtime(fileToTest)) except: continue if now - mDT > cutoff: try: os.remove(fileToTest) except: pass # ignore problems
def createGlobalDownloadCmd(
self, language, madrigalUrl, output, format, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, kindatList, expName, fileDesc)
createGlobalDownloadCmd returns a string representing a global download as is command to run in a particular language.
Inputs:
language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is output - output directory name format - 'hdf5', 'ascii', or 'netCDF4' user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description.
def createGlobalDownloadCmd(self, language, madrigalUrl, output, format, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, kindatList, expName, fileDesc): """createGlobalDownloadCmd returns a string representing a global download as is command to run in a particular language. Inputs: language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is output - output directory name format - 'hdf5', 'ascii', or 'netCDF4' user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description. """ if language not in ('python', 'Matlab', 'IDL'): raise ValueError, 'language %s not supported' % (str(language)) # url if language == 'python': cmd = 'globalDownload.py --verbose --url=%s ' % (madrigalUrl) elif language == 'Matlab': cmd = "globalDownload('%s', ...\n " % (madrigalUrl) elif language == 'IDL': cmd = "madglobaldownload, '%s', $\n " % (madrigalUrl) # output if language == 'python': cmd += '--outputDir=%s ' % (output) elif language == 'Matlab': cmd += "'%s', ...\n " % (output) elif language == 'IDL': cmd += "'%s', $\n " % (output) # user_fullname if language == 'python': cmd += '--user_fullname="%s" ' % (user_fullname) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_fullname) elif language == 'IDL': cmd += "'%s', $\n " % (user_fullname) # user_email if language == 'python': cmd += '--user_email=%s ' % (user_email) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_email) elif language == 'IDL': cmd += "'%s', $\n " % (user_email) # user_affiliation if language == 'python': cmd += '--user_affiliation="%s" ' % (user_affiliation) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_affiliation) elif language == 'IDL': cmd += "'%s', $\n " % (user_affiliation) # format part 1 (format is not in same order in Matlab and IDL) if format not in ('hdf5', 'ascii', 'netCDF4'): raise ValueError, 'format not in hdf5, ascii or netCDF4' if language == 'python': cmd += '--format="%s" ' % (format) elif language == 'Matlab': cmd += "'%s', ...\n " % (format) # start_datetime if language == 'python': cmd += '--startDate="%s" ' % (start_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (start_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (start_datetime.month, start_datetime.day, start_datetime.year, start_datetime.hour, start_datetime.minute, start_datetime.second) # end_datetime if language == 'python': cmd += '--endDate="%s" ' % (end_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (end_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (end_datetime.month, end_datetime.day, end_datetime.year, end_datetime.hour, end_datetime.minute, end_datetime.second) # instrument if language == 'python': cmd += '--inst=%i ' % (instCode) elif language == 'Matlab': cmd += "%i, ...\n " % (instCode) elif language == 'IDL': cmd += "%i, $\n " % (instCode) # kindatList if language == 'python': if len(kindatList) == 0: pass else: kindatStr = '--kindat=' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' cmd += '%s ' % (kindatStr) elif language == 'Matlab': kindatStr = '[' for kindat in kindatList: if kindat == 0: continue kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, ...\n " % (kindatStr) elif language == 'IDL': # make sure 0 not in list try: kindatList.remove(0) except ValueError: pass if len(kindatList) == 0: kindatStr = 'PTR_NEW()' else: kindatStr = '[' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, $\n " % (kindatStr) # now deal with IDL format if needed if language == 'IDL': cmd += "'%s', $\n " % (format) # expName if language == 'python': if len(expName) > 0: cmd += '--expName="%s" ' % (expName) elif language == 'Matlab': expName = expName.replace('*', '.*') expName = expName.replace('?', '.?') cmd += "'%s', ...\n " % (expName) elif language == 'IDL': cmd += "'%s', $\n " % (expName) # fileDesc if language == 'python': if len(fileDesc) > 0: cmd += '--fileDesc="%s" ' % (fileDesc) elif language == 'Matlab': fileDesc = fileDesc.replace('*', '.*') fileDesc = fileDesc.replace('?', '.?') cmd += "'%s') " % (fileDesc) elif language == 'IDL': cmd += "'%s' " % (fileDesc) return(cmd)
def createGlobalIsprintCmd(
self, language, madrigalUrl, parmList, output, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, filterList, kindatList, expName, fileDesc, seasonalStartDate, seasonalEndDate, format=None)
createGlobalIsprintCmd returns a string representing a global isprint command to run in a particular language.
Inputs:
language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is parmList - ordered list of parameters requested. output - output file name user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) filterList - a list of strings in form "mnem,lower,upper" where lower and/or upper may be empty kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description. seasonalStartDate - a string in form 'MM/DD'. Dates before then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal start date. seasonalEndDate - a string in form 'MM/DD'. Dates after then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal end date. format - 'hdf5', 'ascii', or 'netCDF4'. If None, not specified (Madrigal 2 does not support this)
def createGlobalIsprintCmd(self, language, madrigalUrl, parmList, output, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, filterList, kindatList, expName, fileDesc, seasonalStartDate, seasonalEndDate, format=None): """createGlobalIsprintCmd returns a string representing a global isprint command to run in a particular language. Inputs: language - which language to use. Allowed values are ('python', 'Matlab', 'IDL') madrigalUrl - url to madrigal home page where data is parmList - ordered list of parameters requested. output - output file name user_fullname user_email user_affiliation start_datetime - a datetime object. Reject experiments before that datetime end_datetime - a datetime object. Reject experiments after that datetime instCode - instrument code (integer) filterList - a list of strings in form "mnem,lower,upper" where lower and/or upper may be empty kindatList - a list of kindat codes. An empty list selects all kindats expName - filter experiments by the experiment name. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by experiment name. fileDesc - filter files by the file description string. Matching is case insensitive and fnmatch characters * and ? are allowed. Empty string is no filtering by file description. seasonalStartDate - a string in form 'MM/DD'. Dates before then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal start date. seasonalEndDate - a string in form 'MM/DD'. Dates after then in any year will be ignored. Assumes non-leap year. Empty string means no filtering by seasonal end date. format - 'hdf5', 'ascii', or 'netCDF4'. If None, not specified (Madrigal 2 does not support this) """ if language not in ('python', 'Matlab', 'IDL'): raise ValueError, 'language %s not supported' % (str(language)) # url if language == 'python': cmd = 'globalIsprint.py --verbose --url=%s ' % (madrigalUrl) elif language == 'Matlab': cmd = "globalIsprint('%s', ...\n " % (madrigalUrl) elif language == 'IDL': cmd = "madglobalprint, '%s', $\n " % (madrigalUrl) # parms if len(parmList) == 0: raise ValueError, 'parmList cannot be empty' parmStr = '' for parm in parmList: parmStr += str(parm) if parm != parmList[-1]: parmStr += ',' if language == 'python': cmd += '--parms=%s ' % (parmStr) elif language == 'Matlab': cmd += "'%s', ...\n " % (parmStr) elif language == 'IDL': cmd += "'%s', $\n " % (parmStr) # output if language == 'python': cmd += '--output=%s ' % (output) elif language == 'Matlab': cmd += "'%s', ...\n " % (output) elif language == 'IDL': cmd += "'%s', $\n " % (output) # user_fullname if language == 'python': cmd += '--user_fullname="%s" ' % (user_fullname) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_fullname) elif language == 'IDL': cmd += "'%s', $\n " % (user_fullname) # user_email if language == 'python': cmd += '--user_email=%s ' % (user_email) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_email) elif language == 'IDL': cmd += "'%s', $\n " % (user_email) # user_affiliation if language == 'python': cmd += '--user_affiliation="%s" ' % (user_affiliation) elif language == 'Matlab': cmd += "'%s', ...\n " % (user_affiliation) elif language == 'IDL': cmd += "'%s', $\n " % (user_affiliation) # start_datetime if language == 'python': cmd += '--startDate="%s" ' % (start_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (start_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (start_datetime.month, start_datetime.day, start_datetime.year, start_datetime.hour, start_datetime.minute, start_datetime.second) # end_datetime if language == 'python': cmd += '--endDate="%s" ' % (end_datetime.strftime('%m/%d/%Y')) elif language == 'Matlab': cmd += "datenum('%s'), ...\n " % (end_datetime.strftime('%d-%b-%Y %H:%M:%S')) elif language == 'IDL': cmd += "julday(%i,%i,%i,%i,%i,%i), $\n " % (end_datetime.month, end_datetime.day, end_datetime.year, end_datetime.hour, end_datetime.minute, end_datetime.second) # instrument if language == 'python': cmd += '--inst=%i ' % (instCode) elif language == 'Matlab': cmd += "%i, ...\n " % (instCode) elif language == 'IDL': cmd += "%i, $\n " % (instCode) # format is here for Matlab or python if language in ('Matlab', 'python'): if language == 'python': if not format is None: if format.lower() == 'hdf5': cmd += '--format=%s ' % ('Hdf5') elif format in ('netCDF4', 'ascii'): cmd += '--format=%s ' % (format) elif language == 'Matlab': if format is None: cmd += "'%s', ...\n " % ('ascii') elif format.lower() == 'hdf5': cmd += "'%s', ...\n " % ('Hdf5') elif format in ('netCDF4', 'ascii'): cmd += "'%s', ...\n " % (format) # filterList # add seasonal filters if needed if len(seasonalStartDate) or len(seasonalEndDate): daynoFilterStr = self._getDaynoFilter(seasonalStartDate, seasonalEndDate) filterList.append(daynoFilterStr) if language == 'python': for filterItem in filterList: cmd += '--filter=%s ' % (filterItem) elif language == 'Matlab': filterStr = '' for filterItem in filterList: filterStr += 'filter=%s ' % (filterItem) cmd += "'%s', ...\n " % (filterStr) elif language == 'IDL': filterStr = '' for filterItem in filterList: filterStr += 'filter=%s ' % (filterItem) cmd += "'%s', $\n " % (filterStr) # kindatList if language == 'python': if len(kindatList) == 0: pass else: kindatStr = '--kindat=' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' cmd += '%s ' % (kindatStr) elif language == 'Matlab': kindatStr = '[' for kindat in kindatList: if kindat == 0: continue kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, ...\n " % (kindatStr) elif language == 'IDL': # make sure 0 not in list try: kindatList.remove(0) except ValueError: pass if len(kindatList) == 0: kindatStr = 'PTR_NEW()' else: kindatStr = '[' for kindat in kindatList: kindatStr += '%i' % (kindat) if kindat != kindatList[-1]: kindatStr += ',' kindatStr += ']' cmd += "%s, $\n " % (kindatStr) # expName if language == 'python': if len(expName) > 0: cmd += '--expName="%s" ' % (expName) elif language == 'Matlab': expName = expName.replace('*', '.*') expName = expName.replace('?', '.?') cmd += "'%s', ...\n " % (expName) elif language == 'IDL': cmd += "'%s', $\n " % (expName) # fileDesc if language == 'python': if len(fileDesc) > 0: cmd += '--fileDesc="%s" ' % (fileDesc) elif language == 'Matlab': fileDesc = fileDesc.replace('*', '.*') fileDesc = fileDesc.replace('?', '.?') cmd += "'%s') " % (fileDesc) elif language == 'IDL': cmd += "'%s', $\n " % (fileDesc) # format is here for idl if language == 'IDL': if format is None: cmd += "'ascii', $\n " elif format.lower() == 'hdf5': cmd += "'hdf5', $\n " elif format in ('netCDF4', 'ascii'): cmd += "'%s', $\n " % (format) return(cmd)
def downloadFileAsIs(
self, expId, basename, user_fullname, user_email, user_affiliation)
downloadFileAsIs returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters)
Inputs: expId - experiment id of experiment basename - basename of file. May have .txt or .nc extension, in which case Hdf5 file is converted user_fullname, user_email, user_affiliation - user identification strings
def downloadFileAsIs(self, expId, basename, user_fullname, user_email, user_affiliation): """downloadFileAsIs returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters) Inputs: expId - experiment id of experiment basename - basename of file. May have .txt or .nc extension, in which case Hdf5 file is converted user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() hdf5Extensions = ('.hdf5', '.h5', '.hdf') fileName, fileExtension = os.path.splitext(basename) if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) expDir = self._madExpObjExpID.getExpDirByExpId(int(expId)) if expDir is None: raise ValueError, 'No expDir found for exp_id %i' % (int(expId)) if fileExtension in hdf5Extensions: baseHdf5 = basename else: # we need to search for the hdf5 basename madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) baseHdf5 = None for i in range(madFileObj.getFileCount()): thisFileName, thisFileExt = os.path.splitext(madFileObj.getFilenameByPosition(i)) if thisFileName == fileName and thisFileExt in hdf5Extensions: baseHdf5 = madFileObj.getFilenameByPosition(i) break if baseHdf5 is None: raise ValueError, 'No valid file for %s found in %s' % (basename, expDir) if basename == baseHdf5: fullFilename = os.path.join(expDir, basename) fullHdf5Filename = fullFilename tmpDir = None else: fullHdf5Filename = os.path.join(expDir, baseHdf5) # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass fullFilename = os.path.join(tmpDir, basename) if os.access(fullFilename, os.R_OK): try: os.remove(fullFilename) except: pass if fileExtension == '.txt': cachedFile = os.path.join(expDir, 'overview', baseHdf5 + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(fullHdf5Filename, fullFilename) elif fileExtension == '.nc': cachedFile = os.path.join(expDir, 'overview', baseHdf5 + '.nc') if os.access(cachedFile, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(fullHdf5Filename, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(fullHdf5Filename) cedarObj.write('netCDF4', fullFilename) # log access self.logDataAccess(fullHdf5Filename, user_fullname, user_email, user_affiliation) return(fullFilename)
def downloadFullFileAsIs(
self, fullHdf5Filename, format, user_fullname, user_email, user_affiliation)
downloadFullFileAsIs is similar to downloadFileAsIs with fullFilename input instead of expId .
Returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters)
Inputs: fullHdf5Filename - full path to Madrigal Hdf5 file format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings
def downloadFullFileAsIs(self, fullHdf5Filename, format, user_fullname, user_email, user_affiliation): """downloadFullFileAsIs is similar to downloadFileAsIs with fullFilename input instead of expId . Returns a path to a Madrigal file to download as is (that is, with parms in file, and no filters) Inputs: fullHdf5Filename - full path to Madrigal Hdf5 file format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() if format not in ('hdf5', 'netCDF4', 'ascii'): raise ValueError, 'Illegal format %s' % (str(format)) if not os.access(fullHdf5Filename, os.R_OK): raise IOError, 'Unable to find Hdf5 file %s' % (str(fullHdf5Filename)) if format == 'hdf5': fullFilename = fullHdf5Filename tmpDir = None else: # dynamically create file if format == 'netCDF4': thisExt = '.nc' elif format == 'ascii': thisExt = '.txt' # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass base, file_extension = os.path.splitext(fullHdf5Filename) basename = os.path.basename(base + thisExt) fullFilename = os.path.join(tmpDir, basename) if os.access(fullFilename, os.R_OK): try: os.remove(fullFilename) except: pass if format == 'ascii': cachedFile = os.path.join(os.path.dirname(fullHdf5Filename), 'overview', os.path.basename(fullHdf5Filename) + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(fullHdf5Filename, fullFilename) elif format == 'netCDF4': cachedFile = os.path.join(os.path.dirname(fullHdf5Filename), 'overview', os.path.basename(fullHdf5Filename) + '.nc') if os.access(cachedFile, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(fullHdf5Filename, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(fullHdf5Filename) cedarObj.write('netCDF4', fullFilename) # log access self.logDataAccess(fullHdf5Filename, user_fullname, user_email, user_affiliation) return(fullFilename)
def downloadIsprintFileFromIsprintForm(
self, isprintForm, user_fullname, user_email, user_affiliation)
downloadIsprintFileFromIsprintForm returns a full path to the temp file is experiments/stage created by isprint to download.
Inputs:
isprintForm - the django form that encapsulates all information from get_advanced web page.
user_fullname, user_email, user_affiliation - user identification strings
def downloadIsprintFileFromIsprintForm(self, isprintForm, user_fullname, user_email, user_affiliation): """downloadIsprintFileFromIsprintForm returns a full path to the temp file is experiments/stage created by isprint to download. Inputs: isprintForm - the django form that encapsulates all information from get_advanced web page. user_fullname, user_email, user_affiliation - user identification strings """ # defaults arguments showHeaders=False missing=None assumed=None knownbad=None orgFilename = isprintForm['fullFilename'] requestedParms = isprintForm['parameters'] # make unique uniqueRequestedParms = [] for s in requestedParms: asciiParm = s.encode('ascii').lower().strip() if asciiParm not in uniqueRequestedParms: uniqueRequestedParms.append(asciiParm) start_date = isprintForm['start_date'] end_date = isprintForm['end_date'] format = isprintForm['formats'] # create name of temp file basename_noext, ext = os.path.splitext(os.path.basename(orgFilename)) randint = random.randint(0,999999) if format in ('ascii', 'netCDF4'): # need new basename if format == 'ascii': basename = basename_noext + '_%06i.txt' % (randint) else: basename = basename_noext + '_%06i.nc' % (randint) else: basename = basename_noext + '_%06i.hdf5' % (randint) tmpFile = os.path.join(self._madDB.getMadroot(), 'experiments/stage', basename) if format == 'ascii': # reset defaults showHeaders = isprintForm['showHeaders'] missing = isprintForm['missing'] assumed = isprintForm['missing'] knownbad = isprintForm['missing'] # next task - create a list of filters, but only if actually modified madFilters = [] madFileObj = madrigal.data.MadrigalFile(orgFilename, self._madDB) # used to determine default values # check if we need a time filer earliestTime = madFileObj.getEarliestTime() latestTime = madFileObj.getLatestTime() earliestDT = datetime.datetime(*earliestTime) latestDT = datetime.datetime(*latestTime) earliest_unix = calendar.timegm(earliestDT.timetuple()) latest_unix = calendar.timegm(latestDT.timetuple()) start_unix = calendar.timegm(start_date.timetuple()) end_unix = calendar.timegm(end_date.timetuple()) if earliest_unix < start_unix or latest_unix > end_unix: # we need a time filter madFilters.append(madrigal.derivation.MadrigalFilter('ut1_unix', [(start_unix, end_unix)])) # altitude filter if isprintForm.has_key('min_alt'): file_min_alt = madFileObj.getMinValidAltitude() file_max_alt = madFileObj.getMaxValidAltitude() try: min_alt = float(isprintForm['min_alt']) except ValueError: min_alt = float('nan') try: max_alt = float(isprintForm['max_alt']) except ValueError: max_alt = float('nan') is_needed = False if not math.isnan(min_alt): if min_alt > file_min_alt + 1.0E-6: is_needed = True if not math.isnan(max_alt): if max_alt < file_max_alt - 1.0E-6: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('gdalt', [(min_alt, max_alt)])) # azimuth filter if isprintForm.has_key('min_az'): try: min_az = float(isprintForm['min_az']) except ValueError: min_az = float('nan') try: max_az = float(isprintForm['max_az']) except ValueError: max_az = float('nan') try: min_az2 = float(isprintForm['min_az2']) except ValueError: min_az2 = float('nan') try: max_az2 = float(isprintForm['max_az2']) except ValueError: max_az2 = float('nan') is_needed = False if not math.isnan(min_az): if min_az > -180.0: is_needed = True if not math.isnan(max_az): if max_az < 180.0: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('azm', [(min_az, max_az), (min_az2, max_az2)])) # elevation filter if isprintForm.has_key('min_el'): try: min_el = float(isprintForm['min_el']) except ValueError: min_el = float('nan') try: max_el = float(isprintForm['max_el']) except ValueError: max_el = float('nan') try: min_el2 = float(isprintForm['min_el2']) except ValueError: min_el2 = float('nan') try: max_el2 = float(isprintForm['max_el2']) except ValueError: max_el2 = float('nan') is_needed = False if not math.isnan(min_el): if min_el > 0.0: is_needed = True if not math.isnan(max_el): if max_el < 90.0: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('azm', [(min_el, max_el), (min_el2, max_el2)])) # pulse length filter if isprintForm.has_key('min_pl'): file_min_pl = madFileObj.getMinPulseLength() file_max_pl = madFileObj.getMaxPulseLength() try: min_pl = float(isprintForm['min_pl']) except ValueError: min_pl = float('nan') try: max_pl = float(isprintForm['max_pl']) except ValueError: max_pl = float('nan') is_needed = False if not math.isnan(min_pl): if min_pl > file_min_pl + 1.0E-9: is_needed = True if not math.isnan(max_pl): if max_pl < file_max_pl - 1.0E-9: is_needed = True if is_needed: madFilters.append(madrigal.derivation.MadrigalFilter('pl', [(min_pl, max_pl)])) # free parameters for i in range(1, 4): parm_name = isprintForm['parm_%i' % (i)] parm_lower = isprintForm['parm_%i_lower' % (i)] parm_upper = isprintForm['parm_%i_upper' % (i)] if parm_name == 'None': continue if len(parm_lower) == 0 and len(parm_upper) == 0: continue # filter needed try: min_value = float(parm_lower) except ValueError: min_value = float('nan') try: max_value = float(parm_upper) except ValueError: max_value = float('nan') madFilters.append(madrigal.derivation.MadrigalFilter(parm_name, [(min_value, max_value)])) # create new temp file madrigal.isprint.Isprint(orgFilename, tmpFile, uniqueRequestedParms, madFilters, showHeaders=showHeaders, missing=missing, assumed=assumed, knownbad=knownbad) # log access self.logDataAccess(orgFilename, user_fullname, user_email, user_affiliation) return(tmpFile)
def downloadMultipleFiles(
self, fileList, format, user_fullname, user_email, user_affiliation)
downloadMultipleFiles downloads multiple files in tarred format
Returns a path to a Madrigal tarred file to download in given format as is (that is, with parms in file, and no filters)
Inputs: fileList - list of full paths to Madrigal Hdf5 files format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings
def downloadMultipleFiles(self, fileList, format, user_fullname, user_email, user_affiliation): """downloadMultipleFiles downloads multiple files in tarred format Returns a path to a Madrigal tarred file to download in given format as is (that is, with parms in file, and no filters) Inputs: fileList - list of full paths to Madrigal Hdf5 files format - 'hdf5', 'netCDF4', or 'ascii' user_fullname, user_email, user_affiliation - user identification strings """ self.cleanStage() if format not in ('hdf5', 'netCDF4', 'ascii'): raise ValueError, 'Illegal format %s' % (str(format)) # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass # be sure no duplicate file names basenameList = [] if format == 'hdf5': finalFileList = [] for thisFile in fileList: basename = os.path.basename(thisFile) dirname = os.path.dirname(thisFile) if basename in basenameList: basename = self.modifyBasename(basename) fullFilename = os.path.join(tmpDir, basename) shutil.copy(thisFile, fullFilename) else: fullFilename = os.path.join(dirname, basename) basenameList.append(basename) finalFileList.append(fullFilename) else: # dynamically create files if format == 'netCDF4': thisExt = '.nc' elif format == 'ascii': thisExt = '.txt' finalFileList = [] for thisFile in fileList: if not os.access(thisFile, os.R_OK): raise IOError, 'Unable to find Hdf5 file %s' % (str(thisFile)) base, file_extension = os.path.splitext(thisFile) basename = os.path.basename(base + thisExt) if basename in basenameList: basename = self.modifyBasename(basename) basenameList.append(basename) fullFilename = os.path.join(tmpDir, basename) if format == 'ascii': cachedFile = os.path.join(os.path.dirname(thisFile), 'overview', os.path.basename(thisFile) + '.txt.gz') if os.access(cachedFile, os.R_OK): fullFilename += '.gz' if not os.access(fullFilename, os.R_OK): shutil.copy(cachedFile, fullFilename) else: madrigal.cedar.convertToText(thisFile, fullFilename) elif format == 'netCDF4': cachedFile = os.path.join(os.path.dirname(thisFile), 'overview', os.path.basename(thisFile) + '.nc') if os.access(cachedFile, os.R_OK): if not os.access(fullFilename, os.R_OK): shutil.copy(cachedFile, fullFilename) else: try: madrigal.cedar.convertToNetCDF4(thisFile, fullFilename) except IOError: cedarObj = madrigal.cedar.MadrigalCedarFile(thisFile) cedarObj.write('netCDF4', fullFilename) finalFileList.append(fullFilename) # create tar file now = datetime.datetime.now() tar_filename = os.path.join(tmpDir, 'madrigalFiles_%s.tar' % (now.strftime('%Y%m%dT%H%M%S'))) tar = tarfile.open(tar_filename, "w") for thisFile in finalFileList: # log access self.logDataAccess(thisFile, user_fullname, user_email, user_affiliation) tar.add(thisFile) # let clean stage do this - safer return(tar_filename)
def filterLog(
self, tmpFile, kinstList=None, accessStartDate=None, accessEndDate=None)
filterLog writes a subsection of the access log to a temporary file
Inputs:
tmpFile - temporary file to write subsection of log to kinstList - list of kinsts to accept. If None (the default), accept all instruments accessStartDate - if not None (the default), reject all access dates before datetime accessStartDate accessEndDate - if not None (the default), reject all access dates after datetime accessEndDate
def filterLog(self, tmpFile, kinstList=None, accessStartDate=None, accessEndDate=None): """filterLog writes a subsection of the access log to a temporary file Inputs: tmpFile - temporary file to write subsection of log to kinstList - list of kinsts to accept. If None (the default), accept all instruments accessStartDate - if not None (the default), reject all access dates before datetime accessStartDate accessEndDate - if not None (the default), reject all access dates after datetime accessEndDate """ f = open(tmpFile, 'w') accessLogs = glob.glob(os.path.join(self._madDB.getMadroot(), 'metadata/userdata/access_*.log')) accessLogs.sort() # addition for cedar only """accessLogs2 = glob.glob('/opt/cedar/metadata/userdata/access_*[0-9].log') accessLogs += accessLogs2 accessLogs.sort()""" if kinstList: # create a dictionary of key = 3 letter inst mnem, value = kinstList instDict = {} instList = self._instObj.getInstrumentList() for inst in instList: if instDict.has_key(inst[1]): instDict[inst[1]].append(inst[2]) else: instDict[inst[1]] = [inst[2]] for accessLog in accessLogs: # see if we can skip this year basename = os.path.basename(accessLog) year = int(basename[7:-4]) startYear = datetime.datetime(year,1,1,0,0,0) endYear = datetime.datetime(year,12,31,23,59,59) if accessStartDate: if accessStartDate > endYear: continue if accessEndDate: if accessEndDate < startYear: continue # this file can be huge, so read one line at a time fl = open(accessLog) while True: line = fl.readline() if len(line) == 0: break items = line.strip().split(',') if len(items) != 5: continue # walk through filters # kinst if kinstList: # get the instrument mnemonic dirs = items[-1].split('/') found = False try: for kinst in instDict[dirs[-3]]: if kinst in kinstList: found = True break except KeyError: continue if not found: continue # access time if accessStartDate or accessEndDate: thisDT = datetime.datetime.strptime(items[-2], '%Y-%m-%d %H-%M-%S') if accessStartDate: if accessStartDate > thisDT: continue if accessEndDate: if accessEndDate < thisDT: continue # all filters passed f.write(line) fl.close() f.close()
def generateDownloadFileScriptFromForm(
self, form, user_fullname, user_email, user_affiliation)
generateDownloadFileScriptFromForm converts the Django form into arguments so that if can then call createGlobalDownloadCmd.
form is a dict with keys: instruments, start_date, end_date, format_select, language_select, kindat_select, expName, fileDesc user_fullname, user_email, user_affiliation - strings
def generateDownloadFileScriptFromForm(self, form, user_fullname, user_email, user_affiliation): """generateDownloadFileScriptFromForm converts the Django form into arguments so that if can then call createGlobalDownloadCmd. form is a dict with keys: instruments, start_date, end_date, format_select, language_select, kindat_select, expName, fileDesc user_fullname, user_email, user_affiliation - strings """ instCode = int(form['instruments']) start_datetime = datetime.datetime(form['start_date'].year, form['start_date'].month, form['start_date'].day) end_datetime = datetime.datetime(form['end_date'].year, form['end_date'].month, form['end_date'].day) format = form['format_select'] language = form['language_select'] kindatList = [int(kindat) for kindat in form['kindat_select']] expName = form['expName'].strip() fileDesc = form['fileDesc'].strip() madrigalUrl = self._madDB.getTopLevelUrl() return(self.createGlobalDownloadCmd(language, madrigalUrl, '/tmp', format, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, kindatList, expName, fileDesc))
def generateGlobalIsprintScriptFromForm(
self, form1, form2, form3, user_fullname, user_email, user_affiliation)
generateGlobalIsprintScriptFromForm converts the three Django forms into arguments so that if can then call createGlobalIsprintCmd. Separate forms used because some parts are created by Ajax.
form1 is a dict with keys: instruments, start_date, end_date, format_select, directory_select, language_select, kindat_select, expName, fileDesc form2 is a dict with keys parameters form3 is a dict with keys parm_#, parm_#lower, parm#_upper, where # is 1, 2, and 3 user_fullname, user_email, user_affiliation - strings
def generateGlobalIsprintScriptFromForm(self, form1, form2, form3, user_fullname, user_email, user_affiliation): """generateGlobalIsprintScriptFromForm converts the three Django forms into arguments so that if can then call createGlobalIsprintCmd. Separate forms used because some parts are created by Ajax. form1 is a dict with keys: instruments, start_date, end_date, format_select, directory_select, language_select, kindat_select, expName, fileDesc form2 is a dict with keys parameters form3 is a dict with keys parm_#, parm_#_lower, parm_#_upper, where # is 1, 2, and 3 user_fullname, user_email, user_affiliation - strings """ instCode = int(form1['instruments']) start_datetime = datetime.datetime(form1['start_date'].year, form1['start_date'].month, form1['start_date'].day) end_datetime = datetime.datetime(form1['end_date'].year, form1['end_date'].month, form1['end_date'].day) format = form1['format_select'] if format == 'ascii' and form1['directory_select'] == 'File': output = 'example.txt' else: output = '/tmp' language = form1['language_select'] kindatList = [int(kindat) for kindat in form1['kindat_select']] expName = form1['expName'].strip() fileDesc = form1['fileDesc'].strip() seasonalStartDay = int(form1['seasonalStartDay']) seasonalStartMonth = int(form1['seasonalStartMonth']) seasonalEndDay = int(form1['seasonalEndDay']) seasonalEndMonth = int(form1['seasonalEndMonth']) if seasonalStartDay == 1 and seasonalStartMonth == 1 and \ seasonalEndDay == 31 and seasonalEndMonth == 12: seasonalStartDate = '' seasonalEndDate = '' else: seasonalStartDate = '%02i/%02i' % (seasonalStartMonth, seasonalStartDay) seasonalEndDate = '%02i/%02i' % (seasonalEndMonth, seasonalStartDay) madrigalUrl = self._madDB.getTopLevelUrl() parmList = form2['parameters'] filterList = [] for i in (1,2,3): try: parm = form3['parm_%i' % (i)] except KeyError: continue if len(parm) == 0: continue filterStr = '%s,' % (parm) try: parm_lower = form3['parm_%i_lower' % (i)] filterStr += '%s,' % (str(parm_lower)) except KeyError: filterStr += ',' try: parm_upper = form3['parm_%i_upper' % (i)] filterStr += '%s' % (str(parm_upper)) except KeyError: pass filterList.append(filterStr) return(self.createGlobalIsprintCmd(language, madrigalUrl, parmList, output, user_fullname, user_email, user_affiliation, start_datetime, end_datetime, instCode, filterList, kindatList, expName, fileDesc, seasonalStartDate, seasonalEndDate, format))
def generateLogout(
self, fileName, expName)
generateLogout generates a java script which sends a user to the madLogin page to logout automatically.
Inputs: fileName: the madrigal file to return to expName: the experiment name of the file to return to
Returns: a java script which sends a user to the madLogin page to logout automatically
Affects: None.
Exceptions: None.
def generateLogout(self, fileName, expName): """ generateLogout generates a java script which sends a user to the madLogin page to logout automatically. Inputs: fileName: the madrigal file to return to expName: the experiment name of the file to return to Returns: a java script which sends a user to the madLogin page to logout automatically Affects: None. Exceptions: None. """ print '<script language = "JavaScript">' print '\twindow.location = "madLogin?fileName=' + \ fileName + '&expName=' + \ expName + '&state=autoLogout"' print '</script>'
def getDays(
self, kinst, year, month=None, optimize=True)
getDays returns a sorted list of datetime.date objects where there is local data for kinst & year & possibly month combination
Inputs: kinst - instrument id (int) year - year (int) month (1-12) if None, include all months optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning
def getDays(self, kinst, year, month=None, optimize=True): """getDays returns a sorted list of datetime.date objects where there is local data for kinst & year & possibly month combination Inputs: kinst - instrument id (int) year - year (int) month (1-12) if None, include all months optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning """ retList = [] sDT = datetime.datetime(year,1,1) eDT = datetime.datetime(year,12,31,23,59,59) if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue # loop over all days delta = datetime.timedelta(days=1) loopDT = max(sDT, datetime.datetime(thisSDT.year, thisSDT.month, thisSDT.day)) while (loopDT <= thisEDT): if loopDT > eDT: break if not month is None: if month > loopDT.month: loopDT += delta continue elif month < loopDT.month: break thisDate = loopDT.date() if thisDate not in retList: retList.append(thisDate) loopDT += delta retList.sort() return(retList)
def getExpIDFromExpPath(
self, expPath, matchAnyExpNum=False)
getExpIDFromExpPath returns the expId for given expPath (starts with 'experiments')
If matchAnyExpNum is False, it will only match the right experiments directory (default).
It True, matches via re to experiments[0-9]/
Returns None if not found
Inputs: expPath - experiment path (starts with 'experiments')
def getExpIDFromExpPath(self, expPath, matchAnyExpNum=False): """getExpIDFromExpPath returns the expId for given expPath (starts with 'experiments') If matchAnyExpNum is False, it will only match the right experiments* directory (default). It True, matches via re to experiments[0-9]*/<remaining exp path> Returns None if not found Inputs: expPath - experiment path (starts with 'experiments') """ madExpObj = madrigal.metadata.MadrigalExperiment(self._madDB) if matchAnyExpNum: expPathRE = 'experiments[0-9]*/' + expPath[expPath.find('/')+1:] for i in range(madExpObj.getExpCount()): if not matchAnyExpNum: if madExpObj.getExpPathByPosition(i) == expPath: return(madExpObj.getExpIdByPosition(i)) else: if len(re.findall(expPathRE, madExpObj.getExpPathByPosition(i))) > 0: return(madExpObj.getExpIdByPosition(i)) return(None)
def getExpInfoFromExpID(
self, expID)
getExpInfoFromExpID returns a tuple of (pi_name, pi_email, expUrl, kinst, expDesc, kinstDesc) for given expID
expUrl is url from getRealExpUrlByExpId
Inputs: expID - experimentID (int)
def getExpInfoFromExpID(self, expID): """getExpInfoFromExpID returns a tuple of (pi_name, pi_email, expUrl, kinst, expDesc, kinstDesc) for given expID expUrl is url from getRealExpUrlByExpId Inputs: expID - experimentID (int) """ expID = int(expID) if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) kinst = self._madExpObjExpID.getKinstByExpId(expID) kinstDesc = self._instObj.getInstrumentName(kinst) expPI = self._madExpObjExpID.getPIByExpId(expID) expPIEmail = self._madExpObjExpID.getPIEmailByExpId(expID) expUrl = self._madExpObjExpID.getRealExpUrlByExpId(expID) if expPI in (None, ''): expPI = self._instObj.getContactName(kinst) expPIEmail = self._instObj.getContactEmail(kinst) thisSDTList = self._madExpObjExpID.getExpStartDateTimeByExpId(expID) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjExpID.getExpEndDateTimeByExpId(expID) thisEDT = datetime.datetime(*thisEDTList[0:6]) thisExpName = self._madExpObjExpID.getExpNameByExpId(expID) thisExpDesc = '%s: %s-%s' % (thisExpName, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S')) return((expPI, expPIEmail, expUrl, kinst, thisExpDesc, kinstDesc))
def getExperimentList(
self, kinstList, startDT, endDT, localOnly)
getExperimentList returns a sorted list of tuples of (expId, expUrl, expName, instName, kinst, expStartDT, expEndDT, siteId, siteName)
Inputs: kinstList - a list of instrument id (int) - may include 0 startDT - start datetime to search endDT - end datetime to search localOnly - if True, only search locally. If False, search globally
def getExperimentList(self, kinstList, startDT, endDT, localOnly): """getExperimentList returns a sorted list of tuples of (expId, expUrl, expName, instName, kinst, expStartDT, expEndDT, siteId, siteName) Inputs: kinstList - a list of instrument id (int) - may include 0 startDT - start datetime to search endDT - end datetime to search localOnly - if True, only search locally. If False, search globally """ retList = [] madroot = self._madDB.getMadroot() siteId = self._madDB.getSiteID() siteObj = madrigal.metadata.MadrigalSite(self._madDB) if localOnly: expTabFile = os.path.join(madroot, 'metadata/expTab.txt') else: expTabFile = os.path.join(madroot, 'metadata/expTabAll.txt') madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB, expTabFile) startIndex = madExpObjDate.getStartPosition(startDT) for i in range(startIndex, madExpObjDate.getExpCount()): thisKinst = madExpObjDate.getKinstByPosition(i) if thisKinst not in kinstList and 0 not in kinstList: continue thisSDTList = madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < startDT: continue if thisSDT > endDT: break # check for security security = madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue thisSiteId = madExpObjDate.getExpSiteIdByPosition(i) if siteId == thisSiteId: isLocal = True else: isLocal = False thisExpId = madExpObjDate.getExpIdByPosition(i) thisExpPath = madExpObjDate.getExpPathByPosition(i) thisExpName = madExpObjDate.getExpNameByPosition(i) if isLocal: thisExpUrl = django.core.urlresolvers.reverse('show_experiment') + \ '?experiment_list=%i' % (thisExpId) elif siteObj.getSiteVersion(thisSiteId) == '2.6': thisExpUrl = 'http://' + os.path.join(siteObj.getSiteServer(thisSiteId), siteObj.getSiteRelativeCGI(thisSiteId), 'madExperiment.cgi?exp=%s' % (thisExpPath)) thisExpUrl += '&displayLevel=0&expTitle=%s' % (django.utils.http.urlquote(thisExpName)) else: # remote Madrigal 3.0 site baseUrl = os.path.basename(django.core.urlresolvers.reverse('show_experiment')[:-1]) thisExpUrl = baseUrl + '?experiment_list=%s' % (thisExpPath) thisSiteUrl = 'http://%s' % (siteObj.getSiteServer(thisSiteId)) relativeUrl = siteObj.getSiteDocRoot(thisSiteId) if relativeUrl not in ('', None): thisSiteUrl = os.path.join(thisSiteUrl, relativeUrl) if thisSiteUrl[-1] != '/' and thisExpUrl[0] != '/': thisSiteUrl += '/' thisExpUrl = thisSiteUrl + thisExpUrl thisSiteName = siteObj.getSiteName(thisSiteId) instName = self._instObj.getInstrumentName(thisKinst) retList.append((thisExpId, thisExpUrl, thisExpName, instName, thisKinst, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S'), thisSiteId, thisSiteName)) return(retList)
def getExpsOnDate(
self, kinst, year, month, day, optimize=True)
getExpsOnDate returns a sorted list of tuples of (expId, expDesc, expDir, pi_name, pi_email)
Inputs: kinst - instrument id (int) year - year (int) month - month (int) day - day (int) optimize - if True, only start search at beginning of day. But may miss long experiments, so if optimization if False, starts at beginning
def getExpsOnDate(self, kinst, year, month, day, optimize=True): """getExpsOnDate returns a sorted list of tuples of (expId, expDesc, expDir, pi_name, pi_email) Inputs: kinst - instrument id (int) year - year (int) month - month (int) day - day (int) optimize - if True, only start search at beginning of day. But may miss long experiments, so if optimization if False, starts at beginning """ retList = [] sDT = datetime.datetime(year,month,day) eDT = datetime.datetime(year,month,day,23,59,59) if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue thisExpId = self._madExpObjDate.getExpIdByPosition(i) thisExpName = self._madExpObjDate.getExpNameByPosition(i) thisExpDesc = '%s: %s-%s' % (thisExpName, thisSDT.strftime('%Y-%m-%d %H:%M:%S'), thisEDT.strftime('%Y-%m-%d %H:%M:%S')) thisExpDir = self._madExpObjDate.getExpDirByPosition(i) thisExpPI = self._madExpObjDate.getPIByPosition(i) thisExpPIEmail = self._madExpObjDate.getPIEmailByPosition(i) if thisExpPI in (None, ''): thisExpPI = self._instObj.getContactName(kinst) thisExpPIEmail = self._instObj.getContactEmail(kinst) retList.append((thisExpId, thisExpDesc, thisExpDir)) return(retList)
def getFileFromExpDir(
self, expDir, kinst, includeNonDefault=False)
getFileFromExpDir returns a list of tuples of (basename, fileDesc)
Inputs: expDir - full path to exp directory kinst - instrument id (used to look up kindat descriptions) includeNonDefault - if True, include variant and history files. If False (the default), do not
def getFileFromExpDir(self, expDir, kinst, includeNonDefault=False): """getFileFromExpDir returns a list of tuples of (basename, fileDesc) Inputs: expDir - full path to exp directory kinst - instrument id (used to look up kindat descriptions) includeNonDefault - if True, include variant and history files. If False (the default), do not """ retList = [] realTimeList = [] # in case no default files if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK): # no files in this experiment return(retList) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) for i in range(madFileObj.getFileCount()): if madFileObj.getAccessByPosition(i) == 1 and not self.isTrusted(): continue category = madFileObj.getCategoryByPosition(i) if category in (2,3) and not includeNonDefault: continue basename = madFileObj.getFilenameByPosition(i) kindat = madFileObj.getKindatByPosition(i) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) status = madFileObj.getStatusByPosition(i) fileDesc = '%s: %s - %s' % (basename, kindatDesc, status) if category != 4: retList.append((basename, fileDesc)) else: realTimeList.append((basename, fileDesc)) if len(retList) > 0: return(retList) else: return(realTimeList)
def getFileFromExpID(
self, expID, includeNonDefault=False)
getFileFromExpDir returns a list of tuples of (basename, fileDesc)
Inputs: expID - experimentID (int) includeNonDefault - if True, include variant and history files. If False (the default), do not
def getFileFromExpID(self, expID, includeNonDefault=False): """getFileFromExpDir returns a list of tuples of (basename, fileDesc) Inputs: expID - experimentID (int) includeNonDefault - if True, include variant and history files. If False (the default), do not """ retList = [] realTimeList = [] # in case no default files if self._madExpObjExpID is None: self._madExpObjExpID = madrigal.metadata.MadrigalExperiment(self._madDB) expDir = self._madExpObjExpID.getExpDirByExpId(expID) kinst = self._madExpObjExpID.getKinstByExpId(expID) if not os.access(os.path.join(expDir, 'fileTab.txt'), os.R_OK): # no files in this experiment return(retList) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) for i in range(madFileObj.getFileCount()): if madFileObj.getAccessByPosition(i) == 1 and not self.isTrusted(): continue category = madFileObj.getCategoryByPosition(i) if category in (2,3) and not includeNonDefault: continue if category == 2: categoryStr = '<variant file> ' elif category == 3: categoryStr = '<history file> ' else: categoryStr = '' basename = madFileObj.getFilenameByPosition(i) kindat = madFileObj.getKindatByPosition(i) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) status = madFileObj.getStatusByPosition(i) fileDesc = '%s: %s%s - %s' % (basename, categoryStr, kindatDesc, status) if category != 4: retList.append((basename, fileDesc)) else: realTimeList.append((basename, fileDesc)) if len(retList) > 0: return(retList) else: return(realTimeList)
def getInfoFromFile(
self, filePath)
getInfoFromFile returns a tuple of (expName, kindatDesc) for a given input file
def getInfoFromFile(self, filePath): """getInfoFromFile returns a tuple of (expName, kindatDesc) for a given input file """ expDir = os.path.dirname(filePath) basename = os.path.basename(filePath) madExpObj = madrigal.metadata.MadrigalExperiment(self._madDB, os.path.join(expDir, 'expTab.txt')) expName = madExpObj.getExpNameByPosition(0) kinst = madExpObj.getKinstByPosition(0) madFileObj = madrigal.metadata.MadrigalMetaFile(self._madDB, os.path.join(expDir, 'fileTab.txt')) kindat = madFileObj.getKindatByFilename(basename) kindatDesc = self._madKindatObj.getKindatDescription(kindat, kinst) return((expName, kindatDesc))
def getMonths(
self, kinst, year, optimize=True)
getMonths returns a list of tuples of (monthNumber, monthName) where monthNumber is 1-12, and monthName is the form January, Febuary, etc. for the the months where there is local data for kinst & year combination
Inputs: kinst - instrument id (int) year - year (int) optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning
def getMonths(self, kinst, year, optimize=True): """getMonths returns a list of tuples of (monthNumber, monthName) where monthNumber is 1-12, and monthName is the form January, Febuary, etc. for the the months where there is local data for kinst & year combination Inputs: kinst - instrument id (int) year - year (int) optimize - if True, only start search at beginning of year. But may miss long experiments, so if optimization if False, starts at beginning """ tempDict = {} # dict with key = month number, value = month name sDT = datetime.datetime(year,1,1) eDT = datetime.datetime(year,12,31,23,59,59) madroot = self._madDB.getMadroot() if self._madExpObjDate is None: self._madExpObjDate = madrigal.metadata.MadrigalExperiment(self._madDB) if optimize: startIndex = self._madExpObjDate.getStartPosition(sDT) else: startIndex = 0 for i in range(startIndex, self._madExpObjDate.getExpCount()): thisKinst = self._madExpObjDate.getKinstByPosition(i) if kinst != thisKinst: continue thisSDTList = self._madExpObjDate.getExpStartDateTimeByPosition(i) thisSDT = datetime.datetime(*thisSDTList[0:6]) thisEDTList = self._madExpObjDate.getExpEndDateTimeByPosition(i) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < sDT: continue if thisSDT > eDT: break # check for security security = self._madExpObjDate.getSecurityByPosition(i) if not self.isTrusted(): if security not in (0,2): continue else: if security not in (0,1,2,3): continue if thisSDT.year == year: startMonth = thisSDT.month else: startMonth = 1 if thisEDT.year == year: endMonth = thisEDT.month else: endMonth = 12 monthList = range(startMonth, endMonth + 1) for thisMonth in monthList: if thisMonth not in tempDict.keys(): tempDict[thisMonth] = calendar.month_name[thisMonth] monthKeys = tempDict.keys() monthKeys.sort() retList = [(monthKey, tempDict[monthKey]) for monthKey in monthKeys] return(retList)
def getRulesOfTheRoad(
self, PI=None, PIEmail=None)
getRulesOfTheRoad returns a string giving the rules in html formal for using madrigal data.
Inputs: PI - contact name. Default is site name. PIEmail - email link. Default is site admin.
Returns: a string giving the rules in html formal for using madrigal data
Affects: None.
Exceptions: None.
def getRulesOfTheRoad(self, PI=None, PIEmail=None): """ getRulesOfTheRoad returns a string giving the rules in html formal for using madrigal data. Inputs: PI - contact name. Default is site name. PIEmail - email link. Default is site admin. Returns: a string giving the rules in html formal for using madrigal data Affects: None. Exceptions: None. """ if not PI or not PIEmail: # get the site name siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteID = self._madDB.getSiteID() contactName = str(siteObj.getSiteName(siteID)) contactEmail = str(siteObj.getSiteEmail(siteID)) else: contactName = str(PI) contactEmail = str(PIEmail) returnStr = 'Please contact %s at ' % (contactName) returnStr = returnStr + '<a href="mailto:' + contactEmail + '">' + \ contactEmail + '</a> before using this data in a report or publication.' return returnStr
def getSingleRedirectList(
self)
getSingleRedirectList returns a list with tuples (kinst, url) where url is url to redirect single UI to if instrument not local. If no redirect needed because instrument local, url is empty string
def getSingleRedirectList(self): """getSingleRedirectList returns a list with tuples (kinst, url) where url is url to redirect single UI to if instrument not local. If no redirect needed because instrument local, url is empty string """ madInstData = madrigal.metadata.MadrigalInstrumentData(self._madDB, True) siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteID = self._madDB.getSiteID() # create a dict with key = siteID, value = redirect url for speed getStr = '?isGlobal=True&categories=%i&instruments=%i' addUrl = django.core.urlresolvers.reverse('view_single') cedarUrl = 'http://cedar.openmadrigal.org/' + addUrl + getStr siteDict = {} for thisSiteID, siteDesc in siteObj.getSiteList(): if thisSiteID == siteID: siteDict[thisSiteID] = '' # local case elif siteObj.getSiteVersion(thisSiteID) == '2.6': # redirect to cedar because other site below Madrigal 3 siteDict[thisSiteID] = cedarUrl else: siteDict[thisSiteID] = os.path.join(siteObj.getSiteServer(thisSiteID), siteObj.getSiteDocRoot(thisSiteID), addUrl + getStr) retList = [] for kinst, desc, thisSiteID in madInstData.getInstruments(): if len(siteDict[thisSiteID]) > 0: url = siteDict[thisSiteID] % (self._instObj.getCategoryId(kinst), kinst) else: url = '' retList.append((kinst, url)) return(retList)
def getSiteInfo(
self)
getSiteInfo returns a tuple of two items: 1. local site name 2. list of tuples of (siteName, url) of non-local sites
def getSiteInfo(self): """getSiteInfo returns a tuple of two items: 1. local site name 2. list of tuples of (siteName, url) of non-local sites """ siteID = self._madDB.getSiteID() siteObj = madrigal.metadata.MadrigalSite(self._madDB) siteName = siteObj.getSiteName(siteID) retList = [] siteList = siteObj.getSiteList() for thisSiteID, thisSiteName in siteList: if thisSiteID == siteID: continue thisSiteServer = siteObj.getSiteServer(thisSiteID) thisSiteDocRoot = siteObj.getSiteDocRoot(thisSiteID) thisSiteUrl = urlparse.urlunparse(('http', thisSiteServer, thisSiteDocRoot, '','','')) retList.append((thisSiteName, thisSiteUrl)) return((siteName, retList))
def isTrusted(
self)
isTrusted returns 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise.
Inputs: None
Returns: 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise. Also returns 0 if no browser ip available or trustedIPs.txt cannot be opened.
Affects: None.
Exceptions: None.
def isTrusted(self): """ isTrusted returns 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise. Inputs: None Returns: 1 if browser ip matches any in the trustedIPs.txt file; 0 otherwise. Also returns 0 if no browser ip available or trustedIPs.txt cannot be opened. Affects: None. Exceptions: None. """ if self._isTrusted_ != None: return(self._isTrusted_) try: trustFile = open(self._madDB.getMadroot() + '/trustedIPs.txt', 'r') except: return 0 # try to read env var REMOTE_ADDR and HTTP_X_FORWARDED_FOR userIPList = [] if os.environ.get('REMOTE_ADDR')!= None: userIPList.append(os.environ.get('REMOTE_ADDR')) if os.environ.get('HTTP_X_FORWARDED_FOR') != None: ips = os.environ.get('HTTP_X_FORWARDED_FOR').split(',') for ip in ips: userIPList.append(ip.strip()) if len(userIPList) == 0: self._isTrusted_ = 0 return 0 if len(userIPList[0]) < 7: # ip address too short self._isTrusted_ = 0 return 0 # loop through trustedIPs.txt to find a match ipList = trustFile.readlines() for userIP in userIPList: for ipItem in ipList: # match using filename matching with * if fnmatch.fnmatch(userIP, ipItem.strip()): self._isTrusted_ = 1 return 1 # out of loop, no match found self._isTrusted_ = 0 return 0
def listRecords(
self, fullFilename)
listRecords returns the list records html for fullFilename
def listRecords(self, fullFilename): """listRecords returns the list records html for fullFilename """ # check if record plots exist basename = os.path.basename(fullFilename) thisDir = os.path.dirname(fullFilename) pngFiles = glob.glob(os.path.join(thisDir, 'plots', basename, 'records/*.png')) if len(pngFiles) > 0: url = '<a href="javascript:plotRecno(%i)">View record plot</a>' else: url = None output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.cedar.listRecords(fullFilename, output, url) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip())
def logDataAccess(
self, fullFilenameList, user_fullname=None, user_email=None, user_affiliation=None)
logDataAccess logs queries that access low-level data.
Records user name, email, affiliation, datetime, and full path the file(s) accessed.
Inputs:
fullFilenameList either a list of full filenames, or a string with one filename user_fullname - if None, try to read from cookie. Also, any commas replaced by spaces. user_email - if None, try to read from cookie. Also, any commas replaced by spaces. user_affiliation - if None, try to read from cookie. Also, any commas replaced by spaces.
Outputs: None
Affects: Write line to log file with 5 or more comma-delimited columns. Example:
Bill Rideout,brideout@haystack.mit.edu,MIT Haystack,2002-12-25 00:00:00, /opt/madrigal/experiments/2005/mlh/01sep05/mlh050901g.001,/opt/madrigal/experiments/2005/mlh/02sep05/mlh050902g.001
Uses _getLock and _dropLock to ensure single users access to log file
def logDataAccess(self, fullFilenameList, user_fullname=None, user_email=None, user_affiliation=None): """ logDataAccess logs queries that access low-level data. Records user name, email, affiliation, datetime, and full path the file(s) accessed. Inputs: fullFilenameList either a list of full filenames, or a string with one filename user_fullname - if None, try to read from cookie. Also, any commas replaced by spaces. user_email - if None, try to read from cookie. Also, any commas replaced by spaces. user_affiliation - if None, try to read from cookie. Also, any commas replaced by spaces. Outputs: None Affects: Write line to log file with 5 or more comma-delimited columns. Example: Bill Rideout,brideout@haystack.mit.edu,MIT Haystack,2002-12-25 00:00:00, \ /opt/madrigal/experiments/2005/mlh/01sep05/mlh050901g.001,/opt/madrigal/experiments/2005/mlh/02sep05/mlh050902g.001 Uses _getLock and _dropLock to ensure single users access to log file """ if user_fullname == None or user_email == None or user_affiliation == None: # try to get name, email, affiliation from cookie cookie = Cookie.SimpleCookie() if os.environ.has_key('HTTP_COOKIE'): cookie.load(os.environ['HTTP_COOKIE']) try: user_fullname = cookie["user_fullname"].value user_email = cookie["user_email"].value user_affiliation = cookie["user_affiliation"].value except: # no way to write log return if user_fullname == None or user_email == None or user_affiliation == None: return # strip out any commas user_fullname = user_fullname.replace(',', ' ') user_email = user_email.replace(',', ' ') user_affiliation = user_affiliation.replace(',', ' ') if type(fullFilenameList) in (types.ListType, types.TupleType): delimiter = ',' fileStr = delimiter.join(fullFilenameList) else: fileStr = str(fullFilenameList) now = datetime.datetime.now() nowStr = now.strftime('%Y-%m-%d %H-%M-%S') # lock out any method that writes to log file self._getLock(self._logFile) f = open(self._logFile, 'a') f.write('%s,%s,%s,%s,%s\n' % (user_fullname.encode('utf8'), user_email.encode('utf8'), user_affiliation.encode('utf8'), nowStr, fileStr)) f.close() # done with log file - allow access to other writing calls self._dropLock(self._logFile)
def modifyBasename(
self, basename)
modifyBasename adds _
def modifyBasename(self, basename): """modifyBasename adds _<num> to make sure basename unique """ base, file_extension = os.path.splitext(basename) index = base.rfind('_') if index != -1: try: version = int(base[index+1:]) return('%s%i%s' % (base[:index], version+1, file_extension)) except: pass return('%s_%i%s' % (base, 1, file_extension))
def printFileAsIs(
self, fullFilename, user_fullname, user_email, user_affiliation, html=True)
printFileAsIs returns the full path to a temp file representing file as plain text or html to print as is (that is, with parms rom file, and no filters)
Inputs: fullFilename - full path to Madrigal Hdf5 file to convert to string user_fullname, user_email, user_affiliation - user identification strings html - if True (the default) return as Html with popup parm names. If False, pure text
def printFileAsIs(self, fullFilename, user_fullname, user_email, user_affiliation, html=True): """printFileAsIs returns the full path to a temp file representing file as plain text or html to print as is (that is, with parms rom file, and no filters) Inputs: fullFilename - full path to Madrigal Hdf5 file to convert to string user_fullname, user_email, user_affiliation - user identification strings html - if True (the default) return as Html with popup parm names. If False, pure text """ self.cleanStage() # create tmp dir if needed tmpDir = os.path.join(self._madDB.getMadroot(), 'experiments/stage') try: os.mkdir(tmpDir) except: pass fileName, fileExtension = os.path.splitext(fullFilename) fullTmpFilename = os.path.join(tmpDir, os.path.basename(fileName + '.txt')) if os.access(fullTmpFilename, os.R_OK): try: os.remove(fullTmpFilename) except: pass if html: summary = 'html' else: summary = 'plain' madrigal.cedar.convertToText(fullFilename, fullTmpFilename, summary=summary) # log access self.logDataAccess(fullFilename, user_fullname, user_email, user_affiliation) return(fullTmpFilename)
def runLookerFromForm(
self, form)
runLookerFromForm returns the text output of looker from one of the looker forms
Inputs: form - the django form that encapsulates all information from looker web page.
def runLookerFromForm(self, form): """runLookerFromForm returns the text output of looker from one of the looker forms Inputs: form - the django form that encapsulates all information from looker web page. """ looker_cmd = os.path.join(self._madDB.getMadroot(), 'bin/looker1') looker_options = int(form['looker_options']) if looker_options in (1,2): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, float(form['start_lat']), float(form['stop_lat']), float(form['step_lat']), float(form['start_lon']), float(form['stop_lon']), float(form['step_lon']), float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) looker_cmd += argStr try: text = subprocess.check_output(looker_cmd.split()) except: raise IOError, 'Unable to run cmd <%s>' % (looker_cmd) return(text) elif looker_options in (3,): try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, 0.0, 0.0, 0.0, float(form['start_lat']), float(form['stop_lat']), float(form['step_lat']), float(form['start_lon']), float(form['stop_lon']), float(form['step_lon']), float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (4,): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, float(form['start_az']), float(form['stop_az']), float(form['step_az']), float(form['start_el']), float(form['stop_el']), float(form['step_el']), float(form['start_range']), float(form['stop_range']), float(form['step_range'])) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (5,6,7): try: kinst = int(form['instruments']) if kinst == 0: raise ValueError, '' slatgd = self._instObj.getLatitude(kinst) slon = self._instObj.getLongitude(kinst) if slon > 180.0: slon -= 360.0 saltgd = self._instObj.getAltitude(kinst) except: slatgd = float(form['inst_lat']) slon = float(form['inst_lon']) saltgd = float(form['inst_alt']) try: year = float(form['year']) except: year = 2000.0 argStr = ' %i ' + '%f ' * 13 if looker_options == 5: p1 = float(form['fl_az']) p2 = float(form['fl_el']) p3 = float(form['fl_range']) elif looker_options == 6: p1 = float(form['fl_lat']) p2 = float(form['fl_lon']) p3 = float(form['fl_alt']) elif looker_options == 7: p1 = float(form['fl_apex_lat']) p2 = float(form['fl_apex_lon']) p3 = 0.0 argStr = argStr % (looker_options, year, slatgd, slon, saltgd, p1, p2, p3, float(form['start_alt']), float(form['stop_alt']), float(form['step_alt']), 0.0, 0.0, 0.0) looker_cmd += argStr text = subprocess.check_output(looker_cmd.split()) return(text) elif looker_options in (8,): latList = numpy.arange(float(form['start_lat']), float(form['stop_lat']), float(form['step_lat'])) latList = latList.tolist() # check for null list if len(latList) == 0 and abs(float(form['start_lat']) - float(form['stop_lat'])) < 1.0E-6: latList = [float(form['start_lat'])] lonList = numpy.arange(float(form['start_lon']), float(form['stop_lon']), float(form['step_lon'])) lonList = lonList.tolist() # check for null list if len(lonList) == 0 and abs(float(form['start_lon']) - float(form['stop_lon'])) < 1.0E-6: lonList = [float(form['start_lon'])] altList = numpy.arange(float(form['start_alt']), float(form['stop_alt']), float(form['step_alt'])) altList = altList.tolist() # check for null list if len(altList) == 0 and abs(float(form['start_alt']) - float(form['stop_alt'])) < 1.0E-6: altList = [float(form['start_alt'])] requestedParms = ['gdlat', 'glon', 'gdalt'] + [str(parm.lower()) for parm in form['pList']] dtList = [form['datetime']] output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.isprint.MadCalculatorGrid(output, requestedParms, dtList, latList, lonList, altList) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip()) else: raise ValueError, 'Unknown looker_options %s' % (str(looker_options))
def runMadrigalCalculatorFromForm(
self, madCalculatorForm)
runMadrigalCalculatorFromForm returns the text output of madCalculator from the MadCalulatorForm
Inputs: madCalculatorForm - the django form that encapsulates all information from madrigal_calculator web page.
def runMadrigalCalculatorFromForm(self, madCalculatorForm): """runMadrigalCalculatorFromForm returns the text output of madCalculator from the MadCalulatorForm Inputs: madCalculatorForm - the django form that encapsulates all information from madrigal_calculator web page. """ requestedParms = madCalculatorForm['parameters'] requestedParms = ['gdlat', 'glon', 'gdalt'] + [str(parm) for parm in requestedParms] thisDT = madCalculatorForm['datetime'] min_latitude = madCalculatorForm['min_latitude'] max_latitude = madCalculatorForm['max_latitude'] delta_latitude = madCalculatorForm['delta_latitude'] min_longitude = madCalculatorForm['min_longitude'] max_longitude = madCalculatorForm['max_longitude'] delta_longitude = madCalculatorForm['delta_longitude'] min_altitude = madCalculatorForm['min_altitude'] max_altitude = madCalculatorForm['max_altitude'] delta_altitude = madCalculatorForm['delta_altitude'] latList = numpy.arange(min_latitude, max_latitude+0.001*delta_latitude, delta_latitude).tolist() lonList = numpy.arange(min_longitude, max_longitude+0.001*delta_longitude, delta_longitude).tolist() altList = numpy.arange(min_altitude, max_altitude+0.001*delta_altitude, delta_altitude).tolist() if 0 in (len(latList), len(lonList), len(altList)): raise ValueError, 'Got 0 length spatial range' output = os.path.join(tempfile.gettempdir(), 'tmp_%i.txt' % (random.randint(0,999999))) madrigal.isprint.MadCalculatorGrid(output, requestedParms, [thisDT], latList, lonList, altList) f = open(output) text = f.read() f.close() os.remove(output) return(text.strip())
class MadrigalWebFormat
MadrigalWebFormat defines the format of an web interface.
Information about how a web page is formatted is stored in this class. In particular, the possible derived parameters to display for a given format (such as Short or Comprehensive) are set in this class. Edit this class to create new formats or modify existing ones.
Non-standard Python modules used: None
No exceptions thrown
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Oct. 29, 2001
class MadrigalWebFormat: """MadrigalWebFormat defines the format of an web interface. Information about how a web page is formatted is stored in this class. In particular, the possible derived parameters to display for a given format (such as Short or Comprehensive) are set in this class. Edit this class to create new formats or modify existing ones. Non-standard Python modules used: None No exceptions thrown Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Oct. 29, 2001 """ # constants # Edit this data to change which parameters to display # or to add new formats # # Format Parameters # ------ ---------- _privateDict = {'Comprehensive': [ 'year', 'month', 'day', 'bmonth', 'bday', 'hour', 'min', 'sec', 'md', 'dayno', 'bhm', 'bhhmmss', 'ehhmmss', 'uth', 'b_uth', 'ut', 'ut1_unix', 'ut2_unix', 'beg_ut', 'slt', 'sltc', 'fyear', 'sunrise_hour', 'sunset_hour', 'conj_sunrise_h', 'conj_sunset_h', 'aplt', 'julian_date', 'gdalt', 'range', 'resl', 'azm', 'az1', 'az2', 'elm', 'el1', 'el2', 'gdlat', 'glon', 'szen', 'szenc', 'sdwht', 'beamid', 'bn', 'be', 'bd', 'magh', 'magd', 'magzu', 'bmag', 'bdec', 'binc', 'lshell', 'diplat', 'invlat', 'aplat', 'aplon', 'e_reg_s_lat', 'e_reg_s_lon', 'e_reg_s_sdwht', 'e_reg_n_lat', 'e_reg_n_lon', 'e_reg_n_sdwht', 'magconjlat', 'magconjlon', 'magconjsdwht', 'mlt', 'tsyg_eq_xgsm', 'tsyg_eq_ygsm', 'tsyg_eq_xgse', 'tsyg_eq_ygse', 'aacgm_lat', 'aacgm_long', 'aspect', 'cxr', 'cyr', 'czr', 'pl', 'snp3', 'chisq', 'gfit', 'mhdqc1', 'systmp', 'systmi', 'power', 'tfreq', 'popl', 'ne', 'nel', 'ti', 'te', 'tr', 'vo', 'ph+', 'pm', 'co', 'vdopp', 'dvdopp', 'dco', 'dpm', 'dph+', 'dvo', 'dtr', 'dte', 'dti', 'dpopl', 'dne', 'ne_model', 'nel_model', 'te_model', 'ti_model', 'vo_model', 'hmax_model', 'nmax_model', 'ne_modeldiff', 'nel_modeldiff', 'te_modeldiff', 'ti_modeldiff', 'vo_modeldiff', 'tn', 'tnm', 'tinfm', 'mol', 'nn2l', 'no2l', 'nol', 'narl', 'nhel', 'nhl', 'nn4sl', 'fa', 'pnrmd', 'pnrmdi', 'ut1', 'ut2', 'dut21', 'kinst', 'recno', 'kindat', 'fof2', 'dfa', 'dst', 'kp', 'ap', 'ap3', 'f10.7', 'fbar', 'pdcon', 'dpdcon', 'hlcon', 'dhlcon', 'ne_iri', 'nel_iri', 'tn_iri', 'te_iri', 'ti_iri', 'po+_iri', 'pno+_iri', 'po2+_iri', 'phe+_iri', 'ph+_iri', 'pn+_iri', 'bxgsm', 'bygsm', 'bzgsm', 'bimf', 'bxgse', 'bygse', 'bzgse', 'swden', 'swspd', 'swq'], 'Short': [ 'year', 'md', 'dayno', 'uth', 'b_uth', 'ut', 'beg_ut', 'lt', 'aplt', 'jdayno', 'gdalt', 'range', 'azm', 'az1', 'az2', 'elm', 'el1', 'el2', 'gdlat', 'glon', 'popl', 'nel', 'ti', 'te', 'tr', 'vo', 'ph+', 'pm', 'co', 'vdopp', 'dvdopp', 'dco', 'dpm', 'dph+', 'dvo', 'dtr', 'dte', 'dti', 'dpopl', 'dne', 'kp', 'ap', 'ap3', 'f10.7', 'fbar']} def getFormat(self, formatName): return self._privateDict[formatName]
Ancestors (in MRO)
Methods
def getFormat(
self, formatName)
def getFormat(self, formatName): return self._privateDict[formatName]