madrigal.metadata module
The metadata module provides access to all metadata about one particular madrigal database.
This metadata is presently read from the files in the metadata directory, and from the madrigal.cfg file. If that file cannot be found, hard-coded values set during installation are used instead. If the madrigal.cfg file is found at the location specified by either the madroot enviroment variable or at the hard-coded value of madroot set at installation, then the parameters are read from the madrigal.cfg file. Note that madroot with caps is only written once in this file before installation, since it will be automatically replaced, so it is referred to by MAD_ROOT or MAD+ROOT.
$Id: metadata_original.py 6207 2017-09-29 15:52:00Z brideout $
# -*- coding: utf-8 -*- """The metadata module provides access to all metadata about one particular madrigal database. This metadata is presently read from the files in the metadata directory, and from the madrigal.cfg file. If that file cannot be found, hard-coded values set during installation are used instead. If the madrigal.cfg file is found at the location specified by either the madroot enviroment variable or at the hard-coded value of madroot set at installation, then the parameters are read from the madrigal.cfg file. Note that madroot with caps is only written once in this file before installation, since it will be automatically replaced, so it is referred to by MAD_ROOT or MAD+ROOT. $Id: metadata_original.py 6207 2017-09-29 15:52:00Z brideout $ """ # standard python imports import StringIO import ConfigParser import os, sys, traceback import os.path import shutil import time import datetime import calendar import fnmatch import types import re import random import copy import glob import distutils.version # Madrigal imports import madrigal.admin import madrigal._derive # helper functions def getMadrigalUTFromDT(dt): """getMadrigalUTFromDT returns the number of seconds since midnight UT 1950-01-01 for datetime dt """ t1950 = 631152000.0 # offset from Unix time return(calendar.timegm(dt.timetuple()) + t1950) def getMadrigalUTFromDate(year, month, day, hour, minute, second, microsecond): """getMadrigalUTFromDate returns the number of seconds since midnight UT 1950-01-01 for date specified in arguments """ dt = datetime.datetime(year, month, day, hour, minute, second, microsecond) return(getMadrigalUTFromDT(dt)) def getUnixUTFromDT(dt): """getUnixUTFromDT returns the number of float seconds since midnight UT 1970-01-01 for datetime dt """ return(calendar.timegm(dt.timetuple()) + dt.microsecond/1.0E6) def getUnixUTFromDate(year, month, day, hour, minute, second, microsecond): """getUnixUTFromDate returns the number of seconds since midnight UT 1970-01-01 for date specified in arguments """ dt = datetime.datetime(year, month, day, hour, minute, second, microsecond) return(getUnixUTFromDT(dt)) class MadrigalDB: """MadrigalDB is an object that provides access to an entire Madrigal database. This object provides complete high-level access to an entire Madrigal database. Presently, all its information comes from madrigal.cfg, or another path passed in by the user. If env variable madroot is not set, or if madrigal.cfg cannot be opened, the default values automatically editted during installation are used Usage example:: import madrigal.metadata try: test = madrigal.metadata.MadrigalDB() except madrigal.admin.MadrigalError, e: print e.getExceptionStr() else: print test.toString() Non-standard Python modules used: None MadrigalError exception thrown if: 1. If the constructor is called with an configuration file path as an argument, and that file cannot be read. 2. If the configuration file cannot be parsed by ConfigParser. 3. If any of the following keys cannot be found in the configuration file: *__MAD_SERVER *__MAD_SERVERROOT *__SITE_ID *__HTML_STYLE *__INDEX_HEAD *__CON_TACTLINK Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Oct. 4, 2001 """ # lines that are edited during installation __hardCodeMadRoot = '/Users/brideout/madroot3_2' __hardCodeMadServer = 'localhost:8000' __hardCodeMadServerRoot = '' __hardCodeSiteId = '999' __hardCodeHtmlStyle = '<BODY BGCOLOR=#FFFFFF LINK=#008000 VLINK=#003366>' __hardCodeIndexHead = 'Welcome to the Madrigal Database at mad31' __hardCodeContact = '<A HREF="MAILTO:brideout@mit.edu">Madrigal administator</A><BR>' __hardCodeMailServer = 'localhost' #constants __webProtocol = "http" """ Change the above string to use another web protocol such as https. """ __binDir = "/bin" """ Sets the relative path from madrigal root to the bin directory. """ __metadataDir = "/metadata" """ Sets the relative path from madrigal root to the metadata directory. """ __experimentDir = "/experiments" """ Sets the relative path from madrigal root to the experiment directory. """ __MAD_ROOT = "MAD" + "ROOT" """ Sets the name of the environment variable that points to the madrigal root directory. """ __confFileName = "madrigal.cfg" """ Sets the name of the default madrigal configuration file. """ __MAD_SERVER = "MAD" + "SERVER" """ Sets the key name in the configuration file to find the main madrigal url. """ __MAD_SERVERROOT = "MAD" + "SERVERROOT" """ Sets the key name in the configuration file to find the top level directory. """ __SITE_ID = "SITE" + "ID" """ Sets the key name in the configuration file to find the site id. """ __HTML_STYLE = "HTML" + "STYLE" """ Sets the key name in the configuration file to find the html body style tag. """ __INDEX_HEAD = "INDEX" + "HEAD" """ Sets the key name in the configuration file to find the heading in the top level madrigal page. """ __CON_TACTLINK = "CON" + "TACT" """ Sets the key name in the configuration file to find the contact link line. """ __MAIL_SERVER = "MAIL" + "SERVER" """ Sets the key name in the configuration file to find the mailserver. """ __PYTHON_EXE = "PYTHON" + "EXE" """ Sets the key name in the configuration file to find the python executable. """ def __init__(self, initFile=None): """__init__ initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString). Inputs: String representing the full path to the configuration file. Default is None, in which case configuration file is $(__MAD_ROOT)/__confFileName (now madrigal.cfg). Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown if error (see class description for specific conditions). Notes: Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile, this constructor goes about loading some basic data about the database, including information such as: -the database utility directory -the www home base Implemented using ConfigParser module. This reads ini type files. The only difference between ini files and madrigal.cfg is that ini files are broken into sections of the form: [sectionTitle] key1 = value1 key2 = value2 Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal" is appended to the beginning of the file. """ # if madroot not set, use hard coded madroot self.__madRootDir = os.environ.get(self.__MAD_ROOT) if (self.__madRootDir == None): self.__madRootDir = self.__hardCodeMadRoot # Set configuration file if (initFile == None): self.__confFilePath = self.__madRootDir + "/" + self.__confFileName else: self.__confFilePath = initFile # open configuration file try: self.__confFile = open(self.__confFilePath, "r") except IOError: # can't read from file - use all hard-coded values self.__initFromHardCode() self.__finishInit() return # create Parser using standard module ConfigParser self.__parser = ConfigParser.ConfigParser() # read conf file into a StringIO with "[madrigal]\n" section heading prepended strConfFile = StringIO.StringIO("[madrigal]\n" + self.__confFile.read()) # parse StringIO configuration file try: self.__parser.readfp(strConfFile) except: raise madrigal.admin.MadrigalError("Unable to parse configuration file " + self.__confFilePath, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # read information from configuration file self.__readConfFile() # close conf file self.__confFile.close() self.__finishInit() # end __init__ def __finishInit(self): """__finishInit is a private helper function that finishes initialization by combining attributes to form new ones. Inputs: None Returns: Void. Affects: Initializes class member variables that are combinations of existing ones. Exceptions: None. """ # combine information # Set the database utility directory self.__databaseUtilityDirectory = self.__madRootDir + self.__binDir # Set the wwwHomeBase self.__wwwHomeBase = self.__webProtocol + "://" + self.__mainUrl # Set the top level url if self.__topLevel.strip() not in ('', '.'): self.__topLevelUrl = self.__wwwHomeBase + '/' + self.__topLevel else: self.__topLevelUrl = self.__wwwHomeBase # Set contactEmail by stripping ContactLink line # Look for colon at end of Mailto: begIndexEmail = self.__contactlink.find(':') # look for "> at end of email address endIndexEmail = self.__contactlink.find('">') # if not found, try '> if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('\'>') # if still not found, try > alone if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('>') #check that both were found if begIndexEmail != -1 and endIndexEmail != -1: self.__contactEmail = self.__contactlink[begIndexEmail + 1 : endIndexEmail] if not self.__isValidEmail(self.__contactEmail): self.__contactEmail = None elif self.__isValidEmail(self.__contactlink): # user just entered an email self.__contactEmail = self.__contactlink # make contactlink proper link self.__contactlink = '<A HREF="MAILTO:%s”>%s</A>' % (self.__contactEmail, self.__contactEmail) else: self.__contactEmail = None # set pythonexe to default self.__pythonexe = self.__databaseUtilityDirectory + '/python' # end __init__ def __readConfFile(self): """__readConfFile is a private helper function that reads information from the parsed config file. Inputs: None Returns: Void. Affects: Initializes class member variables that are found in the config file. Exceptions: MadrigalError thrown if any key not found. """ # get the main url info try: self.__mainUrl = self.__parser.get("madrigal", self.__MAD_SERVER) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__MAD_SERVER + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the top level info try: self.__topLevel = self.__parser.get("madrigal", self.__MAD_SERVERROOT) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__MAD_SERVERROOT + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the site id try: self.__siteIdValue = self.__parser.get("madrigal", self.__SITE_ID) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__SITE_ID + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the htmlstyle try: self.__htmlstyle = self.__parser.get("madrigal", self.__HTML_STYLE) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__HTML_STYLE + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the indexhead try: self.__indexhead = self.__parser.get("madrigal", self.__INDEX_HEAD) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__INDEX_HEAD + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the contactlink try: self.__contactlink = self.__parser.get("madrigal", self.__CON_TACTLINK) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__CON_TACTLINK + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the mailserver try: self.__mailserver = self.__parser.get("madrigal", self.__MAIL_SERVER) except: # no error in this case - simply default to localhost # this avoids an error if the old version of madrigal.cfg is present self.__mailserver = 'localhost' # end __readConfFile def __initFromHardCode(self): """__initFromHardCode is a private helper function that reads information from the automatically editted lines. Inputs: None Returns: Void. Affects: Initializes class member variables that are found in the constants at the top of this file. These constants were set during installation. Exceptions: None. """ self.__mainUrl = self.__hardCodeMadServer self.__topLevel = self.__hardCodeMadServerRoot self.__siteIdValue = int(self.__hardCodeSiteId) self.__htmlstyle = self.__hardCodeHtmlStyle self.__indexhead = self.__hardCodeIndexHead self.__contactlink = self.__hardCodeContact # mailserver may be empty if len(self.__hardCodeMailServer) > 0: self.__mailserver = self.__hardCodeMailServer else: self.__mailserver = 'localhost' # public methods def getDatabaseUtilityDirectory(self): """getDatabaseUtilityDirectory returns the full path to the database utility directory. Inputs: None Returns: String representing the full path to the database utility directory. (eg, /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__databaseUtilityDirectory def getWWWHomeBase(self): """getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__wwwHomeBase def getMadServer(self): """getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__mainUrl def getTopLevelUrl(self): """getTopLevelUrl returns the full url of the top level directory in main database website. Inputs: None Returns: String representing the full url to the top level directory in main database website. (eg, http://haystack.mit.edu/madrigal) Affects: Nothing Exceptions: None """ return self.__topLevelUrl def getRelativeTopLevel(self): """getRelativeTopLevel returns the relative url of the top level directory in main database website. Inputs: None Returns: String representing the relative url to the top level directory in main database website. (eg, madrigal) Affects: Nothing Exceptions: None """ return self.__topLevel def getSiteID(self): """getSiteID returns the site id number. Inputs: None Returns: The site id (integer) of the madrigal installation. Affects: Nothing Exceptions: If non-integer found """ try: return int(self.__siteIdValue) except: raise madrigal.admin.MadrigalError("Site id not an integer in madrigal configuration file " + \ self.__confFile, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def getMadrootEnvVarName(self): """getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT). Inputs: None Returns: The name of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__MAD_ROOT def getMadroot(self): """getMadroot returns the value of the environment variable __MAD_ROOT. Inputs: None Returns: The value of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__madRootDir def getMetadataDir(self): """getMetadataDir returns the metadata directory. Inputs: None Returns: The full metadata directory path. (eg. /opt/madrigal/metadata) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__metadataDir def getExperimentDir(self): """getExperimentDir returns the main experiment directory. No longer guarenteed to be unique,but will hold test files and geophysical experiments. Inputs: None Returns: The full experiment directory path. (eg. /opt/madrigal/experiments) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__experimentDir def getExperimentDirs(self): """getExperimentDirs returns a list of full paths of all valid experiment directories. Inputs: None Returns: list of full paths of all valid experiment directories. Valid experiment directories are those of the form of the regular expression $/Users/brideout/madroot3_2/experiments[0-9]* Affects: Nothing Exceptions: None """ reStr = '/experiments[0-9]*$' possibleExpDirs = glob.glob(os.path.join(self.__madRootDir, 'experiments*')) retList = [] for possibleExpDir in possibleExpDirs: result = re.search(reStr, possibleExpDir) if result: retList.append(possibleExpDir) return(retList) def getBinDir(self): """getBinDir returns the madrigal bin directory. Inputs: None Returns: The madrigal bin directory path. (eg /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__binDir def getHtmlStyle(self): """getHtmlSyle returns the default html body tag for the site. Inputs: None Returns: The default html body tag for the site. (eg. <BODY BGCOLOR=#FFFF88 LINK=#008000 VLINK=#003366>) Affects: Nothing Exceptions: None """ return self.__htmlstyle def getBackgroundColor(self): """getBackgroundColor returns the string representing background color from getHtmlStyle Inputs: None Returns the color string from the body tag in madrigal.cfg. For example, will return '#FFFF88; if <BODY BGCOLOR=#FFFFFF LINK=#008000 VLINK=#003366> is <BODY BGCOLOR=#FFFF88 LINK=#008000 VLINK=#003366> Returns empty string if no BGCOLOR= """ line = self.getHtmlStyle().upper() items = line.split() for item in items: index = item.find('BGCOLOR=') if index != -1: return(item[len('BGCOLOR='):]) return('') # not found def getIndexHead(self): """getIndexHead returns the heading of the top level madrigal page. Inputs: None Returns: The heading of the top level madrigal page. (eg. Welcome to the Madrigal Database <BR> at Ishtar) Affects: Nothing Exceptions: None """ return self.__indexhead def getContactLink(self): """getContactLink returns contact email link tag (see getContactEmail for the email alone). Inputs: None Returns: The contact email link tag. (eg. <A HREF="MAILTO:brideout@haystack.mit.edu">madrigal@haystack</A><BR>) Affects: Nothing Exceptions: None """ return self.__contactlink def getContactEmail(self): """getContactEmail returns the email address of the site administrator. Inputs: None Returns: The email address of the site administrator. Affects: Nothing Exceptions: None """ if not self.__contactEmail is None: return(self.__contactEmail) else: madSite = madrigal.metadata.MadrigalSite() return(madSite.getSiteEmail(self.getSiteID())) def getMailserver(self): """getMailserver returns the mailserver name. Inputs: None Returns: The mailserver name. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to localhost Affects: Nothing Exceptions: None """ return self.__mailserver def getPythonExecutable(self): """getPythonExecutable returns the full path to the python executable. Inputs: None Returns: the full path to the python executable. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to madroot/bin/python Affects: Nothing Exceptions: None """ return self.__pythonexe def getLocalRulesOfRoad(self): """getLocalRulesOfRoad returns the local rules of the road. Inputs: None Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that. Else returns a default rules_of_road statement Affects: Nothing Exceptions: None """ default_rules_of_road = 'Use of the Madrigal Database is generally subject to the ' + \ 'CEDAR Rules-of-the-Road . ' + \ 'Prior permission to access the data is not required. However, the user is required to establish ' + \ 'early contact with any organization whose data are involved in the project to discuss the ' + \ 'intended usage. Data are often subject to limitations which are not immediately evident to ' + \ 'new users. Before they are formally submitted, draft copies of all reports and publications ' + \ 'must be sent to the contact scientist at all data-supplying organizations along with an offer ' + \ 'of co-authorship to scientists who have provided data. This offer may be declined. The ' + \ 'Database and the organizations that contributed data must be acknowledged in all reports and ' + \ 'publications, and whenever this data is made available through another database. If you have ' + \ 'any questions about appropriate use of these data, contact <a href="mailto:%s">%s</a>' % (self.getContactEmail(), self.getContactEmail()) localRules = None try: f = open(os.path.join(self.getMadroot(), 'local_rules_of_the_road.txt')) localRules = f.read() f.close() except: pass if localRules == None: return default_rules_of_road else: return localRules def getFullPathFromPartial(self, partialPath): """getFullPathFromPartial returns the full path to a file or directory based on a partial path. Follows the rule that if partialPath begins with [/]experiments, then the full path is madroot + partialPath. Otherwise (pre madrigal 2.6) full path is madroot + experiments + partialPath Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10) Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10) """ if partialPath[:12].find('experiments') != -1: fullPath = os.path.join(self.getMadroot(), partialPath) else: fullPath = os.path.join(self.getMadroot(), 'experiments', partialPath) return(fullPath) def getExpList(self, expName = None, kinstList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, showIgnoredExperiments = False, enforcePathConvention = False): """getExpList returns a list of full experiment directory names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. startDate: a datetime date. If None (default), do not reject any experiments. endDate: a datetime date. If None (default), do not reject any experiments. startDayOfYear: a Julian day number (1-366) after which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any experiments. endDayOfYear: a Julian day number (1-366) before which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any experiments. showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If False (the default), only required path in the form 1999/mlh/*. Returns: a list of full experiment directory names that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None expList = [] # get all experiment directories expDirList = self.getExperimentDirs() # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: os.path.walk(thisExpDir, self.__getExperiments, (expName, kinstList, startDate, endDate, startDayOfYear, endDayOfYear, expList, showIgnoredExperiments, enforcePathConvention)) return expList def getFileList(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, enforcePathConvention = 0, includeNonDefault = 0, includeNonMadrigal = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0, path = None): """getFileList returns a list of full file names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. path if not None (the default), search only experiment directory set by path. If None, search add experiments. Returns: a list of full path file names (strings) that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None fileList = [] # get all experiment directories if path is None: expDirList = self.getExperimentDirs() else: expDirList = [path] # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: os.path.walk(thisExpDir, self.__getFiles, (expName, kinstList, kindatList, startDate, endDate, startDayOfYear, endDayOfYear, fileList, publicAccessOnly, enforcePathConvention, includeNonDefault, includeNonMadrigal, appendKinst, appendStartTime, includeRealtime)) return fileList def getFileListFromMetadata(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, includeNonDefault = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0): """getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only. This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't search the actual experiment directories. It is therefore faster, but less robust. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and starttime. Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None # the list to be returned fileList = [] # load a Madrigal experiment object madExpObj = MadrigalExperiment(self) # load a Madrigal metafile object madFileObj = MadrigalMetaFile(self) # convert input arguments to needed forms if expName != None: expNameArg = expName.replace(' ', '_') if startDate != None: startDateTime = madrigal.metadata.getMadrigalUTFromDate(startDate[0], startDate[1], startDate[2], startDate[3], startDate[4], startDate[5], 0) if endDate != None: endDateTime = madrigal.metadata.getMadrigalUTFromDate(endDate[0], endDate[1], endDate[2], endDate[3], endDate[4], endDate[5], 0) # loop through all experiments to create four lists: # acceptedExpIdList, acceptedExpKinstList, acceptedExpStartTimeList, and acceptedExpDirList acceptedExpIdList = [] acceptedExpKinstList = [] acceptedExpStartTimeList = [] acceptedExpDirList = [] position = -1 while 1: position += 1 # first position is zero thisName = madExpObj.getExpNameByPosition(position) #check if we're at the end if thisName == None: break # check experiment access thisSecurity = madExpObj.getSecurityByPosition(position) if thisSecurity == -1: continue elif publicAccessOnly == 0: if thisSecurity not in (0,1): continue elif publicAccessOnly == 1: if thisSecurity != 0: continue elif publicAccessOnly == 2: if thisSecurity not in (0,2): continue # apply expName filter if desired if expName != None: # since we are using file name matching, all spaces are # converted to underscores thisName = thisName.replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(thisName.lower(), expNameArg.lower()): continue # apply kinstList filter thisKinst = madExpObj.getKinstByPosition(position) if kinstList != None: if thisKinst not in kinstList: continue # startDate filter - we always need thisStartTime whether or not filter used thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) thisStartTime = madrigal.metadata.getMadrigalUTFromDate(thisStartDate[0], thisStartDate[1], thisStartDate[2], thisStartDate[3], thisStartDate[4], thisStartDate[5], 0) if endDate != None: if thisStartTime > endDateTime: continue # endDate filter if endDate != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) thisEndTime = madrigal.metadata.getMadrigalUTFromDate(thisEndDate[0], thisEndDate[1], thisEndDate[2], thisEndDate[3], thisEndDate[4], thisEndDate[5], 0) if startDate != None: if thisEndTime < startDateTime: continue # apply startDayOfYear filter if startDayOfYear != None: thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) if thisStartDate[7] < startDayOfYear: # index 7 refers to Day of year continue # apply endDayOfYear filter if endDayOfYear != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) if thisEndDate[7] > endDayOfYear: # index 7 refers to Day of year continue # this experiment has made it through all filters, append its id, kinst, startTime, and dir acceptedExpIdList.append(madExpObj.getExpIdByPosition(position)) acceptedExpKinstList.append(thisKinst) acceptedExpStartTimeList.append(thisStartTime) # find the directory based on url dir = self.getMadroot() if dir[-1] != '/': dir += '/' url = madExpObj.getExpUrlByPosition(position) index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: dir += 'experiments/' dir += url[index+8:] acceptedExpDirList.append(dir) # now loop through the file object to find all files position = -1 while 1: position += 1 # first position is zero thisExpId = madFileObj.getExpIdByPosition(position) #check if we're at the end if thisExpId == None: break # skip this file if expId not in acceptedExpIdList if thisExpId not in acceptedExpIdList: continue # apply kindatList filter thisKindat = madFileObj.getKindatByPosition(position) if kindatList != None: if thisKindat not in kindatList: continue # check file access if madFileObj.getAccessByPosition(position) != 0: if publicAccessOnly in (1,2): continue # apply includeNonDefault filter if includeNonDefault == 0: category = madFileObj.getCategoryByPosition(position) if includeRealtime == 0: if category != 1: # not default continue else: if category not in (1,4): # not default or realtime continue # this file has been accepted by all filters - first, get its experiment index expIndex = acceptedExpIdList.index(thisExpId) thisFilename = acceptedExpDirList[expIndex] + '/' + madFileObj.getFilenameByPosition(position) # append result to fileList according to appendKinst and appendStartTime if appendKinst == 0 and appendStartTime == 0: fileList.append(thisFilename) elif appendKinst == 1 and appendStartTime == 0: fileList.append((thisFilename, acceptedExpKinstList[expIndex])) elif appendKinst == 0 and appendStartTime == 1: fileList.append((thisFilename, acceptedExpStartTimeList[expIndex])) else: fileList.append((thisFilename, acceptedExpKinstList[expIndex], acceptedExpStartTimeList[expIndex])) return fileList def setFileAccess(self, expDirectory, accessMode): """setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Inputs: expDirectory: The full path to a directory in the experiment directory. That is, it may be madroot/experiments[0-9]* or any directory under it. accessMode: either 0 for public access, or 1 for private access. Returns: None Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Exceptions: If accessMode is not 1 or 0. """ if (accessMode != 0 and accessMode != 1): raise madrigal.admin.MadrigalError('MadrigalDB.setFileAccess called with accessMode = ' + \ str(accessMode) + ', must be either 0 or 1', None) # walk the experiments directory to find all files meeting criteria os.path.walk(expDirectory, self.__setFileAccess, accessMode) def tarExperiments(self, tarFileName, startDate = None, endDate = None, excludePrivData = 0, ignoreDirCon = 1, includeNonDefData = 0, onlyData = 0, filetype = 0, verbose = 0): """tarExperiments creates a tar file containing files from madroot/experiments[0-9]*. Note: this method sometimes requires the modification of the fileTab.txt files found in the experiments directory. This is because some data files might be excluded, so that the fileTab.txt file will no longer be accurate. Because of this, all files to be tar'ed will be copied to /tmp/temp<random num>/experiments, where the fileTab.txt files will be modified. When done, this temp dir will be deleted. Inputs: tarFileName: The full path to a tar file to be created. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. excludePrivData: if 1, allow data marked as private to be omitted (and the line from the fileTab.txt to be removed). If 0 (the default), all data, public and private, will be included. ignoreDirCon: if 1, ignore convention that directory must be in form 1999/mlh/03sep99 (the default). If 0 , reject non-standard directories. includeNonDefData: if 1, include all files listed in fileTab.txt. If 0 (the default), reject non-default files, and modify fileTab.txt to remove non-default listings. onlyData: if 1, reject all files not listed in fileTab.txt. If 0 (the default), accept all files in a directory not mentioned in fileTab.txt. filetype: format to save data files as. Default 0 is to leave present format unchanged. <type> is an integer as follows: type = 0 Leave present format unchanged (default) type = 1 Madrigal type = 2 Blocked Binary type = 3 Cbf type = 4 Unblocked binary type = 5 Ascii verbose: if 1, print to std out the list of files included (relative path). If 0, (the default) print nothing. Returns: None Affects: created tar file tarFileName of selected files from madroot/experiments. Exceptions: If unable to read any experiment file. """ if ignoreDirCon == 1: enforcePathConvention = 0 else: enforcePathConvention = 1 if onlyData == 1: includeNonMadrigal = 0 else: includeNonMadrigal = 1 # create a random temp dir tempDir = '/tmp/temp' + str(random.randrange(1,10000000)) if verbose == 1: print 'Creating list of files to tar...' # get list of files to tar tarFileList = self.getFileList(None, None, None, startDate, endDate, None, None, excludePrivData, enforcePathConvention, includeNonDefData, includeNonMadrigal) # now create a new list of filenames, using relative paths relTarFileList = [] for file in tarFileList: newFilename = file.split(self.getMadroot() + '/')[1] relTarFileList.append(newFilename) # copy all these files to tempDir if verbose == 1: print 'The following is a list of files included:' for file in relTarFileList: # make sure dir exists try: os.makedirs(tempDir + '/' + os.path.dirname(file)) except: pass # now copy file shutil.copyfile(self.getMadroot() + '/' + file, tempDir + '/' + file) if verbose == 1: print '\t' + file if onlyData: # now check if that data file needs to be converted to another filetype # since onlyData is set, convert every file if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + file, tempDir + '/' + file + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + file + '.backup 1 100000000 -o ' + \ tempDir + '/' + file + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + file + '.backup') if verbose == 1: print 'Modifying fileTab.txt files if needed...' # modify fileTab.txt files, if required if (not onlyData) and (excludePrivData or not includeNonDefData): # first loop through each fileTab.txt file in list just to check permissions for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # make sure fileTab.txt is writable if not os.access(tempDir + '/' + filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(filename), None) # make sure the directory is writable if not os.access(os.path.dirname(tempDir + '/' + filename), os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(os.path.dirname(filename)), None) # no exceptions raised - all permissions are okay # this time loop through and modify the fileTab.txt files for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # create a MadrigalMetaFile object fileMeta = MadrigalMetaFile(self, tempDir + '/' + filename) # loop through each file name to see if its in tarFileList: fileNum = 0 while 1: madFilename = fileMeta.getFilenameByPosition(fileNum) if madFilename == None: break # get madRelFilename madRelFilename = os.path.dirname(filename) + '/' + madFilename # if its not in relTarFileList, delete it from fileMeta if not madRelFilename in relTarFileList: fileMeta.deleteRowByFilename(madFilename) fileNum = 0 continue else: # we know madRelFilename is a data file since its in fileTab.txt - # now check if that data file needs to be converted to another filetype if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + madRelFilename, tempDir + '/' + madRelFilename + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + madRelFilename + '.backup 1 100000000 -o ' + \ tempDir + '/' + madRelFilename + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + madRelFilename + '.backup') # get next madFilename fileNum = fileNum + 1 # if fileMeta not empty, write it out if fileMeta.getFileCount() > 0: fileMeta.writeMetadata() # else if its empty, simply delete it else: os.remove(tempDir + '/' + filename) # done modifying fileTab.txt - ready to tar # need to change working directory to tempDir to get relative paths in tar. # Since this directory will soon be deleted, will need to change it back to # whatever it is now when we're done pwd = os.getcwd() # check if tarFileName is absolute or relative to pwd if tarFileName[0] != '/': tarFileName = pwd + '/' + tarFileName if verbose == 1: print 'Creating tar file...' os.chdir(tempDir) os.system('tar -cf ' + tarFileName + ' experiments*') os.chdir(pwd) if verbose == 1: print 'Removing temp files...' # finally remove temp dir try: shutil.rmtree(tempDir) except: raise madrigal.admin.MadrigalError('In tarExperiments could not remove dir ' + tempDir, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def listFileTimes(self, expDir=None, relative=True): """listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory Inputs: expDir - the particular subdirectory of an experiment directory to list. If None (the default), will list all files for all experiment directories. May be an absolute path, or may start with experiments[0-9]*. relative - if True (the default) give the path relative to the experiments[0-9]* directory. If False, give full path Returns: a list of tuples, where each tuple has 1. the file path, and 2. a UT datetime object of the last file modification. Exceptions: raised if expDir is not a valid experiments directory or subdirectory """ expDirs = self.getExperimentDirs() if expDir == None: dirsToExamine = expDirs else: # verify a valid directory found = False if expDir[-1] == '/': expDir = expDir[:-1] # strip trailing / if expDir[0:11] == 'experiments': # convert to absolute path expDir = os.path.join(self.getMadroot(), expDir) for thisDir in expDirs: if expDir.find(thisDir) != -1: found = True if found: dirsToExamine = [expDir] else: raise ValueError, 'expDir %s not a valid experiment directory' % (expDir) retList = [] for dirToExamine in dirsToExamine: for root, dirs, files in os.walk(dirToExamine): for name in files: fullname = os.path.join(root, name) relIndex = fullname[len(self.__madRootDir):].find('/experiments') relativeName = fullname[relIndex + 1 + (len(self.__madRootDir)):] ts = os.stat(fullname).st_mtime if relative: retList.append((relativeName, datetime.datetime.utcfromtimestamp(ts))) else: retList.append((fullname, datetime.datetime.utcfromtimestamp(ts))) return(retList) def getKinstKindatConfig(self, kinst, kindat, iniFile=None): """getKinstKindatConfig gets information for the /Users/brideout/madroot3_2/cachedFiles.ini needed to create Madrigal3 files if not specified. Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them. Inputs: kinst - the instrument kinst (integer) kindat - the data kindat (integer) iniFile - the ini file to use. If None, uses default ini file $/Users/brideout/madroot3_2/cachedFiles.ini Returns: a tuple with three items: 1. a list of extra parameters (string mnemonics) 2. a list of independent spatial parameters 3. a list of array splitting parameters Algorithm: 1. If iniFile == None and no default file, returns ([], [], []) 2. Searches ini file for section [%i] % (kinst). If not found, returns ([], [], []) 3. Searches right section for key %i_parms % (kindat). If not found, searches for default_parms. If not found, extra parameters are [] 4. Searches right section for key %i_formats % (kindat). If not found, searches for default_formats. If not found, indepedent spatial parms and extra splitting parameters are [] and []. If found, then parses dictionary, and returns key 'array'. If value has only one item, returns it as a one item list of independent spatial parameters. If two items, they are ind spatial parms and array splitting parms. """ if not iniFile: thisIniFile = os.path.join(self.getMadroot(), 'cachedFiles.ini') if not os.access(thisIniFile, os.R_OK): return(([], [], [])) else: thisIniFile = iniFile instSection = '%i' % (kinst) parser = ConfigParser.SafeConfigParser() parser.read(thisIniFile) if not parser.has_section(instSection): return(([], [], [])) extraParms = [] indSpatialParms = [] arraySplitParms = [] # get extra parms if parser.has_option(instSection, '%i_parms' % (kindat)): extraParms = parser.get(instSection, '%i_parms' % (kindat)) extraParms = extraParms.split(',') elif parser.has_option(instSection, 'default_parms'): extraParms = parser.get(instSection, 'default_parms') extraParms = extraParms.split(',') # make sure no empty parms snuck in finalExtraParms = [] for extraParm in extraParms: if len(extraParm.strip()) > 0: finalExtraParms.append(extraParm.strip()) # get format dict formatDict = None if parser.has_option(instSection, '%i_formats' % (kindat)): formatDict = parser.get(instSection, '%i_formats' % (kindat)) formatDict = eval(formatDict) elif parser.has_option(instSection, 'default_formats'): formatDict = parser.get(instSection, 'default_formats') formatDict = eval(formatDict) if not formatDict is None: if formatDict.has_key('array'): value = formatDict['array'] if type(value) in (types.ListType, types.TupleType): indSpatialParms = value[0] arraySplitParms = value[1] else: indSpatialParms = [value] return((finalExtraParms, indSpatialParms, arraySplitParms)) def toString(self): """toString returns a simple string representation of a MadrigalDB object. Inputs: None Returns: String describing a simple representation of a MadrigalDB object. Affects: Nothing Exceptions: None """ output = "Object type: MadrigalDB\n" output += "Database utility directory = " + self.getDatabaseUtilityDirectory() + "\n" output += "WWW home base = " + self.getWWWHomeBase() + "\n" output += "Server name = " + self.getMadServer() + "\n" output += "Top level url = " + self.getTopLevelUrl() + "\n" output += "Relative Top level url = " + self.getRelativeTopLevel() + "\n" output += "Site ID = " + str(self.getSiteID()) + "\n" output += "MAD_ROOT env. variable name = " + self.getMadrootEnvVarName() + "\n" output += "MAD_ROOT env. variable value = " + self.getMadroot() + "\n" output += "Madrigal metadata dir = " + self.getMetadataDir() + "\n" output += "Madrigal bin dir = " + self.getBinDir() + "\n" output += "Madrigal html body style = " + self.getHtmlStyle() + "\n" output += "Madrigal top level heading = " + self.getIndexHead() + "\n" output += "Madrigal contact link = " + self.getContactLink() + "\n" output += "Madrigal contact email = " + str(self.getContactEmail()) + "\n" output += "Madrigal mailserver = " + self.getMailserver() + "\n" output += "Madrigal python exe = " + self.getPythonExecutable() + "\n" return output def isTestExperiment(self, url, siteId=None): """isTestExperiment returns True if the given experiment url is a test of this Madrigal server. Url can be either real or form in expTab.txt, or can be the experiment directory. If siteId not given, use local site id. """ # Skip test experiments if siteId == None: siteId = self.getSiteID() if url.find('1998/mlh/20jan98') != -1 and siteId != 1: return(True) if url.find('1997/aro/06jan97') != -1 and siteId != 7: return(True) if url.find('1997/lyr/08apr97') != -1 and siteId != 2: return(True) if url.find('1997/son/06jan97') != -1 and siteId != 3: return(True) if url.find('1995/jro/01feb95') != -1 and siteId != 6: return(True) if url.find('1998/jro/27apr98') != -1 and siteId != 6: return(True) return(False) def __str__(self): """ __str__ simply calls toString """ return (self.toString()) def __isValidEmail(self, emailStr): """ __isValidEmail is a private helper function that does some checking to ensure a valid email address was found. Inputs: emailStr - email address string to verify Returns: 1 if no problems, 0 if not valid. Affects: Nothing Exceptions: None """ emailStr = emailStr.strip() if emailStr.find(' ') != -1: return 0 if emailStr.find('"') != -1: return 0 if emailStr.find('<') != -1: return 0 if emailStr.find('>') != -1: return 0 if emailStr.find('/') != -1: return 0 if emailStr.find(':') != -1: return 0 if emailStr.find('@') == -1: return(0) # otherwise okay return 1 def __getExperiments(self, arg, dirname, names): """ __getExperiment is a private helper function called by os.path.walk in getExpList. __getExperiments is called for each sub-directory in the experiments directory. Its purpose is to add any experiment directory paths from that directory that match the search criteria to the expList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. startDate: a python datetime in UTC. If None, do not reject any files. endDate: a python datetime in UTC. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. expList: the list of valid experiment paths to which is appended any newly-found valid experiment directories showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If False (the default), only required path in the form 1999/mlh/*. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __startDate = 2 __endDate = 3 __startDayOfYear = 4 __endDayOfYear = 5 __expList = 6 __showIgnored = 7 __enforcePath = 8 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt if not 'expTab.txt' in names: return # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply startDate filter expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError, 'Error getting expStartDate from dir %s' % (dirname) time1 = datetime.datetime(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5]) if arg[__endDate] != None: if time1 > arg[__endDate]: # stop going down tree names[:] = [] return # apply endDate filter if arg[__startDate] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() time1 = datetime.datetime(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5]) if time1 < arg[__startDate]: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply ignore filter if expMeta.getSecurityByPosition() == -1 and not arg[__showIgnored]: # stop going down tree names[:] = [] return # now reject paths not matching present path convention if not enforcePathConvention if not arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention if arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay arg[__expList].append(dirname) def __getFiles(self, arg, dirname, names): """ __getFiles is a private helper function called by os.path.walk in getFileList. __getFiles is called for each sub-directory in the experiments directory. Its purpose is to add any file paths from that directory that match the search criteria to the fileList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None, all kindat values are accepted. startDate: a python date/time in UTC (see time module - actually a tuple of nine integers) after which to accept files. If None, do not reject any files. endDate: a python date/time in UTC (see time module - actually a tuple of nine integers) before which to accept files. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. fileList: the list of valid file paths to which is appended any newly-found valid file paths publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __kindatList = 2 __startDate = 3 __endDate = 4 __startDayOfYear = 5 __endDayOfYear = 6 __fileList = 7 __pubAccessOnly = 8 __enforcePath = 9 __includeNonDef = 10 __includeNonMad = 11 __appendKinst = 12 __appendStartTime = 13 __includeRealtime = 14 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt and fileTab.txt if not includeNonMadrigal if 'expTab.txt' in names: self.__hasExpTab = 1 else: self.__hasExpTab = 0 if 'fileTab.txt' in names: self.__hasFileTab = 1 else: self.__hasFileTab = 0 if arg[__includeNonMad] == 0: if not self.__hasExpTab or not self.__hasFileTab: return # apply all filters relating to expTab.txt if self.__hasExpTab if self.__hasExpTab: # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply date filters expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError, 'Error getting expStartDate from dir %s' % (dirname) expStartUt = madrigal.metadata.getMadrigalUTFromDate(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5], 0) expEndDate = expMeta.getExpEndDateTimeByPosition() expEndUt = madrigal.metadata.getMadrigalUTFromDate(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5], 0) if arg[__startDate] != None: time1 = madrigal.metadata.getMadrigalUTFromDate(arg[__startDate][0], arg[__startDate][1], arg[__startDate][2], arg[__startDate][3], arg[__startDate][4], arg[__startDate][5], 0) if time1 > expEndUt: # stop going down tree names[:] = [] return # save time1 in case we need to appendStartTime thisStartTime = expStartUt # apply endDate filter if arg[__endDate] != None: time2 = madrigal.metadata.getMadrigalUTFromDate(arg[__endDate][0], arg[__endDate][1], arg[__endDate][2], arg[__endDate][3], arg[__endDate][4], arg[__endDate][5], 0) if time2 < expStartUt: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # check experiment access thisSecurity = expMeta.getSecurityByPosition() if thisSecurity == -1: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 0: if thisSecurity not in (0,1): # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 1: if thisSecurity != 0: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 2: if thisSecurity not in (0,2): # stop going down tree names[:] = [] return else: # no expTab.txt - set kinst to None expKinst = None # now reject paths not matching present path convention if enforcePathConvention == 0 if arg[__enforcePath] == 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention != 0 if arg[__enforcePath] != 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay, now start looking at individual files if self.__hasFileTab: fileMeta = MadrigalMetaFile(self, dirname + '/' + 'fileTab.txt') # get numFiles numFiles = fileMeta.getFileCount() # loop through each file, and add it if okay fileCount = 0 while fileCount < numFiles: # skip non-default and non-realtime files if not includeNonDef if fileMeta.getCategoryByPosition(fileCount) not in (1,4) and arg[__includeNonDef] == 0: fileCount = fileCount + 1 continue # skip realtime files if not includeNonDef and not includeRealtime if fileMeta.getCategoryByPosition(fileCount) == 4 and arg[__includeNonDef] == 0 and arg[__includeRealtime] == 0: fileCount = fileCount + 1 continue # check if file has right kindat if arg[__kindatList] != None: if fileMeta.getKindatByPosition(fileCount) not in arg[__kindatList]: fileCount = fileCount + 1 continue # check file access if fileMeta.getAccessByPosition(fileCount) != 0: if arg[__pubAccessOnly] in (1,2): fileCount = fileCount + 1 continue # the file needs to be added filename = fileMeta.getFilenameByPosition(fileCount) if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + filename) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + filename, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + filename, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) fileCount = fileCount + 1 # now add all non-madrigal files if includeNonMad if arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue # if its not in fileTab.txt, add it if fileMeta.getExpIdByFilename(name) == None: if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) # else the file has no fileTab.txt and includeNonMad is true - include all files elif arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) def __setFileAccess(self, arg, dirname, names): """setFileAccess is a private helper function called by os.path.walk in setFileAccess. Inputs: arg: either 0 for public access, or 1 for private access. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None Affects: sets all fileTab.txt files in dirname to be public or private, depending on arg. Exceptions: None. """ for name in names: if name == 'fileTab.txt': try: fileMetaObj = madrigal.metadata.MadrigalMetaFile(self, dirname + '/' + name) fileMetaObj.setAccess(arg) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() class MadrigalSite: """MadrigalSite is an object that provides access to Madrigal site info from the metadata. This object provides access to all Madrigal site information in the metadata file siteTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: siteObj = madrigal.metadata.MadrigalSite() print siteObj.getSiteName(1) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __siteMetadataFile = "siteTab.txt" _defaultVersion = '2.6' # column positions __siteIDCol = 0 __siteNameCol = 1 __madServerCol = 2 __madDocRootCol = 3 __madCGICol = 4 __madServletCol = 5 __contactNameCol = 6 __contactAddr1Col = 7 __contactAddr2Col = 8 __contactAddr3Col = 9 __contactCityCol = 10 __contactStateCol = 11 __contactZipCol = 12 __contactCountryCol = 13 __contactPhoneCol = 14 __contactEmailCol = 15 __siteVersionCol = 16 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalSite by reading from siteTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/siteTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get site metadata file if (initFile == None): self.__filename = self.__siteMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without version allowedLens = (self.__contactEmailCol+1, self.__siteVersionCol+1) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens).getList() def getSiteName(self, siteID): """getSiteName returns the site name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__siteNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteServer(self, siteID): """getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found. Inputs: siteID integer to get site server. Returns: the site server that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madServerCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteDocRoot(self, siteID): """getSiteDocRoot returns the relative document root (e.g. madrigal) that matches siteID argument, or None if not found. Inputs: siteID integer to get document root path. Returns: the document root path that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madDocRootCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteRelativeCGI(self, siteID): """getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal) that matches siteID argument, or None if not found. For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored. Inputs: siteID integer to get relative cgi path. Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ version = self.getSiteVersion(siteID) if distutils.version.LooseVersion(version) >= distutils.version.LooseVersion('3.0'): return(self.getSiteDocRoot(siteID)) # this code only runs for Madrigal 2.0 for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madCGICol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return None def getSiteContactName(self, siteID): """getSiteContactName returns the site contact name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress1(self, siteID): """getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 1 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr1Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress2(self, siteID): """getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 2 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr2Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress3(self, siteID): """getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 3 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr3Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteCity(self, siteID): """getSiteCity returns the site city that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact city that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCityCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteState(self, siteID): """getSiteState returns the site state that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact state that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactStateCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSitePostalCode(self, siteID): """getSitePostalCode returns the site postal code that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact postal code that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactZipCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteCountry(self, siteID): """getSiteCountry returns the site country that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact country that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCountryCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteTelephone(self, siteID): """getSiteTelephone returns the site telephone that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact telephone that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactPhoneCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteEmail(self, siteID): """getSiteEmail returns the site email address that matches siteID argument, or None if not found. Inputs: siteID integer to get Site email address. Returns: the site email address that matches siteID argument, or None if not found. To list multiple email addresses in this field separate them with semicolons (since commas are delimiters). getSiteEmail will automatically replace semicolons with commas, as required by multiple email addresses. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactEmailCol].replace(';', ',') except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteList(self): """getSiteList returns a list of all site ids and names. Inputs: None. Returns: a list of all site ids and names. Each item in the list is a tuple of the form (Site id (integer), site name (string)). Example item: (1,'Millstone') Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for site in self.__fileList: try: item = (int(site[self.__siteIDCol]), site[self.__siteNameCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in siteTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList def getSiteVersion(self, siteID): """getSiteVersion returns the site version string that matches siteID argument, or None if not found. Inputs: siteID integer Returns: the site version that matches siteID argument, or None if not found. If file does not have this field, returns default value of 2.6. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): if len(site) >= self.__siteVersionCol + 1: return site[self.__siteVersionCol] else: return(self._defaultVersion) except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def setSiteVersionBySiteID(self, siteID, version): """setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers at given siteID. Inputs: siteID - siteID of site in list. version - string Madrigal version as period delimited integers (eg, 2.6) Returns: None. Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or siteID not found. This method added in Madrigal 3.0 """ # verify no illegal commas if version.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) # parse test version = version.strip() try: siteID = int(siteID) if len(version) == 0: raise ValueError, '' for item in version.split('.'): int(item) except: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) rightPosition = None for position, site in enumerate(self.__fileList): # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): rightPosition = position except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if rightPosition is None: raise madrigal.admin.MadrigalError('siteId %i not found in siteTab.txt' % (siteID)) if len(self.__fileList[position]) == self.__siteVersionCol: # not yet set self.__fileList[position].append(version) else: self.__fileList[position][self.__siteVersionCol] = version def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the siteTab.txt file. Inputs: newFullPath: a new path to write the siteTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) class MadrigalInstrument: """MadrigalInstrument is an object that provides access to Madrigal instrument info from the metadata. This object provides access to all Madrigal instrument information in the metadata files instTab.txt and instType.Tab. key = self.__instKinstCol (used in searches) Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrument() print test.getInstrumentName(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __instMetadataFile = "instTab.txt" # column positions __instKinstCol = 0 __instMnemonicCol = 1 __instNameCol = 2 __latitudeCol = 3 __longitudeCol = 4 __altitudeCol = 5 __contactNameCol = 6 __contactAddr1Col = 7 __contactAddr2Col = 8 __contactAddr3Col = 9 __contactCityCol = 10 __contactStateCol = 11 __contactZipCol = 12 __contactCountryCol = 13 __contactPhoneCol = 14 __contactEmailCol = 15 __categoryCol = 16 # instType.txt file __inst2MetadataFile = "instType.txt" # column positions __inst2CategoryIdCol = 0 __inst2CategoryDescCol = 1 def __init__(self, madDB=None, initFile=None, init2File=None): """__init__ initializes MadrigalInstrument by reading from instTab.txt (or initFile) and instType.txt file (or init2File). Inputs: madDB - Existing MadrigalDB object, by default = None. initFile - String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instTab.txt. init2File - String representing the full path to the metadata file instType.txt. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instTab.txt file was updated with the release of the madrigal python api, and this function will throw an error if used with the old file. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument metadata file if (initFile == None): self.__filename = self.__instMetadataFile else: self.__filename = initFile genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, key=self.__instKinstCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # get instrument metadata file 2 if (init2File == None): self.__filename2 = self.__inst2MetadataFile else: self.__filename2 = initFile self.__fileList2 = madrigal.metadata.MadrigalMetadata(self.__filename2, self.__madDB).getList() def getInstrumentName(self, kinst): """getInstrumentName returns the instrument name that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument name. Returns: the instrument name that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instNameCol]) def getInstrumentMnemonic(self, kinst): """getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument mnemonic. Returns: the instrument mnemonic that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instMnemonicCol]) def getLatitude(self, kinst): """getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get latitude. Returns: the latitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__latitudeCol])) except ValueError: return(None) def getLongitude(self, kinst): """getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get longitude. Returns: the longitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__longitudeCol])) except ValueError: return(None) def getAltitude(self, kinst): """getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get altitude. Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__altitudeCol])) except ValueError: return(None) def getContactName(self, kinst): """getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact name as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactNameCol]) def getContactAddress1(self, kinst): """getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 3 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactAddr1Col]) def getContactEmail(self, kinst): """getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact email. Returns: the contact email as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactEmailCol]) def getCategory(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ categoryId = None try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None if categoryId == None: return(None) # now loop until categoryId found for inst2 in self.__fileList2: try: if (int(inst2[self.__inst2CategoryIdCol]) == categoryId): return(inst2[self.__inst2CategoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in instType.txt parsing metadata row: ' + str(inst2), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # type not found return(None) def getCategoryId(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None return(categoryId) def getInstrumentList(self): """getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values. Inputs: None. Returns: a list of all instrument names, mnemonics, and their kinst values. Each item in the list is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)). Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], int(inst[self.__instKinstCol])) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList def getOrderedInstrumentList(self): """getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId), ordered by categoryId and then kinst. Inputs: None. Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and then kinst. Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: kinst = int(inst[self.__instKinstCol]) item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], kinst, self.getCategory(kinst), self.getCategoryId(kinst)) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) retList.sort(self.__instrumentSort) return retList def getOrderedInstrumentListWithData(self, isTrusted, localOnly=False, localExpObj=None, globalExpObj=None, allowArchive=False, requireFiles=False, requireFilesOrPlots=False): """getInstrumentList returns information about which instruments have local and global data. Inputs: isTrusted - True if client is trusted, False otherwise localOnly - if False (the default), will return information about both local and global data. If True, then global data ignored. localExpObj - an MadrigalExperiment object for the local data. If None (the default), will be created. globalExpObj - an MadrigalExperiment object for the global data. If None (the default), will be created if not localOnly. allowArchive - if True, allow experiments marked as security==2(public arcive), and if isTrusted, allow security==3(private archive) requireFiles - if True, only include experiments with Madrigal data files. If False (default), not not require that. requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots. If False (default), not not require that. Returns: an ordered tuple of two items: 1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear, globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst. If localOnly, globalStartYear and globalEndYear = 0. If not local only, localStartYear and localEndYear will be zero if no local data for that instrument. 2. categoryDict - key = categoryId, value = category Description Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if (not localOnly) and (globalExpObj==None): globalExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt')) if localExpObj == None: localExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) orderInstList = [] categoryDict = {} # next task - create local dict and global dict of key=kinst, value = (startYear, endYear) tuple localInstDict = {} globalInstDict = {} # Skip test experiments testExp = ('1998/mlh/20jan98', '1997/aro/06jan97', '1997/lyr/08apr97', '1997/son/06jan97', '1995/jro/01feb95', '1998/jro/27apr98') for i in range(localExpObj.getExpCount()): thisUrl = localExpObj.getExpUrlByPosition(i) if thisUrl[-16:] in testExp: continue thisSecurity = localExpObj.getSecurityByPosition(i) if thisSecurity == -1: # blocked to all continue if thisSecurity == 1 and not isTrusted: # no access continue if thisSecurity == 2 and not allowArchive: # no access continue if thisSecurity == 3 and ((not isTrusted) or (not allowArchive)): # no access continue kinst = localExpObj.getKinstByPosition(i) startYear = localExpObj.getExpStartDateTimeByPosition(i)[0] endYear = localExpObj.getExpEndDateTimeByPosition(i)[0] if localInstDict.has_key(kinst): oldStartYear, oldEndYear = localInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue expDir = localExpObj.getExpDirByPosition(i) if requireFilesOrPlots: links = localExpObj.getExpLinksByPosition(i) if len(links) == 0: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if requireFiles: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if localInstDict.has_key(kinst): localInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) else: localInstDict[kinst] = (startYear, endYear) if not localOnly: for i in range(globalExpObj.getExpCount()): kinst = globalExpObj.getKinstByPosition(i) startYear = globalExpObj.getExpStartDateTimeByPosition(i)[0] endYear = globalExpObj.getExpEndDateTimeByPosition(i)[0] thisUrl = globalExpObj.getExpUrlByPosition(i) thisSecurity = globalExpObj.getSecurityByPosition(i) thisSite = globalExpObj.getExpSiteIdByPosition(i) if thisUrl[-16:] in testExp: continue if not globalInstDict.has_key(kinst): globalInstDict[kinst] = (startYear, endYear) continue oldStartYear, oldEndYear = globalInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue if thisSecurity != 0 and self.__madDB.getSiteID() != thisSite: # no search remote sites for non-public experiments allowed continue if requireFiles: if globalExpObj.getExpIdByPosition(i) not in expIdList: continue globalInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) # populate orderInstList and categoryDict instList = self.getOrderedInstrumentList() for inst in instList: if not localOnly: if globalInstDict.has_key(inst[2]): if not categoryDict.has_key(inst[4]): categoryDict[inst[4]] = inst[3] globalStartYear, globalEndYear = globalInstDict[inst[2]] try: localStartYear, localEndYear = localInstDict[inst[2]] except KeyError: localStartYear, localEndYear = (0, 0) orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) else: if localInstDict.has_key(inst[2]): if not categoryDict.has_key(inst[4]): categoryDict[inst[4]] = inst[3] globalStartYear = 0 globalEndYear = 0 localStartYear, localEndYear = localInstDict[inst[2]] orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) return ((orderInstList, categoryDict)) def __instrumentSort(self, thisInst, otherInst): """instrumentSort is a private method used to sort tuples of instrument data """ result = cmp(thisInst[4], otherInst[4]) if result != 0: return(result) return(cmp(thisInst[2], otherInst[2])) class MadrigalInstrumentParameters: """MadrigalInstrumentParameters is an object that provides access to the metadata file that summarizes the parameters associated with each instrument. This object provides access to all Madrigal instrument parameter information in the metadata file instParmTab.txt. The metadata file instParmTab.txt lists, for any given instrument, all the measured parameters found in all the data files in the database associated with that instrument. This class also contains a method to rebuild the table instParmTab.txt by examining every data file in the database. This is presumably a slow process and should be done in the background. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentParameters() print test.getParameters(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jul. 17, 2002 """ #constants __instParmMetadataFile = "instParmTab.txt" # column positions __instParmKinstCol = 0 __instParmListCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instParmTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument parameter metadata file if (initFile == None): self.__filename = self.__instParmMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getParameters(self, kinst): """getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get parameters. If 0, get parameters from all instruments. Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank. Affects: None Exceptions: if error in metadata file """ parmList = [] for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instParmKinstCol]) == kinst or kinst == 0): if len(inst[self.__instParmListCol]) == 0: continue tempList = inst[self.__instParmListCol].split() for parm in tempList: if not parm.lower() in parmList: if parm.find('-32767') == -1: parmList.append(parm.lower()) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(parmList) == 0: return(None) return parmList def rebuildInstParmTable(self, completeRebuildFlag = 0): """rebuildInstParmTable rebuilds the instParmTab.txt metadata file. The table instParmTab.txt is a listing of every measured parameter found in any data file for a given instrument. It now will also import data from other Madrigal sites instParmTab.txt files. Since these files are constantly updated, this table needs to be updated on a regular basis. This methods works in one of two ways, depending on the value of completeRebuildFlag and whether a file called instParmLastUpdate.txt exists in the metadata directory. If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable loops through each instrument in the instTab.txt. For each instrument, it loops through every data file associated with that instrument. For every data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. Since this process involves every file in the database, it may take a great deal of time and should be run in the background. If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable first stores all the existing parameters from the instParmTab.txt. It reads the date of the last update from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with each instrument. For every new data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. This makes rebuildInstParmTable faster, but possibly keeps invalid parameters if experiments are ever deleted. Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are also added. Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means newer than the date in the instParmLastUpdate.txt file (stored as a float). If 1, rebuild the table completely. This will eliminate any parameters no longer found in files, but will take longer. If no instParmLastUpdate.txt file is found, the table is always rebuilt completely. Returns: None. Affects: Writes file instParmTab.txt in metadata directory Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instParmTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # if madroot not set, set it now if os.environ.get('MAD' + 'ROOT') == None: os.environ['MAD' + 'ROOT'] = self.__madDB.getMadroot() # now its safe to import madrigal.data since MAD + ROOT set import madrigal.data # get date of files to examine if completeRebuildFlag == 0: # try to open and read instParmLastUpdate.txt try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'r') lastUpdateTime = float(dateFile.read()) dateFile.close() except: # if failure, set time to 1/1/1970 and set completeRebuildFlag = 1 lastUpdateTime = time.mktime([1970,1,1,0,0,0,0,0,0]) completeRebuildFlag = 1 else: # set time to 1/1/1971 lastUpdateTime = time.mktime([1971,1,1,0,0,0,0,0,0]) # now write new version of instParmLastUpdate.txt # if not writable, exception thrown try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'w') except: raise madrigal.admin.MadrigalError('Unable to write: ' + \ self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', None) newUpdateTime = time.time() dateFile.write(str(newUpdateTime)) dateFile.close() # finally, make sure new instParmLastUpdate.txt is world-writable try: os.chmod(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 0666) except: pass # create a needed MadrigalParameters obj madParmObj = madrigal.data.MadrigalParameters(self.__madDB) # create a needed MadrigalSite obj madSiteObj = madrigal.metadata.MadrigalSite(self.__madDB) # create a string to hold all the text for the new file newFileStr = '' # create a dictionary with all instrument codes as keys instObj = madrigal.metadata.MadrigalInstrument(self.__madDB) instList = instObj.getInstrumentList() instCodeDict = {} for inst in instList: if inst[2] == 0: continue if inst[2] in instCodeDict.keys(): continue if completeRebuildFlag == 1: # start with empty list instCodeDict[inst[2]]= [] else: # start with present list tempParmList = [] oldParmList = self.getParameters(inst[2]) if oldParmList != None: for item in oldParmList: # check that its not -32767 try: if madParmObj.getParmCodeFromMnemonic(item) == -32767: continue except ValueError: print('skipping parm %s since no longer valid' % (str(item))) continue tempParmList.append(madParmObj.getParmCodeFromMnemonic(item)) instCodeDict[inst[2]] = tempParmList else: instCodeDict[inst[2]] = [] # get a list of all default and realtime files filelist = self.__madDB.getFileListFromMetadata(appendKinst = 1, includeRealtime = 1) # appendKinst, allow realtime # loop through each file - file is tuple of # (file name, inst code) for file in filelist: # check whether file should be skipped try: if lastUpdateTime > os.stat(file[0])[8]: continue except: continue # try to create MadrigalFile object for that file try: print '\t\tAdding parameters to instParmTab.txt from ' + str(file) madFileObj = madrigal.data.MadrigalFile(file[0], self.__madDB) fileParmList = madFileObj.getMeasuredParmList() # append unique parameters for parm in fileParmList: if not parm in instCodeDict[file[1]]: instCodeDict[file[1]].append(parm) # if a problem with file, skip it except: continue # now create a new file string from instCodeDict # create a sorted list of keys keyList = instCodeDict.keys() keyList.sort() # step through each key delimiter = ' ' for key in keyList: # sort that instrument's list instParmList = madParmObj.normalizeParmList(instCodeDict[key]) # convert from codes to mnemonics instParmList = madParmObj.getParmMnemonicList(instParmList) # append that instrument's data to newFileStr newFileStr = newFileStr + str(key) + ',' newFileStr = newFileStr + delimiter.join(instParmList).lower() + '\n' # if no data found, throw error if len(newFileStr) == 0: raise madrigal.admin.MadrigalError('No data found for: ' + str(filename), None) # write new instParmTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close class MadrigalKindat: """MadrigalKindat is an object that provides access to Madrigal kind of data info from the metadata. This object provides access to all Madrigal kind of data information in the metadata file typeTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalKindat() print test.getKindatDescription(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __typeMetadataFile = "typeTab.txt" # column positions __typeCodeCol = 0 __typeDescCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalKindat by reading from typeTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/typeTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get kindat metadata file if (initFile == None): self.__filename = self.__typeMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getKindatDescription(self, code, kinst=None): """getKindatDescription returns the kindat description that matches code argument, or None if not found. Inputs: code integer to get kindat description. or integer as string kinst - used to look up kindat description in form '%i_%i' % (kinst, code). If None, (the default) only return exact code match as integer. If given, first try to match '%i_%i' % (kinst, code). If not found, then try just code. May be integer or string integer Returns: the kindat description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) if kinst != None: kinst = int(kinst) for i, type in enumerate(self.__fileList): # find matching kinst and code try: items = type[self.__typeCodeCol].split('_') if len(items) == 2: thisKinst = int(items[0]) thisKindat = int(items[1]) if thisKinst == kinst and thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # just search for code for i, type in enumerate(self.__fileList): try: items = type[self.__typeCodeCol].split('_') if len(items) == 1: thisKindat = int(items[0]) if thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getKindatList(self): """getKindatList returns a list of all kindat descriptions and codes. Inputs: None. Returns: a list of all kindat descriptions and codes. Each item in the list is a tuple of the form (Kindat description (string), kindat code (integer or string in form '%i_%i' % (kinst, code))). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, kindat in enumerate(self.__fileList): try: # works if its a standard code item = (kindat[self.__typeDescCol], int(kindat[self.__typeCodeCol])) retList.append(item) except: try: # works for kinst_kindat item = (kindat[self.__typeDescCol], kindat[self.__typeCodeCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(kindat), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList class MadrigalInstrumentKindats: """MadrigalInstrumentKindats is an object that provides access to the metadata file that summarizes the kindat codes associated with each instrument. This object provides access to all Madrigal instrument kindat information in the metadata file instKindatTab.txt. The metadata file instKindatTab.txt lists, for any given instrument and year, all the kindat codes found in all the data files in the local database associated with that instrument. This class also contains a method to rebuild the table instKindatTab.txt by examining all the metadata in the database. This is presumably a somewhat slow process and should be done in the background. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentKindats() print test.getKindatListForInstruments([20,30]) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Aug. 15, 2002 """ #constants __instKindatMetadataFile = "instKindatTab.txt" # column positions __instKindatKinstCol = 0 __instKindatListCol = 1 __instKindatYearCol = 2 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument kindat metadata file if (initFile == None): self.__filename = self.__instKindatMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getKindatListForInstruments(self, kinstList): """getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list. A kindat code is returned if it appears in any year Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer. Returns: a list of kindat codes as integers associated with the given instrument list. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # if kinstList is just a single integer, convert it into a list if type(kinstList) == types.IntType: kinstList = [kinstList] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instKindatKinstCol]) in kinstList): if len(inst[self.__instKindatListCol]) == 0: continue tempList = inst[self.__instKindatListCol].split() for parm in tempList: if not int(parm) in retKindatList: retKindatList.append(int(parm)) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return(retKindatList) def getKindatListForInstrumentYear(self, kinst, year): """getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year. Inputs: kinst, year Returns: a list of kindat codes as integers associated with the given instrument and year, or None if not found. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if int(inst[self.__instKindatKinstCol]) != kinst: continue if int(inst[self.__instKindatYearCol]) != year: continue return([int(kindat) for kindat in inst[self.__instKindatListCol].split()]) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def rebuildInstKindatTable(self): """rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file. The table instKindatTab.txt is a listing of every kindat found for a given instrument for any given year. Since these files are constantly updated, this table needs to be updated on a regular basis. Data from other Madrigal sites is also imported into this table. How it works: For each instrument in instTab.txt, this method first creates a list of all the experiments that used that instrument. It then loops through the file table. For each file listing, if the file is the default one, and its experiment id is in the list of experiments just created, the kindat is added to the list if its not already there. Data from all other Madrigal sites is then added via getMetadata. It then writes the metadata file in the form: 10, 1001 1002 1003, where the first column in the instrument code and the second column is a space delimited list of kindat codes. Inputs: None. Returns: None. Affects: Writes file instKindatTab.txt in metadata directory Exceptions: If unable to write instKindatTab.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instKindatTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # get experiment metadata madExp = MadrigalExperiment(self.__madDB) kindatDict = {} # key = year, value = list of kindats for i in range(madExp.getExpCount()): kinst = madExp.getKinstByPosition(i) if kinst not in kindatDict.keys(): kindatDict[kinst] = {} expDir = madExp.getExpDirByPosition(i) # get year range timeTuple = madExp.getExpStartDateTimeByPosition(i) syear = timeTuple[0] timeTuple = madExp.getExpEndDateTimeByPosition(i) eyear = timeTuple[0] for year in range(syear, eyear+1): if year not in kindatDict[kinst].keys(): kindatDict[kinst][year] = [] try: madFileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt')) except: continue for j in range(madFileObj.getFileCount()): kindat = madFileObj.getKindatByPosition(j) if kindat not in kindatDict[kinst][year]: kindatDict[kinst][year].append(kindat) # create newFileStr from instCodeDict newFileStr = '' kinstList = kindatDict.keys() kinstList.sort() delimiter = ' ' for kinst in kinstList: yearList = kindatDict[kinst].keys() yearList.sort() for year in yearList: kindatDict[kinst][year].sort() kindatStrList = [str(kindat) for kindat in kindatDict[kinst][year]] kindatStr = delimiter.join(kindatStrList) newFileStr += '%i,%s,%i\n' % (kinst, kindatStr, year) # finally, write new instKindatTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close class MadrigalExperiment: """MadrigalExperiment is an object that provides access to Madrigal experiment info from the metadata. This object provides access to all Madrigal experiment information in the metadata file expTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalExperiment() print test.getExperimentName(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Apr. 17, 2002 """ #constants __expMetadataFile = "expTab.txt" # column positions __expIdCol = 0 __expUrlCol = 1 __expNameCol = 2 __expSiteIdCol = 3 __expStartDateCol = 4 __expStartTimeCol = 5 __expEndDateCol = 6 __expEndTimeCol = 7 __expKinstCol = 8 __expSecurityCol = 9 __expPICol = 10 __expPIEmailCol = 11 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalExperiment by reading from expTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/expTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get experiment metadata file if (initFile == None): self.__filename = self.__expMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without PI and email allowedLens = (self.__expSecurityCol+1, self.__expPIEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__expIdCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expIdCol] = str(int(expId)) self._dict[position] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str((position, expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpUrlByPosition(self, position = 0): """getExpUrlByPosition returns the experiment url of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment url, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expUrlCol] else: return(None) def getExpUrlByExpId(self, expId): """getExpUrlByExpId returns the experiment url for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpUrlByPosition(position)) def getRealExpUrlByPosition(self, position = 0): """getRealExpUrlByPosition returns the real experiment url of the experiment at given position. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the real experiment url, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index == -1: return(thisUrl) if thisUrl.find('experiments') != -1: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + thisUrl[index+8:]) else: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + 'experiments', thisUrl[index+8:]) return(realUrl) def getRealExpUrlByExpId(self, expId): """getRealExpUrlByExpId returns the real experiment url for a given experiment id. The url in the metadata contains /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the real experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getRealExpUrlByPosition(position)) def getExpPathByPosition(self, position = 0): """getExpPathByPosition returns the experiment path of the experiment at given position. Experiment path is the path to the experiment from the madroot directory, and always begins "experiments" Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index != -1: return(thisUrl[index+8:]) index = thisUrl.find('experiments') if index == -1: return(None) return(thisUrl[index:]) def getExpPathByExpId(self, expId): """getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the path to the experiment from the madroot directory (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpPathByPosition(position)) def setExpUrlByPosition(self, position, expUrl): """setExpUrlByPosition sets the experiment url of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expUrl - the new experiment url to use Returns: None. Affects: sets the experiment url of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expUrl.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_exc()])) try: self.__fileList[position][self.__expUrlCol] = str(expUrl) except: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_ex()])) def getExpDirByPosition(self, position = 0): """getExpDirByPosition returns the full experiment directory of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the full experiment directory, or None if position >= number of experiments. Uses experiment url to determine directory. Affects: None Exceptions: None """ if len(self.__fileList) > position: url = self.__fileList[position][self.__expUrlCol] # find the directory based on url maddir = self.__madDB.getMadroot() index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: maddir = os.path.join(maddir, 'experiments', url[index+8:]) else: maddir = os.path.join(maddir, url[index+8:]) return(maddir) else: return(None) def getExpDirByExpId(self, expId): """getExpDirByExpId returns the full experiment directory for a given experiment id. Inputs: Experiment Id (integer). Returns: the full experiment directory (string). Returns None if experiment id not found. Uses experiment url to determine directory. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpDirByPosition(position)) def getExpNameByPosition(self, position = 0): """getExpNameByPosition returns the experiment name of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment name, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expNameCol] else: return(None) def getExpNameByExpId(self, expId): """getExpNameByExpId returns the experiment name for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment name (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpNameByPosition(position)) def setExpNameByPosition(self, position, expName): """setExpNameByPosition sets the experiment name of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expName - the new experiment name to use Returns: None. Affects: sets the experiment name of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expName.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), [traceback.format_exc()])) try: self.__fileList[position][self.__expNameCol] = str(expName) except: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpSiteIdByExpId(self, expId): """getExpSiteIdByExpId returns the site id (int) for a given experiment id. Inputs: Experiment Id (integer). Returns: the site id for this experiment. Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpSiteIdByPosition(position)) def getExpSiteIdByPosition(self, position = 0): """getExpSiteIdByPosition returns the experiment site id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment site id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSiteIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpSiteIdByPosition(self, position, expSiteId): """setExpSiteIdByPosition sets the experiment site id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expSiteId - the new experiment site id to use Returns: None. Affects: sets the experiment site id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSiteIdCol] = str(int(expSiteId)) except: raise madrigal.admin.MadrigalError('Error in setExpSiteIdByPosition with args %s: %s' % \ (str(position, expSiteId), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpStartDateTimeByPosition(self, position = 0): """getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings startTime = [int(self.__fileList[position][self.__expStartDateCol][0:4]), int(self.__fileList[position][self.__expStartDateCol][4:6]), int(self.__fileList[position][self.__expStartDateCol][6:8]), int(self.__fileList[position][self.__expStartTimeCol][0:2]), int(self.__fileList[position][self.__expStartTimeCol][2:4]), int(self.__fileList[position][self.__expStartTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if startTime[3] == 24: startTime[3] = 23 startTime[4] = 59 startTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(startTime[0], startTime[1], startTime[2], startTime[3], startTime[4], startTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) startTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return startTime else: return(None) def getExpEndDateTimeByPosition(self, position = 0): """getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings endTime = [int(self.__fileList[position][self.__expEndDateCol][0:4]), int(self.__fileList[position][self.__expEndDateCol][4:6]), int(self.__fileList[position][self.__expEndDateCol][6:8]), int(self.__fileList[position][self.__expEndTimeCol][0:2]), int(self.__fileList[position][self.__expEndTimeCol][2:4]), int(self.__fileList[position][self.__expEndTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if endTime[3] == 24: endTime[3] = 23 endTime[4] = 59 endTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(endTime[0], endTime[1], endTime[2], endTime[3], endTime[4], endTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) endTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return endTime else: return(None) def getExpStartDateTimeByExpId(self, expId): """getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpStartDateTimeByPosition(position)) def getExpEndDateTimeByExpId(self, expId): """getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpEndDateTimeByPosition(position)) def setExpStartDateTimeByPosition(self, startDateTime, position = 0): """setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position. Inputs: startDateTime - a python datetime object to set the exp start date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp start date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expStartDateCol] = '%04i%02i%02i' % (startDateTime.year, startDateTime.month, startDateTime.day) self.__fileList[position][self.__expStartTimeCol] = '%02i%02i%02i' % (startDateTime.hour, startDateTime.minute, startDateTime.second) def setExpEndDateTimeByPosition(self, endDateTime, position = 0): """setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position. Inputs: endDateTime - a python datetime object to set the exp end date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp end date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expEndDateCol] = '%04i%02i%02i' % (endDateTime.year, endDateTime.month, endDateTime.day) self.__fileList[position][self.__expEndTimeCol] = '%02i%02i%02i' % (endDateTime.hour, endDateTime.minute, endDateTime.second) def getKinstByPosition(self, position = 0): """getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (instrument code) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expKinstCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getKinstByExpId(self, expId): """getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment kinst (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getKinstByPosition(position)) def setExpKinstByPosition(self, position, expKinst): """setExpKinstByPosition sets the experiment kinst of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expKinst - the new experiment kinst to use Returns: None. Affects: sets the experiment kinst of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expKinstCol] = str(int(expKinst)) except: raise madrigal.admin.MadrigalError('Error in setExpKinstByPosition with args %s: %s' % \ (str(position, expKinst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getSecurityByPosition(self, position = 0): """getSecurityByPosition returns the security code of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the security code (integer) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if security column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSecurityCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getSecurityByExpId(self, expId): """getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the security code (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getSecurityByPosition(position)) def setSecurityByPosition(self, position, securityCode): """setSecurityByPosition sets the security code (integer) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). securityCode - the new experiment security code (integer) to use Returns: None. Affects: sets the security code of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSecurityCol] = str(int(securityCode)) except: raise madrigal.admin.MadrigalError('Error in setSecurityByPosition with args %s: %s' % \ (str(position, securityCode), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getPIByPosition(self, position = 0): """getPIByPosition returns the principal investigator of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's name (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: pi = self.__fileList[position][self.__expPICol] if len(pi) > 0: return(pi) else: return(None) except: return(None) else: return(None) def getPIByExpId(self, expId): """getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's name (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIByPosition(position)) def setPIByPosition(self, position, PI): """setPIByPosition sets the principal investigator (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PI - the new experiment principal investigator's name (string) to use Returns: None. Affects: sets the principal investigator of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PI.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add PI and blank email self.__fileList[position].append(str(PI)) self.__fileList[position].append('') else: try: self.__fileList[position][self.__expPICol] = str(PI) except: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()])) def getPIEmailByPosition(self, position = 0): """getPIEmailByPosition returns the principal investigator email of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's email (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: piEmail = self.__fileList[position][self.__expPIEmailCol] if len(piEmail) > 0: return(piEmail) else: return(None) except: return(None) else: return(None) def getPIEmailByExpId(self, expId): """getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's email (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIEmailByPosition(position)) def setPIEmailByPosition(self, position, PIEmail): """setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PIEmail - the new experiment principal investigator's email (string) to use Returns: None. Affects: sets the principal investigator email of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PIEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add blank PI and email self.__fileList[position].append('') self.__fileList[position].append(str(PIEmail)) else: try: self.__fileList[position][self.__expPIEmailCol] = str(PIEmail) except: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()])) def getExpLinksByExpId(self, expId): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: Experiment Id (integer). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpLinksByPosition(position)) def getExpLinksByPosition(self, position = 0): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: position - position of experiment in list (first position is zero). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ retList = [] allowedExtensions = ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif'] # find the experiment directory based on url topdir = self.__madDB.getMadroot() url = self.getExpUrlByPosition(position) if url == None: return retList index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: expdir = os.path.join(topdir, 'experiments', url[index+8:]) else: expdir = os.path.join(topdir, url[index+8:]) # now create a list of html files htmlFiles = [] l1 = glob.glob(os.path.join(expdir, '*.html')) l2 = glob.glob(os.path.join(expdir, '*/index.html')) l3 = glob.glob(os.path.join(expdir, '*/*/index.html')) for item in l1: htmlFiles.append(item) for item in l2: htmlFiles.append(item) for item in l3: htmlFiles.append(item) for htmlFile in htmlFiles: # get title if possible f = open(htmlFile) text = f.read() f.close() i1 = text.upper().find('<TITLE>') + len('<TITLE>') i2 = text.upper().find('</TITLE>') if i1 > -1 and i2 > -1: title = text[i1:i2] else: title = 'No title' # get url i1 = htmlFile.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', htmlFile[i1:]) retList.append((title, url)) for extension in allowedExtensions: plotList = glob.glob(os.path.join(expdir, '*.%s' % (extension))) for plot in plotList: i1 = plot.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', plot[i1:]) retList.append((os.path.basename(plot), url)) return retList def getExpCount(self): """getExpCount returns number of experiments in MadrigalExperiment object """ return(len(self.__fileList)) def sortByDateSite(self): """sortByDateSite will resort self.__fileList so that experiments are listed first by experiment start date, and then by site """ self.__fileList.sort(self.__compareDateSite__) # rebuild self._dict self._dict = {} for i, items in enumerate(self.__fileList): self._dict[items[self.__expIdCol]] = i def getStartPosition(self, startDT): """getStartPosition returns the position of the first experiment with a start datetime after startDT after sorting by start time. Inputs: startDT - start datetime Returns: returns the position of the first experiment with a start datetime after startDT. """ self.sortByDateSite() top = len(self.__fileList) - 1 bottom = 0 while(top >= bottom): if top == bottom: return(max(top-1, 0)) middle = (top+bottom)/2 thisSDTList = self.getExpStartDateTimeByPosition(middle) thisSDT = datetime.datetime(*thisSDTList[0:6]) if thisSDT < startDT: bottom = middle + 1 continue if middle == 0: return(0) prevSDTList = self.getExpStartDateTimeByPosition(middle-1) prevSDT = datetime.datetime(*prevSDTList[0:6]) if prevSDT < startDT: return(middle-1) else: top = middle -1 def __compareDateSite__(self, first, second): """__compareDateSite__ is a private method to help sort by start date and then site. first, second - lists of experiment information as parsed from expTab.txt files """ result = cmp(first[self.__expStartDateCol], second[self.__expStartDateCol]) if result != 0: return(result) result = cmp(first[self.__expStartTimeCol], second[self.__expStartTimeCol]) if result != 0: return(result) result = cmp(first[self.__expEndDateCol], second[self.__expEndDateCol]) if result != 0: return(result) result = cmp(first[self.__expEndTimeCol], second[self.__expEndTimeCol]) if result != 0: return(result) return(cmp(first[self.__expSiteIdCol], second[self.__expSiteIdCol])) def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n') def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the expTab.txt file. Inputs: newFullPath: a new path to write the expTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def __str__(self): """return possibly modified file as a string in same format as writeMetadata """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' return(metaFileStr) class MadrigalMetaFile: """MadrigalMetaFile is an object that provides access to Madrigal file info from the metadata. This object provides access to all Madrigal experiment information in the metadata file fileTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalMetaFile() print test.getFileCount() except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu May. 7, 2002 """ #constants __fileMetadataFile = "fileTab.txt" # column positions __fileNameCol = 0 __fileExpIdCol = 1 __fileKindatCol = 2 __fileCategoryCol = 3 __fileSizeCol = 4 __fileHasCatalogCol = 5 __fileHasHeaderCol = 6 __fileModDateCol = 7 __fileModTimeCol = 8 __fileStatusCol = 9 __fileAccessCol = 10 __fileAnalystCol = 11 __fileAnalystEmailCol = 12 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalMetaFile by reading from fileTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/fileTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get file metadata file if (initFile == None): self.__filename = self.__fileMetadataFile else: self.__filename = initFile # MadrigalMetafile can now legal have two different lengths - with or without file analyst and email allowedLens = (self.__fileAccessCol+1, self.__fileAnalystEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__fileNameCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # check that at least first row has right number of columns if len(self.__fileList) > 0: # analyst and analyst email columns may or may not exist if len(self.__fileList[0]) not in allowedLens: raise madrigal.admin.MadrigalError('Error in count of first row of ' + str(self.__filename), None) def getFileCount(self): """getFileCount returns the number of files (rows) in the metadata file. Inputs: None Returns: the number of files (rows) in the metadata file. Affects: None Exceptions: None """ return len(self.__fileList) def getFilenameByPosition(self, position = 0): """getFilenameByPosition returns the filename of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the filename of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileNameCol] else: return(None) def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id (integer) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the experiment id (integer) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst exp id cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileExpIdCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the file at given position. Inputs: position - position of file in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the file at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__fileExpIdCol] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str(position), str(expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def getKindatByPosition(self, position = 0): """getKindatByPosition returns the kindat (kind of data code) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (kind of instrument code) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileKindatCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getCategoryByPosition(self, position = 0): """getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileCategoryCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getCategoryByFilename(self, filename): """getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename. Inputs: filename - name of file to search for. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileCategoryCol])) def getHasCatalogByPosition(self, position = 0): """getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any catalog records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasCatalogCol]) == 0: return False else: return True else: return(None) def getHasCatalogByFilename(self,filename): """getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any catalog records, False otherwise. Returns none if name not found Affects: None Exceptions: None. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasCatalogCol])) def setHasCatalogByPosition(self, position, hasCatalog): """setHasCatalogByPosition sets the value of hasCatalog for the file at the given position. Inputs: position - position of file in list (first position is zero). hasCatalog - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasCatalog not in (0, 1, True, False): raise ValueError, 'Illegal value for hasCatalog in setHasCatalogByPosition: %s' % (str(hasCatalog)) if len(self.__fileList) > position: if hasCatalog in (1, True): self.__fileList[position][self.__fileHasCatalogCol] = '1' else: self.__fileList[position][self.__fileHasCatalogCol] = '0' else: raise ValueError, 'setHasCatalogByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getHasHeaderByPosition(self, position = 0): """getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any header records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasHeaderCol]) == 0: return False else: return True else: return(None) def getHasHeaderByFilename(self,filename): """getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any header records, False otherwise. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasHeaderCol])) def setHasHeaderByPosition(self, position, hasHeader): """setHasHeaderByPosition sets the value of hasHeader for the file at the given position. Inputs: position - position of file in list (first position is zero). hasHeader - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasHeader not in (0, 1, True, False): raise ValueError, 'Illegal value for hasHeader in setHasHeaderByPosition: %s' % (str(hasHeader)) if len(self.__fileList) > position: if hasHeader in (1, True): self.__fileList[position][self.__fileHasHeaderCol] = '1' else: self.__fileList[position][self.__fileHasHeaderCol] = '0' else: raise ValueError, 'setHasHeaderByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getStatusByPosition(self, position = 0): """getStatusByPosition returns the status description of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the status description of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileStatusCol] else: return(None) def getStatusByFilename(self,filename): """getStatusByFilename returns the status description of the file with the given name. Inputs: filename - name of file to search for. Returns: the status description of the file with the given name. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.__fileList[position][self.__fileStatusCol]) def getAccessByPosition(self, position = 0): """getAccessByPosition returns the access (0=public, 1=private) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the access level (0=public, 1=private) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if access column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileAccessCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getFileDatetimeByPosition(self, position = 0): """getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set. Inputs: position of file in list (first position is zero). Defaults to first. Returns: a datetime of the file at given position, or None if not set. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: dateStr = self.__fileList[position][self.__fileModDateCol] if len(dateStr) < 8: return(None) timeStr = self.__fileList[position][self.__fileModTimeCol] if len(timeStr) < 6: return(None) return(datetime.datetime.strptime('%s %s' % (dateStr, timeStr), '%Y%m%d %H%M%S')) else: return(None) def getFileDatetimeByFilename(self,filename): """getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set. Inputs: filename - name of file to search for. Returns: a datetime of the file with the given filenme, or None if not set. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.getFileDatetimeByPosition(position)) def setFileDatetimeByPosition(self, position, dt): """setFileDatetimeByPosition sets the file datetime for the given position Inputs: position - position of file in list (first position is zero). dt - datetime (UT) of file being added. If None, use file datetime. Will raise exception if dt is None and fileTab.txt in the metadata version, not the expDir version. Returns: None. Affects: None Exceptions: If position beyond length. """ if dt is None: basename = self.getFilenameByPosition(position) expDir = os.path.dirname(self.__filename) if len(expDir) < 2: raise IOError('Cannot set datetime to None using the metadata version of fileTab.txt') dataFile = os.path.join(expDir, basename) if not os.access(dataFile, os.R_OK): raise IOError('Cannot access %s' % (dataFile)) dt = datetime.datetime.utcfromtimestamp(os.path.getmtime(dataFile)) dateStr = dt.strftime('%Y%m%d') timeStr = dt.strftime('%H%M%S') if len(self.__fileList) > position: self.__fileList[position][self.__fileModDateCol] = dateStr self.__fileList[position][self.__fileModTimeCol] = timeStr else: raise ValueError('setFileDatetimeByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def deleteRowByFilename(self, filename): """deleteRowByFilename deletes a row with a given filename. Inputs: filename - name of file to search for. Returns: None. Affects: Removes item from self.__fileList if filename found Exceptions: Thrown if filename not found. """ for file in self.__fileList: if filename == file[self.__fileNameCol]: self.__fileList.remove(file) return # no matches found raise madrigal.admin.MadrigalError('Could not delete file ' + filename + ' from ' + self.__filename, None) def getExpIdByFilename(self, filename): """getExpIdByFilename returns the first experiment id (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the experiment id (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if exp id cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileExpIdCol])) def getKindatByFilename(self, filename): """getKindatByFilename returns the first kindat (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the kindat (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if kindat cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileKindatCol])) def setAccessByPosition(self, position, access): """setAccessByPosition sets the value of access for the file at the given position. Inputs: position - position of file in list (first position is zero). access - 0 of False for public, 1 or True for private Returns: None. Affects: None Exceptions: If position beyond length. """ if access not in (0, 1, True, False): raise ValueError, 'Illegal value for access in setAccessByPosition: %s' % (str(access)) if len(self.__fileList) > position: if access in (1, True): self.__fileList[position][self.__fileAccessCol] = '1' else: self.__fileList[position][self.__fileAccessCol] = '0' else: raise ValueError, 'setAccessByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def setAccess(self, accessType): """setAccess sets the access column to all 0's (public) or all 1's (private). Inputs: accessType - either 0 to set to public access, or 1 to set to private access. Returns: None. Affects: Overwrite fileTab.txt file with access column set to all 0's (public) or all 1's (private). Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1 """ if (accessType != 0 and accessType != 1): raise madrigal.admin.MadrigalError('MadrigalMetaFile.setAccess called with arg = ' + \ str(accessType) + ', must be either 0 or 1', None) for row in self.__fileList: row[self.__fileAccessCol] = str(accessType) self.writeMetadata() def setKindatByPosition(self, position, kindat): """setKindatByPosition sets the value of kindat for the file at the given position. Inputs: position - position of file in list (first position is zero). kindat - integer kindat value Returns: None. Affects: None Exceptions: If position beyond length. """ kindat = int(kindat) if len(self.__fileList) > position: self.__fileList[position][self.__fileKindatCol] = str(kindat) else: raise ValueError, 'setKindatByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def setCategoryByPosition(self, position, category): """setCategoryByPosition sets the value of category for the file at the given position. Inputs: position - position of file in list (first position is zero). category - 1=default, 2=variant, 3=history, 4=real-time Returns: None. Affects: None Exceptions: If position beyond length. """ if int(category) not in (1, 2, 3, 4): raise ValueError, 'Illegal value for category in setCategoryByPosition: %s' % (str(category)) if len(self.__fileList) > position: self.__fileList[position][self.__fileCategoryCol] = str(category) else: raise ValueError, 'setCategoryByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def setStatusByPosition(self, position, status): """setStatusByPosition sets the value of status string for the file at the given position. Inputs: position - position of file in list (first position is zero). status - string describing status Returns: None. Affects: None Exceptions: If position beyond length. """ if type(status) != types.StringType: raise ValueError, 'Illegal value for status in setStatusByPosition: %s' % (str(status)) # check that string does not illegally contain a comma if status.find(',') != -1: raise ValueError, 'status string in fileTab.txt cannot contain a comma: <%s> is illegal' % (status) if len(self.__fileList) > position: self.__fileList[position][self.__fileStatusCol] = status else: raise ValueError, 'setStatusByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getAnalystByPosition(self, position = 0): """getAnalystByPosition returns file analyst name if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None) else: return(None) def getAnalystByFilename(self, filename): """getAnalystByFilename returns file analyst name if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None) def setAnalystByPosition(self, position, analyst): """setAnalystByPosition sets the file analyst name for the given position Inputs: position - position of file in list (first position is zero). analyst - name of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analyst.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystByPosition with args %s: %s' % \ (str(position), analyst), [traceback.format_exc()]) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol+1: # append two extra columns self.__fileList[position].append(str(analyst)) self.__fileList[position].append('') else: self.__fileList[position][self.__fileAnalystCol] = str(analyst) else: raise ValueError, 'setAnalystByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getAnalystEmailByPosition(self, position = 0): """getAnalystEmailByPosition returns file analyst email if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None) else: return(None) def getAnalystEmailByFilename(self, filename): """getAnalystEmailByFilename returns file analyst email if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None) def setAnalystEmailByPosition(self, position, analystEmail): """setAnalystEmailByPosition sets the file analyst email for the given position Inputs: position - position of file in list (first position is zero). analystEmail - email of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analystEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystEmailByPosition with args %s: %s' % \ (str(position, analystEmail), [traceback.format_exc()])) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol + 1: # append two extra columns self.__fileList[position].append('') self.__fileList[position].append(str(analystEmail)) else: self.__fileList[position][self.__fileAnalystEmailCol] = str(analystEmail) else: raise ValueError, 'setAnalystEmailByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getMetadataSummaryByFilename(self, filename, madExpObj = None, madInstObj = None, madKindatObj = None): """getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename. Inputs: filename - name of file to search for. The next three inputs are other metadata objects. If these objects already exist, performance will be improved by passing them in rather than recreating them. If they do not exist, they will be created: madExpObj - a MadrigalExperiment object to get experiment metadata from. If None (the default), a new MadrigalExperiment object is created. madInstObj - a MadrigalInstrument object to get experiment metadata from. If None (the default), a new MadrigalInstrument object is created. madKindatObj - a MadrigalKindat object to get experiment metadata from. If None (the default), a new MadrigalKindat object is created. Returns: A string summarizing the metadata about the file. The format is:: Start Date and Time: 01/06/1997 14:07 End Date and Time: 01/10/1997 23:45 Instrument: Millstone Hill Incoherent Scatter Radar Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study Kind of data: INSCAL (8.0) Basic Derived Parameters Affects: None Exceptions: Thrown if any parsing error in metadata. """ # build return String retStr = '' # create any needed metadata objects if madExpObj == None: madExpObj = MadrigalExperiment(self.__madDB) if madInstObj == None: madInstObj = MadrigalInstrument(self.__madDB) if madKindatObj == None: madKindatObj = MadrigalKindat(self.__madDB) # get the experiment id - if None, return message to update Metadata expId = self.getExpIdByFilename(filename) if expId == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' # get the experiment start date - if not found, return message to update Metadata starttime = madExpObj.getExpStartDateTimeByExpId(expId) if starttime == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' retStr = retStr + '\tExperiment start: ' + time.asctime(starttime) + '\n' # endtime endtime = madExpObj.getExpEndDateTimeByExpId(expId) retStr = retStr + '\tExperiment end: ' + time.asctime(endtime) + '\n' # get instrument name kinst = madExpObj.getKinstByExpId(expId) # if kinst not found, metadata is corrupt if kinst == None: return 'Could not find kinst for experiment id = ' + str(expId) instName = madInstObj.getInstrumentName(kinst) retStr = retStr + '\tInstrument name: ' + str(instName) + '\n' # get experiment name expName = madExpObj.getExpNameByExpId(expId) retStr = retStr + '\tExperiment name: ' + str(expName) + '\n' # get kind of data kindat = self.getKindatByFilename(filename) kindatName = madKindatObj.getKindatDescription(kindat, kinst) retStr = retStr + '\tKind of data: ' + str(kindatName) + '\n' return retStr def getFileDOIUrlByPosition(self, position): """getFileDOIUrlByPosition returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByPosition(position) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) basename = self.getFilenameByPosition(position) if basename is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, basename)) def getFileDOIUrlByFilename(self, filename): """getFileDOIUrlByFilename returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByFilename(filename) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, filename)) def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n') def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the fileTab.txt file. Inputs: newFullPath: a new path to write the fileTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def __str__(self): """return possibly modified file as a string in same format as writeMetadata """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' return(metaFileStr) class MadrigalParmCategory: """MadrigalParmCategory is an object that provides access to Madrigal parameter category info from the metadata. This object provides access to all Madrigal kind of data information in the metadata file madCatTab.txt. Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jan. 18, 2013 """ #constants _categoryMetadataFile = "madCatTab.txt" # column positions _categoryCodeCol = 0 _categoryDescCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get category metadata file if (initFile == None): self.__filename = self._categoryMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getCategoryDesc(self, code): """getCategoryDesc returns the category description that matches code argument, or None if not found. Inputs: code integer to get category description. or integer as string Returns: the category description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) for i, category in enumerate(self.__fileList): # find matching code try: if code == int(category[self._categoryCodeCol]): return(category[self._categoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in madCatTab.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getCategoryList(self): """getCategoryList returns a list of all category descriptions and codes. Inputs: None. Returns: a list of all category descriptions and codes. Each item in the list is a tuple of the form (Category description (string), code (integer)). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, category in enumerate(self.__fileList): item = (category[self._categoryDescCol], int(category[self._categoryCodeCol])) retList.append(item) return(retList) class MadrigalInstrumentData: """MadrigalInstrumentData is an object that provides access to Madrigal instrument data info from the metadata. This object provides access to all Madrigal kind of data information in the metadata files instData.txt and instDataPriv.txt. Those files summarize years data is available by instrument. Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Feb. 2, 2015 """ #constants _instDataMetadataFile = "instData.txt" # column positions _siteIDCol = 0 _kinstCol = 1 _yearsCol = 2 def __init__(self, madDB=None, priv=False, initFile=None, madInstObj=None): """__init__ initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt). Inputs: madDB - Existing MadrigalDB object, by default = None. priv - if True, use instDataPriv.txt instead of instData.txt. If False (the default), use instData.txt. initFile - String representing the full path to the metadata file. Default is None, in which case file read depends on priv argument. priv arg ingored if this not None madInstObj - m madrigal.metadata.MadrigalInstrument object. If None, one is created. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB if priv: self._instDataMetadataFile = "instDataPriv.txt" # get instData metadata file if (initFile == None): self.__filename = self._instDataMetadataFile else: self.__filename = initFile if madInstObj is not None: self._madInstObj = madInstObj else: self._madInstObj = madrigal.metadata.MadrigalInstrument(self.__madDB) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getCategories(self, local=False): """getCategories returns the a list of (category id, category desc) tuples. Inputs: local - if False, return all categories for which there is data anywhere in Madrigal. If True, only return local categories. Returns: a list of (category id, category desc) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is category id, value is category desc str. Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: if local: if int(data[self._siteIDCol]) != siteID: continue kinst = int(data[self._kinstCol]) categoryID = self._madInstObj.getCategoryId(kinst) if categoryID not in localDict.keys(): localDict[categoryID] = self._madInstObj.getCategory(kinst) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) categoryIDKeys = localDict.keys() categoryIDKeys.sort() retList = [(catID, localDict[catID]) for catID in categoryIDKeys] return(retList) def getInstruments(self, categoryID=0, local=False): """getInstruments returns the a list of (kinst, instrument desc, siteID) tuples. Inputs: categoryID - category id to return instruments with data for. If 0, return all local - if False, return all instruments with that category for which there is data anywhere in Madrigal. If True, only return local instruments, in which case siteID is always the local siteID. Returns: a list of (kinst, instrument desc, siteID) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is kinst, value is tuple if (instrument desc str, siteID). # Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: kinst = int(data[self._kinstCol]) thisSiteID = int(data[self._siteIDCol]) thisCategoryID = self._madInstObj.getCategoryId(kinst) if thisCategoryID != categoryID and categoryID != 0: continue if local: if thisSiteID != siteID: continue if kinst not in localDict.keys(): localDict[kinst] = (self._madInstObj.getInstrumentName(kinst), thisSiteID) else: raise ValueError, 'Duplicate kinst %i found in instData' % (kinst) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) kinstKeys = localDict.keys() kinstKeys.sort() retList = [(kinst, localDict[kinst][0], localDict[kinst][1]) for kinst in kinstKeys] return(retList) def getInstrumentYears(self, kinst): """getInstrumentYears returns the a list of years (int) for instrument. Inputs: kinst - the instrument id Returns: an ordered list of years as integers. If none found, raises error Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, data in enumerate(self.__fileList): # find matching code try: thisKinst = int(data[self._kinstCol]) if thisKinst != kinst: continue yearsStr = data[self._yearsCol] retList = [int(year) for year in yearsStr.split()] retList.sort() except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(retList) == 0: raise madrigal.admin.MadrigalError('No data found for kinst %i' % (kinst), '') return(retList) class MadrigalMetadata: """MadrigalMetadata is a private class that parses a Madrigal metadata file. This private class is used by all classes that need to parse a Madrigal metadata file. If the class is called with the name of the metadata file only, the metadata file is assumed to be at $MAD_ROOT/metadata. If a full path name is given that includes a directory separator, then that is used instead. The getList method returns a list with one item for each line in the file. That item for each line is simply a list of strings found in the line. The following is an example metadata file and the list the method getList would return. Metadata file example:: Tom, Dick,,Harry ,,Joe, Sally, Jane,Joe, Dick The list returned by getList example:: [['Tom', 'Dick', '', 'Harry'], ['', '', 'Joe', ''], ['Sally', 'Jane', 'Joe', 'Dick']] Non-standard Python modules used: None MadrigalError exception thrown if: 1. Unable to open metadata file. 2. All lines in metadata file do not have same number of items Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 8, 2001 """ # constants # file delimiter - presently a comma __DELIMITER = ',' def __init__(self, metadataFileName, madDB=None, allowedLenList=None, key=None): """__init__ initializes the MadrigalCategoryList by reading from __privateList. Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir is included. Existing MadrigalDB object, by default = None. allowedLenList - a list of allowed lengths (integers) for data lines. If None (the default), then the rule is that all lengths must be the same. key - which position in the file to use as a key. Creates a attribute called self._dict, which is a dictionary with keys = item in key position, value = line index. Used for faster lookup by key. If None (the default), self._dict = None Returns: void Affects: Initializes private member variable __categoryList. Exceptions: None. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # create empty list to hold parsed data self.__fileList = [] # used to check that every line has same number of words if allowedLenList == None self.__numWords = 0 self._allowedLenList = allowedLenList if not key is None: self._dict = {} else:self._dict = None # get real filename self.__fileName = self.__getFullPath(metadataFileName) # open configuration file try: self.__file = open(self.__fileName, "r") except IOError: raise madrigal.admin.MadrigalError("Unable to open metadata file " + self.__fileName, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # loop over each line in file, creating a list for each line = self.__file.readline() count = 0 while (len(line.strip())): self.__parseLine(line, key, count) line = self.__file.readline() count += 1 # close metadata file self.__file.close() def __parseLine(self, line, key=None, count=None): """__parseLine adds a list of items in line to __fileList. Inputs: Line of file to parse (String). key - position to add key for self._dict, value = line position If None (the default), ignore count - if None, ignore. Else add to self._dict as value (see key above) Returns: void Affects: Adds a list of items in line to self.__fileList. Exceptions: None. """ list = line.split(self.__DELIMITER) # create new list with leading and trailing whitespace stripped strippedList = [] for word in list: strippedList.append(word.strip()) self.__fileList.append(strippedList) if not self._dict is None: self._dict[list[key]] = count # check correct number of words if self._allowedLenList: if len(strippedList) not in self._allowedLenList: raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \ self.__fileName + " at line " + \ str(len(self.__fileList)), [traceback.format_exc()]) elif self.__numWords == 0: self.__numWords = len(strippedList) elif self.__numWords != len(strippedList): raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \ self.__fileName + " at line " + \ str(len(self.__fileList)), [traceback.format_exc()]) def __getFullPath(self, filename): """getFullPath returns the full path name of the metafile. Inputs: filename passed in as argument Returns: The full path name of the metafile. If the filename argument already is a full path, filename is returned unchanged. If the filename is simply the name of a metadata file, then $MAD_ROOT/metadata is appended Affects: Nothing Exceptions: None """ if (len(os.path.dirname(filename)) != 0): return filename fullName = self.__madDB.getMetadataDir() + "/" + filename # normalize in case we run on a system without / as a separator return os.path.normpath(fullName) # public methods def getList(self): """getList returns the list of lists of items in each line in the metafile. Inputs: None Returns: The list of lists of items in each line in the metafile. That is, each item in the returned list is itself a list, representing a single line in the metafile. That single line's list is the list of items in that line. Affects: Nothing Exceptions: None """ return(self.__fileList) def getDict(self): """getDict returns self_dict, which will be a dict with keys= key column set, value = line number, of None if no key passed into constructor """ return(self._dict) def toString(self): """toString returns a simple string representation of a MadrigalMetadata object. Inputs: None Returns: String describing a simple representation of a __MadrigalMetadat object. Affects: Nothing Exceptions: None """ return str(self.__fileList)
Functions
def getMadrigalUTFromDT(
dt)
getMadrigalUTFromDT returns the number of seconds since midnight UT 1950-01-01 for datetime dt
def getMadrigalUTFromDT(dt): """getMadrigalUTFromDT returns the number of seconds since midnight UT 1950-01-01 for datetime dt """ t1950 = 631152000.0 # offset from Unix time return(calendar.timegm(dt.timetuple()) + t1950)
def getMadrigalUTFromDate(
year, month, day, hour, minute, second, microsecond)
getMadrigalUTFromDate returns the number of seconds since midnight UT 1950-01-01 for date specified in arguments
def getMadrigalUTFromDate(year, month, day, hour, minute, second, microsecond): """getMadrigalUTFromDate returns the number of seconds since midnight UT 1950-01-01 for date specified in arguments """ dt = datetime.datetime(year, month, day, hour, minute, second, microsecond) return(getMadrigalUTFromDT(dt))
def getUnixUTFromDT(
dt)
getUnixUTFromDT returns the number of float seconds since midnight UT 1970-01-01 for datetime dt
def getUnixUTFromDT(dt): """getUnixUTFromDT returns the number of float seconds since midnight UT 1970-01-01 for datetime dt """ return(calendar.timegm(dt.timetuple()) + dt.microsecond/1.0E6)
def getUnixUTFromDate(
year, month, day, hour, minute, second, microsecond)
getUnixUTFromDate returns the number of seconds since midnight UT 1970-01-01 for date specified in arguments
def getUnixUTFromDate(year, month, day, hour, minute, second, microsecond): """getUnixUTFromDate returns the number of seconds since midnight UT 1970-01-01 for date specified in arguments """ dt = datetime.datetime(year, month, day, hour, minute, second, microsecond) return(getUnixUTFromDT(dt))
Classes
class MadrigalDB
MadrigalDB is an object that provides access to an entire Madrigal database.
This object provides complete high-level access to an entire Madrigal database. Presently, all its information comes from madrigal.cfg, or another path passed in by the user. If env variable madroot is not set, or if madrigal.cfg cannot be opened, the default values automatically editted during installation are used
Usage example::
import madrigal.metadata try: test = madrigal.metadata.MadrigalDB() except madrigal.admin.MadrigalError, e: print e.getExceptionStr() else: print test.toString()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. If the constructor is called with an configuration file path as an argument, and that file cannot be read. 2. If the configuration file cannot be parsed by ConfigParser. 3. If any of the following keys cannot be found in the configuration file: *__MAD_SERVER *__MAD_SERVERROOT *__SITE_ID *__HTML_STYLE *__INDEX_HEAD *__CON_TACTLINK
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Oct. 4, 2001
class MadrigalDB: """MadrigalDB is an object that provides access to an entire Madrigal database. This object provides complete high-level access to an entire Madrigal database. Presently, all its information comes from madrigal.cfg, or another path passed in by the user. If env variable madroot is not set, or if madrigal.cfg cannot be opened, the default values automatically editted during installation are used Usage example:: import madrigal.metadata try: test = madrigal.metadata.MadrigalDB() except madrigal.admin.MadrigalError, e: print e.getExceptionStr() else: print test.toString() Non-standard Python modules used: None MadrigalError exception thrown if: 1. If the constructor is called with an configuration file path as an argument, and that file cannot be read. 2. If the configuration file cannot be parsed by ConfigParser. 3. If any of the following keys cannot be found in the configuration file: *__MAD_SERVER *__MAD_SERVERROOT *__SITE_ID *__HTML_STYLE *__INDEX_HEAD *__CON_TACTLINK Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Oct. 4, 2001 """ # lines that are edited during installation __hardCodeMadRoot = '/Users/brideout/madroot3_2' __hardCodeMadServer = 'localhost:8000' __hardCodeMadServerRoot = '' __hardCodeSiteId = '999' __hardCodeHtmlStyle = '<BODY BGCOLOR=#FFFFFF LINK=#008000 VLINK=#003366>' __hardCodeIndexHead = 'Welcome to the Madrigal Database at mad31' __hardCodeContact = '<A HREF="MAILTO:brideout@mit.edu">Madrigal administator</A><BR>' __hardCodeMailServer = 'localhost' #constants __webProtocol = "http" """ Change the above string to use another web protocol such as https. """ __binDir = "/bin" """ Sets the relative path from madrigal root to the bin directory. """ __metadataDir = "/metadata" """ Sets the relative path from madrigal root to the metadata directory. """ __experimentDir = "/experiments" """ Sets the relative path from madrigal root to the experiment directory. """ __MAD_ROOT = "MAD" + "ROOT" """ Sets the name of the environment variable that points to the madrigal root directory. """ __confFileName = "madrigal.cfg" """ Sets the name of the default madrigal configuration file. """ __MAD_SERVER = "MAD" + "SERVER" """ Sets the key name in the configuration file to find the main madrigal url. """ __MAD_SERVERROOT = "MAD" + "SERVERROOT" """ Sets the key name in the configuration file to find the top level directory. """ __SITE_ID = "SITE" + "ID" """ Sets the key name in the configuration file to find the site id. """ __HTML_STYLE = "HTML" + "STYLE" """ Sets the key name in the configuration file to find the html body style tag. """ __INDEX_HEAD = "INDEX" + "HEAD" """ Sets the key name in the configuration file to find the heading in the top level madrigal page. """ __CON_TACTLINK = "CON" + "TACT" """ Sets the key name in the configuration file to find the contact link line. """ __MAIL_SERVER = "MAIL" + "SERVER" """ Sets the key name in the configuration file to find the mailserver. """ __PYTHON_EXE = "PYTHON" + "EXE" """ Sets the key name in the configuration file to find the python executable. """ def __init__(self, initFile=None): """__init__ initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString). Inputs: String representing the full path to the configuration file. Default is None, in which case configuration file is $(__MAD_ROOT)/__confFileName (now madrigal.cfg). Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown if error (see class description for specific conditions). Notes: Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile, this constructor goes about loading some basic data about the database, including information such as: -the database utility directory -the www home base Implemented using ConfigParser module. This reads ini type files. The only difference between ini files and madrigal.cfg is that ini files are broken into sections of the form: [sectionTitle] key1 = value1 key2 = value2 Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal" is appended to the beginning of the file. """ # if madroot not set, use hard coded madroot self.__madRootDir = os.environ.get(self.__MAD_ROOT) if (self.__madRootDir == None): self.__madRootDir = self.__hardCodeMadRoot # Set configuration file if (initFile == None): self.__confFilePath = self.__madRootDir + "/" + self.__confFileName else: self.__confFilePath = initFile # open configuration file try: self.__confFile = open(self.__confFilePath, "r") except IOError: # can't read from file - use all hard-coded values self.__initFromHardCode() self.__finishInit() return # create Parser using standard module ConfigParser self.__parser = ConfigParser.ConfigParser() # read conf file into a StringIO with "[madrigal]\n" section heading prepended strConfFile = StringIO.StringIO("[madrigal]\n" + self.__confFile.read()) # parse StringIO configuration file try: self.__parser.readfp(strConfFile) except: raise madrigal.admin.MadrigalError("Unable to parse configuration file " + self.__confFilePath, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # read information from configuration file self.__readConfFile() # close conf file self.__confFile.close() self.__finishInit() # end __init__ def __finishInit(self): """__finishInit is a private helper function that finishes initialization by combining attributes to form new ones. Inputs: None Returns: Void. Affects: Initializes class member variables that are combinations of existing ones. Exceptions: None. """ # combine information # Set the database utility directory self.__databaseUtilityDirectory = self.__madRootDir + self.__binDir # Set the wwwHomeBase self.__wwwHomeBase = self.__webProtocol + "://" + self.__mainUrl # Set the top level url if self.__topLevel.strip() not in ('', '.'): self.__topLevelUrl = self.__wwwHomeBase + '/' + self.__topLevel else: self.__topLevelUrl = self.__wwwHomeBase # Set contactEmail by stripping ContactLink line # Look for colon at end of Mailto: begIndexEmail = self.__contactlink.find(':') # look for "> at end of email address endIndexEmail = self.__contactlink.find('">') # if not found, try '> if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('\'>') # if still not found, try > alone if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('>') #check that both were found if begIndexEmail != -1 and endIndexEmail != -1: self.__contactEmail = self.__contactlink[begIndexEmail + 1 : endIndexEmail] if not self.__isValidEmail(self.__contactEmail): self.__contactEmail = None elif self.__isValidEmail(self.__contactlink): # user just entered an email self.__contactEmail = self.__contactlink # make contactlink proper link self.__contactlink = '<A HREF="MAILTO:%s”>%s</A>' % (self.__contactEmail, self.__contactEmail) else: self.__contactEmail = None # set pythonexe to default self.__pythonexe = self.__databaseUtilityDirectory + '/python' # end __init__ def __readConfFile(self): """__readConfFile is a private helper function that reads information from the parsed config file. Inputs: None Returns: Void. Affects: Initializes class member variables that are found in the config file. Exceptions: MadrigalError thrown if any key not found. """ # get the main url info try: self.__mainUrl = self.__parser.get("madrigal", self.__MAD_SERVER) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__MAD_SERVER + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the top level info try: self.__topLevel = self.__parser.get("madrigal", self.__MAD_SERVERROOT) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__MAD_SERVERROOT + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the site id try: self.__siteIdValue = self.__parser.get("madrigal", self.__SITE_ID) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__SITE_ID + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the htmlstyle try: self.__htmlstyle = self.__parser.get("madrigal", self.__HTML_STYLE) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__HTML_STYLE + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the indexhead try: self.__indexhead = self.__parser.get("madrigal", self.__INDEX_HEAD) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__INDEX_HEAD + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the contactlink try: self.__contactlink = self.__parser.get("madrigal", self.__CON_TACTLINK) except: raise madrigal.admin.MadrigalError("Unable to find key " + self.__CON_TACTLINK + \ " in configuration file due to ConfigParser error", traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # get the mailserver try: self.__mailserver = self.__parser.get("madrigal", self.__MAIL_SERVER) except: # no error in this case - simply default to localhost # this avoids an error if the old version of madrigal.cfg is present self.__mailserver = 'localhost' # end __readConfFile def __initFromHardCode(self): """__initFromHardCode is a private helper function that reads information from the automatically editted lines. Inputs: None Returns: Void. Affects: Initializes class member variables that are found in the constants at the top of this file. These constants were set during installation. Exceptions: None. """ self.__mainUrl = self.__hardCodeMadServer self.__topLevel = self.__hardCodeMadServerRoot self.__siteIdValue = int(self.__hardCodeSiteId) self.__htmlstyle = self.__hardCodeHtmlStyle self.__indexhead = self.__hardCodeIndexHead self.__contactlink = self.__hardCodeContact # mailserver may be empty if len(self.__hardCodeMailServer) > 0: self.__mailserver = self.__hardCodeMailServer else: self.__mailserver = 'localhost' # public methods def getDatabaseUtilityDirectory(self): """getDatabaseUtilityDirectory returns the full path to the database utility directory. Inputs: None Returns: String representing the full path to the database utility directory. (eg, /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__databaseUtilityDirectory def getWWWHomeBase(self): """getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__wwwHomeBase def getMadServer(self): """getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__mainUrl def getTopLevelUrl(self): """getTopLevelUrl returns the full url of the top level directory in main database website. Inputs: None Returns: String representing the full url to the top level directory in main database website. (eg, http://haystack.mit.edu/madrigal) Affects: Nothing Exceptions: None """ return self.__topLevelUrl def getRelativeTopLevel(self): """getRelativeTopLevel returns the relative url of the top level directory in main database website. Inputs: None Returns: String representing the relative url to the top level directory in main database website. (eg, madrigal) Affects: Nothing Exceptions: None """ return self.__topLevel def getSiteID(self): """getSiteID returns the site id number. Inputs: None Returns: The site id (integer) of the madrigal installation. Affects: Nothing Exceptions: If non-integer found """ try: return int(self.__siteIdValue) except: raise madrigal.admin.MadrigalError("Site id not an integer in madrigal configuration file " + \ self.__confFile, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def getMadrootEnvVarName(self): """getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT). Inputs: None Returns: The name of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__MAD_ROOT def getMadroot(self): """getMadroot returns the value of the environment variable __MAD_ROOT. Inputs: None Returns: The value of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__madRootDir def getMetadataDir(self): """getMetadataDir returns the metadata directory. Inputs: None Returns: The full metadata directory path. (eg. /opt/madrigal/metadata) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__metadataDir def getExperimentDir(self): """getExperimentDir returns the main experiment directory. No longer guarenteed to be unique,but will hold test files and geophysical experiments. Inputs: None Returns: The full experiment directory path. (eg. /opt/madrigal/experiments) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__experimentDir def getExperimentDirs(self): """getExperimentDirs returns a list of full paths of all valid experiment directories. Inputs: None Returns: list of full paths of all valid experiment directories. Valid experiment directories are those of the form of the regular expression $/Users/brideout/madroot3_2/experiments[0-9]* Affects: Nothing Exceptions: None """ reStr = '/experiments[0-9]*$' possibleExpDirs = glob.glob(os.path.join(self.__madRootDir, 'experiments*')) retList = [] for possibleExpDir in possibleExpDirs: result = re.search(reStr, possibleExpDir) if result: retList.append(possibleExpDir) return(retList) def getBinDir(self): """getBinDir returns the madrigal bin directory. Inputs: None Returns: The madrigal bin directory path. (eg /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__binDir def getHtmlStyle(self): """getHtmlSyle returns the default html body tag for the site. Inputs: None Returns: The default html body tag for the site. (eg. <BODY BGCOLOR=#FFFF88 LINK=#008000 VLINK=#003366>) Affects: Nothing Exceptions: None """ return self.__htmlstyle def getBackgroundColor(self): """getBackgroundColor returns the string representing background color from getHtmlStyle Inputs: None Returns the color string from the body tag in madrigal.cfg. For example, will return '#FFFF88; if <BODY BGCOLOR=#FFFFFF LINK=#008000 VLINK=#003366> is <BODY BGCOLOR=#FFFF88 LINK=#008000 VLINK=#003366> Returns empty string if no BGCOLOR= """ line = self.getHtmlStyle().upper() items = line.split() for item in items: index = item.find('BGCOLOR=') if index != -1: return(item[len('BGCOLOR='):]) return('') # not found def getIndexHead(self): """getIndexHead returns the heading of the top level madrigal page. Inputs: None Returns: The heading of the top level madrigal page. (eg. Welcome to the Madrigal Database <BR> at Ishtar) Affects: Nothing Exceptions: None """ return self.__indexhead def getContactLink(self): """getContactLink returns contact email link tag (see getContactEmail for the email alone). Inputs: None Returns: The contact email link tag. (eg. <A HREF="MAILTO:brideout@haystack.mit.edu">madrigal@haystack</A><BR>) Affects: Nothing Exceptions: None """ return self.__contactlink def getContactEmail(self): """getContactEmail returns the email address of the site administrator. Inputs: None Returns: The email address of the site administrator. Affects: Nothing Exceptions: None """ if not self.__contactEmail is None: return(self.__contactEmail) else: madSite = madrigal.metadata.MadrigalSite() return(madSite.getSiteEmail(self.getSiteID())) def getMailserver(self): """getMailserver returns the mailserver name. Inputs: None Returns: The mailserver name. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to localhost Affects: Nothing Exceptions: None """ return self.__mailserver def getPythonExecutable(self): """getPythonExecutable returns the full path to the python executable. Inputs: None Returns: the full path to the python executable. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to madroot/bin/python Affects: Nothing Exceptions: None """ return self.__pythonexe def getLocalRulesOfRoad(self): """getLocalRulesOfRoad returns the local rules of the road. Inputs: None Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that. Else returns a default rules_of_road statement Affects: Nothing Exceptions: None """ default_rules_of_road = 'Use of the Madrigal Database is generally subject to the ' + \ 'CEDAR Rules-of-the-Road . ' + \ 'Prior permission to access the data is not required. However, the user is required to establish ' + \ 'early contact with any organization whose data are involved in the project to discuss the ' + \ 'intended usage. Data are often subject to limitations which are not immediately evident to ' + \ 'new users. Before they are formally submitted, draft copies of all reports and publications ' + \ 'must be sent to the contact scientist at all data-supplying organizations along with an offer ' + \ 'of co-authorship to scientists who have provided data. This offer may be declined. The ' + \ 'Database and the organizations that contributed data must be acknowledged in all reports and ' + \ 'publications, and whenever this data is made available through another database. If you have ' + \ 'any questions about appropriate use of these data, contact <a href="mailto:%s">%s</a>' % (self.getContactEmail(), self.getContactEmail()) localRules = None try: f = open(os.path.join(self.getMadroot(), 'local_rules_of_the_road.txt')) localRules = f.read() f.close() except: pass if localRules == None: return default_rules_of_road else: return localRules def getFullPathFromPartial(self, partialPath): """getFullPathFromPartial returns the full path to a file or directory based on a partial path. Follows the rule that if partialPath begins with [/]experiments, then the full path is madroot + partialPath. Otherwise (pre madrigal 2.6) full path is madroot + experiments + partialPath Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10) Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10) """ if partialPath[:12].find('experiments') != -1: fullPath = os.path.join(self.getMadroot(), partialPath) else: fullPath = os.path.join(self.getMadroot(), 'experiments', partialPath) return(fullPath) def getExpList(self, expName = None, kinstList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, showIgnoredExperiments = False, enforcePathConvention = False): """getExpList returns a list of full experiment directory names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. startDate: a datetime date. If None (default), do not reject any experiments. endDate: a datetime date. If None (default), do not reject any experiments. startDayOfYear: a Julian day number (1-366) after which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any experiments. endDayOfYear: a Julian day number (1-366) before which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any experiments. showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If False (the default), only required path in the form 1999/mlh/*. Returns: a list of full experiment directory names that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None expList = [] # get all experiment directories expDirList = self.getExperimentDirs() # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: os.path.walk(thisExpDir, self.__getExperiments, (expName, kinstList, startDate, endDate, startDayOfYear, endDayOfYear, expList, showIgnoredExperiments, enforcePathConvention)) return expList def getFileList(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, enforcePathConvention = 0, includeNonDefault = 0, includeNonMadrigal = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0, path = None): """getFileList returns a list of full file names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. path if not None (the default), search only experiment directory set by path. If None, search add experiments. Returns: a list of full path file names (strings) that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None fileList = [] # get all experiment directories if path is None: expDirList = self.getExperimentDirs() else: expDirList = [path] # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: os.path.walk(thisExpDir, self.__getFiles, (expName, kinstList, kindatList, startDate, endDate, startDayOfYear, endDayOfYear, fileList, publicAccessOnly, enforcePathConvention, includeNonDefault, includeNonMadrigal, appendKinst, appendStartTime, includeRealtime)) return fileList def getFileListFromMetadata(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, includeNonDefault = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0): """getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only. This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't search the actual experiment directories. It is therefore faster, but less robust. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and starttime. Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None # the list to be returned fileList = [] # load a Madrigal experiment object madExpObj = MadrigalExperiment(self) # load a Madrigal metafile object madFileObj = MadrigalMetaFile(self) # convert input arguments to needed forms if expName != None: expNameArg = expName.replace(' ', '_') if startDate != None: startDateTime = madrigal.metadata.getMadrigalUTFromDate(startDate[0], startDate[1], startDate[2], startDate[3], startDate[4], startDate[5], 0) if endDate != None: endDateTime = madrigal.metadata.getMadrigalUTFromDate(endDate[0], endDate[1], endDate[2], endDate[3], endDate[4], endDate[5], 0) # loop through all experiments to create four lists: # acceptedExpIdList, acceptedExpKinstList, acceptedExpStartTimeList, and acceptedExpDirList acceptedExpIdList = [] acceptedExpKinstList = [] acceptedExpStartTimeList = [] acceptedExpDirList = [] position = -1 while 1: position += 1 # first position is zero thisName = madExpObj.getExpNameByPosition(position) #check if we're at the end if thisName == None: break # check experiment access thisSecurity = madExpObj.getSecurityByPosition(position) if thisSecurity == -1: continue elif publicAccessOnly == 0: if thisSecurity not in (0,1): continue elif publicAccessOnly == 1: if thisSecurity != 0: continue elif publicAccessOnly == 2: if thisSecurity not in (0,2): continue # apply expName filter if desired if expName != None: # since we are using file name matching, all spaces are # converted to underscores thisName = thisName.replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(thisName.lower(), expNameArg.lower()): continue # apply kinstList filter thisKinst = madExpObj.getKinstByPosition(position) if kinstList != None: if thisKinst not in kinstList: continue # startDate filter - we always need thisStartTime whether or not filter used thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) thisStartTime = madrigal.metadata.getMadrigalUTFromDate(thisStartDate[0], thisStartDate[1], thisStartDate[2], thisStartDate[3], thisStartDate[4], thisStartDate[5], 0) if endDate != None: if thisStartTime > endDateTime: continue # endDate filter if endDate != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) thisEndTime = madrigal.metadata.getMadrigalUTFromDate(thisEndDate[0], thisEndDate[1], thisEndDate[2], thisEndDate[3], thisEndDate[4], thisEndDate[5], 0) if startDate != None: if thisEndTime < startDateTime: continue # apply startDayOfYear filter if startDayOfYear != None: thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) if thisStartDate[7] < startDayOfYear: # index 7 refers to Day of year continue # apply endDayOfYear filter if endDayOfYear != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) if thisEndDate[7] > endDayOfYear: # index 7 refers to Day of year continue # this experiment has made it through all filters, append its id, kinst, startTime, and dir acceptedExpIdList.append(madExpObj.getExpIdByPosition(position)) acceptedExpKinstList.append(thisKinst) acceptedExpStartTimeList.append(thisStartTime) # find the directory based on url dir = self.getMadroot() if dir[-1] != '/': dir += '/' url = madExpObj.getExpUrlByPosition(position) index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: dir += 'experiments/' dir += url[index+8:] acceptedExpDirList.append(dir) # now loop through the file object to find all files position = -1 while 1: position += 1 # first position is zero thisExpId = madFileObj.getExpIdByPosition(position) #check if we're at the end if thisExpId == None: break # skip this file if expId not in acceptedExpIdList if thisExpId not in acceptedExpIdList: continue # apply kindatList filter thisKindat = madFileObj.getKindatByPosition(position) if kindatList != None: if thisKindat not in kindatList: continue # check file access if madFileObj.getAccessByPosition(position) != 0: if publicAccessOnly in (1,2): continue # apply includeNonDefault filter if includeNonDefault == 0: category = madFileObj.getCategoryByPosition(position) if includeRealtime == 0: if category != 1: # not default continue else: if category not in (1,4): # not default or realtime continue # this file has been accepted by all filters - first, get its experiment index expIndex = acceptedExpIdList.index(thisExpId) thisFilename = acceptedExpDirList[expIndex] + '/' + madFileObj.getFilenameByPosition(position) # append result to fileList according to appendKinst and appendStartTime if appendKinst == 0 and appendStartTime == 0: fileList.append(thisFilename) elif appendKinst == 1 and appendStartTime == 0: fileList.append((thisFilename, acceptedExpKinstList[expIndex])) elif appendKinst == 0 and appendStartTime == 1: fileList.append((thisFilename, acceptedExpStartTimeList[expIndex])) else: fileList.append((thisFilename, acceptedExpKinstList[expIndex], acceptedExpStartTimeList[expIndex])) return fileList def setFileAccess(self, expDirectory, accessMode): """setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Inputs: expDirectory: The full path to a directory in the experiment directory. That is, it may be madroot/experiments[0-9]* or any directory under it. accessMode: either 0 for public access, or 1 for private access. Returns: None Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Exceptions: If accessMode is not 1 or 0. """ if (accessMode != 0 and accessMode != 1): raise madrigal.admin.MadrigalError('MadrigalDB.setFileAccess called with accessMode = ' + \ str(accessMode) + ', must be either 0 or 1', None) # walk the experiments directory to find all files meeting criteria os.path.walk(expDirectory, self.__setFileAccess, accessMode) def tarExperiments(self, tarFileName, startDate = None, endDate = None, excludePrivData = 0, ignoreDirCon = 1, includeNonDefData = 0, onlyData = 0, filetype = 0, verbose = 0): """tarExperiments creates a tar file containing files from madroot/experiments[0-9]*. Note: this method sometimes requires the modification of the fileTab.txt files found in the experiments directory. This is because some data files might be excluded, so that the fileTab.txt file will no longer be accurate. Because of this, all files to be tar'ed will be copied to /tmp/temp<random num>/experiments, where the fileTab.txt files will be modified. When done, this temp dir will be deleted. Inputs: tarFileName: The full path to a tar file to be created. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. excludePrivData: if 1, allow data marked as private to be omitted (and the line from the fileTab.txt to be removed). If 0 (the default), all data, public and private, will be included. ignoreDirCon: if 1, ignore convention that directory must be in form 1999/mlh/03sep99 (the default). If 0 , reject non-standard directories. includeNonDefData: if 1, include all files listed in fileTab.txt. If 0 (the default), reject non-default files, and modify fileTab.txt to remove non-default listings. onlyData: if 1, reject all files not listed in fileTab.txt. If 0 (the default), accept all files in a directory not mentioned in fileTab.txt. filetype: format to save data files as. Default 0 is to leave present format unchanged. <type> is an integer as follows: type = 0 Leave present format unchanged (default) type = 1 Madrigal type = 2 Blocked Binary type = 3 Cbf type = 4 Unblocked binary type = 5 Ascii verbose: if 1, print to std out the list of files included (relative path). If 0, (the default) print nothing. Returns: None Affects: created tar file tarFileName of selected files from madroot/experiments. Exceptions: If unable to read any experiment file. """ if ignoreDirCon == 1: enforcePathConvention = 0 else: enforcePathConvention = 1 if onlyData == 1: includeNonMadrigal = 0 else: includeNonMadrigal = 1 # create a random temp dir tempDir = '/tmp/temp' + str(random.randrange(1,10000000)) if verbose == 1: print 'Creating list of files to tar...' # get list of files to tar tarFileList = self.getFileList(None, None, None, startDate, endDate, None, None, excludePrivData, enforcePathConvention, includeNonDefData, includeNonMadrigal) # now create a new list of filenames, using relative paths relTarFileList = [] for file in tarFileList: newFilename = file.split(self.getMadroot() + '/')[1] relTarFileList.append(newFilename) # copy all these files to tempDir if verbose == 1: print 'The following is a list of files included:' for file in relTarFileList: # make sure dir exists try: os.makedirs(tempDir + '/' + os.path.dirname(file)) except: pass # now copy file shutil.copyfile(self.getMadroot() + '/' + file, tempDir + '/' + file) if verbose == 1: print '\t' + file if onlyData: # now check if that data file needs to be converted to another filetype # since onlyData is set, convert every file if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + file, tempDir + '/' + file + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + file + '.backup 1 100000000 -o ' + \ tempDir + '/' + file + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + file + '.backup') if verbose == 1: print 'Modifying fileTab.txt files if needed...' # modify fileTab.txt files, if required if (not onlyData) and (excludePrivData or not includeNonDefData): # first loop through each fileTab.txt file in list just to check permissions for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # make sure fileTab.txt is writable if not os.access(tempDir + '/' + filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(filename), None) # make sure the directory is writable if not os.access(os.path.dirname(tempDir + '/' + filename), os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(os.path.dirname(filename)), None) # no exceptions raised - all permissions are okay # this time loop through and modify the fileTab.txt files for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # create a MadrigalMetaFile object fileMeta = MadrigalMetaFile(self, tempDir + '/' + filename) # loop through each file name to see if its in tarFileList: fileNum = 0 while 1: madFilename = fileMeta.getFilenameByPosition(fileNum) if madFilename == None: break # get madRelFilename madRelFilename = os.path.dirname(filename) + '/' + madFilename # if its not in relTarFileList, delete it from fileMeta if not madRelFilename in relTarFileList: fileMeta.deleteRowByFilename(madFilename) fileNum = 0 continue else: # we know madRelFilename is a data file since its in fileTab.txt - # now check if that data file needs to be converted to another filetype if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + madRelFilename, tempDir + '/' + madRelFilename + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + madRelFilename + '.backup 1 100000000 -o ' + \ tempDir + '/' + madRelFilename + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + madRelFilename + '.backup') # get next madFilename fileNum = fileNum + 1 # if fileMeta not empty, write it out if fileMeta.getFileCount() > 0: fileMeta.writeMetadata() # else if its empty, simply delete it else: os.remove(tempDir + '/' + filename) # done modifying fileTab.txt - ready to tar # need to change working directory to tempDir to get relative paths in tar. # Since this directory will soon be deleted, will need to change it back to # whatever it is now when we're done pwd = os.getcwd() # check if tarFileName is absolute or relative to pwd if tarFileName[0] != '/': tarFileName = pwd + '/' + tarFileName if verbose == 1: print 'Creating tar file...' os.chdir(tempDir) os.system('tar -cf ' + tarFileName + ' experiments*') os.chdir(pwd) if verbose == 1: print 'Removing temp files...' # finally remove temp dir try: shutil.rmtree(tempDir) except: raise madrigal.admin.MadrigalError('In tarExperiments could not remove dir ' + tempDir, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def listFileTimes(self, expDir=None, relative=True): """listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory Inputs: expDir - the particular subdirectory of an experiment directory to list. If None (the default), will list all files for all experiment directories. May be an absolute path, or may start with experiments[0-9]*. relative - if True (the default) give the path relative to the experiments[0-9]* directory. If False, give full path Returns: a list of tuples, where each tuple has 1. the file path, and 2. a UT datetime object of the last file modification. Exceptions: raised if expDir is not a valid experiments directory or subdirectory """ expDirs = self.getExperimentDirs() if expDir == None: dirsToExamine = expDirs else: # verify a valid directory found = False if expDir[-1] == '/': expDir = expDir[:-1] # strip trailing / if expDir[0:11] == 'experiments': # convert to absolute path expDir = os.path.join(self.getMadroot(), expDir) for thisDir in expDirs: if expDir.find(thisDir) != -1: found = True if found: dirsToExamine = [expDir] else: raise ValueError, 'expDir %s not a valid experiment directory' % (expDir) retList = [] for dirToExamine in dirsToExamine: for root, dirs, files in os.walk(dirToExamine): for name in files: fullname = os.path.join(root, name) relIndex = fullname[len(self.__madRootDir):].find('/experiments') relativeName = fullname[relIndex + 1 + (len(self.__madRootDir)):] ts = os.stat(fullname).st_mtime if relative: retList.append((relativeName, datetime.datetime.utcfromtimestamp(ts))) else: retList.append((fullname, datetime.datetime.utcfromtimestamp(ts))) return(retList) def getKinstKindatConfig(self, kinst, kindat, iniFile=None): """getKinstKindatConfig gets information for the /Users/brideout/madroot3_2/cachedFiles.ini needed to create Madrigal3 files if not specified. Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them. Inputs: kinst - the instrument kinst (integer) kindat - the data kindat (integer) iniFile - the ini file to use. If None, uses default ini file $/Users/brideout/madroot3_2/cachedFiles.ini Returns: a tuple with three items: 1. a list of extra parameters (string mnemonics) 2. a list of independent spatial parameters 3. a list of array splitting parameters Algorithm: 1. If iniFile == None and no default file, returns ([], [], []) 2. Searches ini file for section [%i] % (kinst). If not found, returns ([], [], []) 3. Searches right section for key %i_parms % (kindat). If not found, searches for default_parms. If not found, extra parameters are [] 4. Searches right section for key %i_formats % (kindat). If not found, searches for default_formats. If not found, indepedent spatial parms and extra splitting parameters are [] and []. If found, then parses dictionary, and returns key 'array'. If value has only one item, returns it as a one item list of independent spatial parameters. If two items, they are ind spatial parms and array splitting parms. """ if not iniFile: thisIniFile = os.path.join(self.getMadroot(), 'cachedFiles.ini') if not os.access(thisIniFile, os.R_OK): return(([], [], [])) else: thisIniFile = iniFile instSection = '%i' % (kinst) parser = ConfigParser.SafeConfigParser() parser.read(thisIniFile) if not parser.has_section(instSection): return(([], [], [])) extraParms = [] indSpatialParms = [] arraySplitParms = [] # get extra parms if parser.has_option(instSection, '%i_parms' % (kindat)): extraParms = parser.get(instSection, '%i_parms' % (kindat)) extraParms = extraParms.split(',') elif parser.has_option(instSection, 'default_parms'): extraParms = parser.get(instSection, 'default_parms') extraParms = extraParms.split(',') # make sure no empty parms snuck in finalExtraParms = [] for extraParm in extraParms: if len(extraParm.strip()) > 0: finalExtraParms.append(extraParm.strip()) # get format dict formatDict = None if parser.has_option(instSection, '%i_formats' % (kindat)): formatDict = parser.get(instSection, '%i_formats' % (kindat)) formatDict = eval(formatDict) elif parser.has_option(instSection, 'default_formats'): formatDict = parser.get(instSection, 'default_formats') formatDict = eval(formatDict) if not formatDict is None: if formatDict.has_key('array'): value = formatDict['array'] if type(value) in (types.ListType, types.TupleType): indSpatialParms = value[0] arraySplitParms = value[1] else: indSpatialParms = [value] return((finalExtraParms, indSpatialParms, arraySplitParms)) def toString(self): """toString returns a simple string representation of a MadrigalDB object. Inputs: None Returns: String describing a simple representation of a MadrigalDB object. Affects: Nothing Exceptions: None """ output = "Object type: MadrigalDB\n" output += "Database utility directory = " + self.getDatabaseUtilityDirectory() + "\n" output += "WWW home base = " + self.getWWWHomeBase() + "\n" output += "Server name = " + self.getMadServer() + "\n" output += "Top level url = " + self.getTopLevelUrl() + "\n" output += "Relative Top level url = " + self.getRelativeTopLevel() + "\n" output += "Site ID = " + str(self.getSiteID()) + "\n" output += "MAD_ROOT env. variable name = " + self.getMadrootEnvVarName() + "\n" output += "MAD_ROOT env. variable value = " + self.getMadroot() + "\n" output += "Madrigal metadata dir = " + self.getMetadataDir() + "\n" output += "Madrigal bin dir = " + self.getBinDir() + "\n" output += "Madrigal html body style = " + self.getHtmlStyle() + "\n" output += "Madrigal top level heading = " + self.getIndexHead() + "\n" output += "Madrigal contact link = " + self.getContactLink() + "\n" output += "Madrigal contact email = " + str(self.getContactEmail()) + "\n" output += "Madrigal mailserver = " + self.getMailserver() + "\n" output += "Madrigal python exe = " + self.getPythonExecutable() + "\n" return output def isTestExperiment(self, url, siteId=None): """isTestExperiment returns True if the given experiment url is a test of this Madrigal server. Url can be either real or form in expTab.txt, or can be the experiment directory. If siteId not given, use local site id. """ # Skip test experiments if siteId == None: siteId = self.getSiteID() if url.find('1998/mlh/20jan98') != -1 and siteId != 1: return(True) if url.find('1997/aro/06jan97') != -1 and siteId != 7: return(True) if url.find('1997/lyr/08apr97') != -1 and siteId != 2: return(True) if url.find('1997/son/06jan97') != -1 and siteId != 3: return(True) if url.find('1995/jro/01feb95') != -1 and siteId != 6: return(True) if url.find('1998/jro/27apr98') != -1 and siteId != 6: return(True) return(False) def __str__(self): """ __str__ simply calls toString """ return (self.toString()) def __isValidEmail(self, emailStr): """ __isValidEmail is a private helper function that does some checking to ensure a valid email address was found. Inputs: emailStr - email address string to verify Returns: 1 if no problems, 0 if not valid. Affects: Nothing Exceptions: None """ emailStr = emailStr.strip() if emailStr.find(' ') != -1: return 0 if emailStr.find('"') != -1: return 0 if emailStr.find('<') != -1: return 0 if emailStr.find('>') != -1: return 0 if emailStr.find('/') != -1: return 0 if emailStr.find(':') != -1: return 0 if emailStr.find('@') == -1: return(0) # otherwise okay return 1 def __getExperiments(self, arg, dirname, names): """ __getExperiment is a private helper function called by os.path.walk in getExpList. __getExperiments is called for each sub-directory in the experiments directory. Its purpose is to add any experiment directory paths from that directory that match the search criteria to the expList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. startDate: a python datetime in UTC. If None, do not reject any files. endDate: a python datetime in UTC. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. expList: the list of valid experiment paths to which is appended any newly-found valid experiment directories showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If False (the default), only required path in the form 1999/mlh/*. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __startDate = 2 __endDate = 3 __startDayOfYear = 4 __endDayOfYear = 5 __expList = 6 __showIgnored = 7 __enforcePath = 8 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt if not 'expTab.txt' in names: return # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply startDate filter expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError, 'Error getting expStartDate from dir %s' % (dirname) time1 = datetime.datetime(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5]) if arg[__endDate] != None: if time1 > arg[__endDate]: # stop going down tree names[:] = [] return # apply endDate filter if arg[__startDate] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() time1 = datetime.datetime(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5]) if time1 < arg[__startDate]: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply ignore filter if expMeta.getSecurityByPosition() == -1 and not arg[__showIgnored]: # stop going down tree names[:] = [] return # now reject paths not matching present path convention if not enforcePathConvention if not arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention if arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay arg[__expList].append(dirname) def __getFiles(self, arg, dirname, names): """ __getFiles is a private helper function called by os.path.walk in getFileList. __getFiles is called for each sub-directory in the experiments directory. Its purpose is to add any file paths from that directory that match the search criteria to the fileList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None, all kindat values are accepted. startDate: a python date/time in UTC (see time module - actually a tuple of nine integers) after which to accept files. If None, do not reject any files. endDate: a python date/time in UTC (see time module - actually a tuple of nine integers) before which to accept files. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. fileList: the list of valid file paths to which is appended any newly-found valid file paths publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __kindatList = 2 __startDate = 3 __endDate = 4 __startDayOfYear = 5 __endDayOfYear = 6 __fileList = 7 __pubAccessOnly = 8 __enforcePath = 9 __includeNonDef = 10 __includeNonMad = 11 __appendKinst = 12 __appendStartTime = 13 __includeRealtime = 14 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt and fileTab.txt if not includeNonMadrigal if 'expTab.txt' in names: self.__hasExpTab = 1 else: self.__hasExpTab = 0 if 'fileTab.txt' in names: self.__hasFileTab = 1 else: self.__hasFileTab = 0 if arg[__includeNonMad] == 0: if not self.__hasExpTab or not self.__hasFileTab: return # apply all filters relating to expTab.txt if self.__hasExpTab if self.__hasExpTab: # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply date filters expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError, 'Error getting expStartDate from dir %s' % (dirname) expStartUt = madrigal.metadata.getMadrigalUTFromDate(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5], 0) expEndDate = expMeta.getExpEndDateTimeByPosition() expEndUt = madrigal.metadata.getMadrigalUTFromDate(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5], 0) if arg[__startDate] != None: time1 = madrigal.metadata.getMadrigalUTFromDate(arg[__startDate][0], arg[__startDate][1], arg[__startDate][2], arg[__startDate][3], arg[__startDate][4], arg[__startDate][5], 0) if time1 > expEndUt: # stop going down tree names[:] = [] return # save time1 in case we need to appendStartTime thisStartTime = expStartUt # apply endDate filter if arg[__endDate] != None: time2 = madrigal.metadata.getMadrigalUTFromDate(arg[__endDate][0], arg[__endDate][1], arg[__endDate][2], arg[__endDate][3], arg[__endDate][4], arg[__endDate][5], 0) if time2 < expStartUt: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # check experiment access thisSecurity = expMeta.getSecurityByPosition() if thisSecurity == -1: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 0: if thisSecurity not in (0,1): # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 1: if thisSecurity != 0: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 2: if thisSecurity not in (0,2): # stop going down tree names[:] = [] return else: # no expTab.txt - set kinst to None expKinst = None # now reject paths not matching present path convention if enforcePathConvention == 0 if arg[__enforcePath] == 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention != 0 if arg[__enforcePath] != 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay, now start looking at individual files if self.__hasFileTab: fileMeta = MadrigalMetaFile(self, dirname + '/' + 'fileTab.txt') # get numFiles numFiles = fileMeta.getFileCount() # loop through each file, and add it if okay fileCount = 0 while fileCount < numFiles: # skip non-default and non-realtime files if not includeNonDef if fileMeta.getCategoryByPosition(fileCount) not in (1,4) and arg[__includeNonDef] == 0: fileCount = fileCount + 1 continue # skip realtime files if not includeNonDef and not includeRealtime if fileMeta.getCategoryByPosition(fileCount) == 4 and arg[__includeNonDef] == 0 and arg[__includeRealtime] == 0: fileCount = fileCount + 1 continue # check if file has right kindat if arg[__kindatList] != None: if fileMeta.getKindatByPosition(fileCount) not in arg[__kindatList]: fileCount = fileCount + 1 continue # check file access if fileMeta.getAccessByPosition(fileCount) != 0: if arg[__pubAccessOnly] in (1,2): fileCount = fileCount + 1 continue # the file needs to be added filename = fileMeta.getFilenameByPosition(fileCount) if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + filename) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + filename, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + filename, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) fileCount = fileCount + 1 # now add all non-madrigal files if includeNonMad if arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue # if its not in fileTab.txt, add it if fileMeta.getExpIdByFilename(name) == None: if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) # else the file has no fileTab.txt and includeNonMad is true - include all files elif arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) def __setFileAccess(self, arg, dirname, names): """setFileAccess is a private helper function called by os.path.walk in setFileAccess. Inputs: arg: either 0 for public access, or 1 for private access. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None Affects: sets all fileTab.txt files in dirname to be public or private, depending on arg. Exceptions: None. """ for name in names: if name == 'fileTab.txt': try: fileMetaObj = madrigal.metadata.MadrigalMetaFile(self, dirname + '/' + name) fileMetaObj.setAccess(arg) except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Ancestors (in MRO)
Methods
def __init__(
self, initFile=None)
init initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString).
Inputs: String representing the full path to the configuration file. Default is None, in which case configuration file is $(MAD_ROOT)/confFileName (now madrigal.cfg).
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown if error (see class description for specific conditions).
Notes:
Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile, this constructor goes about loading some basic data about the database, including information such as: -the database utility directory -the www home base Implemented using ConfigParser module. This reads ini type files. The only difference between ini files and madrigal.cfg is that ini files are broken into sections of the form: [sectionTitle] key1 = value1 key2 = value2 Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal" is appended to the beginning of the file.
def __init__(self, initFile=None): """__init__ initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString). Inputs: String representing the full path to the configuration file. Default is None, in which case configuration file is $(__MAD_ROOT)/__confFileName (now madrigal.cfg). Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown if error (see class description for specific conditions). Notes: Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile, this constructor goes about loading some basic data about the database, including information such as: -the database utility directory -the www home base Implemented using ConfigParser module. This reads ini type files. The only difference between ini files and madrigal.cfg is that ini files are broken into sections of the form: [sectionTitle] key1 = value1 key2 = value2 Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal" is appended to the beginning of the file. """ # if madroot not set, use hard coded madroot self.__madRootDir = os.environ.get(self.__MAD_ROOT) if (self.__madRootDir == None): self.__madRootDir = self.__hardCodeMadRoot # Set configuration file if (initFile == None): self.__confFilePath = self.__madRootDir + "/" + self.__confFileName else: self.__confFilePath = initFile # open configuration file try: self.__confFile = open(self.__confFilePath, "r") except IOError: # can't read from file - use all hard-coded values self.__initFromHardCode() self.__finishInit() return # create Parser using standard module ConfigParser self.__parser = ConfigParser.ConfigParser() # read conf file into a StringIO with "[madrigal]\n" section heading prepended strConfFile = StringIO.StringIO("[madrigal]\n" + self.__confFile.read()) # parse StringIO configuration file try: self.__parser.readfp(strConfFile) except: raise madrigal.admin.MadrigalError("Unable to parse configuration file " + self.__confFilePath, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # read information from configuration file self.__readConfFile() # close conf file self.__confFile.close() self.__finishInit()
def getBackgroundColor(
self)
getBackgroundColor returns the string representing background color from getHtmlStyle
Inputs: None
Returns the color string from the body tag in madrigal.cfg. For example, will return '#FFFF88; if
is Returns empty string if no BGCOLOR=def getBackgroundColor(self): """getBackgroundColor returns the string representing background color from getHtmlStyle Inputs: None Returns the color string from the body tag in madrigal.cfg. For example, will return '#FFFF88; if <BODY BGCOLOR=#FFFFFF LINK=#008000 VLINK=#003366> is <BODY BGCOLOR=#FFFF88 LINK=#008000 VLINK=#003366> Returns empty string if no BGCOLOR= """ line = self.getHtmlStyle().upper() items = line.split() for item in items: index = item.find('BGCOLOR=') if index != -1: return(item[len('BGCOLOR='):]) return('') # not found
def getBinDir(
self)
getBinDir returns the madrigal bin directory.
Inputs: None
Returns: The madrigal bin directory path. (eg /opt/madrigal/bin)
Affects: Nothing
Exceptions: None
def getBinDir(self): """getBinDir returns the madrigal bin directory. Inputs: None Returns: The madrigal bin directory path. (eg /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__binDir
def getContactEmail(
self)
getContactEmail returns the email address of the site administrator.
Inputs: None
Returns: The email address of the site administrator.
Affects: Nothing
Exceptions: None
def getContactEmail(self): """getContactEmail returns the email address of the site administrator. Inputs: None Returns: The email address of the site administrator. Affects: Nothing Exceptions: None """ if not self.__contactEmail is None: return(self.__contactEmail) else: madSite = madrigal.metadata.MadrigalSite() return(madSite.getSiteEmail(self.getSiteID()))
def getContactLink(
self)
getContactLink returns contact email link tag (see getContactEmail for the email alone).
Inputs: None
Returns: The contact email link tag.
(eg. madrigal@haystack
)
Affects: Nothing
Exceptions: None
def getContactLink(self): """getContactLink returns contact email link tag (see getContactEmail for the email alone). Inputs: None Returns: The contact email link tag. (eg. <A HREF="MAILTO:brideout@haystack.mit.edu">madrigal@haystack</A><BR>) Affects: Nothing Exceptions: None """ return self.__contactlink
def getDatabaseUtilityDirectory(
self)
getDatabaseUtilityDirectory returns the full path to the database utility directory.
Inputs: None
Returns: String representing the full path to the database utility directory. (eg, /opt/madrigal/bin)
Affects: Nothing
Exceptions: None
def getDatabaseUtilityDirectory(self): """getDatabaseUtilityDirectory returns the full path to the database utility directory. Inputs: None Returns: String representing the full path to the database utility directory. (eg, /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__databaseUtilityDirectory
def getExpList(
self, expName=None, kinstList=None, startDate=None, endDate=None, startDayOfYear=None, endDayOfYear=None, showIgnoredExperiments=False, enforcePathConvention=False)
getExpList returns a list of full experiment directory names that match the search arguments.
Inputs:
expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. startDate: a datetime date. If None (default), do not reject any experiments. endDate: a datetime date. If None (default), do not reject any experiments. startDayOfYear: a Julian day number (1-366) after which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any experiments. endDayOfYear: a Julian day number (1-366) before which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any experiments. showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If False (the default), only required path in the form 1999/mlh/*.
Returns: a list of full experiment directory names that match the search arguments
Affects: Nothing
Exceptions: None
def getExpList(self, expName = None, kinstList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, showIgnoredExperiments = False, enforcePathConvention = False): """getExpList returns a list of full experiment directory names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. startDate: a datetime date. If None (default), do not reject any experiments. endDate: a datetime date. If None (default), do not reject any experiments. startDayOfYear: a Julian day number (1-366) after which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any experiments. endDayOfYear: a Julian day number (1-366) before which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any experiments. showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If False (the default), only required path in the form 1999/mlh/*. Returns: a list of full experiment directory names that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None expList = [] # get all experiment directories expDirList = self.getExperimentDirs() # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: os.path.walk(thisExpDir, self.__getExperiments, (expName, kinstList, startDate, endDate, startDayOfYear, endDayOfYear, expList, showIgnoredExperiments, enforcePathConvention)) return expList
def getExperimentDir(
self)
getExperimentDir returns the main experiment directory. No longer guarenteed to be unique,but will hold test files and geophysical experiments.
Inputs: None
Returns: The full experiment directory path. (eg. /opt/madrigal/experiments)
Affects: Nothing
Exceptions: None
def getExperimentDir(self): """getExperimentDir returns the main experiment directory. No longer guarenteed to be unique,but will hold test files and geophysical experiments. Inputs: None Returns: The full experiment directory path. (eg. /opt/madrigal/experiments) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__experimentDir
def getExperimentDirs(
self)
getExperimentDirs returns a list of full paths of all valid experiment directories.
Inputs: None
Returns: list of full paths of all valid experiment directories. Valid experiment directories are those of the form of the regular expression $/Users/brideout/madroot3_2/experiments[0-9]*
Affects: Nothing
Exceptions: None
def getExperimentDirs(self): """getExperimentDirs returns a list of full paths of all valid experiment directories. Inputs: None Returns: list of full paths of all valid experiment directories. Valid experiment directories are those of the form of the regular expression $/Users/brideout/madroot3_2/experiments[0-9]* Affects: Nothing Exceptions: None """ reStr = '/experiments[0-9]*$' possibleExpDirs = glob.glob(os.path.join(self.__madRootDir, 'experiments*')) retList = [] for possibleExpDir in possibleExpDirs: result = re.search(reStr, possibleExpDir) if result: retList.append(possibleExpDir) return(retList)
def getFileList(
self, expName=None, kinstList=None, kindatList=None, startDate=None, endDate=None, startDayOfYear=None, endDayOfYear=None, publicAccessOnly=3, enforcePathConvention=0, includeNonDefault=0, includeNonMadrigal=0, appendKinst=0, appendStartTime=0, includeRealtime=0, path=None)
getFileList returns a list of full file names that match the search arguments.
Inputs:
expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. path if not None (the default), search only experiment directory set by path. If None, search add experiments.
Returns: a list of full path file names (strings) that match the search arguments
Affects: Nothing
Exceptions: None
def getFileList(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, enforcePathConvention = 0, includeNonDefault = 0, includeNonMadrigal = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0, path = None): """getFileList returns a list of full file names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional single character>. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. path if not None (the default), search only experiment directory set by path. If None, search add experiments. Returns: a list of full path file names (strings) that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None fileList = [] # get all experiment directories if path is None: expDirList = self.getExperimentDirs() else: expDirList = [path] # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: os.path.walk(thisExpDir, self.__getFiles, (expName, kinstList, kindatList, startDate, endDate, startDayOfYear, endDayOfYear, fileList, publicAccessOnly, enforcePathConvention, includeNonDefault, includeNonMadrigal, appendKinst, appendStartTime, includeRealtime)) return fileList
def getFileListFromMetadata(
self, expName=None, kinstList=None, kindatList=None, startDate=None, endDate=None, startDayOfYear=None, endDayOfYear=None, publicAccessOnly=3, includeNonDefault=0, appendKinst=0, appendStartTime=0, includeRealtime=0)
getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only.
This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't search the actual experiment directories. It is therefore faster, but less robust.
Inputs:
expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0.
Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and starttime.
Affects: Nothing
Exceptions: None
def getFileListFromMetadata(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, includeNonDefault = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0): """getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only. This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't search the actual experiment directories. It is therefore faster, but less robust. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and starttime. Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None # the list to be returned fileList = [] # load a Madrigal experiment object madExpObj = MadrigalExperiment(self) # load a Madrigal metafile object madFileObj = MadrigalMetaFile(self) # convert input arguments to needed forms if expName != None: expNameArg = expName.replace(' ', '_') if startDate != None: startDateTime = madrigal.metadata.getMadrigalUTFromDate(startDate[0], startDate[1], startDate[2], startDate[3], startDate[4], startDate[5], 0) if endDate != None: endDateTime = madrigal.metadata.getMadrigalUTFromDate(endDate[0], endDate[1], endDate[2], endDate[3], endDate[4], endDate[5], 0) # loop through all experiments to create four lists: # acceptedExpIdList, acceptedExpKinstList, acceptedExpStartTimeList, and acceptedExpDirList acceptedExpIdList = [] acceptedExpKinstList = [] acceptedExpStartTimeList = [] acceptedExpDirList = [] position = -1 while 1: position += 1 # first position is zero thisName = madExpObj.getExpNameByPosition(position) #check if we're at the end if thisName == None: break # check experiment access thisSecurity = madExpObj.getSecurityByPosition(position) if thisSecurity == -1: continue elif publicAccessOnly == 0: if thisSecurity not in (0,1): continue elif publicAccessOnly == 1: if thisSecurity != 0: continue elif publicAccessOnly == 2: if thisSecurity not in (0,2): continue # apply expName filter if desired if expName != None: # since we are using file name matching, all spaces are # converted to underscores thisName = thisName.replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(thisName.lower(), expNameArg.lower()): continue # apply kinstList filter thisKinst = madExpObj.getKinstByPosition(position) if kinstList != None: if thisKinst not in kinstList: continue # startDate filter - we always need thisStartTime whether or not filter used thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) thisStartTime = madrigal.metadata.getMadrigalUTFromDate(thisStartDate[0], thisStartDate[1], thisStartDate[2], thisStartDate[3], thisStartDate[4], thisStartDate[5], 0) if endDate != None: if thisStartTime > endDateTime: continue # endDate filter if endDate != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) thisEndTime = madrigal.metadata.getMadrigalUTFromDate(thisEndDate[0], thisEndDate[1], thisEndDate[2], thisEndDate[3], thisEndDate[4], thisEndDate[5], 0) if startDate != None: if thisEndTime < startDateTime: continue # apply startDayOfYear filter if startDayOfYear != None: thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) if thisStartDate[7] < startDayOfYear: # index 7 refers to Day of year continue # apply endDayOfYear filter if endDayOfYear != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) if thisEndDate[7] > endDayOfYear: # index 7 refers to Day of year continue # this experiment has made it through all filters, append its id, kinst, startTime, and dir acceptedExpIdList.append(madExpObj.getExpIdByPosition(position)) acceptedExpKinstList.append(thisKinst) acceptedExpStartTimeList.append(thisStartTime) # find the directory based on url dir = self.getMadroot() if dir[-1] != '/': dir += '/' url = madExpObj.getExpUrlByPosition(position) index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: dir += 'experiments/' dir += url[index+8:] acceptedExpDirList.append(dir) # now loop through the file object to find all files position = -1 while 1: position += 1 # first position is zero thisExpId = madFileObj.getExpIdByPosition(position) #check if we're at the end if thisExpId == None: break # skip this file if expId not in acceptedExpIdList if thisExpId not in acceptedExpIdList: continue # apply kindatList filter thisKindat = madFileObj.getKindatByPosition(position) if kindatList != None: if thisKindat not in kindatList: continue # check file access if madFileObj.getAccessByPosition(position) != 0: if publicAccessOnly in (1,2): continue # apply includeNonDefault filter if includeNonDefault == 0: category = madFileObj.getCategoryByPosition(position) if includeRealtime == 0: if category != 1: # not default continue else: if category not in (1,4): # not default or realtime continue # this file has been accepted by all filters - first, get its experiment index expIndex = acceptedExpIdList.index(thisExpId) thisFilename = acceptedExpDirList[expIndex] + '/' + madFileObj.getFilenameByPosition(position) # append result to fileList according to appendKinst and appendStartTime if appendKinst == 0 and appendStartTime == 0: fileList.append(thisFilename) elif appendKinst == 1 and appendStartTime == 0: fileList.append((thisFilename, acceptedExpKinstList[expIndex])) elif appendKinst == 0 and appendStartTime == 1: fileList.append((thisFilename, acceptedExpStartTimeList[expIndex])) else: fileList.append((thisFilename, acceptedExpKinstList[expIndex], acceptedExpStartTimeList[expIndex])) return fileList
def getFullPathFromPartial(
self, partialPath)
getFullPathFromPartial returns the full path to a file or directory based on a partial path.
Follows the rule that if partialPath begins with [/]experiments, then the full path is madroot + partialPath. Otherwise (pre madrigal 2.6) full path is madroot + experiments + partialPath
Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10)
Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10)
def getFullPathFromPartial(self, partialPath): """getFullPathFromPartial returns the full path to a file or directory based on a partial path. Follows the rule that if partialPath begins with [/]experiments, then the full path is madroot + partialPath. Otherwise (pre madrigal 2.6) full path is madroot + experiments + partialPath Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10) Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10) """ if partialPath[:12].find('experiments') != -1: fullPath = os.path.join(self.getMadroot(), partialPath) else: fullPath = os.path.join(self.getMadroot(), 'experiments', partialPath) return(fullPath)
def getHtmlStyle(
self)
getHtmlSyle returns the default html body tag for the site.
Inputs: None
Returns: The default html body tag for the site. (eg.
)Affects: Nothing
Exceptions: None
def getHtmlStyle(self): """getHtmlSyle returns the default html body tag for the site. Inputs: None Returns: The default html body tag for the site. (eg. <BODY BGCOLOR=#FFFF88 LINK=#008000 VLINK=#003366>) Affects: Nothing Exceptions: None """ return self.__htmlstyle
def getIndexHead(
self)
getIndexHead returns the heading of the top level madrigal page.
Inputs: None
Returns: The heading of the top level madrigal page.
(eg. Welcome to the Madrigal Database
at Ishtar)
Affects: Nothing
Exceptions: None
def getIndexHead(self): """getIndexHead returns the heading of the top level madrigal page. Inputs: None Returns: The heading of the top level madrigal page. (eg. Welcome to the Madrigal Database <BR> at Ishtar) Affects: Nothing Exceptions: None """ return self.__indexhead
def getKinstKindatConfig(
self, kinst, kindat, iniFile=None)
getKinstKindatConfig gets information for the /Users/brideout/madroot3_2/cachedFiles.ini needed to create Madrigal3 files if not specified.
Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them.
Inputs:
kinst - the instrument kinst (integer) kindat - the data kindat (integer) iniFile - the ini file to use. If None, uses default ini file $/Users/brideout/madroot3_2/cachedFiles.ini
Returns: a tuple with three items: 1. a list of extra parameters (string mnemonics) 2. a list of independent spatial parameters 3. a list of array splitting parameters
Algorithm:
- If iniFile == None and no default file, returns ([], [], [])
- Searches ini file for section [%i] % (kinst). If not found, returns ([], [], [])
- Searches right section for key %i_parms % (kindat). If not found, searches for default_parms. If not found, extra parameters are []
- Searches right section for key %i_formats % (kindat). If not found, searches for default_formats. If not found, indepedent spatial parms and extra splitting parameters are [] and []. If found, then parses dictionary, and returns key 'array'. If value has only one item, returns it as a one item list of independent spatial parameters. If two items, they are ind spatial parms and array splitting parms.
def getKinstKindatConfig(self, kinst, kindat, iniFile=None): """getKinstKindatConfig gets information for the /Users/brideout/madroot3_2/cachedFiles.ini needed to create Madrigal3 files if not specified. Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them. Inputs: kinst - the instrument kinst (integer) kindat - the data kindat (integer) iniFile - the ini file to use. If None, uses default ini file $/Users/brideout/madroot3_2/cachedFiles.ini Returns: a tuple with three items: 1. a list of extra parameters (string mnemonics) 2. a list of independent spatial parameters 3. a list of array splitting parameters Algorithm: 1. If iniFile == None and no default file, returns ([], [], []) 2. Searches ini file for section [%i] % (kinst). If not found, returns ([], [], []) 3. Searches right section for key %i_parms % (kindat). If not found, searches for default_parms. If not found, extra parameters are [] 4. Searches right section for key %i_formats % (kindat). If not found, searches for default_formats. If not found, indepedent spatial parms and extra splitting parameters are [] and []. If found, then parses dictionary, and returns key 'array'. If value has only one item, returns it as a one item list of independent spatial parameters. If two items, they are ind spatial parms and array splitting parms. """ if not iniFile: thisIniFile = os.path.join(self.getMadroot(), 'cachedFiles.ini') if not os.access(thisIniFile, os.R_OK): return(([], [], [])) else: thisIniFile = iniFile instSection = '%i' % (kinst) parser = ConfigParser.SafeConfigParser() parser.read(thisIniFile) if not parser.has_section(instSection): return(([], [], [])) extraParms = [] indSpatialParms = [] arraySplitParms = [] # get extra parms if parser.has_option(instSection, '%i_parms' % (kindat)): extraParms = parser.get(instSection, '%i_parms' % (kindat)) extraParms = extraParms.split(',') elif parser.has_option(instSection, 'default_parms'): extraParms = parser.get(instSection, 'default_parms') extraParms = extraParms.split(',') # make sure no empty parms snuck in finalExtraParms = [] for extraParm in extraParms: if len(extraParm.strip()) > 0: finalExtraParms.append(extraParm.strip()) # get format dict formatDict = None if parser.has_option(instSection, '%i_formats' % (kindat)): formatDict = parser.get(instSection, '%i_formats' % (kindat)) formatDict = eval(formatDict) elif parser.has_option(instSection, 'default_formats'): formatDict = parser.get(instSection, 'default_formats') formatDict = eval(formatDict) if not formatDict is None: if formatDict.has_key('array'): value = formatDict['array'] if type(value) in (types.ListType, types.TupleType): indSpatialParms = value[0] arraySplitParms = value[1] else: indSpatialParms = [value] return((finalExtraParms, indSpatialParms, arraySplitParms))
def getLocalRulesOfRoad(
self)
getLocalRulesOfRoad returns the local rules of the road.
Inputs: None
Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that. Else returns a default rules_of_road statement
Affects: Nothing
Exceptions: None
def getLocalRulesOfRoad(self): """getLocalRulesOfRoad returns the local rules of the road. Inputs: None Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that. Else returns a default rules_of_road statement Affects: Nothing Exceptions: None """ default_rules_of_road = 'Use of the Madrigal Database is generally subject to the ' + \ 'CEDAR Rules-of-the-Road . ' + \ 'Prior permission to access the data is not required. However, the user is required to establish ' + \ 'early contact with any organization whose data are involved in the project to discuss the ' + \ 'intended usage. Data are often subject to limitations which are not immediately evident to ' + \ 'new users. Before they are formally submitted, draft copies of all reports and publications ' + \ 'must be sent to the contact scientist at all data-supplying organizations along with an offer ' + \ 'of co-authorship to scientists who have provided data. This offer may be declined. The ' + \ 'Database and the organizations that contributed data must be acknowledged in all reports and ' + \ 'publications, and whenever this data is made available through another database. If you have ' + \ 'any questions about appropriate use of these data, contact <a href="mailto:%s">%s</a>' % (self.getContactEmail(), self.getContactEmail()) localRules = None try: f = open(os.path.join(self.getMadroot(), 'local_rules_of_the_road.txt')) localRules = f.read() f.close() except: pass if localRules == None: return default_rules_of_road else: return localRules
def getMadServer(
self)
getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu).
Inputs: None
Returns: String representing the url to the main database website.
Affects: Nothing
Exceptions: None
def getMadServer(self): """getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__mainUrl
def getMadroot(
self)
getMadroot returns the value of the environment variable __MAD_ROOT.
Inputs: None
Returns: The value of the environment variable __MAD_ROOT.
Affects: Nothing
Exceptions: None
def getMadroot(self): """getMadroot returns the value of the environment variable __MAD_ROOT. Inputs: None Returns: The value of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__madRootDir
def getMadrootEnvVarName(
self)
getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT).
Inputs: None
Returns: The name of the environment variable __MAD_ROOT.
Affects: Nothing
Exceptions: None
def getMadrootEnvVarName(self): """getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT). Inputs: None Returns: The name of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__MAD_ROOT
def getMailserver(
self)
getMailserver returns the mailserver name.
Inputs: None
Returns: The mailserver name. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to localhost
Affects: Nothing
Exceptions: None
def getMailserver(self): """getMailserver returns the mailserver name. Inputs: None Returns: The mailserver name. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to localhost Affects: Nothing Exceptions: None """ return self.__mailserver
def getMetadataDir(
self)
getMetadataDir returns the metadata directory.
Inputs: None
Returns: The full metadata directory path. (eg. /opt/madrigal/metadata)
Affects: Nothing
Exceptions: None
def getMetadataDir(self): """getMetadataDir returns the metadata directory. Inputs: None Returns: The full metadata directory path. (eg. /opt/madrigal/metadata) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__metadataDir
def getPythonExecutable(
self)
getPythonExecutable returns the full path to the python executable.
Inputs: None
Returns: the full path to the python executable. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to madroot/bin/python
Affects: Nothing
Exceptions: None
def getPythonExecutable(self): """getPythonExecutable returns the full path to the python executable. Inputs: None Returns: the full path to the python executable. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to madroot/bin/python Affects: Nothing Exceptions: None """ return self.__pythonexe
def getRelativeTopLevel(
self)
getRelativeTopLevel returns the relative url of the top level directory in main database website.
Inputs: None
Returns: String representing the relative url to the top level directory in main database website. (eg, madrigal)
Affects: Nothing
Exceptions: None
def getRelativeTopLevel(self): """getRelativeTopLevel returns the relative url of the top level directory in main database website. Inputs: None Returns: String representing the relative url to the top level directory in main database website. (eg, madrigal) Affects: Nothing Exceptions: None """ return self.__topLevel
def getSiteID(
self)
getSiteID returns the site id number.
Inputs: None
Returns: The site id (integer) of the madrigal installation.
Affects: Nothing
Exceptions: If non-integer found
def getSiteID(self): """getSiteID returns the site id number. Inputs: None Returns: The site id (integer) of the madrigal installation. Affects: Nothing Exceptions: If non-integer found """ try: return int(self.__siteIdValue) except: raise madrigal.admin.MadrigalError("Site id not an integer in madrigal configuration file " + \ self.__confFile, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
def getTopLevelUrl(
self)
getTopLevelUrl returns the full url of the top level directory in main database website.
Inputs: None
Returns: String representing the full url to the top level directory in main database website. (eg, http://haystack.mit.edu/madrigal)
Affects: Nothing
Exceptions: None
def getTopLevelUrl(self): """getTopLevelUrl returns the full url of the top level directory in main database website. Inputs: None Returns: String representing the full url to the top level directory in main database website. (eg, http://haystack.mit.edu/madrigal) Affects: Nothing Exceptions: None """ return self.__topLevelUrl
def getWWWHomeBase(
self)
getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu).
Inputs: None
Returns: String representing the url to the main database website.
Affects: Nothing
Exceptions: None
def getWWWHomeBase(self): """getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__wwwHomeBase
def isTestExperiment(
self, url, siteId=None)
isTestExperiment returns True if the given experiment url is a test of this Madrigal server. Url can be either real or form in expTab.txt, or can be the experiment directory. If siteId not given, use local site id.
def isTestExperiment(self, url, siteId=None): """isTestExperiment returns True if the given experiment url is a test of this Madrigal server. Url can be either real or form in expTab.txt, or can be the experiment directory. If siteId not given, use local site id. """ # Skip test experiments if siteId == None: siteId = self.getSiteID() if url.find('1998/mlh/20jan98') != -1 and siteId != 1: return(True) if url.find('1997/aro/06jan97') != -1 and siteId != 7: return(True) if url.find('1997/lyr/08apr97') != -1 and siteId != 2: return(True) if url.find('1997/son/06jan97') != -1 and siteId != 3: return(True) if url.find('1995/jro/01feb95') != -1 and siteId != 6: return(True) if url.find('1998/jro/27apr98') != -1 and siteId != 6: return(True) return(False)
def listFileTimes(
self, expDir=None, relative=True)
listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory
Inputs:
expDir - the particular subdirectory of an experiment directory to list. If None (the default), will list all files for all experiment directories. May be an absolute path, or may start with experiments[0-9]*. relative - if True (the default) give the path relative to the experiments[0-9]* directory. If False, give full path
Returns: a list of tuples, where each tuple has 1. the file path, and 2. a UT datetime object of the last file modification.
Exceptions: raised if expDir is not a valid experiments directory or subdirectory
def listFileTimes(self, expDir=None, relative=True): """listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory Inputs: expDir - the particular subdirectory of an experiment directory to list. If None (the default), will list all files for all experiment directories. May be an absolute path, or may start with experiments[0-9]*. relative - if True (the default) give the path relative to the experiments[0-9]* directory. If False, give full path Returns: a list of tuples, where each tuple has 1. the file path, and 2. a UT datetime object of the last file modification. Exceptions: raised if expDir is not a valid experiments directory or subdirectory """ expDirs = self.getExperimentDirs() if expDir == None: dirsToExamine = expDirs else: # verify a valid directory found = False if expDir[-1] == '/': expDir = expDir[:-1] # strip trailing / if expDir[0:11] == 'experiments': # convert to absolute path expDir = os.path.join(self.getMadroot(), expDir) for thisDir in expDirs: if expDir.find(thisDir) != -1: found = True if found: dirsToExamine = [expDir] else: raise ValueError, 'expDir %s not a valid experiment directory' % (expDir) retList = [] for dirToExamine in dirsToExamine: for root, dirs, files in os.walk(dirToExamine): for name in files: fullname = os.path.join(root, name) relIndex = fullname[len(self.__madRootDir):].find('/experiments') relativeName = fullname[relIndex + 1 + (len(self.__madRootDir)):] ts = os.stat(fullname).st_mtime if relative: retList.append((relativeName, datetime.datetime.utcfromtimestamp(ts))) else: retList.append((fullname, datetime.datetime.utcfromtimestamp(ts))) return(retList)
def setFileAccess(
self, expDirectory, accessMode)
setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private.
Inputs:
expDirectory: The full path to a directory in the experiment directory. That is, it may be madroot/experiments[0-9]* or any directory under it. accessMode: either 0 for public access, or 1 for private access.
Returns: None
Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private.
Exceptions: If accessMode is not 1 or 0.
def setFileAccess(self, expDirectory, accessMode): """setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Inputs: expDirectory: The full path to a directory in the experiment directory. That is, it may be madroot/experiments[0-9]* or any directory under it. accessMode: either 0 for public access, or 1 for private access. Returns: None Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Exceptions: If accessMode is not 1 or 0. """ if (accessMode != 0 and accessMode != 1): raise madrigal.admin.MadrigalError('MadrigalDB.setFileAccess called with accessMode = ' + \ str(accessMode) + ', must be either 0 or 1', None) # walk the experiments directory to find all files meeting criteria os.path.walk(expDirectory, self.__setFileAccess, accessMode)
def tarExperiments(
self, tarFileName, startDate=None, endDate=None, excludePrivData=0, ignoreDirCon=1, includeNonDefData=0, onlyData=0, filetype=0, verbose=0)
tarExperiments creates a tar file containing files from madroot/experiments[0-9]*.
Note: this method sometimes requires the modification of the fileTab.txt
files found in the experiments directory. This is because some data files might be
excluded, so that the fileTab.txt file will no longer be accurate. Because of this,
all files to be tar'ed will be copied to /tmp/temp
Inputs:
tarFileName: The full path to a tar file to be created. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. excludePrivData: if 1, allow data marked as private to be omitted (and the line from the fileTab.txt to be removed). If 0 (the default), all data, public and private, will be included. ignoreDirCon: if 1, ignore convention that directory must be in form 1999/mlh/03sep99 (the default). If 0 , reject non-standard directories. includeNonDefData: if 1, include all files listed in fileTab.txt. If 0 (the default), reject non-default files, and modify fileTab.txt to remove non-default listings. onlyData: if 1, reject all files not listed in fileTab.txt. If 0 (the default), accept all files in a directory not mentioned in fileTab.txt. filetype: format to save data files as. Default 0 is to leave present format unchanged. <type> is an integer as follows: type = 0 Leave present format unchanged (default) type = 1 Madrigal type = 2 Blocked Binary type = 3 Cbf type = 4 Unblocked binary type = 5 Ascii verbose: if 1, print to std out the list of files included (relative path). If 0, (the default) print nothing.
Returns: None
Affects: created tar file tarFileName of selected files from madroot/experiments.
Exceptions: If unable to read any experiment file.
def tarExperiments(self, tarFileName, startDate = None, endDate = None, excludePrivData = 0, ignoreDirCon = 1, includeNonDefData = 0, onlyData = 0, filetype = 0, verbose = 0): """tarExperiments creates a tar file containing files from madroot/experiments[0-9]*. Note: this method sometimes requires the modification of the fileTab.txt files found in the experiments directory. This is because some data files might be excluded, so that the fileTab.txt file will no longer be accurate. Because of this, all files to be tar'ed will be copied to /tmp/temp<random num>/experiments, where the fileTab.txt files will be modified. When done, this temp dir will be deleted. Inputs: tarFileName: The full path to a tar file to be created. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. excludePrivData: if 1, allow data marked as private to be omitted (and the line from the fileTab.txt to be removed). If 0 (the default), all data, public and private, will be included. ignoreDirCon: if 1, ignore convention that directory must be in form 1999/mlh/03sep99 (the default). If 0 , reject non-standard directories. includeNonDefData: if 1, include all files listed in fileTab.txt. If 0 (the default), reject non-default files, and modify fileTab.txt to remove non-default listings. onlyData: if 1, reject all files not listed in fileTab.txt. If 0 (the default), accept all files in a directory not mentioned in fileTab.txt. filetype: format to save data files as. Default 0 is to leave present format unchanged. <type> is an integer as follows: type = 0 Leave present format unchanged (default) type = 1 Madrigal type = 2 Blocked Binary type = 3 Cbf type = 4 Unblocked binary type = 5 Ascii verbose: if 1, print to std out the list of files included (relative path). If 0, (the default) print nothing. Returns: None Affects: created tar file tarFileName of selected files from madroot/experiments. Exceptions: If unable to read any experiment file. """ if ignoreDirCon == 1: enforcePathConvention = 0 else: enforcePathConvention = 1 if onlyData == 1: includeNonMadrigal = 0 else: includeNonMadrigal = 1 # create a random temp dir tempDir = '/tmp/temp' + str(random.randrange(1,10000000)) if verbose == 1: print 'Creating list of files to tar...' # get list of files to tar tarFileList = self.getFileList(None, None, None, startDate, endDate, None, None, excludePrivData, enforcePathConvention, includeNonDefData, includeNonMadrigal) # now create a new list of filenames, using relative paths relTarFileList = [] for file in tarFileList: newFilename = file.split(self.getMadroot() + '/')[1] relTarFileList.append(newFilename) # copy all these files to tempDir if verbose == 1: print 'The following is a list of files included:' for file in relTarFileList: # make sure dir exists try: os.makedirs(tempDir + '/' + os.path.dirname(file)) except: pass # now copy file shutil.copyfile(self.getMadroot() + '/' + file, tempDir + '/' + file) if verbose == 1: print '\t' + file if onlyData: # now check if that data file needs to be converted to another filetype # since onlyData is set, convert every file if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + file, tempDir + '/' + file + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + file + '.backup 1 100000000 -o ' + \ tempDir + '/' + file + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + file + '.backup') if verbose == 1: print 'Modifying fileTab.txt files if needed...' # modify fileTab.txt files, if required if (not onlyData) and (excludePrivData or not includeNonDefData): # first loop through each fileTab.txt file in list just to check permissions for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # make sure fileTab.txt is writable if not os.access(tempDir + '/' + filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(filename), None) # make sure the directory is writable if not os.access(os.path.dirname(tempDir + '/' + filename), os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(os.path.dirname(filename)), None) # no exceptions raised - all permissions are okay # this time loop through and modify the fileTab.txt files for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # create a MadrigalMetaFile object fileMeta = MadrigalMetaFile(self, tempDir + '/' + filename) # loop through each file name to see if its in tarFileList: fileNum = 0 while 1: madFilename = fileMeta.getFilenameByPosition(fileNum) if madFilename == None: break # get madRelFilename madRelFilename = os.path.dirname(filename) + '/' + madFilename # if its not in relTarFileList, delete it from fileMeta if not madRelFilename in relTarFileList: fileMeta.deleteRowByFilename(madFilename) fileNum = 0 continue else: # we know madRelFilename is a data file since its in fileTab.txt - # now check if that data file needs to be converted to another filetype if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + madRelFilename, tempDir + '/' + madRelFilename + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + madRelFilename + '.backup 1 100000000 -o ' + \ tempDir + '/' + madRelFilename + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + madRelFilename + '.backup') # get next madFilename fileNum = fileNum + 1 # if fileMeta not empty, write it out if fileMeta.getFileCount() > 0: fileMeta.writeMetadata() # else if its empty, simply delete it else: os.remove(tempDir + '/' + filename) # done modifying fileTab.txt - ready to tar # need to change working directory to tempDir to get relative paths in tar. # Since this directory will soon be deleted, will need to change it back to # whatever it is now when we're done pwd = os.getcwd() # check if tarFileName is absolute or relative to pwd if tarFileName[0] != '/': tarFileName = pwd + '/' + tarFileName if verbose == 1: print 'Creating tar file...' os.chdir(tempDir) os.system('tar -cf ' + tarFileName + ' experiments*') os.chdir(pwd) if verbose == 1: print 'Removing temp files...' # finally remove temp dir try: shutil.rmtree(tempDir) except: raise madrigal.admin.MadrigalError('In tarExperiments could not remove dir ' + tempDir, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
def toString(
self)
toString returns a simple string representation of a MadrigalDB object.
Inputs: None
Returns: String describing a simple representation of a MadrigalDB object.
Affects: Nothing
Exceptions: None
def toString(self): """toString returns a simple string representation of a MadrigalDB object. Inputs: None Returns: String describing a simple representation of a MadrigalDB object. Affects: Nothing Exceptions: None """ output = "Object type: MadrigalDB\n" output += "Database utility directory = " + self.getDatabaseUtilityDirectory() + "\n" output += "WWW home base = " + self.getWWWHomeBase() + "\n" output += "Server name = " + self.getMadServer() + "\n" output += "Top level url = " + self.getTopLevelUrl() + "\n" output += "Relative Top level url = " + self.getRelativeTopLevel() + "\n" output += "Site ID = " + str(self.getSiteID()) + "\n" output += "MAD_ROOT env. variable name = " + self.getMadrootEnvVarName() + "\n" output += "MAD_ROOT env. variable value = " + self.getMadroot() + "\n" output += "Madrigal metadata dir = " + self.getMetadataDir() + "\n" output += "Madrigal bin dir = " + self.getBinDir() + "\n" output += "Madrigal html body style = " + self.getHtmlStyle() + "\n" output += "Madrigal top level heading = " + self.getIndexHead() + "\n" output += "Madrigal contact link = " + self.getContactLink() + "\n" output += "Madrigal contact email = " + str(self.getContactEmail()) + "\n" output += "Madrigal mailserver = " + self.getMailserver() + "\n" output += "Madrigal python exe = " + self.getPythonExecutable() + "\n" return output
class MadrigalExperiment
MadrigalExperiment is an object that provides access to Madrigal experiment info from the metadata.
This object provides access to all Madrigal experiment information in the metadata file expTab.txt.
Usage example::
import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalExperiment() print test.getExperimentName(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Apr. 17, 2002
class MadrigalExperiment: """MadrigalExperiment is an object that provides access to Madrigal experiment info from the metadata. This object provides access to all Madrigal experiment information in the metadata file expTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalExperiment() print test.getExperimentName(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Apr. 17, 2002 """ #constants __expMetadataFile = "expTab.txt" # column positions __expIdCol = 0 __expUrlCol = 1 __expNameCol = 2 __expSiteIdCol = 3 __expStartDateCol = 4 __expStartTimeCol = 5 __expEndDateCol = 6 __expEndTimeCol = 7 __expKinstCol = 8 __expSecurityCol = 9 __expPICol = 10 __expPIEmailCol = 11 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalExperiment by reading from expTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/expTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get experiment metadata file if (initFile == None): self.__filename = self.__expMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without PI and email allowedLens = (self.__expSecurityCol+1, self.__expPIEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__expIdCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expIdCol] = str(int(expId)) self._dict[position] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str((position, expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpUrlByPosition(self, position = 0): """getExpUrlByPosition returns the experiment url of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment url, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expUrlCol] else: return(None) def getExpUrlByExpId(self, expId): """getExpUrlByExpId returns the experiment url for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpUrlByPosition(position)) def getRealExpUrlByPosition(self, position = 0): """getRealExpUrlByPosition returns the real experiment url of the experiment at given position. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the real experiment url, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index == -1: return(thisUrl) if thisUrl.find('experiments') != -1: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + thisUrl[index+8:]) else: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + 'experiments', thisUrl[index+8:]) return(realUrl) def getRealExpUrlByExpId(self, expId): """getRealExpUrlByExpId returns the real experiment url for a given experiment id. The url in the metadata contains /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the real experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getRealExpUrlByPosition(position)) def getExpPathByPosition(self, position = 0): """getExpPathByPosition returns the experiment path of the experiment at given position. Experiment path is the path to the experiment from the madroot directory, and always begins "experiments" Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index != -1: return(thisUrl[index+8:]) index = thisUrl.find('experiments') if index == -1: return(None) return(thisUrl[index:]) def getExpPathByExpId(self, expId): """getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the path to the experiment from the madroot directory (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpPathByPosition(position)) def setExpUrlByPosition(self, position, expUrl): """setExpUrlByPosition sets the experiment url of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expUrl - the new experiment url to use Returns: None. Affects: sets the experiment url of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expUrl.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_exc()])) try: self.__fileList[position][self.__expUrlCol] = str(expUrl) except: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_ex()])) def getExpDirByPosition(self, position = 0): """getExpDirByPosition returns the full experiment directory of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the full experiment directory, or None if position >= number of experiments. Uses experiment url to determine directory. Affects: None Exceptions: None """ if len(self.__fileList) > position: url = self.__fileList[position][self.__expUrlCol] # find the directory based on url maddir = self.__madDB.getMadroot() index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: maddir = os.path.join(maddir, 'experiments', url[index+8:]) else: maddir = os.path.join(maddir, url[index+8:]) return(maddir) else: return(None) def getExpDirByExpId(self, expId): """getExpDirByExpId returns the full experiment directory for a given experiment id. Inputs: Experiment Id (integer). Returns: the full experiment directory (string). Returns None if experiment id not found. Uses experiment url to determine directory. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpDirByPosition(position)) def getExpNameByPosition(self, position = 0): """getExpNameByPosition returns the experiment name of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment name, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expNameCol] else: return(None) def getExpNameByExpId(self, expId): """getExpNameByExpId returns the experiment name for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment name (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpNameByPosition(position)) def setExpNameByPosition(self, position, expName): """setExpNameByPosition sets the experiment name of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expName - the new experiment name to use Returns: None. Affects: sets the experiment name of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expName.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), [traceback.format_exc()])) try: self.__fileList[position][self.__expNameCol] = str(expName) except: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpSiteIdByExpId(self, expId): """getExpSiteIdByExpId returns the site id (int) for a given experiment id. Inputs: Experiment Id (integer). Returns: the site id for this experiment. Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpSiteIdByPosition(position)) def getExpSiteIdByPosition(self, position = 0): """getExpSiteIdByPosition returns the experiment site id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment site id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSiteIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpSiteIdByPosition(self, position, expSiteId): """setExpSiteIdByPosition sets the experiment site id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expSiteId - the new experiment site id to use Returns: None. Affects: sets the experiment site id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSiteIdCol] = str(int(expSiteId)) except: raise madrigal.admin.MadrigalError('Error in setExpSiteIdByPosition with args %s: %s' % \ (str(position, expSiteId), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpStartDateTimeByPosition(self, position = 0): """getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings startTime = [int(self.__fileList[position][self.__expStartDateCol][0:4]), int(self.__fileList[position][self.__expStartDateCol][4:6]), int(self.__fileList[position][self.__expStartDateCol][6:8]), int(self.__fileList[position][self.__expStartTimeCol][0:2]), int(self.__fileList[position][self.__expStartTimeCol][2:4]), int(self.__fileList[position][self.__expStartTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if startTime[3] == 24: startTime[3] = 23 startTime[4] = 59 startTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(startTime[0], startTime[1], startTime[2], startTime[3], startTime[4], startTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) startTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return startTime else: return(None) def getExpEndDateTimeByPosition(self, position = 0): """getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings endTime = [int(self.__fileList[position][self.__expEndDateCol][0:4]), int(self.__fileList[position][self.__expEndDateCol][4:6]), int(self.__fileList[position][self.__expEndDateCol][6:8]), int(self.__fileList[position][self.__expEndTimeCol][0:2]), int(self.__fileList[position][self.__expEndTimeCol][2:4]), int(self.__fileList[position][self.__expEndTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if endTime[3] == 24: endTime[3] = 23 endTime[4] = 59 endTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(endTime[0], endTime[1], endTime[2], endTime[3], endTime[4], endTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) endTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return endTime else: return(None) def getExpStartDateTimeByExpId(self, expId): """getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpStartDateTimeByPosition(position)) def getExpEndDateTimeByExpId(self, expId): """getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpEndDateTimeByPosition(position)) def setExpStartDateTimeByPosition(self, startDateTime, position = 0): """setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position. Inputs: startDateTime - a python datetime object to set the exp start date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp start date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expStartDateCol] = '%04i%02i%02i' % (startDateTime.year, startDateTime.month, startDateTime.day) self.__fileList[position][self.__expStartTimeCol] = '%02i%02i%02i' % (startDateTime.hour, startDateTime.minute, startDateTime.second) def setExpEndDateTimeByPosition(self, endDateTime, position = 0): """setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position. Inputs: endDateTime - a python datetime object to set the exp end date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp end date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expEndDateCol] = '%04i%02i%02i' % (endDateTime.year, endDateTime.month, endDateTime.day) self.__fileList[position][self.__expEndTimeCol] = '%02i%02i%02i' % (endDateTime.hour, endDateTime.minute, endDateTime.second) def getKinstByPosition(self, position = 0): """getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (instrument code) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expKinstCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getKinstByExpId(self, expId): """getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment kinst (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getKinstByPosition(position)) def setExpKinstByPosition(self, position, expKinst): """setExpKinstByPosition sets the experiment kinst of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expKinst - the new experiment kinst to use Returns: None. Affects: sets the experiment kinst of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expKinstCol] = str(int(expKinst)) except: raise madrigal.admin.MadrigalError('Error in setExpKinstByPosition with args %s: %s' % \ (str(position, expKinst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getSecurityByPosition(self, position = 0): """getSecurityByPosition returns the security code of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the security code (integer) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if security column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSecurityCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getSecurityByExpId(self, expId): """getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the security code (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getSecurityByPosition(position)) def setSecurityByPosition(self, position, securityCode): """setSecurityByPosition sets the security code (integer) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). securityCode - the new experiment security code (integer) to use Returns: None. Affects: sets the security code of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSecurityCol] = str(int(securityCode)) except: raise madrigal.admin.MadrigalError('Error in setSecurityByPosition with args %s: %s' % \ (str(position, securityCode), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getPIByPosition(self, position = 0): """getPIByPosition returns the principal investigator of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's name (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: pi = self.__fileList[position][self.__expPICol] if len(pi) > 0: return(pi) else: return(None) except: return(None) else: return(None) def getPIByExpId(self, expId): """getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's name (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIByPosition(position)) def setPIByPosition(self, position, PI): """setPIByPosition sets the principal investigator (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PI - the new experiment principal investigator's name (string) to use Returns: None. Affects: sets the principal investigator of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PI.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add PI and blank email self.__fileList[position].append(str(PI)) self.__fileList[position].append('') else: try: self.__fileList[position][self.__expPICol] = str(PI) except: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()])) def getPIEmailByPosition(self, position = 0): """getPIEmailByPosition returns the principal investigator email of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's email (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: piEmail = self.__fileList[position][self.__expPIEmailCol] if len(piEmail) > 0: return(piEmail) else: return(None) except: return(None) else: return(None) def getPIEmailByExpId(self, expId): """getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's email (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIEmailByPosition(position)) def setPIEmailByPosition(self, position, PIEmail): """setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PIEmail - the new experiment principal investigator's email (string) to use Returns: None. Affects: sets the principal investigator email of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PIEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add blank PI and email self.__fileList[position].append('') self.__fileList[position].append(str(PIEmail)) else: try: self.__fileList[position][self.__expPIEmailCol] = str(PIEmail) except: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()])) def getExpLinksByExpId(self, expId): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: Experiment Id (integer). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpLinksByPosition(position)) def getExpLinksByPosition(self, position = 0): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: position - position of experiment in list (first position is zero). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ retList = [] allowedExtensions = ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif'] # find the experiment directory based on url topdir = self.__madDB.getMadroot() url = self.getExpUrlByPosition(position) if url == None: return retList index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: expdir = os.path.join(topdir, 'experiments', url[index+8:]) else: expdir = os.path.join(topdir, url[index+8:]) # now create a list of html files htmlFiles = [] l1 = glob.glob(os.path.join(expdir, '*.html')) l2 = glob.glob(os.path.join(expdir, '*/index.html')) l3 = glob.glob(os.path.join(expdir, '*/*/index.html')) for item in l1: htmlFiles.append(item) for item in l2: htmlFiles.append(item) for item in l3: htmlFiles.append(item) for htmlFile in htmlFiles: # get title if possible f = open(htmlFile) text = f.read() f.close() i1 = text.upper().find('<TITLE>') + len('<TITLE>') i2 = text.upper().find('</TITLE>') if i1 > -1 and i2 > -1: title = text[i1:i2] else: title = 'No title' # get url i1 = htmlFile.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', htmlFile[i1:]) retList.append((title, url)) for extension in allowedExtensions: plotList = glob.glob(os.path.join(expdir, '*.%s' % (extension))) for plot in plotList: i1 = plot.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', plot[i1:]) retList.append((os.path.basename(plot), url)) return retList def getExpCount(self): """getExpCount returns number of experiments in MadrigalExperiment object """ return(len(self.__fileList)) def sortByDateSite(self): """sortByDateSite will resort self.__fileList so that experiments are listed first by experiment start date, and then by site """ self.__fileList.sort(self.__compareDateSite__) # rebuild self._dict self._dict = {} for i, items in enumerate(self.__fileList): self._dict[items[self.__expIdCol]] = i def getStartPosition(self, startDT): """getStartPosition returns the position of the first experiment with a start datetime after startDT after sorting by start time. Inputs: startDT - start datetime Returns: returns the position of the first experiment with a start datetime after startDT. """ self.sortByDateSite() top = len(self.__fileList) - 1 bottom = 0 while(top >= bottom): if top == bottom: return(max(top-1, 0)) middle = (top+bottom)/2 thisSDTList = self.getExpStartDateTimeByPosition(middle) thisSDT = datetime.datetime(*thisSDTList[0:6]) if thisSDT < startDT: bottom = middle + 1 continue if middle == 0: return(0) prevSDTList = self.getExpStartDateTimeByPosition(middle-1) prevSDT = datetime.datetime(*prevSDTList[0:6]) if prevSDT < startDT: return(middle-1) else: top = middle -1 def __compareDateSite__(self, first, second): """__compareDateSite__ is a private method to help sort by start date and then site. first, second - lists of experiment information as parsed from expTab.txt files """ result = cmp(first[self.__expStartDateCol], second[self.__expStartDateCol]) if result != 0: return(result) result = cmp(first[self.__expStartTimeCol], second[self.__expStartTimeCol]) if result != 0: return(result) result = cmp(first[self.__expEndDateCol], second[self.__expEndDateCol]) if result != 0: return(result) result = cmp(first[self.__expEndTimeCol], second[self.__expEndTimeCol]) if result != 0: return(result) return(cmp(first[self.__expSiteIdCol], second[self.__expSiteIdCol])) def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n') def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the expTab.txt file. Inputs: newFullPath: a new path to write the expTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def __str__(self): """return possibly modified file as a string in same format as writeMetadata """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' return(metaFileStr)
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None)
init initializes MadrigalExperiment by reading from expTab.txt (or initFile).
Inputs: Existing MadrigalDB object, by default = None.
String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/expTab.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.
def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalExperiment by reading from expTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/expTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get experiment metadata file if (initFile == None): self.__filename = self.__expMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without PI and email allowedLens = (self.__expSecurityCol+1, self.__expPIEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__expIdCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict()
def getExpCount(
self)
getExpCount returns number of experiments in MadrigalExperiment object
def getExpCount(self): """getExpCount returns number of experiments in MadrigalExperiment object """ return(len(self.__fileList))
def getExpDirByExpId(
self, expId)
getExpDirByExpId returns the full experiment directory for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the full experiment directory (string). Returns None if experiment id not found. Uses experiment url to determine directory.
Affects: None
Exceptions: None
def getExpDirByExpId(self, expId): """getExpDirByExpId returns the full experiment directory for a given experiment id. Inputs: Experiment Id (integer). Returns: the full experiment directory (string). Returns None if experiment id not found. Uses experiment url to determine directory. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpDirByPosition(position))
def getExpDirByPosition(
self, position=0)
getExpDirByPosition returns the full experiment directory of the experiment at given position.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the full experiment directory, or None if position >= number of experiments. Uses experiment url to determine directory.
Affects: None
Exceptions: None
def getExpDirByPosition(self, position = 0): """getExpDirByPosition returns the full experiment directory of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the full experiment directory, or None if position >= number of experiments. Uses experiment url to determine directory. Affects: None Exceptions: None """ if len(self.__fileList) > position: url = self.__fileList[position][self.__expUrlCol] # find the directory based on url maddir = self.__madDB.getMadroot() index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: maddir = os.path.join(maddir, 'experiments', url[index+8:]) else: maddir = os.path.join(maddir, url[index+8:]) return(maddir) else: return(None)
def getExpEndDateTimeByExpId(
self, expId)
getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.
Returns None if experiment id not found. Since mktime does not go before 1970, I use
date.c methods, and miss only day of week and DST flag
Affects: None
Exceptions: None
def getExpEndDateTimeByExpId(self, expId): """getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpEndDateTimeByPosition(position))
def getExpEndDateTimeByPosition(
self, position=0)
getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.
Returns None if position >= number of experiments. Since mktime does not go before 1970, I use
date.c methods, and miss only day of week and DST flag
Affects: None
Exceptions: None
def getExpEndDateTimeByPosition(self, position = 0): """getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings endTime = [int(self.__fileList[position][self.__expEndDateCol][0:4]), int(self.__fileList[position][self.__expEndDateCol][4:6]), int(self.__fileList[position][self.__expEndDateCol][6:8]), int(self.__fileList[position][self.__expEndTimeCol][0:2]), int(self.__fileList[position][self.__expEndTimeCol][2:4]), int(self.__fileList[position][self.__expEndTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if endTime[3] == 24: endTime[3] = 23 endTime[4] = 59 endTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(endTime[0], endTime[1], endTime[2], endTime[3], endTime[4], endTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) endTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return endTime else: return(None)
def getExpIdByPosition(
self, position=0)
getExpIdByPosition returns the experiment id of the experiment at given position.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the experiment id (integer), or None if position >= number of experiments.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getExpLinksByExpId(
self, expId)
getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.
Inputs:
Inputs: Experiment Id (integer).
Returns: a list of (title, url) tuples containing all links for this experiment.
In order to be a link, a file must be in the experiment directory in the form .html, /index.html, or //index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename.
Affects: None
Exceptions: None
def getExpLinksByExpId(self, expId): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: Experiment Id (integer). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpLinksByPosition(position))
def getExpLinksByPosition(
self, position=0)
getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.
Inputs:
Inputs: position - position of experiment in list (first position is zero).
Returns: a list of (title, url) tuples containing all links for this experiment.
In order to be a link, a file must be in the experiment directory in the form .html, /index.html, or //index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename.
Affects: None
Exceptions: None
def getExpLinksByPosition(self, position = 0): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: position - position of experiment in list (first position is zero). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ retList = [] allowedExtensions = ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif'] # find the experiment directory based on url topdir = self.__madDB.getMadroot() url = self.getExpUrlByPosition(position) if url == None: return retList index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: expdir = os.path.join(topdir, 'experiments', url[index+8:]) else: expdir = os.path.join(topdir, url[index+8:]) # now create a list of html files htmlFiles = [] l1 = glob.glob(os.path.join(expdir, '*.html')) l2 = glob.glob(os.path.join(expdir, '*/index.html')) l3 = glob.glob(os.path.join(expdir, '*/*/index.html')) for item in l1: htmlFiles.append(item) for item in l2: htmlFiles.append(item) for item in l3: htmlFiles.append(item) for htmlFile in htmlFiles: # get title if possible f = open(htmlFile) text = f.read() f.close() i1 = text.upper().find('<TITLE>') + len('<TITLE>') i2 = text.upper().find('</TITLE>') if i1 > -1 and i2 > -1: title = text[i1:i2] else: title = 'No title' # get url i1 = htmlFile.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', htmlFile[i1:]) retList.append((title, url)) for extension in allowedExtensions: plotList = glob.glob(os.path.join(expdir, '*.%s' % (extension))) for plot in plotList: i1 = plot.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', plot[i1:]) retList.append((os.path.basename(plot), url)) return retList
def getExpNameByExpId(
self, expId)
getExpNameByExpId returns the experiment name for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the experiment name (string). Returns None if experiment id not found.
Affects: None
Exceptions: None
def getExpNameByExpId(self, expId): """getExpNameByExpId returns the experiment name for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment name (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpNameByPosition(position))
def getExpNameByPosition(
self, position=0)
getExpNameByPosition returns the experiment name of the experiment at given position.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the experiment name, or None if position >= number of experiments.
Affects: None
Exceptions: None
def getExpNameByPosition(self, position = 0): """getExpNameByPosition returns the experiment name of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment name, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expNameCol] else: return(None)
def getExpPathByExpId(
self, expId)
getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id.
The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one.
Inputs: Experiment Id (integer).
Returns: the path to the experiment from the madroot directory (string). Returns None if experiment id not found.
Affects: None
Exceptions: None
def getExpPathByExpId(self, expId): """getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the path to the experiment from the madroot directory (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpPathByPosition(position))
def getExpPathByPosition(
self, position=0)
getExpPathByPosition returns the experiment path of the experiment at given position.
Experiment path is the path to the experiment from the madroot directory, and always begins "experiments"
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments.
Affects: None
Exceptions:
def getExpPathByPosition(self, position = 0): """getExpPathByPosition returns the experiment path of the experiment at given position. Experiment path is the path to the experiment from the madroot directory, and always begins "experiments" Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index != -1: return(thisUrl[index+8:]) index = thisUrl.find('experiments') if index == -1: return(None) return(thisUrl[index:])
def getExpSiteIdByExpId(
self, expId)
getExpSiteIdByExpId returns the site id (int) for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the site id for this experiment. Returns None if experiment id not found.
Affects: None
Exceptions: None
def getExpSiteIdByExpId(self, expId): """getExpSiteIdByExpId returns the site id (int) for a given experiment id. Inputs: Experiment Id (integer). Returns: the site id for this experiment. Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpSiteIdByPosition(position))
def getExpSiteIdByPosition(
self, position=0)
getExpSiteIdByPosition returns the experiment site id of the experiment at given position.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the experiment site id (integer), or None if position >= number of experiments.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getExpSiteIdByPosition(self, position = 0): """getExpSiteIdByPosition returns the experiment site id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment site id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSiteIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getExpStartDateTimeByExpId(
self, expId)
getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.
Returns None if experiment id not found. Since mktime does not go before 1970, I use
date.c methods, and miss only day of week and DST flag
Affects: None
Exceptions: None
def getExpStartDateTimeByExpId(self, expId): """getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpStartDateTimeByPosition(position))
def getExpStartDateTimeByPosition(
self, position=0)
getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.
Returns None if position >= number of experiments. Since mktime does not go before 1970, I use
date.c methods, and miss only day of week and DST flag
Affects: None
Exceptions: None
def getExpStartDateTimeByPosition(self, position = 0): """getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings startTime = [int(self.__fileList[position][self.__expStartDateCol][0:4]), int(self.__fileList[position][self.__expStartDateCol][4:6]), int(self.__fileList[position][self.__expStartDateCol][6:8]), int(self.__fileList[position][self.__expStartTimeCol][0:2]), int(self.__fileList[position][self.__expStartTimeCol][2:4]), int(self.__fileList[position][self.__expStartTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if startTime[3] == 24: startTime[3] = 23 startTime[4] = 59 startTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(startTime[0], startTime[1], startTime[2], startTime[3], startTime[4], startTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) startTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return startTime else: return(None)
def getExpUrlByExpId(
self, expId)
getExpUrlByExpId returns the experiment url for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the experiment url (string). Returns None if experiment id not found.
Affects: None
Exceptions: None
def getExpUrlByExpId(self, expId): """getExpUrlByExpId returns the experiment url for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpUrlByPosition(position))
def getExpUrlByPosition(
self, position=0)
getExpUrlByPosition returns the experiment url of the experiment at given position.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the experiment url, or None if position >= number of experiments.
Affects: None
Exceptions: None
def getExpUrlByPosition(self, position = 0): """getExpUrlByPosition returns the experiment url of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment url, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expUrlCol] else: return(None)
def getKinstByExpId(
self, expId)
getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the experiment kinst (integer). Returns None if experiment id not found.
Affects: None
Exceptions: None
def getKinstByExpId(self, expId): """getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment kinst (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getKinstByPosition(position))
def getKinstByPosition(
self, position=0)
getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the kinst (instrument code) of the file at given position as an integer.
Returns None if position >= number of files.
Affects: None
Exceptions: Thrown if kinst column cannot be parsed into an integer
def getKinstByPosition(self, position = 0): """getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (instrument code) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expKinstCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getLine(
self, position)
getLine returns the line at a given position. Returns None if position > number of lines.
Inputs: position - position in file. First line = 0
def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n')
def getPIByExpId(
self, expId)
getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the principal investigator's name (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist.
Affects: None
Exceptions: None
This method added in Madrigal 2.6
def getPIByExpId(self, expId): """getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's name (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIByPosition(position))
def getPIByPosition(
self, position=0)
getPIByPosition returns the principal investigator of the experiment at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the principal investigator's name (string) of the file at given position as a string.
Returns None if position >= number of files. Since not all expTab.txt files may have this
column, returns None if the column does not exist.
Affects: None
Exceptions: None
This method added in Madrigal 2.6
def getPIByPosition(self, position = 0): """getPIByPosition returns the principal investigator of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's name (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: pi = self.__fileList[position][self.__expPICol] if len(pi) > 0: return(pi) else: return(None) except: return(None) else: return(None)
def getPIEmailByExpId(
self, expId)
getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the principal investigator's email (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist.
Affects: None
Exceptions: None
This method added in Madrigal 2.6
def getPIEmailByExpId(self, expId): """getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's email (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIEmailByPosition(position))
def getPIEmailByPosition(
self, position=0)
getPIEmailByPosition returns the principal investigator email of the experiment at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the principal investigator's email (string) of the file at given position as a string.
Returns None if position >= number of files. Since not all expTab.txt files may have this
column, returns None if the column does not exist.
Affects: None
Exceptions: None
This method added in Madrigal 2.6
def getPIEmailByPosition(self, position = 0): """getPIEmailByPosition returns the principal investigator email of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's email (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: piEmail = self.__fileList[position][self.__expPIEmailCol] if len(piEmail) > 0: return(piEmail) else: return(None) except: return(None) else: return(None)
def getRealExpUrlByExpId(
self, expId)
getRealExpUrlByExpId returns the real experiment url for a given experiment id.
The url in the metadata contains /madtoc/ for historical reasons. This method converts that url to the real one.
Inputs: Experiment Id (integer).
Returns: the real experiment url (string). Returns None if experiment id not found.
Affects: None
Exceptions: None
def getRealExpUrlByExpId(self, expId): """getRealExpUrlByExpId returns the real experiment url for a given experiment id. The url in the metadata contains /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the real experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getRealExpUrlByPosition(position))
def getRealExpUrlByPosition(
self, position=0)
getRealExpUrlByPosition returns the real experiment url of the experiment at given position.
The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one.
Inputs: position of experiment in list (first position is zero). Defaults to first.
Returns: the real experiment url, or None if position >= number of experiments.
Affects: None
Exceptions:
def getRealExpUrlByPosition(self, position = 0): """getRealExpUrlByPosition returns the real experiment url of the experiment at given position. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the real experiment url, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index == -1: return(thisUrl) if thisUrl.find('experiments') != -1: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + thisUrl[index+8:]) else: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + 'experiments', thisUrl[index+8:]) return(realUrl)
def getSecurityByExpId(
self, expId)
getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id.
Inputs: Experiment Id (integer).
Returns: the security code (integer). Returns None if experiment id not found.
Affects: None
Exceptions: None
def getSecurityByExpId(self, expId): """getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the security code (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getSecurityByPosition(position))
def getSecurityByPosition(
self, position=0)
getSecurityByPosition returns the security code of the experiment at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the security code (integer) of the file at given position as an integer.
Returns None if position >= number of files.
Affects: None
Exceptions: Thrown if security column cannot be parsed into an integer
def getSecurityByPosition(self, position = 0): """getSecurityByPosition returns the security code of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the security code (integer) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if security column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSecurityCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getStartPosition(
self, startDT)
getStartPosition returns the position of the first experiment with a start datetime after startDT after sorting by start time.
Inputs: startDT - start datetime Returns: returns the position of the first experiment with a start datetime after
startDT.
def getStartPosition(self, startDT): """getStartPosition returns the position of the first experiment with a start datetime after startDT after sorting by start time. Inputs: startDT - start datetime Returns: returns the position of the first experiment with a start datetime after startDT. """ self.sortByDateSite() top = len(self.__fileList) - 1 bottom = 0 while(top >= bottom): if top == bottom: return(max(top-1, 0)) middle = (top+bottom)/2 thisSDTList = self.getExpStartDateTimeByPosition(middle) thisSDT = datetime.datetime(*thisSDTList[0:6]) if thisSDT < startDT: bottom = middle + 1 continue if middle == 0: return(0) prevSDTList = self.getExpStartDateTimeByPosition(middle-1) prevSDT = datetime.datetime(*prevSDTList[0:6]) if prevSDT < startDT: return(middle-1) else: top = middle -1
def setExpEndDateTimeByPosition(
self, endDateTime, position=0)
setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position.
Inputs:
endDateTime - a python datetime object to set the exp end date and time to. position - which experiment row to change - defaults to 0
Returns: None.
Affects: sets exp end date and time in self.__fileList.
Exceptions: None
def setExpEndDateTimeByPosition(self, endDateTime, position = 0): """setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position. Inputs: endDateTime - a python datetime object to set the exp end date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp end date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expEndDateCol] = '%04i%02i%02i' % (endDateTime.year, endDateTime.month, endDateTime.day) self.__fileList[position][self.__expEndTimeCol] = '%02i%02i%02i' % (endDateTime.hour, endDateTime.minute, endDateTime.second)
def setExpIdByPosition(
self, position, expId)
setExpIdByPosition sets the experiment id of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). expId - the new experiment id to use
Returns: None.
Affects: sets the experiment id of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expIdCol] = str(int(expId)) self._dict[position] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str((position, expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
def setExpKinstByPosition(
self, position, expKinst)
setExpKinstByPosition sets the experiment kinst of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). expKinst - the new experiment kinst to use
Returns: None.
Affects: sets the experiment kinst of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
def setExpKinstByPosition(self, position, expKinst): """setExpKinstByPosition sets the experiment kinst of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expKinst - the new experiment kinst to use Returns: None. Affects: sets the experiment kinst of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expKinstCol] = str(int(expKinst)) except: raise madrigal.admin.MadrigalError('Error in setExpKinstByPosition with args %s: %s' % \ (str(position, expKinst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
def setExpNameByPosition(
self, position, expName)
setExpNameByPosition sets the experiment name of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). expName - the new experiment name to use
Returns: None.
Affects: sets the experiment name of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
def setExpNameByPosition(self, position, expName): """setExpNameByPosition sets the experiment name of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expName - the new experiment name to use Returns: None. Affects: sets the experiment name of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expName.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), [traceback.format_exc()])) try: self.__fileList[position][self.__expNameCol] = str(expName) except: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
def setExpSiteIdByPosition(
self, position, expSiteId)
setExpSiteIdByPosition sets the experiment site id of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). expSiteId - the new experiment site id to use
Returns: None.
Affects: sets the experiment site id of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
def setExpSiteIdByPosition(self, position, expSiteId): """setExpSiteIdByPosition sets the experiment site id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expSiteId - the new experiment site id to use Returns: None. Affects: sets the experiment site id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSiteIdCol] = str(int(expSiteId)) except: raise madrigal.admin.MadrigalError('Error in setExpSiteIdByPosition with args %s: %s' % \ (str(position, expSiteId), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
def setExpStartDateTimeByPosition(
self, startDateTime, position=0)
setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position.
Inputs:
startDateTime - a python datetime object to set the exp start date and time to. position - which experiment row to change - defaults to 0
Returns: None.
Affects: sets exp start date and time in self.__fileList.
Exceptions: None
def setExpStartDateTimeByPosition(self, startDateTime, position = 0): """setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position. Inputs: startDateTime - a python datetime object to set the exp start date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp start date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expStartDateCol] = '%04i%02i%02i' % (startDateTime.year, startDateTime.month, startDateTime.day) self.__fileList[position][self.__expStartTimeCol] = '%02i%02i%02i' % (startDateTime.hour, startDateTime.minute, startDateTime.second)
def setExpUrlByPosition(
self, position, expUrl)
setExpUrlByPosition sets the experiment url of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). expUrl - the new experiment url to use
Returns: None.
Affects: sets the experiment url of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
def setExpUrlByPosition(self, position, expUrl): """setExpUrlByPosition sets the experiment url of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expUrl - the new experiment url to use Returns: None. Affects: sets the experiment url of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expUrl.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_exc()])) try: self.__fileList[position][self.__expUrlCol] = str(expUrl) except: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_ex()]))
def setPIByPosition(
self, position, PI)
setPIByPosition sets the principal investigator (string) of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). PI - the new experiment principal investigator's name (string) to use
Returns: None.
Affects: sets the principal investigator of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
This method added in Madrigal 2.6
def setPIByPosition(self, position, PI): """setPIByPosition sets the principal investigator (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PI - the new experiment principal investigator's name (string) to use Returns: None. Affects: sets the principal investigator of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PI.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add PI and blank email self.__fileList[position].append(str(PI)) self.__fileList[position].append('') else: try: self.__fileList[position][self.__expPICol] = str(PI) except: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()]))
def setPIEmailByPosition(
self, position, PIEmail)
setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). PIEmail - the new experiment principal investigator's email (string) to use
Returns: None.
Affects: sets the principal investigator email of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
This method added in Madrigal 2.6
def setPIEmailByPosition(self, position, PIEmail): """setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PIEmail - the new experiment principal investigator's email (string) to use Returns: None. Affects: sets the principal investigator email of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PIEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add blank PI and email self.__fileList[position].append('') self.__fileList[position].append(str(PIEmail)) else: try: self.__fileList[position][self.__expPIEmailCol] = str(PIEmail) except: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()]))
def setSecurityByPosition(
self, position, securityCode)
setSecurityByPosition sets the security code (integer) of the experiment at given position.
Inputs:
position - position of experiment in list (first position is zero). securityCode - the new experiment security code (integer) to use
Returns: None.
Affects: sets the security code of the experiment at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
def setSecurityByPosition(self, position, securityCode): """setSecurityByPosition sets the security code (integer) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). securityCode - the new experiment security code (integer) to use Returns: None. Affects: sets the security code of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSecurityCol] = str(int(securityCode)) except: raise madrigal.admin.MadrigalError('Error in setSecurityByPosition with args %s: %s' % \ (str(position, securityCode), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])))
def sortByDateSite(
self)
sortByDateSite will resort self.__fileList so that experiments are listed first by experiment start date, and then by site
def sortByDateSite(self): """sortByDateSite will resort self.__fileList so that experiments are listed first by experiment start date, and then by site """ self.__fileList.sort(self.__compareDateSite__) # rebuild self._dict self._dict = {} for i, items in enumerate(self.__fileList): self._dict[items[self.__expIdCol]] = i
def writeMetadata(
self, newFullPath=None)
writeMetadata writes a new version of the expTab.txt file.
Inputs: newFullPath: a new path to write the expTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from.
Returns: None.
Affects: Writes updated version of metadata file.
Exceptions: If unable to write file
def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the expTab.txt file. Inputs: newFullPath: a new path to write the expTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
class MadrigalInstrument
MadrigalInstrument is an object that provides access to Madrigal instrument info from the metadata.
This object provides access to all Madrigal instrument information in the metadata files instTab.txt and instType.Tab.
key = self.__instKinstCol (used in searches)
Usage example::
import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrument() print test.getInstrumentName(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001
class MadrigalInstrument: """MadrigalInstrument is an object that provides access to Madrigal instrument info from the metadata. This object provides access to all Madrigal instrument information in the metadata files instTab.txt and instType.Tab. key = self.__instKinstCol (used in searches) Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrument() print test.getInstrumentName(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __instMetadataFile = "instTab.txt" # column positions __instKinstCol = 0 __instMnemonicCol = 1 __instNameCol = 2 __latitudeCol = 3 __longitudeCol = 4 __altitudeCol = 5 __contactNameCol = 6 __contactAddr1Col = 7 __contactAddr2Col = 8 __contactAddr3Col = 9 __contactCityCol = 10 __contactStateCol = 11 __contactZipCol = 12 __contactCountryCol = 13 __contactPhoneCol = 14 __contactEmailCol = 15 __categoryCol = 16 # instType.txt file __inst2MetadataFile = "instType.txt" # column positions __inst2CategoryIdCol = 0 __inst2CategoryDescCol = 1 def __init__(self, madDB=None, initFile=None, init2File=None): """__init__ initializes MadrigalInstrument by reading from instTab.txt (or initFile) and instType.txt file (or init2File). Inputs: madDB - Existing MadrigalDB object, by default = None. initFile - String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instTab.txt. init2File - String representing the full path to the metadata file instType.txt. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instTab.txt file was updated with the release of the madrigal python api, and this function will throw an error if used with the old file. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument metadata file if (initFile == None): self.__filename = self.__instMetadataFile else: self.__filename = initFile genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, key=self.__instKinstCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # get instrument metadata file 2 if (init2File == None): self.__filename2 = self.__inst2MetadataFile else: self.__filename2 = initFile self.__fileList2 = madrigal.metadata.MadrigalMetadata(self.__filename2, self.__madDB).getList() def getInstrumentName(self, kinst): """getInstrumentName returns the instrument name that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument name. Returns: the instrument name that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instNameCol]) def getInstrumentMnemonic(self, kinst): """getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument mnemonic. Returns: the instrument mnemonic that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instMnemonicCol]) def getLatitude(self, kinst): """getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get latitude. Returns: the latitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__latitudeCol])) except ValueError: return(None) def getLongitude(self, kinst): """getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get longitude. Returns: the longitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__longitudeCol])) except ValueError: return(None) def getAltitude(self, kinst): """getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get altitude. Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__altitudeCol])) except ValueError: return(None) def getContactName(self, kinst): """getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact name as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactNameCol]) def getContactAddress1(self, kinst): """getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 3 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactAddr1Col]) def getContactEmail(self, kinst): """getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact email. Returns: the contact email as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactEmailCol]) def getCategory(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ categoryId = None try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None if categoryId == None: return(None) # now loop until categoryId found for inst2 in self.__fileList2: try: if (int(inst2[self.__inst2CategoryIdCol]) == categoryId): return(inst2[self.__inst2CategoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in instType.txt parsing metadata row: ' + str(inst2), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # type not found return(None) def getCategoryId(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None return(categoryId) def getInstrumentList(self): """getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values. Inputs: None. Returns: a list of all instrument names, mnemonics, and their kinst values. Each item in the list is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)). Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], int(inst[self.__instKinstCol])) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList def getOrderedInstrumentList(self): """getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId), ordered by categoryId and then kinst. Inputs: None. Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and then kinst. Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: kinst = int(inst[self.__instKinstCol]) item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], kinst, self.getCategory(kinst), self.getCategoryId(kinst)) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) retList.sort(self.__instrumentSort) return retList def getOrderedInstrumentListWithData(self, isTrusted, localOnly=False, localExpObj=None, globalExpObj=None, allowArchive=False, requireFiles=False, requireFilesOrPlots=False): """getInstrumentList returns information about which instruments have local and global data. Inputs: isTrusted - True if client is trusted, False otherwise localOnly - if False (the default), will return information about both local and global data. If True, then global data ignored. localExpObj - an MadrigalExperiment object for the local data. If None (the default), will be created. globalExpObj - an MadrigalExperiment object for the global data. If None (the default), will be created if not localOnly. allowArchive - if True, allow experiments marked as security==2(public arcive), and if isTrusted, allow security==3(private archive) requireFiles - if True, only include experiments with Madrigal data files. If False (default), not not require that. requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots. If False (default), not not require that. Returns: an ordered tuple of two items: 1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear, globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst. If localOnly, globalStartYear and globalEndYear = 0. If not local only, localStartYear and localEndYear will be zero if no local data for that instrument. 2. categoryDict - key = categoryId, value = category Description Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if (not localOnly) and (globalExpObj==None): globalExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt')) if localExpObj == None: localExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) orderInstList = [] categoryDict = {} # next task - create local dict and global dict of key=kinst, value = (startYear, endYear) tuple localInstDict = {} globalInstDict = {} # Skip test experiments testExp = ('1998/mlh/20jan98', '1997/aro/06jan97', '1997/lyr/08apr97', '1997/son/06jan97', '1995/jro/01feb95', '1998/jro/27apr98') for i in range(localExpObj.getExpCount()): thisUrl = localExpObj.getExpUrlByPosition(i) if thisUrl[-16:] in testExp: continue thisSecurity = localExpObj.getSecurityByPosition(i) if thisSecurity == -1: # blocked to all continue if thisSecurity == 1 and not isTrusted: # no access continue if thisSecurity == 2 and not allowArchive: # no access continue if thisSecurity == 3 and ((not isTrusted) or (not allowArchive)): # no access continue kinst = localExpObj.getKinstByPosition(i) startYear = localExpObj.getExpStartDateTimeByPosition(i)[0] endYear = localExpObj.getExpEndDateTimeByPosition(i)[0] if localInstDict.has_key(kinst): oldStartYear, oldEndYear = localInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue expDir = localExpObj.getExpDirByPosition(i) if requireFilesOrPlots: links = localExpObj.getExpLinksByPosition(i) if len(links) == 0: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if requireFiles: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if localInstDict.has_key(kinst): localInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) else: localInstDict[kinst] = (startYear, endYear) if not localOnly: for i in range(globalExpObj.getExpCount()): kinst = globalExpObj.getKinstByPosition(i) startYear = globalExpObj.getExpStartDateTimeByPosition(i)[0] endYear = globalExpObj.getExpEndDateTimeByPosition(i)[0] thisUrl = globalExpObj.getExpUrlByPosition(i) thisSecurity = globalExpObj.getSecurityByPosition(i) thisSite = globalExpObj.getExpSiteIdByPosition(i) if thisUrl[-16:] in testExp: continue if not globalInstDict.has_key(kinst): globalInstDict[kinst] = (startYear, endYear) continue oldStartYear, oldEndYear = globalInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue if thisSecurity != 0 and self.__madDB.getSiteID() != thisSite: # no search remote sites for non-public experiments allowed continue if requireFiles: if globalExpObj.getExpIdByPosition(i) not in expIdList: continue globalInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) # populate orderInstList and categoryDict instList = self.getOrderedInstrumentList() for inst in instList: if not localOnly: if globalInstDict.has_key(inst[2]): if not categoryDict.has_key(inst[4]): categoryDict[inst[4]] = inst[3] globalStartYear, globalEndYear = globalInstDict[inst[2]] try: localStartYear, localEndYear = localInstDict[inst[2]] except KeyError: localStartYear, localEndYear = (0, 0) orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) else: if localInstDict.has_key(inst[2]): if not categoryDict.has_key(inst[4]): categoryDict[inst[4]] = inst[3] globalStartYear = 0 globalEndYear = 0 localStartYear, localEndYear = localInstDict[inst[2]] orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) return ((orderInstList, categoryDict)) def __instrumentSort(self, thisInst, otherInst): """instrumentSort is a private method used to sort tuples of instrument data """ result = cmp(thisInst[4], otherInst[4]) if result != 0: return(result) return(cmp(thisInst[2], otherInst[2]))
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None, init2File=None)
init initializes MadrigalInstrument by reading from instTab.txt (or initFile) and instType.txt file (or init2File).
Inputs:
madDB - Existing MadrigalDB object, by default = None. initFile - String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instTab.txt. init2File - String representing the full path to the metadata file instType.txt. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instTab.txt file was updated with the release of the madrigal python api, and this function will throw an error if used with the old file.
def __init__(self, madDB=None, initFile=None, init2File=None): """__init__ initializes MadrigalInstrument by reading from instTab.txt (or initFile) and instType.txt file (or init2File). Inputs: madDB - Existing MadrigalDB object, by default = None. initFile - String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instTab.txt. init2File - String representing the full path to the metadata file instType.txt. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instTab.txt file was updated with the release of the madrigal python api, and this function will throw an error if used with the old file. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument metadata file if (initFile == None): self.__filename = self.__instMetadataFile else: self.__filename = initFile genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, key=self.__instKinstCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # get instrument metadata file 2 if (init2File == None): self.__filename2 = self.__inst2MetadataFile else: self.__filename2 = initFile self.__fileList2 = madrigal.metadata.MadrigalMetadata(self.__filename2, self.__madDB).getList()
def getAltitude(
self, kinst)
getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank.
Inputs: kinst integer to get altitude.
Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getAltitude(self, kinst): """getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get altitude. Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__altitudeCol])) except ValueError: return(None)
def getCategory(
self, kinst)
getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found.
Inputs: kinst integer to get altitude.
Returns: the instrument category that matches kinst argument as a string, or None if kinst not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getCategory(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ categoryId = None try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None if categoryId == None: return(None) # now loop until categoryId found for inst2 in self.__fileList2: try: if (int(inst2[self.__inst2CategoryIdCol]) == categoryId): return(inst2[self.__inst2CategoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in instType.txt parsing metadata row: ' + str(inst2), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # type not found return(None)
def getCategoryId(
self, kinst)
getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found.
Inputs: kinst integer to get altitude.
Returns: the instrument category that matches kinst argument as a string, or None if kinst not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getCategoryId(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None return(categoryId)
def getContactAddress1(
self, kinst)
getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank.
Inputs: kinst integer to get contact name.
Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
This method added in Madrigal 3
def getContactAddress1(self, kinst): """getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 3 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactAddr1Col])
def getContactEmail(
self, kinst)
getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank.
Inputs: kinst integer to get contact email.
Returns: the contact email as a string that matches kinst argument, or None if not found or blank.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
This method added in Madrigal 2.6
def getContactEmail(self, kinst): """getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact email. Returns: the contact email as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactEmailCol])
def getContactName(
self, kinst)
getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank.
Inputs: kinst integer to get contact name.
Returns: the contact name as a string that matches kinst argument, or None if not found or blank.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
This method added in Madrigal 2.6
def getContactName(self, kinst): """getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact name as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactNameCol])
def getInstrumentList(
self)
getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values.
Inputs: None.
Returns: a list of all instrument names, mnemonics, and their kinst values. Each item in the list is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)). Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31)
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getInstrumentList(self): """getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values. Inputs: None. Returns: a list of all instrument names, mnemonics, and their kinst values. Each item in the list is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)). Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], int(inst[self.__instKinstCol])) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList
def getInstrumentMnemonic(
self, kinst)
getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found.
Inputs: kinst integer to get instrument mnemonic.
Returns: the instrument mnemonic that matches kinst argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getInstrumentMnemonic(self, kinst): """getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument mnemonic. Returns: the instrument mnemonic that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instMnemonicCol])
def getInstrumentName(
self, kinst)
getInstrumentName returns the instrument name that matches kinst argument, or None if not found.
Inputs: kinst integer to get instrument name.
Returns: the instrument name that matches kinst argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getInstrumentName(self, kinst): """getInstrumentName returns the instrument name that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument name. Returns: the instrument name that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instNameCol])
def getLatitude(
self, kinst)
getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank.
Inputs: kinst integer to get latitude.
Returns: the latitude as a float that matches kinst argument, or None if not found or blank.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getLatitude(self, kinst): """getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get latitude. Returns: the latitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__latitudeCol])) except ValueError: return(None)
def getLongitude(
self, kinst)
getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank.
Inputs: kinst integer to get longitude.
Returns: the longitude as a float that matches kinst argument, or None if not found or blank.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getLongitude(self, kinst): """getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get longitude. Returns: the longitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__longitudeCol])) except ValueError: return(None)
def getOrderedInstrumentList(
self)
getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId), ordered by categoryId and then kinst.
Inputs: None.
Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and then kinst. Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1)
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getOrderedInstrumentList(self): """getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId), ordered by categoryId and then kinst. Inputs: None. Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and then kinst. Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: kinst = int(inst[self.__instKinstCol]) item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], kinst, self.getCategory(kinst), self.getCategoryId(kinst)) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) retList.sort(self.__instrumentSort) return retList
def getOrderedInstrumentListWithData(
self, isTrusted, localOnly=False, localExpObj=None, globalExpObj=None, allowArchive=False, requireFiles=False, requireFilesOrPlots=False)
getInstrumentList returns information about which instruments have local and global data.
Inputs:
isTrusted - True if client is trusted, False otherwise localOnly - if False (the default), will return information about both local and global data. If True, then global data ignored. localExpObj - an MadrigalExperiment object for the local data. If None (the default), will be created. globalExpObj - an MadrigalExperiment object for the global data. If None (the default), will be created if not localOnly. allowArchive - if True, allow experiments marked as security==2(public arcive), and if isTrusted, allow security==3(private archive) requireFiles - if True, only include experiments with Madrigal data files. If False (default), not not require that. requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots. If False (default), not not require that.
Returns: an ordered tuple of two items: 1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear, globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst. If localOnly, globalStartYear and globalEndYear = 0. If not local only, localStartYear and localEndYear will be zero if no local data for that instrument. 2. categoryDict - key = categoryId, value = category Description
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getOrderedInstrumentListWithData(self, isTrusted, localOnly=False, localExpObj=None, globalExpObj=None, allowArchive=False, requireFiles=False, requireFilesOrPlots=False): """getInstrumentList returns information about which instruments have local and global data. Inputs: isTrusted - True if client is trusted, False otherwise localOnly - if False (the default), will return information about both local and global data. If True, then global data ignored. localExpObj - an MadrigalExperiment object for the local data. If None (the default), will be created. globalExpObj - an MadrigalExperiment object for the global data. If None (the default), will be created if not localOnly. allowArchive - if True, allow experiments marked as security==2(public arcive), and if isTrusted, allow security==3(private archive) requireFiles - if True, only include experiments with Madrigal data files. If False (default), not not require that. requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots. If False (default), not not require that. Returns: an ordered tuple of two items: 1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear, globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst. If localOnly, globalStartYear and globalEndYear = 0. If not local only, localStartYear and localEndYear will be zero if no local data for that instrument. 2. categoryDict - key = categoryId, value = category Description Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if (not localOnly) and (globalExpObj==None): globalExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt')) if localExpObj == None: localExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) orderInstList = [] categoryDict = {} # next task - create local dict and global dict of key=kinst, value = (startYear, endYear) tuple localInstDict = {} globalInstDict = {} # Skip test experiments testExp = ('1998/mlh/20jan98', '1997/aro/06jan97', '1997/lyr/08apr97', '1997/son/06jan97', '1995/jro/01feb95', '1998/jro/27apr98') for i in range(localExpObj.getExpCount()): thisUrl = localExpObj.getExpUrlByPosition(i) if thisUrl[-16:] in testExp: continue thisSecurity = localExpObj.getSecurityByPosition(i) if thisSecurity == -1: # blocked to all continue if thisSecurity == 1 and not isTrusted: # no access continue if thisSecurity == 2 and not allowArchive: # no access continue if thisSecurity == 3 and ((not isTrusted) or (not allowArchive)): # no access continue kinst = localExpObj.getKinstByPosition(i) startYear = localExpObj.getExpStartDateTimeByPosition(i)[0] endYear = localExpObj.getExpEndDateTimeByPosition(i)[0] if localInstDict.has_key(kinst): oldStartYear, oldEndYear = localInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue expDir = localExpObj.getExpDirByPosition(i) if requireFilesOrPlots: links = localExpObj.getExpLinksByPosition(i) if len(links) == 0: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if requireFiles: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if localInstDict.has_key(kinst): localInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) else: localInstDict[kinst] = (startYear, endYear) if not localOnly: for i in range(globalExpObj.getExpCount()): kinst = globalExpObj.getKinstByPosition(i) startYear = globalExpObj.getExpStartDateTimeByPosition(i)[0] endYear = globalExpObj.getExpEndDateTimeByPosition(i)[0] thisUrl = globalExpObj.getExpUrlByPosition(i) thisSecurity = globalExpObj.getSecurityByPosition(i) thisSite = globalExpObj.getExpSiteIdByPosition(i) if thisUrl[-16:] in testExp: continue if not globalInstDict.has_key(kinst): globalInstDict[kinst] = (startYear, endYear) continue oldStartYear, oldEndYear = globalInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue if thisSecurity != 0 and self.__madDB.getSiteID() != thisSite: # no search remote sites for non-public experiments allowed continue if requireFiles: if globalExpObj.getExpIdByPosition(i) not in expIdList: continue globalInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) # populate orderInstList and categoryDict instList = self.getOrderedInstrumentList() for inst in instList: if not localOnly: if globalInstDict.has_key(inst[2]): if not categoryDict.has_key(inst[4]): categoryDict[inst[4]] = inst[3] globalStartYear, globalEndYear = globalInstDict[inst[2]] try: localStartYear, localEndYear = localInstDict[inst[2]] except KeyError: localStartYear, localEndYear = (0, 0) orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) else: if localInstDict.has_key(inst[2]): if not categoryDict.has_key(inst[4]): categoryDict[inst[4]] = inst[3] globalStartYear = 0 globalEndYear = 0 localStartYear, localEndYear = localInstDict[inst[2]] orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) return ((orderInstList, categoryDict))
class MadrigalInstrumentData
MadrigalInstrumentData is an object that provides access to Madrigal instrument data info from the metadata.
This object provides access to all Madrigal kind of data information in the metadata files instData.txt and instDataPriv.txt. Those files summarize years data is available by instrument.
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Feb. 2, 2015
class MadrigalInstrumentData: """MadrigalInstrumentData is an object that provides access to Madrigal instrument data info from the metadata. This object provides access to all Madrigal kind of data information in the metadata files instData.txt and instDataPriv.txt. Those files summarize years data is available by instrument. Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Feb. 2, 2015 """ #constants _instDataMetadataFile = "instData.txt" # column positions _siteIDCol = 0 _kinstCol = 1 _yearsCol = 2 def __init__(self, madDB=None, priv=False, initFile=None, madInstObj=None): """__init__ initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt). Inputs: madDB - Existing MadrigalDB object, by default = None. priv - if True, use instDataPriv.txt instead of instData.txt. If False (the default), use instData.txt. initFile - String representing the full path to the metadata file. Default is None, in which case file read depends on priv argument. priv arg ingored if this not None madInstObj - m madrigal.metadata.MadrigalInstrument object. If None, one is created. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB if priv: self._instDataMetadataFile = "instDataPriv.txt" # get instData metadata file if (initFile == None): self.__filename = self._instDataMetadataFile else: self.__filename = initFile if madInstObj is not None: self._madInstObj = madInstObj else: self._madInstObj = madrigal.metadata.MadrigalInstrument(self.__madDB) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getCategories(self, local=False): """getCategories returns the a list of (category id, category desc) tuples. Inputs: local - if False, return all categories for which there is data anywhere in Madrigal. If True, only return local categories. Returns: a list of (category id, category desc) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is category id, value is category desc str. Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: if local: if int(data[self._siteIDCol]) != siteID: continue kinst = int(data[self._kinstCol]) categoryID = self._madInstObj.getCategoryId(kinst) if categoryID not in localDict.keys(): localDict[categoryID] = self._madInstObj.getCategory(kinst) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) categoryIDKeys = localDict.keys() categoryIDKeys.sort() retList = [(catID, localDict[catID]) for catID in categoryIDKeys] return(retList) def getInstruments(self, categoryID=0, local=False): """getInstruments returns the a list of (kinst, instrument desc, siteID) tuples. Inputs: categoryID - category id to return instruments with data for. If 0, return all local - if False, return all instruments with that category for which there is data anywhere in Madrigal. If True, only return local instruments, in which case siteID is always the local siteID. Returns: a list of (kinst, instrument desc, siteID) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is kinst, value is tuple if (instrument desc str, siteID). # Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: kinst = int(data[self._kinstCol]) thisSiteID = int(data[self._siteIDCol]) thisCategoryID = self._madInstObj.getCategoryId(kinst) if thisCategoryID != categoryID and categoryID != 0: continue if local: if thisSiteID != siteID: continue if kinst not in localDict.keys(): localDict[kinst] = (self._madInstObj.getInstrumentName(kinst), thisSiteID) else: raise ValueError, 'Duplicate kinst %i found in instData' % (kinst) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) kinstKeys = localDict.keys() kinstKeys.sort() retList = [(kinst, localDict[kinst][0], localDict[kinst][1]) for kinst in kinstKeys] return(retList) def getInstrumentYears(self, kinst): """getInstrumentYears returns the a list of years (int) for instrument. Inputs: kinst - the instrument id Returns: an ordered list of years as integers. If none found, raises error Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, data in enumerate(self.__fileList): # find matching code try: thisKinst = int(data[self._kinstCol]) if thisKinst != kinst: continue yearsStr = data[self._yearsCol] retList = [int(year) for year in yearsStr.split()] retList.sort() except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(retList) == 0: raise madrigal.admin.MadrigalError('No data found for kinst %i' % (kinst), '') return(retList)
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, priv=False, initFile=None, madInstObj=None)
init initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt).
Inputs: madDB - Existing MadrigalDB object, by default = None.
priv - if True, use instDataPriv.txt instead of instData.txt. If False (the default), use instData.txt. initFile - String representing the full path to the metadata file. Default is None, in which case file read depends on priv argument. priv arg ingored if this not None madInstObj - m madrigal.metadata.MadrigalInstrument object. If None, one is created.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.
def __init__(self, madDB=None, priv=False, initFile=None, madInstObj=None): """__init__ initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt). Inputs: madDB - Existing MadrigalDB object, by default = None. priv - if True, use instDataPriv.txt instead of instData.txt. If False (the default), use instData.txt. initFile - String representing the full path to the metadata file. Default is None, in which case file read depends on priv argument. priv arg ingored if this not None madInstObj - m madrigal.metadata.MadrigalInstrument object. If None, one is created. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB if priv: self._instDataMetadataFile = "instDataPriv.txt" # get instData metadata file if (initFile == None): self.__filename = self._instDataMetadataFile else: self.__filename = initFile if madInstObj is not None: self._madInstObj = madInstObj else: self._madInstObj = madrigal.metadata.MadrigalInstrument(self.__madDB) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()
def getCategories(
self, local=False)
getCategories returns the a list of (category id, category desc) tuples.
Inputs:
local - if False, return all categories for which there is data anywhere in Madrigal. If True, only return local categories.
Returns: a list of (category id, category desc) tuples
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getCategories(self, local=False): """getCategories returns the a list of (category id, category desc) tuples. Inputs: local - if False, return all categories for which there is data anywhere in Madrigal. If True, only return local categories. Returns: a list of (category id, category desc) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is category id, value is category desc str. Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: if local: if int(data[self._siteIDCol]) != siteID: continue kinst = int(data[self._kinstCol]) categoryID = self._madInstObj.getCategoryId(kinst) if categoryID not in localDict.keys(): localDict[categoryID] = self._madInstObj.getCategory(kinst) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) categoryIDKeys = localDict.keys() categoryIDKeys.sort() retList = [(catID, localDict[catID]) for catID in categoryIDKeys] return(retList)
def getInstrumentYears(
self, kinst)
getInstrumentYears returns the a list of years (int) for instrument.
Inputs:
kinst - the instrument id
Returns: an ordered list of years as integers. If none found, raises error
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getInstrumentYears(self, kinst): """getInstrumentYears returns the a list of years (int) for instrument. Inputs: kinst - the instrument id Returns: an ordered list of years as integers. If none found, raises error Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, data in enumerate(self.__fileList): # find matching code try: thisKinst = int(data[self._kinstCol]) if thisKinst != kinst: continue yearsStr = data[self._yearsCol] retList = [int(year) for year in yearsStr.split()] retList.sort() except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(retList) == 0: raise madrigal.admin.MadrigalError('No data found for kinst %i' % (kinst), '') return(retList)
def getInstruments(
self, categoryID=0, local=False)
getInstruments returns the a list of (kinst, instrument desc, siteID) tuples.
Inputs:
categoryID - category id to return instruments with data for. If 0, return all local - if False, return all instruments with that category for which there is data anywhere in Madrigal. If True, only return local instruments, in which case siteID is always the local siteID.
Returns: a list of (kinst, instrument desc, siteID) tuples
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getInstruments(self, categoryID=0, local=False): """getInstruments returns the a list of (kinst, instrument desc, siteID) tuples. Inputs: categoryID - category id to return instruments with data for. If 0, return all local - if False, return all instruments with that category for which there is data anywhere in Madrigal. If True, only return local instruments, in which case siteID is always the local siteID. Returns: a list of (kinst, instrument desc, siteID) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is kinst, value is tuple if (instrument desc str, siteID). # Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: kinst = int(data[self._kinstCol]) thisSiteID = int(data[self._siteIDCol]) thisCategoryID = self._madInstObj.getCategoryId(kinst) if thisCategoryID != categoryID and categoryID != 0: continue if local: if thisSiteID != siteID: continue if kinst not in localDict.keys(): localDict[kinst] = (self._madInstObj.getInstrumentName(kinst), thisSiteID) else: raise ValueError, 'Duplicate kinst %i found in instData' % (kinst) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) kinstKeys = localDict.keys() kinstKeys.sort() retList = [(kinst, localDict[kinst][0], localDict[kinst][1]) for kinst in kinstKeys] return(retList)
class MadrigalInstrumentKindats
MadrigalInstrumentKindats is an object that provides access to the metadata file that summarizes the kindat codes associated with each instrument.
This object provides access to all Madrigal instrument kindat information in the metadata file instKindatTab.txt. The metadata file instKindatTab.txt lists, for any given instrument and year, all the kindat codes found in all the data files in the local database associated with that instrument.
This class also contains a method to rebuild the table instKindatTab.txt by examining all the metadata in the database. This is presumably a somewhat slow process and should be done in the background.
Usage example::
import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentKindats() print test.getKindatListForInstruments([20,30]) except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Aug. 15, 2002
class MadrigalInstrumentKindats: """MadrigalInstrumentKindats is an object that provides access to the metadata file that summarizes the kindat codes associated with each instrument. This object provides access to all Madrigal instrument kindat information in the metadata file instKindatTab.txt. The metadata file instKindatTab.txt lists, for any given instrument and year, all the kindat codes found in all the data files in the local database associated with that instrument. This class also contains a method to rebuild the table instKindatTab.txt by examining all the metadata in the database. This is presumably a somewhat slow process and should be done in the background. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentKindats() print test.getKindatListForInstruments([20,30]) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Aug. 15, 2002 """ #constants __instKindatMetadataFile = "instKindatTab.txt" # column positions __instKindatKinstCol = 0 __instKindatListCol = 1 __instKindatYearCol = 2 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument kindat metadata file if (initFile == None): self.__filename = self.__instKindatMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getKindatListForInstruments(self, kinstList): """getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list. A kindat code is returned if it appears in any year Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer. Returns: a list of kindat codes as integers associated with the given instrument list. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # if kinstList is just a single integer, convert it into a list if type(kinstList) == types.IntType: kinstList = [kinstList] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instKindatKinstCol]) in kinstList): if len(inst[self.__instKindatListCol]) == 0: continue tempList = inst[self.__instKindatListCol].split() for parm in tempList: if not int(parm) in retKindatList: retKindatList.append(int(parm)) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return(retKindatList) def getKindatListForInstrumentYear(self, kinst, year): """getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year. Inputs: kinst, year Returns: a list of kindat codes as integers associated with the given instrument and year, or None if not found. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if int(inst[self.__instKindatKinstCol]) != kinst: continue if int(inst[self.__instKindatYearCol]) != year: continue return([int(kindat) for kindat in inst[self.__instKindatListCol].split()]) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def rebuildInstKindatTable(self): """rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file. The table instKindatTab.txt is a listing of every kindat found for a given instrument for any given year. Since these files are constantly updated, this table needs to be updated on a regular basis. Data from other Madrigal sites is also imported into this table. How it works: For each instrument in instTab.txt, this method first creates a list of all the experiments that used that instrument. It then loops through the file table. For each file listing, if the file is the default one, and its experiment id is in the list of experiments just created, the kindat is added to the list if its not already there. Data from all other Madrigal sites is then added via getMetadata. It then writes the metadata file in the form: 10, 1001 1002 1003, where the first column in the instrument code and the second column is a space delimited list of kindat codes. Inputs: None. Returns: None. Affects: Writes file instKindatTab.txt in metadata directory Exceptions: If unable to write instKindatTab.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instKindatTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # get experiment metadata madExp = MadrigalExperiment(self.__madDB) kindatDict = {} # key = year, value = list of kindats for i in range(madExp.getExpCount()): kinst = madExp.getKinstByPosition(i) if kinst not in kindatDict.keys(): kindatDict[kinst] = {} expDir = madExp.getExpDirByPosition(i) # get year range timeTuple = madExp.getExpStartDateTimeByPosition(i) syear = timeTuple[0] timeTuple = madExp.getExpEndDateTimeByPosition(i) eyear = timeTuple[0] for year in range(syear, eyear+1): if year not in kindatDict[kinst].keys(): kindatDict[kinst][year] = [] try: madFileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt')) except: continue for j in range(madFileObj.getFileCount()): kindat = madFileObj.getKindatByPosition(j) if kindat not in kindatDict[kinst][year]: kindatDict[kinst][year].append(kindat) # create newFileStr from instCodeDict newFileStr = '' kinstList = kindatDict.keys() kinstList.sort() delimiter = ' ' for kinst in kinstList: yearList = kindatDict[kinst].keys() yearList.sort() for year in yearList: kindatDict[kinst][year].sort() kindatStrList = [str(kindat) for kindat in kindatDict[kinst][year]] kindatStr = delimiter.join(kindatStrList) newFileStr += '%i,%s,%i\n' % (kinst, kindatStr, year) # finally, write new instKindatTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None)
init initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile).
Inputs: Existing MadrigalDB object, by default = None.
String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there.
def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument kindat metadata file if (initFile == None): self.__filename = self.__instKindatMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()
def getKindatListForInstrumentYear(
self, kinst, year)
getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year.
Inputs: kinst, year
Returns: a list of kindat codes as integers associated with the given instrument and year, or None if not found.
Affects: None
Exceptions: if error in metadata file
def getKindatListForInstrumentYear(self, kinst, year): """getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year. Inputs: kinst, year Returns: a list of kindat codes as integers associated with the given instrument and year, or None if not found. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if int(inst[self.__instKindatKinstCol]) != kinst: continue if int(inst[self.__instKindatYearCol]) != year: continue return([int(kindat) for kindat in inst[self.__instKindatListCol].split()]) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
def getKindatListForInstruments(
self, kinstList)
getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list. A kindat code is returned if it appears in any year
Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer.
Returns: a list of kindat codes as integers associated with the given instrument list.
Affects: None
Exceptions: if error in metadata file
def getKindatListForInstruments(self, kinstList): """getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list. A kindat code is returned if it appears in any year Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer. Returns: a list of kindat codes as integers associated with the given instrument list. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # if kinstList is just a single integer, convert it into a list if type(kinstList) == types.IntType: kinstList = [kinstList] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instKindatKinstCol]) in kinstList): if len(inst[self.__instKindatListCol]) == 0: continue tempList = inst[self.__instKindatListCol].split() for parm in tempList: if not int(parm) in retKindatList: retKindatList.append(int(parm)) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return(retKindatList)
def rebuildInstKindatTable(
self)
rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file.
The table instKindatTab.txt is a listing of every kindat found for a given instrument for any given year. Since these files are constantly updated, this table needs to be updated on a regular basis. Data from other Madrigal sites is also imported into this table.
How it works: For each instrument in instTab.txt, this method first creates a list of all the experiments that used that instrument. It then loops through the file table. For each file listing, if the file is the default one, and its experiment id is in the list of experiments just created, the kindat is added to the list if its not already there. Data from all other Madrigal sites is then added via getMetadata. It then writes the metadata file in the form: 10, 1001 1002 1003, where the first column in the instrument code and the second column is a space delimited list of kindat codes.
Inputs: None.
Returns: None.
Affects: Writes file instKindatTab.txt in metadata directory
Exceptions: If unable to write instKindatTab.txt file.
def rebuildInstKindatTable(self): """rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file. The table instKindatTab.txt is a listing of every kindat found for a given instrument for any given year. Since these files are constantly updated, this table needs to be updated on a regular basis. Data from other Madrigal sites is also imported into this table. How it works: For each instrument in instTab.txt, this method first creates a list of all the experiments that used that instrument. It then loops through the file table. For each file listing, if the file is the default one, and its experiment id is in the list of experiments just created, the kindat is added to the list if its not already there. Data from all other Madrigal sites is then added via getMetadata. It then writes the metadata file in the form: 10, 1001 1002 1003, where the first column in the instrument code and the second column is a space delimited list of kindat codes. Inputs: None. Returns: None. Affects: Writes file instKindatTab.txt in metadata directory Exceptions: If unable to write instKindatTab.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instKindatTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # get experiment metadata madExp = MadrigalExperiment(self.__madDB) kindatDict = {} # key = year, value = list of kindats for i in range(madExp.getExpCount()): kinst = madExp.getKinstByPosition(i) if kinst not in kindatDict.keys(): kindatDict[kinst] = {} expDir = madExp.getExpDirByPosition(i) # get year range timeTuple = madExp.getExpStartDateTimeByPosition(i) syear = timeTuple[0] timeTuple = madExp.getExpEndDateTimeByPosition(i) eyear = timeTuple[0] for year in range(syear, eyear+1): if year not in kindatDict[kinst].keys(): kindatDict[kinst][year] = [] try: madFileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt')) except: continue for j in range(madFileObj.getFileCount()): kindat = madFileObj.getKindatByPosition(j) if kindat not in kindatDict[kinst][year]: kindatDict[kinst][year].append(kindat) # create newFileStr from instCodeDict newFileStr = '' kinstList = kindatDict.keys() kinstList.sort() delimiter = ' ' for kinst in kinstList: yearList = kindatDict[kinst].keys() yearList.sort() for year in yearList: kindatDict[kinst][year].sort() kindatStrList = [str(kindat) for kindat in kindatDict[kinst][year]] kindatStr = delimiter.join(kindatStrList) newFileStr += '%i,%s,%i\n' % (kinst, kindatStr, year) # finally, write new instKindatTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close
class MadrigalInstrumentParameters
MadrigalInstrumentParameters is an object that provides access to the metadata file that summarizes the parameters associated with each instrument.
This object provides access to all Madrigal instrument parameter information in the metadata file instParmTab.txt. The metadata file instParmTab.txt lists, for any given instrument, all the measured parameters found in all the data files in the database associated with that instrument.
This class also contains a method to rebuild the table instParmTab.txt by examining every data file in the database. This is presumably a slow process and should be done in the background.
Usage example::
import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentParameters() print test.getParameters(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jul. 17, 2002
class MadrigalInstrumentParameters: """MadrigalInstrumentParameters is an object that provides access to the metadata file that summarizes the parameters associated with each instrument. This object provides access to all Madrigal instrument parameter information in the metadata file instParmTab.txt. The metadata file instParmTab.txt lists, for any given instrument, all the measured parameters found in all the data files in the database associated with that instrument. This class also contains a method to rebuild the table instParmTab.txt by examining every data file in the database. This is presumably a slow process and should be done in the background. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentParameters() print test.getParameters(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jul. 17, 2002 """ #constants __instParmMetadataFile = "instParmTab.txt" # column positions __instParmKinstCol = 0 __instParmListCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instParmTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument parameter metadata file if (initFile == None): self.__filename = self.__instParmMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getParameters(self, kinst): """getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get parameters. If 0, get parameters from all instruments. Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank. Affects: None Exceptions: if error in metadata file """ parmList = [] for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instParmKinstCol]) == kinst or kinst == 0): if len(inst[self.__instParmListCol]) == 0: continue tempList = inst[self.__instParmListCol].split() for parm in tempList: if not parm.lower() in parmList: if parm.find('-32767') == -1: parmList.append(parm.lower()) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(parmList) == 0: return(None) return parmList def rebuildInstParmTable(self, completeRebuildFlag = 0): """rebuildInstParmTable rebuilds the instParmTab.txt metadata file. The table instParmTab.txt is a listing of every measured parameter found in any data file for a given instrument. It now will also import data from other Madrigal sites instParmTab.txt files. Since these files are constantly updated, this table needs to be updated on a regular basis. This methods works in one of two ways, depending on the value of completeRebuildFlag and whether a file called instParmLastUpdate.txt exists in the metadata directory. If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable loops through each instrument in the instTab.txt. For each instrument, it loops through every data file associated with that instrument. For every data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. Since this process involves every file in the database, it may take a great deal of time and should be run in the background. If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable first stores all the existing parameters from the instParmTab.txt. It reads the date of the last update from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with each instrument. For every new data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. This makes rebuildInstParmTable faster, but possibly keeps invalid parameters if experiments are ever deleted. Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are also added. Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means newer than the date in the instParmLastUpdate.txt file (stored as a float). If 1, rebuild the table completely. This will eliminate any parameters no longer found in files, but will take longer. If no instParmLastUpdate.txt file is found, the table is always rebuilt completely. Returns: None. Affects: Writes file instParmTab.txt in metadata directory Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instParmTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # if madroot not set, set it now if os.environ.get('MAD' + 'ROOT') == None: os.environ['MAD' + 'ROOT'] = self.__madDB.getMadroot() # now its safe to import madrigal.data since MAD + ROOT set import madrigal.data # get date of files to examine if completeRebuildFlag == 0: # try to open and read instParmLastUpdate.txt try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'r') lastUpdateTime = float(dateFile.read()) dateFile.close() except: # if failure, set time to 1/1/1970 and set completeRebuildFlag = 1 lastUpdateTime = time.mktime([1970,1,1,0,0,0,0,0,0]) completeRebuildFlag = 1 else: # set time to 1/1/1971 lastUpdateTime = time.mktime([1971,1,1,0,0,0,0,0,0]) # now write new version of instParmLastUpdate.txt # if not writable, exception thrown try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'w') except: raise madrigal.admin.MadrigalError('Unable to write: ' + \ self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', None) newUpdateTime = time.time() dateFile.write(str(newUpdateTime)) dateFile.close() # finally, make sure new instParmLastUpdate.txt is world-writable try: os.chmod(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 0666) except: pass # create a needed MadrigalParameters obj madParmObj = madrigal.data.MadrigalParameters(self.__madDB) # create a needed MadrigalSite obj madSiteObj = madrigal.metadata.MadrigalSite(self.__madDB) # create a string to hold all the text for the new file newFileStr = '' # create a dictionary with all instrument codes as keys instObj = madrigal.metadata.MadrigalInstrument(self.__madDB) instList = instObj.getInstrumentList() instCodeDict = {} for inst in instList: if inst[2] == 0: continue if inst[2] in instCodeDict.keys(): continue if completeRebuildFlag == 1: # start with empty list instCodeDict[inst[2]]= [] else: # start with present list tempParmList = [] oldParmList = self.getParameters(inst[2]) if oldParmList != None: for item in oldParmList: # check that its not -32767 try: if madParmObj.getParmCodeFromMnemonic(item) == -32767: continue except ValueError: print('skipping parm %s since no longer valid' % (str(item))) continue tempParmList.append(madParmObj.getParmCodeFromMnemonic(item)) instCodeDict[inst[2]] = tempParmList else: instCodeDict[inst[2]] = [] # get a list of all default and realtime files filelist = self.__madDB.getFileListFromMetadata(appendKinst = 1, includeRealtime = 1) # appendKinst, allow realtime # loop through each file - file is tuple of # (file name, inst code) for file in filelist: # check whether file should be skipped try: if lastUpdateTime > os.stat(file[0])[8]: continue except: continue # try to create MadrigalFile object for that file try: print '\t\tAdding parameters to instParmTab.txt from ' + str(file) madFileObj = madrigal.data.MadrigalFile(file[0], self.__madDB) fileParmList = madFileObj.getMeasuredParmList() # append unique parameters for parm in fileParmList: if not parm in instCodeDict[file[1]]: instCodeDict[file[1]].append(parm) # if a problem with file, skip it except: continue # now create a new file string from instCodeDict # create a sorted list of keys keyList = instCodeDict.keys() keyList.sort() # step through each key delimiter = ' ' for key in keyList: # sort that instrument's list instParmList = madParmObj.normalizeParmList(instCodeDict[key]) # convert from codes to mnemonics instParmList = madParmObj.getParmMnemonicList(instParmList) # append that instrument's data to newFileStr newFileStr = newFileStr + str(key) + ',' newFileStr = newFileStr + delimiter.join(instParmList).lower() + '\n' # if no data found, throw error if len(newFileStr) == 0: raise madrigal.admin.MadrigalError('No data found for: ' + str(filename), None) # write new instParmTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None)
init initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile).
Inputs: Existing MadrigalDB object, by default = None.
String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instParmTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there.
def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instParmTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument parameter metadata file if (initFile == None): self.__filename = self.__instParmMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()
def getParameters(
self, kinst)
getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank.
Inputs: kinst integer to get parameters. If 0, get parameters from all instruments.
Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank.
Affects: None
Exceptions: if error in metadata file
def getParameters(self, kinst): """getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get parameters. If 0, get parameters from all instruments. Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank. Affects: None Exceptions: if error in metadata file """ parmList = [] for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instParmKinstCol]) == kinst or kinst == 0): if len(inst[self.__instParmListCol]) == 0: continue tempList = inst[self.__instParmListCol].split() for parm in tempList: if not parm.lower() in parmList: if parm.find('-32767') == -1: parmList.append(parm.lower()) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(parmList) == 0: return(None) return parmList
def rebuildInstParmTable(
self, completeRebuildFlag=0)
rebuildInstParmTable rebuilds the instParmTab.txt metadata file.
The table instParmTab.txt is a listing of every measured parameter found in any data file for a given instrument. It now will also import data from other Madrigal sites instParmTab.txt files. Since these files are constantly updated, this table needs to be updated on a regular basis. This methods works in one of two ways, depending on the value of completeRebuildFlag and whether a file called instParmLastUpdate.txt exists in the metadata directory.
If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable loops through each instrument in the instTab.txt. For each instrument, it loops through every data file associated with that instrument. For every data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. Since this process involves every file in the database, it may take a great deal of time and should be run in the background.
If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable first stores all the existing parameters from the instParmTab.txt. It reads the date of the last update from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with each instrument. For every new data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. This makes rebuildInstParmTable faster, but possibly keeps invalid parameters if experiments are ever deleted.
Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are also added.
Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means newer than the date in the instParmLastUpdate.txt file (stored as a float). If 1, rebuild the table completely. This will eliminate any parameters no longer found in files, but will take longer. If no instParmLastUpdate.txt file is found, the table is always rebuilt completely.
Returns: None.
Affects: Writes file instParmTab.txt in metadata directory
Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file.
def rebuildInstParmTable(self, completeRebuildFlag = 0): """rebuildInstParmTable rebuilds the instParmTab.txt metadata file. The table instParmTab.txt is a listing of every measured parameter found in any data file for a given instrument. It now will also import data from other Madrigal sites instParmTab.txt files. Since these files are constantly updated, this table needs to be updated on a regular basis. This methods works in one of two ways, depending on the value of completeRebuildFlag and whether a file called instParmLastUpdate.txt exists in the metadata directory. If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable loops through each instrument in the instTab.txt. For each instrument, it loops through every data file associated with that instrument. For every data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. Since this process involves every file in the database, it may take a great deal of time and should be run in the background. If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable first stores all the existing parameters from the instParmTab.txt. It reads the date of the last update from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with each instrument. For every new data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. This makes rebuildInstParmTable faster, but possibly keeps invalid parameters if experiments are ever deleted. Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are also added. Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means newer than the date in the instParmLastUpdate.txt file (stored as a float). If 1, rebuild the table completely. This will eliminate any parameters no longer found in files, but will take longer. If no instParmLastUpdate.txt file is found, the table is always rebuilt completely. Returns: None. Affects: Writes file instParmTab.txt in metadata directory Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instParmTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # if madroot not set, set it now if os.environ.get('MAD' + 'ROOT') == None: os.environ['MAD' + 'ROOT'] = self.__madDB.getMadroot() ow its safe to import madrigal.data since MAD + ROOT set import madrigal.data et date of files to examine if completeRebuildFlag == 0: # try to open and read instParmLastUpdate.txt try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'r') lastUpdateTime = float(dateFile.read()) dateFile.close() except: # if failure, set time to 1/1/1970 and set completeRebuildFlag = 1 lastUpdateTime = time.mktime([1970,1,1,0,0,0,0,0,0]) completeRebuildFlag = 1 else: # set time to 1/1/1971 lastUpdateTime = time.mktime([1971,1,1,0,0,0,0,0,0]) # now write new version of instParmLastUpdate.txt # if not writable, exception thrown try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'w') except: raise madrigal.admin.MadrigalError('Unable to write: ' + \ self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', None) newUpdateTime = time.time() dateFile.write(str(newUpdateTime)) dateFile.close() # finally, make sure new instParmLastUpdate.txt is world-writable try: os.chmod(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 0666) except: pass # create a needed MadrigalParameters obj madParmObj = madrigal.data.MadrigalParameters(self.__madDB) # create a needed MadrigalSite obj madSiteObj = madrigal.metadata.MadrigalSite(self.__madDB) # create a string to hold all the text for the new file newFileStr = '' # create a dictionary with all instrument codes as keys instObj = madrigal.metadata.MadrigalInstrument(self.__madDB) instList = instObj.getInstrumentList() instCodeDict = {} for inst in instList: if inst[2] == 0: continue if inst[2] in instCodeDict.keys(): continue if completeRebuildFlag == 1: # start with empty list instCodeDict[inst[2]]= [] else: # start with present list tempParmList = [] oldParmList = self.getParameters(inst[2]) if oldParmList != None: for item in oldParmList: # check that its not -32767 try: if madParmObj.getParmCodeFromMnemonic(item) == -32767: continue except ValueError: print('skipping parm %s since no longer valid' % (str(item))) continue tempParmList.append(madParmObj.getParmCodeFromMnemonic(item)) instCodeDict[inst[2]] = tempParmList else: instCodeDict[inst[2]] = [] # get a list of all default and realtime files filelist = self.__madDB.getFileListFromMetadata(appendKinst = 1, includeRealtime = 1) # appendKinst, allow realtime # loop through each file - file is tuple of # (file name, inst code) for file in filelist: # check whether file should be skipped try: if lastUpdateTime > os.stat(file[0])[8]: continue except: continue # try to create MadrigalFile object for that file try: print '\t\tAdding parameters to instParmTab.txt from ' + str(file) madFileObj = madrigal.data.MadrigalFile(file[0], self.__madDB) fileParmList = madFileObj.getMeasuredParmList() # append unique parameters for parm in fileParmList: if not parm in instCodeDict[file[1]]: instCodeDict[file[1]].append(parm) # if a problem with file, skip it except: continue # now create a new file string from instCodeDict # create a sorted list of keys keyList = instCodeDict.keys() keyList.sort() # step through each key delimiter = ' ' for key in keyList: # sort that instrument's list instParmList = madParmObj.normalizeParmList(instCodeDict[key]) # convert from codes to mnemonics instParmList = madParmObj.getParmMnemonicList(instParmList) # append that instrument's data to newFileStr newFileStr = newFileStr + str(key) + ',' newFileStr = newFileStr + delimiter.join(instParmList).lower() + '\n' # if no data found, throw error if len(newFileStr) == 0: raise madrigal.admin.MadrigalError('No data found for: ' + str(filename), None) # write new instParmTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close
class MadrigalKindat
MadrigalKindat is an object that provides access to Madrigal kind of data info from the metadata.
This object provides access to all Madrigal kind of data information in the metadata file typeTab.txt.
Usage example::
import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalKindat() print test.getKindatDescription(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001
class MadrigalKindat: """MadrigalKindat is an object that provides access to Madrigal kind of data info from the metadata. This object provides access to all Madrigal kind of data information in the metadata file typeTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalKindat() print test.getKindatDescription(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __typeMetadataFile = "typeTab.txt" # column positions __typeCodeCol = 0 __typeDescCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalKindat by reading from typeTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/typeTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get kindat metadata file if (initFile == None): self.__filename = self.__typeMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getKindatDescription(self, code, kinst=None): """getKindatDescription returns the kindat description that matches code argument, or None if not found. Inputs: code integer to get kindat description. or integer as string kinst - used to look up kindat description in form '%i_%i' % (kinst, code). If None, (the default) only return exact code match as integer. If given, first try to match '%i_%i' % (kinst, code). If not found, then try just code. May be integer or string integer Returns: the kindat description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) if kinst != None: kinst = int(kinst) for i, type in enumerate(self.__fileList): # find matching kinst and code try: items = type[self.__typeCodeCol].split('_') if len(items) == 2: thisKinst = int(items[0]) thisKindat = int(items[1]) if thisKinst == kinst and thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # just search for code for i, type in enumerate(self.__fileList): try: items = type[self.__typeCodeCol].split('_') if len(items) == 1: thisKindat = int(items[0]) if thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getKindatList(self): """getKindatList returns a list of all kindat descriptions and codes. Inputs: None. Returns: a list of all kindat descriptions and codes. Each item in the list is a tuple of the form (Kindat description (string), kindat code (integer or string in form '%i_%i' % (kinst, code))). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, kindat in enumerate(self.__fileList): try: # works if its a standard code item = (kindat[self.__typeDescCol], int(kindat[self.__typeCodeCol])) retList.append(item) except: try: # works for kinst_kindat item = (kindat[self.__typeDescCol], kindat[self.__typeCodeCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(kindat), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None)
init initializes MadrigalKindat by reading from typeTab.txt (or initFile).
Inputs: Existing MadrigalDB object, by default = None.
String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/typeTab.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.
def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalKindat by reading from typeTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/typeTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get kindat metadata file if (initFile == None): self.__filename = self.__typeMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()
def getKindatDescription(
self, code, kinst=None)
getKindatDescription returns the kindat description that matches code argument, or None if not found.
Inputs:
code integer to get kindat description. or integer as string kinst - used to look up kindat description in form '%i_%i' % (kinst, code). If None, (the default) only return exact code match as integer. If given, first try to match '%i_%i' % (kinst, code). If not found, then try just code. May be integer or string integer
Returns: the kindat description that matches code argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getKindatDescription(self, code, kinst=None): """getKindatDescription returns the kindat description that matches code argument, or None if not found. Inputs: code integer to get kindat description. or integer as string kinst - used to look up kindat description in form '%i_%i' % (kinst, code). If None, (the default) only return exact code match as integer. If given, first try to match '%i_%i' % (kinst, code). If not found, then try just code. May be integer or string integer Returns: the kindat description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) if kinst != None: kinst = int(kinst) for i, type in enumerate(self.__fileList): # find matching kinst and code try: items = type[self.__typeCodeCol].split('_') if len(items) == 2: thisKinst = int(items[0]) thisKindat = int(items[1]) if thisKinst == kinst and thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # just search for code for i, type in enumerate(self.__fileList): try: items = type[self.__typeCodeCol].split('_') if len(items) == 1: thisKindat = int(items[0]) if thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getKindatList(
self)
getKindatList returns a list of all kindat descriptions and codes.
Inputs: None.
Returns: a list of all kindat descriptions and codes. Each item in the list is a tuple of the form (Kindat description (string), kindat code (integer or string in form '%i_%i' % (kinst, code))). Example item: ('INSCAL Basic Derived Parameters', 3001)
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getKindatList(self): """getKindatList returns a list of all kindat descriptions and codes. Inputs: None. Returns: a list of all kindat descriptions and codes. Each item in the list is a tuple of the form (Kindat description (string), kindat code (integer or string in form '%i_%i' % (kinst, code))). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, kindat in enumerate(self.__fileList): try: # works if its a standard code item = (kindat[self.__typeDescCol], int(kindat[self.__typeCodeCol])) retList.append(item) except: try: # works for kinst_kindat item = (kindat[self.__typeDescCol], kindat[self.__typeCodeCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(kindat), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList
class MadrigalMetaFile
MadrigalMetaFile is an object that provides access to Madrigal file info from the metadata.
This object provides access to all Madrigal experiment information in the metadata file fileTab.txt.
Usage example::
import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalMetaFile() print test.getFileCount() except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu May. 7, 2002
class MadrigalMetaFile: """MadrigalMetaFile is an object that provides access to Madrigal file info from the metadata. This object provides access to all Madrigal experiment information in the metadata file fileTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalMetaFile() print test.getFileCount() except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu May. 7, 2002 """ #constants __fileMetadataFile = "fileTab.txt" # column positions __fileNameCol = 0 __fileExpIdCol = 1 __fileKindatCol = 2 __fileCategoryCol = 3 __fileSizeCol = 4 __fileHasCatalogCol = 5 __fileHasHeaderCol = 6 __fileModDateCol = 7 __fileModTimeCol = 8 __fileStatusCol = 9 __fileAccessCol = 10 __fileAnalystCol = 11 __fileAnalystEmailCol = 12 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalMetaFile by reading from fileTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/fileTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get file metadata file if (initFile == None): self.__filename = self.__fileMetadataFile else: self.__filename = initFile # MadrigalMetafile can now legal have two different lengths - with or without file analyst and email allowedLens = (self.__fileAccessCol+1, self.__fileAnalystEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__fileNameCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # check that at least first row has right number of columns if len(self.__fileList) > 0: # analyst and analyst email columns may or may not exist if len(self.__fileList[0]) not in allowedLens: raise madrigal.admin.MadrigalError('Error in count of first row of ' + str(self.__filename), None) def getFileCount(self): """getFileCount returns the number of files (rows) in the metadata file. Inputs: None Returns: the number of files (rows) in the metadata file. Affects: None Exceptions: None """ return len(self.__fileList) def getFilenameByPosition(self, position = 0): """getFilenameByPosition returns the filename of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the filename of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileNameCol] else: return(None) def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id (integer) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the experiment id (integer) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst exp id cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileExpIdCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the file at given position. Inputs: position - position of file in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the file at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__fileExpIdCol] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str(position), str(expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def getKindatByPosition(self, position = 0): """getKindatByPosition returns the kindat (kind of data code) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (kind of instrument code) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileKindatCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getCategoryByPosition(self, position = 0): """getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileCategoryCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getCategoryByFilename(self, filename): """getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename. Inputs: filename - name of file to search for. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileCategoryCol])) def getHasCatalogByPosition(self, position = 0): """getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any catalog records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasCatalogCol]) == 0: return False else: return True else: return(None) def getHasCatalogByFilename(self,filename): """getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any catalog records, False otherwise. Returns none if name not found Affects: None Exceptions: None. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasCatalogCol])) def setHasCatalogByPosition(self, position, hasCatalog): """setHasCatalogByPosition sets the value of hasCatalog for the file at the given position. Inputs: position - position of file in list (first position is zero). hasCatalog - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasCatalog not in (0, 1, True, False): raise ValueError, 'Illegal value for hasCatalog in setHasCatalogByPosition: %s' % (str(hasCatalog)) if len(self.__fileList) > position: if hasCatalog in (1, True): self.__fileList[position][self.__fileHasCatalogCol] = '1' else: self.__fileList[position][self.__fileHasCatalogCol] = '0' else: raise ValueError, 'setHasCatalogByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getHasHeaderByPosition(self, position = 0): """getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any header records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasHeaderCol]) == 0: return False else: return True else: return(None) def getHasHeaderByFilename(self,filename): """getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any header records, False otherwise. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasHeaderCol])) def setHasHeaderByPosition(self, position, hasHeader): """setHasHeaderByPosition sets the value of hasHeader for the file at the given position. Inputs: position - position of file in list (first position is zero). hasHeader - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasHeader not in (0, 1, True, False): raise ValueError, 'Illegal value for hasHeader in setHasHeaderByPosition: %s' % (str(hasHeader)) if len(self.__fileList) > position: if hasHeader in (1, True): self.__fileList[position][self.__fileHasHeaderCol] = '1' else: self.__fileList[position][self.__fileHasHeaderCol] = '0' else: raise ValueError, 'setHasHeaderByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getStatusByPosition(self, position = 0): """getStatusByPosition returns the status description of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the status description of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileStatusCol] else: return(None) def getStatusByFilename(self,filename): """getStatusByFilename returns the status description of the file with the given name. Inputs: filename - name of file to search for. Returns: the status description of the file with the given name. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.__fileList[position][self.__fileStatusCol]) def getAccessByPosition(self, position = 0): """getAccessByPosition returns the access (0=public, 1=private) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the access level (0=public, 1=private) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if access column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileAccessCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getFileDatetimeByPosition(self, position = 0): """getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set. Inputs: position of file in list (first position is zero). Defaults to first. Returns: a datetime of the file at given position, or None if not set. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: dateStr = self.__fileList[position][self.__fileModDateCol] if len(dateStr) < 8: return(None) timeStr = self.__fileList[position][self.__fileModTimeCol] if len(timeStr) < 6: return(None) return(datetime.datetime.strptime('%s %s' % (dateStr, timeStr), '%Y%m%d %H%M%S')) else: return(None) def getFileDatetimeByFilename(self,filename): """getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set. Inputs: filename - name of file to search for. Returns: a datetime of the file with the given filenme, or None if not set. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.getFileDatetimeByPosition(position)) def setFileDatetimeByPosition(self, position, dt): """setFileDatetimeByPosition sets the file datetime for the given position Inputs: position - position of file in list (first position is zero). dt - datetime (UT) of file being added. If None, use file datetime. Will raise exception if dt is None and fileTab.txt in the metadata version, not the expDir version. Returns: None. Affects: None Exceptions: If position beyond length. """ if dt is None: basename = self.getFilenameByPosition(position) expDir = os.path.dirname(self.__filename) if len(expDir) < 2: raise IOError('Cannot set datetime to None using the metadata version of fileTab.txt') dataFile = os.path.join(expDir, basename) if not os.access(dataFile, os.R_OK): raise IOError('Cannot access %s' % (dataFile)) dt = datetime.datetime.utcfromtimestamp(os.path.getmtime(dataFile)) dateStr = dt.strftime('%Y%m%d') timeStr = dt.strftime('%H%M%S') if len(self.__fileList) > position: self.__fileList[position][self.__fileModDateCol] = dateStr self.__fileList[position][self.__fileModTimeCol] = timeStr else: raise ValueError('setFileDatetimeByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def deleteRowByFilename(self, filename): """deleteRowByFilename deletes a row with a given filename. Inputs: filename - name of file to search for. Returns: None. Affects: Removes item from self.__fileList if filename found Exceptions: Thrown if filename not found. """ for file in self.__fileList: if filename == file[self.__fileNameCol]: self.__fileList.remove(file) return # no matches found raise madrigal.admin.MadrigalError('Could not delete file ' + filename + ' from ' + self.__filename, None) def getExpIdByFilename(self, filename): """getExpIdByFilename returns the first experiment id (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the experiment id (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if exp id cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileExpIdCol])) def getKindatByFilename(self, filename): """getKindatByFilename returns the first kindat (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the kindat (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if kindat cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileKindatCol])) def setAccessByPosition(self, position, access): """setAccessByPosition sets the value of access for the file at the given position. Inputs: position - position of file in list (first position is zero). access - 0 of False for public, 1 or True for private Returns: None. Affects: None Exceptions: If position beyond length. """ if access not in (0, 1, True, False): raise ValueError, 'Illegal value for access in setAccessByPosition: %s' % (str(access)) if len(self.__fileList) > position: if access in (1, True): self.__fileList[position][self.__fileAccessCol] = '1' else: self.__fileList[position][self.__fileAccessCol] = '0' else: raise ValueError, 'setAccessByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def setAccess(self, accessType): """setAccess sets the access column to all 0's (public) or all 1's (private). Inputs: accessType - either 0 to set to public access, or 1 to set to private access. Returns: None. Affects: Overwrite fileTab.txt file with access column set to all 0's (public) or all 1's (private). Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1 """ if (accessType != 0 and accessType != 1): raise madrigal.admin.MadrigalError('MadrigalMetaFile.setAccess called with arg = ' + \ str(accessType) + ', must be either 0 or 1', None) for row in self.__fileList: row[self.__fileAccessCol] = str(accessType) self.writeMetadata() def setKindatByPosition(self, position, kindat): """setKindatByPosition sets the value of kindat for the file at the given position. Inputs: position - position of file in list (first position is zero). kindat - integer kindat value Returns: None. Affects: None Exceptions: If position beyond length. """ kindat = int(kindat) if len(self.__fileList) > position: self.__fileList[position][self.__fileKindatCol] = str(kindat) else: raise ValueError, 'setKindatByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def setCategoryByPosition(self, position, category): """setCategoryByPosition sets the value of category for the file at the given position. Inputs: position - position of file in list (first position is zero). category - 1=default, 2=variant, 3=history, 4=real-time Returns: None. Affects: None Exceptions: If position beyond length. """ if int(category) not in (1, 2, 3, 4): raise ValueError, 'Illegal value for category in setCategoryByPosition: %s' % (str(category)) if len(self.__fileList) > position: self.__fileList[position][self.__fileCategoryCol] = str(category) else: raise ValueError, 'setCategoryByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def setStatusByPosition(self, position, status): """setStatusByPosition sets the value of status string for the file at the given position. Inputs: position - position of file in list (first position is zero). status - string describing status Returns: None. Affects: None Exceptions: If position beyond length. """ if type(status) != types.StringType: raise ValueError, 'Illegal value for status in setStatusByPosition: %s' % (str(status)) # check that string does not illegally contain a comma if status.find(',') != -1: raise ValueError, 'status string in fileTab.txt cannot contain a comma: <%s> is illegal' % (status) if len(self.__fileList) > position: self.__fileList[position][self.__fileStatusCol] = status else: raise ValueError, 'setStatusByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getAnalystByPosition(self, position = 0): """getAnalystByPosition returns file analyst name if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None) else: return(None) def getAnalystByFilename(self, filename): """getAnalystByFilename returns file analyst name if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None) def setAnalystByPosition(self, position, analyst): """setAnalystByPosition sets the file analyst name for the given position Inputs: position - position of file in list (first position is zero). analyst - name of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analyst.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystByPosition with args %s: %s' % \ (str(position), analyst), [traceback.format_exc()]) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol+1: # append two extra columns self.__fileList[position].append(str(analyst)) self.__fileList[position].append('') else: self.__fileList[position][self.__fileAnalystCol] = str(analyst) else: raise ValueError, 'setAnalystByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getAnalystEmailByPosition(self, position = 0): """getAnalystEmailByPosition returns file analyst email if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None) else: return(None) def getAnalystEmailByFilename(self, filename): """getAnalystEmailByFilename returns file analyst email if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None) def setAnalystEmailByPosition(self, position, analystEmail): """setAnalystEmailByPosition sets the file analyst email for the given position Inputs: position - position of file in list (first position is zero). analystEmail - email of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analystEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystEmailByPosition with args %s: %s' % \ (str(position, analystEmail), [traceback.format_exc()])) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol + 1: # append two extra columns self.__fileList[position].append('') self.__fileList[position].append(str(analystEmail)) else: self.__fileList[position][self.__fileAnalystEmailCol] = str(analystEmail) else: raise ValueError, 'setAnalystEmailByPosition called for position %i beyond length %i' % (position, len(self.__fileList)) def getMetadataSummaryByFilename(self, filename, madExpObj = None, madInstObj = None, madKindatObj = None): """getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename. Inputs: filename - name of file to search for. The next three inputs are other metadata objects. If these objects already exist, performance will be improved by passing them in rather than recreating them. If they do not exist, they will be created: madExpObj - a MadrigalExperiment object to get experiment metadata from. If None (the default), a new MadrigalExperiment object is created. madInstObj - a MadrigalInstrument object to get experiment metadata from. If None (the default), a new MadrigalInstrument object is created. madKindatObj - a MadrigalKindat object to get experiment metadata from. If None (the default), a new MadrigalKindat object is created. Returns: A string summarizing the metadata about the file. The format is:: Start Date and Time: 01/06/1997 14:07 End Date and Time: 01/10/1997 23:45 Instrument: Millstone Hill Incoherent Scatter Radar Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study Kind of data: INSCAL (8.0) Basic Derived Parameters Affects: None Exceptions: Thrown if any parsing error in metadata. """ # build return String retStr = '' # create any needed metadata objects if madExpObj == None: madExpObj = MadrigalExperiment(self.__madDB) if madInstObj == None: madInstObj = MadrigalInstrument(self.__madDB) if madKindatObj == None: madKindatObj = MadrigalKindat(self.__madDB) # get the experiment id - if None, return message to update Metadata expId = self.getExpIdByFilename(filename) if expId == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' # get the experiment start date - if not found, return message to update Metadata starttime = madExpObj.getExpStartDateTimeByExpId(expId) if starttime == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' retStr = retStr + '\tExperiment start: ' + time.asctime(starttime) + '\n' # endtime endtime = madExpObj.getExpEndDateTimeByExpId(expId) retStr = retStr + '\tExperiment end: ' + time.asctime(endtime) + '\n' # get instrument name kinst = madExpObj.getKinstByExpId(expId) # if kinst not found, metadata is corrupt if kinst == None: return 'Could not find kinst for experiment id = ' + str(expId) instName = madInstObj.getInstrumentName(kinst) retStr = retStr + '\tInstrument name: ' + str(instName) + '\n' # get experiment name expName = madExpObj.getExpNameByExpId(expId) retStr = retStr + '\tExperiment name: ' + str(expName) + '\n' # get kind of data kindat = self.getKindatByFilename(filename) kindatName = madKindatObj.getKindatDescription(kindat, kinst) retStr = retStr + '\tKind of data: ' + str(kindatName) + '\n' return retStr def getFileDOIUrlByPosition(self, position): """getFileDOIUrlByPosition returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByPosition(position) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) basename = self.getFilenameByPosition(position) if basename is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, basename)) def getFileDOIUrlByFilename(self, filename): """getFileDOIUrlByFilename returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByFilename(filename) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, filename)) def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n') def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the fileTab.txt file. Inputs: newFullPath: a new path to write the fileTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def __str__(self): """return possibly modified file as a string in same format as writeMetadata """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' return(metaFileStr)
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None)
init initializes MadrigalMetaFile by reading from fileTab.txt (or initFile).
Inputs: Existing MadrigalDB object, by default = None.
String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/fileTab.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.
def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalMetaFile by reading from fileTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/fileTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get file metadata file if (initFile == None): self.__filename = self.__fileMetadataFile else: self.__filename = initFile # MadrigalMetafile can now legal have two different lengths - with or without file analyst and email allowedLens = (self.__fileAccessCol+1, self.__fileAnalystEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__fileNameCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # check that at least first row has right number of columns if len(self.__fileList) > 0: # analyst and analyst email columns may or may not exist if len(self.__fileList[0]) not in allowedLens: raise madrigal.admin.MadrigalError('Error in count of first row of ' + str(self.__filename), None)
def deleteRowByFilename(
self, filename)
deleteRowByFilename deletes a row with a given filename.
Inputs: filename - name of file to search for.
Returns: None.
Affects: Removes item from self.__fileList if filename found
Exceptions: Thrown if filename not found.
def deleteRowByFilename(self, filename): """deleteRowByFilename deletes a row with a given filename. Inputs: filename - name of file to search for. Returns: None. Affects: Removes item from self.__fileList if filename found Exceptions: Thrown if filename not found. """ for file in self.__fileList: if filename == file[self.__fileNameCol]: self.__fileList.remove(file) return # no matches found raise madrigal.admin.MadrigalError('Could not delete file ' + filename + ' from ' + self.__filename, None)
def getAccessByPosition(
self, position=0)
getAccessByPosition returns the access (0=public, 1=private) of the file at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the access level (0=public, 1=private) of the file at given position as an integer.
Returns None if position >= number of experiments.
Affects: None
Exceptions: Thrown if access column cannot be parsed into an integer
def getAccessByPosition(self, position = 0): """getAccessByPosition returns the access (0=public, 1=private) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the access level (0=public, 1=private) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if access column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileAccessCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getAnalystByFilename(
self, filename)
getAnalystByFilename returns file analyst name if there, None otherwise.
Inputs: filename - name of file to search for.
Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist.
Affects: None
Exceptions: None
def getAnalystByFilename(self, filename): """getAnalystByFilename returns file analyst name if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None)
def getAnalystByPosition(
self, position=0)
getAnalystByPosition returns file analyst name if there, None otherwise.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments.
Affects: None
Exceptions: None
def getAnalystByPosition(self, position = 0): """getAnalystByPosition returns file analyst name if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None) else: return(None)
def getAnalystEmailByFilename(
self, filename)
getAnalystEmailByFilename returns file analyst email if there, None otherwise.
Inputs: filename - name of file to search for.
Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist.
Affects: None
Exceptions: None
def getAnalystEmailByFilename(self, filename): """getAnalystEmailByFilename returns file analyst email if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None)
def getAnalystEmailByPosition(
self, position=0)
getAnalystEmailByPosition returns file analyst email if there, None otherwise.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments.
Affects: None
Exceptions: None
def getAnalystEmailByPosition(self, position = 0): """getAnalystEmailByPosition returns file analyst email if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None) else: return(None)
def getCategoryByFilename(
self, filename)
getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename.
Inputs: filename - name of file to search for.
Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None.
Affects: None
Exceptions: Thrown if category column cannot be parsed into an integer
def getCategoryByFilename(self, filename): """getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename. Inputs: filename - name of file to search for. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileCategoryCol]))
def getCategoryByPosition(
self, position=0)
getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer.
Returns None if position >= number of experiments.
Affects: None
Exceptions: Thrown if category column cannot be parsed into an integer
def getCategoryByPosition(self, position = 0): """getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileCategoryCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getExpIdByFilename(
self, filename)
getExpIdByFilename returns the first experiment id (integer) with the given filename.
Inputs: filename - name of file to search for.
Returns: the experiment id (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None.
Affects: None
Exceptions: Thrown if exp id cannot be parsed into an integer
def getExpIdByFilename(self, filename): """getExpIdByFilename returns the first experiment id (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the experiment id (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if exp id cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileExpIdCol]))
def getExpIdByPosition(
self, position=0)
getExpIdByPosition returns the experiment id (integer) of the file at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the experiment id (integer) of the file at given position as an integer.
Returns None if position >= number of experiments.
Affects: None
Exceptions: Thrown if kinst exp id cannot be parsed into an integer
def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id (integer) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the experiment id (integer) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst exp id cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileExpIdCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getFileCount(
self)
getFileCount returns the number of files (rows) in the metadata file.
Inputs: None
Returns: the number of files (rows) in the metadata file.
Affects: None
Exceptions: None
def getFileCount(self): """getFileCount returns the number of files (rows) in the metadata file. Inputs: None Returns: the number of files (rows) in the metadata file. Affects: None Exceptions: None """ return len(self.__fileList)
def getFileDOIUrlByFilename(
self, filename)
getFileDOIUrlByFilename returns the full url to the file on the archive site.
Meant to be used as a referencable url, so always refers to cedar.openmadigal.org
Returns permanent URL to file, or None if not found
def getFileDOIUrlByFilename(self, filename): """getFileDOIUrlByFilename returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByFilename(filename) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, filename))
def getFileDOIUrlByPosition(
self, position)
getFileDOIUrlByPosition returns the full url to the file on the archive site.
Meant to be used as a referencable url, so always refers to cedar.openmadigal.org
Returns permanent URL to file, or None if not found
def getFileDOIUrlByPosition(self, position): """getFileDOIUrlByPosition returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByPosition(position) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) basename = self.getFilenameByPosition(position) if basename is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, basename))
def getFileDatetimeByFilename(
self, filename)
getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set.
Inputs: filename - name of file to search for.
Returns: a datetime of the file with the given filenme, or None if not set. Returns none if name not found
Affects: None
Exceptions: Thrown if filename not found.
def getFileDatetimeByFilename(self,filename): """getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set. Inputs: filename - name of file to search for. Returns: a datetime of the file with the given filenme, or None if not set. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.getFileDatetimeByPosition(position))
def getFileDatetimeByPosition(
self, position=0)
getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: a datetime of the file at given position, or None if not set.
Returns None if position >= number of experiments.
Affects: None
Exceptions: None
def getFileDatetimeByPosition(self, position = 0): """getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set. Inputs: position of file in list (first position is zero). Defaults to first. Returns: a datetime of the file at given position, or None if not set. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: dateStr = self.__fileList[position][self.__fileModDateCol] if len(dateStr) < 8: return(None) timeStr = self.__fileList[position][self.__fileModTimeCol] if len(timeStr) < 6: return(None) return(datetime.datetime.strptime('%s %s' % (dateStr, timeStr), '%Y%m%d %H%M%S')) else: return(None)
def getFilenameByPosition(
self, position=0)
getFilenameByPosition returns the filename of the file at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the filename of the file at given position as a string.
Returns None if position >= number of experiments.
Affects: None
Exceptions: None
def getFilenameByPosition(self, position = 0): """getFilenameByPosition returns the filename of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the filename of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileNameCol] else: return(None)
def getHasCatalogByFilename(
self, filename)
getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise.
Inputs: filename - name of file to search for.
Returns: true if the file with the given name has any catalog records, False otherwise. Returns none if name not found
Affects: None
Exceptions: None.
def getHasCatalogByFilename(self,filename): """getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any catalog records, False otherwise. Returns none if name not found Affects: None Exceptions: None. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasCatalogCol]))
def getHasCatalogByPosition(
self, position=0)
getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: True if the file at given position has any catalog records, False otherwise Returns None if position >= number of experiments.
Affects: None
Exceptions: None
def getHasCatalogByPosition(self, position = 0): """getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any catalog records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasCatalogCol]) == 0: return False else: return True else: return(None)
def getHasHeaderByFilename(
self, filename)
getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise.
Inputs: filename - name of file to search for.
Returns: true if the file with the given name has any header records, False otherwise. Returns none if name not found
Affects: None
Exceptions: Thrown if filename not found.
def getHasHeaderByFilename(self,filename): """getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any header records, False otherwise. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasHeaderCol]))
def getHasHeaderByPosition(
self, position=0)
getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: True if the file at given position has any header records, False otherwise Returns None if position >= number of experiments.
Affects: None
Exceptions: None
def getHasHeaderByPosition(self, position = 0): """getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any header records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasHeaderCol]) == 0: return False else: return True else: return(None)
def getKindatByFilename(
self, filename)
getKindatByFilename returns the first kindat (integer) with the given filename.
Inputs: filename - name of file to search for.
Returns: the kindat (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None.
Affects: None
Exceptions: Thrown if kindat cannot be parsed into an integer
def getKindatByFilename(self, filename): """getKindatByFilename returns the first kindat (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the kindat (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if kindat cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileKindatCol]))
def getKindatByPosition(
self, position=0)
getKindatByPosition returns the kindat (kind of data code) of the file at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the kinst (kind of instrument code) of the file at given position as an integer.
Returns None if position >= number of experiments.
Affects: None
Exceptions: Thrown if kinst column cannot be parsed into an integer
def getKindatByPosition(self, position = 0): """getKindatByPosition returns the kindat (kind of data code) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (kind of instrument code) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileKindatCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None)
def getLine(
self, position)
getLine returns the line at a given position. Returns None if position > number of lines.
Inputs: position - position in file. First line = 0
def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n')
def getMetadataSummaryByFilename(
self, filename, madExpObj=None, madInstObj=None, madKindatObj=None)
getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename.
Inputs: filename - name of file to search for.
The next three inputs are other metadata objects. If these objects already exist, performance will be improved by passing them in rather than recreating them. If they do not exist, they will be created: madExpObj - a MadrigalExperiment object to get experiment metadata from. If None (the default), a new MadrigalExperiment object is created. madInstObj - a MadrigalInstrument object to get experiment metadata from. If None (the default), a new MadrigalInstrument object is created. madKindatObj - a MadrigalKindat object to get experiment metadata from. If None (the default), a new MadrigalKindat object is created.
Returns: A string summarizing the metadata about the file. The format is::
Start Date and Time: 01/06/1997 14:07 End Date and Time: 01/10/1997 23:45 Instrument: Millstone Hill Incoherent Scatter Radar Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study Kind of data: INSCAL (8.0) Basic Derived Parameters
Affects: None
Exceptions: Thrown if any parsing error in metadata.
def getMetadataSummaryByFilename(self, filename, madExpObj = None, madInstObj = None, madKindatObj = None): """getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename. Inputs: filename - name of file to search for. The next three inputs are other metadata objects. If these objects already exist, performance will be improved by passing them in rather than recreating them. If they do not exist, they will be created: madExpObj - a MadrigalExperiment object to get experiment metadata from. If None (the default), a new MadrigalExperiment object is created. madInstObj - a MadrigalInstrument object to get experiment metadata from. If None (the default), a new MadrigalInstrument object is created. madKindatObj - a MadrigalKindat object to get experiment metadata from. If None (the default), a new MadrigalKindat object is created. Returns: A string summarizing the metadata about the file. The format is:: Start Date and Time: 01/06/1997 14:07 End Date and Time: 01/10/1997 23:45 Instrument: Millstone Hill Incoherent Scatter Radar Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study Kind of data: INSCAL (8.0) Basic Derived Parameters Affects: None Exceptions: Thrown if any parsing error in metadata. """ # build return String retStr = '' # create any needed metadata objects if madExpObj == None: madExpObj = MadrigalExperiment(self.__madDB) if madInstObj == None: madInstObj = MadrigalInstrument(self.__madDB) if madKindatObj == None: madKindatObj = MadrigalKindat(self.__madDB) # get the experiment id - if None, return message to update Metadata expId = self.getExpIdByFilename(filename) if expId == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' # get the experiment start date - if not found, return message to update Metadata starttime = madExpObj.getExpStartDateTimeByExpId(expId) if starttime == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' retStr = retStr + '\tExperiment start: ' + time.asctime(starttime) + '\n' # endtime endtime = madExpObj.getExpEndDateTimeByExpId(expId) retStr = retStr + '\tExperiment end: ' + time.asctime(endtime) + '\n' # get instrument name kinst = madExpObj.getKinstByExpId(expId) # if kinst not found, metadata is corrupt if kinst == None: return 'Could not find kinst for experiment id = ' + str(expId) instName = madInstObj.getInstrumentName(kinst) retStr = retStr + '\tInstrument name: ' + str(instName) + '\n' # get experiment name expName = madExpObj.getExpNameByExpId(expId) retStr = retStr + '\tExperiment name: ' + str(expName) + '\n' # get kind of data kindat = self.getKindatByFilename(filename) kindatName = madKindatObj.getKindatDescription(kindat, kinst) retStr = retStr + '\tKind of data: ' + str(kindatName) + '\n' return retStr
def getStatusByFilename(
self, filename)
getStatusByFilename returns the status description of the file with the given name.
Inputs: filename - name of file to search for.
Returns: the status description of the file with the given name. Returns none if name not found
Affects: None
Exceptions: Thrown if filename not found.
def getStatusByFilename(self,filename): """getStatusByFilename returns the status description of the file with the given name. Inputs: filename - name of file to search for. Returns: the status description of the file with the given name. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.__fileList[position][self.__fileStatusCol])
def getStatusByPosition(
self, position=0)
getStatusByPosition returns the status description of the file at given position.
Inputs: position of file in list (first position is zero). Defaults to first.
Returns: the status description of the file at given position as a string.
Returns None if position >= number of experiments.
Affects: None
Exceptions: None
def getStatusByPosition(self, position = 0): """getStatusByPosition returns the status description of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the status description of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileStatusCol] else: return(None)
def setAccess(
self, accessType)
setAccess sets the access column to all 0's (public) or all 1's (private).
Inputs: accessType - either 0 to set to public access, or 1 to set to private access.
Returns: None.
Affects: Overwrite fileTab.txt file with access column set to all 0's (public) or all 1's (private).
Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1
def setAccess(self, accessType): """setAccess sets the access column to all 0's (public) or all 1's (private). Inputs: accessType - either 0 to set to public access, or 1 to set to private access. Returns: None. Affects: Overwrite fileTab.txt file with access column set to all 0's (public) or all 1's (private). Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1 """ if (accessType != 0 and accessType != 1): raise madrigal.admin.MadrigalError('MadrigalMetaFile.setAccess called with arg = ' + \ str(accessType) + ', must be either 0 or 1', None) for row in self.__fileList: row[self.__fileAccessCol] = str(accessType) self.writeMetadata()
def setAccessByPosition(
self, position, access)
setAccessByPosition sets the value of access for the file at the given position.
Inputs:
position - position of file in list (first position is zero). access - 0 of False for public, 1 or True for private
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setAccessByPosition(self, position, access): """setAccessByPosition sets the value of access for the file at the given position. Inputs: position - position of file in list (first position is zero). access - 0 of False for public, 1 or True for private Returns: None. Affects: None Exceptions: If position beyond length. """ if access not in (0, 1, True, False): raise ValueError, 'Illegal value for access in setAccessByPosition: %s' % (str(access)) if len(self.__fileList) > position: if access in (1, True): self.__fileList[position][self.__fileAccessCol] = '1' else: self.__fileList[position][self.__fileAccessCol] = '0' else: raise ValueError, 'setAccessByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def setAnalystByPosition(
self, position, analyst)
setAnalystByPosition sets the file analyst name for the given position
Inputs:
position - position of file in list (first position is zero). analyst - name of file analyst
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setAnalystByPosition(self, position, analyst): """setAnalystByPosition sets the file analyst name for the given position Inputs: position - position of file in list (first position is zero). analyst - name of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analyst.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystByPosition with args %s: %s' % \ (str(position), analyst), [traceback.format_exc()]) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol+1: # append two extra columns self.__fileList[position].append(str(analyst)) self.__fileList[position].append('') else: self.__fileList[position][self.__fileAnalystCol] = str(analyst) else: raise ValueError, 'setAnalystByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def setAnalystEmailByPosition(
self, position, analystEmail)
setAnalystEmailByPosition sets the file analyst email for the given position
Inputs:
position - position of file in list (first position is zero). analystEmail - email of file analyst
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setAnalystEmailByPosition(self, position, analystEmail): """setAnalystEmailByPosition sets the file analyst email for the given position Inputs: position - position of file in list (first position is zero). analystEmail - email of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analystEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystEmailByPosition with args %s: %s' % \ (str(position, analystEmail), [traceback.format_exc()])) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol + 1: # append two extra columns self.__fileList[position].append('') self.__fileList[position].append(str(analystEmail)) else: self.__fileList[position][self.__fileAnalystEmailCol] = str(analystEmail) else: raise ValueError, 'setAnalystEmailByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def setCategoryByPosition(
self, position, category)
setCategoryByPosition sets the value of category for the file at the given position.
Inputs:
position - position of file in list (first position is zero). category - 1=default, 2=variant, 3=history, 4=real-time
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setCategoryByPosition(self, position, category): """setCategoryByPosition sets the value of category for the file at the given position. Inputs: position - position of file in list (first position is zero). category - 1=default, 2=variant, 3=history, 4=real-time Returns: None. Affects: None Exceptions: If position beyond length. """ if int(category) not in (1, 2, 3, 4): raise ValueError, 'Illegal value for category in setCategoryByPosition: %s' % (str(category)) if len(self.__fileList) > position: self.__fileList[position][self.__fileCategoryCol] = str(category) else: raise ValueError, 'setCategoryByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def setExpIdByPosition(
self, position, expId)
setExpIdByPosition sets the experiment id of the file at given position.
Inputs:
position - position of file in list (first position is zero). expId - the new experiment id to use
Returns: None.
Affects: sets the experiment id of the file at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.
def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the file at given position. Inputs: position - position of file in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the file at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__fileExpIdCol] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str(position), str(expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
def setFileDatetimeByPosition(
self, position, dt)
setFileDatetimeByPosition sets the file datetime for the given position
Inputs:
position - position of file in list (first position is zero). dt - datetime (UT) of file being added. If None, use file datetime. Will raise exception if dt is None and fileTab.txt in the metadata version, not the expDir version.
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setFileDatetimeByPosition(self, position, dt): """setFileDatetimeByPosition sets the file datetime for the given position Inputs: position - position of file in list (first position is zero). dt - datetime (UT) of file being added. If None, use file datetime. Will raise exception if dt is None and fileTab.txt in the metadata version, not the expDir version. Returns: None. Affects: None Exceptions: If position beyond length. """ if dt is None: basename = self.getFilenameByPosition(position) expDir = os.path.dirname(self.__filename) if len(expDir) < 2: raise IOError('Cannot set datetime to None using the metadata version of fileTab.txt') dataFile = os.path.join(expDir, basename) if not os.access(dataFile, os.R_OK): raise IOError('Cannot access %s' % (dataFile)) dt = datetime.datetime.utcfromtimestamp(os.path.getmtime(dataFile)) dateStr = dt.strftime('%Y%m%d') timeStr = dt.strftime('%H%M%S') if len(self.__fileList) > position: self.__fileList[position][self.__fileModDateCol] = dateStr self.__fileList[position][self.__fileModTimeCol] = timeStr else: raise ValueError('setFileDatetimeByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))
def setHasCatalogByPosition(
self, position, hasCatalog)
setHasCatalogByPosition sets the value of hasCatalog for the file at the given position.
Inputs:
position - position of file in list (first position is zero). hasCatalog - 1 or True for yes, 0 or False for no
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setHasCatalogByPosition(self, position, hasCatalog): """setHasCatalogByPosition sets the value of hasCatalog for the file at the given position. Inputs: position - position of file in list (first position is zero). hasCatalog - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasCatalog not in (0, 1, True, False): raise ValueError, 'Illegal value for hasCatalog in setHasCatalogByPosition: %s' % (str(hasCatalog)) if len(self.__fileList) > position: if hasCatalog in (1, True): self.__fileList[position][self.__fileHasCatalogCol] = '1' else: self.__fileList[position][self.__fileHasCatalogCol] = '0' else: raise ValueError, 'setHasCatalogByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def setHasHeaderByPosition(
self, position, hasHeader)
setHasHeaderByPosition sets the value of hasHeader for the file at the given position.
Inputs:
position - position of file in list (first position is zero). hasHeader - 1 or True for yes, 0 or False for no
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setHasHeaderByPosition(self, position, hasHeader): """setHasHeaderByPosition sets the value of hasHeader for the file at the given position. Inputs: position - position of file in list (first position is zero). hasHeader - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasHeader not in (0, 1, True, False): raise ValueError, 'Illegal value for hasHeader in setHasHeaderByPosition: %s' % (str(hasHeader)) if len(self.__fileList) > position: if hasHeader in (1, True): self.__fileList[position][self.__fileHasHeaderCol] = '1' else: self.__fileList[position][self.__fileHasHeaderCol] = '0' else: raise ValueError, 'setHasHeaderByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def setKindatByPosition(
self, position, kindat)
setKindatByPosition sets the value of kindat for the file at the given position.
Inputs:
position - position of file in list (first position is zero). kindat - integer kindat value
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setKindatByPosition(self, position, kindat): """setKindatByPosition sets the value of kindat for the file at the given position. Inputs: position - position of file in list (first position is zero). kindat - integer kindat value Returns: None. Affects: None Exceptions: If position beyond length. """ kindat = int(kindat) if len(self.__fileList) > position: self.__fileList[position][self.__fileKindatCol] = str(kindat) else: raise ValueError, 'setKindatByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def setStatusByPosition(
self, position, status)
setStatusByPosition sets the value of status string for the file at the given position.
Inputs:
position - position of file in list (first position is zero). status - string describing status
Returns: None.
Affects: None
Exceptions: If position beyond length.
def setStatusByPosition(self, position, status): """setStatusByPosition sets the value of status string for the file at the given position. Inputs: position - position of file in list (first position is zero). status - string describing status Returns: None. Affects: None Exceptions: If position beyond length. """ if type(status) != types.StringType: raise ValueError, 'Illegal value for status in setStatusByPosition: %s' % (str(status)) # check that string does not illegally contain a comma if status.find(',') != -1: raise ValueError, 'status string in fileTab.txt cannot contain a comma: <%s> is illegal' % (status) if len(self.__fileList) > position: self.__fileList[position][self.__fileStatusCol] = status else: raise ValueError, 'setStatusByPosition called for position %i beyond length %i' % (position, len(self.__fileList))
def writeMetadata(
self, newFullPath=None)
writeMetadata writes a new version of the fileTab.txt file.
Inputs: newFullPath: a new path to write the fileTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from.
Returns: None.
Affects: Writes updated version of metadata file.
Exceptions: If unable to write file
def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the fileTab.txt file. Inputs: newFullPath: a new path to write the fileTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
class MadrigalMetadata
MadrigalMetadata is a private class that parses a Madrigal metadata file.
This private class is used by all classes that need to parse a Madrigal metadata file. If the class is called with the name of the metadata file only, the metadata file is assumed to be at $MAD_ROOT/metadata. If a full path name is given that includes a directory separator, then that is used instead. The getList method returns a list with one item for each line in the file. That item for each line is simply a list of strings found in the line. The following is an example metadata file and the list the method getList would return.
Metadata file example::
Tom, Dick,,Harry ,,Joe, Sally, Jane,Joe, Dick
The list returned by getList example::
[['Tom', 'Dick', '', 'Harry'], ['', '', 'Joe', ''], ['Sally', 'Jane', 'Joe', 'Dick']]
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. Unable to open metadata file. 2. All lines in metadata file do not have same number of items
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 8, 2001
class MadrigalMetadata: """MadrigalMetadata is a private class that parses a Madrigal metadata file. This private class is used by all classes that need to parse a Madrigal metadata file. If the class is called with the name of the metadata file only, the metadata file is assumed to be at $MAD_ROOT/metadata. If a full path name is given that includes a directory separator, then that is used instead. The getList method returns a list with one item for each line in the file. That item for each line is simply a list of strings found in the line. The following is an example metadata file and the list the method getList would return. Metadata file example:: Tom, Dick,,Harry ,,Joe, Sally, Jane,Joe, Dick The list returned by getList example:: [['Tom', 'Dick', '', 'Harry'], ['', '', 'Joe', ''], ['Sally', 'Jane', 'Joe', 'Dick']] Non-standard Python modules used: None MadrigalError exception thrown if: 1. Unable to open metadata file. 2. All lines in metadata file do not have same number of items Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 8, 2001 """ # constants # file delimiter - presently a comma __DELIMITER = ',' def __init__(self, metadataFileName, madDB=None, allowedLenList=None, key=None): """__init__ initializes the MadrigalCategoryList by reading from __privateList. Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir is included. Existing MadrigalDB object, by default = None. allowedLenList - a list of allowed lengths (integers) for data lines. If None (the default), then the rule is that all lengths must be the same. key - which position in the file to use as a key. Creates a attribute called self._dict, which is a dictionary with keys = item in key position, value = line index. Used for faster lookup by key. If None (the default), self._dict = None Returns: void Affects: Initializes private member variable __categoryList. Exceptions: None. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # create empty list to hold parsed data self.__fileList = [] # used to check that every line has same number of words if allowedLenList == None self.__numWords = 0 self._allowedLenList = allowedLenList if not key is None: self._dict = {} else:self._dict = None # get real filename self.__fileName = self.__getFullPath(metadataFileName) # open configuration file try: self.__file = open(self.__fileName, "r") except IOError: raise madrigal.admin.MadrigalError("Unable to open metadata file " + self.__fileName, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # loop over each line in file, creating a list for each line = self.__file.readline() count = 0 while (len(line.strip())): self.__parseLine(line, key, count) line = self.__file.readline() count += 1 # close metadata file self.__file.close() def __parseLine(self, line, key=None, count=None): """__parseLine adds a list of items in line to __fileList. Inputs: Line of file to parse (String). key - position to add key for self._dict, value = line position If None (the default), ignore count - if None, ignore. Else add to self._dict as value (see key above) Returns: void Affects: Adds a list of items in line to self.__fileList. Exceptions: None. """ list = line.split(self.__DELIMITER) # create new list with leading and trailing whitespace stripped strippedList = [] for word in list: strippedList.append(word.strip()) self.__fileList.append(strippedList) if not self._dict is None: self._dict[list[key]] = count # check correct number of words if self._allowedLenList: if len(strippedList) not in self._allowedLenList: raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \ self.__fileName + " at line " + \ str(len(self.__fileList)), [traceback.format_exc()]) elif self.__numWords == 0: self.__numWords = len(strippedList) elif self.__numWords != len(strippedList): raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \ self.__fileName + " at line " + \ str(len(self.__fileList)), [traceback.format_exc()]) def __getFullPath(self, filename): """getFullPath returns the full path name of the metafile. Inputs: filename passed in as argument Returns: The full path name of the metafile. If the filename argument already is a full path, filename is returned unchanged. If the filename is simply the name of a metadata file, then $MAD_ROOT/metadata is appended Affects: Nothing Exceptions: None """ if (len(os.path.dirname(filename)) != 0): return filename fullName = self.__madDB.getMetadataDir() + "/" + filename # normalize in case we run on a system without / as a separator return os.path.normpath(fullName) # public methods def getList(self): """getList returns the list of lists of items in each line in the metafile. Inputs: None Returns: The list of lists of items in each line in the metafile. That is, each item in the returned list is itself a list, representing a single line in the metafile. That single line's list is the list of items in that line. Affects: Nothing Exceptions: None """ return(self.__fileList) def getDict(self): """getDict returns self_dict, which will be a dict with keys= key column set, value = line number, of None if no key passed into constructor """ return(self._dict) def toString(self): """toString returns a simple string representation of a MadrigalMetadata object. Inputs: None Returns: String describing a simple representation of a __MadrigalMetadat object. Affects: Nothing Exceptions: None """ return str(self.__fileList)
Ancestors (in MRO)
Methods
def __init__(
self, metadataFileName, madDB=None, allowedLenList=None, key=None)
init initializes the MadrigalCategoryList by reading from __privateList.
Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir is included.
Existing MadrigalDB object, by default = None. allowedLenList - a list of allowed lengths (integers) for data lines. If None (the default), then the rule is that all lengths must be the same. key - which position in the file to use as a key. Creates a attribute called self._dict, which is a dictionary with keys = item in key position, value = line index. Used for faster lookup by key. If None (the default), self._dict = None
Returns: void
Affects: Initializes private member variable __categoryList.
Exceptions: None.
def __init__(self, metadataFileName, madDB=None, allowedLenList=None, key=None): """__init__ initializes the MadrigalCategoryList by reading from __privateList. Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir is included. Existing MadrigalDB object, by default = None. allowedLenList - a list of allowed lengths (integers) for data lines. If None (the default), then the rule is that all lengths must be the same. key - which position in the file to use as a key. Creates a attribute called self._dict, which is a dictionary with keys = item in key position, value = line index. Used for faster lookup by key. If None (the default), self._dict = None Returns: void Affects: Initializes private member variable __categoryList. Exceptions: None. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # create empty list to hold parsed data self.__fileList = [] # used to check that every line has same number of words if allowedLenList == None self.__numWords = 0 self._allowedLenList = allowedLenList if not key is None: self._dict = {} else:self._dict = None # get real filename self.__fileName = self.__getFullPath(metadataFileName) # open configuration file try: self.__file = open(self.__fileName, "r") except IOError: raise madrigal.admin.MadrigalError("Unable to open metadata file " + self.__fileName, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # loop over each line in file, creating a list for each line = self.__file.readline() count = 0 while (len(line.strip())): self.__parseLine(line, key, count) line = self.__file.readline() count += 1 # close metadata file self.__file.close()
def getDict(
self)
getDict returns self_dict, which will be a dict with keys= key column set, value = line number, of None if no key passed into constructor
def getDict(self): """getDict returns self_dict, which will be a dict with keys= key column set, value = line number, of None if no key passed into constructor """ return(self._dict)
def getList(
self)
getList returns the list of lists of items in each line in the metafile.
Inputs: None
Returns: The list of lists of items in each line in the metafile. That is, each item in the returned list is itself a list, representing a single line in the metafile. That single line's list is the list of items in that line.
Affects: Nothing
Exceptions: None
def getList(self): """getList returns the list of lists of items in each line in the metafile. Inputs: None Returns: The list of lists of items in each line in the metafile. That is, each item in the returned list is itself a list, representing a single line in the metafile. That single line's list is the list of items in that line. Affects: Nothing Exceptions: None """ return(self.__fileList)
def toString(
self)
toString returns a simple string representation of a MadrigalMetadata object.
Inputs: None
Returns: String describing a simple representation of a __MadrigalMetadat object.
Affects: Nothing
Exceptions: None
def toString(self): """toString returns a simple string representation of a MadrigalMetadata object. Inputs: None Returns: String describing a simple representation of a __MadrigalMetadat object. Affects: Nothing Exceptions: None """ return str(self.__fileList)
class MadrigalParmCategory
MadrigalParmCategory is an object that provides access to Madrigal parameter category info from the metadata.
This object provides access to all Madrigal kind of data information in the metadata file madCatTab.txt.
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jan. 18, 2013
class MadrigalParmCategory: """MadrigalParmCategory is an object that provides access to Madrigal parameter category info from the metadata. This object provides access to all Madrigal kind of data information in the metadata file madCatTab.txt. Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jan. 18, 2013 """ #constants _categoryMetadataFile = "madCatTab.txt" # column positions _categoryCodeCol = 0 _categoryDescCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get category metadata file if (initFile == None): self.__filename = self._categoryMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getCategoryDesc(self, code): """getCategoryDesc returns the category description that matches code argument, or None if not found. Inputs: code integer to get category description. or integer as string Returns: the category description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) for i, category in enumerate(self.__fileList): # find matching code try: if code == int(category[self._categoryCodeCol]): return(category[self._categoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in madCatTab.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getCategoryList(self): """getCategoryList returns a list of all category descriptions and codes. Inputs: None. Returns: a list of all category descriptions and codes. Each item in the list is a tuple of the form (Category description (string), code (integer)). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, category in enumerate(self.__fileList): item = (category[self._categoryDescCol], int(category[self._categoryCodeCol])) retList.append(item) return(retList)
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None)
init initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile).
Inputs: Existing MadrigalDB object, by default = None.
String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.
def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get category metadata file if (initFile == None): self.__filename = self._categoryMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()
def getCategoryDesc(
self, code)
getCategoryDesc returns the category description that matches code argument, or None if not found.
Inputs:
code integer to get category description. or integer as string
Returns: the category description that matches code argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getCategoryDesc(self, code): """getCategoryDesc returns the category description that matches code argument, or None if not found. Inputs: code integer to get category description. or integer as string Returns: the category description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) for i, category in enumerate(self.__fileList): # find matching code try: if code == int(category[self._categoryCodeCol]): return(category[self._categoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in madCatTab.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getCategoryList(
self)
getCategoryList returns a list of all category descriptions and codes.
Inputs: None.
Returns: a list of all category descriptions and codes. Each item in the list is a tuple of the form (Category description (string), code (integer)). Example item: ('INSCAL Basic Derived Parameters', 3001)
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getCategoryList(self): """getCategoryList returns a list of all category descriptions and codes. Inputs: None. Returns: a list of all category descriptions and codes. Each item in the list is a tuple of the form (Category description (string), code (integer)). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, category in enumerate(self.__fileList): item = (category[self._categoryDescCol], int(category[self._categoryCodeCol])) retList.append(item) return(retList)
class MadrigalSite
MadrigalSite is an object that provides access to Madrigal site info from the metadata.
This object provides access to all Madrigal site information in the metadata file siteTab.txt.
Usage example::
import madrigal.metadata import madrigal.admin try: siteObj = madrigal.metadata.MadrigalSite() print siteObj.getSiteName(1) except madrigal.admin.MadrigalError, e: print e.getExceptionStr()
Non-standard Python modules used: None
MadrigalError exception thrown if:
1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted
Change history:
Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001
class MadrigalSite: """MadrigalSite is an object that provides access to Madrigal site info from the metadata. This object provides access to all Madrigal site information in the metadata file siteTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: siteObj = madrigal.metadata.MadrigalSite() print siteObj.getSiteName(1) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __siteMetadataFile = "siteTab.txt" _defaultVersion = '2.6' # column positions __siteIDCol = 0 __siteNameCol = 1 __madServerCol = 2 __madDocRootCol = 3 __madCGICol = 4 __madServletCol = 5 __contactNameCol = 6 __contactAddr1Col = 7 __contactAddr2Col = 8 __contactAddr3Col = 9 __contactCityCol = 10 __contactStateCol = 11 __contactZipCol = 12 __contactCountryCol = 13 __contactPhoneCol = 14 __contactEmailCol = 15 __siteVersionCol = 16 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalSite by reading from siteTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/siteTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get site metadata file if (initFile == None): self.__filename = self.__siteMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without version allowedLens = (self.__contactEmailCol+1, self.__siteVersionCol+1) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens).getList() def getSiteName(self, siteID): """getSiteName returns the site name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__siteNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteServer(self, siteID): """getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found. Inputs: siteID integer to get site server. Returns: the site server that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madServerCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteDocRoot(self, siteID): """getSiteDocRoot returns the relative document root (e.g. madrigal) that matches siteID argument, or None if not found. Inputs: siteID integer to get document root path. Returns: the document root path that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madDocRootCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteRelativeCGI(self, siteID): """getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal) that matches siteID argument, or None if not found. For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored. Inputs: siteID integer to get relative cgi path. Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ version = self.getSiteVersion(siteID) if distutils.version.LooseVersion(version) >= distutils.version.LooseVersion('3.0'): return(self.getSiteDocRoot(siteID)) # this code only runs for Madrigal 2.0 for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madCGICol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return None def getSiteContactName(self, siteID): """getSiteContactName returns the site contact name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress1(self, siteID): """getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 1 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr1Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress2(self, siteID): """getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 2 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr2Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress3(self, siteID): """getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 3 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr3Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteCity(self, siteID): """getSiteCity returns the site city that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact city that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCityCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteState(self, siteID): """getSiteState returns the site state that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact state that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactStateCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSitePostalCode(self, siteID): """getSitePostalCode returns the site postal code that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact postal code that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactZipCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteCountry(self, siteID): """getSiteCountry returns the site country that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact country that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCountryCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteTelephone(self, siteID): """getSiteTelephone returns the site telephone that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact telephone that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactPhoneCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteEmail(self, siteID): """getSiteEmail returns the site email address that matches siteID argument, or None if not found. Inputs: siteID integer to get Site email address. Returns: the site email address that matches siteID argument, or None if not found. To list multiple email addresses in this field separate them with semicolons (since commas are delimiters). getSiteEmail will automatically replace semicolons with commas, as required by multiple email addresses. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactEmailCol].replace(';', ',') except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteList(self): """getSiteList returns a list of all site ids and names. Inputs: None. Returns: a list of all site ids and names. Each item in the list is a tuple of the form (Site id (integer), site name (string)). Example item: (1,'Millstone') Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for site in self.__fileList: try: item = (int(site[self.__siteIDCol]), site[self.__siteNameCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in siteTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList def getSiteVersion(self, siteID): """getSiteVersion returns the site version string that matches siteID argument, or None if not found. Inputs: siteID integer Returns: the site version that matches siteID argument, or None if not found. If file does not have this field, returns default value of 2.6. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): if len(site) >= self.__siteVersionCol + 1: return site[self.__siteVersionCol] else: return(self._defaultVersion) except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def setSiteVersionBySiteID(self, siteID, version): """setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers at given siteID. Inputs: siteID - siteID of site in list. version - string Madrigal version as period delimited integers (eg, 2.6) Returns: None. Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or siteID not found. This method added in Madrigal 3.0 """ # verify no illegal commas if version.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) # parse test version = version.strip() try: siteID = int(siteID) if len(version) == 0: raise ValueError, '' for item in version.split('.'): int(item) except: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) rightPosition = None for position, site in enumerate(self.__fileList): # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): rightPosition = position except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if rightPosition is None: raise madrigal.admin.MadrigalError('siteId %i not found in siteTab.txt' % (siteID)) if len(self.__fileList[position]) == self.__siteVersionCol: # not yet set self.__fileList[position].append(version) else: self.__fileList[position][self.__siteVersionCol] = version def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the siteTab.txt file. Inputs: newFullPath: a new path to write the siteTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
Ancestors (in MRO)
Methods
def __init__(
self, madDB=None, initFile=None)
init initializes MadrigalSite by reading from siteTab.txt (or initFile).
Inputs: Existing MadrigalDB object, by default = None.
String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/siteTab.txt.
Returns: void
Affects: Initializes all the class member variables.
Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.
def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalSite by reading from siteTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/siteTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get site metadata file if (initFile == None): self.__filename = self.__siteMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without version allowedLens = (self.__contactEmailCol+1, self.__siteVersionCol+1) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens).getList()
def getSiteAddress1(
self, siteID)
getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact address 1 that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteAddress1(self, siteID): """getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 1 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr1Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteAddress2(
self, siteID)
getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact address 2 that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteAddress2(self, siteID): """getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 2 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr2Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteAddress3(
self, siteID)
getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact address 3 that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteAddress3(self, siteID): """getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 3 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr3Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteCity(
self, siteID)
getSiteCity returns the site city that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact city that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteCity(self, siteID): """getSiteCity returns the site city that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact city that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCityCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteContactName(
self, siteID)
getSiteContactName returns the site contact name that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact name that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteContactName(self, siteID): """getSiteContactName returns the site contact name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteCountry(
self, siteID)
getSiteCountry returns the site country that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact country that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteCountry(self, siteID): """getSiteCountry returns the site country that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact country that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCountryCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteDocRoot(
self, siteID)
getSiteDocRoot returns the relative document root (e.g. madrigal) that matches siteID argument, or None if not found.
Inputs: siteID integer to get document root path.
Returns: the document root path that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteDocRoot(self, siteID): """getSiteDocRoot returns the relative document root (e.g. madrigal) that matches siteID argument, or None if not found. Inputs: siteID integer to get document root path. Returns: the document root path that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madDocRootCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteEmail(
self, siteID)
getSiteEmail returns the site email address that matches siteID argument, or None if not found.
Inputs: siteID integer to get Site email address.
Returns: the site email address that matches siteID argument, or None if not found. To list multiple email addresses in this field separate them with semicolons (since commas are delimiters). getSiteEmail will automatically replace semicolons with commas, as required by multiple email addresses.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteEmail(self, siteID): """getSiteEmail returns the site email address that matches siteID argument, or None if not found. Inputs: siteID integer to get Site email address. Returns: the site email address that matches siteID argument, or None if not found. To list multiple email addresses in this field separate them with semicolons (since commas are delimiters). getSiteEmail will automatically replace semicolons with commas, as required by multiple email addresses. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactEmailCol].replace(';', ',') except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteList(
self)
getSiteList returns a list of all site ids and names.
Inputs: None.
Returns: a list of all site ids and names. Each item in the list is a tuple of the form (Site id (integer), site name (string)). Example item: (1,'Millstone')
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteList(self): """getSiteList returns a list of all site ids and names. Inputs: None. Returns: a list of all site ids and names. Each item in the list is a tuple of the form (Site id (integer), site name (string)). Example item: (1,'Millstone') Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for site in self.__fileList: try: item = (int(site[self.__siteIDCol]), site[self.__siteNameCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in siteTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList
def getSiteName(
self, siteID)
getSiteName returns the site name that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site name that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteName(self, siteID): """getSiteName returns the site name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__siteNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSitePostalCode(
self, siteID)
getSitePostalCode returns the site postal code that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact postal code that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSitePostalCode(self, siteID): """getSitePostalCode returns the site postal code that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact postal code that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactZipCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteRelativeCGI(
self, siteID)
getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal) that matches siteID argument, or None if not found.
For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored.
Inputs: siteID integer to get relative cgi path.
Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteRelativeCGI(self, siteID): """getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal) that matches siteID argument, or None if not found. For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored. Inputs: siteID integer to get relative cgi path. Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ version = self.getSiteVersion(siteID) if distutils.version.LooseVersion(version) >= distutils.version.LooseVersion('3.0'): return(self.getSiteDocRoot(siteID)) # this code only runs for Madrigal 2.0 for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madCGICol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return None
def getSiteServer(
self, siteID)
getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found.
Inputs: siteID integer to get site server.
Returns: the site server that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteServer(self, siteID): """getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found. Inputs: siteID integer to get site server. Returns: the site server that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madServerCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteState(
self, siteID)
getSiteState returns the site state that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact state that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteState(self, siteID): """getSiteState returns the site state that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact state that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactStateCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteTelephone(
self, siteID)
getSiteTelephone returns the site telephone that matches siteID argument, or None if not found.
Inputs: siteID integer to get SiteName.
Returns: the site contact telephone that matches siteID argument, or None if not found.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteTelephone(self, siteID): """getSiteTelephone returns the site telephone that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact telephone that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactPhoneCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def getSiteVersion(
self, siteID)
getSiteVersion returns the site version string that matches siteID argument, or None if not found.
Inputs: siteID integer
Returns: the site version that matches siteID argument, or None if not found. If file does not have this field, returns default value of 2.6.
Affects: None
Exceptions: MadrigalError if any item in row cannot be cast to correct format
def getSiteVersion(self, siteID): """getSiteVersion returns the site version string that matches siteID argument, or None if not found. Inputs: siteID integer Returns: the site version that matches siteID argument, or None if not found. If file does not have this field, returns default value of 2.6. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): if len(site) >= self.__siteVersionCol + 1: return site[self.__siteVersionCol] else: return(self._defaultVersion) except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None)
def setSiteVersionBySiteID(
self, siteID, version)
setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers at given siteID.
Inputs:
siteID - siteID of site in list. version - string Madrigal version as period delimited integers (eg, 2.6)
Returns: None.
Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position
Exceptions: MadrigalError if any item in row cannot be cast to correct format, or siteID not found.
This method added in Madrigal 3.0
def setSiteVersionBySiteID(self, siteID, version): """setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers at given siteID. Inputs: siteID - siteID of site in list. version - string Madrigal version as period delimited integers (eg, 2.6) Returns: None. Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or siteID not found. This method added in Madrigal 3.0 """ # verify no illegal commas if version.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) # parse test version = version.strip() try: siteID = int(siteID) if len(version) == 0: raise ValueError, '' for item in version.split('.'): int(item) except: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) rightPosition = None for position, site in enumerate(self.__fileList): # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): rightPosition = position except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if rightPosition is None: raise madrigal.admin.MadrigalError('siteId %i not found in siteTab.txt' % (siteID)) if len(self.__fileList[position]) == self.__siteVersionCol: # not yet set self.__fileList[position].append(version) else: self.__fileList[position][self.__siteVersionCol] = version
def writeMetadata(
self, newFullPath=None)
writeMetadata writes a new version of the siteTab.txt file.
Inputs: newFullPath: a new path to write the siteTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from.
Returns: None.
Affects: Writes updated version of metadata file.
Exceptions: If unable to write file
def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the siteTab.txt file. Inputs: newFullPath: a new path to write the siteTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))