Previous: madrigal.isprint   Up: Internal Madrigal Python API   Next: madrigal.openmadrigal

Top

madrigal.metadata module

The metadata module provides access to all metadata about one particular madrigal database.

This metadata is presently read from the files in the metadata directory, and from the madrigal.cfg file. If that file cannot be found, hard-coded values set during installation are used instead. If the madrigal.cfg file is found at the location specified by either the madroot enviroment variable or at the hard-coded value of madroot set at installation, then the parameters are read from the madrigal.cfg file. Note that madroot with caps is only written once in this file before installation, since it will be automatically replaced, so it is referred to by MAD_ROOT or MAD+ROOT.

$Id: metadata_original.py 7044 2019-10-07 19:13:16Z brideout $

# -*- coding: utf-8 -*-

"""The metadata module provides access to all metadata about one particular madrigal database.

This metadata is presently read from the files in the metadata directory, and from the
madrigal.cfg file.  If that file cannot be found, hard-coded values set during installation
are used instead.  If the madrigal.cfg file is found at the location specified by either
the madroot enviroment variable or at the hard-coded value of madroot set at installation, then
the parameters are read from the madrigal.cfg file.  Note that madroot with caps is only written
once in this file before installation, since it will be automatically replaced, so it is referred
to by MAD_ROOT or MAD+ROOT.

$Id: metadata_original.py 7044 2019-10-07 19:13:16Z brideout $
"""
# standard python imports
import io
import configparser
import os, sys, traceback
import os.path
import shutil
import time
import datetime
import calendar
import fnmatch
import types
import re
import random
import copy
import glob
import distutils.version

# Madrigal imports
import madrigal.admin
import madrigal._derive


# helper functions
def getMadrigalUTFromDT(dt):
    """getMadrigalUTFromDT returns the number of seconds since midnight UT 1950-01-01 
    for datetime dt
    """
    t1950 = 631152000.0 # offset from Unix time
    return(calendar.timegm(dt.timetuple()) + t1950)

def getMadrigalUTFromDate(year, month, day, hour, minute, second, microsecond):
    """getMadrigalUTFromDate returns the number of seconds since midnight UT 1950-01-01 
    for date specified in arguments
    """
    dt = datetime.datetime(year, month, day, hour, minute, second, microsecond)
    return(getMadrigalUTFromDT(dt))

def getUnixUTFromDT(dt):
    """getUnixUTFromDT returns the number of float seconds since midnight UT 1970-01-01 
    for datetime dt
    """
    return(calendar.timegm(dt.timetuple()) + dt.microsecond/1.0E6)

def getUnixUTFromDate(year, month, day, hour, minute, second, microsecond):
    """getUnixUTFromDate returns the number of seconds since midnight UT 1970-01-01 
    for date specified in arguments
    """
    dt = datetime.datetime(year, month, day, hour, minute, second, microsecond)
    return(getUnixUTFromDT(dt))


class MadrigalDB:
    """MadrigalDB is an object that provides access to an entire Madrigal database.

    This object provides complete high-level access to an entire Madrigal database.
    Presently, all its information comes from madrigal.cfg, or another path passed in by the
    user.  If env variable madroot is not set, or if madrigal.cfg cannot be opened, the
    default values automatically editted during installation are used

    Usage example::

        import madrigal.metadata
    
        try:
        
            test =  madrigal.metadata.MadrigalDB()
            
        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()
            
        else:
        
            print test.toString()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1.  If the constructor is called with an configuration file path as an argument, and that file cannot be
            read.
        
        2.  If the configuration file cannot be parsed by ConfigParser.
    
        3.  If any of the following keys cannot be found in the configuration file:
    
            *__MAD_SERVER

            *__MAD_SERVERROOT
            
            *__SITE_ID

            *__HTML_STYLE

            *__INDEX_HEAD

            *__CON_TACTLINK


    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Oct. 4, 2001

    """

    # lines that are edited during installation
    __hardCodeMadRoot          = '/Volumes/dropbox/mad31'
    __hardCodeMadServer        = 'localhost:8000'
    __hardCodeMadServerRoot    = '.'
    __hardCodeSiteId           = '999'
    __hardCodeHtmlStyle        = ''
    __hardCodeIndexHead        = 'Welcome to the Madrigal Database 
at Dropbox' __hardCodeContact = 'Madrigal administator
' __hardCodeMailServer = 'localhost' #constants __webProtocol = "http" """ Change the above string to use another web protocol such as https. """ __binDir = "/bin" """ Sets the relative path from madrigal root to the bin directory. """ __metadataDir = "/metadata" """ Sets the relative path from madrigal root to the metadata directory. """ __experimentDir = "/experiments" """ Sets the relative path from madrigal root to the experiment directory. """ __MAD_ROOT = "MAD" + "ROOT" """ Sets the name of the environment variable that points to the madrigal root directory. """ __confFileName = "madrigal.cfg" """ Sets the name of the default madrigal configuration file. """ __MAD_SERVER = "MAD" + "SERVER" """ Sets the key name in the configuration file to find the main madrigal url. """ __MAD_SERVERROOT = "MAD" + "SERVERROOT" """ Sets the key name in the configuration file to find the top level directory. """ __SITE_ID = "SITE" + "ID" """ Sets the key name in the configuration file to find the site id. """ __HTML_STYLE = "HTML" + "STYLE" """ Sets the key name in the configuration file to find the html body style tag. """ __INDEX_HEAD = "INDEX" + "HEAD" """ Sets the key name in the configuration file to find the heading in the top level madrigal page. """ __CON_TACTLINK = "CON" + "TACT" """ Sets the key name in the configuration file to find the contact link line. """ __MAIL_SERVER = "MAIL" + "SERVER" """ Sets the key name in the configuration file to find the mailserver. """ __PYTHON_EXE = "PYTHON" + "EXE" """ Sets the key name in the configuration file to find the python executable. """ def __init__(self, initFile=None): """__init__ initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString). Inputs: String representing the full path to the configuration file. Default is None, in which case configuration file is $(__MAD_ROOT)/__confFileName (now madrigal.cfg). Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown if error (see class description for specific conditions). Notes: Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile, this constructor goes about loading some basic data about the database, including information such as: -the database utility directory -the www home base Implemented using ConfigParser module. This reads ini type files. The only difference between ini files and madrigal.cfg is that ini files are broken into sections of the form: [sectionTitle] key1 = value1 key2 = value2 Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal" is appended to the beginning of the file. """ # if madroot not set, use hard coded madroot self.__madRootDir = os.environ.get(self.__MAD_ROOT) if (self.__madRootDir == None): self.__madRootDir = self.__hardCodeMadRoot # Set configuration file if (initFile == None): self.__confFilePath = self.__madRootDir + "/" + self.__confFileName else: self.__confFilePath = initFile # open configuration file try: self.__confFile = open(self.__confFilePath, "r") except IOError: # can't read from file - use all hard-coded values self.__initFromHardCode() self.__finishInit() return # create Parser using standard module ConfigParser self.__parser = configparser.ConfigParser() # read conf file into a StringIO with "[madrigal]\n" section heading prepended strConfFile = io.StringIO("[madrigal]\n" + self.__confFile.read()) # parse StringIO configuration file try: self.__parser.readfp(strConfFile) except: raise madrigal.admin.MadrigalError("Unable to parse configuration file " + self.__confFilePath, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # read information from configuration file self.__readConfFile() # close conf file self.__confFile.close() self.__finishInit() # end __init__ def __finishInit(self): """__finishInit is a private helper function that finishes initialization by combining attributes to form new ones. Inputs: None Returns: Void. Affects: Initializes class member variables that are combinations of existing ones. Exceptions: None. """ # combine information # Set the database utility directory self.__databaseUtilityDirectory = self.__madRootDir + self.__binDir # Set the wwwHomeBase if self.__mainUrl.find('http') == -1: self.__wwwHomeBase = self.__webProtocol + "://" + self.__mainUrl else: self.__wwwHomeBase = self.__mainUrl # Set the top level url if self.__topLevel.strip() not in ('', '.'): self.__topLevelUrl = self.__wwwHomeBase + '/' + self.__topLevel else: self.__topLevelUrl = self.__wwwHomeBase # Set contactEmail by stripping ContactLink line # Look for colon at end of Mailto: begIndexEmail = self.__contactlink.find(':') # look for "> at end of email address endIndexEmail = self.__contactlink.find('">') # if not found, try '> if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('\'>') # if still not found, try > alone if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('>') #check that both were found if begIndexEmail != -1 and endIndexEmail != -1: self.__contactEmail = self.__contactlink[begIndexEmail + 1 : endIndexEmail] if not self.__isValidEmail(self.__contactEmail): self.__contactEmail = None elif self.__isValidEmail(self.__contactlink): # user just entered an email self.__contactEmail = self.__contactlink # make contactlink proper link self.__contactlink = ' 0: self.__mailserver = self.__hardCodeMailServer else: self.__mailserver = 'localhost' # public methods def getDatabaseUtilityDirectory(self): """getDatabaseUtilityDirectory returns the full path to the database utility directory. Inputs: None Returns: String representing the full path to the database utility directory. (eg, /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__databaseUtilityDirectory def getWWWHomeBase(self): """getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__wwwHomeBase def getMadServer(self): """getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__mainUrl def getTopLevelUrl(self): """getTopLevelUrl returns the full url of the top level directory in main database website. Inputs: None Returns: String representing the full url to the top level directory in main database website. (eg, http://haystack.mit.edu/madrigal) Affects: Nothing Exceptions: None """ return self.__topLevelUrl def getRelativeTopLevel(self): """getRelativeTopLevel returns the relative url of the top level directory in main database website. Inputs: None Returns: String representing the relative url to the top level directory in main database website. (eg, madrigal) Affects: Nothing Exceptions: None """ return self.__topLevel def getSiteID(self): """getSiteID returns the site id number. Inputs: None Returns: The site id (integer) of the madrigal installation. Affects: Nothing Exceptions: If non-integer found """ try: return int(self.__siteIdValue) except: raise madrigal.admin.MadrigalError("Site id not an integer in madrigal configuration file " + \ self.__confFile, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def getMadrootEnvVarName(self): """getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT). Inputs: None Returns: The name of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__MAD_ROOT def getMadroot(self): """getMadroot returns the value of the environment variable __MAD_ROOT. Inputs: None Returns: The value of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__madRootDir def getMetadataDir(self): """getMetadataDir returns the metadata directory. Inputs: None Returns: The full metadata directory path. (eg. /opt/madrigal/metadata) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__metadataDir def getExperimentDir(self): """getExperimentDir returns the main experiment directory. No longer guarenteed to be unique,but will hold test files and geophysical experiments. Inputs: None Returns: The full experiment directory path. (eg. /opt/madrigal/experiments) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__experimentDir def getExperimentDirs(self): """getExperimentDirs returns a list of full paths of all valid experiment directories. Inputs: None Returns: list of full paths of all valid experiment directories. Valid experiment directories are those of the form of the regular expression $/Volumes/dropbox/mad31/experiments[0-9]* Affects: Nothing Exceptions: None """ reStr = '/experiments[0-9]*$' possibleExpDirs = glob.glob(os.path.join(self.__madRootDir, 'experiments*')) retList = [] for possibleExpDir in possibleExpDirs: result = re.search(reStr, possibleExpDir) if result: retList.append(possibleExpDir) return(retList) def getBinDir(self): """getBinDir returns the madrigal bin directory. Inputs: None Returns: The madrigal bin directory path. (eg /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__binDir def getHtmlStyle(self): """getHtmlSyle returns the default html body tag for the site. Inputs: None Returns: The default html body tag for the site. (eg. ) Affects: Nothing Exceptions: None """ return self.__htmlstyle def getBackgroundColor(self): """getBackgroundColor returns the string representing background color from getHtmlStyle Inputs: None Returns the color string from the body tag in madrigal.cfg. For example, will return '#FFFF88; if is Returns empty string if no BGCOLOR= """ line = self.getHtmlStyle().upper() items = line.split() for item in items: index = item.find('BGCOLOR=') if index != -1: return(item[len('BGCOLOR='):]) return('') # not found def getIndexHead(self): """getIndexHead returns the heading of the top level madrigal page. Inputs: None Returns: The heading of the top level madrigal page. (eg. Welcome to the Madrigal Database
at Ishtar) Affects: Nothing Exceptions: None """ return self.__indexhead def getContactLink(self): """getContactLink returns contact email link tag (see getContactEmail for the email alone). Inputs: None Returns: The contact email link tag. (eg.
madrigal@haystack
) Affects: Nothing Exceptions: None """ return self.__contactlink def getContactEmail(self): """getContactEmail returns the email address of the site administrator. Inputs: None Returns: The email address of the site administrator. Affects: Nothing Exceptions: None """ if not self.__contactEmail is None: return(self.__contactEmail) else: madSite = madrigal.metadata.MadrigalSite() return(madSite.getSiteEmail(self.getSiteID())) def getMailserver(self): """getMailserver returns the mailserver name. Inputs: None Returns: The mailserver name. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to localhost Affects: Nothing Exceptions: None """ return self.__mailserver def getPythonExecutable(self): """getPythonExecutable returns the full path to the python executable. Inputs: None Returns: the full path to the python executable. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to madroot/bin/python Affects: Nothing Exceptions: None """ return self.__pythonexe def getLocalRulesOfRoad(self): """getLocalRulesOfRoad returns the local rules of the road. Inputs: None Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that. Else returns a default rules_of_road statement Affects: Nothing Exceptions: None """ default_rules_of_road = 'Use of the Madrigal Database is generally subject to the ' + \ 'CEDAR Rules-of-the-Road . ' + \ 'Prior permission to access the data is not required. However, the user is required to establish ' + \ 'early contact with any organization whose data are involved in the project to discuss the ' + \ 'intended usage. Data are often subject to limitations which are not immediately evident to ' + \ 'new users. Before they are formally submitted, draft copies of all reports and publications ' + \ 'must be sent to the contact scientist at all data-supplying organizations along with an offer ' + \ 'of co-authorship to scientists who have provided data. This offer may be declined. The ' + \ 'Database and the organizations that contributed data must be acknowledged in all reports and ' + \ 'publications, and whenever this data is made available through another database. If you have ' + \ 'any questions about appropriate use of these data, contact %s' % (self.getContactEmail(), self.getContactEmail()) localRules = None try: f = open(os.path.join(self.getMadroot(), 'local_rules_of_the_road.txt')) localRules = f.read() f.close() except: pass if localRules == None: return default_rules_of_road else: return localRules def getFullPathFromPartial(self, partialPath): """getFullPathFromPartial returns the full path to a file or directory based on a partial path. Follows the rule that if partialPath begins with [/]experiments, then the full path is madroot + partialPath. Otherwise (pre madrigal 2.6) full path is madroot + experiments + partialPath Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10) Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10) """ if partialPath[:12].find('experiments') != -1: fullPath = os.path.join(self.getMadroot(), partialPath) else: fullPath = os.path.join(self.getMadroot(), 'experiments', partialPath) return(fullPath) def getExpList(self, expName = None, kinstList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, showIgnoredExperiments = False, enforcePathConvention = False): """getExpList returns a list of full experiment directory names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. startDate: a datetime date. If None (default), do not reject any experiments. endDate: a datetime date. If None (default), do not reject any experiments. startDayOfYear: a Julian day number (1-366) after which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any experiments. endDayOfYear: a Julian day number (1-366) before which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any experiments. showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If False (the default), only required path in the form 1999/mlh/*. Returns: a list of full experiment directory names that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None expList = [] # get all experiment directories expDirList = self.getExperimentDirs() # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: for root, dirs, files in os.walk(thisExpDir): self.__getExperiments((expName, kinstList, startDate, endDate, startDayOfYear, endDayOfYear, expList, showIgnoredExperiments, enforcePathConvention), root, dirs + files) return expList def getFileList(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, enforcePathConvention = 0, includeNonDefault = 0, includeNonMadrigal = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0, path = None): """getFileList returns a list of full file names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. path if not None (the default), search only experiment directory set by path. If None, search add experiments. Returns: a list of full path file names (strings) that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None fileList = [] # get all experiment directories if path is None: expDirList = self.getExperimentDirs() else: expDirList = [path] # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: for root, dirs, files in os.walk(thisExpDir): self.__getFiles((expName, kinstList, kindatList, startDate, endDate, startDayOfYear, endDayOfYear, fileList, publicAccessOnly, enforcePathConvention, includeNonDefault, includeNonMadrigal, appendKinst, appendStartTime, includeRealtime), root, dirs + files) return fileList def getFileListFromMetadata(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, includeNonDefault = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0): """getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only. This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't search the actual experiment directories. It is therefore faster, but less robust. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and starttime. Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None # the list to be returned fileList = [] # load a Madrigal experiment object madExpObj = MadrigalExperiment(self) # load a Madrigal metafile object madFileObj = MadrigalMetaFile(self) # convert input arguments to needed forms if expName != None: expNameArg = expName.replace(' ', '_') if startDate != None: startDateTime = madrigal.metadata.getMadrigalUTFromDate(startDate[0], startDate[1], startDate[2], startDate[3], startDate[4], startDate[5], 0) if endDate != None: endDateTime = madrigal.metadata.getMadrigalUTFromDate(endDate[0], endDate[1], endDate[2], endDate[3], endDate[4], endDate[5], 0) # loop through all experiments to create four lists: # acceptedExpIdList, acceptedExpKinstList, acceptedExpStartTimeList, and acceptedExpDirList acceptedExpIdList = [] acceptedExpKinstList = [] acceptedExpStartTimeList = [] acceptedExpDirList = [] position = -1 while 1: position += 1 # first position is zero thisName = madExpObj.getExpNameByPosition(position) #check if we're at the end if thisName == None: break # check experiment access thisSecurity = madExpObj.getSecurityByPosition(position) if thisSecurity == -1: continue elif publicAccessOnly == 0: if thisSecurity not in (0,1): continue elif publicAccessOnly == 1: if thisSecurity != 0: continue elif publicAccessOnly == 2: if thisSecurity not in (0,2): continue # apply expName filter if desired if expName != None: # since we are using file name matching, all spaces are # converted to underscores thisName = thisName.replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(thisName.lower(), expNameArg.lower()): continue # apply kinstList filter thisKinst = madExpObj.getKinstByPosition(position) if kinstList != None: if thisKinst not in kinstList: continue # startDate filter - we always need thisStartTime whether or not filter used thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) thisStartTime = madrigal.metadata.getMadrigalUTFromDate(thisStartDate[0], thisStartDate[1], thisStartDate[2], thisStartDate[3], thisStartDate[4], thisStartDate[5], 0) if endDate != None: if thisStartTime > endDateTime: continue # endDate filter if endDate != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) thisEndTime = madrigal.metadata.getMadrigalUTFromDate(thisEndDate[0], thisEndDate[1], thisEndDate[2], thisEndDate[3], thisEndDate[4], thisEndDate[5], 0) if startDate != None: if thisEndTime < startDateTime: continue # apply startDayOfYear filter if startDayOfYear != None: thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) if thisStartDate[7] < startDayOfYear: # index 7 refers to Day of year continue # apply endDayOfYear filter if endDayOfYear != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) if thisEndDate[7] > endDayOfYear: # index 7 refers to Day of year continue # this experiment has made it through all filters, append its id, kinst, startTime, and dir acceptedExpIdList.append(madExpObj.getExpIdByPosition(position)) acceptedExpKinstList.append(thisKinst) acceptedExpStartTimeList.append(thisStartTime) # find the directory based on url dir = self.getMadroot() if dir[-1] != '/': dir += '/' url = madExpObj.getExpUrlByPosition(position) index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: dir += 'experiments/' dir += url[index+8:] acceptedExpDirList.append(dir) # now loop through the file object to find all files position = -1 while 1: position += 1 # first position is zero thisExpId = madFileObj.getExpIdByPosition(position) #check if we're at the end if thisExpId == None: break # skip this file if expId not in acceptedExpIdList if thisExpId not in acceptedExpIdList: continue # apply kindatList filter thisKindat = madFileObj.getKindatByPosition(position) if kindatList != None: if thisKindat not in kindatList: continue # check file access if madFileObj.getAccessByPosition(position) != 0: if publicAccessOnly in (1,2): continue # apply includeNonDefault filter if includeNonDefault == 0: category = madFileObj.getCategoryByPosition(position) if includeRealtime == 0: if category != 1: # not default continue else: if category not in (1,4): # not default or realtime continue # this file has been accepted by all filters - first, get its experiment index expIndex = acceptedExpIdList.index(thisExpId) thisFilename = acceptedExpDirList[expIndex] + '/' + madFileObj.getFilenameByPosition(position) # append result to fileList according to appendKinst and appendStartTime if appendKinst == 0 and appendStartTime == 0: fileList.append(thisFilename) elif appendKinst == 1 and appendStartTime == 0: fileList.append((thisFilename, acceptedExpKinstList[expIndex])) elif appendKinst == 0 and appendStartTime == 1: fileList.append((thisFilename, acceptedExpStartTimeList[expIndex])) else: fileList.append((thisFilename, acceptedExpKinstList[expIndex], acceptedExpStartTimeList[expIndex])) return fileList def setFileAccess(self, expDirectory, accessMode): """setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Inputs: expDirectory: The full path to a directory in the experiment directory. That is, it may be madroot/experiments[0-9]* or any directory under it. accessMode: either 0 for public access, or 1 for private access. Returns: None Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Exceptions: If accessMode is not 1 or 0. """ if (accessMode != 0 and accessMode != 1): raise madrigal.admin.MadrigalError('MadrigalDB.setFileAccess called with accessMode = ' + \ str(accessMode) + ', must be either 0 or 1', None) # walk the experiments directory to find all files meeting criteria for root, dirs, files in os.walk(expDirectory): self.__setFileAccess(accessMode, root, dirs + files) def tarExperiments(self, tarFileName, startDate = None, endDate = None, excludePrivData = 0, ignoreDirCon = 1, includeNonDefData = 0, onlyData = 0, filetype = 0, verbose = 0): """tarExperiments creates a tar file containing files from madroot/experiments[0-9]*. Note: this method sometimes requires the modification of the fileTab.txt files found in the experiments directory. This is because some data files might be excluded, so that the fileTab.txt file will no longer be accurate. Because of this, all files to be tar'ed will be copied to /tmp/temp/experiments, where the fileTab.txt files will be modified. When done, this temp dir will be deleted. Inputs: tarFileName: The full path to a tar file to be created. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. excludePrivData: if 1, allow data marked as private to be omitted (and the line from the fileTab.txt to be removed). If 0 (the default), all data, public and private, will be included. ignoreDirCon: if 1, ignore convention that directory must be in form 1999/mlh/03sep99 (the default). If 0 , reject non-standard directories. includeNonDefData: if 1, include all files listed in fileTab.txt. If 0 (the default), reject non-default files, and modify fileTab.txt to remove non-default listings. onlyData: if 1, reject all files not listed in fileTab.txt. If 0 (the default), accept all files in a directory not mentioned in fileTab.txt. filetype: format to save data files as. Default 0 is to leave present format unchanged. is an integer as follows: type = 0 Leave present format unchanged (default) type = 1 Madrigal type = 2 Blocked Binary type = 3 Cbf type = 4 Unblocked binary type = 5 Ascii verbose: if 1, print to std out the list of files included (relative path). If 0, (the default) print nothing. Returns: None Affects: created tar file tarFileName of selected files from madroot/experiments. Exceptions: If unable to read any experiment file. """ if ignoreDirCon == 1: enforcePathConvention = 0 else: enforcePathConvention = 1 if onlyData == 1: includeNonMadrigal = 0 else: includeNonMadrigal = 1 # create a random temp dir tempDir = '/tmp/temp' + str(random.randrange(1,10000000)) if verbose == 1: print ('Creating list of files to tar...') # get list of files to tar tarFileList = self.getFileList(None, None, None, startDate, endDate, None, None, excludePrivData, enforcePathConvention, includeNonDefData, includeNonMadrigal) # now create a new list of filenames, using relative paths relTarFileList = [] for file in tarFileList: newFilename = file.split(self.getMadroot() + '/')[1] relTarFileList.append(newFilename) # copy all these files to tempDir if verbose == 1: print('The following is a list of files included:') for file in relTarFileList: # make sure dir exists try: os.makedirs(tempDir + '/' + os.path.dirname(file)) except: pass # now copy file shutil.copyfile(self.getMadroot() + '/' + file, tempDir + '/' + file) if verbose == 1: print('\t' + file) if onlyData: # now check if that data file needs to be converted to another filetype # since onlyData is set, convert every file if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + file, tempDir + '/' + file + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + file + '.backup 1 100000000 -o ' + \ tempDir + '/' + file + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + file + '.backup') if verbose == 1: print('Modifying fileTab.txt files if needed...') # modify fileTab.txt files, if required if (not onlyData) and (excludePrivData or not includeNonDefData): # first loop through each fileTab.txt file in list just to check permissions for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # make sure fileTab.txt is writable if not os.access(tempDir + '/' + filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(filename), None) # make sure the directory is writable if not os.access(os.path.dirname(tempDir + '/' + filename), os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(os.path.dirname(filename)), None) # no exceptions raised - all permissions are okay # this time loop through and modify the fileTab.txt files for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # create a MadrigalMetaFile object fileMeta = MadrigalMetaFile(self, tempDir + '/' + filename) # loop through each file name to see if its in tarFileList: fileNum = 0 while 1: madFilename = fileMeta.getFilenameByPosition(fileNum) if madFilename == None: break # get madRelFilename madRelFilename = os.path.dirname(filename) + '/' + madFilename # if its not in relTarFileList, delete it from fileMeta if not madRelFilename in relTarFileList: fileMeta.deleteRowByFilename(madFilename) fileNum = 0 continue else: # we know madRelFilename is a data file since its in fileTab.txt - # now check if that data file needs to be converted to another filetype if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + madRelFilename, tempDir + '/' + madRelFilename + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + madRelFilename + '.backup 1 100000000 -o ' + \ tempDir + '/' + madRelFilename + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + madRelFilename + '.backup') # get next madFilename fileNum = fileNum + 1 # if fileMeta not empty, write it out if fileMeta.getFileCount() > 0: fileMeta.writeMetadata() # else if its empty, simply delete it else: os.remove(tempDir + '/' + filename) # done modifying fileTab.txt - ready to tar # need to change working directory to tempDir to get relative paths in tar. # Since this directory will soon be deleted, will need to change it back to # whatever it is now when we're done pwd = os.getcwd() # check if tarFileName is absolute or relative to pwd if tarFileName[0] != '/': tarFileName = pwd + '/' + tarFileName if verbose == 1: print('Creating tar file...') os.chdir(tempDir) os.system('tar -cf ' + tarFileName + ' experiments*') os.chdir(pwd) if verbose == 1: print('Removing temp files...') # finally remove temp dir try: shutil.rmtree(tempDir) except: raise madrigal.admin.MadrigalError('In tarExperiments could not remove dir ' + tempDir, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def listFileTimes(self, expDir=None, relative=True): """listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory Inputs: expDir - the particular subdirectory of an experiment directory to list. If None (the default), will list all files for all experiment directories. May be an absolute path, or may start with experiments[0-9]*. relative - if True (the default) give the path relative to the experiments[0-9]* directory. If False, give full path Returns: a list of tuples, where each tuple has 1. the file path, and 2. a UT datetime object of the last file modification. Exceptions: raised if expDir is not a valid experiments directory or subdirectory """ expDirs = self.getExperimentDirs() if expDir == None: dirsToExamine = expDirs else: # verify a valid directory found = False if expDir[-1] == '/': expDir = expDir[:-1] # strip trailing / if expDir[0:11] == 'experiments': # convert to absolute path expDir = os.path.join(self.getMadroot(), expDir) for thisDir in expDirs: if expDir.find(thisDir) != -1: found = True if found: dirsToExamine = [expDir] else: raise ValueError('expDir %s not a valid experiment directory' % (expDir)) retList = [] for dirToExamine in dirsToExamine: for root, dirs, files in os.walk(dirToExamine): for name in files: fullname = os.path.join(root, name) relIndex = fullname[len(self.__madRootDir):].find('/experiments') relativeName = fullname[relIndex + 1 + (len(self.__madRootDir)):] ts = os.stat(fullname).st_mtime if relative: retList.append((relativeName, datetime.datetime.utcfromtimestamp(ts))) else: retList.append((fullname, datetime.datetime.utcfromtimestamp(ts))) return(retList) def getKinstKindatConfig(self, kinst, kindat, iniFile=None): """getKinstKindatConfig gets information for the /Volumes/dropbox/mad31/cachedFiles.ini needed to create Madrigal3 files if not specified. Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them. Inputs: kinst - the instrument kinst (integer) kindat - the data kindat (integer) iniFile - the ini file to use. If None, uses default ini file $/Volumes/dropbox/mad31/cachedFiles.ini Returns: a tuple with three items: 1. a list of extra parameters (string mnemonics) 2. a list of independent spatial parameters 3. a list of array splitting parameters Algorithm: 1. If iniFile == None and no default file, returns ([], [], []) 2. Searches ini file for section [%i] % (kinst). If not found, returns ([], [], []) 3. Searches right section for key %i_parms % (kindat). If not found, searches for default_parms. If not found, extra parameters are [] 4. Searches right section for key %i_formats % (kindat). If not found, searches for default_formats. If not found, indepedent spatial parms and extra splitting parameters are [] and []. If found, then parses dictionary, and returns key 'array'. If value has only one item, returns it as a one item list of independent spatial parameters. If two items, they are ind spatial parms and array splitting parms. """ if not iniFile: thisIniFile = os.path.join(self.getMadroot(), 'cachedFiles.ini') if not os.access(thisIniFile, os.R_OK): return(([], [], [])) else: thisIniFile = iniFile instSection = '%i' % (kinst) parser = configparser.SafeConfigParser() parser.read(thisIniFile) if not parser.has_section(instSection): return(([], [], [])) extraParms = [] indSpatialParms = [] arraySplitParms = [] # get extra parms if parser.has_option(instSection, '%i_parms' % (kindat)): extraParms = parser.get(instSection, '%i_parms' % (kindat)) extraParms = extraParms.split(',') elif parser.has_option(instSection, 'default_parms'): extraParms = parser.get(instSection, 'default_parms') extraParms = extraParms.split(',') # make sure no empty parms snuck in finalExtraParms = [] for extraParm in extraParms: if len(extraParm.strip()) > 0: finalExtraParms.append(extraParm.strip()) # get format dict formatDict = None if parser.has_option(instSection, '%i_formats' % (kindat)): formatDict = parser.get(instSection, '%i_formats' % (kindat)) formatDict = eval(formatDict) elif parser.has_option(instSection, 'default_formats'): formatDict = parser.get(instSection, 'default_formats') formatDict = eval(formatDict) if not formatDict is None: if 'array' in formatDict: value = formatDict['array'] if type(value) in (list, tuple): indSpatialParms = value[0] arraySplitParms = value[1] else: indSpatialParms = [value] return((finalExtraParms, indSpatialParms, arraySplitParms)) def toString(self): """toString returns a simple string representation of a MadrigalDB object. Inputs: None Returns: String describing a simple representation of a MadrigalDB object. Affects: Nothing Exceptions: None """ output = "Object type: MadrigalDB\n" output += "Database utility directory = " + self.getDatabaseUtilityDirectory() + "\n" output += "WWW home base = " + self.getWWWHomeBase() + "\n" output += "Server name = " + self.getMadServer() + "\n" output += "Top level url = " + self.getTopLevelUrl() + "\n" output += "Relative Top level url = " + self.getRelativeTopLevel() + "\n" output += "Site ID = " + str(self.getSiteID()) + "\n" output += "MAD_ROOT env. variable name = " + self.getMadrootEnvVarName() + "\n" output += "MAD_ROOT env. variable value = " + self.getMadroot() + "\n" output += "Madrigal metadata dir = " + self.getMetadataDir() + "\n" output += "Madrigal bin dir = " + self.getBinDir() + "\n" output += "Madrigal html body style = " + self.getHtmlStyle() + "\n" output += "Madrigal top level heading = " + self.getIndexHead() + "\n" output += "Madrigal contact link = " + self.getContactLink() + "\n" output += "Madrigal contact email = " + str(self.getContactEmail()) + "\n" output += "Madrigal mailserver = " + self.getMailserver() + "\n" output += "Madrigal python exe = " + self.getPythonExecutable() + "\n" return output def isTestExperiment(self, url, siteId=None): """isTestExperiment returns True if the given experiment url is a test of this Madrigal server. Url can be either real or form in expTab.txt, or can be the experiment directory. If siteId not given, use local site id. """ # Skip test experiments if siteId == None: siteId = self.getSiteID() if url.find('1998/mlh/20jan98') != -1 and siteId != 1: return(True) if url.find('1997/aro/06jan97') != -1 and siteId != 7: return(True) if url.find('1997/lyr/08apr97') != -1 and siteId != 2: return(True) if url.find('1997/son/06jan97') != -1 and siteId != 3: return(True) if url.find('1995/jro/01feb95') != -1 and siteId != 6: return(True) if url.find('1998/jro/27apr98') != -1 and siteId != 6: return(True) return(False) def __str__(self): """ __str__ simply calls toString """ return (self.toString()) def __isValidEmail(self, emailStr): """ __isValidEmail is a private helper function that does some checking to ensure a valid email address was found. Inputs: emailStr - email address string to verify Returns: 1 if no problems, 0 if not valid. Affects: Nothing Exceptions: None """ emailStr = emailStr.strip() if emailStr.find(' ') != -1: return 0 if emailStr.find('"') != -1: return 0 if emailStr.find('<') != -1: return 0 if emailStr.find('>') != -1: return 0 if emailStr.find('/') != -1: return 0 if emailStr.find(':') != -1: return 0 if emailStr.find('@') == -1: return(0) # otherwise okay return 1 def __getExperiments(self, arg, dirname, names): """ __getExperiment is a private helper function called by os.path.walk in getExpList. __getExperiments is called for each sub-directory in the experiments directory. Its purpose is to add any experiment directory paths from that directory that match the search criteria to the expList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. startDate: a python datetime in UTC. If None, do not reject any files. endDate: a python datetime in UTC. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. expList: the list of valid experiment paths to which is appended any newly-found valid experiment directories showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If False (the default), only required path in the form 1999/mlh/*. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __startDate = 2 __endDate = 3 __startDayOfYear = 4 __endDayOfYear = 5 __expList = 6 __showIgnored = 7 __enforcePath = 8 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt if not 'expTab.txt' in names: return # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply startDate filter expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError('Error getting expStartDate from dir %s' % (dirname)) time1 = datetime.datetime(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5]) if arg[__endDate] != None: if time1 > arg[__endDate]: # stop going down tree names[:] = [] return # apply endDate filter if arg[__startDate] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() time1 = datetime.datetime(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5]) if time1 < arg[__startDate]: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply ignore filter if expMeta.getSecurityByPosition() == -1 and not arg[__showIgnored]: # stop going down tree names[:] = [] return # now reject paths not matching present path convention if not enforcePathConvention if not arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention if arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay arg[__expList].append(dirname) def __getFiles(self, arg, dirname, names): """ __getFiles is a private helper function called by os.path.walk in getFileList. __getFiles is called for each sub-directory in the experiments directory. Its purpose is to add any file paths from that directory that match the search criteria to the fileList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None, all kindat values are accepted. startDate: a python date/time in UTC (see time module - actually a tuple of nine integers) after which to accept files. If None, do not reject any files. endDate: a python date/time in UTC (see time module - actually a tuple of nine integers) before which to accept files. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. fileList: the list of valid file paths to which is appended any newly-found valid file paths publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __kindatList = 2 __startDate = 3 __endDate = 4 __startDayOfYear = 5 __endDayOfYear = 6 __fileList = 7 __pubAccessOnly = 8 __enforcePath = 9 __includeNonDef = 10 __includeNonMad = 11 __appendKinst = 12 __appendStartTime = 13 __includeRealtime = 14 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt and fileTab.txt if not includeNonMadrigal if 'expTab.txt' in names: self.__hasExpTab = 1 else: self.__hasExpTab = 0 if 'fileTab.txt' in names: self.__hasFileTab = 1 else: self.__hasFileTab = 0 if arg[__includeNonMad] == 0: if not self.__hasExpTab or not self.__hasFileTab: return # apply all filters relating to expTab.txt if self.__hasExpTab if self.__hasExpTab: # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply date filters expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError('Error getting expStartDate from dir %s' % (dirname)) expStartUt = madrigal.metadata.getMadrigalUTFromDate(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5], 0) expEndDate = expMeta.getExpEndDateTimeByPosition() expEndUt = madrigal.metadata.getMadrigalUTFromDate(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5], 0) if arg[__startDate] != None: time1 = madrigal.metadata.getMadrigalUTFromDate(arg[__startDate][0], arg[__startDate][1], arg[__startDate][2], arg[__startDate][3], arg[__startDate][4], arg[__startDate][5], 0) if time1 > expEndUt: # stop going down tree names[:] = [] return # save time1 in case we need to appendStartTime thisStartTime = expStartUt # apply endDate filter if arg[__endDate] != None: time2 = madrigal.metadata.getMadrigalUTFromDate(arg[__endDate][0], arg[__endDate][1], arg[__endDate][2], arg[__endDate][3], arg[__endDate][4], arg[__endDate][5], 0) if time2 < expStartUt: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # check experiment access thisSecurity = expMeta.getSecurityByPosition() if thisSecurity == -1: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 0: if thisSecurity not in (0,1): # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 1: if thisSecurity != 0: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 2: if thisSecurity not in (0,2): # stop going down tree names[:] = [] return else: # no expTab.txt - set kinst to None expKinst = None # now reject paths not matching present path convention if enforcePathConvention == 0 if arg[__enforcePath] == 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention != 0 if arg[__enforcePath] != 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay, now start looking at individual files if self.__hasFileTab: fileMeta = MadrigalMetaFile(self, dirname + '/' + 'fileTab.txt') # get numFiles numFiles = fileMeta.getFileCount() # loop through each file, and add it if okay fileCount = 0 while fileCount < numFiles: # skip non-default and non-realtime files if not includeNonDef if fileMeta.getCategoryByPosition(fileCount) not in (1,4) and arg[__includeNonDef] == 0: fileCount = fileCount + 1 continue # skip realtime files if not includeNonDef and not includeRealtime if fileMeta.getCategoryByPosition(fileCount) == 4 and arg[__includeNonDef] == 0 and arg[__includeRealtime] == 0: fileCount = fileCount + 1 continue # check if file has right kindat if arg[__kindatList] != None: if fileMeta.getKindatByPosition(fileCount) not in arg[__kindatList]: fileCount = fileCount + 1 continue # check file access if fileMeta.getAccessByPosition(fileCount) != 0: if arg[__pubAccessOnly] in (1,2): fileCount = fileCount + 1 continue # the file needs to be added filename = fileMeta.getFilenameByPosition(fileCount) if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + filename) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + filename, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + filename, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) fileCount = fileCount + 1 # now add all non-madrigal files if includeNonMad if arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue # if its not in fileTab.txt, add it if fileMeta.getExpIdByFilename(name) == None: if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) # else the file has no fileTab.txt and includeNonMad is true - include all files elif arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) def __setFileAccess(self, arg, dirname, names): """setFileAccess is a private helper function called by os.path.walk in setFileAccess. Inputs: arg: either 0 for public access, or 1 for private access. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None Affects: sets all fileTab.txt files in dirname to be public or private, depending on arg. Exceptions: None. """ for name in names: if name == 'fileTab.txt': try: fileMetaObj = madrigal.metadata.MadrigalMetaFile(self, dirname + '/' + name) fileMetaObj.setAccess(arg) except madrigal.admin.MadrigalError as e: print(e.getExceptionStr()) class MadrigalSite: """MadrigalSite is an object that provides access to Madrigal site info from the metadata. This object provides access to all Madrigal site information in the metadata file siteTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: siteObj = madrigal.metadata.MadrigalSite() print siteObj.getSiteName(1) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __siteMetadataFile = "siteTab.txt" _defaultVersion = '2.6' # column positions __siteIDCol = 0 __siteNameCol = 1 __madServerCol = 2 __madDocRootCol = 3 __madCGICol = 4 __madServletCol = 5 __contactNameCol = 6 __contactAddr1Col = 7 __contactAddr2Col = 8 __contactAddr3Col = 9 __contactCityCol = 10 __contactStateCol = 11 __contactZipCol = 12 __contactCountryCol = 13 __contactPhoneCol = 14 __contactEmailCol = 15 __siteVersionCol = 16 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalSite by reading from siteTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/siteTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get site metadata file if (initFile == None): self.__filename = self.__siteMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without version allowedLens = (self.__contactEmailCol+1, self.__siteVersionCol+1) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens).getList() def getSiteName(self, siteID): """getSiteName returns the site name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__siteNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteServer(self, siteID): """getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found. Inputs: siteID integer to get site server. Returns: the site server that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madServerCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteDocRoot(self, siteID): """getSiteDocRoot returns the relative document root (e.g. madrigal) that matches siteID argument, or None if not found. Inputs: siteID integer to get document root path. Returns: the document root path that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madDocRootCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteRelativeCGI(self, siteID): """getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal) that matches siteID argument, or None if not found. For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored. Inputs: siteID integer to get relative cgi path. Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ version = self.getSiteVersion(siteID) if distutils.version.LooseVersion(version) >= distutils.version.LooseVersion('3.0'): return(self.getSiteDocRoot(siteID)) # this code only runs for Madrigal 2.0 for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__madCGICol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return None def getSiteContactName(self, siteID): """getSiteContactName returns the site contact name that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact name that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactNameCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress1(self, siteID): """getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 1 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr1Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress2(self, siteID): """getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 2 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr2Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteAddress3(self, siteID): """getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact address 3 that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactAddr3Col] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteCity(self, siteID): """getSiteCity returns the site city that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact city that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCityCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteState(self, siteID): """getSiteState returns the site state that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact state that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactStateCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSitePostalCode(self, siteID): """getSitePostalCode returns the site postal code that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact postal code that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactZipCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteCountry(self, siteID): """getSiteCountry returns the site country that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact country that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactCountryCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteTelephone(self, siteID): """getSiteTelephone returns the site telephone that matches siteID argument, or None if not found. Inputs: siteID integer to get SiteName. Returns: the site contact telephone that matches siteID argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactPhoneCol] except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteEmail(self, siteID): """getSiteEmail returns the site email address that matches siteID argument, or None if not found. Inputs: siteID integer to get Site email address. Returns: the site email address that matches siteID argument, or None if not found. To list multiple email addresses in this field separate them with semicolons (since commas are delimiters). getSiteEmail will automatically replace semicolons with commas, as required by multiple email addresses. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): return site[self.__contactEmailCol].replace(';', ',') except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getSiteList(self): """getSiteList returns a list of all site ids and names. Inputs: None. Returns: a list of all site ids and names. Each item in the list is a tuple of the form (Site id (integer), site name (string)). Example item: (1,'Millstone') Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for site in self.__fileList: try: item = (int(site[self.__siteIDCol]), site[self.__siteNameCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in siteTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList def getSiteVersion(self, siteID): """getSiteVersion returns the site version string that matches siteID argument, or None if not found. Inputs: siteID integer Returns: the site version that matches siteID argument, or None if not found. If file does not have this field, returns default value of 2.6. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ for site in self.__fileList: # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): if len(site) >= self.__siteVersionCol + 1: return site[self.__siteVersionCol] else: return(self._defaultVersion) except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def setSiteVersionBySiteID(self, siteID, version): """setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers at given siteID. Inputs: siteID - siteID of site in list. version - string Madrigal version as period delimited integers (eg, 2.6) Returns: None. Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or siteID not found. This method added in Madrigal 3.0 """ # verify no illegal commas if version.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) # parse test version = version.strip() try: siteID = int(siteID) if len(version) == 0: raise ValueError('') for item in version.split('.'): int(item) except: raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' % \ (str(position, version), [traceback.format_exc()])) rightPosition = None for position, site in enumerate(self.__fileList): # find matching siteid try: if (int(site[self.__siteIDCol]) == siteID): rightPosition = position except: raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if rightPosition is None: raise madrigal.admin.MadrigalError('siteId %i not found in siteTab.txt' % (siteID)) if len(self.__fileList[position]) == self.__siteVersionCol: # not yet set self.__fileList[position].append(version) else: self.__fileList[position][self.__siteVersionCol] = version def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the siteTab.txt file. Inputs: newFullPath: a new path to write the siteTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) class MadrigalInstrument: """MadrigalInstrument is an object that provides access to Madrigal instrument info from the metadata. This object provides access to all Madrigal instrument information in the metadata files instTab.txt and instType.Tab. key = self.__instKinstCol (used in searches) Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrument() print test.getInstrumentName(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __instMetadataFile = "instTab.txt" # column positions __instKinstCol = 0 __instMnemonicCol = 1 __instNameCol = 2 __latitudeCol = 3 __longitudeCol = 4 __altitudeCol = 5 __contactNameCol = 6 __contactAddr1Col = 7 __contactAddr2Col = 8 __contactAddr3Col = 9 __contactCityCol = 10 __contactStateCol = 11 __contactZipCol = 12 __contactCountryCol = 13 __contactPhoneCol = 14 __contactEmailCol = 15 __categoryCol = 16 # instType.txt file __inst2MetadataFile = "instType.txt" # column positions __inst2CategoryIdCol = 0 __inst2CategoryDescCol = 1 def __init__(self, madDB=None, initFile=None, init2File=None): """__init__ initializes MadrigalInstrument by reading from instTab.txt (or initFile) and instType.txt file (or init2File). Inputs: madDB - Existing MadrigalDB object, by default = None. initFile - String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instTab.txt. init2File - String representing the full path to the metadata file instType.txt. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instTab.txt file was updated with the release of the madrigal python api, and this function will throw an error if used with the old file. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument metadata file if (initFile == None): self.__filename = self.__instMetadataFile else: self.__filename = initFile genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, key=self.__instKinstCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # get instrument metadata file 2 if (init2File == None): self.__filename2 = self.__inst2MetadataFile else: self.__filename2 = initFile self.__fileList2 = madrigal.metadata.MadrigalMetadata(self.__filename2, self.__madDB).getList() def getInstrumentName(self, kinst): """getInstrumentName returns the instrument name that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument name. Returns: the instrument name that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instNameCol]) def getInstrumentMnemonic(self, kinst): """getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found. Inputs: kinst integer to get instrument mnemonic. Returns: the instrument mnemonic that matches kinst argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__instMnemonicCol]) def getLatitude(self, kinst): """getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get latitude. Returns: the latitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__latitudeCol])) except ValueError: return(None) def getLongitude(self, kinst): """getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get longitude. Returns: the longitude as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__longitudeCol])) except ValueError: return(None) def getAltitude(self, kinst): """getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get altitude. Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] except KeyError: return(None) try: return(float(self.__fileList[position][self.__altitudeCol])) except ValueError: return(None) def getContactName(self, kinst): """getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact name as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactNameCol]) def getContactAddress1(self, kinst): """getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact name. Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 3 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactAddr1Col]) def getContactEmail(self, kinst): """getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get contact email. Returns: the contact email as a string that matches kinst argument, or None if not found or blank. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format This method added in Madrigal 2.6 """ try: position = self._dict[str(kinst)] except KeyError: return(None) return(self.__fileList[position][self.__contactEmailCol]) def getCategory(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ categoryId = None try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None if categoryId == None: return(None) # now loop until categoryId found for inst2 in self.__fileList2: try: if (int(inst2[self.__inst2CategoryIdCol]) == categoryId): return(inst2[self.__inst2CategoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in instType.txt parsing metadata row: ' + str(inst2), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # type not found return(None) def getCategoryId(self, kinst): """getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found. Inputs: kinst integer to get altitude. Returns: the instrument category that matches kinst argument as a string, or None if kinst not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ try: position = self._dict[str(kinst)] categoryId = int(self.__fileList[position][self.__categoryCol]) except KeyError: categoryId = None return(categoryId) def getInstrumentList(self): """getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values. Inputs: None. Returns: a list of all instrument names, mnemonics, and their kinst values. Each item in the list is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)). Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], int(inst[self.__instKinstCol])) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList def getOrderedInstrumentList(self): """getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId), ordered by categoryId and then kinst. Inputs: None. Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and then kinst. Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for inst in self.__fileList: try: kinst = int(inst[self.__instKinstCol]) item = (inst[self.__instNameCol], inst[self.__instMnemonicCol], kinst, self.getCategory(kinst), self.getCategoryId(kinst)) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) retList.sort(key=self.__instrumentSort) return retList def getOrderedInstrumentListWithData(self, isTrusted, localOnly=False, localExpObj=None, globalExpObj=None, allowArchive=False, requireFiles=False, requireFilesOrPlots=False): """getInstrumentList returns information about which instruments have local and global data. Inputs: isTrusted - True if client is trusted, False otherwise localOnly - if False (the default), will return information about both local and global data. If True, then global data ignored. localExpObj - an MadrigalExperiment object for the local data. If None (the default), will be created. globalExpObj - an MadrigalExperiment object for the global data. If None (the default), will be created if not localOnly. allowArchive - if True, allow experiments marked as security==2(public arcive), and if isTrusted, allow security==3(private archive) requireFiles - if True, only include experiments with Madrigal data files. If False (default), not not require that. requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots. If False (default), not not require that. Returns: an ordered tuple of two items: 1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear, globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst. If localOnly, globalStartYear and globalEndYear = 0. If not local only, localStartYear and localEndYear will be zero if no local data for that instrument. 2. categoryDict - key = categoryId, value = category Description Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if (not localOnly) and (globalExpObj==None): globalExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB, os.path.join(self.__madDB.getMadroot(), 'metadata/expTabAll.txt')) if localExpObj == None: localExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) orderInstList = [] categoryDict = {} # next task - create local dict and global dict of key=kinst, value = (startYear, endYear) tuple localInstDict = {} globalInstDict = {} # Skip test experiments testExp = ('1998/mlh/20jan98', '1997/aro/06jan97', '1997/lyr/08apr97', '1997/son/06jan97', '1995/jro/01feb95', '1998/jro/27apr98') for i in range(localExpObj.getExpCount()): thisUrl = localExpObj.getExpUrlByPosition(i) if thisUrl[-16:] in testExp: continue thisSecurity = localExpObj.getSecurityByPosition(i) if thisSecurity == -1: # blocked to all continue if thisSecurity == 1 and not isTrusted: # no access continue if thisSecurity == 2 and not allowArchive: # no access continue if thisSecurity == 3 and ((not isTrusted) or (not allowArchive)): # no access continue kinst = localExpObj.getKinstByPosition(i) startYear = localExpObj.getExpStartDateTimeByPosition(i)[0] endYear = localExpObj.getExpEndDateTimeByPosition(i)[0] if kinst in localInstDict: oldStartYear, oldEndYear = localInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue expDir = localExpObj.getExpDirByPosition(i) if requireFilesOrPlots: links = localExpObj.getExpLinksByPosition(i) if len(links) == 0: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if requireFiles: try: size = os.path.getsize(os.path.join(expDir, 'fileTab.txt')) if size < 3: continue except: continue if kinst in localInstDict: localInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) else: localInstDict[kinst] = (startYear, endYear) if not localOnly: for i in range(globalExpObj.getExpCount()): kinst = globalExpObj.getKinstByPosition(i) startYear = globalExpObj.getExpStartDateTimeByPosition(i)[0] endYear = globalExpObj.getExpEndDateTimeByPosition(i)[0] thisUrl = globalExpObj.getExpUrlByPosition(i) thisSecurity = globalExpObj.getSecurityByPosition(i) thisSite = globalExpObj.getExpSiteIdByPosition(i) if thisUrl[-16:] in testExp: continue if kinst not in globalInstDict: globalInstDict[kinst] = (startYear, endYear) continue oldStartYear, oldEndYear = globalInstDict[kinst] if startYear >= oldStartYear and endYear <= oldEndYear: # no new info continue if thisSecurity != 0 and self.__madDB.getSiteID() != thisSite: # no search remote sites for non-public experiments allowed continue if requireFiles: if globalExpObj.getExpIdByPosition(i) not in expIdList: continue globalInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear)) # populate orderInstList and categoryDict instList = self.getOrderedInstrumentList() for inst in instList: if not localOnly: if inst[2] in globalInstDict: if inst[4] not in categoryDict: categoryDict[inst[4]] = inst[3] globalStartYear, globalEndYear = globalInstDict[inst[2]] try: localStartYear, localEndYear = localInstDict[inst[2]] except KeyError: localStartYear, localEndYear = (0, 0) orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) else: if inst[2] in localInstDict: if inst[4] not in categoryDict: categoryDict[inst[4]] = inst[3] globalStartYear = 0 globalEndYear = 0 localStartYear, localEndYear = localInstDict[inst[2]] orderInstList.append((inst[0], localStartYear, localEndYear, globalStartYear, globalEndYear, inst[2], inst[4])) return ((orderInstList, categoryDict)) def __instrumentSort(self, thisInst): """instrumentSort is a private method used to sort tuples of instrument data """ return((thisInst[4], thisInst[2])) class MadrigalInstrumentParameters: """MadrigalInstrumentParameters is an object that provides access to the metadata file that summarizes the parameters associated with each instrument. This object provides access to all Madrigal instrument parameter information in the metadata file instParmTab.txt. The metadata file instParmTab.txt lists, for any given instrument, all the measured parameters found in all the data files in the database associated with that instrument. This class also contains a method to rebuild the table instParmTab.txt by examining every data file in the database. This is presumably a slow process and should be done in the background. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentParameters() print test.getParameters(30) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jul. 17, 2002 """ #constants __instParmMetadataFile = "instParmTab.txt" # column positions __instParmKinstCol = 0 __instParmListCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instParmTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument parameter metadata file if (initFile == None): self.__filename = self.__instParmMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getParameters(self, kinst): """getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank. Inputs: kinst integer to get parameters. If 0, get parameters from all instruments. Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank. Affects: None Exceptions: if error in metadata file """ parmList = [] for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instParmKinstCol]) == kinst or kinst == 0): if len(inst[self.__instParmListCol]) == 0: continue tempList = inst[self.__instParmListCol].split() for parm in tempList: if not parm.lower() in parmList: if parm.find('-32767') == -1: parmList.append(parm.lower()) except: raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(parmList) == 0: return(None) return parmList def rebuildInstParmTable(self, completeRebuildFlag = 0): """rebuildInstParmTable rebuilds the instParmTab.txt metadata file. The table instParmTab.txt is a listing of every measured parameter found in any data file for a given instrument. It now will also import data from other Madrigal sites instParmTab.txt files. Since these files are constantly updated, this table needs to be updated on a regular basis. This methods works in one of two ways, depending on the value of completeRebuildFlag and whether a file called instParmLastUpdate.txt exists in the metadata directory. If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable loops through each instrument in the instTab.txt. For each instrument, it loops through every data file associated with that instrument. For every data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. Since this process involves every file in the database, it may take a great deal of time and should be run in the background. If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable first stores all the existing parameters from the instParmTab.txt. It reads the date of the last update from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with each instrument. For every new data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. This makes rebuildInstParmTable faster, but possibly keeps invalid parameters if experiments are ever deleted. Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are also added. Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means newer than the date in the instParmLastUpdate.txt file (stored as a float). If 1, rebuild the table completely. This will eliminate any parameters no longer found in files, but will take longer. If no instParmLastUpdate.txt file is found, the table is always rebuilt completely. Returns: None. Affects: Writes file instParmTab.txt in metadata directory Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instParmTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # if madroot not set, set it now if os.environ.get('MAD' + 'ROOT') == None: os.environ['MAD' + 'ROOT'] = self.__madDB.getMadroot() # now its safe to import madrigal.data since MAD + ROOT set import madrigal.data # get date of files to examine if completeRebuildFlag == 0: # try to open and read instParmLastUpdate.txt try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'r') lastUpdateTime = float(dateFile.read()) dateFile.close() except: # if failure, set time to 1/1/1970 and set completeRebuildFlag = 1 lastUpdateTime = time.mktime((1970,1,1,0,0,0,0,0,0)) completeRebuildFlag = 1 else: # set time to 1/1/1971 lastUpdateTime = time.mktime((1971,1,1,0,0,0,0,0,0)) # now write new version of instParmLastUpdate.txt # if not writable, exception thrown try: dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'w') except: raise madrigal.admin.MadrigalError('Unable to write: ' + \ self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', None) newUpdateTime = time.time() dateFile.write(str(newUpdateTime)) dateFile.close() # finally, make sure new instParmLastUpdate.txt is world-writable try: os.chmod(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 0o666) except: pass # create a needed MadrigalParameters obj madParmObj = madrigal.data.MadrigalParameters(self.__madDB) # create a needed MadrigalSite obj madSiteObj = madrigal.metadata.MadrigalSite(self.__madDB) # create a string to hold all the text for the new file newFileStr = '' # create a dictionary with all instrument codes as keys instObj = madrigal.metadata.MadrigalInstrument(self.__madDB) instList = instObj.getInstrumentList() instCodeDict = {} for inst in instList: if inst[2] == 0: continue if inst[2] in list(instCodeDict.keys()): continue if completeRebuildFlag == 1: # start with empty list instCodeDict[inst[2]]= [] else: # start with present list tempParmList = [] oldParmList = self.getParameters(inst[2]) if oldParmList != None: for item in oldParmList: # check that its not -32767 try: if madParmObj.getParmCodeFromMnemonic(item) == -32767: continue except ValueError: print(('skipping parm %s since no longer valid' % (str(item)))) continue tempParmList.append(madParmObj.getParmCodeFromMnemonic(item)) instCodeDict[inst[2]] = tempParmList else: instCodeDict[inst[2]] = [] # get a list of all default and realtime files filelist = self.__madDB.getFileListFromMetadata(appendKinst = 1, includeRealtime = 1) # appendKinst, allow realtime # loop through each file - file is tuple of # (file name, inst code) for file in filelist: # check whether file should be skipped try: if lastUpdateTime > os.stat(file[0])[8]: continue except: continue # try to create MadrigalFile object for that file try: print('\t\tAdding parameters to instParmTab.txt from ' + str(file)) madFileObj = madrigal.data.MadrigalFile(file[0], self.__madDB) fileParmList = madFileObj.getMeasuredParmList() # append unique parameters for parm in fileParmList: if not parm in instCodeDict[file[1]]: instCodeDict[file[1]].append(parm) # if a problem with file, skip it except: continue # now create a new file string from instCodeDict # create a sorted list of keys keyList = list(instCodeDict.keys()) keyList.sort() # step through each key delimiter = ' ' for key in keyList: # sort that instrument's list instParmList = madParmObj.normalizeParmList(instCodeDict[key]) # convert from codes to mnemonics instParmList = madParmObj.getParmMnemonicList(instParmList) # append that instrument's data to newFileStr newFileStr = newFileStr + str(key) + ',' newFileStr = newFileStr + delimiter.join(instParmList).lower() + '\n' # if no data found, throw error if len(newFileStr) == 0: raise madrigal.admin.MadrigalError('No data found for: ' + str(filename), None) # write new instParmTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close class MadrigalKindat: """MadrigalKindat is an object that provides access to Madrigal kind of data info from the metadata. This object provides access to all Madrigal kind of data information in the metadata file typeTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalKindat() print test.getKindatDescription(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001 """ #constants __typeMetadataFile = "typeTab.txt" # column positions __typeCodeCol = 0 __typeDescCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalKindat by reading from typeTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/typeTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get kindat metadata file if (initFile == None): self.__filename = self.__typeMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getKindatDescription(self, code, kinst=None): """getKindatDescription returns the kindat description that matches code argument, or None if not found. Inputs: code integer to get kindat description. or integer as string kinst - used to look up kindat description in form '%i_%i' % (kinst, code). If None, (the default) only return exact code match as integer. If given, first try to match '%i_%i' % (kinst, code). If not found, then try just code. May be integer or string integer Returns: the kindat description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) if kinst != None: kinst = int(kinst) for i, type in enumerate(self.__fileList): # find matching kinst and code try: items = type[self.__typeCodeCol].split('_') if len(items) == 2: thisKinst = int(items[0]) thisKindat = int(items[1]) if thisKinst == kinst and thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # just search for code for i, type in enumerate(self.__fileList): try: items = type[self.__typeCodeCol].split('_') if len(items) == 1: thisKindat = int(items[0]) if thisKindat == code: return type[self.__typeDescCol] except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getKindatList(self): """getKindatList returns a list of all kindat descriptions and codes. Inputs: None. Returns: a list of all kindat descriptions and codes. Each item in the list is a tuple of the form (Kindat description (string), kindat code (integer or string in form '%i_%i' % (kinst, code))). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, kindat in enumerate(self.__fileList): try: if kindat[self.__typeCodeCol].find('_') != -1: raise ValueError('') # works if its a standard code - int no longer raises an error if underscore found item = (kindat[self.__typeDescCol], int(kindat[self.__typeCodeCol])) retList.append(item) except: try: # works for kinst_kindat item = (kindat[self.__typeDescCol], kindat[self.__typeCodeCol]) retList.append(item) except: raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(kindat), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return retList class MadrigalInstrumentKindats: """MadrigalInstrumentKindats is an object that provides access to the metadata file that summarizes the kindat codes associated with each instrument. This object provides access to all Madrigal instrument kindat information in the metadata file instKindatTab.txt. The metadata file instKindatTab.txt lists, for any given instrument and year, all the kindat codes found in all the data files in the local database associated with that instrument. This class also contains a method to rebuild the table instKindatTab.txt by examining all the metadata in the database. This is presumably a somewhat slow process and should be done in the background. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalInstrumentKindats() print test.getKindatListForInstruments([20,30]) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Aug. 15, 2002 """ #constants __instKindatMetadataFile = "instKindatTab.txt" # column positions __instKindatKinstCol = 0 __instKindatListCol = 1 __instKindatYearCol = 2 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get instrument kindat metadata file if (initFile == None): self.__filename = self.__instKindatMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getKindatListForInstruments(self, kinstList): """getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list. A kindat code is returned if it appears in any year Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer. Returns: a list of kindat codes as integers associated with the given instrument list. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # if kinstList is just a single integer, convert it into a list if type(kinstList) == int: kinstList = [kinstList] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if (int(inst[self.__instKindatKinstCol]) in kinstList): if len(inst[self.__instKindatListCol]) == 0: continue tempList = inst[self.__instKindatListCol].split() for parm in tempList: if not int(parm) in retKindatList: retKindatList.append(int(parm)) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) return(retKindatList) def getKindatListForInstrumentYear(self, kinst, year): """getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year. Inputs: kinst, year Returns: a list of kindat codes as integers associated with the given instrument and year, or None if not found. Affects: None Exceptions: if error in metadata file """ retKindatList = [] # loop through instKindatTab.txt for inst in self.__fileList: # find matching kinst try: if int(inst[self.__instKindatKinstCol]) != kinst: continue if int(inst[self.__instKindatYearCol]) != year: continue return([int(kindat) for kindat in inst[self.__instKindatListCol].split()]) except: raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def rebuildInstKindatTable(self): """rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file. The table instKindatTab.txt is a listing of every kindat found for a given instrument for any given year. Since these files are constantly updated, this table needs to be updated on a regular basis. Data from other Madrigal sites is also imported into this table. How it works: For each instrument in instTab.txt, this method first creates a list of all the experiments that used that instrument. It then loops through the file table. For each file listing, if the file is the default one, and its experiment id is in the list of experiments just created, the kindat is added to the list if its not already there. Data from all other Madrigal sites is then added via getMetadata. It then writes the metadata file in the form: 10, 1001 1002 1003, where the first column in the instrument code and the second column is a space delimited list of kindat codes. Inputs: None. Returns: None. Affects: Writes file instKindatTab.txt in metadata directory Exceptions: If unable to write instKindatTab.txt file. """ # get full file name filename = self.__madDB.getMetadataDir() + '/' + self.__filename # throw an exception if instKindatTab.txt file not writable if not os.access(filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None) # get experiment metadata madExp = MadrigalExperiment(self.__madDB) kindatDict = {} # key = year, value = list of kindats for i in range(madExp.getExpCount()): kinst = madExp.getKinstByPosition(i) if kinst not in list(kindatDict.keys()): kindatDict[kinst] = {} expDir = madExp.getExpDirByPosition(i) # get year range timeTuple = madExp.getExpStartDateTimeByPosition(i) syear = timeTuple[0] timeTuple = madExp.getExpEndDateTimeByPosition(i) eyear = timeTuple[0] for year in range(syear, eyear+1): if year not in list(kindatDict[kinst].keys()): kindatDict[kinst][year] = [] try: madFileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB, os.path.join(expDir, 'fileTab.txt')) except: continue for j in range(madFileObj.getFileCount()): kindat = madFileObj.getKindatByPosition(j) if kindat not in kindatDict[kinst][year]: kindatDict[kinst][year].append(kindat) # create newFileStr from instCodeDict newFileStr = '' kinstList = list(kindatDict.keys()) kinstList.sort() delimiter = ' ' for kinst in kinstList: yearList = list(kindatDict[kinst].keys()) yearList.sort() for year in yearList: kindatDict[kinst][year].sort() kindatStrList = [str(kindat) for kindat in kindatDict[kinst][year]] kindatStr = delimiter.join(kindatStrList) newFileStr += '%i,%s,%i\n' % (kinst, kindatStr, year) # finally, write new instKindatTab.txt newFile = open(filename, 'w') newFile.write(newFileStr) newFile.close class MadrigalExperiment: """MadrigalExperiment is an object that provides access to Madrigal experiment info from the metadata. This object provides access to all Madrigal experiment information in the metadata file expTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalExperiment() print test.getExperimentName(3001) except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Apr. 17, 2002 """ #constants __expMetadataFile = "expTab.txt" # column positions __expIdCol = 0 __expUrlCol = 1 __expNameCol = 2 __expSiteIdCol = 3 __expStartDateCol = 4 __expStartTimeCol = 5 __expEndDateCol = 6 __expEndTimeCol = 7 __expKinstCol = 8 __expSecurityCol = 9 __expPICol = 10 __expPIEmailCol = 11 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalExperiment by reading from expTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/expTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get experiment metadata file if (initFile == None): self.__filename = self.__expMetadataFile else: self.__filename = initFile # MadrigalExperiment can now legal have two different lengths - with or without PI and email allowedLens = (self.__expSecurityCol+1, self.__expPIEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__expIdCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expIdCol] = str(int(expId)) self._dict[position] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str((position, expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpUrlByPosition(self, position = 0): """getExpUrlByPosition returns the experiment url of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment url, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expUrlCol] else: return(None) def getExpUrlByExpId(self, expId): """getExpUrlByExpId returns the experiment url for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpUrlByPosition(position)) def getRealExpUrlByPosition(self, position = 0): """getRealExpUrlByPosition returns the real experiment url of the experiment at given position. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the real experiment url, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index == -1: return(thisUrl) if thisUrl.find('experiments') != -1: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + thisUrl[index+8:]) else: realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + 'experiments', thisUrl[index+8:]) return(realUrl) def getRealExpUrlByExpId(self, expId): """getRealExpUrlByExpId returns the real experiment url for a given experiment id. The url in the metadata contains /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the real experiment url (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getRealExpUrlByPosition(position)) def getExpPathByPosition(self, position = 0): """getExpPathByPosition returns the experiment path of the experiment at given position. Experiment path is the path to the experiment from the madroot directory, and always begins "experiments" Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments. Affects: None Exceptions: """ thisUrl = self.getExpUrlByPosition(position) if not thisUrl: return(None) index = thisUrl.find('/madtoc/') if index != -1: return(thisUrl[index+8:]) index = thisUrl.find('experiments') if index == -1: return(None) return(thisUrl[index:]) def getExpPathByExpId(self, expId): """getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id. The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one. Inputs: Experiment Id (integer). Returns: the path to the experiment from the madroot directory (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpPathByPosition(position)) def setExpUrlByPosition(self, position, expUrl): """setExpUrlByPosition sets the experiment url of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expUrl - the new experiment url to use Returns: None. Affects: sets the experiment url of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expUrl.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_exc()])) try: self.__fileList[position][self.__expUrlCol] = str(expUrl) except: raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' % \ (str(position, expUrl), [traceback.format_ex()])) def getExpDirByPosition(self, position = 0): """getExpDirByPosition returns the full experiment directory of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the full experiment directory, or None if position >= number of experiments. Uses experiment url to determine directory. Affects: None Exceptions: None """ if len(self.__fileList) > position: url = self.__fileList[position][self.__expUrlCol] # find the directory based on url maddir = self.__madDB.getMadroot() index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: maddir = os.path.join(maddir, 'experiments', url[index+8:]) else: maddir = os.path.join(maddir, url[index+8:]) return(maddir) else: return(None) def getExpDirByExpId(self, expId): """getExpDirByExpId returns the full experiment directory for a given experiment id. Inputs: Experiment Id (integer). Returns: the full experiment directory (string). Returns None if experiment id not found. Uses experiment url to determine directory. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpDirByPosition(position)) def getExpNameByPosition(self, position = 0): """getExpNameByPosition returns the experiment name of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment name, or None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__expNameCol] else: return(None) def getExpNameByExpId(self, expId): """getExpNameByExpId returns the experiment name for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment name (string). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpNameByPosition(position)) def setExpNameByPosition(self, position, expName): """setExpNameByPosition sets the experiment name of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expName - the new experiment name to use Returns: None. Affects: sets the experiment name of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ # verify no illegal commas if expName.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), [traceback.format_exc()])) try: self.__fileList[position][self.__expNameCol] = str(expName) except: raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' % \ (str(position, expName), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpSiteIdByExpId(self, expId): """getExpSiteIdByExpId returns the site id (int) for a given experiment id. Inputs: Experiment Id (integer). Returns: the site id for this experiment. Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpSiteIdByPosition(position)) def getExpSiteIdByPosition(self, position = 0): """getExpSiteIdByPosition returns the experiment site id of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment site id (integer), or None if position >= number of experiments. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSiteIdCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpSiteIdByPosition(self, position, expSiteId): """setExpSiteIdByPosition sets the experiment site id of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expSiteId - the new experiment site id to use Returns: None. Affects: sets the experiment site id of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSiteIdCol] = str(int(expSiteId)) except: raise madrigal.admin.MadrigalError('Error in setExpSiteIdByPosition with args %s: %s' % \ (str(position, expSiteId), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getExpStartDateTimeByPosition(self, position = 0): """getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings startTime = [int(self.__fileList[int(position)][self.__expStartDateCol][0:4]), int(self.__fileList[int(position)][self.__expStartDateCol][4:6]), int(self.__fileList[int(position)][self.__expStartDateCol][6:8]), int(self.__fileList[int(position)][self.__expStartTimeCol][0:2]), int(self.__fileList[int(position)][self.__expStartTimeCol][2:4]), int(self.__fileList[int(position)][self.__expStartTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if startTime[3] == 24: startTime[3] = 23 startTime[4] = 59 startTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(startTime[0], startTime[1], startTime[2], startTime[3], startTime[4], startTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) startTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return startTime else: return(None) def getExpEndDateTimeByPosition(self, position = 0): """getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position. Inputs: position of experiment in list (first position is zero). Defaults to first. Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ position = int(position) if len(self.__fileList) > position: # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings endTime = [int(self.__fileList[position][self.__expEndDateCol][0:4]), int(self.__fileList[position][self.__expEndDateCol][4:6]), int(self.__fileList[position][self.__expEndDateCol][6:8]), int(self.__fileList[position][self.__expEndTimeCol][0:2]), int(self.__fileList[position][self.__expEndTimeCol][2:4]), int(self.__fileList[position][self.__expEndTimeCol][4:6]), 0, 0, 0] # handle hour = 24 case if endTime[3] == 24: endTime[3] = 23 endTime[4] = 59 endTime[5] = 59 # we still need day of year utcTime = madrigal.metadata.getMadrigalUTFromDate(endTime[0], endTime[1], endTime[2], endTime[3], endTime[4], endTime[5], 0) utcDate = madrigal._derive.getDateFromUt(utcTime) endTime[7] = utcDate[7] # return python time tuple, missing only day of week and DST flag return endTime else: return(None) def getExpStartDateTimeByExpId(self, expId): """getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpStartDateTimeByPosition(position)) def getExpEndDateTimeByExpId(self, expId): """getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC. Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpEndDateTimeByPosition(position)) def setExpStartDateTimeByPosition(self, startDateTime, position = 0): """setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position. Inputs: startDateTime - a python datetime object to set the exp start date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp start date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expStartDateCol] = '%04i%02i%02i' % (startDateTime.year, startDateTime.month, startDateTime.day) self.__fileList[position][self.__expStartTimeCol] = '%02i%02i%02i' % (startDateTime.hour, startDateTime.minute, startDateTime.second) def setExpEndDateTimeByPosition(self, endDateTime, position = 0): """setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position. Inputs: endDateTime - a python datetime object to set the exp end date and time to. position - which experiment row to change - defaults to 0 Returns: None. Affects: sets exp end date and time in self.__fileList. Exceptions: None """ self.__fileList[position][self.__expEndDateCol] = '%04i%02i%02i' % (endDateTime.year, endDateTime.month, endDateTime.day) self.__fileList[position][self.__expEndTimeCol] = '%02i%02i%02i' % (endDateTime.hour, endDateTime.minute, endDateTime.second) def getKinstByPosition(self, position = 0): """getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (instrument code) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expKinstCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getKinstByExpId(self, expId): """getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the experiment kinst (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getKinstByPosition(position)) def setExpKinstByPosition(self, position, expKinst): """setExpKinstByPosition sets the experiment kinst of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). expKinst - the new experiment kinst to use Returns: None. Affects: sets the experiment kinst of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expKinstCol] = str(int(expKinst)) except: raise madrigal.admin.MadrigalError('Error in setExpKinstByPosition with args %s: %s' % \ (str(position, expKinst), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getSecurityByPosition(self, position = 0): """getSecurityByPosition returns the security code of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the security code (integer) of the file at given position as an integer. Returns None if position >= number of files. Affects: None Exceptions: Thrown if security column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__expSecurityCol]) except: raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getSecurityByExpId(self, expId): """getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the security code (integer). Returns None if experiment id not found. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getSecurityByPosition(position)) def setSecurityByPosition(self, position, securityCode): """setSecurityByPosition sets the security code (integer) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). securityCode - the new experiment security code (integer) to use Returns: None. Affects: sets the security code of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__expSecurityCol] = str(int(securityCode)) except: raise madrigal.admin.MadrigalError('Error in setSecurityByPosition with args %s: %s' % \ (str(position, securityCode), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))) def getPIByPosition(self, position = 0): """getPIByPosition returns the principal investigator of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's name (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: pi = self.__fileList[position][self.__expPICol] if len(pi) > 0: return(pi) else: return(None) except: return(None) else: return(None) def getPIByExpId(self, expId): """getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's name (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIByPosition(position)) def setPIByPosition(self, position, PI): """setPIByPosition sets the principal investigator (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PI - the new experiment principal investigator's name (string) to use Returns: None. Affects: sets the principal investigator of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PI.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add PI and blank email self.__fileList[position].append(str(PI)) self.__fileList[position].append('') else: try: self.__fileList[position][self.__expPICol] = str(PI) except: raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' % \ (str(position, PI), [traceback.format_exc()])) def getPIEmailByPosition(self, position = 0): """getPIEmailByPosition returns the principal investigator email of the experiment at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the principal investigator's email (string) of the file at given position as a string. Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ if len(self.__fileList) > position: try: piEmail = self.__fileList[position][self.__expPIEmailCol] if len(piEmail) > 0: return(piEmail) else: return(None) except: return(None) else: return(None) def getPIEmailByExpId(self, expId): """getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id. Inputs: Experiment Id (integer). Returns: the principal investigator's email (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None This method added in Madrigal 2.6 """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getPIEmailByPosition(position)) def setPIEmailByPosition(self, position, PIEmail): """setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position. Inputs: position - position of experiment in list (first position is zero). PIEmail - the new experiment principal investigator's email (string) to use Returns: None. Affects: sets the principal investigator email of the experiment at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. This method added in Madrigal 2.6 """ # verify no illegal commas if PIEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()])) if len(self.__fileList[position]) == self.__expPICol: # add blank PI and email self.__fileList[position].append('') self.__fileList[position].append(str(PIEmail)) else: try: self.__fileList[position][self.__expPIEmailCol] = str(PIEmail) except: raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' % \ (str(position, PIEmail), [traceback.format_exc()])) def getExpLinksByExpId(self, expId): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: Experiment Id (integer). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ try: position = self._dict[str(expId)] except KeyError: return(None) return(self.getExpLinksByPosition(position)) def getExpLinksByPosition(self, position = 0): """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment. Inputs: Inputs: position - position of experiment in list (first position is zero). Returns: a list of (title, url) tuples containing all links for this experiment. In order to be a link, a file must be in the experiment directory in the form *.html, */index.html, or */*/index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename. Affects: None Exceptions: None """ retList = [] allowedExtensions = ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif', 'tar.gz'] # find the experiment directory based on url topdir = self.__madDB.getMadroot() url = self.getExpUrlByPosition(position) if url == None: return retList index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: expdir = os.path.join(topdir, 'experiments', url[index+8:]) else: expdir = os.path.join(topdir, url[index+8:]) # now create a list of html files htmlFiles = [] l1 = glob.glob(os.path.join(expdir, '*.html')) l2 = glob.glob(os.path.join(expdir, '*/index.html')) l3 = glob.glob(os.path.join(expdir, '*/*/index.html')) for item in l1: htmlFiles.append(item) for item in l2: htmlFiles.append(item) for item in l3: htmlFiles.append(item) for htmlFile in htmlFiles: # get title if possible f = open(htmlFile) text = f.read() f.close() i1 = text.upper().find('') + len('<TITLE>') i2 = text.upper().find('') if i1 > -1 and i2 > -1: title = text[i1:i2] else: title = 'No title' # get url i1 = htmlFile.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', htmlFile[i1:]) retList.append((title, url)) for extension in allowedExtensions: plotList = glob.glob(os.path.join(expdir, '*.%s' % (extension))) for plot in plotList: i1 = plot.find('experiments') url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', plot[i1:]) retList.append((os.path.basename(plot), url)) return retList def getExpCount(self): """getExpCount returns number of experiments in MadrigalExperiment object """ return(len(self.__fileList)) def sortByDateSite(self): """sortByDateSite will resort self.__fileList so that experiments are listed first by experiment end date, and then by site """ self.__fileList.sort(key=self.__compareDateSite__) # rebuild self._dict self._dict = {} for i, items in enumerate(self.__fileList): self._dict[items[self.__expIdCol]] = i def getStartPosition(self, startDT): """getStartPosition returns the position of the first experiment with a start datetime after endDT after sorting by end time. Inputs: startDT - start datetime Returns: returns the position of the first experiment with a start datetime after startDT. """ self.sortByDateSite() top = len(self.__fileList) - 1 bottom = 0 while(top >= bottom): if top == bottom: return(max(top-1, 0)) middle = int((top+bottom)/2) thisEDTList = self.getExpEndDateTimeByPosition(middle) thisEDT = datetime.datetime(*thisEDTList[0:6]) if thisEDT < startDT: bottom = middle + 1 continue if middle == 0: return(0) prevEDTList = self.getExpEndDateTimeByPosition(middle-1) prevEDT = datetime.datetime(*prevEDTList[0:6]) if prevEDT < startDT: return(middle-1) else: top = middle -1 def __compareDateSite__(self, first): """__compareDateSite__ is a private method to help sort by start date and then site. first, second - lists of experiment information as parsed from expTab.txt files """ result = (first[self.__expEndDateCol]) if result != 0: return(result) result = (first[self.__expEndTimeCol]) if result != 0: return(result) result = (first[self.__expStartDateCol]) if result != 0: return(result) result = (first[self.__expStartTimeCol]) if result != 0: return(result) return((first[self.__expSiteIdCol])) def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n') def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the expTab.txt file. Inputs: newFullPath: a new path to write the expTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def __str__(self): """return possibly modified file as a string in same format as writeMetadata """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' return(metaFileStr) class MadrigalMetaFile: """MadrigalMetaFile is an object that provides access to Madrigal file info from the metadata. This object provides access to all Madrigal experiment information in the metadata file fileTab.txt. Usage example:: import madrigal.metadata import madrigal.admin try: test = madrigal.metadata.MadrigalMetaFile() print test.getFileCount() except madrigal.admin.MadrigalError, e: print e.getExceptionStr() Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu May. 7, 2002 """ #constants __fileMetadataFile = "fileTab.txt" # column positions __fileNameCol = 0 __fileExpIdCol = 1 __fileKindatCol = 2 __fileCategoryCol = 3 __fileSizeCol = 4 __fileHasCatalogCol = 5 __fileHasHeaderCol = 6 __fileModDateCol = 7 __fileModTimeCol = 8 __fileStatusCol = 9 __fileAccessCol = 10 __fileAnalystCol = 11 __fileAnalystEmailCol = 12 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalMetaFile by reading from fileTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/fileTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get file metadata file if (initFile == None): self.__filename = self.__fileMetadataFile else: self.__filename = initFile # MadrigalMetafile can now legal have two different lengths - with or without file analyst and email allowedLens = (self.__fileAccessCol+1, self.__fileAnalystEmailCol+1) genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__fileNameCol) self.__fileList = genericObj.getList() self._dict = genericObj.getDict() # check that at least first row has right number of columns if len(self.__fileList) > 0: # analyst and analyst email columns may or may not exist if len(self.__fileList[0]) not in allowedLens: raise madrigal.admin.MadrigalError('Error in count of first row of ' + str(self.__filename), None) def getFileCount(self): """getFileCount returns the number of files (rows) in the metadata file. Inputs: None Returns: the number of files (rows) in the metadata file. Affects: None Exceptions: None """ return len(self.__fileList) def getFilenameByPosition(self, position = 0): """getFilenameByPosition returns the filename of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the filename of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileNameCol] else: return(None) def getExpIdByPosition(self, position = 0): """getExpIdByPosition returns the experiment id (integer) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the experiment id (integer) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst exp id cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileExpIdCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def setExpIdByPosition(self, position, expId): """setExpIdByPosition sets the experiment id of the file at given position. Inputs: position - position of file in list (first position is zero). expId - the new experiment id to use Returns: None. Affects: sets the experiment id of the file at given position Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found. """ try: self.__fileList[position][self.__fileExpIdCol] = str(int(expId)) except: raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' % \ (str(position), str(expId)), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def getKindatByPosition(self, position = 0): """getKindatByPosition returns the kindat (kind of data code) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the kinst (kind of instrument code) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if kinst column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileKindatCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getCategoryByPosition(self, position = 0): """getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileCategoryCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getCategoryByFilename(self, filename): """getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename. Inputs: filename - name of file to search for. Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if category column cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileCategoryCol])) def getHasCatalogByPosition(self, position = 0): """getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any catalog records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasCatalogCol]) == 0: return False else: return True else: return(None) def getHasCatalogByFilename(self,filename): """getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any catalog records, False otherwise. Returns none if name not found Affects: None Exceptions: None. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasCatalogCol])) def setHasCatalogByPosition(self, position, hasCatalog): """setHasCatalogByPosition sets the value of hasCatalog for the file at the given position. Inputs: position - position of file in list (first position is zero). hasCatalog - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasCatalog not in (0, 1, True, False): raise ValueError('Illegal value for hasCatalog in setHasCatalogByPosition: %s' % (str(hasCatalog))) if len(self.__fileList) > position: if hasCatalog in (1, True): self.__fileList[position][self.__fileHasCatalogCol] = '1' else: self.__fileList[position][self.__fileHasCatalogCol] = '0' else: raise ValueError('setHasCatalogByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def getHasHeaderByPosition(self, position = 0): """getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: True if the file at given position has any header records, False otherwise Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: if int(self.__fileList[position][self.__fileHasHeaderCol]) == 0: return False else: return True else: return(None) def getHasHeaderByFilename(self,filename): """getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise. Inputs: filename - name of file to search for. Returns: true if the file with the given name has any header records, False otherwise. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileHasHeaderCol])) def setHasHeaderByPosition(self, position, hasHeader): """setHasHeaderByPosition sets the value of hasHeader for the file at the given position. Inputs: position - position of file in list (first position is zero). hasHeader - 1 or True for yes, 0 or False for no Returns: None. Affects: None Exceptions: If position beyond length. """ if hasHeader not in (0, 1, True, False): raise ValueError('Illegal value for hasHeader in setHasHeaderByPosition: %s' % (str(hasHeader))) if len(self.__fileList) > position: if hasHeader in (1, True): self.__fileList[position][self.__fileHasHeaderCol] = '1' else: self.__fileList[position][self.__fileHasHeaderCol] = '0' else: raise ValueError('setHasHeaderByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def getStatusByPosition(self, position = 0): """getStatusByPosition returns the status description of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the status description of the file at given position as a string. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: return self.__fileList[position][self.__fileStatusCol] else: return(None) def getStatusByFilename(self,filename): """getStatusByFilename returns the status description of the file with the given name. Inputs: filename - name of file to search for. Returns: the status description of the file with the given name. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.__fileList[position][self.__fileStatusCol]) def getAccessByPosition(self, position = 0): """getAccessByPosition returns the access (0=public, 1=private) of the file at given position. Inputs: position of file in list (first position is zero). Defaults to first. Returns: the access level (0=public, 1=private) of the file at given position as an integer. Returns None if position >= number of experiments. Affects: None Exceptions: Thrown if access column cannot be parsed into an integer """ if len(self.__fileList) > position: try: return int(self.__fileList[position][self.__fileAccessCol]) except: raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) else: return(None) def getFileDatetimeByPosition(self, position = 0): """getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set. Inputs: position of file in list (first position is zero). Defaults to first. Returns: a datetime of the file at given position, or None if not set. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: dateStr = self.__fileList[position][self.__fileModDateCol] if len(dateStr) < 8: return(None) timeStr = self.__fileList[position][self.__fileModTimeCol] if len(timeStr) < 6: return(None) return(datetime.datetime.strptime('%s %s' % (dateStr, timeStr), '%Y%m%d %H%M%S')) else: return(None) def getFileDatetimeByFilename(self,filename): """getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set. Inputs: filename - name of file to search for. Returns: a datetime of the file with the given filenme, or None if not set. Returns none if name not found Affects: None Exceptions: Thrown if filename not found. """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(self.getFileDatetimeByPosition(position)) def setFileDatetimeByPosition(self, position, dt): """setFileDatetimeByPosition sets the file datetime for the given position Inputs: position - position of file in list (first position is zero). dt - datetime (UT) of file being added. If None, use file datetime. Will raise exception if dt is None and fileTab.txt in the metadata version, not the expDir version. Returns: None. Affects: None Exceptions: If position beyond length. """ if dt is None: basename = self.getFilenameByPosition(position) expDir = os.path.dirname(self.__filename) if len(expDir) < 2: raise IOError('Cannot set datetime to None using the metadata version of fileTab.txt') dataFile = os.path.join(expDir, basename) if not os.access(dataFile, os.R_OK): raise IOError('Cannot access %s' % (dataFile)) dt = datetime.datetime.utcfromtimestamp(os.path.getmtime(dataFile)) dateStr = dt.strftime('%Y%m%d') timeStr = dt.strftime('%H%M%S') if len(self.__fileList) > position: self.__fileList[position][self.__fileModDateCol] = dateStr self.__fileList[position][self.__fileModTimeCol] = timeStr else: raise ValueError('setFileDatetimeByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def deleteRowByFilename(self, filename): """deleteRowByFilename deletes a row with a given filename. Inputs: filename - name of file to search for. Returns: None. Affects: Removes item from self.__fileList if filename found Exceptions: Thrown if filename not found. """ for file in self.__fileList: if filename == file[self.__fileNameCol]: self.__fileList.remove(file) return # no matches found raise madrigal.admin.MadrigalError('Could not delete file ' + filename + ' from ' + self.__filename, None) def getExpIdByFilename(self, filename): """getExpIdByFilename returns the first experiment id (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the experiment id (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if exp id cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileExpIdCol])) def getKindatByFilename(self, filename): """getKindatByFilename returns the first kindat (integer) with the given filename. Inputs: filename - name of file to search for. Returns: the kindat (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None. Affects: None Exceptions: Thrown if kindat cannot be parsed into an integer """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) return(int(self.__fileList[position][self.__fileKindatCol])) def setAccessByPosition(self, position, access): """setAccessByPosition sets the value of access for the file at the given position. Inputs: position - position of file in list (first position is zero). access - 0 of False for public, 1 or True for private Returns: None. Affects: None Exceptions: If position beyond length. """ if access not in (0, 1, True, False): raise ValueError('Illegal value for access in setAccessByPosition: %s' % (str(access))) if len(self.__fileList) > position: if access in (1, True): self.__fileList[position][self.__fileAccessCol] = '1' else: self.__fileList[position][self.__fileAccessCol] = '0' else: raise ValueError('setAccessByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def setAccess(self, accessType): """setAccess sets the access column to all 0's (public) or all 1's (private). Inputs: accessType - either 0 to set to public access, or 1 to set to private access. Returns: None. Affects: Overwrite fileTab.txt file with access column set to all 0's (public) or all 1's (private). Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1 """ if (accessType != 0 and accessType != 1): raise madrigal.admin.MadrigalError('MadrigalMetaFile.setAccess called with arg = ' + \ str(accessType) + ', must be either 0 or 1', None) for row in self.__fileList: row[self.__fileAccessCol] = str(accessType) self.writeMetadata() def setKindatByPosition(self, position, kindat): """setKindatByPosition sets the value of kindat for the file at the given position. Inputs: position - position of file in list (first position is zero). kindat - integer kindat value Returns: None. Affects: None Exceptions: If position beyond length. """ kindat = int(kindat) if len(self.__fileList) > position: self.__fileList[position][self.__fileKindatCol] = str(kindat) else: raise ValueError('setKindatByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def setCategoryByPosition(self, position, category): """setCategoryByPosition sets the value of category for the file at the given position. Inputs: position - position of file in list (first position is zero). category - 1=default, 2=variant, 3=history, 4=real-time Returns: None. Affects: None Exceptions: If position beyond length. """ if int(category) not in (1, 2, 3, 4): raise ValueError('Illegal value for category in setCategoryByPosition: %s' % (str(category))) if len(self.__fileList) > position: self.__fileList[position][self.__fileCategoryCol] = str(category) else: raise ValueError('setCategoryByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def setStatusByPosition(self, position, status): """setStatusByPosition sets the value of status string for the file at the given position. Inputs: position - position of file in list (first position is zero). status - string describing status Returns: None. Affects: None Exceptions: If position beyond length. """ statusTypes = [bytes, str] if type(status) not in statusTypes: raise ValueError('Illegal value for status in setStatusByPosition: %s' % (str(status))) # check that string does not illegally contain a comma if status.find(',') != -1: raise ValueError('status string in fileTab.txt cannot contain a comma: <%s> is illegal' % (status)) if len(self.__fileList) > position: self.__fileList[position][self.__fileStatusCol] = status else: raise ValueError('setStatusByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def getAnalystByPosition(self, position = 0): """getAnalystByPosition returns file analyst name if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None) else: return(None) def getAnalystByFilename(self, filename): """getAnalystByFilename returns file analyst name if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystCol]) except: return(None) def setAnalystByPosition(self, position, analyst): """setAnalystByPosition sets the file analyst name for the given position Inputs: position - position of file in list (first position is zero). analyst - name of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analyst.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystByPosition with args %s: %s' % \ (str(position), analyst), [traceback.format_exc()]) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol+1: # append two extra columns self.__fileList[position].append(str(analyst)) self.__fileList[position].append('') else: self.__fileList[position][self.__fileAnalystCol] = str(analyst) else: raise ValueError('setAnalystByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def getAnalystEmailByPosition(self, position = 0): """getAnalystEmailByPosition returns file analyst email if there, None otherwise. Inputs: position of file in list (first position is zero). Defaults to first. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments. Affects: None Exceptions: None """ if len(self.__fileList) > position: try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None) else: return(None) def getAnalystEmailByFilename(self, filename): """getAnalystEmailByFilename returns file analyst email if there, None otherwise. Inputs: filename - name of file to search for. Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Affects: None Exceptions: None """ filename = os.path.basename(filename) try: position = self._dict[filename] except KeyError: return(None) try: return(self.__fileList[position][self.__fileAnalystEmailCol]) except: return(None) def setAnalystEmailByPosition(self, position, analystEmail): """setAnalystEmailByPosition sets the file analyst email for the given position Inputs: position - position of file in list (first position is zero). analystEmail - email of file analyst Returns: None. Affects: None Exceptions: If position beyond length. """ # verify no illegal commas if analystEmail.find(',') != -1: raise madrigal.admin.MadrigalError('Error in setAnalystEmailByPosition with args %s: %s' % \ (str(position, analystEmail), [traceback.format_exc()])) if len(self.__fileList) > position: if len(self.__fileList[position]) == self.__fileAccessCol + 1: # append two extra columns self.__fileList[position].append('') self.__fileList[position].append(str(analystEmail)) else: self.__fileList[position][self.__fileAnalystEmailCol] = str(analystEmail) else: raise ValueError('setAnalystEmailByPosition called for position %i beyond length %i' % (position, len(self.__fileList))) def getMetadataSummaryByFilename(self, filename, madExpObj = None, madInstObj = None, madKindatObj = None): """getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename. Inputs: filename - name of file to search for. The next three inputs are other metadata objects. If these objects already exist, performance will be improved by passing them in rather than recreating them. If they do not exist, they will be created: madExpObj - a MadrigalExperiment object to get experiment metadata from. If None (the default), a new MadrigalExperiment object is created. madInstObj - a MadrigalInstrument object to get experiment metadata from. If None (the default), a new MadrigalInstrument object is created. madKindatObj - a MadrigalKindat object to get experiment metadata from. If None (the default), a new MadrigalKindat object is created. Returns: A string summarizing the metadata about the file. The format is:: Start Date and Time: 01/06/1997 14:07 End Date and Time: 01/10/1997 23:45 Instrument: Millstone Hill Incoherent Scatter Radar Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study Kind of data: INSCAL (8.0) Basic Derived Parameters Affects: None Exceptions: Thrown if any parsing error in metadata. """ # build return String retStr = '' # create any needed metadata objects if madExpObj == None: madExpObj = MadrigalExperiment(self.__madDB) if madInstObj == None: madInstObj = MadrigalInstrument(self.__madDB) if madKindatObj == None: madKindatObj = MadrigalKindat(self.__madDB) # get the experiment id - if None, return message to update Metadata expId = self.getExpIdByFilename(filename) if expId == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' # get the experiment start date - if not found, return message to update Metadata starttime = madExpObj.getExpStartDateTimeByExpId(expId) if starttime == None: return '\tMetadata for file ' + str(filename) + \ ' not found. Please notify Madrigal administrator that metadata ' + \ 'needs to be updated.' retStr = retStr + '\tExperiment start: ' + time.asctime(starttime) + '\n' # endtime endtime = madExpObj.getExpEndDateTimeByExpId(expId) retStr = retStr + '\tExperiment end: ' + time.asctime(endtime) + '\n' # get instrument name kinst = madExpObj.getKinstByExpId(expId) # if kinst not found, metadata is corrupt if kinst == None: return 'Could not find kinst for experiment id = ' + str(expId) instName = madInstObj.getInstrumentName(kinst) retStr = retStr + '\tInstrument name: ' + str(instName) + '\n' # get experiment name expName = madExpObj.getExpNameByExpId(expId) retStr = retStr + '\tExperiment name: ' + str(expName) + '\n' # get kind of data kindat = self.getKindatByFilename(filename) kindatName = madKindatObj.getKindatDescription(kindat, kinst) retStr = retStr + '\tKind of data: ' + str(kindatName) + '\n' return retStr def getFileDOIUrlByPosition(self, position): """getFileDOIUrlByPosition returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByPosition(position) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) basename = self.getFilenameByPosition(position) if basename is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, basename)) def getFileDOIUrlByFilename(self, filename): """getFileDOIUrlByFilename returns the full url to the file on the archive site. Meant to be used as a referencable url, so always refers to cedar.openmadigal.org Returns permanent URL to file, or None if not found """ # get expDir if self.__fileMetadataFile != self.__filename: # local copy expDir = os.path.dirname(self.__filename) else: expId = self.getExpIdByFilename(filename) if expId is None: return(None) # need a MadrigalExperiment object madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB) expDir = madExpObj.getExpDirByExpId(expId) if expDir is None: return(None) # change so that it starts with experiments expDir = expDir[expDir.find('experiments'):] url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s' return(url_template % (expDir, filename)) def getLine(self, position): """getLine returns the line at a given position. Returns None if position > number of lines. Inputs: position - position in file. First line = 0 """ delimiter = ',' if position >= len(self.__fileList): return(None) return(delimiter.join(self.__fileList[position]) + '\n') def writeMetadata(self, newFullPath=None): """writeMetadata writes a new version of the fileTab.txt file. Inputs: newFullPath: a new path to write the fileTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from. Returns: None. Affects: Writes updated version of metadata file. Exceptions: If unable to write file """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' # try to write file, if not, raise exception try: if newFullPath == None: if (len(os.path.dirname(self.__filename)) != 0): newFullPath = self.__filename else: newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename newFile = open(newFullPath, "w") newFile.write(metaFileStr) newFile.close() except: raise madrigal.admin.MadrigalError("Unable to write metadata file " + \ str(newFullPath), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def __str__(self): """return possibly modified file as a string in same format as writeMetadata """ # create string to hold file metaFileStr = '' delimiter = ',' for lineList in self.__fileList: metaFileStr += delimiter.join(lineList) + '\n' return(metaFileStr) class MadrigalParmCategory: """MadrigalParmCategory is an object that provides access to Madrigal parameter category info from the metadata. This object provides access to all Madrigal kind of data information in the metadata file madCatTab.txt. Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jan. 18, 2013 """ #constants _categoryMetadataFile = "madCatTab.txt" # column positions _categoryCodeCol = 0 _categoryDescCol = 1 def __init__(self, madDB=None, initFile=None): """__init__ initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile). Inputs: Existing MadrigalDB object, by default = None. String representing the full path to the metadata file. Default is None, in which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # get category metadata file if (initFile == None): self.__filename = self._categoryMetadataFile else: self.__filename = initFile self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getCategoryDesc(self, code): """getCategoryDesc returns the category description that matches code argument, or None if not found. Inputs: code integer to get category description. or integer as string Returns: the category description that matches code argument, or None if not found. Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ code = int(code) for i, category in enumerate(self.__fileList): # find matching code try: if code == int(category[self._categoryCodeCol]): return(category[self._categoryDescCol]) except: raise madrigal.admin.MadrigalError('Error in madCatTab.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # not found return(None) def getCategoryList(self): """getCategoryList returns a list of all category descriptions and codes. Inputs: None. Returns: a list of all category descriptions and codes. Each item in the list is a tuple of the form (Category description (string), code (integer)). Example item: ('INSCAL Basic Derived Parameters', 3001) Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, category in enumerate(self.__fileList): item = (category[self._categoryDescCol], int(category[self._categoryCodeCol])) retList.append(item) return(retList) class MadrigalInstrumentData: """MadrigalInstrumentData is an object that provides access to Madrigal instrument data info from the metadata. This object provides access to all Madrigal kind of data information in the metadata files instData.txt and instDataPriv.txt. Those files summarize years data is available by instrument. Non-standard Python modules used: None MadrigalError exception thrown if: 1. MadrigalMetadata fails to open or parse metadata file 2. Columns expected to be ints or floats cannot be converted Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Feb. 2, 2015 """ #constants _instDataMetadataFile = "instData.txt" # column positions _siteIDCol = 0 _kinstCol = 1 _yearsCol = 2 def __init__(self, madDB=None, priv=False, initFile=None, madInstObj=None): """__init__ initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt). Inputs: madDB - Existing MadrigalDB object, by default = None. priv - if True, use instDataPriv.txt instead of instData.txt. If False (the default), use instData.txt. initFile - String representing the full path to the metadata file. Default is None, in which case file read depends on priv argument. priv arg ingored if this not None madInstObj - m madrigal.metadata.MadrigalInstrument object. If None, one is created. Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB if priv: self._instDataMetadataFile = "instDataPriv.txt" # get instData metadata file if (initFile == None): self.__filename = self._instDataMetadataFile else: self.__filename = initFile if madInstObj is not None: self._madInstObj = madInstObj else: self._madInstObj = madrigal.metadata.MadrigalInstrument(self.__madDB) self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList() def getCategories(self, local=False): """getCategories returns the a list of (category id, category desc) tuples. Inputs: local - if False, return all categories for which there is data anywhere in Madrigal. If True, only return local categories. Returns: a list of (category id, category desc) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is category id, value is category desc str. Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: if local: if int(data[self._siteIDCol]) != siteID: continue kinst = int(data[self._kinstCol]) categoryID = self._madInstObj.getCategoryId(kinst) if categoryID not in list(localDict.keys()): localDict[categoryID] = self._madInstObj.getCategory(kinst) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) categoryIDKeys = list(localDict.keys()) categoryIDKeys.sort() retList = [(catID, localDict[catID]) for catID in categoryIDKeys] return(retList) def getInstruments(self, categoryID=0, local=False): """getInstruments returns the a list of (kinst, instrument desc, siteID) tuples. Inputs: categoryID - category id to return instruments with data for. If 0, return all local - if False, return all instruments with that category for which there is data anywhere in Madrigal. If True, only return local instruments, in which case siteID is always the local siteID. Returns: a list of (kinst, instrument desc, siteID) tuples Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ if local: siteID = self.__madDB.getSiteID() localDict = {} # key is kinst, value is tuple if (instrument desc str, siteID). # Will be converted to list before returned for i, data in enumerate(self.__fileList): # find matching code try: kinst = int(data[self._kinstCol]) thisSiteID = int(data[self._siteIDCol]) thisCategoryID = self._madInstObj.getCategoryId(kinst) if thisCategoryID != categoryID and categoryID != 0: continue if local: if thisSiteID != siteID: continue if kinst not in list(localDict.keys()): localDict[kinst] = (self._madInstObj.getInstrumentName(kinst), thisSiteID) else: raise ValueError('Duplicate kinst %i found in instData' % (kinst)) except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) kinstKeys = list(localDict.keys()) kinstKeys.sort() retList = [(kinst, localDict[kinst][0], localDict[kinst][1]) for kinst in kinstKeys] return(retList) def getInstrumentYears(self, kinst): """getInstrumentYears returns the a list of years (int) for instrument. Inputs: kinst - the instrument id Returns: an ordered list of years as integers. If none found, raises error Affects: None Exceptions: MadrigalError if any item in row cannot be cast to correct format """ retList = [] for i, data in enumerate(self.__fileList): # find matching code try: thisKinst = int(data[self._kinstCol]) if thisKinst != kinst: continue yearsStr = data[self._yearsCol] retList = [int(year) for year in yearsStr.split()] retList.sort() except: raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category), traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) if len(retList) == 0: raise madrigal.admin.MadrigalError('No data found for kinst %i' % (kinst), '') return(retList) class MadrigalMetadata: """MadrigalMetadata is a private class that parses a Madrigal metadata file. This private class is used by all classes that need to parse a Madrigal metadata file. If the class is called with the name of the metadata file only, the metadata file is assumed to be at $MAD_ROOT/metadata. If a full path name is given that includes a directory separator, then that is used instead. The getList method returns a list with one item for each line in the file. That item for each line is simply a list of strings found in the line. The following is an example metadata file and the list the method getList would return. Metadata file example:: Tom, Dick,,Harry ,,Joe, Sally, Jane,Joe, Dick The list returned by getList example:: [['Tom', 'Dick', '', 'Harry'], ['', '', 'Joe', ''], ['Sally', 'Jane', 'Joe', 'Dick']] Non-standard Python modules used: None MadrigalError exception thrown if: 1. Unable to open metadata file. 2. All lines in metadata file do not have same number of items Change history: Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 8, 2001 """ # constants # file delimiter - presently a comma __DELIMITER = ',' def __init__(self, metadataFileName, madDB=None, allowedLenList=None, key=None): """__init__ initializes the MadrigalCategoryList by reading from __privateList. Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir is included. Existing MadrigalDB object, by default = None. allowedLenList - a list of allowed lengths (integers) for data lines. If None (the default), then the rule is that all lengths must be the same. key - which position in the file to use as a key. Creates a attribute called self._dict, which is a dictionary with keys = item in key position, value = line index. Used for faster lookup by key. If None (the default), self._dict = None Returns: void Affects: Initializes private member variable __categoryList. Exceptions: None. """ # get metadata dir if madDB == None: self.__madDB = madrigal.metadata.MadrigalDB() else: self.__madDB = madDB # create empty list to hold parsed data self.__fileList = [] # used to check that every line has same number of words if allowedLenList == None self.__numWords = 0 self._allowedLenList = allowedLenList if not key is None: self._dict = {} else:self._dict = None # get real filename self.__fileName = self.__getFullPath(metadataFileName) # open configuration file try: self.__file = open(self.__fileName, "r") except IOError: raise madrigal.admin.MadrigalError("Unable to open metadata file " + self.__fileName, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # loop over each line in file, creating a list for each line = self.__file.readline() count = 0 while (len(line.strip())): self.__parseLine(line, key, count) line = self.__file.readline() count += 1 # close metadata file self.__file.close() def __parseLine(self, line, key=None, count=None): """__parseLine adds a list of items in line to __fileList. Inputs: Line of file to parse (String). key - position to add key for self._dict, value = line position If None (the default), ignore count - if None, ignore. Else add to self._dict as value (see key above) Returns: void Affects: Adds a list of items in line to self.__fileList. Exceptions: None. """ list = line.split(self.__DELIMITER) # create new list with leading and trailing whitespace stripped strippedList = [] for word in list: strippedList.append(word.strip()) self.__fileList.append(strippedList) if not self._dict is None: self._dict[list[key]] = count # check correct number of words if self._allowedLenList: if len(strippedList) not in self._allowedLenList: raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \ self.__fileName + " at line " + \ str(len(self.__fileList)), [traceback.format_exc()]) elif self.__numWords == 0: self.__numWords = len(strippedList) elif self.__numWords != len(strippedList): raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \ self.__fileName + " at line " + \ str(len(self.__fileList)), [traceback.format_exc()]) def __getFullPath(self, filename): """getFullPath returns the full path name of the metafile. Inputs: filename passed in as argument Returns: The full path name of the metafile. If the filename argument already is a full path, filename is returned unchanged. If the filename is simply the name of a metadata file, then $MAD_ROOT/metadata is appended Affects: Nothing Exceptions: None """ if (len(os.path.dirname(filename)) != 0): return filename fullName = self.__madDB.getMetadataDir() + "/" + filename # normalize in case we run on a system without / as a separator return os.path.normpath(fullName) # public methods def getList(self): """getList returns the list of lists of items in each line in the metafile. Inputs: None Returns: The list of lists of items in each line in the metafile. That is, each item in the returned list is itself a list, representing a single line in the metafile. That single line's list is the list of items in that line. Affects: Nothing Exceptions: None """ return(self.__fileList) def getDict(self): """getDict returns self_dict, which will be a dict with keys= key column set, value = line number, of None if no key passed into constructor """ return(self._dict) def toString(self): """toString returns a simple string representation of a MadrigalMetadata object. Inputs: None Returns: String describing a simple representation of a __MadrigalMetadat object. Affects: Nothing Exceptions: None """ return str(self.__fileList)

Functions

def getMadrigalUTFromDT(

dt)

getMadrigalUTFromDT returns the number of seconds since midnight UT 1950-01-01 for datetime dt

def getMadrigalUTFromDT(dt):
    """getMadrigalUTFromDT returns the number of seconds since midnight UT 1950-01-01 
    for datetime dt
    """
    t1950 = 631152000.0 # offset from Unix time
    return(calendar.timegm(dt.timetuple()) + t1950)

def getMadrigalUTFromDate(

year, month, day, hour, minute, second, microsecond)

getMadrigalUTFromDate returns the number of seconds since midnight UT 1950-01-01 for date specified in arguments

def getMadrigalUTFromDate(year, month, day, hour, minute, second, microsecond):
    """getMadrigalUTFromDate returns the number of seconds since midnight UT 1950-01-01 
    for date specified in arguments
    """
    dt = datetime.datetime(year, month, day, hour, minute, second, microsecond)
    return(getMadrigalUTFromDT(dt))

def getUnixUTFromDT(

dt)

getUnixUTFromDT returns the number of float seconds since midnight UT 1970-01-01 for datetime dt

def getUnixUTFromDT(dt):
    """getUnixUTFromDT returns the number of float seconds since midnight UT 1970-01-01 
    for datetime dt
    """
    return(calendar.timegm(dt.timetuple()) + dt.microsecond/1.0E6)

def getUnixUTFromDate(

year, month, day, hour, minute, second, microsecond)

getUnixUTFromDate returns the number of seconds since midnight UT 1970-01-01 for date specified in arguments

def getUnixUTFromDate(year, month, day, hour, minute, second, microsecond):
    """getUnixUTFromDate returns the number of seconds since midnight UT 1970-01-01 
    for date specified in arguments
    """
    dt = datetime.datetime(year, month, day, hour, minute, second, microsecond)
    return(getUnixUTFromDT(dt))

Classes

class MadrigalDB

MadrigalDB is an object that provides access to an entire Madrigal database.

This object provides complete high-level access to an entire Madrigal database. Presently, all its information comes from madrigal.cfg, or another path passed in by the user. If env variable madroot is not set, or if madrigal.cfg cannot be opened, the default values automatically editted during installation are used

Usage example::

import madrigal.metadata

try:

    test =  madrigal.metadata.MadrigalDB()

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

else:

    print test.toString()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1.  If the constructor is called with an configuration file path as an argument, and that file cannot be
    read.

2.  If the configuration file cannot be parsed by ConfigParser.

3.  If any of the following keys cannot be found in the configuration file:

    *__MAD_SERVER

    *__MAD_SERVERROOT

    *__SITE_ID

    *__HTML_STYLE

    *__INDEX_HEAD

    *__CON_TACTLINK

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Oct. 4, 2001

class MadrigalDB:
    """MadrigalDB is an object that provides access to an entire Madrigal database.

    This object provides complete high-level access to an entire Madrigal database.
    Presently, all its information comes from madrigal.cfg, or another path passed in by the
    user.  If env variable madroot is not set, or if madrigal.cfg cannot be opened, the
    default values automatically editted during installation are used

    Usage example::

        import madrigal.metadata
    
        try:
        
            test =  madrigal.metadata.MadrigalDB()
            
        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()
            
        else:
        
            print test.toString()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1.  If the constructor is called with an configuration file path as an argument, and that file cannot be
            read.
        
        2.  If the configuration file cannot be parsed by ConfigParser.
    
        3.  If any of the following keys cannot be found in the configuration file:
    
            *__MAD_SERVER

            *__MAD_SERVERROOT
            
            *__SITE_ID

            *__HTML_STYLE

            *__INDEX_HEAD

            *__CON_TACTLINK


    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Oct. 4, 2001

    """

    # lines that are edited during installation
    __hardCodeMadRoot          = '/Volumes/dropbox/mad31'
    __hardCodeMadServer        = 'localhost:8000'
    __hardCodeMadServerRoot    = '.'
    __hardCodeSiteId           = '999'
    __hardCodeHtmlStyle        = ''
    __hardCodeIndexHead        = 'Welcome to the Madrigal Database 
at Dropbox' __hardCodeContact = 'Madrigal administator
' __hardCodeMailServer = 'localhost' #constants __webProtocol = "http" """ Change the above string to use another web protocol such as https. """ __binDir = "/bin" """ Sets the relative path from madrigal root to the bin directory. """ __metadataDir = "/metadata" """ Sets the relative path from madrigal root to the metadata directory. """ __experimentDir = "/experiments" """ Sets the relative path from madrigal root to the experiment directory. """ __MAD_ROOT = "MAD" + "ROOT" """ Sets the name of the environment variable that points to the madrigal root directory. """ __confFileName = "madrigal.cfg" """ Sets the name of the default madrigal configuration file. """ __MAD_SERVER = "MAD" + "SERVER" """ Sets the key name in the configuration file to find the main madrigal url. """ __MAD_SERVERROOT = "MAD" + "SERVERROOT" """ Sets the key name in the configuration file to find the top level directory. """ __SITE_ID = "SITE" + "ID" """ Sets the key name in the configuration file to find the site id. """ __HTML_STYLE = "HTML" + "STYLE" """ Sets the key name in the configuration file to find the html body style tag. """ __INDEX_HEAD = "INDEX" + "HEAD" """ Sets the key name in the configuration file to find the heading in the top level madrigal page. """ __CON_TACTLINK = "CON" + "TACT" """ Sets the key name in the configuration file to find the contact link line. """ __MAIL_SERVER = "MAIL" + "SERVER" """ Sets the key name in the configuration file to find the mailserver. """ __PYTHON_EXE = "PYTHON" + "EXE" """ Sets the key name in the configuration file to find the python executable. """ def __init__(self, initFile=None): """__init__ initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString). Inputs: String representing the full path to the configuration file. Default is None, in which case configuration file is $(__MAD_ROOT)/__confFileName (now madrigal.cfg). Returns: void Affects: Initializes all the class member variables. Exceptions: MadrigalError thrown if error (see class description for specific conditions). Notes: Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile, this constructor goes about loading some basic data about the database, including information such as: -the database utility directory -the www home base Implemented using ConfigParser module. This reads ini type files. The only difference between ini files and madrigal.cfg is that ini files are broken into sections of the form: [sectionTitle] key1 = value1 key2 = value2 Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal" is appended to the beginning of the file. """ # if madroot not set, use hard coded madroot self.__madRootDir = os.environ.get(self.__MAD_ROOT) if (self.__madRootDir == None): self.__madRootDir = self.__hardCodeMadRoot # Set configuration file if (initFile == None): self.__confFilePath = self.__madRootDir + "/" + self.__confFileName else: self.__confFilePath = initFile # open configuration file try: self.__confFile = open(self.__confFilePath, "r") except IOError: # can't read from file - use all hard-coded values self.__initFromHardCode() self.__finishInit() return # create Parser using standard module ConfigParser self.__parser = configparser.ConfigParser() # read conf file into a StringIO with "[madrigal]\n" section heading prepended strConfFile = io.StringIO("[madrigal]\n" + self.__confFile.read()) # parse StringIO configuration file try: self.__parser.readfp(strConfFile) except: raise madrigal.admin.MadrigalError("Unable to parse configuration file " + self.__confFilePath, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) # read information from configuration file self.__readConfFile() # close conf file self.__confFile.close() self.__finishInit() # end __init__ def __finishInit(self): """__finishInit is a private helper function that finishes initialization by combining attributes to form new ones. Inputs: None Returns: Void. Affects: Initializes class member variables that are combinations of existing ones. Exceptions: None. """ # combine information # Set the database utility directory self.__databaseUtilityDirectory = self.__madRootDir + self.__binDir # Set the wwwHomeBase if self.__mainUrl.find('http') == -1: self.__wwwHomeBase = self.__webProtocol + "://" + self.__mainUrl else: self.__wwwHomeBase = self.__mainUrl # Set the top level url if self.__topLevel.strip() not in ('', '.'): self.__topLevelUrl = self.__wwwHomeBase + '/' + self.__topLevel else: self.__topLevelUrl = self.__wwwHomeBase # Set contactEmail by stripping ContactLink line # Look for colon at end of Mailto: begIndexEmail = self.__contactlink.find(':') # look for "> at end of email address endIndexEmail = self.__contactlink.find('">') # if not found, try '> if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('\'>') # if still not found, try > alone if endIndexEmail == -1: endIndexEmail = self.__contactlink.find('>') #check that both were found if begIndexEmail != -1 and endIndexEmail != -1: self.__contactEmail = self.__contactlink[begIndexEmail + 1 : endIndexEmail] if not self.__isValidEmail(self.__contactEmail): self.__contactEmail = None elif self.__isValidEmail(self.__contactlink): # user just entered an email self.__contactEmail = self.__contactlink # make contactlink proper link self.__contactlink = ' 0: self.__mailserver = self.__hardCodeMailServer else: self.__mailserver = 'localhost' # public methods def getDatabaseUtilityDirectory(self): """getDatabaseUtilityDirectory returns the full path to the database utility directory. Inputs: None Returns: String representing the full path to the database utility directory. (eg, /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__databaseUtilityDirectory def getWWWHomeBase(self): """getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__wwwHomeBase def getMadServer(self): """getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu). Inputs: None Returns: String representing the url to the main database website. Affects: Nothing Exceptions: None """ return self.__mainUrl def getTopLevelUrl(self): """getTopLevelUrl returns the full url of the top level directory in main database website. Inputs: None Returns: String representing the full url to the top level directory in main database website. (eg, http://haystack.mit.edu/madrigal) Affects: Nothing Exceptions: None """ return self.__topLevelUrl def getRelativeTopLevel(self): """getRelativeTopLevel returns the relative url of the top level directory in main database website. Inputs: None Returns: String representing the relative url to the top level directory in main database website. (eg, madrigal) Affects: Nothing Exceptions: None """ return self.__topLevel def getSiteID(self): """getSiteID returns the site id number. Inputs: None Returns: The site id (integer) of the madrigal installation. Affects: Nothing Exceptions: If non-integer found """ try: return int(self.__siteIdValue) except: raise madrigal.admin.MadrigalError("Site id not an integer in madrigal configuration file " + \ self.__confFile, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def getMadrootEnvVarName(self): """getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT). Inputs: None Returns: The name of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__MAD_ROOT def getMadroot(self): """getMadroot returns the value of the environment variable __MAD_ROOT. Inputs: None Returns: The value of the environment variable __MAD_ROOT. Affects: Nothing Exceptions: None """ return self.__madRootDir def getMetadataDir(self): """getMetadataDir returns the metadata directory. Inputs: None Returns: The full metadata directory path. (eg. /opt/madrigal/metadata) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__metadataDir def getExperimentDir(self): """getExperimentDir returns the main experiment directory. No longer guarenteed to be unique,but will hold test files and geophysical experiments. Inputs: None Returns: The full experiment directory path. (eg. /opt/madrigal/experiments) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__experimentDir def getExperimentDirs(self): """getExperimentDirs returns a list of full paths of all valid experiment directories. Inputs: None Returns: list of full paths of all valid experiment directories. Valid experiment directories are those of the form of the regular expression $/Volumes/dropbox/mad31/experiments[0-9]* Affects: Nothing Exceptions: None """ reStr = '/experiments[0-9]*$' possibleExpDirs = glob.glob(os.path.join(self.__madRootDir, 'experiments*')) retList = [] for possibleExpDir in possibleExpDirs: result = re.search(reStr, possibleExpDir) if result: retList.append(possibleExpDir) return(retList) def getBinDir(self): """getBinDir returns the madrigal bin directory. Inputs: None Returns: The madrigal bin directory path. (eg /opt/madrigal/bin) Affects: Nothing Exceptions: None """ return self.__madRootDir + self.__binDir def getHtmlStyle(self): """getHtmlSyle returns the default html body tag for the site. Inputs: None Returns: The default html body tag for the site. (eg. ) Affects: Nothing Exceptions: None """ return self.__htmlstyle def getBackgroundColor(self): """getBackgroundColor returns the string representing background color from getHtmlStyle Inputs: None Returns the color string from the body tag in madrigal.cfg. For example, will return '#FFFF88; if is Returns empty string if no BGCOLOR= """ line = self.getHtmlStyle().upper() items = line.split() for item in items: index = item.find('BGCOLOR=') if index != -1: return(item[len('BGCOLOR='):]) return('') # not found def getIndexHead(self): """getIndexHead returns the heading of the top level madrigal page. Inputs: None Returns: The heading of the top level madrigal page. (eg. Welcome to the Madrigal Database
at Ishtar) Affects: Nothing Exceptions: None """ return self.__indexhead def getContactLink(self): """getContactLink returns contact email link tag (see getContactEmail for the email alone). Inputs: None Returns: The contact email link tag. (eg.
madrigal@haystack
) Affects: Nothing Exceptions: None """ return self.__contactlink def getContactEmail(self): """getContactEmail returns the email address of the site administrator. Inputs: None Returns: The email address of the site administrator. Affects: Nothing Exceptions: None """ if not self.__contactEmail is None: return(self.__contactEmail) else: madSite = madrigal.metadata.MadrigalSite() return(madSite.getSiteEmail(self.getSiteID())) def getMailserver(self): """getMailserver returns the mailserver name. Inputs: None Returns: The mailserver name. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to localhost Affects: Nothing Exceptions: None """ return self.__mailserver def getPythonExecutable(self): """getPythonExecutable returns the full path to the python executable. Inputs: None Returns: the full path to the python executable. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to madroot/bin/python Affects: Nothing Exceptions: None """ return self.__pythonexe def getLocalRulesOfRoad(self): """getLocalRulesOfRoad returns the local rules of the road. Inputs: None Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that. Else returns a default rules_of_road statement Affects: Nothing Exceptions: None """ default_rules_of_road = 'Use of the Madrigal Database is generally subject to the ' + \ 'CEDAR Rules-of-the-Road . ' + \ 'Prior permission to access the data is not required. However, the user is required to establish ' + \ 'early contact with any organization whose data are involved in the project to discuss the ' + \ 'intended usage. Data are often subject to limitations which are not immediately evident to ' + \ 'new users. Before they are formally submitted, draft copies of all reports and publications ' + \ 'must be sent to the contact scientist at all data-supplying organizations along with an offer ' + \ 'of co-authorship to scientists who have provided data. This offer may be declined. The ' + \ 'Database and the organizations that contributed data must be acknowledged in all reports and ' + \ 'publications, and whenever this data is made available through another database. If you have ' + \ 'any questions about appropriate use of these data, contact %s' % (self.getContactEmail(), self.getContactEmail()) localRules = None try: f = open(os.path.join(self.getMadroot(), 'local_rules_of_the_road.txt')) localRules = f.read() f.close() except: pass if localRules == None: return default_rules_of_road else: return localRules def getFullPathFromPartial(self, partialPath): """getFullPathFromPartial returns the full path to a file or directory based on a partial path. Follows the rule that if partialPath begins with [/]experiments, then the full path is madroot + partialPath. Otherwise (pre madrigal 2.6) full path is madroot + experiments + partialPath Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10) Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10) """ if partialPath[:12].find('experiments') != -1: fullPath = os.path.join(self.getMadroot(), partialPath) else: fullPath = os.path.join(self.getMadroot(), 'experiments', partialPath) return(fullPath) def getExpList(self, expName = None, kinstList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, showIgnoredExperiments = False, enforcePathConvention = False): """getExpList returns a list of full experiment directory names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. startDate: a datetime date. If None (default), do not reject any experiments. endDate: a datetime date. If None (default), do not reject any experiments. startDayOfYear: a Julian day number (1-366) after which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any experiments. endDayOfYear: a Julian day number (1-366) before which to accept experiments from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any experiments. showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If False (the default), only required path in the form 1999/mlh/*. Returns: a list of full experiment directory names that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None expList = [] # get all experiment directories expDirList = self.getExperimentDirs() # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: for root, dirs, files in os.walk(thisExpDir): self.__getExperiments((expName, kinstList, startDate, endDate, startDayOfYear, endDayOfYear, expList, showIgnoredExperiments, enforcePathConvention), root, dirs + files) return expList def getFileList(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, enforcePathConvention = 0, includeNonDefault = 0, includeNonMadrigal = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0, path = None): """getFileList returns a list of full file names that match the search arguments. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. path if not None (the default), search only experiment directory set by path. If None, search add experiments. Returns: a list of full path file names (strings) that match the search arguments Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None fileList = [] # get all experiment directories if path is None: expDirList = self.getExperimentDirs() else: expDirList = [path] # walk the experiments directory to find all files meeting criteria for thisExpDir in expDirList: for root, dirs, files in os.walk(thisExpDir): self.__getFiles((expName, kinstList, kindatList, startDate, endDate, startDayOfYear, endDayOfYear, fileList, publicAccessOnly, enforcePathConvention, includeNonDefault, includeNonMadrigal, appendKinst, appendStartTime, includeRealtime), root, dirs + files) return fileList def getFileListFromMetadata(self, expName = None, kinstList = None, kindatList = None, startDate = None, endDate = None, startDayOfYear = None, endDayOfYear = None, publicAccessOnly = 3, includeNonDefault = 0, appendKinst = 0, appendStartTime = 0, includeRealtime = 0): """getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only. This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't search the actual experiment directories. It is therefore faster, but less robust. Inputs: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None (default), any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None (default) or if list contains 0, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None (default) or if list contains 0, all kindat values are accepted. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None (default), do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None (default), do not reject any files. publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. (the default) includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and starttime. Affects: Nothing Exceptions: None """ if kinstList != None: if 0 in kinstList: kinstList = None if kindatList != None: if 0 in kindatList: kindatList = None # the list to be returned fileList = [] # load a Madrigal experiment object madExpObj = MadrigalExperiment(self) # load a Madrigal metafile object madFileObj = MadrigalMetaFile(self) # convert input arguments to needed forms if expName != None: expNameArg = expName.replace(' ', '_') if startDate != None: startDateTime = madrigal.metadata.getMadrigalUTFromDate(startDate[0], startDate[1], startDate[2], startDate[3], startDate[4], startDate[5], 0) if endDate != None: endDateTime = madrigal.metadata.getMadrigalUTFromDate(endDate[0], endDate[1], endDate[2], endDate[3], endDate[4], endDate[5], 0) # loop through all experiments to create four lists: # acceptedExpIdList, acceptedExpKinstList, acceptedExpStartTimeList, and acceptedExpDirList acceptedExpIdList = [] acceptedExpKinstList = [] acceptedExpStartTimeList = [] acceptedExpDirList = [] position = -1 while 1: position += 1 # first position is zero thisName = madExpObj.getExpNameByPosition(position) #check if we're at the end if thisName == None: break # check experiment access thisSecurity = madExpObj.getSecurityByPosition(position) if thisSecurity == -1: continue elif publicAccessOnly == 0: if thisSecurity not in (0,1): continue elif publicAccessOnly == 1: if thisSecurity != 0: continue elif publicAccessOnly == 2: if thisSecurity not in (0,2): continue # apply expName filter if desired if expName != None: # since we are using file name matching, all spaces are # converted to underscores thisName = thisName.replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(thisName.lower(), expNameArg.lower()): continue # apply kinstList filter thisKinst = madExpObj.getKinstByPosition(position) if kinstList != None: if thisKinst not in kinstList: continue # startDate filter - we always need thisStartTime whether or not filter used thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) thisStartTime = madrigal.metadata.getMadrigalUTFromDate(thisStartDate[0], thisStartDate[1], thisStartDate[2], thisStartDate[3], thisStartDate[4], thisStartDate[5], 0) if endDate != None: if thisStartTime > endDateTime: continue # endDate filter if endDate != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) thisEndTime = madrigal.metadata.getMadrigalUTFromDate(thisEndDate[0], thisEndDate[1], thisEndDate[2], thisEndDate[3], thisEndDate[4], thisEndDate[5], 0) if startDate != None: if thisEndTime < startDateTime: continue # apply startDayOfYear filter if startDayOfYear != None: thisStartDate = madExpObj.getExpStartDateTimeByPosition(position) if thisStartDate[7] < startDayOfYear: # index 7 refers to Day of year continue # apply endDayOfYear filter if endDayOfYear != None: thisEndDate = madExpObj.getExpEndDateTimeByPosition(position) if thisEndDate[7] > endDayOfYear: # index 7 refers to Day of year continue # this experiment has made it through all filters, append its id, kinst, startTime, and dir acceptedExpIdList.append(madExpObj.getExpIdByPosition(position)) acceptedExpKinstList.append(thisKinst) acceptedExpStartTimeList.append(thisStartTime) # find the directory based on url dir = self.getMadroot() if dir[-1] != '/': dir += '/' url = madExpObj.getExpUrlByPosition(position) index = url.find('/madtoc/') partialExpDir = url[index+8:] # added default experiments if not there already if partialExpDir.find('experiments') == -1: dir += 'experiments/' dir += url[index+8:] acceptedExpDirList.append(dir) # now loop through the file object to find all files position = -1 while 1: position += 1 # first position is zero thisExpId = madFileObj.getExpIdByPosition(position) #check if we're at the end if thisExpId == None: break # skip this file if expId not in acceptedExpIdList if thisExpId not in acceptedExpIdList: continue # apply kindatList filter thisKindat = madFileObj.getKindatByPosition(position) if kindatList != None: if thisKindat not in kindatList: continue # check file access if madFileObj.getAccessByPosition(position) != 0: if publicAccessOnly in (1,2): continue # apply includeNonDefault filter if includeNonDefault == 0: category = madFileObj.getCategoryByPosition(position) if includeRealtime == 0: if category != 1: # not default continue else: if category not in (1,4): # not default or realtime continue # this file has been accepted by all filters - first, get its experiment index expIndex = acceptedExpIdList.index(thisExpId) thisFilename = acceptedExpDirList[expIndex] + '/' + madFileObj.getFilenameByPosition(position) # append result to fileList according to appendKinst and appendStartTime if appendKinst == 0 and appendStartTime == 0: fileList.append(thisFilename) elif appendKinst == 1 and appendStartTime == 0: fileList.append((thisFilename, acceptedExpKinstList[expIndex])) elif appendKinst == 0 and appendStartTime == 1: fileList.append((thisFilename, acceptedExpStartTimeList[expIndex])) else: fileList.append((thisFilename, acceptedExpKinstList[expIndex], acceptedExpStartTimeList[expIndex])) return fileList def setFileAccess(self, expDirectory, accessMode): """setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Inputs: expDirectory: The full path to a directory in the experiment directory. That is, it may be madroot/experiments[0-9]* or any directory under it. accessMode: either 0 for public access, or 1 for private access. Returns: None Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private. Exceptions: If accessMode is not 1 or 0. """ if (accessMode != 0 and accessMode != 1): raise madrigal.admin.MadrigalError('MadrigalDB.setFileAccess called with accessMode = ' + \ str(accessMode) + ', must be either 0 or 1', None) # walk the experiments directory to find all files meeting criteria for root, dirs, files in os.walk(expDirectory): self.__setFileAccess(accessMode, root, dirs + files) def tarExperiments(self, tarFileName, startDate = None, endDate = None, excludePrivData = 0, ignoreDirCon = 1, includeNonDefData = 0, onlyData = 0, filetype = 0, verbose = 0): """tarExperiments creates a tar file containing files from madroot/experiments[0-9]*. Note: this method sometimes requires the modification of the fileTab.txt files found in the experiments directory. This is because some data files might be excluded, so that the fileTab.txt file will no longer be accurate. Because of this, all files to be tar'ed will be copied to /tmp/temp/experiments, where the fileTab.txt files will be modified. When done, this temp dir will be deleted. Inputs: tarFileName: The full path to a tar file to be created. startDate: a python date (see time module - actually a tuple of nine integers) after which to accept files. If None (default), do not reject any files. endDate: a python date (see time module - actually a tuple of nine integers) before which to accept files. If None (default), do not reject any files. excludePrivData: if 1, allow data marked as private to be omitted (and the line from the fileTab.txt to be removed). If 0 (the default), all data, public and private, will be included. ignoreDirCon: if 1, ignore convention that directory must be in form 1999/mlh/03sep99 (the default). If 0 , reject non-standard directories. includeNonDefData: if 1, include all files listed in fileTab.txt. If 0 (the default), reject non-default files, and modify fileTab.txt to remove non-default listings. onlyData: if 1, reject all files not listed in fileTab.txt. If 0 (the default), accept all files in a directory not mentioned in fileTab.txt. filetype: format to save data files as. Default 0 is to leave present format unchanged. is an integer as follows: type = 0 Leave present format unchanged (default) type = 1 Madrigal type = 2 Blocked Binary type = 3 Cbf type = 4 Unblocked binary type = 5 Ascii verbose: if 1, print to std out the list of files included (relative path). If 0, (the default) print nothing. Returns: None Affects: created tar file tarFileName of selected files from madroot/experiments. Exceptions: If unable to read any experiment file. """ if ignoreDirCon == 1: enforcePathConvention = 0 else: enforcePathConvention = 1 if onlyData == 1: includeNonMadrigal = 0 else: includeNonMadrigal = 1 # create a random temp dir tempDir = '/tmp/temp' + str(random.randrange(1,10000000)) if verbose == 1: print ('Creating list of files to tar...') # get list of files to tar tarFileList = self.getFileList(None, None, None, startDate, endDate, None, None, excludePrivData, enforcePathConvention, includeNonDefData, includeNonMadrigal) # now create a new list of filenames, using relative paths relTarFileList = [] for file in tarFileList: newFilename = file.split(self.getMadroot() + '/')[1] relTarFileList.append(newFilename) # copy all these files to tempDir if verbose == 1: print('The following is a list of files included:') for file in relTarFileList: # make sure dir exists try: os.makedirs(tempDir + '/' + os.path.dirname(file)) except: pass # now copy file shutil.copyfile(self.getMadroot() + '/' + file, tempDir + '/' + file) if verbose == 1: print('\t' + file) if onlyData: # now check if that data file needs to be converted to another filetype # since onlyData is set, convert every file if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + file, tempDir + '/' + file + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + file + '.backup 1 100000000 -o ' + \ tempDir + '/' + file + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + file + '.backup') if verbose == 1: print('Modifying fileTab.txt files if needed...') # modify fileTab.txt files, if required if (not onlyData) and (excludePrivData or not includeNonDefData): # first loop through each fileTab.txt file in list just to check permissions for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # make sure fileTab.txt is writable if not os.access(tempDir + '/' + filename, os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(filename), None) # make sure the directory is writable if not os.access(os.path.dirname(tempDir + '/' + filename), os.W_OK): raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \ 'for ' + str(os.path.dirname(filename)), None) # no exceptions raised - all permissions are okay # this time loop through and modify the fileTab.txt files for filename in relTarFileList: if os.path.basename(filename) != 'fileTab.txt': continue # create a MadrigalMetaFile object fileMeta = MadrigalMetaFile(self, tempDir + '/' + filename) # loop through each file name to see if its in tarFileList: fileNum = 0 while 1: madFilename = fileMeta.getFilenameByPosition(fileNum) if madFilename == None: break # get madRelFilename madRelFilename = os.path.dirname(filename) + '/' + madFilename # if its not in relTarFileList, delete it from fileMeta if not madRelFilename in relTarFileList: fileMeta.deleteRowByFilename(madFilename) fileNum = 0 continue else: # we know madRelFilename is a data file since its in fileTab.txt - # now check if that data file needs to be converted to another filetype if filetype != 0: # copy data file to another location shutil.copyfile(tempDir + '/' + madRelFilename, tempDir + '/' + madRelFilename + '.backup') # run mergeCedarFiles execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \ tempDir + '/' + madRelFilename + '.backup 1 100000000 -o ' + \ tempDir + '/' + madRelFilename + \ ' -t ' + str(filetype) os.system(execStr) os.remove(tempDir + '/' + madRelFilename + '.backup') # get next madFilename fileNum = fileNum + 1 # if fileMeta not empty, write it out if fileMeta.getFileCount() > 0: fileMeta.writeMetadata() # else if its empty, simply delete it else: os.remove(tempDir + '/' + filename) # done modifying fileTab.txt - ready to tar # need to change working directory to tempDir to get relative paths in tar. # Since this directory will soon be deleted, will need to change it back to # whatever it is now when we're done pwd = os.getcwd() # check if tarFileName is absolute or relative to pwd if tarFileName[0] != '/': tarFileName = pwd + '/' + tarFileName if verbose == 1: print('Creating tar file...') os.chdir(tempDir) os.system('tar -cf ' + tarFileName + ' experiments*') os.chdir(pwd) if verbose == 1: print('Removing temp files...') # finally remove temp dir try: shutil.rmtree(tempDir) except: raise madrigal.admin.MadrigalError('In tarExperiments could not remove dir ' + tempDir, traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) def listFileTimes(self, expDir=None, relative=True): """listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory Inputs: expDir - the particular subdirectory of an experiment directory to list. If None (the default), will list all files for all experiment directories. May be an absolute path, or may start with experiments[0-9]*. relative - if True (the default) give the path relative to the experiments[0-9]* directory. If False, give full path Returns: a list of tuples, where each tuple has 1. the file path, and 2. a UT datetime object of the last file modification. Exceptions: raised if expDir is not a valid experiments directory or subdirectory """ expDirs = self.getExperimentDirs() if expDir == None: dirsToExamine = expDirs else: # verify a valid directory found = False if expDir[-1] == '/': expDir = expDir[:-1] # strip trailing / if expDir[0:11] == 'experiments': # convert to absolute path expDir = os.path.join(self.getMadroot(), expDir) for thisDir in expDirs: if expDir.find(thisDir) != -1: found = True if found: dirsToExamine = [expDir] else: raise ValueError('expDir %s not a valid experiment directory' % (expDir)) retList = [] for dirToExamine in dirsToExamine: for root, dirs, files in os.walk(dirToExamine): for name in files: fullname = os.path.join(root, name) relIndex = fullname[len(self.__madRootDir):].find('/experiments') relativeName = fullname[relIndex + 1 + (len(self.__madRootDir)):] ts = os.stat(fullname).st_mtime if relative: retList.append((relativeName, datetime.datetime.utcfromtimestamp(ts))) else: retList.append((fullname, datetime.datetime.utcfromtimestamp(ts))) return(retList) def getKinstKindatConfig(self, kinst, kindat, iniFile=None): """getKinstKindatConfig gets information for the /Volumes/dropbox/mad31/cachedFiles.ini needed to create Madrigal3 files if not specified. Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them. Inputs: kinst - the instrument kinst (integer) kindat - the data kindat (integer) iniFile - the ini file to use. If None, uses default ini file $/Volumes/dropbox/mad31/cachedFiles.ini Returns: a tuple with three items: 1. a list of extra parameters (string mnemonics) 2. a list of independent spatial parameters 3. a list of array splitting parameters Algorithm: 1. If iniFile == None and no default file, returns ([], [], []) 2. Searches ini file for section [%i] % (kinst). If not found, returns ([], [], []) 3. Searches right section for key %i_parms % (kindat). If not found, searches for default_parms. If not found, extra parameters are [] 4. Searches right section for key %i_formats % (kindat). If not found, searches for default_formats. If not found, indepedent spatial parms and extra splitting parameters are [] and []. If found, then parses dictionary, and returns key 'array'. If value has only one item, returns it as a one item list of independent spatial parameters. If two items, they are ind spatial parms and array splitting parms. """ if not iniFile: thisIniFile = os.path.join(self.getMadroot(), 'cachedFiles.ini') if not os.access(thisIniFile, os.R_OK): return(([], [], [])) else: thisIniFile = iniFile instSection = '%i' % (kinst) parser = configparser.SafeConfigParser() parser.read(thisIniFile) if not parser.has_section(instSection): return(([], [], [])) extraParms = [] indSpatialParms = [] arraySplitParms = [] # get extra parms if parser.has_option(instSection, '%i_parms' % (kindat)): extraParms = parser.get(instSection, '%i_parms' % (kindat)) extraParms = extraParms.split(',') elif parser.has_option(instSection, 'default_parms'): extraParms = parser.get(instSection, 'default_parms') extraParms = extraParms.split(',') # make sure no empty parms snuck in finalExtraParms = [] for extraParm in extraParms: if len(extraParm.strip()) > 0: finalExtraParms.append(extraParm.strip()) # get format dict formatDict = None if parser.has_option(instSection, '%i_formats' % (kindat)): formatDict = parser.get(instSection, '%i_formats' % (kindat)) formatDict = eval(formatDict) elif parser.has_option(instSection, 'default_formats'): formatDict = parser.get(instSection, 'default_formats') formatDict = eval(formatDict) if not formatDict is None: if 'array' in formatDict: value = formatDict['array'] if type(value) in (list, tuple): indSpatialParms = value[0] arraySplitParms = value[1] else: indSpatialParms = [value] return((finalExtraParms, indSpatialParms, arraySplitParms)) def toString(self): """toString returns a simple string representation of a MadrigalDB object. Inputs: None Returns: String describing a simple representation of a MadrigalDB object. Affects: Nothing Exceptions: None """ output = "Object type: MadrigalDB\n" output += "Database utility directory = " + self.getDatabaseUtilityDirectory() + "\n" output += "WWW home base = " + self.getWWWHomeBase() + "\n" output += "Server name = " + self.getMadServer() + "\n" output += "Top level url = " + self.getTopLevelUrl() + "\n" output += "Relative Top level url = " + self.getRelativeTopLevel() + "\n" output += "Site ID = " + str(self.getSiteID()) + "\n" output += "MAD_ROOT env. variable name = " + self.getMadrootEnvVarName() + "\n" output += "MAD_ROOT env. variable value = " + self.getMadroot() + "\n" output += "Madrigal metadata dir = " + self.getMetadataDir() + "\n" output += "Madrigal bin dir = " + self.getBinDir() + "\n" output += "Madrigal html body style = " + self.getHtmlStyle() + "\n" output += "Madrigal top level heading = " + self.getIndexHead() + "\n" output += "Madrigal contact link = " + self.getContactLink() + "\n" output += "Madrigal contact email = " + str(self.getContactEmail()) + "\n" output += "Madrigal mailserver = " + self.getMailserver() + "\n" output += "Madrigal python exe = " + self.getPythonExecutable() + "\n" return output def isTestExperiment(self, url, siteId=None): """isTestExperiment returns True if the given experiment url is a test of this Madrigal server. Url can be either real or form in expTab.txt, or can be the experiment directory. If siteId not given, use local site id. """ # Skip test experiments if siteId == None: siteId = self.getSiteID() if url.find('1998/mlh/20jan98') != -1 and siteId != 1: return(True) if url.find('1997/aro/06jan97') != -1 and siteId != 7: return(True) if url.find('1997/lyr/08apr97') != -1 and siteId != 2: return(True) if url.find('1997/son/06jan97') != -1 and siteId != 3: return(True) if url.find('1995/jro/01feb95') != -1 and siteId != 6: return(True) if url.find('1998/jro/27apr98') != -1 and siteId != 6: return(True) return(False) def __str__(self): """ __str__ simply calls toString """ return (self.toString()) def __isValidEmail(self, emailStr): """ __isValidEmail is a private helper function that does some checking to ensure a valid email address was found. Inputs: emailStr - email address string to verify Returns: 1 if no problems, 0 if not valid. Affects: Nothing Exceptions: None """ emailStr = emailStr.strip() if emailStr.find(' ') != -1: return 0 if emailStr.find('"') != -1: return 0 if emailStr.find('<') != -1: return 0 if emailStr.find('>') != -1: return 0 if emailStr.find('/') != -1: return 0 if emailStr.find(':') != -1: return 0 if emailStr.find('@') == -1: return(0) # otherwise okay return 1 def __getExperiments(self, arg, dirname, names): """ __getExperiment is a private helper function called by os.path.walk in getExpList. __getExperiments is called for each sub-directory in the experiments directory. Its purpose is to add any experiment directory paths from that directory that match the search criteria to the expList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. startDate: a python datetime in UTC. If None, do not reject any files. endDate: a python datetime in UTC. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. expList: the list of valid experiment paths to which is appended any newly-found valid experiment directories showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code is set to ignore. If True, include those experiments. enforcePathConvention: if True, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If False (the default), only required path in the form 1999/mlh/*. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __startDate = 2 __endDate = 3 __startDayOfYear = 4 __endDayOfYear = 5 __expList = 6 __showIgnored = 7 __enforcePath = 8 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt if not 'expTab.txt' in names: return # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply startDate filter expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError('Error getting expStartDate from dir %s' % (dirname)) time1 = datetime.datetime(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5]) if arg[__endDate] != None: if time1 > arg[__endDate]: # stop going down tree names[:] = [] return # apply endDate filter if arg[__startDate] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() time1 = datetime.datetime(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5]) if time1 < arg[__startDate]: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply ignore filter if expMeta.getSecurityByPosition() == -1 and not arg[__showIgnored]: # stop going down tree names[:] = [] return # now reject paths not matching present path convention if not enforcePathConvention if not arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention if arg[__enforcePath]: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay arg[__expList].append(dirname) def __getFiles(self, arg, dirname, names): """ __getFiles is a private helper function called by os.path.walk in getFileList. __getFiles is called for each sub-directory in the experiments directory. Its purpose is to add any file paths from that directory that match the search criteria to the fileList. Inputs: arg: a tuple containing all the filter arguments, plus the fileList to be appended to. The tuple elements are: expName: a string defining what experiment names to accept (case insensitive). Use of unix file matching characters * and ? are allowed. If None, any experiment name allowed. kinstList: a list of kinst (kind of instrument codes) integers that will be accepted. If None, all kinst values are accepted. kindatList: a list of kindat (kind of data codes) integers that will be accepted. If None, all kindat values are accepted. startDate: a python date/time in UTC (see time module - actually a tuple of nine integers) after which to accept files. If None, do not reject any files. endDate: a python date/time in UTC (see time module - actually a tuple of nine integers) before which to accept files. If None, do not reject any files. startDayOfYear: a Julian day number (1-366) after which to accept files from any given year. This can be used for example to only allow files from Spring, which has a start day of year of 80 (March 21). If None, do not reject any files. endDayOfYear: a Julian day number (1-366) before which to accept files from any given year. This can be used for example to only allow files from Spring, which has a end day of year of 172 (June 21). If None, do not reject any files. fileList: the list of valid file paths to which is appended any newly-found valid file paths publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt as public. If 0, return all non-archive files. If 2, return public and public archive experiments with public files. If 3, return all files, including public, private, archived, and private archived. enforcePathConvention: if 1, only return experiments whose path is of the form convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If 0 (the default), only require paths to be in the form 1999/mlh/*. includeNonDefault: if 1, also include data files that are not default files in they match all other criteria. If 0 (the default), exclude non-default. includeNonMadrigal: if 1, include all experiment files that are not in fileTab.txt. If 0, (the default) limit data files to those in fileTab.txt. appendKinst: if 1, append kind of instrument integer to each file name, so that what is returned is a list of (file name, inst code) tuples. If 0 (the default), do not append instrument code; return a list of file names. appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that what is returned is a list of (file name, startTime) tuples. If 0 (the default), do not append startTimes. includeRealtime: if 1, include realtime files even if includeNonDefault = 0. If 0 (default), do not include realtime if includeNonDefault = 0. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None. Affects: Adds full file paths of any data files found that meet all filter criteria Exceptions: None """ # constants - arg order __expName = 0 __kinstList = 1 __kindatList = 2 __startDate = 3 __endDate = 4 __startDayOfYear = 5 __endDayOfYear = 6 __fileList = 7 __pubAccessOnly = 8 __enforcePath = 9 __includeNonDef = 10 __includeNonMad = 11 __appendKinst = 12 __appendStartTime = 13 __includeRealtime = 14 # regular expression that enforces dir path convention # note that the __dirConvStr2 is the old convention that experiment directories be in the form DDmmmYY[char] __dirConvStr1 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[a-zA-Z0-9\-_]*$' __dirConvStr2 = '/experiments[0-9]*/[0-9][0-9][0-9][0-9]/[a-z][a-z0-9][a-z0-9]/[0-3][0-9][a-z][a-z0-9][a-z0-9][0-9][0-9].?' # ignore any directory without expTab.txt and fileTab.txt if not includeNonMadrigal if 'expTab.txt' in names: self.__hasExpTab = 1 else: self.__hasExpTab = 0 if 'fileTab.txt' in names: self.__hasFileTab = 1 else: self.__hasFileTab = 0 if arg[__includeNonMad] == 0: if not self.__hasExpTab or not self.__hasFileTab: return # apply all filters relating to expTab.txt if self.__hasExpTab if self.__hasExpTab: # get exp metadata expMeta = MadrigalExperiment(self, dirname + '/' + 'expTab.txt') # apply expName filter if arg[__expName] != None: expName = expMeta.getExpNameByPosition() # since we are using file name matching, all spaces are # converted to underscores expName = expName.replace(' ', '_') expNameArg = arg[__expName].replace(' ', '_') # try to match (case insensitive) if not fnmatch.fnmatch(expName.lower(), expNameArg.lower()): # stop going down tree names[:] = [] return # apply kinstList filter expKinst = expMeta.getKinstByPosition() if arg[__kinstList] != None: if expKinst not in arg[__kinstList]: # stop going down tree names[:] = [] return # apply date filters expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate is None: raise IOError('Error getting expStartDate from dir %s' % (dirname)) expStartUt = madrigal.metadata.getMadrigalUTFromDate(expStartDate[0], expStartDate[1], expStartDate[2], expStartDate[3], expStartDate[4], expStartDate[5], 0) expEndDate = expMeta.getExpEndDateTimeByPosition() expEndUt = madrigal.metadata.getMadrigalUTFromDate(expEndDate[0], expEndDate[1], expEndDate[2], expEndDate[3], expEndDate[4], expEndDate[5], 0) if arg[__startDate] != None: time1 = madrigal.metadata.getMadrigalUTFromDate(arg[__startDate][0], arg[__startDate][1], arg[__startDate][2], arg[__startDate][3], arg[__startDate][4], arg[__startDate][5], 0) if time1 > expEndUt: # stop going down tree names[:] = [] return # save time1 in case we need to appendStartTime thisStartTime = expStartUt # apply endDate filter if arg[__endDate] != None: time2 = madrigal.metadata.getMadrigalUTFromDate(arg[__endDate][0], arg[__endDate][1], arg[__endDate][2], arg[__endDate][3], arg[__endDate][4], arg[__endDate][5], 0) if time2 < expStartUt: # stop going down tree names[:] = [] return # apply startDayOfYear filter if arg[__startDayOfYear] != None: expStartDate = expMeta.getExpStartDateTimeByPosition() if expStartDate[7] < arg[__startDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # apply endDayOfYear filter if arg[__endDayOfYear] != None: expEndDate = expMeta.getExpEndDateTimeByPosition() if expEndDate[7] > arg[__endDayOfYear]: # index 7 refers to Day of year # stop going down tree names[:] = [] return # check experiment access thisSecurity = expMeta.getSecurityByPosition() if thisSecurity == -1: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 0: if thisSecurity not in (0,1): # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 1: if thisSecurity != 0: # stop going down tree names[:] = [] return elif arg[__pubAccessOnly] == 2: if thisSecurity not in (0,2): # stop going down tree names[:] = [] return else: # no expTab.txt - set kinst to None expKinst = None # now reject paths not matching present path convention if enforcePathConvention == 0 if arg[__enforcePath] == 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr1) match = reDirPath.search(dirname) if match == None: return # reject paths not matching old path convention if enforcePathConvention != 0 if arg[__enforcePath] != 0: reDirPath = re.compile(self.getMadroot() + __dirConvStr2) match = reDirPath.search(dirname) if match == None: return # experiment is okay, now start looking at individual files if self.__hasFileTab: fileMeta = MadrigalMetaFile(self, dirname + '/' + 'fileTab.txt') # get numFiles numFiles = fileMeta.getFileCount() # loop through each file, and add it if okay fileCount = 0 while fileCount < numFiles: # skip non-default and non-realtime files if not includeNonDef if fileMeta.getCategoryByPosition(fileCount) not in (1,4) and arg[__includeNonDef] == 0: fileCount = fileCount + 1 continue # skip realtime files if not includeNonDef and not includeRealtime if fileMeta.getCategoryByPosition(fileCount) == 4 and arg[__includeNonDef] == 0 and arg[__includeRealtime] == 0: fileCount = fileCount + 1 continue # check if file has right kindat if arg[__kindatList] != None: if fileMeta.getKindatByPosition(fileCount) not in arg[__kindatList]: fileCount = fileCount + 1 continue # check file access if fileMeta.getAccessByPosition(fileCount) != 0: if arg[__pubAccessOnly] in (1,2): fileCount = fileCount + 1 continue # the file needs to be added filename = fileMeta.getFilenameByPosition(fileCount) if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + filename) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + filename, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + filename, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) fileCount = fileCount + 1 # now add all non-madrigal files if includeNonMad if arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue # if its not in fileTab.txt, add it if fileMeta.getExpIdByFilename(name) == None: if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) # else the file has no fileTab.txt and includeNonMad is true - include all files elif arg[__includeNonMad] == 1: for name in names: # skip dir names if os.path.isdir(dirname + '/' + name): continue if arg[__appendKinst] == 0 and arg[__appendStartTime] == 0: # return only file names arg[__fileList].append(dirname + '/' + name) elif arg[__appendStartTime] == 0: # return file name, inst code tuple arg[__fileList].append((dirname + '/' + name, expKinst)) elif arg[__appendKinst] == 0: # return file name, start time tuple arg[__fileList].append((dirname + '/' + name, thisStartTime)) else: # append both arg[__fileList].append((dirname + '/' + filename, expKinst, thisStartTime)) def __setFileAccess(self, arg, dirname, names): """setFileAccess is a private helper function called by os.path.walk in setFileAccess. Inputs: arg: either 0 for public access, or 1 for private access. dirname: the directory name of the present directory names: the list of filenames in the present directory Returns: None Affects: sets all fileTab.txt files in dirname to be public or private, depending on arg. Exceptions: None. """ for name in names: if name == 'fileTab.txt': try: fileMetaObj = madrigal.metadata.MadrigalMetaFile(self, dirname + '/' + name) fileMetaObj.setAccess(arg) except madrigal.admin.MadrigalError as e: print(e.getExceptionStr())

Ancestors (in MRO)

Static methods

def __init__(

self, initFile=None)

init initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString).

Inputs: String representing the full path to the configuration file. Default is None, in which case configuration file is $(MAD_ROOT)/confFileName (now madrigal.cfg).

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown if error (see class description for specific conditions).

Notes:

Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile,
this constructor goes about loading some basic data about the database, including information such as:

-the database utility directory

-the www home base

Implemented using ConfigParser module.  This reads ini type files.  The only difference between
ini files and madrigal.cfg is that ini files are broken into sections of the form:

[sectionTitle]

key1 = value1

key2 = value2

Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal"
is appended to the beginning of the file.
def __init__(self, initFile=None):
    """__init__ initializes the MadrigalDB by reading from $MAD_ROOT/madrigal.cfg (or initString).
    Inputs: String representing the full path to the configuration file. Default is None, in
            which case configuration file is $(__MAD_ROOT)/__confFileName (now madrigal.cfg).
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown if error (see class description for specific conditions).
    Notes:
        Given the file $MAD_ROOT/madrigal.cfg (these names are set by constants above), or initFile,
        this constructor goes about loading some basic data about the database, including information such as:
        
        -the database utility directory
        
        -the www home base
        
        Implemented using ConfigParser module.  This reads ini type files.  The only difference between
        ini files and madrigal.cfg is that ini files are broken into sections of the form:
        
        [sectionTitle]
        
        key1 = value1
        
        key2 = value2
        
        Since madrigal.cfg (or another initFile) has no section header, one section head called "madrigal"
        is appended to the beginning of the file.
    """
    # if madroot not set, use hard coded madroot
    self.__madRootDir = os.environ.get(self.__MAD_ROOT)
    if (self.__madRootDir == None):
        self.__madRootDir = self.__hardCodeMadRoot
    # Set configuration file
    if (initFile == None):
        self.__confFilePath = self.__madRootDir + "/" + self.__confFileName
    else:
        self.__confFilePath = initFile
    # open configuration file
    try:
        self.__confFile = open(self.__confFilePath, "r")
        
    except IOError:
        # can't read from file - use all hard-coded values
        self.__initFromHardCode()
        self.__finishInit()
        return
    
    # create Parser using standard module ConfigParser
    self.__parser = configparser.ConfigParser()
    
    # read conf file into a StringIO with "[madrigal]\n" section heading prepended
    strConfFile = io.StringIO("[madrigal]\n" + self.__confFile.read())
    # parse StringIO configuration file
    try:
        self.__parser.readfp(strConfFile)
    except:
        raise madrigal.admin.MadrigalError("Unable to parse configuration file " + self.__confFilePath,
                                          traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # read information from configuration file
    self.__readConfFile()
    # close conf file
    self.__confFile.close()
    self.__finishInit()

def getBackgroundColor(

self)

getBackgroundColor returns the string representing background color from getHtmlStyle

Inputs: None

Returns the color string from the body tag in madrigal.cfg. For example, will return '#FFFF88; if is Returns empty string if no BGCOLOR=

def getBackgroundColor(self):
    """getBackgroundColor returns the string representing background color from getHtmlStyle
    
    Inputs: None
    
    Returns the color string from the body tag in madrigal.cfg. For example, will return
        '#FFFF88; if  is 
        Returns empty string if no BGCOLOR=
    """
    line = self.getHtmlStyle().upper()
    items = line.split()
    for item in items:
        index = item.find('BGCOLOR=')
        if index != -1:
            return(item[len('BGCOLOR='):])
        
    return('') # not found

def getBinDir(

self)

getBinDir returns the madrigal bin directory.

Inputs: None

Returns: The madrigal bin directory path. (eg /opt/madrigal/bin)

Affects: Nothing

Exceptions: None

def getBinDir(self):
    """getBinDir returns the madrigal bin directory.
    Inputs: None
    
    Returns: The madrigal bin directory path. (eg /opt/madrigal/bin)
    Affects: Nothing
    Exceptions: None
    """
    return self.__madRootDir + self.__binDir

def getContactEmail(

self)

getContactEmail returns the email address of the site administrator.

Inputs: None

Returns: The email address of the site administrator.

Affects: Nothing

Exceptions: None

def getContactEmail(self):
    """getContactEmail returns the email address of the site administrator.
    Inputs: None
    
    Returns: The email address of the site administrator.
    Affects: Nothing
    Exceptions: None
    """
    if not self.__contactEmail is None:
        return(self.__contactEmail)
    else:
        madSite = madrigal.metadata.MadrigalSite()
        return(madSite.getSiteEmail(self.getSiteID()))

getContactLink returns contact email link tag (see getContactEmail for the email alone).

Inputs: None

Returns: The contact email link tag. (eg. madrigal@haystack
)

Affects: Nothing

Exceptions: None

def getDatabaseUtilityDirectory(

self)

getDatabaseUtilityDirectory returns the full path to the database utility directory.

Inputs: None

Returns: String representing the full path to the database utility directory. (eg, /opt/madrigal/bin)

Affects: Nothing

Exceptions: None

def getDatabaseUtilityDirectory(self):
    """getDatabaseUtilityDirectory returns the full path to the database utility directory.
    
    Inputs: None
    
    Returns: String representing the full path to the database utility directory. (eg,
    /opt/madrigal/bin)
    Affects: Nothing
    Exceptions: None
    """
    return self.__databaseUtilityDirectory

def getExpList(

self, expName=None, kinstList=None, startDate=None, endDate=None, startDayOfYear=None, endDayOfYear=None, showIgnoredExperiments=False, enforcePathConvention=False)

getExpList returns a list of full experiment directory names that match the search arguments.

Inputs:

expName: a string defining what experiment names to accept (case insensitive).
Use of unix file matching characters * and ? are allowed.  If None (default),
any experiment name allowed.

kinstList: a list of kinst (kind of instrument codes) integers that will be
accepted.  If None (default) or if list contains 0, all kinst values are accepted.

startDate: a datetime date.  If None (default), do not reject any experiments.

endDate: a datetime date.  If None (default), do not reject any experiments.

startDayOfYear: a Julian day number (1-366) after which to accept experiments from
any given year.  This can be used for example to only allow files from Spring,
which has a start day of year of 80 (March 21). If None (default), do not
reject any experiments.

endDayOfYear: a Julian day number (1-366) before which to accept experiments from
any given year.  This can be used for example to only allow files from Spring,
which has a end day of year of 172 (June 21). If None (default), do not
reject any experiments.

showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code
is set to ignore.  If True, include those experiments.

enforcePathConvention: if True, only return experiments whose path is of the form
convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional
single character>. If False (the default), only required path in the form 1999/mlh/*.

Returns: a list of full experiment directory names that match the search arguments

Affects: Nothing

Exceptions: None

def getExpList(self,
               expName = None,
               kinstList = None,
               startDate = None,
               endDate = None,
               startDayOfYear = None,
               endDayOfYear = None,
               showIgnoredExperiments = False,
               enforcePathConvention = False):
    """getExpList returns a list of full experiment directory names that match the search arguments.
    Inputs:
        expName: a string defining what experiment names to accept (case insensitive).
        Use of unix file matching characters * and ? are allowed.  If None (default),
        any experiment name allowed.
        kinstList: a list of kinst (kind of instrument codes) integers that will be
        accepted.  If None (default) or if list contains 0, all kinst values are accepted.
        startDate: a datetime date.  If None (default), do not reject any experiments.
        endDate: a datetime date.  If None (default), do not reject any experiments.
        startDayOfYear: a Julian day number (1-366) after which to accept experiments from
        any given year.  This can be used for example to only allow files from Spring,
        which has a start day of year of 80 (March 21). If None (default), do not
        reject any experiments.
        endDayOfYear: a Julian day number (1-366) before which to accept experiments from
        any given year.  This can be used for example to only allow files from Spring,
        which has a end day of year of 172 (June 21). If None (default), do not
        reject any experiments.
        showIgnoredExperiments: if False (default), ignore experiments where expTab.txt state code
        is set to ignore.  If True, include those experiments.
        enforcePathConvention: if True, only return experiments whose path is of the form
        convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If False (the default), only required path in the form 1999/mlh/*.
    
    Returns: a list of full experiment directory names that match the search arguments
    Affects: Nothing
    Exceptions: None
    """
    if kinstList != None:
        if 0 in kinstList:
            kinstList = None
    expList = []
    
    # get all experiment directories
    expDirList = self.getExperimentDirs()
    # walk the experiments directory to find all files meeting criteria
    for thisExpDir in expDirList:
            for root, dirs, files in os.walk(thisExpDir):
                    self.__getExperiments((expName,
                      kinstList,
                      startDate,
                      endDate,
                      startDayOfYear,
                      endDayOfYear,
                      expList,
                      showIgnoredExperiments,
                      enforcePathConvention), root, dirs + files)
    return expList

def getExperimentDir(

self)

getExperimentDir returns the main experiment directory. No longer guarenteed to be unique,but will hold test files and geophysical experiments.

Inputs: None

Returns: The full experiment directory path. (eg. /opt/madrigal/experiments)

Affects: Nothing

Exceptions: None

def getExperimentDir(self):
    """getExperimentDir returns the main experiment directory. No longer guarenteed
    to be unique,but will hold test files and geophysical experiments.
    Inputs: None
    
    Returns: The full experiment directory path. (eg. /opt/madrigal/experiments)
    Affects: Nothing
    Exceptions: None
    """
    return self.__madRootDir + self.__experimentDir

def getExperimentDirs(

self)

getExperimentDirs returns a list of full paths of all valid experiment directories.

Inputs: None

Returns: list of full paths of all valid experiment directories. Valid experiment directories are those of the form of the regular expression $/Volumes/dropbox/mad31/experiments[0-9]*

Affects: Nothing

Exceptions: None

def getExperimentDirs(self):
    """getExperimentDirs returns a list of full paths of all valid experiment directories. 
    Inputs: None
    
    Returns:  list of full paths of all valid experiment directories. Valid experiment directories
    are those of the form of the regular expression $/Volumes/dropbox/mad31/experiments[0-9]*
    Affects: Nothing
    Exceptions: None
    """
    reStr = '/experiments[0-9]*$'
    possibleExpDirs = glob.glob(os.path.join(self.__madRootDir, 'experiments*'))
    retList = []
    for possibleExpDir in possibleExpDirs:
        result = re.search(reStr, possibleExpDir)
        if result:
            retList.append(possibleExpDir)
    return(retList)

def getFileList(

self, expName=None, kinstList=None, kindatList=None, startDate=None, endDate=None, startDayOfYear=None, endDayOfYear=None, publicAccessOnly=3, enforcePathConvention=0, includeNonDefault=0, includeNonMadrigal=0, appendKinst=0, appendStartTime=0, includeRealtime=0, path=None)

getFileList returns a list of full file names that match the search arguments.

Inputs:

expName: a string defining what experiment names to accept (case insensitive).
Use of unix file matching characters * and ? are allowed.  If None (default),
any experiment name allowed.

kinstList: a list of kinst (kind of instrument codes) integers that will be
accepted.  If None (default) or if list contains 0, all kinst values are accepted.

kindatList: a list of kindat (kind of data codes) integers that will be
accepted.  If None (default) or if list contains 0, all kindat values are accepted.

startDate: a python date (see time module - actually a tuple of nine integers)
after which to accept files.  If None (default), do not reject any files.

endDate: a python date (see time module - actually a tuple of nine integers)
before which to accept files.  If None (default), do not reject any files.

startDayOfYear: a Julian day number (1-366) after which to accept files from
any given year.  This can be used for example to only allow files from Spring,
which has a start day of year of 80 (March 21). If None (default), do not
reject any files.

endDayOfYear: a Julian day number (1-366) before which to accept files from
any given year.  This can be used for example to only allow files from Spring,
which has a end day of year of 172 (June 21). If None (default), do not
reject any files.

publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt
as public.  If 0, return all non-archive files. If 2, return public
and public archive experiments with public files.  If 3, return all files,
including public, private, archived, and private archived. (the default)

enforcePathConvention: if 1, only return experiments whose path is of the form
convention 1999/mlh/20jan98d (YYYY/<three lower case letters>/DDmmmYY<optional
single character>. If 0 (the default), only require paths to be in the form 1999/mlh/*.

includeNonDefault: if 1, also include data files that are not default files
in they match all other criteria. If 0 (the default), exclude non-default.

includeNonMadrigal: if 1, include all experiment files that are not in
fileTab.txt.  If 0, (the default) limit data files to those in fileTab.txt.

appendKinst: if 1, append kind of instrument integer to each file name, so that
what is returned is a list of (file name, inst code) tuples. If 0 (the default),
do not append instrument code; return a list of file names.

appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that
what is returned is a list of (file name, startTime) tuples. If 0 (the default),
do not append startTimes.

includeRealtime: if 1, include realtime files even if includeNonDefault = 0.  If 0
(default), do not include realtime if includeNonDefault = 0.

path if not None (the default), search only experiment directory set by path.  If None,
search add experiments.

Returns: a list of full path file names (strings) that match the search arguments

Affects: Nothing

Exceptions: None

def getFileList(self,
                expName = None,
                kinstList = None,
                kindatList = None,
                startDate = None,
                endDate = None,
                startDayOfYear = None,
                endDayOfYear = None,
                publicAccessOnly = 3,
                enforcePathConvention = 0,
                includeNonDefault = 0,
                includeNonMadrigal = 0,
                appendKinst = 0,
                appendStartTime = 0,
                includeRealtime = 0,
                path = None):
    """getFileList returns a list of full file names that match the search arguments.
    Inputs:
        expName: a string defining what experiment names to accept (case insensitive).
        Use of unix file matching characters * and ? are allowed.  If None (default),
        any experiment name allowed.
        kinstList: a list of kinst (kind of instrument codes) integers that will be
        accepted.  If None (default) or if list contains 0, all kinst values are accepted.
        
        kindatList: a list of kindat (kind of data codes) integers that will be
        accepted.  If None (default) or if list contains 0, all kindat values are accepted.
        startDate: a python date (see time module - actually a tuple of nine integers)
        after which to accept files.  If None (default), do not reject any files.
        endDate: a python date (see time module - actually a tuple of nine integers)
        before which to accept files.  If None (default), do not reject any files.
        startDayOfYear: a Julian day number (1-366) after which to accept files from
        any given year.  This can be used for example to only allow files from Spring,
        which has a start day of year of 80 (March 21). If None (default), do not
        reject any files.
        endDayOfYear: a Julian day number (1-366) before which to accept files from
        any given year.  This can be used for example to only allow files from Spring,
        which has a end day of year of 172 (June 21). If None (default), do not
        reject any files.
        publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt
        as public.  If 0, return all non-archive files. If 2, return public
        and public archive experiments with public files.  If 3, return all files,
        including public, private, archived, and private archived. (the default)
        enforcePathConvention: if 1, only return experiments whose path is of the form
        convention 1999/mlh/20jan98d (YYYY//DDmmmYY. If 0 (the default), only require paths to be in the form 1999/mlh/*.
        includeNonDefault: if 1, also include data files that are not default files
        in they match all other criteria. If 0 (the default), exclude non-default.
        includeNonMadrigal: if 1, include all experiment files that are not in
        fileTab.txt.  If 0, (the default) limit data files to those in fileTab.txt.
        appendKinst: if 1, append kind of instrument integer to each file name, so that
        what is returned is a list of (file name, inst code) tuples. If 0 (the default),
        do not append instrument code; return a list of file names.
        appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that
        what is returned is a list of (file name, startTime) tuples. If 0 (the default),
        do not append startTimes.
        includeRealtime: if 1, include realtime files even if includeNonDefault = 0.  If 0
        (default), do not include realtime if includeNonDefault = 0.
        
        path if not None (the default), search only experiment directory set by path.  If None,
        search add experiments.
    
    Returns: a list of full path file names (strings) that match the search arguments
    Affects: Nothing
    Exceptions: None
    """
    if kinstList != None:
        if 0 in kinstList:
            kinstList = None
    if kindatList != None:
        if 0 in kindatList:
            kindatList = None
    fileList = []
    
    # get all experiment directories
    if path is None:
        expDirList = self.getExperimentDirs()
    else:
        expDirList = [path]
    # walk the experiments directory to find all files meeting criteria
    for thisExpDir in expDirList:
            for root, dirs, files in os.walk(thisExpDir):
                      self.__getFiles((expName,
                                       kinstList,
                                       kindatList,
                                       startDate,
                                       endDate,
                                       startDayOfYear,
                                       endDayOfYear,
                                       fileList,
                                       publicAccessOnly,
                                       enforcePathConvention,
                                       includeNonDefault,
                                       includeNonMadrigal,
                                       appendKinst,
                                       appendStartTime,
                                       includeRealtime), 
                                       root, dirs + files)
    return fileList

def getFileListFromMetadata(

self, expName=None, kinstList=None, kindatList=None, startDate=None, endDate=None, startDayOfYear=None, endDayOfYear=None, publicAccessOnly=3, includeNonDefault=0, appendKinst=0, appendStartTime=0, includeRealtime=0)

getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only.

This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't search the actual experiment directories. It is therefore faster, but less robust.

Inputs:

expName: a string defining what experiment names to accept (case insensitive).
Use of unix file matching characters * and ? are allowed.  If None (default),
any experiment name allowed.

kinstList: a list of kinst (kind of instrument codes) integers that will be
accepted.  If None (default) or if list contains 0, all kinst values are accepted.

kindatList: a list of kindat (kind of data codes) integers that will be
accepted.  If None (default) or if list contains 0, all kindat values are accepted.

startDate: a python date (see time module - actually a tuple of nine integers)
after which to accept files.  If None (default), do not reject any files.

endDate: a python date (see time module - actually a tuple of nine integers)
before which to accept files.  If None (default), do not reject any files.

startDayOfYear: a Julian day number (1-366) after which to accept files from
any given year.  This can be used for example to only allow files from Spring,
which has a start day of year of 80 (March 21). If None (default), do not
reject any files.

endDayOfYear: a Julian day number (1-366) before which to accept files from
any given year.  This can be used for example to only allow files from Spring,
which has a end day of year of 172 (June 21). If None (default), do not
reject any files.

publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt
as public.  If 0, return all non-archive files. If 2, return public
and public archive experiments with public files.  If 3, return all files,
including public, private, archived, and private archived. (the default)

includeNonDefault: if 1, also include data files that are not default files
in they match all other criteria. If 0 (the default), exclude non-default.

appendKinst: if 1, append kind of instrument integer to each file name, so that
what is returned is a list of (file name, inst code) tuples. If 0 (the default),
do not append instrument code; return a list of file names.

appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that
what is returned is a list of (file name, startTime) tuples. If 0 (the default),
do not append startTimes.

includeRealtime: if 1, include realtime files even if includeNonDefault = 0.  If 0
(default), do not include realtime if includeNonDefault = 0.

Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and starttime.

Affects: Nothing

Exceptions: None

def getFileListFromMetadata(self,
                            expName = None,
                            kinstList = None,
                            kindatList = None,
                            startDate = None,
                            endDate = None,
                            startDayOfYear = None,
                            endDayOfYear = None,
                            publicAccessOnly = 3,
                            includeNonDefault = 0,
                            appendKinst = 0,
                            appendStartTime = 0,
                            includeRealtime = 0):
    """getFileListFromMetadata returns a list of full file names that match the search arguments using metadata only.
    This method is very similar to getFileList, except that it assumes the metadata is correct, and doesn't
    search the actual experiment directories.  It is therefore faster, but less robust.
    Inputs:
        expName: a string defining what experiment names to accept (case insensitive).
        Use of unix file matching characters * and ? are allowed.  If None (default),
        any experiment name allowed.
        kinstList: a list of kinst (kind of instrument codes) integers that will be
        accepted.  If None (default) or if list contains 0, all kinst values are accepted.
        kindatList: a list of kindat (kind of data codes) integers that will be
        accepted.  If None (default) or if list contains 0, all kindat values are accepted.
        startDate: a python date (see time module - actually a tuple of nine integers)
        after which to accept files.  If None (default), do not reject any files.
        endDate: a python date (see time module - actually a tuple of nine integers)
        before which to accept files.  If None (default), do not reject any files.
        startDayOfYear: a Julian day number (1-366) after which to accept files from
        any given year.  This can be used for example to only allow files from Spring,
        which has a start day of year of 80 (March 21). If None (default), do not
        reject any files.
        endDayOfYear: a Julian day number (1-366) before which to accept files from
        any given year.  This can be used for example to only allow files from Spring,
        which has a end day of year of 172 (June 21). If None (default), do not
        reject any files.
        publicAccessOnly: if 1, only return files marked in fileTab.txt and expTab.txt
        as public.  If 0, return all non-archive files. If 2, return public
        and public archive experiments with public files.  If 3, return all files,
        including public, private, archived, and private archived. (the default)
        includeNonDefault: if 1, also include data files that are not default files
        in they match all other criteria. If 0 (the default), exclude non-default.
        appendKinst: if 1, append kind of instrument integer to each file name, so that
        what is returned is a list of (file name, inst code) tuples. If 0 (the default),
        do not append instrument code; return a list of file names.
        appendStartTime: if 1, append start time in seconds since 1950 to each file name, so that
        what is returned is a list of (file name, startTime) tuples. If 0 (the default),
        do not append startTimes.
        includeRealtime: if 1, include realtime files even if includeNonDefault = 0.  If 0
        (default), do not include realtime if includeNonDefault = 0.
    
    Returns: a list of full path file names (strings) that match the search arguments, and possibly kinst and
    starttime.
    Affects: Nothing
    Exceptions: None
    """
    if kinstList != None:
        if 0 in kinstList:
            kinstList = None
    if kindatList != None:
        if 0 in kindatList:
            kindatList = None
    # the list to be returned
    fileList = []
    # load a Madrigal experiment object
    madExpObj = MadrigalExperiment(self)
    # load a Madrigal metafile object
    madFileObj = MadrigalMetaFile(self)
    # convert input arguments to needed forms
    if expName != None:
        expNameArg = expName.replace(' ', '_')
    if startDate != None:
        startDateTime = madrigal.metadata.getMadrigalUTFromDate(startDate[0],
                                                       startDate[1],
                                                       startDate[2],
                                                       startDate[3],
                                                       startDate[4],
                                                       startDate[5],
                                                       0)
    if endDate != None:
        endDateTime = madrigal.metadata.getMadrigalUTFromDate(endDate[0],
                                                     endDate[1],
                                                     endDate[2],
                                                     endDate[3],
                                                     endDate[4],
                                                     endDate[5],
                                                       0)
    # loop through all experiments to create four lists:
    #  acceptedExpIdList, acceptedExpKinstList, acceptedExpStartTimeList, and acceptedExpDirList
    acceptedExpIdList = []
    acceptedExpKinstList = []
    acceptedExpStartTimeList = []
    acceptedExpDirList = []
    position = -1
    while 1:
        position += 1 # first position is zero
        
        thisName = madExpObj.getExpNameByPosition(position)
        #check if we're at the end
        if thisName == None:
            break
        # check experiment access
        thisSecurity = madExpObj.getSecurityByPosition(position)
        if thisSecurity == -1:
            continue
        elif publicAccessOnly == 0:
            if thisSecurity not in (0,1):
                continue
        elif publicAccessOnly == 1:
            if thisSecurity != 0:
                continue
        elif publicAccessOnly == 2:
            if thisSecurity not in (0,2):
                continue
        
        # apply expName filter if desired
        if expName != None:
            # since we are using file name matching, all spaces are
            # converted to underscores
            thisName = thisName.replace(' ', '_')
            # try to match (case insensitive)
            if not fnmatch.fnmatch(thisName.lower(), expNameArg.lower()):
                continue
        # apply kinstList filter
        thisKinst = madExpObj.getKinstByPosition(position)
        
        if kinstList != None:
            if thisKinst not in kinstList:
                continue
        # startDate filter - we always need thisStartTime whether or not filter used
        thisStartDate = madExpObj.getExpStartDateTimeByPosition(position)
        thisStartTime = madrigal.metadata.getMadrigalUTFromDate(thisStartDate[0],
                                                       thisStartDate[1],
                                                       thisStartDate[2],
                                                       thisStartDate[3],
                                                       thisStartDate[4],
                                                       thisStartDate[5],
                                                       0)
        
        if endDate != None:
            if thisStartTime > endDateTime:
                continue
            
        # endDate filter
        if endDate != None:
            thisEndDate = madExpObj.getExpEndDateTimeByPosition(position)
            thisEndTime = madrigal.metadata.getMadrigalUTFromDate(thisEndDate[0],
                                                         thisEndDate[1],
                                                         thisEndDate[2],
                                                         thisEndDate[3],
                                                         thisEndDate[4],
                                                         thisEndDate[5],
                                                         0)
            if startDate != None:
                if thisEndTime < startDateTime:
                    continue
        # apply startDayOfYear filter
        if startDayOfYear != None:
            thisStartDate = madExpObj.getExpStartDateTimeByPosition(position)
            if thisStartDate[7] < startDayOfYear:
                # index 7 refers to Day of year
                continue
        # apply endDayOfYear filter
        if endDayOfYear != None:
            thisEndDate = madExpObj.getExpEndDateTimeByPosition(position)
            if thisEndDate[7] > endDayOfYear:
                # index 7 refers to Day of year
                continue
        # this experiment has made it through all filters, append its id, kinst, startTime, and dir
        acceptedExpIdList.append(madExpObj.getExpIdByPosition(position))
        acceptedExpKinstList.append(thisKinst)
        acceptedExpStartTimeList.append(thisStartTime)
        # find the directory based on url
        dir = self.getMadroot()
        if dir[-1] != '/':
            dir += '/'
        url = madExpObj.getExpUrlByPosition(position)
        index = url.find('/madtoc/')
        partialExpDir = url[index+8:]
        # added default experiments if not there already
        if partialExpDir.find('experiments') == -1:
            dir += 'experiments/'
        dir += url[index+8:]
        acceptedExpDirList.append(dir)
    # now loop through the file object to find all files
    position = -1
    while 1:
        position += 1 # first position is zero
        
        thisExpId = madFileObj.getExpIdByPosition(position)
        #check if we're at the end
        if thisExpId == None:
            break
        
        # skip this file if expId not in acceptedExpIdList
        if thisExpId not in acceptedExpIdList:
            continue
     # apply kindatList filter
        thisKindat = madFileObj.getKindatByPosition(position)
        
        if kindatList != None:
            if thisKindat not in kindatList:
                continue
        # check file access
        if madFileObj.getAccessByPosition(position) != 0:
            if publicAccessOnly in (1,2):
                continue
        # apply includeNonDefault filter
        if includeNonDefault == 0:
            category = madFileObj.getCategoryByPosition(position)
            if includeRealtime == 0:
                if category != 1:
                    # not default
                    continue
            else:
                if category not in (1,4):
                    # not default or realtime
                    continue
            
        # this file has been accepted by all filters - first, get its experiment index
        expIndex = acceptedExpIdList.index(thisExpId)
        thisFilename = acceptedExpDirList[expIndex] + '/' + madFileObj.getFilenameByPosition(position)
        # append result to fileList according to appendKinst and appendStartTime
        if appendKinst == 0 and appendStartTime == 0:
            fileList.append(thisFilename)
        elif appendKinst == 1 and appendStartTime == 0:
            fileList.append((thisFilename, acceptedExpKinstList[expIndex]))
        elif appendKinst == 0 and appendStartTime == 1:
            fileList.append((thisFilename, acceptedExpStartTimeList[expIndex]))
        else:
            fileList.append((thisFilename, acceptedExpKinstList[expIndex], acceptedExpStartTimeList[expIndex]))
    return fileList

def getFullPathFromPartial(

self, partialPath)

getFullPathFromPartial returns the full path to a file or directory based on a partial path.

Follows the rule that if partialPath begins with [/]experiments, then the full path is madroot + partialPath. Otherwise (pre madrigal 2.6) full path is madroot + experiments + partialPath

Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10)

Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10)

def getFullPathFromPartial(self, partialPath):
    """getFullPathFromPartial returns the full path to a file or directory based on
    a partial path.
    
    Follows the rule that if partialPath begins with [/]experiments, then the full
    path is madroot + partialPath.  Otherwise (pre madrigal 2.6) full path is
    madroot + experiments + partialPath
    
    Input: parial path (eg, 2010/mlh/15jan10 or experiments10/2010/mlh/15jan10)
    
    Returns: full path (eg /opt/madrigal/experiments10/2010/mlh/15jan10)
    """
    if partialPath[:12].find('experiments') != -1:
        fullPath = os.path.join(self.getMadroot(), partialPath)
    else:
        fullPath = os.path.join(self.getMadroot(), 'experiments', partialPath)
    return(fullPath)

def getHtmlStyle(

self)

getHtmlSyle returns the default html body tag for the site.

Inputs: None

Returns: The default html body tag for the site. (eg. )

Affects: Nothing

Exceptions: None

def getHtmlStyle(self):
    """getHtmlSyle returns the default html body tag for the site.
    Inputs: None
    
    Returns: The default html body tag for the site.
    (eg. )
    Affects: Nothing
    Exceptions: None
    """
    
    return self.__htmlstyle

def getIndexHead(

self)

getIndexHead returns the heading of the top level madrigal page.

Inputs: None

Returns: The heading of the top level madrigal page. (eg. Welcome to the Madrigal Database
at Ishtar)

Affects: Nothing

Exceptions: None

def getIndexHead(self):
    """getIndexHead returns the heading of the top level madrigal page.
    Inputs: None
    
    Returns: The heading of the top level madrigal page.
    (eg. Welcome to the Madrigal Database 
at Ishtar) Affects: Nothing Exceptions: None """ return self.__indexhead

def getKinstKindatConfig(

self, kinst, kindat, iniFile=None)

getKinstKindatConfig gets information for the /Volumes/dropbox/mad31/cachedFiles.ini needed to create Madrigal3 files if not specified.

Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them.

Inputs:

kinst - the instrument kinst (integer)

kindat - the data kindat (integer)

iniFile - the ini file to use.  If None, uses default ini file $/Volumes/dropbox/mad31/cachedFiles.ini

Returns: a tuple with three items: 1. a list of extra parameters (string mnemonics) 2. a list of independent spatial parameters 3. a list of array splitting parameters

Algorithm:

  1. If iniFile == None and no default file, returns ([], [], [])
  2. Searches ini file for section [%i] % (kinst). If not found, returns ([], [], [])
  3. Searches right section for key %i_parms % (kindat). If not found, searches for default_parms. If not found, extra parameters are []
  4. Searches right section for key %i_formats % (kindat). If not found, searches for default_formats. If not found, indepedent spatial parms and extra splitting parameters are [] and []. If found, then parses dictionary, and returns key 'array'. If value has only one item, returns it as a one item list of independent spatial parameters. If two items, they are ind spatial parms and array splitting parms.
def getKinstKindatConfig(self, kinst, kindat, iniFile=None):
    """getKinstKindatConfig gets information for the /Volumes/dropbox/mad31/cachedFiles.ini needed to create Madrigal3 files if not specified.
    
    Used primarily to get independent spatial parameters and array splitting parms for given files for old loading programs that don't specify them.
    
    Inputs:
    
        kinst - the instrument kinst (integer)
        
        kindat - the data kindat (integer)
        
        iniFile - the ini file to use.  If None, uses default ini file $/Volumes/dropbox/mad31/cachedFiles.ini
        
    Returns: a tuple with three items:
        1. a list of extra parameters (string mnemonics)
        2. a list of independent spatial parameters
        3. a list of array splitting parameters
        
    Algorithm:
    
    1. If iniFile == None and no default file, returns ([], [], [])
    2. Searches ini file for section [%i] % (kinst).  If not found, returns ([], [], [])
    3. Searches right section for key %i_parms % (kindat).  If not found, searches for default_parms.
        If not found, extra parameters are []
    4. Searches right section for key %i_formats % (kindat).  If not found, searches for default_formats.
        If not found, indepedent spatial parms and extra splitting parameters are [] and [].  If found,
        then parses dictionary, and returns key 'array'.  If value has only one item, returns it as a one item
        list of independent spatial parameters.  If two items, they are ind spatial parms and array splitting parms.
    """
    if not iniFile:
        thisIniFile = os.path.join(self.getMadroot(), 'cachedFiles.ini')
        if not os.access(thisIniFile, os.R_OK):
            return(([], [], []))
    else:
        thisIniFile = iniFile
        
    instSection = '%i' % (kinst)
        
    parser = configparser.SafeConfigParser()
    parser.read(thisIniFile)
    if not parser.has_section(instSection):
        return(([], [], []))
    extraParms = []
    indSpatialParms = []
    arraySplitParms = []
    
    # get extra parms
    if parser.has_option(instSection, '%i_parms' % (kindat)):
        extraParms = parser.get(instSection, '%i_parms' % (kindat))
        extraParms = extraParms.split(',')
    elif parser.has_option(instSection, 'default_parms'):
        extraParms = parser.get(instSection, 'default_parms')
        extraParms = extraParms.split(',')
        
    # make sure no empty parms snuck in
    finalExtraParms = []
    for extraParm in extraParms:
        if len(extraParm.strip()) > 0:
            finalExtraParms.append(extraParm.strip())
    
    # get format dict
    formatDict = None
    if parser.has_option(instSection, '%i_formats' % (kindat)):
        formatDict = parser.get(instSection, '%i_formats' % (kindat))
        formatDict = eval(formatDict)
    elif parser.has_option(instSection, 'default_formats'):
        formatDict = parser.get(instSection, 'default_formats')
        formatDict = eval(formatDict)
    if not formatDict is None:
        if 'array' in formatDict:
            value = formatDict['array']
            if type(value) in (list, tuple):
                indSpatialParms = value[0]
                arraySplitParms = value[1]
            else:
                indSpatialParms = [value]
                
                
    return((finalExtraParms, indSpatialParms, arraySplitParms))

def getLocalRulesOfRoad(

self)

getLocalRulesOfRoad returns the local rules of the road.

Inputs: None

Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that. Else returns a default rules_of_road statement

Affects: Nothing

Exceptions: None

def getLocalRulesOfRoad(self):
    """getLocalRulesOfRoad returns the local rules of the road.
    Inputs: None
    
    Returns: If the file madroot/local_rules_of_the_road.txt exists, returns the text of that.
    Else returns a default rules_of_road statement
    Affects: Nothing
    Exceptions: None
    """
    default_rules_of_road = 'Use of the Madrigal Database is generally subject to the ' + \
        'CEDAR Rules-of-the-Road . ' + \
 'Prior permission to access the data is not required.  However, the user is required to establish ' + \
 'early contact with any organization whose data are involved in the project to discuss the ' + \
 'intended usage. Data are often subject to limitations which are not immediately evident to ' + \
 'new users.  Before they are formally submitted, draft copies of all reports and publications ' + \
 'must be sent to the contact scientist at all data-supplying organizations along with an offer ' + \
 'of co-authorship to scientists who have provided data. This offer may be declined.  The ' + \
 'Database and the organizations that contributed data must be acknowledged in all reports and ' + \
 'publications, and whenever this data is made available through another database. If you have ' + \
 'any questions about appropriate use of these data, contact %s' % (self.getContactEmail(), self.getContactEmail())
    localRules = None
    try:
        f = open(os.path.join(self.getMadroot(), 'local_rules_of_the_road.txt'))
        localRules = f.read()
        f.close()
    except:
        pass
    if localRules == None:
        return default_rules_of_road
    else:
        return localRules

def getMadServer(

self)

getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu).

Inputs: None

Returns: String representing the url to the main database website.

Affects: Nothing

Exceptions: None

def getMadServer(self):
    """getMadServer returns the full name of the madrigal server (eg, haystack.mit.edu).
    
    Inputs: None
    
    Returns: String representing the url to the main database website.
    Affects: Nothing
    Exceptions: None
    """
    return self.__mainUrl

def getMadroot(

self)

getMadroot returns the value of the environment variable __MAD_ROOT.

Inputs: None

Returns: The value of the environment variable __MAD_ROOT.

Affects: Nothing

Exceptions: None

def getMadroot(self):
    """getMadroot returns the value of the environment variable __MAD_ROOT.
    Inputs: None
    
    Returns: The value of the environment variable __MAD_ROOT.
    Affects: Nothing
    Exceptions: None
    """
    return self.__madRootDir

def getMadrootEnvVarName(

self)

getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT).

Inputs: None

Returns: The name of the environment variable __MAD_ROOT.

Affects: Nothing

Exceptions: None

def getMadrootEnvVarName(self):
    """getMadrootEnvVarName returns the name of the environment variable __MAD_ROOT (presently = MAD_ROOT).
    Inputs: None
    
    Returns: The name of the environment variable __MAD_ROOT.
    Affects: Nothing
    Exceptions: None
    """
    return self.__MAD_ROOT

def getMailserver(

self)

getMailserver returns the mailserver name.

Inputs: None

Returns: The mailserver name. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to localhost

Affects: Nothing

Exceptions: None

def getMailserver(self):
    """getMailserver returns the mailserver name.
    Inputs: None
    
    Returns: The mailserver name.  If this heading is not found in madrigal.cfg, no
    error is thrown - simply defaults to localhost
    Affects: Nothing
    Exceptions: None
    """
    
    return self.__mailserver

def getMetadataDir(

self)

getMetadataDir returns the metadata directory.

Inputs: None

Returns: The full metadata directory path. (eg. /opt/madrigal/metadata)

Affects: Nothing

Exceptions: None

def getMetadataDir(self):
    """getMetadataDir returns the metadata directory.
    Inputs: None
    
    Returns: The full metadata directory path. (eg. /opt/madrigal/metadata)
    Affects: Nothing
    Exceptions: None
    """
    return self.__madRootDir + self.__metadataDir

def getPythonExecutable(

self)

getPythonExecutable returns the full path to the python executable.

Inputs: None

Returns: the full path to the python executable. If this heading is not found in madrigal.cfg, no error is thrown - simply defaults to madroot/bin/python

Affects: Nothing

Exceptions: None

def getPythonExecutable(self):
    """getPythonExecutable returns the full path to the python executable.
    Inputs: None
    
    Returns: the full path to the python executable.  If this heading is not
    found in madrigal.cfg, no error is thrown - simply defaults to
    madroot/bin/python
    Affects: Nothing
    Exceptions: None
    """
    
    return self.__pythonexe

def getRelativeTopLevel(

self)

getRelativeTopLevel returns the relative url of the top level directory in main database website.

Inputs: None

Returns: String representing the relative url to the top level directory in main database website. (eg, madrigal)

Affects: Nothing

Exceptions: None

def getRelativeTopLevel(self):
    """getRelativeTopLevel returns the relative url of the top level directory in main database website.
    
    Inputs: None
    
    Returns: String representing the relative url to the top level directory in main database website.
    (eg, madrigal)
    Affects: Nothing
    Exceptions: None
    """
    return self.__topLevel

def getSiteID(

self)

getSiteID returns the site id number.

Inputs: None

Returns: The site id (integer) of the madrigal installation.

Affects: Nothing

Exceptions: If non-integer found

def getSiteID(self):
    """getSiteID returns the site id number.
    Inputs: None
    
    Returns: The site id (integer) of the madrigal installation.
    Affects: Nothing
    Exceptions: If non-integer found
    """
    
    try:
        return int(self.__siteIdValue)
    except:
        raise madrigal.admin.MadrigalError("Site id not an integer in madrigal configuration file " + \
                                          self.__confFile,
                                          traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))

def getTopLevelUrl(

self)

getTopLevelUrl returns the full url of the top level directory in main database website.

Inputs: None

Returns: String representing the full url to the top level directory in main database website. (eg, http://haystack.mit.edu/madrigal)

Affects: Nothing

Exceptions: None

def getTopLevelUrl(self):
    """getTopLevelUrl returns the full url of the top level directory in main database website.
    
    Inputs: None
    
    Returns: String representing the full url to the top level directory in main database website.
    (eg, http://haystack.mit.edu/madrigal)
    Affects: Nothing
    Exceptions: None
    """
    return self.__topLevelUrl

def getWWWHomeBase(

self)

getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu).

Inputs: None

Returns: String representing the url to the main database website.

Affects: Nothing

Exceptions: None

def getWWWHomeBase(self):
    """getWWWHomeBase returns the url to the main database website(eg, http://haystack.mit.edu).
    
    Inputs: None
    
    Returns: String representing the url to the main database website.
    Affects: Nothing
    Exceptions: None
    """
    return self.__wwwHomeBase

def isTestExperiment(

self, url, siteId=None)

isTestExperiment returns True if the given experiment url is a test of this Madrigal server. Url can be either real or form in expTab.txt, or can be the experiment directory. If siteId not given, use local site id.

def isTestExperiment(self, url, siteId=None):
    """isTestExperiment returns True if the given experiment url is a test of this Madrigal
    server.  Url can be either real or form in expTab.txt, or can be the experiment directory.
    If siteId not given, use local site id.
    """
    # Skip test experiments
    if siteId == None:
        siteId = self.getSiteID()
    if url.find('1998/mlh/20jan98') != -1 and siteId != 1:
        return(True)
    if url.find('1997/aro/06jan97') != -1 and siteId != 7:
        return(True)
    if url.find('1997/lyr/08apr97') != -1 and siteId != 2:
        return(True)
    if url.find('1997/son/06jan97') != -1 and siteId != 3:
        return(True)
    if url.find('1995/jro/01feb95') != -1 and siteId != 6:
        return(True)
    if url.find('1998/jro/27apr98') != -1 and siteId != 6:
        return(True)
    return(False)

def listFileTimes(

self, expDir=None, relative=True)

listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory

Inputs:

expDir - the particular subdirectory of an experiment directory to list.  If None (the default),
    will list all files for all experiment directories.  May be an absolute path, or may start with
    experiments[0-9]*.

relative - if True (the default) give the path relative to the experiments[0-9]* directory.  If False,
    give full path

Returns: a list of tuples, where each tuple has 1. the file path, and 2. a UT datetime object of the last file modification.

Exceptions: raised if expDir is not a valid experiments directory or subdirectory

def listFileTimes(self, expDir=None, relative=True):
    """listFileTimes returns a list of (filename, datetime_ut) of all files in the experiment directory
    
    Inputs:
    
        expDir - the particular subdirectory of an experiment directory to list.  If None (the default),
            will list all files for all experiment directories.  May be an absolute path, or may start with
            experiments[0-9]*.
            
        relative - if True (the default) give the path relative to the experiments[0-9]* directory.  If False,
            give full path
            
    Returns: a list of tuples, where each tuple has 1.  the file path, and 2. a UT datetime object of the
        last file modification.
        
    Exceptions: raised if expDir is not a valid experiments directory or subdirectory
    """
    expDirs = self.getExperimentDirs()
    
    if expDir == None:
        dirsToExamine = expDirs
    else:
        # verify a valid directory
        found = False
        if expDir[-1] == '/':
            expDir = expDir[:-1] # strip trailing /
        if expDir[0:11] == 'experiments':
            # convert to absolute path
            expDir = os.path.join(self.getMadroot(), expDir)
        for thisDir in expDirs:
            if expDir.find(thisDir) != -1:
                found = True
        if found:
            dirsToExamine = [expDir]
        else:
            raise ValueError('expDir %s not a valid experiment directory' % (expDir))
        
    retList = []
    
    for dirToExamine in dirsToExamine:
        for root, dirs, files in os.walk(dirToExamine):
            for name in files:
                fullname = os.path.join(root, name)
                relIndex = fullname[len(self.__madRootDir):].find('/experiments')
                relativeName = fullname[relIndex + 1 + (len(self.__madRootDir)):]
                ts = os.stat(fullname).st_mtime
                if relative:
                    retList.append((relativeName, datetime.datetime.utcfromtimestamp(ts)))
                else:
                    retList.append((fullname, datetime.datetime.utcfromtimestamp(ts)))
                
    return(retList)

def setFileAccess(

self, expDirectory, accessMode)

setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private.

Inputs:

expDirectory:  The full path to a directory in the experiment directory.  That is, it
may be madroot/experiments[0-9]* or any directory under it.

accessMode: either 0 for public access, or 1 for private access.

Returns: None

Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private.

Exceptions: If accessMode is not 1 or 0.

def setFileAccess(self, expDirectory, accessMode):
    """setFileAccess sets all fileTab.txt files in all subdirectories of expDirectory to be public or private.
    Inputs:
        expDirectory:  The full path to a directory in the experiment directory.  That is, it
        may be madroot/experiments[0-9]* or any directory under it.
        accessMode: either 0 for public access, or 1 for private access.
    
    Returns: None
    Affects: sets all fileTab.txt files in all subdirectories of expDirectory to be public or private.
    Exceptions: If accessMode is not 1 or 0.
    """
    if (accessMode != 0 and accessMode != 1):
        raise madrigal.admin.MadrigalError('MadrigalDB.setFileAccess called with accessMode = ' + \
               str(accessMode) + ', must be either 0 or 1', None)
    # walk the experiments directory to find all files meeting criteria
       
    for root, dirs, files in os.walk(expDirectory):
        self.__setFileAccess(accessMode, root, dirs + files)

def tarExperiments(

self, tarFileName, startDate=None, endDate=None, excludePrivData=0, ignoreDirCon=1, includeNonDefData=0, onlyData=0, filetype=0, verbose=0)

tarExperiments creates a tar file containing files from madroot/experiments[0-9]*.

Note: this method sometimes requires the modification of the fileTab.txt files found in the experiments directory. This is because some data files might be excluded, so that the fileTab.txt file will no longer be accurate. Because of this, all files to be tar'ed will be copied to /tmp/temp/experiments, where the fileTab.txt files will be modified. When done, this temp dir will be deleted.

Inputs:

tarFileName:  The full path to a tar file to be created.

startDate: a python date (see time module - actually a tuple of nine integers)
after which to accept files.  If None (default), do not reject any files.

endDate: a python date (see time module - actually a tuple of nine integers)
before which to accept files.  If None (default), do not reject any files.

excludePrivData: if 1, allow data marked as private to be omitted (and the line
from the fileTab.txt to be removed).  If 0 (the default), all data, public and
private, will be included.

ignoreDirCon: if 1, ignore convention that directory must be in form
1999/mlh/03sep99 (the default).  If 0 , reject non-standard directories.

includeNonDefData:  if 1, include all files listed in fileTab.txt. If 0 (the default),
reject non-default files, and modify fileTab.txt to remove non-default listings.

onlyData: if 1, reject all files not listed in fileTab.txt.  If 0 (the default),
accept all files in a directory not mentioned in fileTab.txt.

filetype: format to save data files as.  Default 0 is to leave present format unchanged.
<type> is an integer as follows:

    type = 0  Leave present format unchanged (default)

    type = 1  Madrigal

    type = 2  Blocked Binary

    type = 3  Cbf

    type = 4  Unblocked binary

    type = 5  Ascii

verbose: if 1, print to std out the list of files included (relative path).  If 0,
(the default) print nothing.

Returns: None

Affects: created tar file tarFileName of selected files from madroot/experiments.

Exceptions: If unable to read any experiment file.

def tarExperiments(self,
                   tarFileName,
                   startDate = None,
                   endDate = None,
                   excludePrivData = 0,
                   ignoreDirCon = 1,
                   includeNonDefData = 0,
                   onlyData = 0,
                   filetype = 0,
                   verbose = 0):
    """tarExperiments creates a tar file containing files from madroot/experiments[0-9]*.
    Note:  this method sometimes requires the modification of the fileTab.txt
    files found in the experiments directory.  This is because some data files might be
    excluded, so that the fileTab.txt file will no longer be accurate.  Because of this,
    all files to be tar'ed will be copied to /tmp/temp/experiments,
    where the fileTab.txt files will be modified.  When done, this temp dir will be deleted.
    Inputs:
        tarFileName:  The full path to a tar file to be created.
        startDate: a python date (see time module - actually a tuple of nine integers)
        after which to accept files.  If None (default), do not reject any files.
        endDate: a python date (see time module - actually a tuple of nine integers)
        before which to accept files.  If None (default), do not reject any files.
        
        excludePrivData: if 1, allow data marked as private to be omitted (and the line
        from the fileTab.txt to be removed).  If 0 (the default), all data, public and
        private, will be included.
        ignoreDirCon: if 1, ignore convention that directory must be in form
        1999/mlh/03sep99 (the default).  If 0 , reject non-standard directories.
        includeNonDefData:  if 1, include all files listed in fileTab.txt. If 0 (the default),
        reject non-default files, and modify fileTab.txt to remove non-default listings.
        onlyData: if 1, reject all files not listed in fileTab.txt.  If 0 (the default),
        accept all files in a directory not mentioned in fileTab.txt.
        filetype: format to save data files as.  Default 0 is to leave present format unchanged.
         is an integer as follows:
            type = 0  Leave present format unchanged (default)
        
            type = 1  Madrigal
            
            type = 2  Blocked Binary
            
            type = 3  Cbf
            
            type = 4  Unblocked binary
            
            type = 5  Ascii
        verbose: if 1, print to std out the list of files included (relative path).  If 0,
        (the default) print nothing.
    
    Returns: None
    Affects: created tar file tarFileName of selected files from madroot/experiments. 
    Exceptions: If unable to read any experiment file.
    """
    
    if ignoreDirCon == 1:
        enforcePathConvention = 0
    else:
        enforcePathConvention = 1
    if onlyData == 1:
        includeNonMadrigal = 0
    else:
        includeNonMadrigal = 1
    # create a random temp dir
    tempDir = '/tmp/temp' + str(random.randrange(1,10000000))
    if verbose == 1:
        print ('Creating list of files to tar...')
    # get list of files to tar
    tarFileList = self.getFileList(None,
                                   None,
                                   None,
                                   startDate,
                                   endDate,
                                   None,
                                   None,
                                   excludePrivData,
                                   enforcePathConvention,
                                   includeNonDefData,
                                   includeNonMadrigal)
    
    # now create a new list of filenames, using relative paths
    relTarFileList = []
    for file in tarFileList:
        newFilename = file.split(self.getMadroot() + '/')[1]
        relTarFileList.append(newFilename)
    # copy all these files to tempDir
    if verbose == 1:
        print('The following is a list of files included:')
    for file in relTarFileList:
        # make sure dir exists
        try:
            os.makedirs(tempDir + '/' + os.path.dirname(file))
        except:
            pass
        # now copy file
        shutil.copyfile(self.getMadroot() + '/' + file, tempDir + '/' + file)
        if verbose == 1:
            print('\t' + file)
        if onlyData:
            # now check if that data file needs to be converted to another filetype
            # since onlyData is set, convert every file
            if filetype != 0:
                # copy data file to another location
                shutil.copyfile(tempDir + '/' + file, tempDir + '/' + file + '.backup')
                # run mergeCedarFiles
                execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \
                          tempDir + '/' + file + '.backup 1 100000000 -o ' + \
                          tempDir + '/' + file + \
                          ' -t ' + str(filetype)
                os.system(execStr)
                os.remove(tempDir + '/' + file + '.backup')
    if verbose == 1:
        print('Modifying fileTab.txt files if needed...')
    # modify fileTab.txt files, if required
    if (not onlyData) and (excludePrivData or not includeNonDefData):
        # first loop through each fileTab.txt file in list just to check permissions
        for filename in relTarFileList:
            if os.path.basename(filename) != 'fileTab.txt':
                continue
            
            # make sure fileTab.txt is writable
            if not os.access(tempDir + '/' + filename, os.W_OK):
                raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \
                                                   'for ' + str(filename), None)
            
            # make sure the directory is writable
            if not os.access(os.path.dirname(tempDir + '/' + filename), os.W_OK):
                raise madrigal.admin.MadrigalError('Unable to tar experiments because denied write permission ' + \
                                                   'for ' + str(os.path.dirname(filename)), None)
        # no exceptions raised - all permissions are okay
        # this time loop through and modify the fileTab.txt files
        for filename in relTarFileList:
            if os.path.basename(filename) != 'fileTab.txt':
                continue
            # create a MadrigalMetaFile object 
            fileMeta = MadrigalMetaFile(self, tempDir + '/' + filename)
            # loop through each file name to see if its in tarFileList:
            fileNum = 0
            while 1:
                madFilename = fileMeta.getFilenameByPosition(fileNum)
                if madFilename == None:
                    break
                # get madRelFilename 
                madRelFilename = os.path.dirname(filename) + '/' + madFilename
                # if its not in relTarFileList, delete it from fileMeta
                if not madRelFilename in relTarFileList:
                    fileMeta.deleteRowByFilename(madFilename)
                    fileNum = 0
                    continue
                else:
                    # we know madRelFilename is a data file since its in fileTab.txt -
                    # now check if that data file needs to be converted to another filetype
                    if filetype != 0:
                        # copy data file to another location
                        shutil.copyfile(tempDir + '/' + madRelFilename, tempDir + '/' + madRelFilename + '.backup')
                        # run mergeCedarFiles
                        execStr = self.getMadroot() + '/bin/mergeCedarFiles -i ' + \
                                  tempDir + '/' + madRelFilename + '.backup 1 100000000 -o ' + \
                                  tempDir + '/' + madRelFilename + \
                                  ' -t ' + str(filetype)
                        os.system(execStr)
                        os.remove(tempDir + '/' + madRelFilename + '.backup')
                # get next madFilename
                fileNum = fileNum + 1
            # if fileMeta not empty, write it out
            if fileMeta.getFileCount() > 0:
                fileMeta.writeMetadata()
            # else if its empty, simply delete it
            else:
                os.remove(tempDir + '/' + filename)
    # done modifying fileTab.txt - ready to tar
    # need to change working directory to tempDir to get relative paths in tar.
    # Since this directory will soon be deleted, will need to change it back to
    # whatever it is now when we're done
    pwd = os.getcwd()
    
    # check if tarFileName is absolute or relative to pwd
    if tarFileName[0] != '/':
        tarFileName = pwd + '/' + tarFileName
    if verbose == 1:
        print('Creating tar file...')
        
    os.chdir(tempDir)
    os.system('tar -cf ' + tarFileName + ' experiments*')
    os.chdir(pwd)
    if verbose == 1:
        print('Removing temp files...')
    # finally remove temp dir
    try:
        shutil.rmtree(tempDir)
    except:
        raise madrigal.admin.MadrigalError('In tarExperiments could not remove dir ' + tempDir,
                                           traceback.format_exception(sys.exc_info()[0],
                                                                      sys.exc_info()[1],
                                                                      sys.exc_info()[2]))

def toString(

self)

toString returns a simple string representation of a MadrigalDB object.

Inputs: None

Returns: String describing a simple representation of a MadrigalDB object.

Affects: Nothing

Exceptions: None

def toString(self):
    """toString returns a simple string representation of a MadrigalDB object.
    Inputs: None
    
    Returns: String describing a simple representation of a MadrigalDB object.
    Affects: Nothing
    Exceptions: None
    """
    output =  "Object type: MadrigalDB\n"
    output += "Database utility directory = "   + self.getDatabaseUtilityDirectory() + "\n"
    output += "WWW home base = "                + self.getWWWHomeBase() + "\n"
    output += "Server name = "                  + self.getMadServer() + "\n"
    output += "Top level url = "                + self.getTopLevelUrl() + "\n"
    output += "Relative Top level url = "       + self.getRelativeTopLevel() + "\n"
    output += "Site ID = "                      + str(self.getSiteID()) + "\n"
    output += "MAD_ROOT env. variable name = "   + self.getMadrootEnvVarName() + "\n"
    output += "MAD_ROOT env. variable value = "  + self.getMadroot() + "\n"
    output += "Madrigal metadata dir = "        + self.getMetadataDir() + "\n"
    output += "Madrigal bin dir = "             + self.getBinDir() + "\n"
    output += "Madrigal html body style = "     + self.getHtmlStyle() + "\n"
    output += "Madrigal top level heading = "   + self.getIndexHead() + "\n"
    output += "Madrigal contact link = "        + self.getContactLink() + "\n"
    output += "Madrigal contact email = "       + str(self.getContactEmail()) + "\n"
    output += "Madrigal mailserver = "          + self.getMailserver() + "\n"
    output += "Madrigal python exe = "          + self.getPythonExecutable() + "\n"
    return output

class MadrigalExperiment

MadrigalExperiment is an object that provides access to Madrigal experiment info from the metadata.

This object provides access to all Madrigal experiment information in the metadata file expTab.txt.

Usage example::

import madrigal.metadata

import madrigal.admin

try:

    test = madrigal.metadata.MadrigalExperiment()

    print test.getExperimentName(3001)

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

2. Columns expected to be ints or floats cannot be converted

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Apr. 17, 2002

class MadrigalExperiment:
    """MadrigalExperiment is an object that provides access to Madrigal experiment info from the metadata.

    This object provides access to all Madrigal experiment information in the metadata file expTab.txt.

    Usage example::

        import madrigal.metadata
        
        import madrigal.admin

        try:
    
            test = madrigal.metadata.MadrigalExperiment()

            print test.getExperimentName(3001)

        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file
        
        2. Columns expected to be ints or floats cannot be converted

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Apr. 17, 2002

    """

    #constants
    __expMetadataFile  = "expTab.txt"

    # column positions
    __expIdCol           =  0
    __expUrlCol          =  1
    __expNameCol         =  2
    __expSiteIdCol       =  3
    __expStartDateCol    =  4
    __expStartTimeCol    =  5
    __expEndDateCol      =  6
    __expEndTimeCol      =  7
    __expKinstCol        =  8
    __expSecurityCol     =  9
    __expPICol           =  10
    __expPIEmailCol      =  11
    

    def __init__(self, madDB=None, initFile=None):
        """__init__ initializes MadrigalExperiment by reading from expTab.txt (or initFile).

        Inputs: Existing MadrigalDB object, by default = None.

            String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/expTab.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get experiment metadata file
        if (initFile == None):
            self.__filename = self.__expMetadataFile
        else:
            self.__filename = initFile
            
        # MadrigalExperiment can now legal have two different lengths - with or without PI and email
        allowedLens = (self.__expSecurityCol+1, self.__expPIEmailCol+1)
        
        genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__expIdCol)

        self.__fileList = genericObj.getList()
        self._dict = genericObj.getDict()


    def getExpIdByPosition(self, position = 0):
        """getExpIdByPosition returns the experiment id of the experiment at given position.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the experiment id (integer), or None if position >= number of experiments.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__expIdCol])

            except:
                raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)


    def setExpIdByPosition(self, position, expId):
        """setExpIdByPosition sets the experiment id of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            expId - the new experiment id to use
        
        Returns: None.

        Affects: sets the experiment id of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        """
        
        try:
            self.__fileList[position][self.__expIdCol] = str(int(expId))
            self._dict[position] = str(int(expId))

        except:
            raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' %  \
                                               (str((position, expId)),
                                                traceback.format_exception(sys.exc_info()[0],
                                                                           sys.exc_info()[1],
                                                                           sys.exc_info()[2])))


    def getExpUrlByPosition(self, position = 0):
        """getExpUrlByPosition returns the experiment url of the experiment at given position.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the experiment url, or None if position >= number of experiments.

        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:
            return self.__fileList[position][self.__expUrlCol]

        else:
            return(None)


    def getExpUrlByExpId(self, expId):
        """getExpUrlByExpId returns the experiment url for a given experiment id.

        Inputs: Experiment Id (integer).
        
        Returns: the experiment url (string).  Returns None if experiment id not found.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpUrlByPosition(position))

    
    
    def getRealExpUrlByPosition(self, position = 0):
        """getRealExpUrlByPosition returns the real experiment url of the experiment at given position.
        
        The url in the metadata may contain /madtoc/ for historical reasons.  This method converts that url to the real one.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the real experiment url, or None if position >= number of experiments.

        Affects: None

        Exceptions: 
        """
        thisUrl = self.getExpUrlByPosition(position)
        if not thisUrl:
            return(None)
        index = thisUrl.find('/madtoc/')
        if index == -1:
            return(thisUrl)
        if thisUrl.find('experiments') != -1:
            realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + thisUrl[index+8:])
        else:
            realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + 'experiments', thisUrl[index+8:])
        return(realUrl)


    def getRealExpUrlByExpId(self, expId):
        """getRealExpUrlByExpId returns the real experiment url for a given experiment id.
        
        The url in the metadata contains /madtoc/ for historical reasons.  This method converts that url to the real one.

        Inputs: Experiment Id (integer).
        
        Returns: the real experiment url (string).  Returns None if experiment id not found.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getRealExpUrlByPosition(position))
    
    
    def getExpPathByPosition(self, position = 0):
        """getExpPathByPosition returns the experiment path of the experiment at given position.
        
        Experiment path is the path to the experiment from the madroot directory, and always begins "experiments"
        
        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments.

        Affects: None

        Exceptions: 
        """
        thisUrl = self.getExpUrlByPosition(position)
        if not thisUrl:
            return(None)
        index = thisUrl.find('/madtoc/')
        if index != -1:
            return(thisUrl[index+8:])
        index = thisUrl.find('experiments')
        if index == -1:
            return(None)
        return(thisUrl[index:])


    def getExpPathByExpId(self, expId):
        """getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id.
        
        The url in the metadata may contain /madtoc/ for historical reasons.  This method converts that url to the real one.

        Inputs: Experiment Id (integer).
        
        Returns: the path to the experiment from the madroot directory (string).  Returns None if experiment id not found.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpPathByPosition(position))


    def setExpUrlByPosition(self, position, expUrl):
        """setExpUrlByPosition sets the experiment url of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            expUrl - the new experiment url to use
        
        Returns: None.

        Affects: sets the experiment url of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        """
        # verify no illegal commas
        if expUrl.find(',') != -1:
            raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' %  \
                                               (str(position, expUrl),
                                                [traceback.format_exc()]))
        
        try:
            self.__fileList[position][self.__expUrlCol] = str(expUrl)

        except:
            raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' %  \
                                               (str(position, expUrl),
                                                [traceback.format_ex()]))

    

    def getExpDirByPosition(self, position = 0):
        """getExpDirByPosition returns the full experiment directory of the experiment at given position.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the full experiment directory, or None if position >= number of experiments.  Uses
        experiment url to determine directory.

        Affects: None

        Exceptions: None
        """
        if len(self.__fileList) > position:
            url = self.__fileList[position][self.__expUrlCol]
            # find the directory based on url
            maddir = self.__madDB.getMadroot()
            index = url.find('/madtoc/')
            partialExpDir = url[index+8:]
            # added default experiments if not there already
            if partialExpDir.find('experiments') == -1:
                maddir = os.path.join(maddir, 'experiments', url[index+8:])
            else:
                maddir = os.path.join(maddir, url[index+8:])
            return(maddir)

        else:
            return(None)


    def getExpDirByExpId(self, expId):
        """getExpDirByExpId returns the full experiment directory for a given experiment id.

        Inputs: Experiment Id (integer).
        
        Returns: the full experiment directory (string).  Returns None if experiment id not found.
        Uses experiment url to determine directory.

        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpDirByPosition(position))


        
    def getExpNameByPosition(self, position = 0):
        """getExpNameByPosition returns the experiment name of the experiment at given position.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the experiment name, or None if position >= number of experiments.

        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:
            return self.__fileList[position][self.__expNameCol]

        else:
            return(None)


    def getExpNameByExpId(self, expId):
        """getExpNameByExpId returns the experiment name for a given experiment id.

        Inputs: Experiment Id (integer).
        
        Returns: the experiment name (string).  Returns None if experiment id not found.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpNameByPosition(position))


    def setExpNameByPosition(self, position, expName):
        """setExpNameByPosition sets the experiment name of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            expName - the new experiment name to use
        
        Returns: None.

        Affects: sets the experiment name of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        """
        # verify no illegal commas
        if expName.find(',') != -1:
            raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' %  \
                                               (str(position, expName),
                                                [traceback.format_exc()]))
        
        try:
            self.__fileList[position][self.__expNameCol] = str(expName)

        except:
            raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' %  \
                                               (str(position, expName),
                                                traceback.format_exception(sys.exc_info()[0],
                                                                           sys.exc_info()[1],
                                                                           sys.exc_info()[2])))


    def getExpSiteIdByExpId(self, expId):
        """getExpSiteIdByExpId returns the site id (int) for a given experiment id.

        Inputs: Experiment Id (integer).
        
        Returns: the site id for this experiment.  Returns None if experiment id not found.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpSiteIdByPosition(position))


    def getExpSiteIdByPosition(self, position = 0):
        """getExpSiteIdByPosition returns the experiment site id of the experiment at given position.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the experiment site id (integer), or None if position >= number of experiments.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__expSiteIdCol])

            except:
                raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)


    def setExpSiteIdByPosition(self, position, expSiteId):
        """setExpSiteIdByPosition sets the experiment site id of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            expSiteId - the new experiment site id to use
        
        Returns: None.

        Affects: sets the experiment site id of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        """
        
        try:
            self.__fileList[position][self.__expSiteIdCol] = str(int(expSiteId))

        except:
            raise madrigal.admin.MadrigalError('Error in setExpSiteIdByPosition with args %s: %s' %  \
                                               (str(position, expSiteId),
                                                traceback.format_exception(sys.exc_info()[0],
                                                                           sys.exc_info()[1],
                                                                           sys.exc_info()[2])))


    def getExpStartDateTimeByPosition(self, position = 0):
        """getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.  
        Returns None if position >= number of experiments.  Since mktime does not go before 1970, I use
        date.c methods, and miss only day of week and DST flag
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:
            # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings
            startTime =  [int(self.__fileList[int(position)][self.__expStartDateCol][0:4]),
                          int(self.__fileList[int(position)][self.__expStartDateCol][4:6]),
                          int(self.__fileList[int(position)][self.__expStartDateCol][6:8]),
                          int(self.__fileList[int(position)][self.__expStartTimeCol][0:2]),
                          int(self.__fileList[int(position)][self.__expStartTimeCol][2:4]),
                          int(self.__fileList[int(position)][self.__expStartTimeCol][4:6]),
                          0,
                          0,
                          0]

            # handle hour = 24 case
            if startTime[3] == 24:
                startTime[3] = 23
                startTime[4] = 59
                startTime[5] = 59
            
            # we still need day of year
            utcTime = madrigal.metadata.getMadrigalUTFromDate(startTime[0],
                                                     startTime[1],
                                                     startTime[2],
                                                     startTime[3],
                                                     startTime[4],
                                                     startTime[5],
                                                     0)

            utcDate = madrigal._derive.getDateFromUt(utcTime)

            startTime[7] = utcDate[7]

            # return python time tuple, missing only day of week and DST flag
            return startTime

        else:
            return(None)



    def getExpEndDateTimeByPosition(self, position = 0):
        """getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position.

        Inputs: position of experiment in list (first position is zero).  Defaults to first.
        
        Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.  
        Returns None if position >= number of experiments.  Since mktime does not go before 1970, I use
        date.c methods, and miss only day of week and DST flag
        
        Affects: None

        Exceptions: None
        """
        position = int(position)

        if len(self.__fileList) > position:
            # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings
            endTime =  [int(self.__fileList[position][self.__expEndDateCol][0:4]),
                        int(self.__fileList[position][self.__expEndDateCol][4:6]),
                        int(self.__fileList[position][self.__expEndDateCol][6:8]),
                        int(self.__fileList[position][self.__expEndTimeCol][0:2]),
                        int(self.__fileList[position][self.__expEndTimeCol][2:4]),
                        int(self.__fileList[position][self.__expEndTimeCol][4:6]),
                        0,
                        0,
                        0]

            # handle hour = 24 case
            if endTime[3] == 24:
                endTime[3] = 23
                endTime[4] = 59
                endTime[5] = 59
            
            # we still need day of year
            utcTime = madrigal.metadata.getMadrigalUTFromDate(endTime[0],
                                                     endTime[1],
                                                     endTime[2],
                                                     endTime[3],
                                                     endTime[4],
                                                     endTime[5],
                                                     0)

            utcDate = madrigal._derive.getDateFromUt(utcTime)

            endTime[7] = utcDate[7]

            # return python time tuple, missing only day of week and DST flag
            return endTime

        else:
            return(None)



    def getExpStartDateTimeByExpId(self, expId):
        """getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id.
        
        Inputs: Experiment Id (integer).
        
        Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.  
        Returns None if experiment id not found.  Since mktime does not go before 1970, I use
        date.c methods, and miss only day of week and DST flag
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpStartDateTimeByPosition(position))



    def getExpEndDateTimeByExpId(self, expId):
        """getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id.
        
        Inputs: Experiment Id (integer).
        
        Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.  
        Returns None if experiment id not found.  Since mktime does not go before 1970, I use
        date.c methods, and miss only day of week and DST flag
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpEndDateTimeByPosition(position))

        

    def setExpStartDateTimeByPosition(self, startDateTime, position = 0):
        """setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position.

        Inputs:

            startDateTime - a python datetime object to set the exp start date and time to.

            position - which experiment row to change - defaults to 0
        
        Returns: None.
        
        Affects: sets exp start date and time in self.__fileList.

        Exceptions: None
        """
        self.__fileList[position][self.__expStartDateCol] = '%04i%02i%02i' % (startDateTime.year,
                                                                              startDateTime.month,
                                                                              startDateTime.day)

        self.__fileList[position][self.__expStartTimeCol] = '%02i%02i%02i' % (startDateTime.hour,
                                                                              startDateTime.minute,
                                                                              startDateTime.second)


    def setExpEndDateTimeByPosition(self, endDateTime, position = 0):
        """setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position.

        Inputs:

            endDateTime - a python datetime object to set the exp end date and time to.

            position - which experiment row to change - defaults to 0
        
        Returns: None.
        
        Affects: sets exp end date and time in self.__fileList.

        Exceptions: None
        """
        self.__fileList[position][self.__expEndDateCol] = '%04i%02i%02i' % (endDateTime.year,
                                                                            endDateTime.month,
                                                                            endDateTime.day)

        self.__fileList[position][self.__expEndTimeCol] = '%02i%02i%02i' % (endDateTime.hour,
                                                                            endDateTime.minute,
                                                                            endDateTime.second)



    def getKinstByPosition(self, position = 0):
        """getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the kinst (instrument code) of the file at given position as an integer.  
        Returns None if position >= number of files.
        
        Affects: None

        Exceptions: Thrown if kinst column cannot be parsed into an integer
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__expKinstCol])

            except:
                raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)



    def getKinstByExpId(self, expId):
        """getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id.
        
        Inputs: Experiment Id (integer).
        
        Returns: the experiment kinst (integer).  Returns None if experiment id not found.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getKinstByPosition(position))

       
    def setExpKinstByPosition(self, position, expKinst):
        """setExpKinstByPosition sets the experiment kinst of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            expKinst - the new experiment kinst to use
        
        Returns: None.

        Affects: sets the experiment kinst of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        """
        
        try:
            self.__fileList[position][self.__expKinstCol] = str(int(expKinst))

        except:
            raise madrigal.admin.MadrigalError('Error in setExpKinstByPosition with args %s: %s' %  \
                                               (str(position, expKinst),
                                                traceback.format_exception(sys.exc_info()[0],
                                                                           sys.exc_info()[1],
                                                                           sys.exc_info()[2])))


    def getSecurityByPosition(self, position = 0):
        """getSecurityByPosition returns the security code of the experiment at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the security code (integer) of the file at given position as an integer.  
        Returns None if position >= number of files.
        
        Affects: None

        Exceptions: Thrown if security column cannot be parsed into an integer
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__expSecurityCol])

            except:
                raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)


    def getSecurityByExpId(self, expId):
        """getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id.
        
        Inputs: Experiment Id (integer).
        
        Returns: the security code (integer).  Returns None if experiment id not found.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getSecurityByPosition(position))


    def setSecurityByPosition(self, position, securityCode):
        """setSecurityByPosition sets the security code (integer) of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            securityCode - the new experiment security code (integer) to use
        
        Returns: None.

        Affects: sets the security code of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        """
        
        try:
            self.__fileList[position][self.__expSecurityCol] = str(int(securityCode))

        except:
            raise madrigal.admin.MadrigalError('Error in setSecurityByPosition with args %s: %s' %  \
                                               (str(position, securityCode),
                                                traceback.format_exception(sys.exc_info()[0],
                                                                           sys.exc_info()[1],
                                                                           sys.exc_info()[2])))
            
            
    def getPIByPosition(self, position = 0):
        """getPIByPosition returns the principal investigator of the experiment at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the principal investigator's name (string) of the file at given position as a string.  
        Returns None if position >= number of files.  Since not all expTab.txt files may have this
        column, returns None if the column does not exist.
        
        Affects: None

        Exceptions: None
        
        This method added in Madrigal 2.6
        """

        if len(self.__fileList) > position:

            try:
                pi = self.__fileList[position][self.__expPICol]
                if len(pi) > 0:
                    return(pi)
                else:
                    return(None)

            except:
                return(None)
        else:
            return(None)


    def getPIByExpId(self, expId):
        """getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id.
        
        Inputs: Experiment Id (integer).
        
        Returns: the principal investigator's name (string).  Returns None if experiment id not found. 
        Since not all expTab.txt files may have this column, returns None if the column does not exist.
        
        Affects: None

        Exceptions: None
        
        This method added in Madrigal 2.6
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getPIByPosition(position))


    def setPIByPosition(self, position, PI):
        """setPIByPosition sets the principal investigator (string) of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            PI - the new experiment principal investigator's name (string) to use
        
        Returns: None.

        Affects: sets the principal investigator of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        
        This method added in Madrigal 2.6
        """
        # verify no illegal commas
        if PI.find(',') != -1:
            raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' %  \
                                               (str(position, PI),
                                                [traceback.format_exc()]))
        
        if len(self.__fileList[position]) == self.__expPICol:
            # add PI and blank email
            self.__fileList[position].append(str(PI))
            self.__fileList[position].append('')
        else:
            try:
                self.__fileList[position][self.__expPICol] = str(PI)

            except:
                raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' %  \
                                                   (str(position, PI),
                                                    [traceback.format_exc()]))
                
    
    def getPIEmailByPosition(self, position = 0):
        """getPIEmailByPosition returns the principal investigator email of the experiment at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the principal investigator's email (string) of the file at given position as a string.  
        Returns None if position >= number of files.  Since not all expTab.txt files may have this
        column, returns None if the column does not exist.
        
        Affects: None

        Exceptions: None
        
        This method added in Madrigal 2.6
        """

        if len(self.__fileList) > position:

            try:
                piEmail = self.__fileList[position][self.__expPIEmailCol]
                if len(piEmail) > 0:
                    return(piEmail)
                else:
                    return(None)

            except:
                return(None)
        else:
            return(None)


    def getPIEmailByExpId(self, expId):
        """getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id.
        
        Inputs: Experiment Id (integer).
        
        Returns: the principal investigator's email (string).  Returns None if experiment id not found. 
        Since not all expTab.txt files may have this column, returns None if the column does not exist.
        
        Affects: None

        Exceptions: None
        
        This method added in Madrigal 2.6
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getPIEmailByPosition(position))


    def setPIEmailByPosition(self, position, PIEmail):
        """setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position.

        Inputs:

            position - position of experiment in list (first position is zero).

            PIEmail - the new experiment principal investigator's email (string) to use
        
        Returns: None.

        Affects: sets the principal investigator email of the experiment at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        
        This method added in Madrigal 2.6
        """
        # verify no illegal commas
        if PIEmail.find(',') != -1:
            raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' %  \
                                               (str(position, PIEmail),
                                                [traceback.format_exc()]))
        
        if len(self.__fileList[position]) == self.__expPICol:
            # add blank PI and email
            self.__fileList[position].append('')
            self.__fileList[position].append(str(PIEmail))
        else:
            try:
                self.__fileList[position][self.__expPIEmailCol] = str(PIEmail)

            except:
                raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' %  \
                                                   (str(position, PIEmail),
                                                    [traceback.format_exc()]))


    def getExpLinksByExpId(self, expId):
        """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.
        
        Inputs:

            Inputs: Experiment Id (integer).
        
        Returns: a list of (title, url) tuples containing all links for this experiment.

        In order to be a link, a file must be in the experiment directory in the form *.html,
        */index.html, or */*/index.html.  The title is parsed from the title in the head; if not found,
        returns 'No title' as title.  The follow file extensions are also links if found in the
        main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif'].  For these
        files, the title is simply the basename.
        
        Affects: None

        Exceptions: None
        """
        try:
            position = self._dict[str(expId)]
        except KeyError:
            return(None)
        return(self.getExpLinksByPosition(position))
        
        
    
    def getExpLinksByPosition(self, position = 0):
        """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.

        Inputs:

            Inputs: position - position of experiment in list (first position is zero).
        
        Returns: a list of (title, url) tuples containing all links for this experiment.

        In order to be a link, a file must be in the experiment directory in the form *.html,
        */index.html, or */*/index.html.  The title is parsed from the title in the head; if not found,
        returns 'No title' as title.  The follow file extensions are also links if found in the
        main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif'].  For these
        files, the title is simply the basename.
        
        Affects: None

        Exceptions: None
        """
        retList = []
        allowedExtensions = ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif', 'tar.gz']
        
        # find the experiment directory based on url
        topdir = self.__madDB.getMadroot()
        url = self.getExpUrlByPosition(position)
        if url == None:
            return retList
        index = url.find('/madtoc/')
        partialExpDir = url[index+8:]
        # added default experiments if not there already
        if partialExpDir.find('experiments') == -1:
            expdir = os.path.join(topdir, 'experiments', url[index+8:])
        else:
            expdir = os.path.join(topdir, url[index+8:])

        # now create a list of html files
        htmlFiles = []
        l1 = glob.glob(os.path.join(expdir, '*.html'))
        l2 = glob.glob(os.path.join(expdir, '*/index.html'))
        l3 = glob.glob(os.path.join(expdir, '*/*/index.html'))

        for item in l1:
            htmlFiles.append(item)
        for item in l2:
            htmlFiles.append(item)
        for item in l3:
            htmlFiles.append(item)
            
        for htmlFile in htmlFiles:
            # get title if possible
            f = open(htmlFile)
            text = f.read()
            f.close()
            i1 = text.upper().find('') + len('<TITLE>')
            i2 = text.upper().find('')
            if i1 > -1 and i2 > -1:
                title = text[i1:i2]
            else:
                title = 'No title'
            # get url
            i1 = htmlFile.find('experiments')
            url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', htmlFile[i1:])
            retList.append((title, url))
            
        
        for extension in allowedExtensions:
            plotList = glob.glob(os.path.join(expdir, '*.%s' % (extension)))
            for plot in plotList:
                i1 = plot.find('experiments')
                url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', plot[i1:])
                retList.append((os.path.basename(plot), url))

        return retList



    def getExpCount(self):
        """getExpCount returns number of experiments in MadrigalExperiment object
        """
        return(len(self.__fileList))


    def sortByDateSite(self):
        """sortByDateSite will resort self.__fileList so that experiments are listed first by experiment
        end date, and then by site
        """
        self.__fileList.sort(key=self.__compareDateSite__)
        # rebuild self._dict 
        self._dict = {}
        for i, items in enumerate(self.__fileList):
            self._dict[items[self.__expIdCol]] = i
        
        
    def getStartPosition(self, startDT):
        """getStartPosition returns the position of the first experiment with a start datetime after
        endDT after sorting by end time.
        
            Inputs: startDT - start datetime
            
            Returns: returns the position of the first experiment with a start datetime after
        startDT.
        """
        self.sortByDateSite()
        top = len(self.__fileList) - 1
        bottom = 0
        while(top >= bottom):
            if top == bottom:
                return(max(top-1, 0))
            middle = int((top+bottom)/2)
            thisEDTList = self.getExpEndDateTimeByPosition(middle)
            thisEDT = datetime.datetime(*thisEDTList[0:6])
            if thisEDT < startDT:
                bottom = middle + 1
                continue
            if middle == 0:
                return(0)
            prevEDTList = self.getExpEndDateTimeByPosition(middle-1)
            prevEDT = datetime.datetime(*prevEDTList[0:6])
            if prevEDT < startDT:
                return(middle-1)
            else:
                top = middle -1
     



    def __compareDateSite__(self, first):
        """__compareDateSite__ is a private method to help sort by start date and then site.

        first, second - lists of experiment information as parsed from expTab.txt files
        """
        result = (first[self.__expEndDateCol])
        if result != 0:
            return(result)
        result = (first[self.__expEndTimeCol])
        if result != 0:
            return(result)
        result = (first[self.__expStartDateCol])
        if result != 0:
            return(result)
        result = (first[self.__expStartTimeCol])
        if result != 0:
            return(result)
        return((first[self.__expSiteIdCol]))
    


    def getLine(self, position):
        """getLine returns the line at a given position.  Returns None if position > number of lines.

        Inputs:  position - position in file.  First line = 0
        """
        delimiter = ','

        if position >= len(self.__fileList):
            return(None)

        return(delimiter.join(self.__fileList[position]) + '\n')
  
        
    def writeMetadata(self, newFullPath=None):
        """writeMetadata writes a new version of the expTab.txt file.
        
        Inputs: newFullPath:  a new path to write the expTab.txt file to, if
        not the same as the original metadata file opened.  Defaults to None, which overwrites
        metadata file that was read from.
        
        Returns: None.

        Affects: Writes updated version of metadata file.

        Exceptions: If unable to write file
        """

        # create string to hold file
        metaFileStr = ''
        delimiter = ','

        for lineList in self.__fileList:
            metaFileStr += delimiter.join(lineList) + '\n'

        # try to write file, if not, raise exception

        try:

            if newFullPath == None:
                if (len(os.path.dirname(self.__filename)) != 0):
                    newFullPath = self.__filename
                else:
                    newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename
            newFile = open(newFullPath, "w")
            newFile.write(metaFileStr)
            newFile.close()

        except:

            raise madrigal.admin.MadrigalError("Unable to write metadata file " + \
                                               str(newFullPath),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))


    def __str__(self):
        """return possibly modified file as a string in same format as writeMetadata
        """
        # create string to hold file
        metaFileStr = ''
        delimiter = ','

        for lineList in self.__fileList:
            metaFileStr += delimiter.join(lineList) + '\n'
            
        return(metaFileStr)

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None)

init initializes MadrigalExperiment by reading from expTab.txt (or initFile).

Inputs: Existing MadrigalDB object, by default = None.

String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/expTab.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.

def __init__(self, madDB=None, initFile=None):
    """__init__ initializes MadrigalExperiment by reading from expTab.txt (or initFile).
    Inputs: Existing MadrigalDB object, by default = None.
        String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/expTab.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get experiment metadata file
    if (initFile == None):
        self.__filename = self.__expMetadataFile
    else:
        self.__filename = initFile
        
    # MadrigalExperiment can now legal have two different lengths - with or without PI and email
    allowedLens = (self.__expSecurityCol+1, self.__expPIEmailCol+1)
    
    genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__expIdCol)
    self.__fileList = genericObj.getList()
    self._dict = genericObj.getDict()

def getExpCount(

self)

getExpCount returns number of experiments in MadrigalExperiment object

def getExpCount(self):
    """getExpCount returns number of experiments in MadrigalExperiment object
    """
    return(len(self.__fileList))

def getExpDirByExpId(

self, expId)

getExpDirByExpId returns the full experiment directory for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the full experiment directory (string). Returns None if experiment id not found. Uses experiment url to determine directory.

Affects: None

Exceptions: None

def getExpDirByExpId(self, expId):
    """getExpDirByExpId returns the full experiment directory for a given experiment id.
    Inputs: Experiment Id (integer).
    
    Returns: the full experiment directory (string).  Returns None if experiment id not found.
    Uses experiment url to determine directory.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpDirByPosition(position))

def getExpDirByPosition(

self, position=0)

getExpDirByPosition returns the full experiment directory of the experiment at given position.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the full experiment directory, or None if position >= number of experiments. Uses experiment url to determine directory.

Affects: None

Exceptions: None

def getExpDirByPosition(self, position = 0):
    """getExpDirByPosition returns the full experiment directory of the experiment at given position.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the full experiment directory, or None if position >= number of experiments.  Uses
    experiment url to determine directory.
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        url = self.__fileList[position][self.__expUrlCol]
        # find the directory based on url
        maddir = self.__madDB.getMadroot()
        index = url.find('/madtoc/')
        partialExpDir = url[index+8:]
        # added default experiments if not there already
        if partialExpDir.find('experiments') == -1:
            maddir = os.path.join(maddir, 'experiments', url[index+8:])
        else:
            maddir = os.path.join(maddir, url[index+8:])
        return(maddir)
    else:
        return(None)

def getExpEndDateTimeByExpId(

self, expId)

getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.
Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag

Affects: None

Exceptions: None

def getExpEndDateTimeByExpId(self, expId):
    """getExpEndDateTimeByExpId returns the ending date/time of the experiment for a given experiment id.
    
    Inputs: Experiment Id (integer).
    
    Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.  
    Returns None if experiment id not found.  Since mktime does not go before 1970, I use
    date.c methods, and miss only day of week and DST flag
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpEndDateTimeByPosition(position))

def getExpEndDateTimeByPosition(

self, position=0)

getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.
Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag

Affects: None

Exceptions: None

def getExpEndDateTimeByPosition(self, position = 0):
    """getExpEndDateTimeByPosition returns the ending date/time of the experiment at given position.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the experiment end date/time in the standard python form of 9 item tuple in UTC.  
    Returns None if position >= number of experiments.  Since mktime does not go before 1970, I use
    date.c methods, and miss only day of week and DST flag
    
    Affects: None
    Exceptions: None
    """
    position = int(position)
    if len(self.__fileList) > position:
        # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings
        endTime =  [int(self.__fileList[position][self.__expEndDateCol][0:4]),
                    int(self.__fileList[position][self.__expEndDateCol][4:6]),
                    int(self.__fileList[position][self.__expEndDateCol][6:8]),
                    int(self.__fileList[position][self.__expEndTimeCol][0:2]),
                    int(self.__fileList[position][self.__expEndTimeCol][2:4]),
                    int(self.__fileList[position][self.__expEndTimeCol][4:6]),
                    0,
                    0,
                    0]
        # handle hour = 24 case
        if endTime[3] == 24:
            endTime[3] = 23
            endTime[4] = 59
            endTime[5] = 59
        
        # we still need day of year
        utcTime = madrigal.metadata.getMadrigalUTFromDate(endTime[0],
                                                 endTime[1],
                                                 endTime[2],
                                                 endTime[3],
                                                 endTime[4],
                                                 endTime[5],
                                                 0)
        utcDate = madrigal._derive.getDateFromUt(utcTime)
        endTime[7] = utcDate[7]
        # return python time tuple, missing only day of week and DST flag
        return endTime
    else:
        return(None)

def getExpIdByPosition(

self, position=0)

getExpIdByPosition returns the experiment id of the experiment at given position.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the experiment id (integer), or None if position >= number of experiments.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getExpIdByPosition(self, position = 0):
    """getExpIdByPosition returns the experiment id of the experiment at given position.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the experiment id (integer), or None if position >= number of experiments.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__expIdCol])
        except:
            raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getExpLinksByExpId(

self, expId)

getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.

Inputs:

Inputs: Experiment Id (integer).

Returns: a list of (title, url) tuples containing all links for this experiment.

In order to be a link, a file must be in the experiment directory in the form .html, /index.html, or //index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename.

Affects: None

Exceptions: None

def getExpLinksByExpId(self, expId):
    """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.
    
    Inputs:
        Inputs: Experiment Id (integer).
    
    Returns: a list of (title, url) tuples containing all links for this experiment.
    In order to be a link, a file must be in the experiment directory in the form *.html,
    */index.html, or */*/index.html.  The title is parsed from the title in the head; if not found,
    returns 'No title' as title.  The follow file extensions are also links if found in the
    main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif'].  For these
    files, the title is simply the basename.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpLinksByPosition(position))

def getExpLinksByPosition(

self, position=0)

getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.

Inputs:

Inputs: position - position of experiment in list (first position is zero).

Returns: a list of (title, url) tuples containing all links for this experiment.

In order to be a link, a file must be in the experiment directory in the form .html, /index.html, or //index.html. The title is parsed from the title in the head; if not found, returns 'No title' as title. The follow file extensions are also links if found in the main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif']. For these files, the title is simply the basename.

Affects: None

Exceptions: None

def getExpLinksByPosition(self, position = 0):
    """getExpLinksByExpId returns a list of (title, url) tuples containing all links for this experiment.
    Inputs:
        Inputs: position - position of experiment in list (first position is zero).
    
    Returns: a list of (title, url) tuples containing all links for this experiment.
    In order to be a link, a file must be in the experiment directory in the form *.html,
    */index.html, or */*/index.html.  The title is parsed from the title in the head; if not found,
    returns 'No title' as title.  The follow file extensions are also links if found in the
    main experiment directory: ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif'].  For these
    files, the title is simply the basename.
    
    Affects: None
    Exceptions: None
    """
    retList = []
    allowedExtensions = ['ps', 'eps', 'png', 'jpg', 'jpeg', 'pdf', 'gif', 'tar.gz']
    
    # find the experiment directory based on url
    topdir = self.__madDB.getMadroot()
    url = self.getExpUrlByPosition(position)
    if url == None:
        return retList
    index = url.find('/madtoc/')
    partialExpDir = url[index+8:]
    # added default experiments if not there already
    if partialExpDir.find('experiments') == -1:
        expdir = os.path.join(topdir, 'experiments', url[index+8:])
    else:
        expdir = os.path.join(topdir, url[index+8:])
    # now create a list of html files
    htmlFiles = []
    l1 = glob.glob(os.path.join(expdir, '*.html'))
    l2 = glob.glob(os.path.join(expdir, '*/index.html'))
    l3 = glob.glob(os.path.join(expdir, '*/*/index.html'))
    for item in l1:
        htmlFiles.append(item)
    for item in l2:
        htmlFiles.append(item)
    for item in l3:
        htmlFiles.append(item)
        
    for htmlFile in htmlFiles:
        # get title if possible
        f = open(htmlFile)
        text = f.read()
        f.close()
        i1 = text.upper().find('') + len('<TITLE>')
        i2 = text.upper().find('')
        if i1 > -1 and i2 > -1:
            title = text[i1:i2]
        else:
            title = 'No title'
        # get url
        i1 = htmlFile.find('experiments')
        url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', htmlFile[i1:])
        retList.append((title, url))
        
    
    for extension in allowedExtensions:
        plotList = glob.glob(os.path.join(expdir, '*.%s' % (extension)))
        for plot in plotList:
            i1 = plot.find('experiments')
            url = os.path.join(self.__madDB.getTopLevelUrl(), 'static', plot[i1:])
            retList.append((os.path.basename(plot), url))
    return retList

def getExpNameByExpId(

self, expId)

getExpNameByExpId returns the experiment name for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the experiment name (string). Returns None if experiment id not found.

Affects: None

Exceptions: None

def getExpNameByExpId(self, expId):
    """getExpNameByExpId returns the experiment name for a given experiment id.
    Inputs: Experiment Id (integer).
    
    Returns: the experiment name (string).  Returns None if experiment id not found.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpNameByPosition(position))

def getExpNameByPosition(

self, position=0)

getExpNameByPosition returns the experiment name of the experiment at given position.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the experiment name, or None if position >= number of experiments.

Affects: None

Exceptions: None

def getExpNameByPosition(self, position = 0):
    """getExpNameByPosition returns the experiment name of the experiment at given position.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the experiment name, or None if position >= number of experiments.
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        return self.__fileList[position][self.__expNameCol]
    else:
        return(None)

def getExpPathByExpId(

self, expId)

getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id.

The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one.

Inputs: Experiment Id (integer).

Returns: the path to the experiment from the madroot directory (string). Returns None if experiment id not found.

Affects: None

Exceptions: None

def getExpPathByExpId(self, expId):
    """getRealExpUrlByExpId returns the experiment path of the experiment for a given experiment id.
    
    The url in the metadata may contain /madtoc/ for historical reasons.  This method converts that url to the real one.
    Inputs: Experiment Id (integer).
    
    Returns: the path to the experiment from the madroot directory (string).  Returns None if experiment id not found.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpPathByPosition(position))

def getExpPathByPosition(

self, position=0)

getExpPathByPosition returns the experiment path of the experiment at given position.

Experiment path is the path to the experiment from the madroot directory, and always begins "experiments"

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments.

Affects: None

Exceptions:

def getExpPathByPosition(self, position = 0):
    """getExpPathByPosition returns the experiment path of the experiment at given position.
    
    Experiment path is the path to the experiment from the madroot directory, and always begins "experiments"
    
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the path to the experiment from the madroot directory, or None if position >= number of experiments.
    Affects: None
    Exceptions: 
    """
    thisUrl = self.getExpUrlByPosition(position)
    if not thisUrl:
        return(None)
    index = thisUrl.find('/madtoc/')
    if index != -1:
        return(thisUrl[index+8:])
    index = thisUrl.find('experiments')
    if index == -1:
        return(None)
    return(thisUrl[index:])

def getExpSiteIdByExpId(

self, expId)

getExpSiteIdByExpId returns the site id (int) for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the site id for this experiment. Returns None if experiment id not found.

Affects: None

Exceptions: None

def getExpSiteIdByExpId(self, expId):
    """getExpSiteIdByExpId returns the site id (int) for a given experiment id.
    Inputs: Experiment Id (integer).
    
    Returns: the site id for this experiment.  Returns None if experiment id not found.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpSiteIdByPosition(position))

def getExpSiteIdByPosition(

self, position=0)

getExpSiteIdByPosition returns the experiment site id of the experiment at given position.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the experiment site id (integer), or None if position >= number of experiments.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getExpSiteIdByPosition(self, position = 0):
    """getExpSiteIdByPosition returns the experiment site id of the experiment at given position.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the experiment site id (integer), or None if position >= number of experiments.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__expSiteIdCol])
        except:
            raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getExpStartDateTimeByExpId(

self, expId)

getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.
Returns None if experiment id not found. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag

Affects: None

Exceptions: None

def getExpStartDateTimeByExpId(self, expId):
    """getExpStartDateTimeByExpId returns the starting date/time of the experiment for a given experiment id.
    
    Inputs: Experiment Id (integer).
    
    Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.  
    Returns None if experiment id not found.  Since mktime does not go before 1970, I use
    date.c methods, and miss only day of week and DST flag
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpStartDateTimeByPosition(position))

def getExpStartDateTimeByPosition(

self, position=0)

getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.
Returns None if position >= number of experiments. Since mktime does not go before 1970, I use date.c methods, and miss only day of week and DST flag

Affects: None

Exceptions: None

def getExpStartDateTimeByPosition(self, position = 0):
    """getExpStartDateTimeByPosition returns the starting date/time of the experiment at given position.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the experiment start date/time in the standard python form of 9 item tuple in UTC.  
    Returns None if position >= number of experiments.  Since mktime does not go before 1970, I use
    date.c methods, and miss only day of week and DST flag
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        # create time from year, month, day, hour, min, sec, weekday, julian day, daylight savings
        startTime =  [int(self.__fileList[int(position)][self.__expStartDateCol][0:4]),
                      int(self.__fileList[int(position)][self.__expStartDateCol][4:6]),
                      int(self.__fileList[int(position)][self.__expStartDateCol][6:8]),
                      int(self.__fileList[int(position)][self.__expStartTimeCol][0:2]),
                      int(self.__fileList[int(position)][self.__expStartTimeCol][2:4]),
                      int(self.__fileList[int(position)][self.__expStartTimeCol][4:6]),
                      0,
                      0,
                      0]
        # handle hour = 24 case
        if startTime[3] == 24:
            startTime[3] = 23
            startTime[4] = 59
            startTime[5] = 59
        
        # we still need day of year
        utcTime = madrigal.metadata.getMadrigalUTFromDate(startTime[0],
                                                 startTime[1],
                                                 startTime[2],
                                                 startTime[3],
                                                 startTime[4],
                                                 startTime[5],
                                                 0)
        utcDate = madrigal._derive.getDateFromUt(utcTime)
        startTime[7] = utcDate[7]
        # return python time tuple, missing only day of week and DST flag
        return startTime
    else:
        return(None)

def getExpUrlByExpId(

self, expId)

getExpUrlByExpId returns the experiment url for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the experiment url (string). Returns None if experiment id not found.

Affects: None

Exceptions: None

def getExpUrlByExpId(self, expId):
    """getExpUrlByExpId returns the experiment url for a given experiment id.
    Inputs: Experiment Id (integer).
    
    Returns: the experiment url (string).  Returns None if experiment id not found.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getExpUrlByPosition(position))

def getExpUrlByPosition(

self, position=0)

getExpUrlByPosition returns the experiment url of the experiment at given position.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the experiment url, or None if position >= number of experiments.

Affects: None

Exceptions: None

def getExpUrlByPosition(self, position = 0):
    """getExpUrlByPosition returns the experiment url of the experiment at given position.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the experiment url, or None if position >= number of experiments.
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        return self.__fileList[position][self.__expUrlCol]
    else:
        return(None)

def getKinstByExpId(

self, expId)

getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the experiment kinst (integer). Returns None if experiment id not found.

Affects: None

Exceptions: None

def getKinstByExpId(self, expId):
    """getKinstByExpId returns the kinst (integer) of the experiment for a given experiment id.
    
    Inputs: Experiment Id (integer).
    
    Returns: the experiment kinst (integer).  Returns None if experiment id not found.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getKinstByPosition(position))

def getKinstByPosition(

self, position=0)

getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the kinst (instrument code) of the file at given position as an integer.
Returns None if position >= number of files.

Affects: None

Exceptions: Thrown if kinst column cannot be parsed into an integer

def getKinstByPosition(self, position = 0):
    """getKinstByPosition returns the kinst (kind of instrument code) of the experiment at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the kinst (instrument code) of the file at given position as an integer.  
    Returns None if position >= number of files.
    
    Affects: None
    Exceptions: Thrown if kinst column cannot be parsed into an integer
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__expKinstCol])
        except:
            raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getLine(

self, position)

getLine returns the line at a given position. Returns None if position > number of lines.

Inputs: position - position in file. First line = 0

def getLine(self, position):
    """getLine returns the line at a given position.  Returns None if position > number of lines.
    Inputs:  position - position in file.  First line = 0
    """
    delimiter = ','
    if position >= len(self.__fileList):
        return(None)
    return(delimiter.join(self.__fileList[position]) + '\n')

def getPIByExpId(

self, expId)

getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the principal investigator's name (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist.

Affects: None

Exceptions: None

This method added in Madrigal 2.6

def getPIByExpId(self, expId):
    """getPIByExpId returns the principal investigator (string) of the experiment for a given experiment id.
    
    Inputs: Experiment Id (integer).
    
    Returns: the principal investigator's name (string).  Returns None if experiment id not found. 
    Since not all expTab.txt files may have this column, returns None if the column does not exist.
    
    Affects: None
    Exceptions: None
    
    This method added in Madrigal 2.6
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getPIByPosition(position))

def getPIByPosition(

self, position=0)

getPIByPosition returns the principal investigator of the experiment at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the principal investigator's name (string) of the file at given position as a string.
Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist.

Affects: None

Exceptions: None

This method added in Madrigal 2.6

def getPIByPosition(self, position = 0):
    """getPIByPosition returns the principal investigator of the experiment at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the principal investigator's name (string) of the file at given position as a string.  
    Returns None if position >= number of files.  Since not all expTab.txt files may have this
    column, returns None if the column does not exist.
    
    Affects: None
    Exceptions: None
    
    This method added in Madrigal 2.6
    """
    if len(self.__fileList) > position:
        try:
            pi = self.__fileList[position][self.__expPICol]
            if len(pi) > 0:
                return(pi)
            else:
                return(None)
        except:
            return(None)
    else:
        return(None)

def getPIEmailByExpId(

self, expId)

getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the principal investigator's email (string). Returns None if experiment id not found. Since not all expTab.txt files may have this column, returns None if the column does not exist.

Affects: None

Exceptions: None

This method added in Madrigal 2.6

def getPIEmailByExpId(self, expId):
    """getPIEmailByExpId returns the principal investigator email (string) of the experiment for a given experiment id.
    
    Inputs: Experiment Id (integer).
    
    Returns: the principal investigator's email (string).  Returns None if experiment id not found. 
    Since not all expTab.txt files may have this column, returns None if the column does not exist.
    
    Affects: None
    Exceptions: None
    
    This method added in Madrigal 2.6
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getPIEmailByPosition(position))

def getPIEmailByPosition(

self, position=0)

getPIEmailByPosition returns the principal investigator email of the experiment at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the principal investigator's email (string) of the file at given position as a string.
Returns None if position >= number of files. Since not all expTab.txt files may have this column, returns None if the column does not exist.

Affects: None

Exceptions: None

This method added in Madrigal 2.6

def getPIEmailByPosition(self, position = 0):
    """getPIEmailByPosition returns the principal investigator email of the experiment at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the principal investigator's email (string) of the file at given position as a string.  
    Returns None if position >= number of files.  Since not all expTab.txt files may have this
    column, returns None if the column does not exist.
    
    Affects: None
    Exceptions: None
    
    This method added in Madrigal 2.6
    """
    if len(self.__fileList) > position:
        try:
            piEmail = self.__fileList[position][self.__expPIEmailCol]
            if len(piEmail) > 0:
                return(piEmail)
            else:
                return(None)
        except:
            return(None)
    else:
        return(None)

def getRealExpUrlByExpId(

self, expId)

getRealExpUrlByExpId returns the real experiment url for a given experiment id.

The url in the metadata contains /madtoc/ for historical reasons. This method converts that url to the real one.

Inputs: Experiment Id (integer).

Returns: the real experiment url (string). Returns None if experiment id not found.

Affects: None

Exceptions: None

def getRealExpUrlByExpId(self, expId):
    """getRealExpUrlByExpId returns the real experiment url for a given experiment id.
    
    The url in the metadata contains /madtoc/ for historical reasons.  This method converts that url to the real one.
    Inputs: Experiment Id (integer).
    
    Returns: the real experiment url (string).  Returns None if experiment id not found.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getRealExpUrlByPosition(position))

def getRealExpUrlByPosition(

self, position=0)

getRealExpUrlByPosition returns the real experiment url of the experiment at given position.

The url in the metadata may contain /madtoc/ for historical reasons. This method converts that url to the real one.

Inputs: position of experiment in list (first position is zero). Defaults to first.

Returns: the real experiment url, or None if position >= number of experiments.

Affects: None

Exceptions:

def getRealExpUrlByPosition(self, position = 0):
    """getRealExpUrlByPosition returns the real experiment url of the experiment at given position.
    
    The url in the metadata may contain /madtoc/ for historical reasons.  This method converts that url to the real one.
    Inputs: position of experiment in list (first position is zero).  Defaults to first.
    
    Returns: the real experiment url, or None if position >= number of experiments.
    Affects: None
    Exceptions: 
    """
    thisUrl = self.getExpUrlByPosition(position)
    if not thisUrl:
        return(None)
    index = thisUrl.find('/madtoc/')
    if index == -1:
        return(thisUrl)
    if thisUrl.find('experiments') != -1:
        realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + thisUrl[index+8:])
    else:
        realUrl = os.path.join(self.__madDB.getTopLevelUrl(), 'showExperiment?experiment_list=' + 'experiments', thisUrl[index+8:])
    return(realUrl)

def getSecurityByExpId(

self, expId)

getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id.

Inputs: Experiment Id (integer).

Returns: the security code (integer). Returns None if experiment id not found.

Affects: None

Exceptions: None

def getSecurityByExpId(self, expId):
    """getSecurityByExpId returns the security code (integer) of the experiment for a given experiment id.
    
    Inputs: Experiment Id (integer).
    
    Returns: the security code (integer).  Returns None if experiment id not found.
    
    Affects: None
    Exceptions: None
    """
    try:
        position = self._dict[str(expId)]
    except KeyError:
        return(None)
    return(self.getSecurityByPosition(position))

def getSecurityByPosition(

self, position=0)

getSecurityByPosition returns the security code of the experiment at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the security code (integer) of the file at given position as an integer.
Returns None if position >= number of files.

Affects: None

Exceptions: Thrown if security column cannot be parsed into an integer

def getSecurityByPosition(self, position = 0):
    """getSecurityByPosition returns the security code of the experiment at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the security code (integer) of the file at given position as an integer.  
    Returns None if position >= number of files.
    
    Affects: None
    Exceptions: Thrown if security column cannot be parsed into an integer
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__expSecurityCol])
        except:
            raise madrigal.admin.MadrigalError('Error in expTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getStartPosition(

self, startDT)

getStartPosition returns the position of the first experiment with a start datetime after endDT after sorting by end time.

Inputs: startDT - start datetime

Returns: returns the position of the first experiment with a start datetime after

startDT.

def getStartPosition(self, startDT):
    """getStartPosition returns the position of the first experiment with a start datetime after
    endDT after sorting by end time.
    
        Inputs: startDT - start datetime
        
        Returns: returns the position of the first experiment with a start datetime after
    startDT.
    """
    self.sortByDateSite()
    top = len(self.__fileList) - 1
    bottom = 0
    while(top >= bottom):
        if top == bottom:
            return(max(top-1, 0))
        middle = int((top+bottom)/2)
        thisEDTList = self.getExpEndDateTimeByPosition(middle)
        thisEDT = datetime.datetime(*thisEDTList[0:6])
        if thisEDT < startDT:
            bottom = middle + 1
            continue
        if middle == 0:
            return(0)
        prevEDTList = self.getExpEndDateTimeByPosition(middle-1)
        prevEDT = datetime.datetime(*prevEDTList[0:6])
        if prevEDT < startDT:
            return(middle-1)
        else:
            top = middle -1

def setExpEndDateTimeByPosition(

self, endDateTime, position=0)

setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position.

Inputs:

endDateTime - a python datetime object to set the exp end date and time to.

position - which experiment row to change - defaults to 0

Returns: None.

Affects: sets exp end date and time in self.__fileList.

Exceptions: None

def setExpEndDateTimeByPosition(self, endDateTime, position = 0):
    """setExpEndDateTimeByPosition sets a new MadrigalExperiment end date and time by position.
    Inputs:
        endDateTime - a python datetime object to set the exp end date and time to.
        position - which experiment row to change - defaults to 0
    
    Returns: None.
    
    Affects: sets exp end date and time in self.__fileList.
    Exceptions: None
    """
    self.__fileList[position][self.__expEndDateCol] = '%04i%02i%02i' % (endDateTime.year,
                                                                        endDateTime.month,
                                                                        endDateTime.day)
    self.__fileList[position][self.__expEndTimeCol] = '%02i%02i%02i' % (endDateTime.hour,
                                                                        endDateTime.minute,
                                                                        endDateTime.second)

def setExpIdByPosition(

self, position, expId)

setExpIdByPosition sets the experiment id of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

expId - the new experiment id to use

Returns: None.

Affects: sets the experiment id of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

def setExpIdByPosition(self, position, expId):
    """setExpIdByPosition sets the experiment id of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        expId - the new experiment id to use
    
    Returns: None.
    Affects: sets the experiment id of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    """
    
    try:
        self.__fileList[position][self.__expIdCol] = str(int(expId))
        self._dict[position] = str(int(expId))
    except:
        raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' %  \
                                           (str((position, expId)),
                                            traceback.format_exception(sys.exc_info()[0],
                                                                       sys.exc_info()[1],
                                                                       sys.exc_info()[2])))

def setExpKinstByPosition(

self, position, expKinst)

setExpKinstByPosition sets the experiment kinst of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

expKinst - the new experiment kinst to use

Returns: None.

Affects: sets the experiment kinst of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

def setExpKinstByPosition(self, position, expKinst):
    """setExpKinstByPosition sets the experiment kinst of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        expKinst - the new experiment kinst to use
    
    Returns: None.
    Affects: sets the experiment kinst of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    """
    
    try:
        self.__fileList[position][self.__expKinstCol] = str(int(expKinst))
    except:
        raise madrigal.admin.MadrigalError('Error in setExpKinstByPosition with args %s: %s' %  \
                                           (str(position, expKinst),
                                            traceback.format_exception(sys.exc_info()[0],
                                                                       sys.exc_info()[1],
                                                                       sys.exc_info()[2])))

def setExpNameByPosition(

self, position, expName)

setExpNameByPosition sets the experiment name of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

expName - the new experiment name to use

Returns: None.

Affects: sets the experiment name of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

def setExpNameByPosition(self, position, expName):
    """setExpNameByPosition sets the experiment name of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        expName - the new experiment name to use
    
    Returns: None.
    Affects: sets the experiment name of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    """
    # verify no illegal commas
    if expName.find(',') != -1:
        raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' %  \
                                           (str(position, expName),
                                            [traceback.format_exc()]))
    
    try:
        self.__fileList[position][self.__expNameCol] = str(expName)
    except:
        raise madrigal.admin.MadrigalError('Error in setExpNameByPosition with args %s: %s' %  \
                                           (str(position, expName),
                                            traceback.format_exception(sys.exc_info()[0],
                                                                       sys.exc_info()[1],
                                                                       sys.exc_info()[2])))

def setExpSiteIdByPosition(

self, position, expSiteId)

setExpSiteIdByPosition sets the experiment site id of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

expSiteId - the new experiment site id to use

Returns: None.

Affects: sets the experiment site id of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

def setExpSiteIdByPosition(self, position, expSiteId):
    """setExpSiteIdByPosition sets the experiment site id of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        expSiteId - the new experiment site id to use
    
    Returns: None.
    Affects: sets the experiment site id of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    """
    
    try:
        self.__fileList[position][self.__expSiteIdCol] = str(int(expSiteId))
    except:
        raise madrigal.admin.MadrigalError('Error in setExpSiteIdByPosition with args %s: %s' %  \
                                           (str(position, expSiteId),
                                            traceback.format_exception(sys.exc_info()[0],
                                                                       sys.exc_info()[1],
                                                                       sys.exc_info()[2])))

def setExpStartDateTimeByPosition(

self, startDateTime, position=0)

setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position.

Inputs:

startDateTime - a python datetime object to set the exp start date and time to.

position - which experiment row to change - defaults to 0

Returns: None.

Affects: sets exp start date and time in self.__fileList.

Exceptions: None

def setExpStartDateTimeByPosition(self, startDateTime, position = 0):
    """setExpStartDateTimeByPosition sets a new MadrigalExperiment start date and time by position.
    Inputs:
        startDateTime - a python datetime object to set the exp start date and time to.
        position - which experiment row to change - defaults to 0
    
    Returns: None.
    
    Affects: sets exp start date and time in self.__fileList.
    Exceptions: None
    """
    self.__fileList[position][self.__expStartDateCol] = '%04i%02i%02i' % (startDateTime.year,
                                                                          startDateTime.month,
                                                                          startDateTime.day)
    self.__fileList[position][self.__expStartTimeCol] = '%02i%02i%02i' % (startDateTime.hour,
                                                                          startDateTime.minute,
                                                                          startDateTime.second)

def setExpUrlByPosition(

self, position, expUrl)

setExpUrlByPosition sets the experiment url of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

expUrl - the new experiment url to use

Returns: None.

Affects: sets the experiment url of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

def setExpUrlByPosition(self, position, expUrl):
    """setExpUrlByPosition sets the experiment url of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        expUrl - the new experiment url to use
    
    Returns: None.
    Affects: sets the experiment url of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    """
    # verify no illegal commas
    if expUrl.find(',') != -1:
        raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' %  \
                                           (str(position, expUrl),
                                            [traceback.format_exc()]))
    
    try:
        self.__fileList[position][self.__expUrlCol] = str(expUrl)
    except:
        raise madrigal.admin.MadrigalError('Error in setExpUrlByPosition with args %s: %s' %  \
                                           (str(position, expUrl),
                                            [traceback.format_ex()]))

def setPIByPosition(

self, position, PI)

setPIByPosition sets the principal investigator (string) of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

PI - the new experiment principal investigator's name (string) to use

Returns: None.

Affects: sets the principal investigator of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

This method added in Madrigal 2.6

def setPIByPosition(self, position, PI):
    """setPIByPosition sets the principal investigator (string) of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        PI - the new experiment principal investigator's name (string) to use
    
    Returns: None.
    Affects: sets the principal investigator of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    
    This method added in Madrigal 2.6
    """
    # verify no illegal commas
    if PI.find(',') != -1:
        raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' %  \
                                           (str(position, PI),
                                            [traceback.format_exc()]))
    
    if len(self.__fileList[position]) == self.__expPICol:
        # add PI and blank email
        self.__fileList[position].append(str(PI))
        self.__fileList[position].append('')
    else:
        try:
            self.__fileList[position][self.__expPICol] = str(PI)
        except:
            raise madrigal.admin.MadrigalError('Error in setPIByPosition with args %s: %s' %  \
                                               (str(position, PI),
                                                [traceback.format_exc()]))

def setPIEmailByPosition(

self, position, PIEmail)

setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

PIEmail - the new experiment principal investigator's email (string) to use

Returns: None.

Affects: sets the principal investigator email of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

This method added in Madrigal 2.6

def setPIEmailByPosition(self, position, PIEmail):
    """setPIEmailByPosition sets the principal investigator email (string) of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        PIEmail - the new experiment principal investigator's email (string) to use
    
    Returns: None.
    Affects: sets the principal investigator email of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    
    This method added in Madrigal 2.6
    """
    # verify no illegal commas
    if PIEmail.find(',') != -1:
        raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' %  \
                                           (str(position, PIEmail),
                                            [traceback.format_exc()]))
    
    if len(self.__fileList[position]) == self.__expPICol:
        # add blank PI and email
        self.__fileList[position].append('')
        self.__fileList[position].append(str(PIEmail))
    else:
        try:
            self.__fileList[position][self.__expPIEmailCol] = str(PIEmail)
        except:
            raise madrigal.admin.MadrigalError('Error in setPIEmailByPosition with args %s: %s' %  \
                                               (str(position, PIEmail),
                                                [traceback.format_exc()]))

def setSecurityByPosition(

self, position, securityCode)

setSecurityByPosition sets the security code (integer) of the experiment at given position.

Inputs:

position - position of experiment in list (first position is zero).

securityCode - the new experiment security code (integer) to use

Returns: None.

Affects: sets the security code of the experiment at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

def setSecurityByPosition(self, position, securityCode):
    """setSecurityByPosition sets the security code (integer) of the experiment at given position.
    Inputs:
        position - position of experiment in list (first position is zero).
        securityCode - the new experiment security code (integer) to use
    
    Returns: None.
    Affects: sets the security code of the experiment at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    """
    
    try:
        self.__fileList[position][self.__expSecurityCol] = str(int(securityCode))
    except:
        raise madrigal.admin.MadrigalError('Error in setSecurityByPosition with args %s: %s' %  \
                                           (str(position, securityCode),
                                            traceback.format_exception(sys.exc_info()[0],
                                                                       sys.exc_info()[1],
                                                                       sys.exc_info()[2])))

def sortByDateSite(

self)

sortByDateSite will resort self.__fileList so that experiments are listed first by experiment end date, and then by site

def sortByDateSite(self):
    """sortByDateSite will resort self.__fileList so that experiments are listed first by experiment
    end date, and then by site
    """
    self.__fileList.sort(key=self.__compareDateSite__)
    # rebuild self._dict 
    self._dict = {}
    for i, items in enumerate(self.__fileList):
        self._dict[items[self.__expIdCol]] = i

def writeMetadata(

self, newFullPath=None)

writeMetadata writes a new version of the expTab.txt file.

Inputs: newFullPath: a new path to write the expTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from.

Returns: None.

Affects: Writes updated version of metadata file.

Exceptions: If unable to write file

def writeMetadata(self, newFullPath=None):
    """writeMetadata writes a new version of the expTab.txt file.
    
    Inputs: newFullPath:  a new path to write the expTab.txt file to, if
    not the same as the original metadata file opened.  Defaults to None, which overwrites
    metadata file that was read from.
    
    Returns: None.
    Affects: Writes updated version of metadata file.
    Exceptions: If unable to write file
    """
    # create string to hold file
    metaFileStr = ''
    delimiter = ','
    for lineList in self.__fileList:
        metaFileStr += delimiter.join(lineList) + '\n'
    # try to write file, if not, raise exception
    try:
        if newFullPath == None:
            if (len(os.path.dirname(self.__filename)) != 0):
                newFullPath = self.__filename
            else:
                newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename
        newFile = open(newFullPath, "w")
        newFile.write(metaFileStr)
        newFile.close()
    except:
        raise madrigal.admin.MadrigalError("Unable to write metadata file " + \
                                           str(newFullPath),
                                           traceback.format_exception(sys.exc_info()[0],
                                                                      sys.exc_info()[1],
                                                                      sys.exc_info()[2]))

class MadrigalInstrument

MadrigalInstrument is an object that provides access to Madrigal instrument info from the metadata.

This object provides access to all Madrigal instrument information in the metadata files instTab.txt and instType.Tab.

key = self.__instKinstCol (used in searches)

Usage example::

import madrigal.metadata

import madrigal.admin

try:

    test = madrigal.metadata.MadrigalInstrument()

    print test.getInstrumentName(30)

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

2. Columns expected to be ints or floats cannot be converted

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001

class MadrigalInstrument:
    """MadrigalInstrument is an object that provides access to Madrigal instrument info from the metadata.

    This object provides access to all Madrigal instrument information in the metadata files instTab.txt
    and instType.Tab.
    
    key = self.__instKinstCol (used in searches)

    Usage example::

        import madrigal.metadata
        
        import madrigal.admin

        try:
    
            test = madrigal.metadata.MadrigalInstrument()

            print test.getInstrumentName(30)

        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file
        
        2. Columns expected to be ints or floats cannot be converted

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Nov. 9, 2001

    """

    #constants
    __instMetadataFile  = "instTab.txt"

    # column positions
    __instKinstCol      =  0
    __instMnemonicCol   =  1
    __instNameCol       =  2
    __latitudeCol       =  3
    __longitudeCol      =  4
    __altitudeCol       =  5
    __contactNameCol    =  6
    __contactAddr1Col   =  7
    __contactAddr2Col   =  8
    __contactAddr3Col   =  9
    __contactCityCol    =  10
    __contactStateCol   =  11
    __contactZipCol     =  12
    __contactCountryCol =  13
    __contactPhoneCol   =  14
    __contactEmailCol   =  15
    __categoryCol       =  16

    # instType.txt file
    __inst2MetadataFile  = "instType.txt"

    # column positions
    __inst2CategoryIdCol    =  0
    __inst2CategoryDescCol  =  1
    

    def __init__(self, madDB=None, initFile=None, init2File=None):
        """__init__ initializes MadrigalInstrument by reading from instTab.txt (or initFile)
        and instType.txt file (or init2File).

        Inputs:

            madDB - Existing MadrigalDB object, by default = None.

            initFile - String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/instTab.txt.

            init2File - String representing the full path to the metadata file instType.txt. Default is
            None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        Note that the instTab.txt file was updated with the release of the madrigal python api, and this
        function will throw an error if used with the old file.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get instrument metadata file
        if (initFile == None):
            self.__filename = self.__instMetadataFile
        else:
            self.__filename = initFile
            
        genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, key=self.__instKinstCol)

        self.__fileList = genericObj.getList()
        self._dict = genericObj.getDict()

        # get instrument metadata file 2
        if (init2File == None):
            self.__filename2 = self.__inst2MetadataFile
        else:
            self.__filename2 = initFile

        self.__fileList2 = madrigal.metadata.MadrigalMetadata(self.__filename2, self.__madDB).getList()


    def getInstrumentName(self, kinst):
        """getInstrumentName returns the instrument name that matches kinst argument, or None if not found.

        Inputs: kinst integer to get instrument name.
        
        Returns: the instrument name that matches kinst argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        return(self.__fileList[position][self.__instNameCol])


    def getInstrumentMnemonic(self, kinst):
        """getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found.

        Inputs: kinst integer to get instrument mnemonic.
        
        Returns: the instrument mnemonic that matches kinst argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        return(self.__fileList[position][self.__instMnemonicCol])


    def getLatitude(self, kinst):
        """getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank.

        Inputs: kinst integer to get latitude.
        
        Returns: the latitude as a float that matches kinst argument, or None if not found or blank.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        try:
            return(float(self.__fileList[position][self.__latitudeCol]))
        except ValueError:
            return(None)


    def getLongitude(self, kinst):
        """getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank.

        Inputs: kinst integer to get longitude.
        
        Returns: the longitude as a float that matches kinst argument, or None if not found or blank.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        try:
            return(float(self.__fileList[position][self.__longitudeCol]))
        except ValueError:
            return(None)
	
	
    def getAltitude(self, kinst):
        """getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank.

        Inputs: kinst integer to get altitude.
        
        Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        try:
            return(float(self.__fileList[position][self.__altitudeCol]))
        except ValueError:
            return(None)
    
    
    def getContactName(self, kinst):
        """getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank.

        Inputs: kinst integer to get contact name.
        
        Returns: the contact name as a string that matches kinst argument, or None if not found or blank.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        
        This method added in Madrigal 2.6
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        return(self.__fileList[position][self.__contactNameCol])
    
    
    def getContactAddress1(self, kinst):
        """getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank.

        Inputs: kinst integer to get contact name.
        
        Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        
        This method added in Madrigal 3
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        return(self.__fileList[position][self.__contactAddr1Col])
    
    
    
    def getContactEmail(self, kinst):
        """getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank.

        Inputs: kinst integer to get contact email.
        
        Returns: the contact email as a string that matches kinst argument, or None if not found or blank.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        
        This method added in Madrigal 2.6
        """
        try:
            position = self._dict[str(kinst)]
        except KeyError:
            return(None)
        return(self.__fileList[position][self.__contactEmailCol])


    def getCategory(self, kinst):
        """getCategory returns the instrument category that matches kinst argument as a string, or
        None if kinst not found.

        Inputs: kinst integer to get altitude.
        
        Returns: the instrument category that matches kinst argument as a string, or
        None if kinst not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        categoryId = None
        try:
            position = self._dict[str(kinst)]
            categoryId = int(self.__fileList[position][self.__categoryCol])
        except KeyError:
            categoryId = None

        if categoryId == None:
            return(None)

        # now loop until categoryId found
        for inst2 in self.__fileList2:
            try:
                if (int(inst2[self.__inst2CategoryIdCol]) == categoryId):
                    return(inst2[self.__inst2CategoryDescCol])
            except:
                raise madrigal.admin.MadrigalError('Error in instType.txt parsing metadata row: ' + str(inst2),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        # type not found
        return(None)



    def getCategoryId(self, kinst):
        """getCategory returns the instrument category that matches kinst argument as a string, or
        None if kinst not found.

        Inputs: kinst integer to get altitude.
        
        Returns: the instrument category that matches kinst argument as a string, or
        None if kinst not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        try:
            position = self._dict[str(kinst)]
            categoryId = int(self.__fileList[position][self.__categoryCol])
        except KeyError:
            categoryId = None
        return(categoryId)

    

    def getInstrumentList(self):
        """getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values.

        Inputs: None.
        
        Returns: a list of all instrument names, mnemonics, and their kinst values.  Each item in the list
        is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)).  Example item:
        ('Millstone Hill UHF Steerable Antenna', 'mlh', 31)

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        retList = []

        for inst in self.__fileList:
            try:
                item = (inst[self.__instNameCol],
                        inst[self.__instMnemonicCol],
                        int(inst[self.__instKinstCol]))
                retList.append(item)
                
            except:
                raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        return retList


    def getOrderedInstrumentList(self):
        """getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId),
        ordered by categoryId and then kinst.

        Inputs: None.
        
        Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and
        then kinst.    Example item:
        ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1)

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        retList = []

        for inst in self.__fileList:
            try:
                kinst = int(inst[self.__instKinstCol])
                item = (inst[self.__instNameCol],
                        inst[self.__instMnemonicCol],
                        kinst,
                        self.getCategory(kinst),
                        self.getCategoryId(kinst))
                retList.append(item)
                
            except:
                raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        retList.sort(key=self.__instrumentSort) 
        return retList


    def getOrderedInstrumentListWithData(self, isTrusted, localOnly=False,
                                         localExpObj=None, globalExpObj=None,
                                         allowArchive=False, requireFiles=False,
                                         requireFilesOrPlots=False):
        """getInstrumentList returns information about which instruments have local and global data.

        Inputs:

                isTrusted - True if client is trusted, False otherwise

                localOnly - if False (the default), will return information about both local and global data.
                    If True, then global data ignored.

                localExpObj - an MadrigalExperiment object for the local data.  If None (the default),
                    will be created.

                globalExpObj - an MadrigalExperiment object for the global data.  If None (the default),
                    will be created if not localOnly.

                allowArchive - if True, allow experiments marked as security==2(public arcive), and if
                    isTrusted, allow security==3(private archive)
                    
                requireFiles - if True, only include experiments with Madrigal data files.  If False
                    (default), not not require that.
                    
                requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots.  
                    If False (default), not not require that.
        
        
        Returns: an ordered tuple of two items:
            1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear,
               globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst.  If
               localOnly, globalStartYear and globalEndYear = 0.  If not local only, localStartYear
               and localEndYear will be zero if no local data for that instrument.
            2. categoryDict - key = categoryId, value = category Description

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        if (not localOnly) and (globalExpObj==None):
            globalExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                                os.path.join(self.__madDB.getMadroot(),
                                                                             'metadata/expTabAll.txt'))
            
        if localExpObj == None:
            localExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)

        orderInstList = []
        categoryDict = {}
        
        # next task - create local dict and global dict of key=kinst, value = (startYear, endYear) tuple
        localInstDict = {}
        globalInstDict = {}
        
        # Skip test experiments
        testExp = ('1998/mlh/20jan98', '1997/aro/06jan97', '1997/lyr/08apr97',
                   '1997/son/06jan97', '1995/jro/01feb95', '1998/jro/27apr98')
        
        for i in range(localExpObj.getExpCount()):
            thisUrl = localExpObj.getExpUrlByPosition(i)
            if thisUrl[-16:] in testExp:
                continue
            thisSecurity = localExpObj.getSecurityByPosition(i)
            if thisSecurity == -1:
                # blocked to all
                continue
            if thisSecurity == 1 and not isTrusted:
                # no access
                continue
            if thisSecurity == 2 and not allowArchive:
                # no access
                continue
            if thisSecurity == 3 and ((not isTrusted) or (not allowArchive)):
                # no access
                continue
            
            kinst = localExpObj.getKinstByPosition(i)
            startYear = localExpObj.getExpStartDateTimeByPosition(i)[0]
            endYear = localExpObj.getExpEndDateTimeByPosition(i)[0]
            if kinst in localInstDict:
                oldStartYear, oldEndYear = localInstDict[kinst]
                if startYear >= oldStartYear and endYear <= oldEndYear:
                    # no new info
                    continue
            
            expDir = localExpObj.getExpDirByPosition(i)
                
            if requireFilesOrPlots:
                links = localExpObj.getExpLinksByPosition(i)
                if len(links) == 0:
                    try:
                        size = os.path.getsize(os.path.join(expDir, 'fileTab.txt'))
                        if size < 3:
                            continue
                    except:
                        continue
            
            if requireFiles:
                try:
                    size = os.path.getsize(os.path.join(expDir, 'fileTab.txt'))
                    if size < 3:
                        continue
                except:
                    continue
            
            if kinst in localInstDict:
                localInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear))
            else:
                localInstDict[kinst] = (startYear, endYear)


        if not localOnly:
            for i in range(globalExpObj.getExpCount()):
                kinst = globalExpObj.getKinstByPosition(i)
                startYear = globalExpObj.getExpStartDateTimeByPosition(i)[0]
                endYear = globalExpObj.getExpEndDateTimeByPosition(i)[0]
                thisUrl = globalExpObj.getExpUrlByPosition(i)
                thisSecurity = globalExpObj.getSecurityByPosition(i)
                thisSite = globalExpObj.getExpSiteIdByPosition(i)
                if thisUrl[-16:] in testExp:
                    continue
                if kinst not in globalInstDict:
                    globalInstDict[kinst] = (startYear, endYear)
                    continue
                oldStartYear, oldEndYear = globalInstDict[kinst]
                if startYear >= oldStartYear and endYear <= oldEndYear:
                    # no new info
                    continue
                if thisSecurity != 0 and self.__madDB.getSiteID() != thisSite:
                    # no search remote sites for non-public experiments allowed
                    continue
                if requireFiles:
                    if globalExpObj.getExpIdByPosition(i) not in expIdList:
                        continue
                globalInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear))
        
        # populate orderInstList and categoryDict
        instList = self.getOrderedInstrumentList() 
        
        for inst in instList:
            if not localOnly:
                if inst[2] in globalInstDict:
                    if inst[4] not in categoryDict:
                        categoryDict[inst[4]] = inst[3]
                    globalStartYear, globalEndYear = globalInstDict[inst[2]]
                    try:
                        localStartYear, localEndYear = localInstDict[inst[2]]
                    except KeyError:
                        localStartYear, localEndYear = (0, 0)
                    orderInstList.append((inst[0], localStartYear, localEndYear,
                                               globalStartYear, globalEndYear, inst[2], inst[4]))
            else:
                if inst[2] in localInstDict:
                    if inst[4] not in categoryDict:
                        categoryDict[inst[4]] = inst[3]
                    globalStartYear = 0
                    globalEndYear = 0
                    localStartYear, localEndYear = localInstDict[inst[2]]
                    orderInstList.append((inst[0], localStartYear, localEndYear,
                                               globalStartYear, globalEndYear, inst[2], inst[4]))

        return ((orderInstList, categoryDict))


    def __instrumentSort(self, thisInst):
        """instrumentSort is a private method used to sort tuples of instrument data
        """
        return((thisInst[4], thisInst[2]))

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None, init2File=None)

init initializes MadrigalInstrument by reading from instTab.txt (or initFile) and instType.txt file (or init2File).

Inputs:

madDB - Existing MadrigalDB object, by default = None.

initFile - String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/instTab.txt.

init2File - String representing the full path to the metadata file instType.txt. Default is
None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instTab.txt file was updated with the release of the madrigal python api, and this function will throw an error if used with the old file.

def __init__(self, madDB=None, initFile=None, init2File=None):
    """__init__ initializes MadrigalInstrument by reading from instTab.txt (or initFile)
    and instType.txt file (or init2File).
    Inputs:
        madDB - Existing MadrigalDB object, by default = None.
        initFile - String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/instTab.txt.
        init2File - String representing the full path to the metadata file instType.txt. Default is
        None, in which case file read is MadrigalDB.getMetadataDir()/instType.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    Note that the instTab.txt file was updated with the release of the madrigal python api, and this
    function will throw an error if used with the old file.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get instrument metadata file
    if (initFile == None):
        self.__filename = self.__instMetadataFile
    else:
        self.__filename = initFile
        
    genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, key=self.__instKinstCol)
    self.__fileList = genericObj.getList()
    self._dict = genericObj.getDict()
    # get instrument metadata file 2
    if (init2File == None):
        self.__filename2 = self.__inst2MetadataFile
    else:
        self.__filename2 = initFile
    self.__fileList2 = madrigal.metadata.MadrigalMetadata(self.__filename2, self.__madDB).getList()

def getAltitude(

self, kinst)

getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank.

Inputs: kinst integer to get altitude.

Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getAltitude(self, kinst):
    """getAltitude returns the altitude as a float that matches kinst argument, or None if not found or blank.
    Inputs: kinst integer to get altitude.
    
    Returns: the altitude in km above sea level as a float that matches kinst argument, or None if not found or blank.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    try:
        return(float(self.__fileList[position][self.__altitudeCol]))
    except ValueError:
        return(None)

def getCategory(

self, kinst)

getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found.

Inputs: kinst integer to get altitude.

Returns: the instrument category that matches kinst argument as a string, or None if kinst not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getCategory(self, kinst):
    """getCategory returns the instrument category that matches kinst argument as a string, or
    None if kinst not found.
    Inputs: kinst integer to get altitude.
    
    Returns: the instrument category that matches kinst argument as a string, or
    None if kinst not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    categoryId = None
    try:
        position = self._dict[str(kinst)]
        categoryId = int(self.__fileList[position][self.__categoryCol])
    except KeyError:
        categoryId = None
    if categoryId == None:
        return(None)
    # now loop until categoryId found
    for inst2 in self.__fileList2:
        try:
            if (int(inst2[self.__inst2CategoryIdCol]) == categoryId):
                return(inst2[self.__inst2CategoryDescCol])
        except:
            raise madrigal.admin.MadrigalError('Error in instType.txt parsing metadata row: ' + str(inst2),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # type not found
    return(None)

def getCategoryId(

self, kinst)

getCategory returns the instrument category that matches kinst argument as a string, or None if kinst not found.

Inputs: kinst integer to get altitude.

Returns: the instrument category that matches kinst argument as a string, or None if kinst not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getCategoryId(self, kinst):
    """getCategory returns the instrument category that matches kinst argument as a string, or
    None if kinst not found.
    Inputs: kinst integer to get altitude.
    
    Returns: the instrument category that matches kinst argument as a string, or
    None if kinst not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    try:
        position = self._dict[str(kinst)]
        categoryId = int(self.__fileList[position][self.__categoryCol])
    except KeyError:
        categoryId = None
    return(categoryId)

def getContactAddress1(

self, kinst)

getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank.

Inputs: kinst integer to get contact name.

Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

This method added in Madrigal 3

def getContactAddress1(self, kinst):
    """getContactAddress1 returns the contact address 1 field (institution) as a string that matches kinst argument, or None if not found or blank.
    Inputs: kinst integer to get contact name.
    
    Returns: the contact address 1 as a string that matches kinst argument, or None if not found or blank.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    
    This method added in Madrigal 3
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    return(self.__fileList[position][self.__contactAddr1Col])

def getContactEmail(

self, kinst)

getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank.

Inputs: kinst integer to get contact email.

Returns: the contact email as a string that matches kinst argument, or None if not found or blank.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

This method added in Madrigal 2.6

def getContactEmail(self, kinst):
    """getContactEmail returns the contact email as a string that matches kinst argument, or None if not found or blank.
    Inputs: kinst integer to get contact email.
    
    Returns: the contact email as a string that matches kinst argument, or None if not found or blank.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    
    This method added in Madrigal 2.6
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    return(self.__fileList[position][self.__contactEmailCol])

def getContactName(

self, kinst)

getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank.

Inputs: kinst integer to get contact name.

Returns: the contact name as a string that matches kinst argument, or None if not found or blank.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

This method added in Madrigal 2.6

def getContactName(self, kinst):
    """getContactName returns the contact name as a string that matches kinst argument, or None if not found or blank.
    Inputs: kinst integer to get contact name.
    
    Returns: the contact name as a string that matches kinst argument, or None if not found or blank.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    
    This method added in Madrigal 2.6
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    return(self.__fileList[position][self.__contactNameCol])

def getInstrumentList(

self)

getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values.

Inputs: None.

Returns: a list of all instrument names, mnemonics, and their kinst values. Each item in the list is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)). Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31)

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getInstrumentList(self):
    """getInstrumentList returns a list of all instrument names, mnemonics, and their kinst values.
    Inputs: None.
    
    Returns: a list of all instrument names, mnemonics, and their kinst values.  Each item in the list
    is a tuple of the form (Instrument Name (string), mnemonic (string), kinst (integer)).  Example item:
    ('Millstone Hill UHF Steerable Antenna', 'mlh', 31)
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    retList = []
    for inst in self.__fileList:
        try:
            item = (inst[self.__instNameCol],
                    inst[self.__instMnemonicCol],
                    int(inst[self.__instKinstCol]))
            retList.append(item)
            
        except:
            raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    return retList

def getInstrumentMnemonic(

self, kinst)

getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found.

Inputs: kinst integer to get instrument mnemonic.

Returns: the instrument mnemonic that matches kinst argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getInstrumentMnemonic(self, kinst):
    """getInstrumentMnemonic returns the 3 char instrument mnemonic that matches kinst argument, or None if not found.
    Inputs: kinst integer to get instrument mnemonic.
    
    Returns: the instrument mnemonic that matches kinst argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    return(self.__fileList[position][self.__instMnemonicCol])

def getInstrumentName(

self, kinst)

getInstrumentName returns the instrument name that matches kinst argument, or None if not found.

Inputs: kinst integer to get instrument name.

Returns: the instrument name that matches kinst argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getInstrumentName(self, kinst):
    """getInstrumentName returns the instrument name that matches kinst argument, or None if not found.
    Inputs: kinst integer to get instrument name.
    
    Returns: the instrument name that matches kinst argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    return(self.__fileList[position][self.__instNameCol])

def getLatitude(

self, kinst)

getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank.

Inputs: kinst integer to get latitude.

Returns: the latitude as a float that matches kinst argument, or None if not found or blank.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getLatitude(self, kinst):
    """getLatitude returns the latitude as a float that matches kinst argument, or None if not found or blank.
    Inputs: kinst integer to get latitude.
    
    Returns: the latitude as a float that matches kinst argument, or None if not found or blank.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    try:
        return(float(self.__fileList[position][self.__latitudeCol]))
    except ValueError:
        return(None)

def getLongitude(

self, kinst)

getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank.

Inputs: kinst integer to get longitude.

Returns: the longitude as a float that matches kinst argument, or None if not found or blank.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getLongitude(self, kinst):
    """getLongitude returns the longitude as a float that matches kinst argument, or None if not found or blank.
    Inputs: kinst integer to get longitude.
    
    Returns: the longitude as a float that matches kinst argument, or None if not found or blank.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    try:
        position = self._dict[str(kinst)]
    except KeyError:
        return(None)
    try:
        return(float(self.__fileList[position][self.__longitudeCol]))
    except ValueError:
        return(None)

def getOrderedInstrumentList(

self)

getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId), ordered by categoryId and then kinst.

Inputs: None.

Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and then kinst. Example item: ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1)

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getOrderedInstrumentList(self):
    """getOrderedInstrumentList returns a list of all (instrument names, mnemonics, kinst, categories, categoryId),
    ordered by categoryId and then kinst.
    Inputs: None.
    
    Returns: a list of tuples of (instrument name, mnemonic, kinst, category, categoryId) ordered by categoryId and
    then kinst.    Example item:
    ('Millstone Hill UHF Steerable Antenna', 'mlh', 31, 'Incoherent Scatter Radars', 1)
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    retList = []
    for inst in self.__fileList:
        try:
            kinst = int(inst[self.__instKinstCol])
            item = (inst[self.__instNameCol],
                    inst[self.__instMnemonicCol],
                    kinst,
                    self.getCategory(kinst),
                    self.getCategoryId(kinst))
            retList.append(item)
            
        except:
            raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    retList.sort(key=self.__instrumentSort) 
    return retList

def getOrderedInstrumentListWithData(

self, isTrusted, localOnly=False, localExpObj=None, globalExpObj=None, allowArchive=False, requireFiles=False, requireFilesOrPlots=False)

getInstrumentList returns information about which instruments have local and global data.

Inputs:

    isTrusted - True if client is trusted, False otherwise

    localOnly - if False (the default), will return information about both local and global data.
        If True, then global data ignored.

    localExpObj - an MadrigalExperiment object for the local data.  If None (the default),
        will be created.

    globalExpObj - an MadrigalExperiment object for the global data.  If None (the default),
        will be created if not localOnly.

    allowArchive - if True, allow experiments marked as security==2(public arcive), and if
        isTrusted, allow security==3(private archive)

    requireFiles - if True, only include experiments with Madrigal data files.  If False
        (default), not not require that.

    requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots.  
        If False (default), not not require that.

Returns: an ordered tuple of two items: 1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear, globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst. If localOnly, globalStartYear and globalEndYear = 0. If not local only, localStartYear and localEndYear will be zero if no local data for that instrument. 2. categoryDict - key = categoryId, value = category Description

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getOrderedInstrumentListWithData(self, isTrusted, localOnly=False,
                                     localExpObj=None, globalExpObj=None,
                                     allowArchive=False, requireFiles=False,
                                     requireFilesOrPlots=False):
    """getInstrumentList returns information about which instruments have local and global data.
    Inputs:
            isTrusted - True if client is trusted, False otherwise
            localOnly - if False (the default), will return information about both local and global data.
                If True, then global data ignored.
            localExpObj - an MadrigalExperiment object for the local data.  If None (the default),
                will be created.
            globalExpObj - an MadrigalExperiment object for the global data.  If None (the default),
                will be created if not localOnly.
            allowArchive - if True, allow experiments marked as security==2(public arcive), and if
                isTrusted, allow security==3(private archive)
                
            requireFiles - if True, only include experiments with Madrigal data files.  If False
                (default), not not require that.
                
            requireFilesOrPlots - if True, only include experiments with Madrigal data files or plots.  
                If False (default), not not require that.
    
    
    Returns: an ordered tuple of two items:
        1. a list of tuples of (instName, localStartYear, localEndYear, globalStartYear,
           globalEndYear, kinst, categoryId), ordered by by categoryId, then kinst.  If
           localOnly, globalStartYear and globalEndYear = 0.  If not local only, localStartYear
           and localEndYear will be zero if no local data for that instrument.
        2. categoryDict - key = categoryId, value = category Description
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    if (not localOnly) and (globalExpObj==None):
        globalExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB,
                                                            os.path.join(self.__madDB.getMadroot(),
                                                                         'metadata/expTabAll.txt'))
        
    if localExpObj == None:
        localExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)
    orderInstList = []
    categoryDict = {}
    
    # next task - create local dict and global dict of key=kinst, value = (startYear, endYear) tuple
    localInstDict = {}
    globalInstDict = {}
    
    # Skip test experiments
    testExp = ('1998/mlh/20jan98', '1997/aro/06jan97', '1997/lyr/08apr97',
               '1997/son/06jan97', '1995/jro/01feb95', '1998/jro/27apr98')
    
    for i in range(localExpObj.getExpCount()):
        thisUrl = localExpObj.getExpUrlByPosition(i)
        if thisUrl[-16:] in testExp:
            continue
        thisSecurity = localExpObj.getSecurityByPosition(i)
        if thisSecurity == -1:
            # blocked to all
            continue
        if thisSecurity == 1 and not isTrusted:
            # no access
            continue
        if thisSecurity == 2 and not allowArchive:
            # no access
            continue
        if thisSecurity == 3 and ((not isTrusted) or (not allowArchive)):
            # no access
            continue
        
        kinst = localExpObj.getKinstByPosition(i)
        startYear = localExpObj.getExpStartDateTimeByPosition(i)[0]
        endYear = localExpObj.getExpEndDateTimeByPosition(i)[0]
        if kinst in localInstDict:
            oldStartYear, oldEndYear = localInstDict[kinst]
            if startYear >= oldStartYear and endYear <= oldEndYear:
                # no new info
                continue
        
        expDir = localExpObj.getExpDirByPosition(i)
            
        if requireFilesOrPlots:
            links = localExpObj.getExpLinksByPosition(i)
            if len(links) == 0:
                try:
                    size = os.path.getsize(os.path.join(expDir, 'fileTab.txt'))
                    if size < 3:
                        continue
                except:
                    continue
        
        if requireFiles:
            try:
                size = os.path.getsize(os.path.join(expDir, 'fileTab.txt'))
                if size < 3:
                    continue
            except:
                continue
        
        if kinst in localInstDict:
            localInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear))
        else:
            localInstDict[kinst] = (startYear, endYear)
    if not localOnly:
        for i in range(globalExpObj.getExpCount()):
            kinst = globalExpObj.getKinstByPosition(i)
            startYear = globalExpObj.getExpStartDateTimeByPosition(i)[0]
            endYear = globalExpObj.getExpEndDateTimeByPosition(i)[0]
            thisUrl = globalExpObj.getExpUrlByPosition(i)
            thisSecurity = globalExpObj.getSecurityByPosition(i)
            thisSite = globalExpObj.getExpSiteIdByPosition(i)
            if thisUrl[-16:] in testExp:
                continue
            if kinst not in globalInstDict:
                globalInstDict[kinst] = (startYear, endYear)
                continue
            oldStartYear, oldEndYear = globalInstDict[kinst]
            if startYear >= oldStartYear and endYear <= oldEndYear:
                # no new info
                continue
            if thisSecurity != 0 and self.__madDB.getSiteID() != thisSite:
                # no search remote sites for non-public experiments allowed
                continue
            if requireFiles:
                if globalExpObj.getExpIdByPosition(i) not in expIdList:
                    continue
            globalInstDict[kinst] = (min(startYear, oldStartYear), max(endYear, oldEndYear))
    
    # populate orderInstList and categoryDict
    instList = self.getOrderedInstrumentList() 
    
    for inst in instList:
        if not localOnly:
            if inst[2] in globalInstDict:
                if inst[4] not in categoryDict:
                    categoryDict[inst[4]] = inst[3]
                globalStartYear, globalEndYear = globalInstDict[inst[2]]
                try:
                    localStartYear, localEndYear = localInstDict[inst[2]]
                except KeyError:
                    localStartYear, localEndYear = (0, 0)
                orderInstList.append((inst[0], localStartYear, localEndYear,
                                           globalStartYear, globalEndYear, inst[2], inst[4]))
        else:
            if inst[2] in localInstDict:
                if inst[4] not in categoryDict:
                    categoryDict[inst[4]] = inst[3]
                globalStartYear = 0
                globalEndYear = 0
                localStartYear, localEndYear = localInstDict[inst[2]]
                orderInstList.append((inst[0], localStartYear, localEndYear,
                                           globalStartYear, globalEndYear, inst[2], inst[4]))
    return ((orderInstList, categoryDict))

class MadrigalInstrumentData

MadrigalInstrumentData is an object that provides access to Madrigal instrument data info from the metadata.

This object provides access to all Madrigal kind of data information in the metadata files instData.txt and instDataPriv.txt. Those files summarize years data is available by instrument.

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

2. Columns expected to be ints or floats cannot be converted

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Feb. 2, 2015

class MadrigalInstrumentData:
    """MadrigalInstrumentData is an object that provides access to Madrigal instrument data info from the metadata.

    This object provides access to all Madrigal kind of data information in the metadata files instData.txt
    and instDataPriv.txt.  Those files summarize years data is available by instrument.

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file
        
        2. Columns expected to be ints or floats cannot be converted

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Feb. 2, 2015

    """

    #constants
    _instDataMetadataFile  = "instData.txt"

    # column positions
    _siteIDCol   =  0
    _kinstCol    =  1
    _yearsCol    = 2

    

    def __init__(self, madDB=None, priv=False, initFile=None, madInstObj=None):
        """__init__ initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt).

        Inputs: madDB - Existing MadrigalDB object, by default = None.

                priv - if True, use instDataPriv.txt instead of instData.txt.  If False (the default),
                    use instData.txt.
                    
                initFile - String representing the full path to the metadata file. Default is None, in
                    which case file read depends on priv argument. priv arg ingored if this not None
                    
                madInstObj - m madrigal.metadata.MadrigalInstrument object.  If None, one is created.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB
            
        if priv:
            self._instDataMetadataFile  = "instDataPriv.txt"

        # get instData metadata file
        if (initFile == None):
            self.__filename = self._instDataMetadataFile
        else:
            self.__filename = initFile
            
        if madInstObj is not None:
            self._madInstObj = madInstObj
        else:
            self._madInstObj = madrigal.metadata.MadrigalInstrument(self.__madDB)

        self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()


    def getCategories(self, local=False):
        """getCategories returns the a list of (category id, category desc) tuples.

        Inputs: 
        
            local - if False, return all categories for which there is data anywhere in Madrigal.
                If True, only return local categories.
        
        Returns:  a list of (category id, category desc) tuples

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        if local:
            siteID = self.__madDB.getSiteID()
            
        localDict = {} # key is category id, value is category desc str.  Will be converted to list before returned
        
        for i, data in enumerate(self.__fileList):
            # find matching  code 
            try:
                if local:
                    if int(data[self._siteIDCol]) != siteID:
                        continue
                kinst = int(data[self._kinstCol])
                categoryID = self._madInstObj.getCategoryId(kinst)
                if categoryID not in list(localDict.keys()):
                    localDict[categoryID] = self._madInstObj.getCategory(kinst)
                    
            except:
                raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        categoryIDKeys = list(localDict.keys())
        categoryIDKeys.sort()
        retList = [(catID, localDict[catID]) for catID in categoryIDKeys]
        return(retList)
        


    def getInstruments(self, categoryID=0, local=False):
        """getInstruments returns the a list of (kinst, instrument desc, siteID) tuples.

        Inputs: 
        
            categoryID - category id to return instruments with data for. If 0, return all
            local - if False, return all instruments with that category for which there is data 
                anywhere in Madrigal. If True, only return local instruments, in which case siteID
                is always the local siteID.
        
        Returns:  a list of (kinst, instrument desc, siteID) tuples

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        if local:
            siteID = self.__madDB.getSiteID()
            
        localDict = {} # key is kinst, value is tuple if (instrument desc str, siteID).  
                       #  Will be converted to list before returned
        
        for i, data in enumerate(self.__fileList):
            # find matching  code 
            try:
                kinst = int(data[self._kinstCol])
                thisSiteID = int(data[self._siteIDCol])
                thisCategoryID = self._madInstObj.getCategoryId(kinst)
                if thisCategoryID != categoryID and categoryID != 0:
                    continue
                if local:
                    if thisSiteID != siteID:
                        continue

                if kinst not in list(localDict.keys()):
                    localDict[kinst] = (self._madInstObj.getInstrumentName(kinst), thisSiteID)
                else:
                    raise ValueError('Duplicate kinst %i found in instData' % (kinst))
                    
            except:
                raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        kinstKeys = list(localDict.keys())
        kinstKeys.sort()
        retList = [(kinst, localDict[kinst][0], localDict[kinst][1]) for kinst in kinstKeys]
        return(retList)
    
    
    def getInstrumentYears(self, kinst):
        """getInstrumentYears returns the a list of years (int) for instrument.

        Inputs: 
        
            kinst - the instrument id
        
        Returns:  an ordered list of years as integers.  If none found, raises error

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        retList = []
        for i, data in enumerate(self.__fileList):
            # find matching  code 
            try:
                thisKinst = int(data[self._kinstCol])
                if thisKinst != kinst:
                    continue
                yearsStr = data[self._yearsCol]

                retList = [int(year) for year in yearsStr.split()]
                retList.sort()
                    
            except:
                raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
                
        if len(retList) == 0:
            raise madrigal.admin.MadrigalError('No data found for kinst %i' % (kinst), '')
        
        return(retList)

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, priv=False, initFile=None, madInstObj=None)

init initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt).

Inputs: madDB - Existing MadrigalDB object, by default = None.

    priv - if True, use instDataPriv.txt instead of instData.txt.  If False (the default),
        use instData.txt.

    initFile - String representing the full path to the metadata file. Default is None, in
        which case file read depends on priv argument. priv arg ingored if this not None

    madInstObj - m madrigal.metadata.MadrigalInstrument object.  If None, one is created.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.

def __init__(self, madDB=None, priv=False, initFile=None, madInstObj=None):
    """__init__ initializes MadrigalInstrumentData by reading from instData.txt (or instDataPriv.txt).
    Inputs: madDB - Existing MadrigalDB object, by default = None.
            priv - if True, use instDataPriv.txt instead of instData.txt.  If False (the default),
                use instData.txt.
                
            initFile - String representing the full path to the metadata file. Default is None, in
                which case file read depends on priv argument. priv arg ingored if this not None
                
            madInstObj - m madrigal.metadata.MadrigalInstrument object.  If None, one is created.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
        
    if priv:
        self._instDataMetadataFile  = "instDataPriv.txt"
    # get instData metadata file
    if (initFile == None):
        self.__filename = self._instDataMetadataFile
    else:
        self.__filename = initFile
        
    if madInstObj is not None:
        self._madInstObj = madInstObj
    else:
        self._madInstObj = madrigal.metadata.MadrigalInstrument(self.__madDB)
    self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()

def getCategories(

self, local=False)

getCategories returns the a list of (category id, category desc) tuples.

Inputs:

local - if False, return all categories for which there is data anywhere in Madrigal.
    If True, only return local categories.

Returns: a list of (category id, category desc) tuples

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getCategories(self, local=False):
    """getCategories returns the a list of (category id, category desc) tuples.
    Inputs: 
    
        local - if False, return all categories for which there is data anywhere in Madrigal.
            If True, only return local categories.
    
    Returns:  a list of (category id, category desc) tuples
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    if local:
        siteID = self.__madDB.getSiteID()
        
    localDict = {} # key is category id, value is category desc str.  Will be converted to list before returned
    
    for i, data in enumerate(self.__fileList):
        # find matching  code 
        try:
            if local:
                if int(data[self._siteIDCol]) != siteID:
                    continue
            kinst = int(data[self._kinstCol])
            categoryID = self._madInstObj.getCategoryId(kinst)
            if categoryID not in list(localDict.keys()):
                localDict[categoryID] = self._madInstObj.getCategory(kinst)
                
        except:
            raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    categoryIDKeys = list(localDict.keys())
    categoryIDKeys.sort()
    retList = [(catID, localDict[catID]) for catID in categoryIDKeys]
    return(retList)

def getInstrumentYears(

self, kinst)

getInstrumentYears returns the a list of years (int) for instrument.

Inputs:

kinst - the instrument id

Returns: an ordered list of years as integers. If none found, raises error

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getInstrumentYears(self, kinst):
    """getInstrumentYears returns the a list of years (int) for instrument.
    Inputs: 
    
        kinst - the instrument id
    
    Returns:  an ordered list of years as integers.  If none found, raises error
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    retList = []
    for i, data in enumerate(self.__fileList):
        # find matching  code 
        try:
            thisKinst = int(data[self._kinstCol])
            if thisKinst != kinst:
                continue
            yearsStr = data[self._yearsCol]
            retList = [int(year) for year in yearsStr.split()]
            retList.sort()
                
        except:
            raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
            
    if len(retList) == 0:
        raise madrigal.admin.MadrigalError('No data found for kinst %i' % (kinst), '')
    
    return(retList)

def getInstruments(

self, categoryID=0, local=False)

getInstruments returns the a list of (kinst, instrument desc, siteID) tuples.

Inputs:

categoryID - category id to return instruments with data for. If 0, return all
local - if False, return all instruments with that category for which there is data 
    anywhere in Madrigal. If True, only return local instruments, in which case siteID
    is always the local siteID.

Returns: a list of (kinst, instrument desc, siteID) tuples

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getInstruments(self, categoryID=0, local=False):
    """getInstruments returns the a list of (kinst, instrument desc, siteID) tuples.
    Inputs: 
    
        categoryID - category id to return instruments with data for. If 0, return all
        local - if False, return all instruments with that category for which there is data 
            anywhere in Madrigal. If True, only return local instruments, in which case siteID
            is always the local siteID.
    
    Returns:  a list of (kinst, instrument desc, siteID) tuples
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    if local:
        siteID = self.__madDB.getSiteID()
        
    localDict = {} # key is kinst, value is tuple if (instrument desc str, siteID).  
                   #  Will be converted to list before returned
    
    for i, data in enumerate(self.__fileList):
        # find matching  code 
        try:
            kinst = int(data[self._kinstCol])
            thisSiteID = int(data[self._siteIDCol])
            thisCategoryID = self._madInstObj.getCategoryId(kinst)
            if thisCategoryID != categoryID and categoryID != 0:
                continue
            if local:
                if thisSiteID != siteID:
                    continue
            if kinst not in list(localDict.keys()):
                localDict[kinst] = (self._madInstObj.getInstrumentName(kinst), thisSiteID)
            else:
                raise ValueError('Duplicate kinst %i found in instData' % (kinst))
                
        except:
            raise madrigal.admin.MadrigalError('Error in instData.txt parsing metadata row %i: ' % (i) + str(category),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    kinstKeys = list(localDict.keys())
    kinstKeys.sort()
    retList = [(kinst, localDict[kinst][0], localDict[kinst][1]) for kinst in kinstKeys]
    return(retList)

class MadrigalInstrumentKindats

MadrigalInstrumentKindats is an object that provides access to the metadata file that summarizes the kindat codes associated with each instrument.

This object provides access to all Madrigal instrument kindat information in the metadata file instKindatTab.txt. The metadata file instKindatTab.txt lists, for any given instrument and year, all the kindat codes found in all the data files in the local database associated with that instrument.

This class also contains a method to rebuild the table instKindatTab.txt by examining all the metadata in the database. This is presumably a somewhat slow process and should be done in the background.

Usage example::

import madrigal.metadata

import madrigal.admin

try:

    test = madrigal.metadata.MadrigalInstrumentKindats()

    print test.getKindatListForInstruments([20,30])

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Aug. 15, 2002

class MadrigalInstrumentKindats:
    """MadrigalInstrumentKindats is an object that provides access to the metadata file that summarizes the kindat codes associated with each instrument.

    This object provides access to all Madrigal instrument kindat information in the metadata file instKindatTab.txt.
    The metadata file instKindatTab.txt lists, for any given instrument and year, all the kindat codes found in all the
    data files in the local database associated with that instrument.

    This class also contains a method to rebuild the table instKindatTab.txt by examining all the metadata in the database.
    This is presumably a somewhat slow process and should be done in the background.

    Usage example::

        import madrigal.metadata
        
        import madrigal.admin

        try:
    
            test = madrigal.metadata.MadrigalInstrumentKindats()

            print test.getKindatListForInstruments([20,30])

        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file


    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Aug. 15, 2002

    """

    #constants
    __instKindatMetadataFile  = "instKindatTab.txt"

    # column positions
    __instKindatKinstCol  =  0
    __instKindatListCol   =  1
    __instKindatYearCol   =  2

    def __init__(self, madDB=None, initFile=None):
        """__init__ initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile).

        Inputs: Existing MadrigalDB object, by default = None.

            String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this
        function will throw an error if file not there.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get instrument kindat metadata file
        if (initFile == None):
            self.__filename = self.__instKindatMetadataFile
        else:
            self.__filename = initFile

        self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()


    def getKindatListForInstruments(self, kinstList):
        """getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list.
        A kindat code is returned if it appears in any year

        Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer.
        
        Returns: a list of kindat codes as integers associated with the given instrument list.

        Affects: None

        Exceptions: if error in metadata file
        """
        retKindatList = []
        
        # if kinstList is just a single integer, convert it into a list
        if type(kinstList) == int:
            kinstList = [kinstList]

        # loop through instKindatTab.txt
        for inst in self.__fileList:
            # find matching kinst
            try:
                if (int(inst[self.__instKindatKinstCol]) in kinstList):
                    if len(inst[self.__instKindatListCol]) == 0:
                        continue
                    tempList = inst[self.__instKindatListCol].split()
                    for parm in tempList:
                        if not int(parm) in retKindatList:
                           retKindatList.append(int(parm))
                
            except:
                raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        return(retKindatList)
    
    
    
    def getKindatListForInstrumentYear(self, kinst, year):
        """getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year.

        Inputs: kinst, year
        
        Returns: a list of kindat codes as integers associated with the given instrument  and year,
            or None if not found.

        Affects: None

        Exceptions: if error in metadata file
        """
        retKindatList = []
        

        # loop through instKindatTab.txt
        for inst in self.__fileList:
            # find matching kinst
            try:
                if int(inst[self.__instKindatKinstCol]) != kinst:
                    continue
                if int(inst[self.__instKindatYearCol]) != year:
                    continue
                return([int(kindat) for kindat in inst[self.__instKindatListCol].split()])
                
            except:
                raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
    

    def rebuildInstKindatTable(self):
        """rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file.

        The table instKindatTab.txt is a listing of every kindat found for a given
        instrument for any given year.  Since these files are constantly updated, this table needs to be updated on a regular
        basis. Data from other Madrigal sites is also imported into this table.

        How it works: For each instrument in instTab.txt, this method first creates a list
        of all the experiments that used that instrument.  It then loops
        through the file table.  For each file listing, if the file is the
        default one, and its experiment id is in the list of experiments just
        created, the kindat is added to the list if its not already there.
        Data from all other Madrigal sites is then added via getMetadata.  It then writes
        the metadata file in the form: 10, 1001 1002 1003, where the first column in the
        instrument code and the second column is a space delimited list of kindat codes.

        Inputs: None.
        
        Returns: None.

        Affects: Writes file instKindatTab.txt in metadata directory

        Exceptions: If unable to write instKindatTab.txt file.
        """
        
        # get full file name
        filename = self.__madDB.getMetadataDir() + '/' + self.__filename
        
        # throw an exception if instKindatTab.txt file not writable
        if not os.access(filename, os.W_OK):
            raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None)


        # get experiment metadata
        madExp = MadrigalExperiment(self.__madDB)


        kindatDict = {} # key = year, value = list of kindats

        for i in range(madExp.getExpCount()):
            kinst = madExp.getKinstByPosition(i)
            if kinst not in list(kindatDict.keys()):
                kindatDict[kinst] = {}
            expDir = madExp.getExpDirByPosition(i)
            # get year range
            timeTuple = madExp.getExpStartDateTimeByPosition(i)
            syear = timeTuple[0]
            timeTuple = madExp.getExpEndDateTimeByPosition(i)
            eyear = timeTuple[0]
            for year in range(syear, eyear+1):
                if year not in list(kindatDict[kinst].keys()):
                    kindatDict[kinst][year] = []
                try:
                    madFileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB,
                                                                    os.path.join(expDir, 'fileTab.txt'))
                except:
                    continue
            
                for j in range(madFileObj.getFileCount()):
                    kindat = madFileObj.getKindatByPosition(j)
                    if kindat not in kindatDict[kinst][year]:
                        kindatDict[kinst][year].append(kindat)
                    
        # create newFileStr from instCodeDict
        newFileStr = ''
        kinstList = list(kindatDict.keys())
        kinstList.sort()
        delimiter = ' '

        for kinst in kinstList:
            yearList = list(kindatDict[kinst].keys())
            yearList.sort()
            for year in yearList:
                kindatDict[kinst][year].sort()
                kindatStrList = [str(kindat) for kindat in kindatDict[kinst][year]]
                kindatStr = delimiter.join(kindatStrList)
                newFileStr += '%i,%s,%i\n' % (kinst, kindatStr, year)

        # finally, write new instKindatTab.txt
        newFile = open(filename, 'w')
        newFile.write(newFileStr)
        newFile.close

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None)

init initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile).

Inputs: Existing MadrigalDB object, by default = None.

String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there.

def __init__(self, madDB=None, initFile=None):
    """__init__ initializes MadrigalInstrumentKindats by reading from instKindatTab.txt (or initFile).
    Inputs: Existing MadrigalDB object, by default = None.
        String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/instKindatTab.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    Note that the instKindatTab.txt file was new with the release of the madrigal python api, and this
    function will throw an error if file not there.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get instrument kindat metadata file
    if (initFile == None):
        self.__filename = self.__instKindatMetadataFile
    else:
        self.__filename = initFile
    self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()

def getKindatListForInstrumentYear(

self, kinst, year)

getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year.

Inputs: kinst, year

Returns: a list of kindat codes as integers associated with the given instrument and year, or None if not found.

Affects: None

Exceptions: if error in metadata file

def getKindatListForInstrumentYear(self, kinst, year):
    """getKindatListForInstrumentYear returns a list of kindat codes as integers for the given instrument and year.
    Inputs: kinst, year
    
    Returns: a list of kindat codes as integers associated with the given instrument  and year,
        or None if not found.
    Affects: None
    Exceptions: if error in metadata file
    """
    retKindatList = []
    
    # loop through instKindatTab.txt
    for inst in self.__fileList:
        # find matching kinst
        try:
            if int(inst[self.__instKindatKinstCol]) != kinst:
                continue
            if int(inst[self.__instKindatYearCol]) != year:
                continue
            return([int(kindat) for kindat in inst[self.__instKindatListCol].split()])
            
        except:
            raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))

def getKindatListForInstruments(

self, kinstList)

getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list. A kindat code is returned if it appears in any year

Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer.

Returns: a list of kindat codes as integers associated with the given instrument list.

Affects: None

Exceptions: if error in metadata file

def getKindatListForInstruments(self, kinstList):
    """getKindatListForInstruments returns a list of kindat codes as integers for the given instrument list.
    A kindat code is returned if it appears in any year
    Inputs: kinstList: a list of kinst integers to get associated kindat list. Also accepts a single integer.
    
    Returns: a list of kindat codes as integers associated with the given instrument list.
    Affects: None
    Exceptions: if error in metadata file
    """
    retKindatList = []
    
    # if kinstList is just a single integer, convert it into a list
    if type(kinstList) == int:
        kinstList = [kinstList]
    # loop through instKindatTab.txt
    for inst in self.__fileList:
        # find matching kinst
        try:
            if (int(inst[self.__instKindatKinstCol]) in kinstList):
                if len(inst[self.__instKindatListCol]) == 0:
                    continue
                tempList = inst[self.__instKindatListCol].split()
                for parm in tempList:
                    if not int(parm) in retKindatList:
                       retKindatList.append(int(parm))
            
        except:
            raise madrigal.admin.MadrigalError('Error in instKindatTab.txt parsing metadata row: ' + str(inst),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    return(retKindatList)

def rebuildInstKindatTable(

self)

rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file.

The table instKindatTab.txt is a listing of every kindat found for a given instrument for any given year. Since these files are constantly updated, this table needs to be updated on a regular basis. Data from other Madrigal sites is also imported into this table.

How it works: For each instrument in instTab.txt, this method first creates a list of all the experiments that used that instrument. It then loops through the file table. For each file listing, if the file is the default one, and its experiment id is in the list of experiments just created, the kindat is added to the list if its not already there. Data from all other Madrigal sites is then added via getMetadata. It then writes the metadata file in the form: 10, 1001 1002 1003, where the first column in the instrument code and the second column is a space delimited list of kindat codes.

Inputs: None.

Returns: None.

Affects: Writes file instKindatTab.txt in metadata directory

Exceptions: If unable to write instKindatTab.txt file.

def rebuildInstKindatTable(self):
    """rebuildInstKindatTable rebuilds the instKindatTab.txt metadata file.
    The table instKindatTab.txt is a listing of every kindat found for a given
    instrument for any given year.  Since these files are constantly updated, this table needs to be updated on a regular
    basis. Data from other Madrigal sites is also imported into this table.
    How it works: For each instrument in instTab.txt, this method first creates a list
    of all the experiments that used that instrument.  It then loops
    through the file table.  For each file listing, if the file is the
    default one, and its experiment id is in the list of experiments just
    created, the kindat is added to the list if its not already there.
    Data from all other Madrigal sites is then added via getMetadata.  It then writes
    the metadata file in the form: 10, 1001 1002 1003, where the first column in the
    instrument code and the second column is a space delimited list of kindat codes.
    Inputs: None.
    
    Returns: None.
    Affects: Writes file instKindatTab.txt in metadata directory
    Exceptions: If unable to write instKindatTab.txt file.
    """
    
    # get full file name
    filename = self.__madDB.getMetadataDir() + '/' + self.__filename
    
    # throw an exception if instKindatTab.txt file not writable
    if not os.access(filename, os.W_OK):
        raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None)
    # get experiment metadata
    madExp = MadrigalExperiment(self.__madDB)
    kindatDict = {} # key = year, value = list of kindats
    for i in range(madExp.getExpCount()):
        kinst = madExp.getKinstByPosition(i)
        if kinst not in list(kindatDict.keys()):
            kindatDict[kinst] = {}
        expDir = madExp.getExpDirByPosition(i)
        # get year range
        timeTuple = madExp.getExpStartDateTimeByPosition(i)
        syear = timeTuple[0]
        timeTuple = madExp.getExpEndDateTimeByPosition(i)
        eyear = timeTuple[0]
        for year in range(syear, eyear+1):
            if year not in list(kindatDict[kinst].keys()):
                kindatDict[kinst][year] = []
            try:
                madFileObj = madrigal.metadata.MadrigalMetaFile(self.__madDB,
                                                                os.path.join(expDir, 'fileTab.txt'))
            except:
                continue
        
            for j in range(madFileObj.getFileCount()):
                kindat = madFileObj.getKindatByPosition(j)
                if kindat not in kindatDict[kinst][year]:
                    kindatDict[kinst][year].append(kindat)
                
    # create newFileStr from instCodeDict
    newFileStr = ''
    kinstList = list(kindatDict.keys())
    kinstList.sort()
    delimiter = ' '
    for kinst in kinstList:
        yearList = list(kindatDict[kinst].keys())
        yearList.sort()
        for year in yearList:
            kindatDict[kinst][year].sort()
            kindatStrList = [str(kindat) for kindat in kindatDict[kinst][year]]
            kindatStr = delimiter.join(kindatStrList)
            newFileStr += '%i,%s,%i\n' % (kinst, kindatStr, year)
    # finally, write new instKindatTab.txt
    newFile = open(filename, 'w')
    newFile.write(newFileStr)
    newFile.close

class MadrigalInstrumentParameters

MadrigalInstrumentParameters is an object that provides access to the metadata file that summarizes the parameters associated with each instrument.

This object provides access to all Madrigal instrument parameter information in the metadata file instParmTab.txt. The metadata file instParmTab.txt lists, for any given instrument, all the measured parameters found in all the data files in the database associated with that instrument.

This class also contains a method to rebuild the table instParmTab.txt by examining every data file in the database. This is presumably a slow process and should be done in the background.

Usage example::

import madrigal.metadata

import madrigal.admin

try:

    test = madrigal.metadata.MadrigalInstrumentParameters()

    print test.getParameters(30)

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jul. 17, 2002

class MadrigalInstrumentParameters:
    """MadrigalInstrumentParameters is an object that provides access to the metadata file that summarizes the parameters associated with each instrument.

    This object provides access to all Madrigal instrument parameter information in the metadata file instParmTab.txt.
    The metadata file instParmTab.txt lists, for any given instrument, all the measured parameters found in all the
    data files in the database associated with that instrument.

    This class also contains a method to rebuild the table instParmTab.txt by examining every data file in the database.
    This is presumably a slow process and should be done in the background.

    Usage example::

        import madrigal.metadata
        
        import madrigal.admin

        try:
    
            test = madrigal.metadata.MadrigalInstrumentParameters()

            print test.getParameters(30)

        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file


    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Jul. 17, 2002

    """

    #constants
    __instParmMetadataFile  = "instParmTab.txt"

    # column positions
    __instParmKinstCol  =  0
    __instParmListCol   =  1

    def __init__(self, madDB=None, initFile=None):
        """__init__ initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile).

        Inputs: Existing MadrigalDB object, by default = None.

            String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        Note that the instParmTab.txt file was new with the release of the madrigal python api, and this
        function will throw an error if file not there.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get instrument parameter metadata file
        if (initFile == None):
            self.__filename = self.__instParmMetadataFile
        else:
            self.__filename = initFile

        self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()


        
    def getParameters(self, kinst):
        """getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank.

        Inputs: kinst integer to get parameters.  If 0, get parameters from all instruments.
        
        Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank.

        Affects: None

        Exceptions: if error in metadata file
        """
        
        parmList = []

        for inst in self.__fileList:
            # find matching kinst
            try:
                if (int(inst[self.__instParmKinstCol]) == kinst or kinst == 0):
                    if len(inst[self.__instParmListCol]) == 0:
                        continue
                    tempList = inst[self.__instParmListCol].split()
                    for parm in tempList:
                        if not parm.lower() in parmList:
                            if parm.find('-32767') == -1:
                                parmList.append(parm.lower())
                
            except:
                raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        if len(parmList) == 0:
            return(None)
        return parmList


    def rebuildInstParmTable(self, completeRebuildFlag = 0):
        """rebuildInstParmTable rebuilds the instParmTab.txt metadata file.

        The table instParmTab.txt is a listing of every measured parameter found in any data file for a given
        instrument. It now will also import data from other Madrigal sites instParmTab.txt files.
        Since these files are constantly updated, this table needs to be updated on a regular
        basis.  This methods works in one of two ways, depending on the value of completeRebuildFlag and whether
        a file called instParmLastUpdate.txt exists in the metadata directory.

        If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable
        loops through each instrument in the instTab.txt.  For each
        instrument, it loops through every data file associated with that instrument.  For every data file, it
        gets the list of parameters in that file, and adds them to the list for that instrument if they are unique.
        Since this process involves every file in the database, it may take a great deal of time and should
        be run in the background.

        If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable
        first stores all the existing parameters from the instParmTab.txt.  It reads the date of the last update
        from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with
        each instrument. For every new data file, it gets the list of parameters in that file, and adds
        them to the list for that instrument if they are unique.  This makes rebuildInstParmTable faster,
        but possibly keeps invalid parameters if experiments are ever deleted.

        Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are
        also added.

        Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means
        newer than the date in the instParmLastUpdate.txt file (stored as a float).  If 1, rebuild the table completely.  This will
        eliminate any parameters no longer found in files, but will take longer.  If no instParmLastUpdate.txt
        file is found, the table is always rebuilt completely.
        
        Returns: None.

        Affects: Writes file instParmTab.txt in metadata directory

        Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file.
        """
        # get full file name
        filename = self.__madDB.getMetadataDir() + '/' + self.__filename
        
        # throw an exception if instParmTab.txt file not writable
        if not os.access(filename, os.W_OK):
            raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None)

        # if madroot not set, set it now
        if os.environ.get('MAD' + 'ROOT') == None:
            os.environ['MAD' + 'ROOT'] = self.__madDB.getMadroot()

	# now its safe to import madrigal.data since MAD + ROOT set
        import madrigal.data

	# get date of files to examine
        if completeRebuildFlag == 0:
            # try to open and read instParmLastUpdate.txt
            try:
                dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'r')
                lastUpdateTime = float(dateFile.read())
                dateFile.close()
                
            except:
                # if failure, set time to 1/1/1970 and set completeRebuildFlag = 1
                lastUpdateTime = time.mktime((1970,1,1,0,0,0,0,0,0))
                completeRebuildFlag = 1
        else:
            # set time to 1/1/1971
            lastUpdateTime = time.mktime((1971,1,1,0,0,0,0,0,0))

        # now write new version of instParmLastUpdate.txt
        # if not writable, exception thrown
        try:
            dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'w')
        except:
            raise madrigal.admin.MadrigalError('Unable to write: ' + \
                                               self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt',
                                               None)
        newUpdateTime = time.time()
        dateFile.write(str(newUpdateTime))
        dateFile.close()
        # finally, make sure new instParmLastUpdate.txt is world-writable
        try:                
            os.chmod(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 0o666)
        except:
            pass

        # create a needed MadrigalParameters obj
        madParmObj = madrigal.data.MadrigalParameters(self.__madDB)

        # create a needed MadrigalSite obj
        madSiteObj = madrigal.metadata.MadrigalSite(self.__madDB)

        # create a string to hold all the text for the new file
        newFileStr = ''
        
        # create a dictionary with all instrument codes as keys
        instObj = madrigal.metadata.MadrigalInstrument(self.__madDB)
        instList = instObj.getInstrumentList()

        instCodeDict = {}
        for inst in instList:
            if inst[2] == 0:
                continue
            if inst[2] in list(instCodeDict.keys()):
                continue
            if completeRebuildFlag == 1:
                # start with empty list
                instCodeDict[inst[2]]= []
            else:
                # start with present list
                tempParmList = []
                oldParmList = self.getParameters(inst[2])
                if oldParmList != None:
                    for item in oldParmList:
                        # check that its not -32767
                        try:
                            if madParmObj.getParmCodeFromMnemonic(item) == -32767:
                                continue
                        except ValueError:
                            print(('skipping parm %s since no longer valid' % (str(item))))
                            continue
                        tempParmList.append(madParmObj.getParmCodeFromMnemonic(item))
                    instCodeDict[inst[2]] = tempParmList
                else:
                    instCodeDict[inst[2]] = []


        # get a list of all default and realtime files 
        filelist = self.__madDB.getFileListFromMetadata(appendKinst = 1,
                                                        includeRealtime = 1) # appendKinst, allow realtime


        # loop through each file - file is tuple of
        # (file name, inst code)
        for file in filelist:
            # check whether file should be skipped
            try:
                if lastUpdateTime > os.stat(file[0])[8]:
                    continue
            except:
                continue
            
            # try to create MadrigalFile object for that file
            try:
                print('\t\tAdding parameters to instParmTab.txt from ' + str(file))
                madFileObj = madrigal.data.MadrigalFile(file[0], self.__madDB)
                fileParmList = madFileObj.getMeasuredParmList()
                # append unique parameters
                for parm in fileParmList:
                    if not parm in instCodeDict[file[1]]:
                        instCodeDict[file[1]].append(parm)
            # if a problem with file, skip it
            except:
                continue

 

        # now create a new file string from instCodeDict
        # create a sorted list of keys
        keyList = list(instCodeDict.keys())
        keyList.sort()
        # step through each key
        delimiter = ' '
        for key in keyList:
            # sort that instrument's list
            instParmList = madParmObj.normalizeParmList(instCodeDict[key])
            # convert from codes to mnemonics
            instParmList = madParmObj.getParmMnemonicList(instParmList)

            # append that instrument's data to newFileStr
            newFileStr = newFileStr + str(key) + ','
            newFileStr = newFileStr + delimiter.join(instParmList).lower() + '\n'

        # if no data found, throw error
        if len(newFileStr) == 0:
            raise madrigal.admin.MadrigalError('No data found for: ' + str(filename), None)
            

        # write new instParmTab.txt
        newFile = open(filename, 'w')
        newFile.write(newFileStr)
        newFile.close

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None)

init initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile).

Inputs: Existing MadrigalDB object, by default = None.

String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully. Note that the instParmTab.txt file was new with the release of the madrigal python api, and this function will throw an error if file not there.

def __init__(self, madDB=None, initFile=None):
    """__init__ initializes MadrigalInstrumentParameters by reading from instParmTab.txt (or initFile).
    Inputs: Existing MadrigalDB object, by default = None.
        String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/instParmTab.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    Note that the instParmTab.txt file was new with the release of the madrigal python api, and this
    function will throw an error if file not there.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get instrument parameter metadata file
    if (initFile == None):
        self.__filename = self.__instParmMetadataFile
    else:
        self.__filename = initFile
    self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()

def getParameters(

self, kinst)

getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank.

Inputs: kinst integer to get parameters. If 0, get parameters from all instruments.

Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank.

Affects: None

Exceptions: if error in metadata file

def getParameters(self, kinst):
    """getParameters returns a list of parameters in mnemonic form (strings or unknown integers as strings) that matches kinst argument, or None if not found or blank.
    Inputs: kinst integer to get parameters.  If 0, get parameters from all instruments.
    
    Returns: a list of mnemonic strings or unknown integer strings, or None if kinst not found or blank.
    Affects: None
    Exceptions: if error in metadata file
    """
    
    parmList = []
    for inst in self.__fileList:
        # find matching kinst
        try:
            if (int(inst[self.__instParmKinstCol]) == kinst or kinst == 0):
                if len(inst[self.__instParmListCol]) == 0:
                    continue
                tempList = inst[self.__instParmListCol].split()
                for parm in tempList:
                    if not parm.lower() in parmList:
                        if parm.find('-32767') == -1:
                            parmList.append(parm.lower())
            
        except:
            raise madrigal.admin.MadrigalError('Error in instTab.txt parsing metadata row: ' + str(inst),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    if len(parmList) == 0:
        return(None)
    return parmList

def rebuildInstParmTable(

self, completeRebuildFlag=0)

rebuildInstParmTable rebuilds the instParmTab.txt metadata file.

The table instParmTab.txt is a listing of every measured parameter found in any data file for a given instrument. It now will also import data from other Madrigal sites instParmTab.txt files. Since these files are constantly updated, this table needs to be updated on a regular basis. This methods works in one of two ways, depending on the value of completeRebuildFlag and whether a file called instParmLastUpdate.txt exists in the metadata directory.

If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable loops through each instrument in the instTab.txt. For each instrument, it loops through every data file associated with that instrument. For every data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. Since this process involves every file in the database, it may take a great deal of time and should be run in the background.

If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable first stores all the existing parameters from the instParmTab.txt. It reads the date of the last update from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with each instrument. For every new data file, it gets the list of parameters in that file, and adds them to the list for that instrument if they are unique. This makes rebuildInstParmTable faster, but possibly keeps invalid parameters if experiments are ever deleted.

Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are also added.

Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means newer than the date in the instParmLastUpdate.txt file (stored as a float). If 1, rebuild the table completely. This will eliminate any parameters no longer found in files, but will take longer. If no instParmLastUpdate.txt file is found, the table is always rebuilt completely.

Returns: None.

Affects: Writes file instParmTab.txt in metadata directory

Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file.

def rebuildInstParmTable(self, completeRebuildFlag = 0):
    """rebuildInstParmTable rebuilds the instParmTab.txt metadata file.
    The table instParmTab.txt is a listing of every measured parameter found in any data file for a given
    instrument. It now will also import data from other Madrigal sites instParmTab.txt files.
    Since these files are constantly updated, this table needs to be updated on a regular
    basis.  This methods works in one of two ways, depending on the value of completeRebuildFlag and whether
    a file called instParmLastUpdate.txt exists in the metadata directory.
    If completeRebuildFlag = 1 or instParmLastUpdate.txt does not exist, the method rebuildInstParmTable
    loops through each instrument in the instTab.txt.  For each
    instrument, it loops through every data file associated with that instrument.  For every data file, it
    gets the list of parameters in that file, and adds them to the list for that instrument if they are unique.
    Since this process involves every file in the database, it may take a great deal of time and should
    be run in the background.
    If completeRebuildFlag = 0 and instParmLastUpdate.txt does exist, the method rebuildInstParmTable
    first stores all the existing parameters from the instParmTab.txt.  It reads the date of the last update
    from instParmLastUpdate.txt, and only reads data files newer than that date that are associated with
    each instrument. For every new data file, it gets the list of parameters in that file, and adds
    them to the list for that instrument if they are unique.  This makes rebuildInstParmTable faster,
    but possibly keeps invalid parameters if experiments are ever deleted.
    Finally, the instParmTab.txt file of every other site is obtained via getMetadata, and those parameters are
    also added.
    Inputs: completeRebuildFlag: if 0 (the default), only add parameters from new files, where new means
    newer than the date in the instParmLastUpdate.txt file (stored as a float).  If 1, rebuild the table completely.  This will
    eliminate any parameters no longer found in files, but will take longer.  If no instParmLastUpdate.txt
    file is found, the table is always rebuilt completely.
    
    Returns: None.
    Affects: Writes file instParmTab.txt in metadata directory
    Exceptions: If unable to write instParmTab.txt file or the instParmLastUpdate.txt file.
    """
    # get full file name
    filename = self.__madDB.getMetadataDir() + '/' + self.__filename
    
    # throw an exception if instParmTab.txt file not writable
    if not os.access(filename, os.W_OK):
        raise madrigal.admin.MadrigalError('Unable to write: ' + str(filename), None)
    # if madroot not set, set it now
    if os.environ.get('MAD' + 'ROOT') == None:
        os.environ['MAD' + 'ROOT'] = self.__madDB.getMadroot()
ow its safe to import madrigal.data since MAD + ROOT set
    import madrigal.data
et date of files to examine
    if completeRebuildFlag == 0:
        # try to open and read instParmLastUpdate.txt
        try:
            dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'r')
            lastUpdateTime = float(dateFile.read())
            dateFile.close()
            
        except:
            # if failure, set time to 1/1/1970 and set completeRebuildFlag = 1
            lastUpdateTime = time.mktime((1970,1,1,0,0,0,0,0,0))
            completeRebuildFlag = 1
    else:
        # set time to 1/1/1971
        lastUpdateTime = time.mktime((1971,1,1,0,0,0,0,0,0))
    # now write new version of instParmLastUpdate.txt
    # if not writable, exception thrown
    try:
        dateFile = open(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 'w')
    except:
        raise madrigal.admin.MadrigalError('Unable to write: ' + \
                                           self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt',
                                           None)
    newUpdateTime = time.time()
    dateFile.write(str(newUpdateTime))
    dateFile.close()
    # finally, make sure new instParmLastUpdate.txt is world-writable
    try:                
        os.chmod(self.__madDB.getMetadataDir() + '/instParmLastUpdate.txt', 0o666)
    except:
        pass
    # create a needed MadrigalParameters obj
    madParmObj = madrigal.data.MadrigalParameters(self.__madDB)
    # create a needed MadrigalSite obj
    madSiteObj = madrigal.metadata.MadrigalSite(self.__madDB)
    # create a string to hold all the text for the new file
    newFileStr = ''
    
    # create a dictionary with all instrument codes as keys
    instObj = madrigal.metadata.MadrigalInstrument(self.__madDB)
    instList = instObj.getInstrumentList()
    instCodeDict = {}
    for inst in instList:
        if inst[2] == 0:
            continue
        if inst[2] in list(instCodeDict.keys()):
            continue
        if completeRebuildFlag == 1:
            # start with empty list
            instCodeDict[inst[2]]= []
        else:
            # start with present list
            tempParmList = []
            oldParmList = self.getParameters(inst[2])
            if oldParmList != None:
                for item in oldParmList:
                    # check that its not -32767
                    try:
                        if madParmObj.getParmCodeFromMnemonic(item) == -32767:
                            continue
                    except ValueError:
                        print(('skipping parm %s since no longer valid' % (str(item))))
                        continue
                    tempParmList.append(madParmObj.getParmCodeFromMnemonic(item))
                instCodeDict[inst[2]] = tempParmList
            else:
                instCodeDict[inst[2]] = []
    # get a list of all default and realtime files 
    filelist = self.__madDB.getFileListFromMetadata(appendKinst = 1,
                                                    includeRealtime = 1) # appendKinst, allow realtime
    # loop through each file - file is tuple of
    # (file name, inst code)
    for file in filelist:
        # check whether file should be skipped
        try:
            if lastUpdateTime > os.stat(file[0])[8]:
                continue
        except:
            continue
        
        # try to create MadrigalFile object for that file
        try:
            print('\t\tAdding parameters to instParmTab.txt from ' + str(file))
            madFileObj = madrigal.data.MadrigalFile(file[0], self.__madDB)
            fileParmList = madFileObj.getMeasuredParmList()
            # append unique parameters
            for parm in fileParmList:
                if not parm in instCodeDict[file[1]]:
                    instCodeDict[file[1]].append(parm)
        # if a problem with file, skip it
        except:
            continue
    # now create a new file string from instCodeDict
    # create a sorted list of keys
    keyList = list(instCodeDict.keys())
    keyList.sort()
    # step through each key
    delimiter = ' '
    for key in keyList:
        # sort that instrument's list
        instParmList = madParmObj.normalizeParmList(instCodeDict[key])
        # convert from codes to mnemonics
        instParmList = madParmObj.getParmMnemonicList(instParmList)
        # append that instrument's data to newFileStr
        newFileStr = newFileStr + str(key) + ','
        newFileStr = newFileStr + delimiter.join(instParmList).lower() + '\n'
    # if no data found, throw error
    if len(newFileStr) == 0:
        raise madrigal.admin.MadrigalError('No data found for: ' + str(filename), None)
        
    # write new instParmTab.txt
    newFile = open(filename, 'w')
    newFile.write(newFileStr)
    newFile.close

class MadrigalKindat

MadrigalKindat is an object that provides access to Madrigal kind of data info from the metadata.

This object provides access to all Madrigal kind of data information in the metadata file typeTab.txt.

Usage example::

import madrigal.metadata
import madrigal.admin

try:

    test = madrigal.metadata.MadrigalKindat()

    print test.getKindatDescription(3001)

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

2. Columns expected to be ints or floats cannot be converted

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001

class MadrigalKindat:
    """MadrigalKindat is an object that provides access to Madrigal kind of data info from the metadata.

    This object provides access to all Madrigal kind of data information in the metadata file typeTab.txt.

    Usage example::

        import madrigal.metadata
        import madrigal.admin

        try:
    
            test = madrigal.metadata.MadrigalKindat()

            print test.getKindatDescription(3001)

        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file
        
        2. Columns expected to be ints or floats cannot be converted

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Nov. 9, 2001

    """

    #constants
    __typeMetadataFile  = "typeTab.txt"

    # column positions
    __typeCodeCol   =  0
    __typeDescCol   =  1

    

    def __init__(self, madDB=None, initFile=None):
        """__init__ initializes MadrigalKindat by reading from typeTab.txt (or initFile).

        Inputs: Existing MadrigalDB object, by default = None.

            String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/typeTab.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get kindat metadata file
        if (initFile == None):
            self.__filename = self.__typeMetadataFile
        else:
            self.__filename = initFile

        self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()


    def getKindatDescription(self, code, kinst=None):
        """getKindatDescription returns the kindat description that matches code argument, or None if not found.

        Inputs: 
        
            code integer to get kindat description. or integer as string
            
            kinst - used to look up kindat description in form '%i_%i' % (kinst, code).  If None,
                (the default) only return exact code match as integer.  If given, first try to match
                '%i_%i' % (kinst, code).  If not found, then try just code. May be integer or string integer
        
        Returns: the kindat description that matches code argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        code = int(code)
        
        if kinst != None:
            kinst = int(kinst)
            for i, type in enumerate(self.__fileList):
                # find matching kinst and code 
                try:
                    items = type[self.__typeCodeCol].split('_')
                    if len(items) == 2:
                        thisKinst = int(items[0])
                        thisKindat = int(items[1])
                        if thisKinst == kinst and thisKindat == code:
                            return type[self.__typeDescCol]
                    
                except:
                    raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                                  sys.exc_info()[1],
                                                                                  sys.exc_info()[2]))
                    
        # just search for code
        for i, type in enumerate(self.__fileList):
            try:
                items = type[self.__typeCodeCol].split('_')
                if len(items) == 1:
                    thisKindat = int(items[0])
                    if thisKindat == code:
                        return type[self.__typeDescCol]
                
            except:
                raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))

        # not found
        return(None)


    def getKindatList(self):
        """getKindatList returns a list of all kindat descriptions and codes.

        Inputs: None.
        
        Returns: a list of all kindat descriptions and codes.  Each item in the list
        is a tuple of the form (Kindat description (string), kindat code (integer or string in
        form '%i_%i' % (kinst, code))).  Example item:
        ('INSCAL Basic Derived Parameters', 3001)

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        retList = []

        for i, kindat in enumerate(self.__fileList):
            
            try:
                if kindat[self.__typeCodeCol].find('_') != -1:
                    raise ValueError('')
                # works if its a standard code - int no longer raises an error if underscore found
                item = (kindat[self.__typeDescCol],
                        int(kindat[self.__typeCodeCol]))
                retList.append(item)
                
            except:
                try:
                    # works for kinst_kindat
                    item = (kindat[self.__typeDescCol],
                            kindat[self.__typeCodeCol])
                    retList.append(item)
                    
                except:
                    raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(kindat),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        return retList

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None)

init initializes MadrigalKindat by reading from typeTab.txt (or initFile).

Inputs: Existing MadrigalDB object, by default = None.

String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/typeTab.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.

def __init__(self, madDB=None, initFile=None):
    """__init__ initializes MadrigalKindat by reading from typeTab.txt (or initFile).
    Inputs: Existing MadrigalDB object, by default = None.
        String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/typeTab.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get kindat metadata file
    if (initFile == None):
        self.__filename = self.__typeMetadataFile
    else:
        self.__filename = initFile
    self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()

def getKindatDescription(

self, code, kinst=None)

getKindatDescription returns the kindat description that matches code argument, or None if not found.

Inputs:

code integer to get kindat description. or integer as string

kinst - used to look up kindat description in form '%i_%i' % (kinst, code).  If None,
    (the default) only return exact code match as integer.  If given, first try to match
    '%i_%i' % (kinst, code).  If not found, then try just code. May be integer or string integer

Returns: the kindat description that matches code argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getKindatDescription(self, code, kinst=None):
    """getKindatDescription returns the kindat description that matches code argument, or None if not found.
    Inputs: 
    
        code integer to get kindat description. or integer as string
        
        kinst - used to look up kindat description in form '%i_%i' % (kinst, code).  If None,
            (the default) only return exact code match as integer.  If given, first try to match
            '%i_%i' % (kinst, code).  If not found, then try just code. May be integer or string integer
    
    Returns: the kindat description that matches code argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    code = int(code)
    
    if kinst != None:
        kinst = int(kinst)
        for i, type in enumerate(self.__fileList):
            # find matching kinst and code 
            try:
                items = type[self.__typeCodeCol].split('_')
                if len(items) == 2:
                    thisKinst = int(items[0])
                    thisKindat = int(items[1])
                    if thisKinst == kinst and thisKindat == code:
                        return type[self.__typeDescCol]
                
            except:
                raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
                
    # just search for code
    for i, type in enumerate(self.__fileList):
        try:
            items = type[self.__typeCodeCol].split('_')
            if len(items) == 1:
                thisKindat = int(items[0])
                if thisKindat == code:
                    return type[self.__typeDescCol]
            
        except:
            raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(type),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    # not found
    return(None)

def getKindatList(

self)

getKindatList returns a list of all kindat descriptions and codes.

Inputs: None.

Returns: a list of all kindat descriptions and codes. Each item in the list is a tuple of the form (Kindat description (string), kindat code (integer or string in form '%i_%i' % (kinst, code))). Example item: ('INSCAL Basic Derived Parameters', 3001)

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getKindatList(self):
    """getKindatList returns a list of all kindat descriptions and codes.
    Inputs: None.
    
    Returns: a list of all kindat descriptions and codes.  Each item in the list
    is a tuple of the form (Kindat description (string), kindat code (integer or string in
    form '%i_%i' % (kinst, code))).  Example item:
    ('INSCAL Basic Derived Parameters', 3001)
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    retList = []
    for i, kindat in enumerate(self.__fileList):
        
        try:
            if kindat[self.__typeCodeCol].find('_') != -1:
                raise ValueError('')
            # works if its a standard code - int no longer raises an error if underscore found
            item = (kindat[self.__typeDescCol],
                    int(kindat[self.__typeCodeCol]))
            retList.append(item)
            
        except:
            try:
                # works for kinst_kindat
                item = (kindat[self.__typeDescCol],
                        kindat[self.__typeCodeCol])
                retList.append(item)
                
            except:
                raise madrigal.admin.MadrigalError('Error in typeTab.txt parsing metadata row %i: ' % (i) + str(kindat),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    return retList

class MadrigalMetaFile

MadrigalMetaFile is an object that provides access to Madrigal file info from the metadata.

This object provides access to all Madrigal experiment information in the metadata file fileTab.txt.

Usage example::

import madrigal.metadata

import madrigal.admin

try:

    test = madrigal.metadata.MadrigalMetaFile()

    print test.getFileCount()

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

2. Columns expected to be ints or floats cannot be converted

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu May. 7, 2002

class MadrigalMetaFile:
    """MadrigalMetaFile is an object that provides access to Madrigal file info from the metadata.

    This object provides access to all Madrigal experiment information in the metadata file fileTab.txt.

    Usage example::

        import madrigal.metadata
        
        import madrigal.admin

        try:
    
            test = madrigal.metadata.MadrigalMetaFile()

            print test.getFileCount()

        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file
        
        2. Columns expected to be ints or floats cannot be converted

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  May. 7, 2002

    """

    #constants
    __fileMetadataFile  = "fileTab.txt"

    # column positions
    __fileNameCol           =  0
    __fileExpIdCol          =  1
    __fileKindatCol         =  2
    __fileCategoryCol       =  3
    __fileSizeCol           =  4
    __fileHasCatalogCol     =  5
    __fileHasHeaderCol      =  6
    __fileModDateCol        =  7
    __fileModTimeCol        =  8
    __fileStatusCol         =  9
    __fileAccessCol         =  10
    __fileAnalystCol        =  11
    __fileAnalystEmailCol   =  12
    

    def __init__(self, madDB=None, initFile=None):
        """__init__ initializes MadrigalMetaFile by reading from fileTab.txt (or initFile).

        Inputs: Existing MadrigalDB object, by default = None.

            String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/fileTab.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get file metadata file
        if (initFile == None):
            self.__filename = self.__fileMetadataFile
        else:
            self.__filename = initFile

        # MadrigalMetafile can now legal have two different lengths - with or without file analyst and email
        allowedLens = (self.__fileAccessCol+1, self.__fileAnalystEmailCol+1)
        
        genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__fileNameCol)

        self.__fileList = genericObj.getList()
        self._dict = genericObj.getDict()
        
        # check that at least first row has right number of columns
        if len(self.__fileList) > 0:
            # analyst and analyst email columns may or may not exist
            if len(self.__fileList[0]) not in allowedLens:
                raise madrigal.admin.MadrigalError('Error in count of first row of ' + str(self.__filename), None)



    def getFileCount(self):
        """getFileCount returns the number of files (rows) in the metadata file.

        Inputs: None
        
        Returns: the number of files (rows) in the metadata file.

        Affects: None

        Exceptions: None
        """

        return len(self.__fileList)


    def getFilenameByPosition(self, position = 0):
        """getFilenameByPosition returns the filename of the file at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the filename of the file at given position as a string.  
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:

            return self.__fileList[position][self.__fileNameCol]

        else:
            return(None)


    def getExpIdByPosition(self, position = 0):
        """getExpIdByPosition returns the experiment id (integer) of the file at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the experiment id (integer) of the file at given position as an integer.  
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: Thrown if kinst exp id cannot be parsed into an integer
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__fileExpIdCol])

            except:
                raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)


    def setExpIdByPosition(self, position, expId):
        """setExpIdByPosition sets the experiment id of the file at given position.

        Inputs:

            position - position of file in list (first position is zero).

            expId - the new experiment id to use
        
        Returns: None.

        Affects: sets the experiment id of the file at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        position not found.
        """
        
        try:
            self.__fileList[position][self.__fileExpIdCol] = str(int(expId))

        except:
            raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' %  \
                                               (str(position), str(expId)),
                                                traceback.format_exception(sys.exc_info()[0],
                                                                           sys.exc_info()[1],
                                                                           sys.exc_info()[2]))


    def getKindatByPosition(self, position = 0):
        """getKindatByPosition returns the kindat (kind of data code) of the file at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the kinst (kind of instrument code) of the file at given position as an integer.  
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: Thrown if kinst column cannot be parsed into an integer
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__fileKindatCol])

            except:
                raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)


    def getCategoryByPosition(self, position = 0):
        """getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer.  
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: Thrown if category column cannot be parsed into an integer
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__fileCategoryCol])

            except:
                raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)


    def getCategoryByFilename(self, filename):
        """getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename.

        Inputs: filename - name of file to search for.
        
        Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the
        given filename.  Since
        filename may not be unique (although it usually is), the first match found is used.  If
        no matches found, returns None.
        
        Affects: None

        Exceptions: Thrown if category column cannot be parsed into an integer
        """
        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        
        return(int(self.__fileList[position][self.__fileCategoryCol]))



    def getHasCatalogByPosition(self, position = 0):
        """getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns:  True if the file at given position has any catalog records, False otherwise 
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:

            if int(self.__fileList[position][self.__fileHasCatalogCol]) == 0:
                return False
            else:
                return True

        else:
            return(None)


    def getHasCatalogByFilename(self,filename):
        """getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise.

        Inputs: filename - name of file to search for.
        
        Returns: true if the file with the given name has any catalog records, False otherwise.  Returns none if name
        not found
        
        Affects: None

        Exceptions: None.
        """

        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        
        return(int(self.__fileList[position][self.__fileHasCatalogCol]))


    def setHasCatalogByPosition(self, position, hasCatalog):
        """setHasCatalogByPosition sets the value of hasCatalog for the file at the given position.

        Inputs:

            position - position of file in list (first position is zero).

            hasCatalog - 1 or True for yes, 0 or False for no
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        if hasCatalog not in (0, 1, True, False):
            raise ValueError('Illegal value for hasCatalog in setHasCatalogByPosition: %s' % (str(hasCatalog)))

        if len(self.__fileList) > position:

            if hasCatalog in (1, True):
                self.__fileList[position][self.__fileHasCatalogCol] = '1'
            else:
                self.__fileList[position][self.__fileHasCatalogCol] = '0'

        else:
            raise ValueError('setHasCatalogByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))


    def getHasHeaderByPosition(self, position = 0):
        """getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns:  True if the file at given position has any header records, False otherwise 
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:

            if int(self.__fileList[position][self.__fileHasHeaderCol]) == 0:
                return False
            else:
                return True

        else:
            return(None)


    def getHasHeaderByFilename(self,filename):
        """getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise.

        Inputs: filename - name of file to search for.
        
        Returns: true if the file with the given name has any header records, False otherwise.  Returns none if name
        not found
        
        Affects: None

        Exceptions: Thrown if filename not found.
        """

        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        
        return(int(self.__fileList[position][self.__fileHasHeaderCol]))



    def setHasHeaderByPosition(self, position, hasHeader):
        """setHasHeaderByPosition sets the value of hasHeader for the file at the given position.

        Inputs:

            position - position of file in list (first position is zero).

            hasHeader - 1 or True for yes, 0 or False for no
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        if hasHeader not in (0, 1, True, False):
            raise ValueError('Illegal value for hasHeader in setHasHeaderByPosition: %s' % (str(hasHeader)))

        if len(self.__fileList) > position:

            if hasHeader in (1, True):
                self.__fileList[position][self.__fileHasHeaderCol] = '1'
            else:
                self.__fileList[position][self.__fileHasHeaderCol] = '0'

        else:
            raise ValueError('setHasHeaderByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))


    def getStatusByPosition(self, position = 0):
        """getStatusByPosition returns the status description of the file at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the status description of the file at given position as a string.  
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:

            return self.__fileList[position][self.__fileStatusCol]

        else:
            return(None)


    def getStatusByFilename(self,filename):
        """getStatusByFilename returns the status description of the file with the given name.

        Inputs: filename - name of file to search for.
        
        Returns: the status description of the file with the given name.  Returns none if name
        not found
        
        Affects: None

        Exceptions: Thrown if filename not found.
        """

        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        
        return(self.__fileList[position][self.__fileStatusCol])



    def getAccessByPosition(self, position = 0):
        """getAccessByPosition returns the access (0=public, 1=private) of the file at given position.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: the access level (0=public, 1=private) of the file at given position as an integer.  
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: Thrown if access column cannot be parsed into an integer
        """

        if len(self.__fileList) > position:

            try:
                return int(self.__fileList[position][self.__fileAccessCol])

            except:
                raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))
        else:
            return(None)
        
        
    def getFileDatetimeByPosition(self, position = 0):
        """getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns: a datetime of the file at given position, or None if not set.  
        Returns None if position >= number of experiments.
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:

            dateStr = self.__fileList[position][self.__fileModDateCol]
            if len(dateStr) < 8:
                return(None)
            timeStr = self.__fileList[position][self.__fileModTimeCol]
            if len(timeStr) < 6:
                return(None)
            
            return(datetime.datetime.strptime('%s %s' % (dateStr, timeStr), '%Y%m%d %H%M%S'))

        else:
            return(None)


    def getFileDatetimeByFilename(self,filename):
        """getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set.

        Inputs: filename - name of file to search for.
        
        Returns: a datetime of the file with the given filenme, or None if not set.  Returns none if name
        not found
        
        Affects: None

        Exceptions: Thrown if filename not found.
        """

        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        
        return(self.getFileDatetimeByPosition(position))

    
    
    def setFileDatetimeByPosition(self, position, dt):
        """setFileDatetimeByPosition sets the file datetime for the given position

        Inputs:

            position - position of file in list (first position is zero).

            dt - datetime (UT) of file being added.  If None, use file datetime.
                Will raise exception if dt is None and fileTab.txt in the metadata
                version, not the expDir version.
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        if dt is None:
            basename = self.getFilenameByPosition(position)
            expDir = os.path.dirname(self.__filename)
            if len(expDir) < 2:
                raise IOError('Cannot set datetime to None using the metadata version of fileTab.txt')
            dataFile = os.path.join(expDir, basename)
            if not os.access(dataFile, os.R_OK):
                raise IOError('Cannot access %s' % (dataFile))
            dt = datetime.datetime.utcfromtimestamp(os.path.getmtime(dataFile))
            
            
        dateStr = dt.strftime('%Y%m%d')
        timeStr = dt.strftime('%H%M%S')

        if len(self.__fileList) > position:

            self.__fileList[position][self.__fileModDateCol] = dateStr
            self.__fileList[position][self.__fileModTimeCol] = timeStr

        else:
            raise ValueError('setFileDatetimeByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))


    def deleteRowByFilename(self, filename):
        """deleteRowByFilename deletes a row with a given filename.

        Inputs: filename - name of file to search for.
        
        Returns: None.
        
        Affects: Removes item from self.__fileList if filename found

        Exceptions: Thrown if filename not found.
        """
        
        for file in self.__fileList:

            if filename == file[self.__fileNameCol]:

                self.__fileList.remove(file)
                return

                
        # no matches found
        raise madrigal.admin.MadrigalError('Could not delete file ' + filename + ' from ' + self.__filename, None)



    def getExpIdByFilename(self, filename):
        """getExpIdByFilename returns the first experiment id (integer) with the given filename.

        Inputs: filename - name of file to search for.
        
        Returns: the experiment id (integer) of the first row found with the given filename.  Since
        filename may not be unique (although it usually is), the first match found is used.  If
        no matches found, returns None.
        
        Affects: None

        Exceptions: Thrown if exp id cannot be parsed into an integer
        """
        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        
        return(int(self.__fileList[position][self.__fileExpIdCol]))



    def getKindatByFilename(self, filename):
        """getKindatByFilename returns the first kindat (integer) with the given filename.

        Inputs: filename - name of file to search for.
        
        Returns: the kindat (integer) of the first row found with the given filename.  Since
        filename may not be unique (although it usually is), the first match found is used.  If
        no matches found, returns None.
        
        Affects: None

        Exceptions: Thrown if kindat cannot be parsed into an integer
        """
        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        
        return(int(self.__fileList[position][self.__fileKindatCol]))



    def setAccessByPosition(self, position, access):
        """setAccessByPosition sets the value of access for the file at the given position.

        Inputs:

            position - position of file in list (first position is zero).

            access - 0 of False for public, 1 or True for private
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        if access not in (0, 1, True, False):
            raise ValueError('Illegal value for access in setAccessByPosition: %s' % (str(access)))

        if len(self.__fileList) > position:

            if access in (1, True):
                self.__fileList[position][self.__fileAccessCol] = '1'
            else:
                self.__fileList[position][self.__fileAccessCol] = '0'

        else:
            raise ValueError('setAccessByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))


    def setAccess(self, accessType):
        """setAccess sets the access column to all 0's (public) or all 1's (private).

        Inputs: accessType - either 0 to set to public access, or 1 to set to private access.
        
        Returns: None.
        
        Affects: Overwrite fileTab.txt file with access column set to all 0's (public)
        or all 1's (private).

        Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1
        """
        if (accessType != 0 and accessType != 1):
            raise madrigal.admin.MadrigalError('MadrigalMetaFile.setAccess called with arg = ' + \
                                               str(accessType) + ', must be either 0 or 1', None)

        for row in self.__fileList:
            row[self.__fileAccessCol] = str(accessType)

        self.writeMetadata()


    def setKindatByPosition(self, position, kindat):
        """setKindatByPosition sets the value of kindat for the file at the given position.

        Inputs:

            position - position of file in list (first position is zero).

            kindat - integer kindat value
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        kindat = int(kindat)
        
        if len(self.__fileList) > position:

            self.__fileList[position][self.__fileKindatCol] = str(kindat)

        else:
            raise ValueError('setKindatByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))



    def setCategoryByPosition(self, position, category):
        """setCategoryByPosition sets the value of category for the file at the given position.

        Inputs:

            position - position of file in list (first position is zero).

            category - 1=default, 2=variant, 3=history, 4=real-time
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        if int(category) not in (1, 2, 3, 4):
            raise ValueError('Illegal value for category in setCategoryByPosition: %s' % (str(category)))

        if len(self.__fileList) > position:

            self.__fileList[position][self.__fileCategoryCol] = str(category)

        else:
            raise ValueError('setCategoryByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))


    def setStatusByPosition(self, position, status):
        """setStatusByPosition sets the value of status string for the file at the given position.

        Inputs:

            position - position of file in list (first position is zero).

            status - string describing status
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        statusTypes = [bytes, str]
        if type(status) not in statusTypes:
            raise ValueError('Illegal value for status in setStatusByPosition: %s' % (str(status)))

        # check that string does not illegally contain a comma
        if status.find(',') != -1:
            raise ValueError('status string in fileTab.txt cannot contain a comma: <%s> is illegal' % (status))

        if len(self.__fileList) > position:

            self.__fileList[position][self.__fileStatusCol] = status

        else:
            raise ValueError('setStatusByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))


    def getAnalystByPosition(self, position = 0):
        """getAnalystByPosition returns file analyst name if there, None otherwise.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns:  file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, 
        returns None if the column does not exist.  Returns None if position >= number of experiments.  
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:

            try:
                return(self.__fileList[position][self.__fileAnalystCol])
            except:
                return(None)

        else:
            return(None)


    def getAnalystByFilename(self, filename):
        """getAnalystByFilename returns file analyst name if there, None otherwise.

        Inputs: filename - name of file to search for.
        
        Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, 
        returns None if the column does not exist.
        
        Affects: None

        Exceptions: None
        """
        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        try:
            return(self.__fileList[position][self.__fileAnalystCol])
        except:
            return(None)



    def setAnalystByPosition(self, position, analyst):
        """setAnalystByPosition sets the file analyst name for the given position

        Inputs:

            position - position of file in list (first position is zero).

            analyst - name of file analyst
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        # verify no illegal commas
        if analyst.find(',') != -1:
            raise madrigal.admin.MadrigalError('Error in setAnalystByPosition with args %s: %s' %  \
                                               (str(position), analyst),
                                                [traceback.format_exc()])

        if len(self.__fileList) > position:

            if len(self.__fileList[position]) == self.__fileAccessCol+1:
                # append two extra columns
                self.__fileList[position].append(str(analyst))
                self.__fileList[position].append('')
            else:
                self.__fileList[position][self.__fileAnalystCol] = str(analyst)

        else:
            raise ValueError('setAnalystByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))


    def getAnalystEmailByPosition(self, position = 0):
        """getAnalystEmailByPosition returns file analyst email if there, None otherwise.

        Inputs: position of file in list (first position is zero).  Defaults to first.
        
        Returns:  file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, 
        returns None if the column does not exist.  Returns None if position >= number of experiments.  
        
        Affects: None

        Exceptions: None
        """

        if len(self.__fileList) > position:

            try:
                return(self.__fileList[position][self.__fileAnalystEmailCol])
            except:
                return(None)

        else:
            return(None)


    def getAnalystEmailByFilename(self, filename):
        """getAnalystEmailByFilename returns file analyst email if there, None otherwise.

        Inputs: filename - name of file to search for.
        
        Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, 
        returns None if the column does not exist.
        
        Affects: None

        Exceptions: None
        """
        filename = os.path.basename(filename)
        try:
            position = self._dict[filename]
        except KeyError:
            return(None)
        try:
            return(self.__fileList[position][self.__fileAnalystEmailCol])
        except:
            return(None)



    def setAnalystEmailByPosition(self, position, analystEmail):
        """setAnalystEmailByPosition sets the file analyst email for the given position

        Inputs:

            position - position of file in list (first position is zero).

            analystEmail - email of file analyst
        
        Returns:  None.
        
        Affects: None

        Exceptions: If position beyond length.
        """
        # verify no illegal commas
        if analystEmail.find(',') != -1:
            raise madrigal.admin.MadrigalError('Error in setAnalystEmailByPosition with args %s: %s' %  \
                                               (str(position, analystEmail),
                                                [traceback.format_exc()]))

        if len(self.__fileList) > position:

            if len(self.__fileList[position]) == self.__fileAccessCol + 1:
                # append two extra columns
                self.__fileList[position].append('')
                self.__fileList[position].append(str(analystEmail))
            else:
                self.__fileList[position][self.__fileAnalystEmailCol] = str(analystEmail)

        else:
            raise ValueError('setAnalystEmailByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))




    def getMetadataSummaryByFilename(self, filename, madExpObj = None, madInstObj = None, madKindatObj = None):
        """getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename.

        Inputs: filename - name of file to search for.

            The next three inputs are other metadata objects.  If these objects already exist, performance will
            be improved by passing them in rather than recreating them.  If they do not exist, they will be
            created:
        
            madExpObj - a MadrigalExperiment object to get experiment metadata from.  If None (the default),
            a new MadrigalExperiment object is created.
            
            madInstObj - a MadrigalInstrument object to get experiment metadata from.  If None (the default),
            a new MadrigalInstrument object is created.
            
            madKindatObj - a MadrigalKindat object to get experiment metadata from.  If None (the default),
            a new MadrigalKindat object is created.
        
        Returns: A string summarizing the metadata about the file.  The format is::

                Start Date and Time: 01/06/1997  14:07  
                End Date and Time:   01/10/1997  23:45
                Instrument: Millstone Hill Incoherent Scatter Radar
                Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study
                Kind of data: INSCAL (8.0) Basic Derived Parameters
        
        Affects: None

        Exceptions: Thrown if any parsing error in metadata.
        """

        # build return String
        retStr = ''

        # create any needed metadata objects

        if madExpObj == None:
            madExpObj = MadrigalExperiment(self.__madDB)

        if madInstObj == None:
            madInstObj = MadrigalInstrument(self.__madDB)

        if madKindatObj == None:
            madKindatObj = MadrigalKindat(self.__madDB)

        # get the experiment id - if None, return message to update Metadata
        expId = self.getExpIdByFilename(filename)

        if expId == None:
            return '\tMetadata for file ' + str(filename) + \
                   ' not found.  Please notify Madrigal administrator that metadata ' + \
                   'needs to be updated.'

        # get the experiment start date - if not found, return message to update Metadata
        starttime = madExpObj.getExpStartDateTimeByExpId(expId)

        if starttime == None:
            return '\tMetadata for file ' + str(filename) + \
                   ' not found.  Please notify Madrigal administrator that metadata ' + \
                   'needs to be updated.'
        

        retStr = retStr + '\tExperiment start:  ' + time.asctime(starttime) + '\n'

        # endtime
        endtime = madExpObj.getExpEndDateTimeByExpId(expId)
        retStr = retStr + '\tExperiment end:    ' + time.asctime(endtime) + '\n'

        # get instrument name
        kinst = madExpObj.getKinstByExpId(expId)
        # if kinst not found, metadata is corrupt
        if kinst == None:
            return 'Could not find kinst for experiment id = ' + str(expId)
        instName = madInstObj.getInstrumentName(kinst)
        retStr = retStr + '\tInstrument name:   ' + str(instName) + '\n'

        # get experiment name
        expName = madExpObj.getExpNameByExpId(expId)
        retStr = retStr + '\tExperiment name:   ' + str(expName) + '\n'

        # get kind of data
        kindat = self.getKindatByFilename(filename)
        kindatName = madKindatObj.getKindatDescription(kindat, kinst)
        retStr = retStr + '\tKind of data:      ' + str(kindatName) + '\n'
        
        return retStr
    
    
    def getFileDOIUrlByPosition(self, position):
        """getFileDOIUrlByPosition returns the full url to the file on the archive site.
        
        Meant to be used as a referencable url, so always refers to cedar.openmadigal.org
        
        Returns permanent URL to file, or None if not found
        """
        
        # get expDir
        if self.__fileMetadataFile != self.__filename:
            # local copy
            expDir = os.path.dirname(self.__filename)
        else:
            expId = self.getExpIdByPosition(position)
            if expId is None:
                return(None)
            # need a MadrigalExperiment object
            madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)
            expDir = madExpObj.getExpDirByExpId(expId)
            if expDir is None:
                return(None)
        
        basename = self.getFilenameByPosition(position)
        if basename is None:
            return(None)
        
        # change so that it starts with experiments
        expDir = expDir[expDir.find('experiments'):]
        
        url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s'
        return(url_template % (expDir, basename))
        
        
    
    def getFileDOIUrlByFilename(self, filename):
        """getFileDOIUrlByFilename returns the full url to the file on the archive site.
        
        Meant to be used as a referencable url, so always refers to cedar.openmadigal.org
        
        Returns permanent URL to file, or None if not found
        """
        # get expDir
        if self.__fileMetadataFile != self.__filename:
            # local copy
            expDir = os.path.dirname(self.__filename)
        else:
            expId = self.getExpIdByFilename(filename)
            if expId is None:
                return(None)
            # need a MadrigalExperiment object
            madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)
            expDir = madExpObj.getExpDirByExpId(expId)
            if expDir is None:
                return(None)
        
        # change so that it starts with experiments
        expDir = expDir[expDir.find('experiments'):]
        
        url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s'
        return(url_template % (expDir, filename))
        
        



    def getLine(self, position):
        """getLine returns the line at a given position.  Returns None if position > number of lines.

        Inputs:  position - position in file.  First line = 0
        """
        delimiter = ','

        if position >= len(self.__fileList):
            return(None)

        return(delimiter.join(self.__fileList[position]) + '\n')
        

    def writeMetadata(self, newFullPath=None):
        """writeMetadata writes a new version of the fileTab.txt file.
        
        Inputs: newFullPath:  a new path to write the fileTab.txt file to, if
        not the same as the original metadata file opened.  Defaults to None, which overwrites
        metadata file that was read from.
        
        Returns: None.

        Affects: Writes updated version of metadata file.

        Exceptions: If unable to write file
        """

        # create string to hold file
        metaFileStr = ''
        delimiter = ','

        for lineList in self.__fileList:
            metaFileStr += delimiter.join(lineList) + '\n'

        # try to write file, if not, raise exception

        try:

            if newFullPath == None:
                if (len(os.path.dirname(self.__filename)) != 0):
                    newFullPath = self.__filename
                else:
                    newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename
            newFile = open(newFullPath, "w")
            newFile.write(metaFileStr)
            newFile.close()

        except:

            raise madrigal.admin.MadrigalError("Unable to write metadata file " + \
                                               str(newFullPath),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
        
    def __str__(self):
        """return possibly modified file as a string in same format as writeMetadata
        """
        # create string to hold file
        metaFileStr = ''
        delimiter = ','

        for lineList in self.__fileList:
            metaFileStr += delimiter.join(lineList) + '\n'
            
        return(metaFileStr)

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None)

init initializes MadrigalMetaFile by reading from fileTab.txt (or initFile).

Inputs: Existing MadrigalDB object, by default = None.

String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/fileTab.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.

def __init__(self, madDB=None, initFile=None):
    """__init__ initializes MadrigalMetaFile by reading from fileTab.txt (or initFile).
    Inputs: Existing MadrigalDB object, by default = None.
        String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/fileTab.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get file metadata file
    if (initFile == None):
        self.__filename = self.__fileMetadataFile
    else:
        self.__filename = initFile
    # MadrigalMetafile can now legal have two different lengths - with or without file analyst and email
    allowedLens = (self.__fileAccessCol+1, self.__fileAnalystEmailCol+1)
    
    genericObj = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens, key=self.__fileNameCol)
    self.__fileList = genericObj.getList()
    self._dict = genericObj.getDict()
    
    # check that at least first row has right number of columns
    if len(self.__fileList) > 0:
        # analyst and analyst email columns may or may not exist
        if len(self.__fileList[0]) not in allowedLens:
            raise madrigal.admin.MadrigalError('Error in count of first row of ' + str(self.__filename), None)

def deleteRowByFilename(

self, filename)

deleteRowByFilename deletes a row with a given filename.

Inputs: filename - name of file to search for.

Returns: None.

Affects: Removes item from self.__fileList if filename found

Exceptions: Thrown if filename not found.

def deleteRowByFilename(self, filename):
    """deleteRowByFilename deletes a row with a given filename.
    Inputs: filename - name of file to search for.
    
    Returns: None.
    
    Affects: Removes item from self.__fileList if filename found
    Exceptions: Thrown if filename not found.
    """
    
    for file in self.__fileList:
        if filename == file[self.__fileNameCol]:
            self.__fileList.remove(file)
            return
            
    # no matches found
    raise madrigal.admin.MadrigalError('Could not delete file ' + filename + ' from ' + self.__filename, None)

def getAccessByPosition(

self, position=0)

getAccessByPosition returns the access (0=public, 1=private) of the file at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the access level (0=public, 1=private) of the file at given position as an integer.
Returns None if position >= number of experiments.

Affects: None

Exceptions: Thrown if access column cannot be parsed into an integer

def getAccessByPosition(self, position = 0):
    """getAccessByPosition returns the access (0=public, 1=private) of the file at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the access level (0=public, 1=private) of the file at given position as an integer.  
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: Thrown if access column cannot be parsed into an integer
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__fileAccessCol])
        except:
            raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getAnalystByFilename(

self, filename)

getAnalystByFilename returns file analyst name if there, None otherwise.

Inputs: filename - name of file to search for.

Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist.

Affects: None

Exceptions: None

def getAnalystByFilename(self, filename):
    """getAnalystByFilename returns file analyst name if there, None otherwise.
    Inputs: filename - name of file to search for.
    
    Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, 
    returns None if the column does not exist.
    
    Affects: None
    Exceptions: None
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    try:
        return(self.__fileList[position][self.__fileAnalystCol])
    except:
        return(None)

def getAnalystByPosition(

self, position=0)

getAnalystByPosition returns file analyst name if there, None otherwise.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments.

Affects: None

Exceptions: None

def getAnalystByPosition(self, position = 0):
    """getAnalystByPosition returns file analyst name if there, None otherwise.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns:  file analyst name if there, None otherwise. Since not all fileTab.txt files may have this column, 
    returns None if the column does not exist.  Returns None if position >= number of experiments.  
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        try:
            return(self.__fileList[position][self.__fileAnalystCol])
        except:
            return(None)
    else:
        return(None)

def getAnalystEmailByFilename(

self, filename)

getAnalystEmailByFilename returns file analyst email if there, None otherwise.

Inputs: filename - name of file to search for.

Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist.

Affects: None

Exceptions: None

def getAnalystEmailByFilename(self, filename):
    """getAnalystEmailByFilename returns file analyst email if there, None otherwise.
    Inputs: filename - name of file to search for.
    
    Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, 
    returns None if the column does not exist.
    
    Affects: None
    Exceptions: None
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    try:
        return(self.__fileList[position][self.__fileAnalystEmailCol])
    except:
        return(None)

def getAnalystEmailByPosition(

self, position=0)

getAnalystEmailByPosition returns file analyst email if there, None otherwise.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, returns None if the column does not exist. Returns None if position >= number of experiments.

Affects: None

Exceptions: None

def getAnalystEmailByPosition(self, position = 0):
    """getAnalystEmailByPosition returns file analyst email if there, None otherwise.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns:  file analyst email if there, None otherwise. Since not all fileTab.txt files may have this column, 
    returns None if the column does not exist.  Returns None if position >= number of experiments.  
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        try:
            return(self.__fileList[position][self.__fileAnalystEmailCol])
        except:
            return(None)
    else:
        return(None)

def getCategoryByFilename(

self, filename)

getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename.

Inputs: filename - name of file to search for.

Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None.

Affects: None

Exceptions: Thrown if category column cannot be parsed into an integer

def getCategoryByFilename(self, filename):
    """getCategoryByFilename returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file with the given filename.
    Inputs: filename - name of file to search for.
    
    Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the first row found with the
    given filename.  Since
    filename may not be unique (although it usually is), the first match found is used.  If
    no matches found, returns None.
    
    Affects: None
    Exceptions: Thrown if category column cannot be parsed into an integer
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    
    return(int(self.__fileList[position][self.__fileCategoryCol]))

def getCategoryByPosition(

self, position=0)

getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer.
Returns None if position >= number of experiments.

Affects: None

Exceptions: Thrown if category column cannot be parsed into an integer

def getCategoryByPosition(self, position = 0):
    """getCategoryByPosition returns the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the category (eg., 1=default, 2=variant, 3=history, 4=real-time) of the file at given position as an integer.  
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: Thrown if category column cannot be parsed into an integer
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__fileCategoryCol])
        except:
            raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getExpIdByFilename(

self, filename)

getExpIdByFilename returns the first experiment id (integer) with the given filename.

Inputs: filename - name of file to search for.

Returns: the experiment id (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None.

Affects: None

Exceptions: Thrown if exp id cannot be parsed into an integer

def getExpIdByFilename(self, filename):
    """getExpIdByFilename returns the first experiment id (integer) with the given filename.
    Inputs: filename - name of file to search for.
    
    Returns: the experiment id (integer) of the first row found with the given filename.  Since
    filename may not be unique (although it usually is), the first match found is used.  If
    no matches found, returns None.
    
    Affects: None
    Exceptions: Thrown if exp id cannot be parsed into an integer
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    
    return(int(self.__fileList[position][self.__fileExpIdCol]))

def getExpIdByPosition(

self, position=0)

getExpIdByPosition returns the experiment id (integer) of the file at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the experiment id (integer) of the file at given position as an integer.
Returns None if position >= number of experiments.

Affects: None

Exceptions: Thrown if kinst exp id cannot be parsed into an integer

def getExpIdByPosition(self, position = 0):
    """getExpIdByPosition returns the experiment id (integer) of the file at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the experiment id (integer) of the file at given position as an integer.  
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: Thrown if kinst exp id cannot be parsed into an integer
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__fileExpIdCol])
        except:
            raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getFileCount(

self)

getFileCount returns the number of files (rows) in the metadata file.

Inputs: None

Returns: the number of files (rows) in the metadata file.

Affects: None

Exceptions: None

def getFileCount(self):
    """getFileCount returns the number of files (rows) in the metadata file.
    Inputs: None
    
    Returns: the number of files (rows) in the metadata file.
    Affects: None
    Exceptions: None
    """
    return len(self.__fileList)

def getFileDOIUrlByFilename(

self, filename)

getFileDOIUrlByFilename returns the full url to the file on the archive site.

Meant to be used as a referencable url, so always refers to cedar.openmadigal.org

Returns permanent URL to file, or None if not found

def getFileDOIUrlByFilename(self, filename):
    """getFileDOIUrlByFilename returns the full url to the file on the archive site.
    
    Meant to be used as a referencable url, so always refers to cedar.openmadigal.org
    
    Returns permanent URL to file, or None if not found
    """
    # get expDir
    if self.__fileMetadataFile != self.__filename:
        # local copy
        expDir = os.path.dirname(self.__filename)
    else:
        expId = self.getExpIdByFilename(filename)
        if expId is None:
            return(None)
        # need a MadrigalExperiment object
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)
        expDir = madExpObj.getExpDirByExpId(expId)
        if expDir is None:
            return(None)
    
    # change so that it starts with experiments
    expDir = expDir[expDir.find('experiments'):]
    
    url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s'
    return(url_template % (expDir, filename))

def getFileDOIUrlByPosition(

self, position)

getFileDOIUrlByPosition returns the full url to the file on the archive site.

Meant to be used as a referencable url, so always refers to cedar.openmadigal.org

Returns permanent URL to file, or None if not found

def getFileDOIUrlByPosition(self, position):
    """getFileDOIUrlByPosition returns the full url to the file on the archive site.
    
    Meant to be used as a referencable url, so always refers to cedar.openmadigal.org
    
    Returns permanent URL to file, or None if not found
    """
    
    # get expDir
    if self.__fileMetadataFile != self.__filename:
        # local copy
        expDir = os.path.dirname(self.__filename)
    else:
        expId = self.getExpIdByPosition(position)
        if expId is None:
            return(None)
        # need a MadrigalExperiment object
        madExpObj = madrigal.metadata.MadrigalExperiment(self.__madDB)
        expDir = madExpObj.getExpDirByExpId(expId)
        if expDir is None:
            return(None)
    
    basename = self.getFilenameByPosition(position)
    if basename is None:
        return(None)
    
    # change so that it starts with experiments
    expDir = expDir[expDir.find('experiments'):]
    
    url_template = 'https://w3id.org/cedar?experiment_list=%s&file_list=%s'
    return(url_template % (expDir, basename))

def getFileDatetimeByFilename(

self, filename)

getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set.

Inputs: filename - name of file to search for.

Returns: a datetime of the file with the given filenme, or None if not set. Returns none if name not found

Affects: None

Exceptions: Thrown if filename not found.

def getFileDatetimeByFilename(self,filename):
    """getFileDatetimeByFilename returns a datetime of the file with the given filenme, or None if not set.
    Inputs: filename - name of file to search for.
    
    Returns: a datetime of the file with the given filenme, or None if not set.  Returns none if name
    not found
    
    Affects: None
    Exceptions: Thrown if filename not found.
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    
    return(self.getFileDatetimeByPosition(position))

def getFileDatetimeByPosition(

self, position=0)

getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: a datetime of the file at given position, or None if not set.
Returns None if position >= number of experiments.

Affects: None

Exceptions: None

def getFileDatetimeByPosition(self, position = 0):
    """getFileDatetimeByPosition returns a datetime of the file at given position, or None if not set.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: a datetime of the file at given position, or None if not set.  
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        dateStr = self.__fileList[position][self.__fileModDateCol]
        if len(dateStr) < 8:
            return(None)
        timeStr = self.__fileList[position][self.__fileModTimeCol]
        if len(timeStr) < 6:
            return(None)
        
        return(datetime.datetime.strptime('%s %s' % (dateStr, timeStr), '%Y%m%d %H%M%S'))
    else:
        return(None)

def getFilenameByPosition(

self, position=0)

getFilenameByPosition returns the filename of the file at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the filename of the file at given position as a string.
Returns None if position >= number of experiments.

Affects: None

Exceptions: None

def getFilenameByPosition(self, position = 0):
    """getFilenameByPosition returns the filename of the file at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the filename of the file at given position as a string.  
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        return self.__fileList[position][self.__fileNameCol]
    else:
        return(None)

def getHasCatalogByFilename(

self, filename)

getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise.

Inputs: filename - name of file to search for.

Returns: true if the file with the given name has any catalog records, False otherwise. Returns none if name not found

Affects: None

Exceptions: None.

def getHasCatalogByFilename(self,filename):
    """getHasCatalogByFilename returns true if the file with the given name has any catalog records, False otherwise.
    Inputs: filename - name of file to search for.
    
    Returns: true if the file with the given name has any catalog records, False otherwise.  Returns none if name
    not found
    
    Affects: None
    Exceptions: None.
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    
    return(int(self.__fileList[position][self.__fileHasCatalogCol]))

def getHasCatalogByPosition(

self, position=0)

getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: True if the file at given position has any catalog records, False otherwise Returns None if position >= number of experiments.

Affects: None

Exceptions: None

def getHasCatalogByPosition(self, position = 0):
    """getHasCatalogByPosition returns True if the file at given position has any catalog records, False otherwise.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns:  True if the file at given position has any catalog records, False otherwise 
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        if int(self.__fileList[position][self.__fileHasCatalogCol]) == 0:
            return False
        else:
            return True
    else:
        return(None)

def getHasHeaderByFilename(

self, filename)

getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise.

Inputs: filename - name of file to search for.

Returns: true if the file with the given name has any header records, False otherwise. Returns none if name not found

Affects: None

Exceptions: Thrown if filename not found.

def getHasHeaderByFilename(self,filename):
    """getHasHeaderByFilename returns true if the file with the given name has any header records, False otherwise.
    Inputs: filename - name of file to search for.
    
    Returns: true if the file with the given name has any header records, False otherwise.  Returns none if name
    not found
    
    Affects: None
    Exceptions: Thrown if filename not found.
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    
    return(int(self.__fileList[position][self.__fileHasHeaderCol]))

def getHasHeaderByPosition(

self, position=0)

getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: True if the file at given position has any header records, False otherwise Returns None if position >= number of experiments.

Affects: None

Exceptions: None

def getHasHeaderByPosition(self, position = 0):
    """getHasHeaderByPosition returns True if the file at given position has any header records, False otherwise.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns:  True if the file at given position has any header records, False otherwise 
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        if int(self.__fileList[position][self.__fileHasHeaderCol]) == 0:
            return False
        else:
            return True
    else:
        return(None)

def getKindatByFilename(

self, filename)

getKindatByFilename returns the first kindat (integer) with the given filename.

Inputs: filename - name of file to search for.

Returns: the kindat (integer) of the first row found with the given filename. Since filename may not be unique (although it usually is), the first match found is used. If no matches found, returns None.

Affects: None

Exceptions: Thrown if kindat cannot be parsed into an integer

def getKindatByFilename(self, filename):
    """getKindatByFilename returns the first kindat (integer) with the given filename.
    Inputs: filename - name of file to search for.
    
    Returns: the kindat (integer) of the first row found with the given filename.  Since
    filename may not be unique (although it usually is), the first match found is used.  If
    no matches found, returns None.
    
    Affects: None
    Exceptions: Thrown if kindat cannot be parsed into an integer
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    
    return(int(self.__fileList[position][self.__fileKindatCol]))

def getKindatByPosition(

self, position=0)

getKindatByPosition returns the kindat (kind of data code) of the file at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the kinst (kind of instrument code) of the file at given position as an integer.
Returns None if position >= number of experiments.

Affects: None

Exceptions: Thrown if kinst column cannot be parsed into an integer

def getKindatByPosition(self, position = 0):
    """getKindatByPosition returns the kindat (kind of data code) of the file at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the kinst (kind of instrument code) of the file at given position as an integer.  
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: Thrown if kinst column cannot be parsed into an integer
    """
    if len(self.__fileList) > position:
        try:
            return int(self.__fileList[position][self.__fileKindatCol])
        except:
            raise madrigal.admin.MadrigalError('Error in fileTab.txt parsing metadata row: ' + str(position),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    else:
        return(None)

def getLine(

self, position)

getLine returns the line at a given position. Returns None if position > number of lines.

Inputs: position - position in file. First line = 0

def getLine(self, position):
    """getLine returns the line at a given position.  Returns None if position > number of lines.
    Inputs:  position - position in file.  First line = 0
    """
    delimiter = ','
    if position >= len(self.__fileList):
        return(None)
    return(delimiter.join(self.__fileList[position]) + '\n')

def getMetadataSummaryByFilename(

self, filename, madExpObj=None, madInstObj=None, madKindatObj=None)

getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename.

Inputs: filename - name of file to search for.

The next three inputs are other metadata objects.  If these objects already exist, performance will
be improved by passing them in rather than recreating them.  If they do not exist, they will be
created:

madExpObj - a MadrigalExperiment object to get experiment metadata from.  If None (the default),
a new MadrigalExperiment object is created.

madInstObj - a MadrigalInstrument object to get experiment metadata from.  If None (the default),
a new MadrigalInstrument object is created.

madKindatObj - a MadrigalKindat object to get experiment metadata from.  If None (the default),
a new MadrigalKindat object is created.

Returns: A string summarizing the metadata about the file. The format is::

    Start Date and Time: 01/06/1997  14:07  
    End Date and Time:   01/10/1997  23:45
    Instrument: Millstone Hill Incoherent Scatter Radar
    Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study
    Kind of data: INSCAL (8.0) Basic Derived Parameters

Affects: None

Exceptions: Thrown if any parsing error in metadata.

def getMetadataSummaryByFilename(self, filename, madExpObj = None, madInstObj = None, madKindatObj = None):
    """getMetadataSummaryByFilename returns a string summarizing the file's metadata given a filename.
    Inputs: filename - name of file to search for.
        The next three inputs are other metadata objects.  If these objects already exist, performance will
        be improved by passing them in rather than recreating them.  If they do not exist, they will be
        created:
    
        madExpObj - a MadrigalExperiment object to get experiment metadata from.  If None (the default),
        a new MadrigalExperiment object is created.
        
        madInstObj - a MadrigalInstrument object to get experiment metadata from.  If None (the default),
        a new MadrigalInstrument object is created.
        
        madKindatObj - a MadrigalKindat object to get experiment metadata from.  If None (the default),
        a new MadrigalKindat object is created.
    
    Returns: A string summarizing the metadata about the file.  The format is::
            Start Date and Time: 01/06/1997  14:07  
            End Date and Time:   01/10/1997  23:45
            Instrument: Millstone Hill Incoherent Scatter Radar
            Experiment name: World Day - Mesosphere/Lower-Thermosphere Coupling Study
            Kind of data: INSCAL (8.0) Basic Derived Parameters
    
    Affects: None
    Exceptions: Thrown if any parsing error in metadata.
    """
    # build return String
    retStr = ''
    # create any needed metadata objects
    if madExpObj == None:
        madExpObj = MadrigalExperiment(self.__madDB)
    if madInstObj == None:
        madInstObj = MadrigalInstrument(self.__madDB)
    if madKindatObj == None:
        madKindatObj = MadrigalKindat(self.__madDB)
    # get the experiment id - if None, return message to update Metadata
    expId = self.getExpIdByFilename(filename)
    if expId == None:
        return '\tMetadata for file ' + str(filename) + \
               ' not found.  Please notify Madrigal administrator that metadata ' + \
               'needs to be updated.'
    # get the experiment start date - if not found, return message to update Metadata
    starttime = madExpObj.getExpStartDateTimeByExpId(expId)
    if starttime == None:
        return '\tMetadata for file ' + str(filename) + \
               ' not found.  Please notify Madrigal administrator that metadata ' + \
               'needs to be updated.'
    
    retStr = retStr + '\tExperiment start:  ' + time.asctime(starttime) + '\n'
    # endtime
    endtime = madExpObj.getExpEndDateTimeByExpId(expId)
    retStr = retStr + '\tExperiment end:    ' + time.asctime(endtime) + '\n'
    # get instrument name
    kinst = madExpObj.getKinstByExpId(expId)
    # if kinst not found, metadata is corrupt
    if kinst == None:
        return 'Could not find kinst for experiment id = ' + str(expId)
    instName = madInstObj.getInstrumentName(kinst)
    retStr = retStr + '\tInstrument name:   ' + str(instName) + '\n'
    # get experiment name
    expName = madExpObj.getExpNameByExpId(expId)
    retStr = retStr + '\tExperiment name:   ' + str(expName) + '\n'
    # get kind of data
    kindat = self.getKindatByFilename(filename)
    kindatName = madKindatObj.getKindatDescription(kindat, kinst)
    retStr = retStr + '\tKind of data:      ' + str(kindatName) + '\n'
    
    return retStr

def getStatusByFilename(

self, filename)

getStatusByFilename returns the status description of the file with the given name.

Inputs: filename - name of file to search for.

Returns: the status description of the file with the given name. Returns none if name not found

Affects: None

Exceptions: Thrown if filename not found.

def getStatusByFilename(self,filename):
    """getStatusByFilename returns the status description of the file with the given name.
    Inputs: filename - name of file to search for.
    
    Returns: the status description of the file with the given name.  Returns none if name
    not found
    
    Affects: None
    Exceptions: Thrown if filename not found.
    """
    filename = os.path.basename(filename)
    try:
        position = self._dict[filename]
    except KeyError:
        return(None)
    
    return(self.__fileList[position][self.__fileStatusCol])

def getStatusByPosition(

self, position=0)

getStatusByPosition returns the status description of the file at given position.

Inputs: position of file in list (first position is zero). Defaults to first.

Returns: the status description of the file at given position as a string.
Returns None if position >= number of experiments.

Affects: None

Exceptions: None

def getStatusByPosition(self, position = 0):
    """getStatusByPosition returns the status description of the file at given position.
    Inputs: position of file in list (first position is zero).  Defaults to first.
    
    Returns: the status description of the file at given position as a string.  
    Returns None if position >= number of experiments.
    
    Affects: None
    Exceptions: None
    """
    if len(self.__fileList) > position:
        return self.__fileList[position][self.__fileStatusCol]
    else:
        return(None)

def setAccess(

self, accessType)

setAccess sets the access column to all 0's (public) or all 1's (private).

Inputs: accessType - either 0 to set to public access, or 1 to set to private access.

Returns: None.

Affects: Overwrite fileTab.txt file with access column set to all 0's (public) or all 1's (private).

Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1

def setAccess(self, accessType):
    """setAccess sets the access column to all 0's (public) or all 1's (private).
    Inputs: accessType - either 0 to set to public access, or 1 to set to private access.
    
    Returns: None.
    
    Affects: Overwrite fileTab.txt file with access column set to all 0's (public)
    or all 1's (private).
    Exceptions: Thrown if file cannot be written, if accessType is not 0 or 1
    """
    if (accessType != 0 and accessType != 1):
        raise madrigal.admin.MadrigalError('MadrigalMetaFile.setAccess called with arg = ' + \
                                           str(accessType) + ', must be either 0 or 1', None)
    for row in self.__fileList:
        row[self.__fileAccessCol] = str(accessType)
    self.writeMetadata()

def setAccessByPosition(

self, position, access)

setAccessByPosition sets the value of access for the file at the given position.

Inputs:

position - position of file in list (first position is zero).

access - 0 of False for public, 1 or True for private

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setAccessByPosition(self, position, access):
    """setAccessByPosition sets the value of access for the file at the given position.
    Inputs:
        position - position of file in list (first position is zero).
        access - 0 of False for public, 1 or True for private
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    if access not in (0, 1, True, False):
        raise ValueError('Illegal value for access in setAccessByPosition: %s' % (str(access)))
    if len(self.__fileList) > position:
        if access in (1, True):
            self.__fileList[position][self.__fileAccessCol] = '1'
        else:
            self.__fileList[position][self.__fileAccessCol] = '0'
    else:
        raise ValueError('setAccessByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setAnalystByPosition(

self, position, analyst)

setAnalystByPosition sets the file analyst name for the given position

Inputs:

position - position of file in list (first position is zero).

analyst - name of file analyst

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setAnalystByPosition(self, position, analyst):
    """setAnalystByPosition sets the file analyst name for the given position
    Inputs:
        position - position of file in list (first position is zero).
        analyst - name of file analyst
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    # verify no illegal commas
    if analyst.find(',') != -1:
        raise madrigal.admin.MadrigalError('Error in setAnalystByPosition with args %s: %s' %  \
                                           (str(position), analyst),
                                            [traceback.format_exc()])
    if len(self.__fileList) > position:
        if len(self.__fileList[position]) == self.__fileAccessCol+1:
            # append two extra columns
            self.__fileList[position].append(str(analyst))
            self.__fileList[position].append('')
        else:
            self.__fileList[position][self.__fileAnalystCol] = str(analyst)
    else:
        raise ValueError('setAnalystByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setAnalystEmailByPosition(

self, position, analystEmail)

setAnalystEmailByPosition sets the file analyst email for the given position

Inputs:

position - position of file in list (first position is zero).

analystEmail - email of file analyst

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setAnalystEmailByPosition(self, position, analystEmail):
    """setAnalystEmailByPosition sets the file analyst email for the given position
    Inputs:
        position - position of file in list (first position is zero).
        analystEmail - email of file analyst
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    # verify no illegal commas
    if analystEmail.find(',') != -1:
        raise madrigal.admin.MadrigalError('Error in setAnalystEmailByPosition with args %s: %s' %  \
                                           (str(position, analystEmail),
                                            [traceback.format_exc()]))
    if len(self.__fileList) > position:
        if len(self.__fileList[position]) == self.__fileAccessCol + 1:
            # append two extra columns
            self.__fileList[position].append('')
            self.__fileList[position].append(str(analystEmail))
        else:
            self.__fileList[position][self.__fileAnalystEmailCol] = str(analystEmail)
    else:
        raise ValueError('setAnalystEmailByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setCategoryByPosition(

self, position, category)

setCategoryByPosition sets the value of category for the file at the given position.

Inputs:

position - position of file in list (first position is zero).

category - 1=default, 2=variant, 3=history, 4=real-time

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setCategoryByPosition(self, position, category):
    """setCategoryByPosition sets the value of category for the file at the given position.
    Inputs:
        position - position of file in list (first position is zero).
        category - 1=default, 2=variant, 3=history, 4=real-time
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    if int(category) not in (1, 2, 3, 4):
        raise ValueError('Illegal value for category in setCategoryByPosition: %s' % (str(category)))
    if len(self.__fileList) > position:
        self.__fileList[position][self.__fileCategoryCol] = str(category)
    else:
        raise ValueError('setCategoryByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setExpIdByPosition(

self, position, expId)

setExpIdByPosition sets the experiment id of the file at given position.

Inputs:

position - position of file in list (first position is zero).

expId - the new experiment id to use

Returns: None.

Affects: sets the experiment id of the file at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or position not found.

def setExpIdByPosition(self, position, expId):
    """setExpIdByPosition sets the experiment id of the file at given position.
    Inputs:
        position - position of file in list (first position is zero).
        expId - the new experiment id to use
    
    Returns: None.
    Affects: sets the experiment id of the file at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    position not found.
    """
    
    try:
        self.__fileList[position][self.__fileExpIdCol] = str(int(expId))
    except:
        raise madrigal.admin.MadrigalError('Error in setExpIdByPosition with args %s: %s' %  \
                                           (str(position), str(expId)),
                                            traceback.format_exception(sys.exc_info()[0],
                                                                       sys.exc_info()[1],
                                                                       sys.exc_info()[2]))

def setFileDatetimeByPosition(

self, position, dt)

setFileDatetimeByPosition sets the file datetime for the given position

Inputs:

position - position of file in list (first position is zero).

dt - datetime (UT) of file being added.  If None, use file datetime.
    Will raise exception if dt is None and fileTab.txt in the metadata
    version, not the expDir version.

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setFileDatetimeByPosition(self, position, dt):
    """setFileDatetimeByPosition sets the file datetime for the given position
    Inputs:
        position - position of file in list (first position is zero).
        dt - datetime (UT) of file being added.  If None, use file datetime.
            Will raise exception if dt is None and fileTab.txt in the metadata
            version, not the expDir version.
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    if dt is None:
        basename = self.getFilenameByPosition(position)
        expDir = os.path.dirname(self.__filename)
        if len(expDir) < 2:
            raise IOError('Cannot set datetime to None using the metadata version of fileTab.txt')
        dataFile = os.path.join(expDir, basename)
        if not os.access(dataFile, os.R_OK):
            raise IOError('Cannot access %s' % (dataFile))
        dt = datetime.datetime.utcfromtimestamp(os.path.getmtime(dataFile))
        
        
    dateStr = dt.strftime('%Y%m%d')
    timeStr = dt.strftime('%H%M%S')
    if len(self.__fileList) > position:
        self.__fileList[position][self.__fileModDateCol] = dateStr
        self.__fileList[position][self.__fileModTimeCol] = timeStr
    else:
        raise ValueError('setFileDatetimeByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setHasCatalogByPosition(

self, position, hasCatalog)

setHasCatalogByPosition sets the value of hasCatalog for the file at the given position.

Inputs:

position - position of file in list (first position is zero).

hasCatalog - 1 or True for yes, 0 or False for no

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setHasCatalogByPosition(self, position, hasCatalog):
    """setHasCatalogByPosition sets the value of hasCatalog for the file at the given position.
    Inputs:
        position - position of file in list (first position is zero).
        hasCatalog - 1 or True for yes, 0 or False for no
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    if hasCatalog not in (0, 1, True, False):
        raise ValueError('Illegal value for hasCatalog in setHasCatalogByPosition: %s' % (str(hasCatalog)))
    if len(self.__fileList) > position:
        if hasCatalog in (1, True):
            self.__fileList[position][self.__fileHasCatalogCol] = '1'
        else:
            self.__fileList[position][self.__fileHasCatalogCol] = '0'
    else:
        raise ValueError('setHasCatalogByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setHasHeaderByPosition(

self, position, hasHeader)

setHasHeaderByPosition sets the value of hasHeader for the file at the given position.

Inputs:

position - position of file in list (first position is zero).

hasHeader - 1 or True for yes, 0 or False for no

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setHasHeaderByPosition(self, position, hasHeader):
    """setHasHeaderByPosition sets the value of hasHeader for the file at the given position.
    Inputs:
        position - position of file in list (first position is zero).
        hasHeader - 1 or True for yes, 0 or False for no
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    if hasHeader not in (0, 1, True, False):
        raise ValueError('Illegal value for hasHeader in setHasHeaderByPosition: %s' % (str(hasHeader)))
    if len(self.__fileList) > position:
        if hasHeader in (1, True):
            self.__fileList[position][self.__fileHasHeaderCol] = '1'
        else:
            self.__fileList[position][self.__fileHasHeaderCol] = '0'
    else:
        raise ValueError('setHasHeaderByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setKindatByPosition(

self, position, kindat)

setKindatByPosition sets the value of kindat for the file at the given position.

Inputs:

position - position of file in list (first position is zero).

kindat - integer kindat value

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setKindatByPosition(self, position, kindat):
    """setKindatByPosition sets the value of kindat for the file at the given position.
    Inputs:
        position - position of file in list (first position is zero).
        kindat - integer kindat value
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    kindat = int(kindat)
    
    if len(self.__fileList) > position:
        self.__fileList[position][self.__fileKindatCol] = str(kindat)
    else:
        raise ValueError('setKindatByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def setStatusByPosition(

self, position, status)

setStatusByPosition sets the value of status string for the file at the given position.

Inputs:

position - position of file in list (first position is zero).

status - string describing status

Returns: None.

Affects: None

Exceptions: If position beyond length.

def setStatusByPosition(self, position, status):
    """setStatusByPosition sets the value of status string for the file at the given position.
    Inputs:
        position - position of file in list (first position is zero).
        status - string describing status
    
    Returns:  None.
    
    Affects: None
    Exceptions: If position beyond length.
    """
    statusTypes = [bytes, str]
    if type(status) not in statusTypes:
        raise ValueError('Illegal value for status in setStatusByPosition: %s' % (str(status)))
    # check that string does not illegally contain a comma
    if status.find(',') != -1:
        raise ValueError('status string in fileTab.txt cannot contain a comma: <%s> is illegal' % (status))
    if len(self.__fileList) > position:
        self.__fileList[position][self.__fileStatusCol] = status
    else:
        raise ValueError('setStatusByPosition called for position %i beyond length %i' % (position, len(self.__fileList)))

def writeMetadata(

self, newFullPath=None)

writeMetadata writes a new version of the fileTab.txt file.

Inputs: newFullPath: a new path to write the fileTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from.

Returns: None.

Affects: Writes updated version of metadata file.

Exceptions: If unable to write file

def writeMetadata(self, newFullPath=None):
    """writeMetadata writes a new version of the fileTab.txt file.
    
    Inputs: newFullPath:  a new path to write the fileTab.txt file to, if
    not the same as the original metadata file opened.  Defaults to None, which overwrites
    metadata file that was read from.
    
    Returns: None.
    Affects: Writes updated version of metadata file.
    Exceptions: If unable to write file
    """
    # create string to hold file
    metaFileStr = ''
    delimiter = ','
    for lineList in self.__fileList:
        metaFileStr += delimiter.join(lineList) + '\n'
    # try to write file, if not, raise exception
    try:
        if newFullPath == None:
            if (len(os.path.dirname(self.__filename)) != 0):
                newFullPath = self.__filename
            else:
                newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename
        newFile = open(newFullPath, "w")
        newFile.write(metaFileStr)
        newFile.close()
    except:
        raise madrigal.admin.MadrigalError("Unable to write metadata file " + \
                                           str(newFullPath),
                                           traceback.format_exception(sys.exc_info()[0],
                                                                      sys.exc_info()[1],
                                                                      sys.exc_info()[2]))

class MadrigalMetadata

MadrigalMetadata is a private class that parses a Madrigal metadata file.

This private class is used by all classes that need to parse a Madrigal metadata file. If the class is called with the name of the metadata file only, the metadata file is assumed to be at $MAD_ROOT/metadata. If a full path name is given that includes a directory separator, then that is used instead. The getList method returns a list with one item for each line in the file. That item for each line is simply a list of strings found in the line. The following is an example metadata file and the list the method getList would return.

Metadata file example::

Tom, Dick,,Harry
,,Joe,
Sally, Jane,Joe, Dick

The list returned by getList example::

[['Tom', 'Dick', '', 'Harry'],
['', '', 'Joe', ''],
['Sally', 'Jane', 'Joe', 'Dick']]

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. Unable to open metadata file.

2. All lines in metadata file do not have same number of items

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 8, 2001

class MadrigalMetadata:
    """MadrigalMetadata is a private class that parses a Madrigal metadata file.


    This private class is used by all classes that need to parse a Madrigal
    metadata file.  If the class is called with the name of the metadata file only,
    the metadata file is assumed to be at $MAD_ROOT/metadata.  If a full path name is
    given that includes a directory separator, then that is used instead.  The getList
    method returns a list with one item for each line in the file.  That item for each
    line is simply a list of strings found in the line.  The following is an example
    metadata file and the list the method getList would return.

    Metadata file example::

        Tom, Dick,,Harry
        ,,Joe,
        Sally, Jane,Joe, Dick

    The list returned by getList example::

        [['Tom', 'Dick', '', 'Harry'],
        ['', '', 'Joe', ''],
        ['Sally', 'Jane', 'Joe', 'Dick']]

        
    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. Unable to open metadata file.
	
        2. All lines in metadata file do not have same number of items

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Nov. 8, 2001

    """

    # constants

    # file delimiter - presently a comma
    __DELIMITER = ','
    

    def __init__(self, metadataFileName, madDB=None, allowedLenList=None, key=None):
        """__init__ initializes the MadrigalCategoryList by reading from __privateList.

        Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir
            is included.

            Existing MadrigalDB object, by default = None.
            
            allowedLenList - a list of allowed lengths (integers) for data lines.  If None (the default),
            then the rule is that all lengths must be the same.
            
            key - which position in the file to use as a key.  Creates a attribute called self._dict,
            which is a dictionary with keys = item in key position, value = line index.  Used for
            faster lookup by key.  If None (the default), self._dict = None
        
        Returns: void

        Affects: Initializes private member variable __categoryList.

        Exceptions: None.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB
        
        # create empty list to hold parsed data
        self.__fileList = []

        # used to check that every line has same number of words if allowedLenList == None
        self.__numWords = 0
        self._allowedLenList = allowedLenList
        
        if not key is None:
            self._dict = {}
        else:self._dict = None
        
        # get real filename 
        self.__fileName = self.__getFullPath(metadataFileName)

        # open configuration file
        try:
            self.__file = open(self.__fileName, "r")
            
        except IOError:
            raise madrigal.admin.MadrigalError("Unable to open metadata file " + self.__fileName,
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
                                                       

        # loop over each line in file, creating a list for each
        line = self.__file.readline()
        
        count = 0
        while (len(line.strip())):
            self.__parseLine(line, key, count)
            line = self.__file.readline()
            count += 1

        # close metadata file
        self.__file.close()
            

    def __parseLine(self, line, key=None, count=None):
        """__parseLine adds a list of items in line to __fileList.

        Inputs: 
            Line of file to parse (String).
            key - position to add key for self._dict, value = line position
                If None (the default), ignore
            count - if None, ignore.  Else add to self._dict as value (see key above)
        
        
        Returns: void

        Affects: Adds a list of items in line to self.__fileList.

        Exceptions: None.
        """
        list = line.split(self.__DELIMITER)

        # create new list with leading and trailing whitespace stripped

        strippedList = []
        
        for word in list:
            strippedList.append(word.strip())

        self.__fileList.append(strippedList)
        
        if not self._dict is None:
            self._dict[list[key]] = count

        # check correct number of words
        if self._allowedLenList:
            if len(strippedList) not in self._allowedLenList:
                raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \
                                                   self.__fileName + " at line " + \
                                                   str(len(self.__fileList)),
                                                   [traceback.format_exc()])
        elif self.__numWords == 0:
            self.__numWords = len(strippedList)
            
        elif self.__numWords != len(strippedList):
            raise madrigal.admin.MadrigalError("Wrong number of items found in metadata file " + \
                                                       self.__fileName + " at line " + \
                                                       str(len(self.__fileList)),
                                                       [traceback.format_exc()])

    def __getFullPath(self, filename):
        """getFullPath returns the full path name of the metafile.
        
        Inputs: filename passed in as argument
        
        Returns: The full path name of the metafile.  If the filename argument already is
        a full path, filename is returned unchanged.  If the filename is simply the name
        of a metadata file, then $MAD_ROOT/metadata is appended

        Affects: Nothing

        Exceptions: None
        """

        if (len(os.path.dirname(filename)) != 0):
            return filename

        fullName = self.__madDB.getMetadataDir() + "/" + filename

        # normalize in case we run on a system without / as a separator

        return os.path.normpath(fullName)


    
    # public methods

    def getList(self):
        """getList returns the list of lists of items in each line in the metafile.
        
        Inputs: None
        
        Returns: The list of lists of items in each line in the metafile.  That is, each
        item in the returned list is itself a list, representing a single line in the
        metafile. That single line's list is the list of items in that line.

        Affects: Nothing

        Exceptions: None
        """

        return(self.__fileList)
    
    
    def getDict(self):
        """getDict returns self_dict, which will be a dict with keys= key column set, value = line number,
        of None if no key passed into constructor
        """
        return(self._dict)
    

    def toString(self):
        """toString returns a simple string representation of a MadrigalMetadata object.

        Inputs: None
        
        Returns: String describing a simple representation of a __MadrigalMetadat object.

        Affects: Nothing

        Exceptions: None
        """

        return str(self.__fileList)

Ancestors (in MRO)

Static methods

def __init__(

self, metadataFileName, madDB=None, allowedLenList=None, key=None)

init initializes the MadrigalCategoryList by reading from __privateList.

Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir is included.

Existing MadrigalDB object, by default = None.

allowedLenList - a list of allowed lengths (integers) for data lines.  If None (the default),
then the rule is that all lengths must be the same.

key - which position in the file to use as a key.  Creates a attribute called self._dict,
which is a dictionary with keys = item in key position, value = line index.  Used for
faster lookup by key.  If None (the default), self._dict = None

Returns: void

Affects: Initializes private member variable __categoryList.

Exceptions: None.

def __init__(self, metadataFileName, madDB=None, allowedLenList=None, key=None):
    """__init__ initializes the MadrigalCategoryList by reading from __privateList.
    Inputs: String metadataFileName - if not a full path, then MadrigalDB.getMetadataDir
        is included.
        Existing MadrigalDB object, by default = None.
        
        allowedLenList - a list of allowed lengths (integers) for data lines.  If None (the default),
        then the rule is that all lengths must be the same.
        
        key - which position in the file to use as a key.  Creates a attribute called self._dict,
        which is a dictionary with keys = item in key position, value = line index.  Used for
        faster lookup by key.  If None (the default), self._dict = None
    
    Returns: void
    Affects: Initializes private member variable __categoryList.
    Exceptions: None.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    
    # create empty list to hold parsed data
    self.__fileList = []
    # used to check that every line has same number of words if allowedLenList == None
    self.__numWords = 0
    self._allowedLenList = allowedLenList
    
    if not key is None:
        self._dict = {}
    else:self._dict = None
    
    # get real filename 
    self.__fileName = self.__getFullPath(metadataFileName)
    # open configuration file
    try:
        self.__file = open(self.__fileName, "r")
        
    except IOError:
        raise madrigal.admin.MadrigalError("Unable to open metadata file " + self.__fileName,
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
                                                   
    # loop over each line in file, creating a list for each
    line = self.__file.readline()
    
    count = 0
    while (len(line.strip())):
        self.__parseLine(line, key, count)
        line = self.__file.readline()
        count += 1
    # close metadata file
    self.__file.close()

def getDict(

self)

getDict returns self_dict, which will be a dict with keys= key column set, value = line number, of None if no key passed into constructor

def getDict(self):
    """getDict returns self_dict, which will be a dict with keys= key column set, value = line number,
    of None if no key passed into constructor
    """
    return(self._dict)

def getList(

self)

getList returns the list of lists of items in each line in the metafile.

Inputs: None

Returns: The list of lists of items in each line in the metafile. That is, each item in the returned list is itself a list, representing a single line in the metafile. That single line's list is the list of items in that line.

Affects: Nothing

Exceptions: None

def getList(self):
    """getList returns the list of lists of items in each line in the metafile.
    
    Inputs: None
    
    Returns: The list of lists of items in each line in the metafile.  That is, each
    item in the returned list is itself a list, representing a single line in the
    metafile. That single line's list is the list of items in that line.
    Affects: Nothing
    Exceptions: None
    """
    return(self.__fileList)

def toString(

self)

toString returns a simple string representation of a MadrigalMetadata object.

Inputs: None

Returns: String describing a simple representation of a __MadrigalMetadat object.

Affects: Nothing

Exceptions: None

def toString(self):
    """toString returns a simple string representation of a MadrigalMetadata object.
    Inputs: None
    
    Returns: String describing a simple representation of a __MadrigalMetadat object.
    Affects: Nothing
    Exceptions: None
    """
    return str(self.__fileList)

class MadrigalParmCategory

MadrigalParmCategory is an object that provides access to Madrigal parameter category info from the metadata.

This object provides access to all Madrigal kind of data information in the metadata file madCatTab.txt.

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

2. Columns expected to be ints or floats cannot be converted

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Jan. 18, 2013

class MadrigalParmCategory:
    """MadrigalParmCategory is an object that provides access to Madrigal parameter category info from the metadata.

    This object provides access to all Madrigal kind of data information in the metadata file madCatTab.txt.

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file
        
        2. Columns expected to be ints or floats cannot be converted

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Jan. 18, 2013

    """

    #constants
    _categoryMetadataFile  = "madCatTab.txt"

    # column positions
    _categoryCodeCol   =  0
    _categoryDescCol   =  1

    

    def __init__(self, madDB=None, initFile=None):
        """__init__ initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile).

        Inputs: Existing MadrigalDB object, by default = None.

            String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get category metadata file
        if (initFile == None):
            self.__filename = self._categoryMetadataFile
        else:
            self.__filename = initFile

        self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()


    def getCategoryDesc(self, code):
        """getCategoryDesc returns the category description that matches code argument, or None if not found.

        Inputs: 
        
            code integer to get category description. or integer as string
        
        Returns: the category description that matches code argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        code = int(code)
        
        for i, category in enumerate(self.__fileList):
            # find matching  code 
            try:
                if code == int(category[self._categoryCodeCol]):
                    return(category[self._categoryDescCol])
                
            except:
                raise madrigal.admin.MadrigalError('Error in madCatTab.txt parsing metadata row %i: ' % (i) + str(category),
                                                   traceback.format_exception(sys.exc_info()[0],
                                                                              sys.exc_info()[1],
                                                                              sys.exc_info()[2]))

        # not found
        return(None)


    def getCategoryList(self):
        """getCategoryList returns a list of all category descriptions and codes.

        Inputs: None.
        
        Returns: a list of all category descriptions and codes.  Each item in the list
        is a tuple of the form (Category description (string),  code (integer)).  Example item:
        ('INSCAL Basic Derived Parameters', 3001)

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        retList = []

        for i, category in enumerate(self.__fileList):
            item = (category[self._categoryDescCol],
                    int(category[self._categoryCodeCol]))
            retList.append(item)
            
        return(retList)

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None)

init initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile).

Inputs: Existing MadrigalDB object, by default = None.

String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.

def __init__(self, madDB=None, initFile=None):
    """__init__ initializes MadrigalParmCategory by reading from madCatTab.txt (or initFile).
    Inputs: Existing MadrigalDB object, by default = None.
        String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/madCatTab.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get category metadata file
    if (initFile == None):
        self.__filename = self._categoryMetadataFile
    else:
        self.__filename = initFile
    self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB).getList()

def getCategoryDesc(

self, code)

getCategoryDesc returns the category description that matches code argument, or None if not found.

Inputs:

code integer to get category description. or integer as string

Returns: the category description that matches code argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getCategoryDesc(self, code):
    """getCategoryDesc returns the category description that matches code argument, or None if not found.
    Inputs: 
    
        code integer to get category description. or integer as string
    
    Returns: the category description that matches code argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    code = int(code)
    
    for i, category in enumerate(self.__fileList):
        # find matching  code 
        try:
            if code == int(category[self._categoryCodeCol]):
                return(category[self._categoryDescCol])
            
        except:
            raise madrigal.admin.MadrigalError('Error in madCatTab.txt parsing metadata row %i: ' % (i) + str(category),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))
    # not found
    return(None)

def getCategoryList(

self)

getCategoryList returns a list of all category descriptions and codes.

Inputs: None.

Returns: a list of all category descriptions and codes. Each item in the list is a tuple of the form (Category description (string), code (integer)). Example item: ('INSCAL Basic Derived Parameters', 3001)

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getCategoryList(self):
    """getCategoryList returns a list of all category descriptions and codes.
    Inputs: None.
    
    Returns: a list of all category descriptions and codes.  Each item in the list
    is a tuple of the form (Category description (string),  code (integer)).  Example item:
    ('INSCAL Basic Derived Parameters', 3001)
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    retList = []
    for i, category in enumerate(self.__fileList):
        item = (category[self._categoryDescCol],
                int(category[self._categoryCodeCol]))
        retList.append(item)
        
    return(retList)

class MadrigalSite

MadrigalSite is an object that provides access to Madrigal site info from the metadata.

This object provides access to all Madrigal site information in the metadata file siteTab.txt.

Usage example::

import madrigal.metadata
import madrigal.admin

try:

    siteObj = madrigal.metadata.MadrigalSite()

    print siteObj.getSiteName(1)

except madrigal.admin.MadrigalError, e:

    print e.getExceptionStr()

Non-standard Python modules used: None

MadrigalError exception thrown if:

1. MadrigalMetadata fails to open or parse metadata file

2. Columns expected to be ints or floats cannot be converted

Change history:

Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu Nov. 9, 2001

class MadrigalSite:
    """MadrigalSite is an object that provides access to Madrigal site info from the metadata.

    This object provides access to all Madrigal site information in the metadata file siteTab.txt.

    Usage example::

        import madrigal.metadata
        import madrigal.admin

        try:
    
            siteObj = madrigal.metadata.MadrigalSite()

            print siteObj.getSiteName(1)

        except madrigal.admin.MadrigalError, e:
        
            print e.getExceptionStr()

    Non-standard Python modules used:
    None

    MadrigalError exception thrown if:

        1. MadrigalMetadata fails to open or parse metadata file
	
        2. Columns expected to be ints or floats cannot be converted

    Change history:

    Written by "Bill Rideout":mailto:wrideout@haystack.mit.edu  Nov. 9, 2001

    """

    #constants
    __siteMetadataFile  = "siteTab.txt"
    _defaultVersion = '2.6'

    # column positions
    __siteIDCol         =  0
    __siteNameCol       =  1
    __madServerCol      =  2
    __madDocRootCol     =  3
    __madCGICol         =  4
    __madServletCol     =  5
    __contactNameCol    =  6
    __contactAddr1Col   =  7
    __contactAddr2Col   =  8
    __contactAddr3Col   =  9
    __contactCityCol    =  10
    __contactStateCol   =  11
    __contactZipCol     =  12
    __contactCountryCol =  13
    __contactPhoneCol   =  14
    __contactEmailCol   =  15
    __siteVersionCol    =  16
    

    def __init__(self, madDB=None, initFile=None):
        """__init__ initializes MadrigalSite by reading from siteTab.txt (or initFile).

        Inputs: Existing MadrigalDB object, by default = None.

            String representing the full path to the metadata file. Default is None, in
            which case file read is MadrigalDB.getMetadataDir()/siteTab.txt.
        
        Returns: void

        Affects: Initializes all the class member variables.

        Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
        """

        # get metadata dir
        if madDB == None:
            self.__madDB = madrigal.metadata.MadrigalDB()
        else:
            self.__madDB = madDB

        # get site metadata file
        if (initFile == None):
            self.__filename = self.__siteMetadataFile
        else:
            self.__filename = initFile
            
        # MadrigalExperiment can now legal have two different lengths - with or without version
        allowedLens = (self.__contactEmailCol+1, self.__siteVersionCol+1)

        self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens).getList()
        
        

    def getSiteName(self, siteID):
        """getSiteName returns the site name that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site name that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__siteNameCol]

            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        # not found
        return(None)


    def getSiteServer(self, siteID):
        """getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found.

        Inputs: siteID integer to get site server.
        
        Returns: the site server that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__madServerCol]

            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        # not found
        return(None)


    def getSiteDocRoot(self, siteID):
        """getSiteDocRoot returns the relative document root (e.g. madrigal)  that matches siteID argument, or None if not found.

        Inputs: siteID integer to get document root path.
        
        Returns: the document root path that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__madDocRootCol]

            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        # not found
        return(None)



    def getSiteRelativeCGI(self, siteID):
        """getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal)  that matches siteID argument, or None if not found.
        
        For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored.

        Inputs: siteID integer to get relative cgi path.
        
        Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        
        version = self.getSiteVersion(siteID)
        if distutils.version.LooseVersion(version) >= distutils.version.LooseVersion('3.0'):
            return(self.getSiteDocRoot(siteID))
        

        # this code only runs for Madrigal 2.0
        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__madCGICol]

            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        # not found
        return None
    
    
    def getSiteContactName(self, siteID):
        """getSiteContactName returns the site contact name that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact name that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactNameCol]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    
    
    def getSiteAddress1(self, siteID):
        """getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact address 1 that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactAddr1Col]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    
    
    def getSiteAddress2(self, siteID):
        """getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact address 2 that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactAddr2Col]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    

    def getSiteAddress3(self, siteID):
        """getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact address 3 that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactAddr3Col]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    

    def getSiteCity(self, siteID):
        """getSiteCity returns the site city that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact city that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactCityCol]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    

    def getSiteState(self, siteID):
        """getSiteState returns the site state that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact state that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactStateCol]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    

    def getSitePostalCode(self, siteID):
        """getSitePostalCode returns the site postal code that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact postal code that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactZipCol]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    

    def getSiteCountry(self, siteID):
        """getSiteCountry returns the site country that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact country that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactCountryCol]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    

    def getSiteTelephone(self, siteID):
        """getSiteTelephone returns the site telephone that matches siteID argument, or None if not found.

        Inputs: siteID integer to get SiteName.
        
        Returns: the site contact telephone that matches siteID argument, or None if not found.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactPhoneCol]
            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))
        # not found
        return(None)
    

    def getSiteEmail(self, siteID):
        """getSiteEmail returns the site email address that matches siteID argument, or None if not found.

        Inputs: siteID integer to get Site email address.
        
        Returns: the site email address that matches siteID argument, or None if not found.  To list multiple
        email addresses in this field separate them with semicolons (since commas are delimiters).  getSiteEmail
        will automatically replace semicolons with commas, as required by multiple email addresses.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    return site[self.__contactEmailCol].replace(';', ',')

            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        # not found
        return(None)


    def getSiteList(self):
        """getSiteList returns a list of all site ids and names.

        Inputs: None.
        
        Returns: a list of all site ids and names.  Each item in the list
        is a tuple of the form (Site id (integer), site name (string)).  Example item:
        (1,'Millstone')

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """
        retList = []

        for site in self.__fileList:
            try:
                item = (int(site[self.__siteIDCol]),
                        site[self.__siteNameCol])
                retList.append(item)
                
            except:
                raise madrigal.admin.MadrigalError('Error in siteTab.txt parsing metadata row: ' + str(inst),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        return retList
    
    
    def getSiteVersion(self, siteID):
        """getSiteVersion returns the site version string that matches siteID argument, or None if not found.

        Inputs: siteID integer
        
        Returns: the site version that matches siteID argument, or None if not found.  If file does
            not have this field, returns default value of 2.6.

        Affects: None

        Exceptions: MadrigalError if any item in row cannot be cast to correct format
        """

        for site in self.__fileList:
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    if len(site) >= self.__siteVersionCol + 1:
                        return site[self.__siteVersionCol]
                    else:
                        return(self._defaultVersion)

            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        # not found
        return(None)
    
    
    
    def setSiteVersionBySiteID(self, siteID, version):
        """setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers 
        at given siteID.

        Inputs:

            siteID - siteID of site in list.

            version - string Madrigal version as period delimited integers (eg, 2.6)
        
        Returns: None.

        Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position

        Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
        siteID not found.
        
        This method added in Madrigal 3.0
        """
        # verify no illegal commas
        if version.find(',') != -1:
            raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' %  \
                                               (str(position, version),
                                                [traceback.format_exc()]))
            
        # parse test
        version = version.strip()
        try:
            siteID = int(siteID)
            if len(version) == 0:
                raise ValueError('')
            for item in version.split('.'):
                int(item)
        except:
            raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' %  \
                                               (str(position, version),
                                                [traceback.format_exc()]))
            
        rightPosition = None
        for position, site in enumerate(self.__fileList):
            # find matching siteid
            try:
                if (int(site[self.__siteIDCol]) == siteID):
                    rightPosition = position

            except:
                raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                           traceback.format_exception(sys.exc_info()[0],
                                                                        sys.exc_info()[1],
                                                                        sys.exc_info()[2]))

        if rightPosition is None:
            raise madrigal.admin.MadrigalError('siteId %i not found in siteTab.txt' % (siteID))
        
        if len(self.__fileList[position]) == self.__siteVersionCol:
            # not yet set
            self.__fileList[position].append(version)
        else:
            self.__fileList[position][self.__siteVersionCol] = version
              
                
                
    def writeMetadata(self, newFullPath=None):
        """writeMetadata writes a new version of the siteTab.txt file.
        
        Inputs: newFullPath:  a new path to write the siteTab.txt file to, if
        not the same as the original metadata file opened.  Defaults to None, which overwrites
        metadata file that was read from.
        
        Returns: None.

        Affects: Writes updated version of metadata file.

        Exceptions: If unable to write file
        """

        # create string to hold file
        metaFileStr = ''
        delimiter = ','

        for lineList in self.__fileList:
            metaFileStr += delimiter.join(lineList) + '\n'

        # try to write file, if not, raise exception

        try:

            if newFullPath == None:
                if (len(os.path.dirname(self.__filename)) != 0):
                    newFullPath = self.__filename
                else:
                    newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename
            newFile = open(newFullPath, "w")
            newFile.write(metaFileStr)
            newFile.close()

        except:

            raise madrigal.admin.MadrigalError("Unable to write metadata file " + \
                                               str(newFullPath),
                                               traceback.format_exception(sys.exc_info()[0],
                                                                          sys.exc_info()[1],
                                                                          sys.exc_info()[2]))

Ancestors (in MRO)

Static methods

def __init__(

self, madDB=None, initFile=None)

init initializes MadrigalSite by reading from siteTab.txt (or initFile).

Inputs: Existing MadrigalDB object, by default = None.

String representing the full path to the metadata file. Default is None, in
which case file read is MadrigalDB.getMetadataDir()/siteTab.txt.

Returns: void

Affects: Initializes all the class member variables.

Exceptions: MadrigalError thrown by MadrigalMetadata if file not opened or parsed successfully.

def __init__(self, madDB=None, initFile=None):
    """__init__ initializes MadrigalSite by reading from siteTab.txt (or initFile).
    Inputs: Existing MadrigalDB object, by default = None.
        String representing the full path to the metadata file. Default is None, in
        which case file read is MadrigalDB.getMetadataDir()/siteTab.txt.
    
    Returns: void
    Affects: Initializes all the class member variables.
    Exceptions: MadrigalError thrown  by MadrigalMetadata if file not opened or parsed successfully.
    """
    # get metadata dir
    if madDB == None:
        self.__madDB = madrigal.metadata.MadrigalDB()
    else:
        self.__madDB = madDB
    # get site metadata file
    if (initFile == None):
        self.__filename = self.__siteMetadataFile
    else:
        self.__filename = initFile
        
    # MadrigalExperiment can now legal have two different lengths - with or without version
    allowedLens = (self.__contactEmailCol+1, self.__siteVersionCol+1)
    self.__fileList = madrigal.metadata.MadrigalMetadata(self.__filename, self.__madDB, allowedLens).getList()

def getSiteAddress1(

self, siteID)

getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact address 1 that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteAddress1(self, siteID):
    """getSiteAddress1 returns the site address 1 that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact address 1 that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactAddr1Col]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteAddress2(

self, siteID)

getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact address 2 that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteAddress2(self, siteID):
    """getSiteAddress2 returns the site address 2 that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact address 2 that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactAddr2Col]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteAddress3(

self, siteID)

getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact address 3 that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteAddress3(self, siteID):
    """getSiteAddress3 returns the site address 3 that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact address 3 that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactAddr3Col]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteCity(

self, siteID)

getSiteCity returns the site city that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact city that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteCity(self, siteID):
    """getSiteCity returns the site city that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact city that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactCityCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteContactName(

self, siteID)

getSiteContactName returns the site contact name that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact name that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteContactName(self, siteID):
    """getSiteContactName returns the site contact name that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact name that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactNameCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteCountry(

self, siteID)

getSiteCountry returns the site country that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact country that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteCountry(self, siteID):
    """getSiteCountry returns the site country that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact country that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactCountryCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteDocRoot(

self, siteID)

getSiteDocRoot returns the relative document root (e.g. madrigal) that matches siteID argument, or None if not found.

Inputs: siteID integer to get document root path.

Returns: the document root path that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteDocRoot(self, siteID):
    """getSiteDocRoot returns the relative document root (e.g. madrigal)  that matches siteID argument, or None if not found.
    Inputs: siteID integer to get document root path.
    
    Returns: the document root path that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__madDocRootCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteEmail(

self, siteID)

getSiteEmail returns the site email address that matches siteID argument, or None if not found.

Inputs: siteID integer to get Site email address.

Returns: the site email address that matches siteID argument, or None if not found. To list multiple email addresses in this field separate them with semicolons (since commas are delimiters). getSiteEmail will automatically replace semicolons with commas, as required by multiple email addresses.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteEmail(self, siteID):
    """getSiteEmail returns the site email address that matches siteID argument, or None if not found.
    Inputs: siteID integer to get Site email address.
    
    Returns: the site email address that matches siteID argument, or None if not found.  To list multiple
    email addresses in this field separate them with semicolons (since commas are delimiters).  getSiteEmail
    will automatically replace semicolons with commas, as required by multiple email addresses.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactEmailCol].replace(';', ',')
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteList(

self)

getSiteList returns a list of all site ids and names.

Inputs: None.

Returns: a list of all site ids and names. Each item in the list is a tuple of the form (Site id (integer), site name (string)). Example item: (1,'Millstone')

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteList(self):
    """getSiteList returns a list of all site ids and names.
    Inputs: None.
    
    Returns: a list of all site ids and names.  Each item in the list
    is a tuple of the form (Site id (integer), site name (string)).  Example item:
    (1,'Millstone')
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    retList = []
    for site in self.__fileList:
        try:
            item = (int(site[self.__siteIDCol]),
                    site[self.__siteNameCol])
            retList.append(item)
            
        except:
            raise madrigal.admin.MadrigalError('Error in siteTab.txt parsing metadata row: ' + str(inst),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    return retList

def getSiteName(

self, siteID)

getSiteName returns the site name that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site name that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteName(self, siteID):
    """getSiteName returns the site name that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site name that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__siteNameCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSitePostalCode(

self, siteID)

getSitePostalCode returns the site postal code that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact postal code that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSitePostalCode(self, siteID):
    """getSitePostalCode returns the site postal code that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact postal code that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactZipCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteRelativeCGI(

self, siteID)

getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal) that matches siteID argument, or None if not found.

For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored.

Inputs: siteID integer to get relative cgi path.

Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteRelativeCGI(self, siteID):
    """getSiteRelativeCGI returns the relative cgi path (e.g.cgi-bin/madrigal)  that matches siteID argument, or None if not found.
    
    For Madrigal 3.0 sites and later, this returns getSiteDocRoot, and the cgi field is ignored.
    Inputs: siteID integer to get relative cgi path.
    
    Returns: the relative cgi path that matches siteID argument, or None if not found. Not meaningful for Madrigal 3.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    
    version = self.getSiteVersion(siteID)
    if distutils.version.LooseVersion(version) >= distutils.version.LooseVersion('3.0'):
        return(self.getSiteDocRoot(siteID))
    
    # this code only runs for Madrigal 2.0
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__madCGICol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return None

def getSiteServer(

self, siteID)

getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found.

Inputs: siteID integer to get site server.

Returns: the site server that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteServer(self, siteID):
    """getSiteServer returns the site server (e.g., www.haystack.mit.edu) that matches siteID argument, or None if not found.
    Inputs: siteID integer to get site server.
    
    Returns: the site server that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__madServerCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteState(

self, siteID)

getSiteState returns the site state that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact state that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteState(self, siteID):
    """getSiteState returns the site state that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact state that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactStateCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteTelephone(

self, siteID)

getSiteTelephone returns the site telephone that matches siteID argument, or None if not found.

Inputs: siteID integer to get SiteName.

Returns: the site contact telephone that matches siteID argument, or None if not found.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteTelephone(self, siteID):
    """getSiteTelephone returns the site telephone that matches siteID argument, or None if not found.
    Inputs: siteID integer to get SiteName.
    
    Returns: the site contact telephone that matches siteID argument, or None if not found.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                return site[self.__contactPhoneCol]
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def getSiteVersion(

self, siteID)

getSiteVersion returns the site version string that matches siteID argument, or None if not found.

Inputs: siteID integer

Returns: the site version that matches siteID argument, or None if not found. If file does not have this field, returns default value of 2.6.

Affects: None

Exceptions: MadrigalError if any item in row cannot be cast to correct format

def getSiteVersion(self, siteID):
    """getSiteVersion returns the site version string that matches siteID argument, or None if not found.
    Inputs: siteID integer
    
    Returns: the site version that matches siteID argument, or None if not found.  If file does
        not have this field, returns default value of 2.6.
    Affects: None
    Exceptions: MadrigalError if any item in row cannot be cast to correct format
    """
    for site in self.__fileList:
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                if len(site) >= self.__siteVersionCol + 1:
                    return site[self.__siteVersionCol]
                else:
                    return(self._defaultVersion)
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    # not found
    return(None)

def setSiteVersionBySiteID(

self, siteID, version)

setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers at given siteID.

Inputs:

siteID - siteID of site in list.

version - string Madrigal version as period delimited integers (eg, 2.6)

Returns: None.

Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position

Exceptions: MadrigalError if any item in row cannot be cast to correct format, or siteID not found.

This method added in Madrigal 3.0

def setSiteVersionBySiteID(self, siteID, version):
    """setSiteVersionBySiteID sets the site Madrigal version (string) as period delimited integers 
    at given siteID.
    Inputs:
        siteID - siteID of site in list.
        version - string Madrigal version as period delimited integers (eg, 2.6)
    
    Returns: None.
    Affects: sets the Madrigal version as period delimited integers (eg, 2.6) at given position
    Exceptions: MadrigalError if any item in row cannot be cast to correct format, or
    siteID not found.
    
    This method added in Madrigal 3.0
    """
    # verify no illegal commas
    if version.find(',') != -1:
        raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' %  \
                                           (str(position, version),
                                            [traceback.format_exc()]))
        
    # parse test
    version = version.strip()
    try:
        siteID = int(siteID)
        if len(version) == 0:
            raise ValueError('')
        for item in version.split('.'):
            int(item)
    except:
        raise madrigal.admin.MadrigalError('Error in setSiteVersionByPosition with args %s: %s' %  \
                                           (str(position, version),
                                            [traceback.format_exc()]))
        
    rightPosition = None
    for position, site in enumerate(self.__fileList):
        # find matching siteid
        try:
            if (int(site[self.__siteIDCol]) == siteID):
                rightPosition = position
        except:
            raise madrigal.admin.MadrigalError('Error parsing metadata row in siteTab.txt: ' + str(site),
                                                       traceback.format_exception(sys.exc_info()[0],
                                                                    sys.exc_info()[1],
                                                                    sys.exc_info()[2]))
    if rightPosition is None:
        raise madrigal.admin.MadrigalError('siteId %i not found in siteTab.txt' % (siteID))
    
    if len(self.__fileList[position]) == self.__siteVersionCol:
        # not yet set
        self.__fileList[position].append(version)
    else:
        self.__fileList[position][self.__siteVersionCol] = version

def writeMetadata(

self, newFullPath=None)

writeMetadata writes a new version of the siteTab.txt file.

Inputs: newFullPath: a new path to write the siteTab.txt file to, if not the same as the original metadata file opened. Defaults to None, which overwrites metadata file that was read from.

Returns: None.

Affects: Writes updated version of metadata file.

Exceptions: If unable to write file

def writeMetadata(self, newFullPath=None):
    """writeMetadata writes a new version of the siteTab.txt file.
    
    Inputs: newFullPath:  a new path to write the siteTab.txt file to, if
    not the same as the original metadata file opened.  Defaults to None, which overwrites
    metadata file that was read from.
    
    Returns: None.
    Affects: Writes updated version of metadata file.
    Exceptions: If unable to write file
    """
    # create string to hold file
    metaFileStr = ''
    delimiter = ','
    for lineList in self.__fileList:
        metaFileStr += delimiter.join(lineList) + '\n'
    # try to write file, if not, raise exception
    try:
        if newFullPath == None:
            if (len(os.path.dirname(self.__filename)) != 0):
                newFullPath = self.__filename
            else:
                newFullPath = self.__madDB.getMetadataDir() + "/" + self.__filename
        newFile = open(newFullPath, "w")
        newFile.write(metaFileStr)
        newFile.close()
    except:
        raise madrigal.admin.MadrigalError("Unable to write metadata file " + \
                                           str(newFullPath),
                                           traceback.format_exception(sys.exc_info()[0],
                                                                      sys.exc_info()[1],
                                                                      sys.exc_info()[2]))
Previous: madrigal.isprint   Up: Internal Madrigal Python API   Next: madrigal.openmadrigal