CedarBackup2.extend.mbox

Package CedarBackup2 :: Package extend :: Module mbox |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup2.extend.mbox

       1  # -*- coding: iso-8859-1 -*-
       2  # vim: set ft=python ts=3 sw=3 expandtab:
       3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
       4  #
       5  #              C E D A R
       6  #          S O L U T I O N S       "Software done right."
       7  #           S O F T W A R E
       8  #
       9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      10  #
      11  # Copyright (c) 2006-2007,2010 Kenneth J. Pronovici.
      12  # All rights reserved.
      13  #
      14  # This program is free software; you can redistribute it and/or
      15  # modify it under the terms of the GNU General Public License,
      16  # Version 2, as published by the Free Software Foundation.
      17  #
      18  # This program is distributed in the hope that it will be useful,
      19  # but WITHOUT ANY WARRANTY; without even the implied warranty of
      20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
      21  #
      22  # Copies of the GNU General Public License are available from
      23  # the Free Software Foundation website, http://www.gnu.org/.
      24  #
      25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      26  #
      27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
      28  # Language : Python 2 (>= 2.7)
      29  # Project  : Official Cedar Backup Extensions
      30  # Purpose  : Provides an extension to back up mbox email files.
      31  #
      32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      33
      34  ########################################################################
      35  # Module documentation
      36  ########################################################################
      37
      38  """
      39  Provides an extension to back up mbox email files.
      40
      41  Backing up email
      42  ================
      43
      44     Email folders (often stored as mbox flatfiles) are not well-suited being backed
      45     up with an incremental backup like the one offered by Cedar Backup.  This is
      46     because mbox files often change on a daily basis, forcing the incremental
      47     backup process to back them up every day in order to avoid losing data.  This
      48     can result in quite a bit of wasted space when backing up large folders.  (Note
      49     that the alternative maildir format does not share this problem, since it
      50     typically uses one file per message.)
      51
      52     One solution to this problem is to design a smarter incremental backup process,
      53     which backs up baseline content on the first day of the week, and then backs up
      54     only new messages added to that folder on every other day of the week.  This way,
      55     the backup for any single day is only as large as the messages placed into the
      56     folder on that day.  The backup isn't as "perfect" as the incremental backup
      57     process, because it doesn't preserve information about messages deleted from
      58     the backed-up folder.  However, it should be much more space-efficient, and
      59     in a recovery situation, it seems better to restore too much data rather
      60     than too little.
      61
      62  What is this extension?
      63  =======================
      64
      65     This is a Cedar Backup extension used to back up mbox email files via the Cedar
      66     Backup command line.  Individual mbox files or directories containing mbox
      67     files can be backed up using the same collect modes allowed for filesystems in
      68     the standard Cedar Backup collect action: weekly, daily, incremental.  It
      69     implements the "smart" incremental backup process discussed above, using
      70     functionality provided by the C{grepmail} utility.
      71
      72     This extension requires a new configuration section <mbox> and is intended to
      73     be run either immediately before or immediately after the standard collect
      74     action.  Aside from its own configuration, it requires the options and collect
      75     configuration sections in the standard Cedar Backup configuration file.
      76
      77     The mbox action is conceptually similar to the standard collect action,
      78     except that mbox directories are not collected recursively.  This implies
      79     some configuration changes (i.e. there's no need for global exclusions or an
      80     ignore file).  If you back up a directory, all of the mbox files in that
      81     directory are backed up into a single tar file using the indicated
      82     compression method.
      83
      84  @author: Kenneth J. Pronovici <pronovic@ieee.org>
      85  """
      86
      87  ########################################################################
      88  # Imported modules
      89  ########################################################################
      90
      91  # System modules
      92  import os
      93  import logging
      94  import datetime
      95  import pickle
      96  import tempfile
      97  from bz2 import BZ2File
      98  from gzip import GzipFile
      99
     100  # Cedar Backup modules
     101  from CedarBackup2.filesystem import FilesystemList, BackupFileList
     102  from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode
     103  from CedarBackup2.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
     104  from CedarBackup2.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
     105  from CedarBackup2.util import isStartOfWeek, buildNormalizedPath
     106  from CedarBackup2.util import resolveCommand, executeCommand
     107  from CedarBackup2.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership
     108
     109
     110  ########################################################################
     111  # Module-wide constants and variables
     112  ########################################################################
     113
     114  logger = logging.getLogger("CedarBackup2.log.extend.mbox")
     115
     116  GREPMAIL_COMMAND = [ "grepmail", ]
     117  REVISION_PATH_EXTENSION = "mboxlast"
     118
     119
     120  ########################################################################
     121  # MboxFile class definition
     122  ########################################################################
     123
     124 -class MboxFile(object):
     125
     126     """
     127     Class representing mbox file configuration..
     128
     129     The following restrictions exist on data in this class:
     130
     131        - The absolute path must be absolute.
     132        - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
     133        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
     134
     135     @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode, compressMode
     136     """
     137
     138 -   def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
     139        """
     140        Constructor for the C{MboxFile} class.
     141
     142        You should never directly instantiate this class.
     143
     144        @param absolutePath: Absolute path to an mbox file on disk.
     145        @param collectMode: Overridden collect mode for this directory.
     146        @param compressMode: Overridden compression mode for this directory.
     147        """
     148        self._absolutePath = None
     149        self._collectMode = None
     150        self._compressMode = None
     151        self.absolutePath = absolutePath
     152        self.collectMode = collectMode
     153        self.compressMode = compressMode
     154
     155 -   def __repr__(self):
     156        """
     157        Official string representation for class instance.
     158        """
     159        return "MboxFile(%s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode)
     160
     161 -   def __str__(self):
     162        """
     163        Informal string representation for class instance.
     164        """
     165        return self.__repr__()
     166
     167 -   def __cmp__(self, other):
     168        """
     169        Definition of equals operator for this class.
     170        @param other: Other object to compare to.
     171        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     172        """
     173        if other is None:
     174           return 1
     175        if self.absolutePath != other.absolutePath:
     176           if self.absolutePath < other.absolutePath:
     177              return -1
     178           else:
     179              return 1
     180        if self.collectMode != other.collectMode:
     181           if self.collectMode < other.collectMode:
     182              return -1
     183           else:
     184              return 1
     185        if self.compressMode != other.compressMode:
     186           if self.compressMode < other.compressMode:
     187              return -1
     188           else:
     189              return 1
     190        return 0
     191
     192 -   def _setAbsolutePath(self, value):
     193        """
     194        Property target used to set the absolute path.
     195        The value must be an absolute path if it is not C{None}.
     196        It does not have to exist on disk at the time of assignment.
     197        @raise ValueError: If the value is not an absolute path.
     198        @raise ValueError: If the value cannot be encoded properly.
     199        """
     200        if value is not None:
     201           if not os.path.isabs(value):
     202              raise ValueError("Absolute path must be, er, an absolute path.")
     203        self._absolutePath = encodePath(value)
     204
     205 -   def _getAbsolutePath(self):
     206        """
     207        Property target used to get the absolute path.
     208        """
     209        return self._absolutePath
     210
     211 -   def _setCollectMode(self, value):
     212        """
     213        Property target used to set the collect mode.
     214        If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
     215        @raise ValueError: If the value is not valid.
     216        """
     217        if value is not None:
     218           if value not in VALID_COLLECT_MODES:
     219              raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
     220        self._collectMode = value
     221
     222 -   def _getCollectMode(self):
     223        """
     224        Property target used to get the collect mode.
     225        """
     226        return self._collectMode
     227
     228 -   def _setCompressMode(self, value):
     229        """
     230        Property target used to set the compress mode.
     231        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
     232        @raise ValueError: If the value is not valid.
     233        """
     234        if value is not None:
     235           if value not in VALID_COMPRESS_MODES:
     236              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
     237        self._compressMode = value
     238
     239 -   def _getCompressMode(self):
     240        """
     241        Property target used to get the compress mode.
     242        """
     243        return self._compressMode
     244
     245     absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.")
     246     collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.")
     247     compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")
     248
     249
     250  ########################################################################
     251  # MboxDir class definition
     252  ########################################################################
     253
     254 -class MboxDir(object):
     255
     256     """
     257     Class representing mbox directory configuration..
     258
     259     The following restrictions exist on data in this class:
     260
     261        - The absolute path must be absolute.
     262        - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
     263        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
     264
     265     Unlike collect directory configuration, this is the only place exclusions
     266     are allowed (no global exclusions at the <mbox> configuration level).  Also,
     267     we only allow relative exclusions and there is no configured ignore file.
     268     This is because mbox directory backups are not recursive.
     269
     270     @sort: __init__, __repr__, __str__, __cmp__, absolutePath, collectMode,
     271            compressMode, relativeExcludePaths, excludePatterns
     272     """
     273
     274 -   def __init__(self, absolutePath=None, collectMode=None, compressMode=None,
     275                  relativeExcludePaths=None, excludePatterns=None):
     276        """
     277        Constructor for the C{MboxDir} class.
     278
     279        You should never directly instantiate this class.
     280
     281        @param absolutePath: Absolute path to a mbox file on disk.
     282        @param collectMode: Overridden collect mode for this directory.
     283        @param compressMode: Overridden compression mode for this directory.
     284        @param relativeExcludePaths: List of relative paths to exclude.
     285        @param excludePatterns: List of regular expression patterns to exclude
     286        """
     287        self._absolutePath = None
     288        self._collectMode = None
     289        self._compressMode = None
     290        self._relativeExcludePaths = None
     291        self._excludePatterns = None
     292        self.absolutePath = absolutePath
     293        self.collectMode = collectMode
     294        self.compressMode = compressMode
     295        self.relativeExcludePaths = relativeExcludePaths
     296        self.excludePatterns = excludePatterns
     297
     298 -   def __repr__(self):
     299        """
     300        Official string representation for class instance.
     301        """
     302        return "MboxDir(%s, %s, %s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode,
     303                                                self.relativeExcludePaths, self.excludePatterns)
     304
     305 -   def __str__(self):
     306        """
     307        Informal string representation for class instance.
     308        """
     309        return self.__repr__()
     310
     311 -   def __cmp__(self, other):
     312        """
     313        Definition of equals operator for this class.
     314        @param other: Other object to compare to.
     315        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     316        """
     317        if other is None:
     318           return 1
     319        if self.absolutePath != other.absolutePath:
     320           if self.absolutePath < other.absolutePath:
     321              return -1
     322           else:
     323              return 1
     324        if self.collectMode != other.collectMode:
     325           if self.collectMode < other.collectMode:
     326              return -1
     327           else:
     328              return 1
     329        if self.compressMode != other.compressMode:
     330           if self.compressMode < other.compressMode:
     331              return -1
     332           else:
     333              return 1
     334        if self.relativeExcludePaths != other.relativeExcludePaths:
     335           if self.relativeExcludePaths < other.relativeExcludePaths:
     336              return -1
     337           else:
     338              return 1
     339        if self.excludePatterns != other.excludePatterns:
     340           if self.excludePatterns < other.excludePatterns:
     341              return -1
     342           else:
     343              return 1
     344        return 0
     345
     346 -   def _setAbsolutePath(self, value):
     347        """
     348        Property target used to set the absolute path.
     349        The value must be an absolute path if it is not C{None}.
     350        It does not have to exist on disk at the time of assignment.
     351        @raise ValueError: If the value is not an absolute path.
     352        @raise ValueError: If the value cannot be encoded properly.
     353        """
     354        if value is not None:
     355           if not os.path.isabs(value):
     356              raise ValueError("Absolute path must be, er, an absolute path.")
     357        self._absolutePath = encodePath(value)
     358
     359 -   def _getAbsolutePath(self):
     360        """
     361        Property target used to get the absolute path.
     362        """
     363        return self._absolutePath
     364
     365 -   def _setCollectMode(self, value):
     366        """
     367        Property target used to set the collect mode.
     368        If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
     369        @raise ValueError: If the value is not valid.
     370        """
     371        if value is not None:
     372           if value not in VALID_COLLECT_MODES:
     373              raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
     374        self._collectMode = value
     375
     376 -   def _getCollectMode(self):
     377        """
     378        Property target used to get the collect mode.
     379        """
     380        return self._collectMode
     381
     382 -   def _setCompressMode(self, value):
     383        """
     384        Property target used to set the compress mode.
     385        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
     386        @raise ValueError: If the value is not valid.
     387        """
     388        if value is not None:
     389           if value not in VALID_COMPRESS_MODES:
     390              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
     391        self._compressMode = value
     392
     393 -   def _getCompressMode(self):
     394        """
     395        Property target used to get the compress mode.
     396        """
     397        return self._compressMode
     398
     399 -   def _setRelativeExcludePaths(self, value):
     400        """
     401        Property target used to set the relative exclude paths list.
     402        Elements do not have to exist on disk at the time of assignment.
     403        """
     404        if value is None:
     405           self._relativeExcludePaths = None
     406        else:
     407           try:
     408              saved = self._relativeExcludePaths
     409              self._relativeExcludePaths = UnorderedList()
     410              self._relativeExcludePaths.extend(value)
     411           except Exception, e:
     412              self._relativeExcludePaths = saved
     413              raise e
     414
     415 -   def _getRelativeExcludePaths(self):
     416        """
     417        Property target used to get the relative exclude paths list.
     418        """
     419        return self._relativeExcludePaths
     420
     421 -   def _setExcludePatterns(self, value):
     422        """
     423        Property target used to set the exclude patterns list.
     424        """
     425        if value is None:
     426           self._excludePatterns = None
     427        else:
     428           try:
     429              saved = self._excludePatterns
     430              self._excludePatterns = RegexList()
     431              self._excludePatterns.extend(value)
     432           except Exception, e:
     433              self._excludePatterns = saved
     434              raise e
     435
     436 -   def _getExcludePatterns(self):
     437        """
     438        Property target used to get the exclude patterns list.
     439        """
     440        return self._excludePatterns
     441
     442     absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.")
     443     collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.")
     444     compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.")
     445     relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
     446     excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
     447
     448
     449  ########################################################################
     450  # MboxConfig class definition
     451  ########################################################################
     452
     453 -class MboxConfig(object):
     454
     455     """
     456     Class representing mbox configuration.
     457
     458     Mbox configuration is used for backing up mbox email files.
     459
     460     The following restrictions exist on data in this class:
     461
     462        - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
     463        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
     464        - The C{mboxFiles} list must be a list of C{MboxFile} objects
     465        - The C{mboxDirs} list must be a list of C{MboxDir} objects
     466
     467     For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished
     468     through the L{util.ObjectTypeList} list implementation that overrides common
     469     list methods and transparently ensures that each element is of the proper
     470     type.
     471
     472     Unlike collect configuration, no global exclusions are allowed on this
     473     level.  We only allow relative exclusions at the mbox directory level.
     474     Also, there is no configured ignore file.  This is because mbox directory
     475     backups are not recursive.
     476
     477     @note: Lists within this class are "unordered" for equality comparisons.
     478
     479     @sort: __init__, __repr__, __str__, __cmp__, collectMode, compressMode, mboxFiles, mboxDirs
     480     """
     481
     482 -   def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
     483        """
     484        Constructor for the C{MboxConfig} class.
     485
     486        @param collectMode: Default collect mode.
     487        @param compressMode: Default compress mode.
     488        @param mboxFiles: List of mbox files to back up
     489        @param mboxDirs: List of mbox directories to back up
     490
     491        @raise ValueError: If one of the values is invalid.
     492        """
     493        self._collectMode = None
     494        self._compressMode = None
     495        self._mboxFiles = None
     496        self._mboxDirs = None
     497        self.collectMode = collectMode
     498        self.compressMode = compressMode
     499        self.mboxFiles = mboxFiles
     500        self.mboxDirs = mboxDirs
     501
     502 -   def __repr__(self):
     503        """
     504        Official string representation for class instance.
     505        """
     506        return "MboxConfig(%s, %s, %s, %s)" % (self.collectMode, self.compressMode, self.mboxFiles, self.mboxDirs)
     507
     508 -   def __str__(self):
     509        """
     510        Informal string representation for class instance.
     511        """
     512        return self.__repr__()
     513
     514 -   def __cmp__(self, other):
     515        """
     516        Definition of equals operator for this class.
     517        Lists within this class are "unordered" for equality comparisons.
     518        @param other: Other object to compare to.
     519        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     520        """
     521        if other is None:
     522           return 1
     523        if self.collectMode != other.collectMode:
     524           if self.collectMode < other.collectMode:
     525              return -1
     526           else:
     527              return 1
     528        if self.compressMode != other.compressMode:
     529           if self.compressMode < other.compressMode:
     530              return -1
     531           else:
     532              return 1
     533        if self.mboxFiles != other.mboxFiles:
     534           if self.mboxFiles < other.mboxFiles:
     535              return -1
     536           else:
     537              return 1
     538        if self.mboxDirs != other.mboxDirs:
     539           if self.mboxDirs < other.mboxDirs:
     540              return -1
     541           else:
     542              return 1
     543        return 0
     544
     545 -   def _setCollectMode(self, value):
     546        """
     547        Property target used to set the collect mode.
     548        If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
     549        @raise ValueError: If the value is not valid.
     550        """
     551        if value is not None:
     552           if value not in VALID_COLLECT_MODES:
     553              raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
     554        self._collectMode = value
     555
     556 -   def _getCollectMode(self):
     557        """
     558        Property target used to get the collect mode.
     559        """
     560        return self._collectMode
     561
     562 -   def _setCompressMode(self, value):
     563        """
     564        Property target used to set the compress mode.
     565        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
     566        @raise ValueError: If the value is not valid.
     567        """
     568        if value is not None:
     569           if value not in VALID_COMPRESS_MODES:
     570              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
     571        self._compressMode = value
     572
     573 -   def _getCompressMode(self):
     574        """
     575        Property target used to get the compress mode.
     576        """
     577        return self._compressMode
     578
     579 -   def _setMboxFiles(self, value):
     580        """
     581        Property target used to set the mboxFiles list.
     582        Either the value must be C{None} or each element must be an C{MboxFile}.
     583        @raise ValueError: If the value is not an C{MboxFile}
     584        """
     585        if value is None:
     586           self._mboxFiles = None
     587        else:
     588           try:
     589              saved = self._mboxFiles
     590              self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile")
     591              self._mboxFiles.extend(value)
     592           except Exception, e:
     593              self._mboxFiles = saved
     594              raise e
     595
     596 -   def _getMboxFiles(self):
     597        """
     598        Property target used to get the mboxFiles list.
     599        """
     600        return self._mboxFiles
     601
     602 -   def _setMboxDirs(self, value):
     603        """
     604        Property target used to set the mboxDirs list.
     605        Either the value must be C{None} or each element must be an C{MboxDir}.
     606        @raise ValueError: If the value is not an C{MboxDir}
     607        """
     608        if value is None:
     609           self._mboxDirs = None
     610        else:
     611           try:
     612              saved = self._mboxDirs
     613              self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir")
     614              self._mboxDirs.extend(value)
     615           except Exception, e:
     616              self._mboxDirs = saved
     617              raise e
     618
     619 -   def _getMboxDirs(self):
     620        """
     621        Property target used to get the mboxDirs list.
     622        """
     623        return self._mboxDirs
     624
     625     collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
     626     compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
     627     mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.")
     628     mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")
     629
     630
     631  ########################################################################
     632  # LocalConfig class definition
     633  ########################################################################
     634
     635 -class LocalConfig(object):
     636
     637     """
     638     Class representing this extension's configuration document.
     639
     640     This is not a general-purpose configuration object like the main Cedar
     641     Backup configuration object.  Instead, it just knows how to parse and emit
     642     Mbox-specific configuration values.  Third parties who need to read and
     643     write configuration related to this extension should access it through the
     644     constructor, C{validate} and C{addConfig} methods.
     645
     646     @note: Lists within this class are "unordered" for equality comparisons.
     647
     648     @sort: __init__, __repr__, __str__, __cmp__, mbox, validate, addConfig
     649     """
     650
     651 -   def __init__(self, xmlData=None, xmlPath=None, validate=True):
     652        """
     653        Initializes a configuration object.
     654
     655        If you initialize the object without passing either C{xmlData} or
     656        C{xmlPath} then configuration will be empty and will be invalid until it
     657        is filled in properly.
     658
     659        No reference to the original XML data or original path is saved off by
     660        this class.  Once the data has been parsed (successfully or not) this
     661        original information is discarded.
     662
     663        Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
     664        method will be called (with its default arguments) against configuration
     665        after successfully parsing any passed-in XML.  Keep in mind that even if
     666        C{validate} is C{False}, it might not be possible to parse the passed-in
     667        XML document if lower-level validations fail.
     668
     669        @note: It is strongly suggested that the C{validate} option always be set
     670        to C{True} (the default) unless there is a specific need to read in
     671        invalid configuration from disk.
     672
     673        @param xmlData: XML data representing configuration.
     674        @type xmlData: String data.
     675
     676        @param xmlPath: Path to an XML file on disk.
     677        @type xmlPath: Absolute path to a file on disk.
     678
     679        @param validate: Validate the document after parsing it.
     680        @type validate: Boolean true/false.
     681
     682        @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
     683        @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
     684        @raise ValueError: If the parsed configuration document is not valid.
     685        """
     686        self._mbox = None
     687        self.mbox = None
     688        if xmlData is not None and xmlPath is not None:
     689           raise ValueError("Use either xmlData or xmlPath, but not both.")
     690        if xmlData is not None:
     691           self._parseXmlData(xmlData)
     692           if validate:
     693              self.validate()
     694        elif xmlPath is not None:
     695           xmlData = open(xmlPath).read()
     696           self._parseXmlData(xmlData)
     697           if validate:
     698              self.validate()
     699
     700 -   def __repr__(self):
     701        """
     702        Official string representation for class instance.
     703        """
     704        return "LocalConfig(%s)" % (self.mbox)
     705
     706 -   def __str__(self):
     707        """
     708        Informal string representation for class instance.
     709        """
     710        return self.__repr__()
     711
     712 -   def __cmp__(self, other):
     713        """
     714        Definition of equals operator for this class.
     715        Lists within this class are "unordered" for equality comparisons.
     716        @param other: Other object to compare to.
     717        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     718        """
     719        if other is None:
     720           return 1
     721        if self.mbox != other.mbox:
     722           if self.mbox < other.mbox:
     723              return -1
     724           else:
     725              return 1
     726        return 0
     727
     728 -   def _setMbox(self, value):
     729        """
     730        Property target used to set the mbox configuration value.
     731        If not C{None}, the value must be a C{MboxConfig} object.
     732        @raise ValueError: If the value is not a C{MboxConfig}
     733        """
     734        if value is None:
     735           self._mbox = None
     736        else:
     737           if not isinstance(value, MboxConfig):
     738              raise ValueError("Value must be a C{MboxConfig} object.")
     739           self._mbox = value
     740
     741 -   def _getMbox(self):
     742        """
     743        Property target used to get the mbox configuration value.
     744        """
     745        return self._mbox
     746
     747     mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.")
     748
     749 -   def validate(self):
     750        """
     751        Validates configuration represented by the object.
     752
     753        Mbox configuration must be filled in.  Within that, the collect mode and
     754        compress mode are both optional, but the list of repositories must
     755        contain at least one entry.
     756
     757        Each configured file or directory must contain an absolute path, and then
     758        must be either able to take collect mode and compress mode configuration
     759        from the parent C{MboxConfig} object, or must set each value on its own.
     760
     761        @raise ValueError: If one of the validations fails.
     762        """
     763        if self.mbox is None:
     764           raise ValueError("Mbox section is required.")
     765        if  (self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \
     766            (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1):
     767           raise ValueError("At least one mbox file or directory must be configured.")
     768        if self.mbox.mboxFiles is not None:
     769           for mboxFile in self.mbox.mboxFiles:
     770              if mboxFile.absolutePath is None:
     771                 raise ValueError("Each mbox file must set an absolute path.")
     772              if self.mbox.collectMode is None and mboxFile.collectMode is None:
     773                 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.")
     774              if self.mbox.compressMode is None and mboxFile.compressMode is None:
     775                 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.")
     776        if self.mbox.mboxDirs is not None:
     777           for mboxDir in self.mbox.mboxDirs:
     778              if mboxDir.absolutePath is None:
     779                 raise ValueError("Each mbox directory must set an absolute path.")
     780              if self.mbox.collectMode is None and mboxDir.collectMode is None:
     781                 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.")
     782              if self.mbox.compressMode is None and mboxDir.compressMode is None:
     783                 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")
     784
     785 -   def addConfig(self, xmlDom, parentNode):
     786        """
     787        Adds an <mbox> configuration section as the next child of a parent.
     788
     789        Third parties should use this function to write configuration related to
     790        this extension.
     791
     792        We add the following fields to the document::
     793
     794           collectMode    //cb_config/mbox/collectMode
     795           compressMode   //cb_config/mbox/compressMode
     796
     797        We also add groups of the following items, one list element per
     798        item::
     799
     800           mboxFiles      //cb_config/mbox/file
     801           mboxDirs       //cb_config/mbox/dir
     802
     803        The mbox files and mbox directories are added by L{_addMboxFile} and
     804        L{_addMboxDir}.
     805
     806        @param xmlDom: DOM tree as from C{impl.createDocument()}.
     807        @param parentNode: Parent that the section should be appended to.
     808        """
     809        if self.mbox is not None:
     810           sectionNode = addContainerNode(xmlDom, parentNode, "mbox")
     811           addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode)
     812           addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode)
     813           if self.mbox.mboxFiles is not None:
     814              for mboxFile in self.mbox.mboxFiles:
     815                 LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile)
     816           if self.mbox.mboxDirs is not None:
     817              for mboxDir in self.mbox.mboxDirs:
     818                 LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)
     819
     820 -   def _parseXmlData(self, xmlData):
     821        """
     822        Internal method to parse an XML string into the object.
     823
     824        This method parses the XML document into a DOM tree (C{xmlDom}) and then
     825        calls a static method to parse the mbox configuration section.
     826
     827        @param xmlData: XML data to be parsed
     828        @type xmlData: String data
     829
     830        @raise ValueError: If the XML cannot be successfully parsed.
     831        """
     832        (xmlDom, parentNode) = createInputDom(xmlData)
     833        self._mbox = LocalConfig._parseMbox(parentNode)
     834
     835     @staticmethod
     836 -   def _parseMbox(parent):
     837        """
     838        Parses an mbox configuration section.
     839
     840        We read the following individual fields::
     841
     842           collectMode    //cb_config/mbox/collect_mode
     843           compressMode   //cb_config/mbox/compress_mode
     844
     845        We also read groups of the following item, one list element per
     846        item::
     847
     848           mboxFiles      //cb_config/mbox/file
     849           mboxDirs       //cb_config/mbox/dir
     850
     851        The mbox files are parsed by L{_parseMboxFiles} and the mbox
     852        directories are parsed by L{_parseMboxDirs}.
     853
     854        @param parent: Parent node to search beneath.
     855
     856        @return: C{MboxConfig} object or C{None} if the section does not exist.
     857        @raise ValueError: If some filled-in value is invalid.
     858        """
     859        mbox = None
     860        section = readFirstChild(parent, "mbox")
     861        if section is not None:
     862           mbox = MboxConfig()
     863           mbox.collectMode = readString(section, "collect_mode")
     864           mbox.compressMode = readString(section, "compress_mode")
     865           mbox.mboxFiles = LocalConfig._parseMboxFiles(section)
     866           mbox.mboxDirs = LocalConfig._parseMboxDirs(section)
     867        return mbox
     868
     869     @staticmethod
     870 -   def _parseMboxFiles(parent):
     871        """
     872        Reads a list of C{MboxFile} objects from immediately beneath the parent.
     873
     874        We read the following individual fields::
     875
     876           absolutePath            abs_path
     877           collectMode             collect_mode
     878           compressMode            compess_mode
     879
     880        @param parent: Parent node to search beneath.
     881
     882        @return: List of C{MboxFile} objects or C{None} if none are found.
     883        @raise ValueError: If some filled-in value is invalid.
     884        """
     885        lst = []
     886        for entry in readChildren(parent, "file"):
     887           if isElement(entry):
     888              mboxFile = MboxFile()
     889              mboxFile.absolutePath = readString(entry, "abs_path")
     890              mboxFile.collectMode = readString(entry, "collect_mode")
     891              mboxFile.compressMode = readString(entry, "compress_mode")
     892              lst.append(mboxFile)
     893        if lst == []:
     894           lst = None
     895        return lst
     896
     897     @staticmethod
     898 -   def _parseMboxDirs(parent):
     899        """
     900        Reads a list of C{MboxDir} objects from immediately beneath the parent.
     901
     902        We read the following individual fields::
     903
     904           absolutePath            abs_path
     905           collectMode             collect_mode
     906           compressMode            compess_mode
     907
     908        We also read groups of the following items, one list element per
     909        item::
     910
     911           relativeExcludePaths    exclude/rel_path
     912           excludePatterns         exclude/pattern
     913
     914        The exclusions are parsed by L{_parseExclusions}.
     915
     916        @param parent: Parent node to search beneath.
     917
     918        @return: List of C{MboxDir} objects or C{None} if none are found.
     919        @raise ValueError: If some filled-in value is invalid.
     920        """
     921        lst = []
     922        for entry in readChildren(parent, "dir"):
     923           if isElement(entry):
     924              mboxDir = MboxDir()
     925              mboxDir.absolutePath = readString(entry, "abs_path")
     926              mboxDir.collectMode = readString(entry, "collect_mode")
     927              mboxDir.compressMode = readString(entry, "compress_mode")
     928              (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry)
     929              lst.append(mboxDir)
     930        if lst == []:
     931           lst = None
     932        return lst
     933
     934     @staticmethod
     935 -   def _parseExclusions(parentNode):
     936        """
     937        Reads exclusions data from immediately beneath the parent.
     938
     939        We read groups of the following items, one list element per item::
     940
     941           relative    exclude/rel_path
     942           patterns    exclude/pattern
     943
     944        If there are none of some pattern (i.e. no relative path items) then
     945        C{None} will be returned for that item in the tuple.
     946
     947        @param parentNode: Parent node to search beneath.
     948
     949        @return: Tuple of (relative, patterns) exclusions.
     950        """
     951        section = readFirstChild(parentNode, "exclude")
     952        if section is None:
     953           return (None, None)
     954        else:
     955           relative = readStringList(section, "rel_path")
     956           patterns = readStringList(section, "pattern")
     957           return (relative, patterns)
     958
     959     @staticmethod
     960 -   def _addMboxFile(xmlDom, parentNode, mboxFile):
     961        """
     962        Adds an mbox file container as the next child of a parent.
     963
     964        We add the following fields to the document::
     965
     966           absolutePath            file/abs_path
     967           collectMode             file/collect_mode
     968           compressMode            file/compress_mode
     969
     970        The <file> node itself is created as the next child of the parent node.
     971        This method only adds one mbox file node.  The parent must loop for each
     972        mbox file in the C{MboxConfig} object.
     973
     974        If C{mboxFile} is C{None}, this method call will be a no-op.
     975
     976        @param xmlDom: DOM tree as from C{impl.createDocument()}.
     977        @param parentNode: Parent that the section should be appended to.
     978        @param mboxFile: MboxFile to be added to the document.
     979        """
     980        if mboxFile is not None:
     981           sectionNode = addContainerNode(xmlDom, parentNode, "file")
     982           addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath)
     983           addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode)
     984           addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)
     985
     986     @staticmethod
     987 -   def _addMboxDir(xmlDom, parentNode, mboxDir):
     988        """
     989        Adds an mbox directory container as the next child of a parent.
     990
     991        We add the following fields to the document::
     992
     993           absolutePath            dir/abs_path
     994           collectMode             dir/collect_mode
     995           compressMode            dir/compress_mode
     996
     997        We also add groups of the following items, one list element per item::
     998
     999           relativeExcludePaths    dir/exclude/rel_path
    1000           excludePatterns         dir/exclude/pattern
    1001
    1002        The <dir> node itself is created as the next child of the parent node.
    1003        This method only adds one mbox directory node.  The parent must loop for
    1004        each mbox directory in the C{MboxConfig} object.
    1005
    1006        If C{mboxDir} is C{None}, this method call will be a no-op.
    1007
    1008        @param xmlDom: DOM tree as from C{impl.createDocument()}.
    1009        @param parentNode: Parent that the section should be appended to.
    1010        @param mboxDir: MboxDir to be added to the document.
    1011        """
    1012        if mboxDir is not None:
    1013           sectionNode = addContainerNode(xmlDom, parentNode, "dir")
    1014           addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath)
    1015           addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode)
    1016           addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode)
    1017           if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or
    1018               (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])):
    1019              excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
    1020              if mboxDir.relativeExcludePaths is not None:
    1021                 for relativePath in mboxDir.relativeExcludePaths:
    1022                    addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
    1023              if mboxDir.excludePatterns is not None:
    1024                 for pattern in mboxDir.excludePatterns:
    1025                    addStringNode(xmlDom, excludeNode, "pattern", pattern)
    1026
    1027
    1028  ########################################################################
    1029  # Public functions
    1030  ########################################################################
    1031
    1032  ###########################
    1033  # executeAction() function
    1034  ###########################
    1035
    1036 -def executeAction(configPath, options, config):
    1037     """
    1038     Executes the mbox backup action.
    1039
    1040     @param configPath: Path to configuration file on disk.
    1041     @type configPath: String representing a path on disk.
    1042
    1043     @param options: Program command-line options.
    1044     @type options: Options object.
    1045
    1046     @param config: Program configuration.
    1047     @type config: Config object.
    1048
    1049     @raise ValueError: Under many generic error conditions
    1050     @raise IOError: If a backup could not be written for some reason.
    1051     """
    1052     logger.debug("Executing mbox extended action.")
    1053     newRevision = datetime.datetime.today()  # mark here so all actions are after this date/time
    1054     if config.options is None or config.collect is None:
    1055        raise ValueError("Cedar Backup configuration is not properly filled in.")
    1056     local = LocalConfig(xmlPath=configPath)
    1057     todayIsStart = isStartOfWeek(config.options.startingDay)
    1058     fullBackup = options.full or todayIsStart
    1059     logger.debug("Full backup flag is [%s]", fullBackup)
    1060     if local.mbox.mboxFiles is not None:
    1061        for mboxFile in local.mbox.mboxFiles:
    1062           logger.debug("Working with mbox file [%s]", mboxFile.absolutePath)
    1063           collectMode = _getCollectMode(local, mboxFile)
    1064           compressMode = _getCompressMode(local, mboxFile)
    1065           lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode)
    1066           if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
    1067              logger.debug("Mbox file meets criteria to be backed up today.")
    1068              _backupMboxFile(config, mboxFile.absolutePath, fullBackup,
    1069                              collectMode, compressMode, lastRevision, newRevision)
    1070           else:
    1071              logger.debug("Mbox file will not be backed up, per collect mode.")
    1072           if collectMode == 'incr':
    1073              _writeNewRevision(config, mboxFile, newRevision)
    1074     if local.mbox.mboxDirs is not None:
    1075        for mboxDir in local.mbox.mboxDirs:
    1076           logger.debug("Working with mbox directory [%s]", mboxDir.absolutePath)
    1077           collectMode = _getCollectMode(local, mboxDir)
    1078           compressMode = _getCompressMode(local, mboxDir)
    1079           lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode)
    1080           (excludePaths, excludePatterns) = _getExclusions(mboxDir)
    1081           if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
    1082              logger.debug("Mbox directory meets criteria to be backed up today.")
    1083              _backupMboxDir(config, mboxDir.absolutePath,
    1084                             fullBackup, collectMode, compressMode,
    1085                             lastRevision, newRevision,
    1086                             excludePaths, excludePatterns)
    1087           else:
    1088              logger.debug("Mbox directory will not be backed up, per collect mode.")
    1089           if collectMode == 'incr':
    1090              _writeNewRevision(config, mboxDir, newRevision)
    1091     logger.info("Executed the mbox extended action successfully.")
    1092
    1093 -def _getCollectMode(local, item):
    1094     """
    1095     Gets the collect mode that should be used for an mbox file or directory.
    1096     Use file- or directory-specific value if possible, otherwise take from mbox section.
    1097     @param local: LocalConfig object.
    1098     @param item: Mbox file or directory
    1099     @return: Collect mode to use.
    1100     """
    1101     if item.collectMode is None:
    1102        collectMode = local.mbox.collectMode
    1103     else:
    1104        collectMode = item.collectMode
    1105     logger.debug("Collect mode is [%s]", collectMode)
    1106     return collectMode
    1107
    1108 -def _getCompressMode(local, item):
    1109     """
    1110     Gets the compress mode that should be used for an mbox file or directory.
    1111     Use file- or directory-specific value if possible, otherwise take from mbox section.
    1112     @param local: LocalConfig object.
    1113     @param item: Mbox file or directory
    1114     @return: Compress mode to use.
    1115     """
    1116     if item.compressMode is None:
    1117        compressMode = local.mbox.compressMode
    1118     else:
    1119        compressMode = item.compressMode
    1120     logger.debug("Compress mode is [%s]", compressMode)
    1121     return compressMode
    1122
    1123 -def _getRevisionPath(config, item):
    1124     """
    1125     Gets the path to the revision file associated with a repository.
    1126     @param config: Cedar Backup configuration.
    1127     @param item: Mbox file or directory
    1128     @return: Absolute path to the revision file associated with the repository.
    1129     """
    1130     normalized = buildNormalizedPath(item.absolutePath)
    1131     filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
    1132     revisionPath = os.path.join(config.options.workingDir, filename)
    1133     logger.debug("Revision file path is [%s]", revisionPath)
    1134     return revisionPath
    1135
    1136 -def _loadLastRevision(config, item, fullBackup, collectMode):
    1137     """
    1138     Loads the last revision date for this item from disk and returns it.
    1139
    1140     If this is a full backup, or if the revision file cannot be loaded for some
    1141     reason, then C{None} is returned.  This indicates that there is no previous
    1142     revision, so the entire mail file or directory should be backed up.
    1143
    1144     @note: We write the actual revision object to disk via pickle, so we don't
    1145     deal with the datetime precision or format at all.  Whatever's in the object
    1146     is what we write.
    1147
    1148     @param config: Cedar Backup configuration.
    1149     @param item: Mbox file or directory
    1150     @param fullBackup: Indicates whether this is a full backup
    1151     @param collectMode: Indicates the collect mode for this item
    1152
    1153     @return: Revision date as a datetime.datetime object or C{None}.
    1154     """
    1155     revisionPath = _getRevisionPath(config, item)
    1156     if fullBackup:
    1157        revisionDate = None
    1158        logger.debug("Revision file ignored because this is a full backup.")
    1159     elif collectMode in ['weekly', 'daily']:
    1160        revisionDate = None
    1161        logger.debug("No revision file based on collect mode [%s].", collectMode)
    1162     else:
    1163        logger.debug("Revision file will be used for non-full incremental backup.")
    1164        if not os.path.isfile(revisionPath):
    1165           revisionDate = None
    1166           logger.debug("Revision file [%s] does not exist on disk.", revisionPath)
    1167        else:
    1168           try:
    1169              revisionDate = pickle.load(open(revisionPath, "r"))
    1170              logger.debug("Loaded revision file [%s] from disk: [%s]", revisionPath, revisionDate)
    1171           except:
    1172              revisionDate = None
    1173              logger.error("Failed loading revision file [%s] from disk.", revisionPath)
    1174     return revisionDate
    1175
    1176 -def _writeNewRevision(config, item, newRevision):
    1177     """
    1178     Writes new revision information to disk.
    1179
    1180     If we can't write the revision file successfully for any reason, we'll log
    1181     the condition but won't throw an exception.
    1182
    1183     @note: We write the actual revision object to disk via pickle, so we don't
    1184     deal with the datetime precision or format at all.  Whatever's in the object
    1185     is what we write.
    1186
    1187     @param config: Cedar Backup configuration.
    1188     @param item: Mbox file or directory
    1189     @param newRevision: Revision date as a datetime.datetime object.
    1190     """
    1191     revisionPath = _getRevisionPath(config, item)
    1192     try:
    1193        pickle.dump(newRevision, open(revisionPath, "w"))
    1194        changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
    1195        logger.debug("Wrote new revision file [%s] to disk: [%s]", revisionPath, newRevision)
    1196     except:
    1197        logger.error("Failed to write revision file [%s] to disk.", revisionPath)
    1198
    1199 -def _getExclusions(mboxDir):
    1200     """
    1201     Gets exclusions (file and patterns) associated with an mbox directory.
    1202
    1203     The returned files value is a list of absolute paths to be excluded from the
    1204     backup for a given directory.  It is derived from the mbox directory's
    1205     relative exclude paths.
    1206
    1207     The returned patterns value is a list of patterns to be excluded from the
    1208     backup for a given directory.  It is derived from the mbox directory's list
    1209     of patterns.
    1210
    1211     @param mboxDir: Mbox directory object.
    1212
    1213     @return: Tuple (files, patterns) indicating what to exclude.
    1214     """
    1215     paths = []
    1216     if mboxDir.relativeExcludePaths is not None:
    1217        for relativePath in mboxDir.relativeExcludePaths:
    1218           paths.append(os.path.join(mboxDir.absolutePath, relativePath))
    1219     patterns = []
    1220     if mboxDir.excludePatterns is not None:
    1221        patterns.extend(mboxDir.excludePatterns)
    1222     logger.debug("Exclude paths: %s", paths)
    1223     logger.debug("Exclude patterns: %s", patterns)
    1224     return(paths, patterns)
    1225
    1226 -def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
    1227     """
    1228     Gets the backup file path (including correct extension) associated with an mbox path.
    1229
    1230     We assume that if the target directory is passed in, that we're backing up a
    1231     directory.  Under these circumstances, we'll just use the basename of the
    1232     individual path as the output file.
    1233
    1234     @note: The backup path only contains the current date in YYYYMMDD format,
    1235     but that's OK because the index information (stored elsewhere) is the actual
    1236     date object.
    1237
    1238     @param config: Cedar Backup configuration.
    1239     @param mboxPath: Path to the indicated mbox file or directory
    1240     @param compressMode: Compress mode to use for this mbox path
    1241     @param newRevision: Revision this backup path represents
    1242     @param targetDir: Target directory in which the path should exist
    1243
    1244     @return: Absolute path to the backup file associated with the repository.
    1245     """
    1246     if targetDir is None:
    1247        normalizedPath = buildNormalizedPath(mboxPath)
    1248        revisionDate = newRevision.strftime("%Y%m%d")
    1249        filename = "mbox-%s-%s" % (revisionDate, normalizedPath)
    1250     else:
    1251        filename = os.path.basename(mboxPath)
    1252     if compressMode == 'gzip':
    1253        filename = "%s.gz" % filename
    1254     elif compressMode == 'bzip2':
    1255        filename = "%s.bz2" % filename
    1256     if targetDir is None:
    1257        backupPath = os.path.join(config.collect.targetDir, filename)
    1258     else:
    1259        backupPath = os.path.join(targetDir, filename)
    1260     logger.debug("Backup file path is [%s]", backupPath)
    1261     return backupPath
    1262
    1263 -def _getTarfilePath(config, mboxPath, compressMode, newRevision):
    1264     """
    1265     Gets the tarfile backup file path (including correct extension) associated
    1266     with an mbox path.
    1267
    1268     Along with the path, the tar archive mode is returned in a form that can
    1269     be used with L{BackupFileList.generateTarfile}.
    1270
    1271     @note: The tarfile path only contains the current date in YYYYMMDD format,
    1272     but that's OK because the index information (stored elsewhere) is the actual
    1273     date object.
    1274
    1275     @param config: Cedar Backup configuration.
    1276     @param mboxPath: Path to the indicated mbox file or directory
    1277     @param compressMode: Compress mode to use for this mbox path
    1278     @param newRevision: Revision this backup path represents
    1279
    1280     @return: Tuple of (absolute path to tarfile, tar archive mode)
    1281     """
    1282     normalizedPath = buildNormalizedPath(mboxPath)
    1283     revisionDate = newRevision.strftime("%Y%m%d")
    1284     filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath)
    1285     if compressMode == 'gzip':
    1286        filename = "%s.gz" % filename
    1287        archiveMode = "targz"
    1288     elif compressMode == 'bzip2':
    1289        filename = "%s.bz2" % filename
    1290        archiveMode = "tarbz2"
    1291     else:
    1292        archiveMode = "tar"
    1293     tarfilePath = os.path.join(config.collect.targetDir, filename)
    1294     logger.debug("Tarfile path is [%s]", tarfilePath)
    1295     return (tarfilePath, archiveMode)
    1296
    1297 -def _getOutputFile(backupPath, compressMode):
    1298     """
    1299     Opens the output file used for saving backup information.
    1300
    1301     If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
    1302     compress mode is "bzip2", we'll open a C{BZ2File}.  Otherwise, we'll just
    1303     return an object from the normal C{open()} method.
    1304
    1305     @param backupPath: Path to file to open.
    1306     @param compressMode: Compress mode of file ("none", "gzip", "bzip").
    1307
    1308     @return: Output file object.
    1309     """
    1310     if compressMode == "gzip":
    1311        return GzipFile(backupPath, "w")
    1312     elif compressMode == "bzip2":
    1313        return BZ2File(backupPath, "w")
    1314     else:
    1315        return open(backupPath, "w")
    1316
    1317 -def _backupMboxFile(config, absolutePath,
    1318                      fullBackup, collectMode, compressMode,
    1319                      lastRevision, newRevision, targetDir=None):
    1320     """
    1321     Backs up an individual mbox file.
    1322
    1323     @param config: Cedar Backup configuration.
    1324     @param absolutePath: Path to mbox file to back up.
    1325     @param fullBackup: Indicates whether this should be a full backup.
    1326     @param collectMode: Indicates the collect mode for this item
    1327     @param compressMode: Compress mode of file ("none", "gzip", "bzip")
    1328     @param lastRevision: Date of last backup as datetime.datetime
    1329     @param newRevision: Date of new (current) backup as datetime.datetime
    1330     @param targetDir: Target directory to write the backed-up file into
    1331
    1332     @raise ValueError: If some value is missing or invalid.
    1333     @raise IOError: If there is a problem backing up the mbox file.
    1334     """
    1335     backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir)
    1336     outputFile = _getOutputFile(backupPath, compressMode)
    1337     if fullBackup or collectMode != "incr" or lastRevision is None:
    1338        args = [ "-a", "-u", absolutePath, ]  # remove duplicates but fetch entire mailbox
    1339     else:
    1340        revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S")  # ISO-8601 format; grepmail calls Date::Parse::str2time()
    1341        args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ]
    1342     command = resolveCommand(GREPMAIL_COMMAND)
    1343     result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0]
    1344     if result != 0:
    1345        raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath))
    1346     logger.debug("Completed backing up mailbox [%s].", absolutePath)
    1347     return backupPath
    1348
    1349 -def _backupMboxDir(config, absolutePath,
    1350                     fullBackup, collectMode, compressMode,
    1351                     lastRevision, newRevision,
    1352                     excludePaths, excludePatterns):
    1353     """
    1354     Backs up a directory containing mbox files.
    1355
    1356     @param config: Cedar Backup configuration.
    1357     @param absolutePath: Path to mbox directory to back up.
    1358     @param fullBackup: Indicates whether this should be a full backup.
    1359     @param collectMode: Indicates the collect mode for this item
    1360     @param compressMode: Compress mode of file ("none", "gzip", "bzip")
    1361     @param lastRevision: Date of last backup as datetime.datetime
    1362     @param newRevision: Date of new (current) backup as datetime.datetime
    1363     @param excludePaths: List of absolute paths to exclude.
    1364     @param excludePatterns: List of patterns to exclude.
    1365
    1366     @raise ValueError: If some value is missing or invalid.
    1367     @raise IOError: If there is a problem backing up the mbox file.
    1368     """
    1369     try:
    1370        tmpdir = tempfile.mkdtemp(dir=config.options.workingDir)
    1371        mboxList = FilesystemList()
    1372        mboxList.excludeDirs = True
    1373        mboxList.excludePaths = excludePaths
    1374        mboxList.excludePatterns = excludePatterns
    1375        mboxList.addDirContents(absolutePath, recursive=False)
    1376        tarList = BackupFileList()
    1377        for item in mboxList:
    1378           backupPath = _backupMboxFile(config, item, fullBackup,
    1379                                        collectMode, "none",  # no need to compress inside compressed tar
    1380                                        lastRevision, newRevision,
    1381                                        targetDir=tmpdir)
    1382           tarList.addFile(backupPath)
    1383        (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision)
    1384        tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True)
    1385        changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
    1386        logger.debug("Completed backing up directory [%s].", absolutePath)
    1387     finally:
    1388        try:
    1389           for item in tarList:
    1390              if os.path.exists(item):
    1391                 try:
    1392                    os.remove(item)
    1393                 except: pass
    1394        except: pass
    1395        try:
    1396           os.rmdir(tmpdir)
    1397        except: pass
    1398