CedarBackup3.extend.mbox

Package CedarBackup3 :: Package extend :: Module mbox |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup3.extend.mbox

       1  # -*- coding: iso-8859-1 -*-
       2  # vim: set ft=python ts=3 sw=3 expandtab:
       3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
       4  #
       5  #              C E D A R
       6  #          S O L U T I O N S       "Software done right."
       7  #           S O F T W A R E
       8  #
       9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      10  #
      11  # Copyright (c) 2006-2007,2010,2015 Kenneth J. Pronovici.
      12  # All rights reserved.
      13  #
      14  # This program is free software; you can redistribute it and/or
      15  # modify it under the terms of the GNU General Public License,
      16  # Version 2, as published by the Free Software Foundation.
      17  #
      18  # This program is distributed in the hope that it will be useful,
      19  # but WITHOUT ANY WARRANTY; without even the implied warranty of
      20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
      21  #
      22  # Copies of the GNU General Public License are available from
      23  # the Free Software Foundation website, http://www.gnu.org/.
      24  #
      25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      26  #
      27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
      28  # Language : Python 3 (>= 3.4)
      29  # Project  : Official Cedar Backup Extensions
      30  # Purpose  : Provides an extension to back up mbox email files.
      31  #
      32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      33
      34  ########################################################################
      35  # Module documentation
      36  ########################################################################
      37
      38  """
      39  Provides an extension to back up mbox email files.
      40
      41  Backing up email
      42  ================
      43
      44     Email folders (often stored as mbox flatfiles) are not well-suited being backed
      45     up with an incremental backup like the one offered by Cedar Backup.  This is
      46     because mbox files often change on a daily basis, forcing the incremental
      47     backup process to back them up every day in order to avoid losing data.  This
      48     can result in quite a bit of wasted space when backing up large folders.  (Note
      49     that the alternative maildir format does not share this problem, since it
      50     typically uses one file per message.)
      51
      52     One solution to this problem is to design a smarter incremental backup process,
      53     which backs up baseline content on the first day of the week, and then backs up
      54     only new messages added to that folder on every other day of the week.  This way,
      55     the backup for any single day is only as large as the messages placed into the
      56     folder on that day.  The backup isn't as "perfect" as the incremental backup
      57     process, because it doesn't preserve information about messages deleted from
      58     the backed-up folder.  However, it should be much more space-efficient, and
      59     in a recovery situation, it seems better to restore too much data rather
      60     than too little.
      61
      62  What is this extension?
      63  =======================
      64
      65     This is a Cedar Backup extension used to back up mbox email files via the Cedar
      66     Backup command line.  Individual mbox files or directories containing mbox
      67     files can be backed up using the same collect modes allowed for filesystems in
      68     the standard Cedar Backup collect action: weekly, daily, incremental.  It
      69     implements the "smart" incremental backup process discussed above, using
      70     functionality provided by the C{grepmail} utility.
      71
      72     This extension requires a new configuration section <mbox> and is intended to
      73     be run either immediately before or immediately after the standard collect
      74     action.  Aside from its own configuration, it requires the options and collect
      75     configuration sections in the standard Cedar Backup configuration file.
      76
      77     The mbox action is conceptually similar to the standard collect action,
      78     except that mbox directories are not collected recursively.  This implies
      79     some configuration changes (i.e. there's no need for global exclusions or an
      80     ignore file).  If you back up a directory, all of the mbox files in that
      81     directory are backed up into a single tar file using the indicated
      82     compression method.
      83
      84  @author: Kenneth J. Pronovici <pronovic@ieee.org>
      85  """
      86
      87  ########################################################################
      88  # Imported modules
      89  ########################################################################
      90
      91  # System modules
      92  import os
      93  import logging
      94  import datetime
      95  import pickle
      96  import tempfile
      97  from bz2 import BZ2File
      98  from gzip import GzipFile
      99  from functools import total_ordering
     100
     101  # Cedar Backup modules
     102  from CedarBackup3.filesystem import FilesystemList, BackupFileList
     103  from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode
     104  from CedarBackup3.xmlutil import isElement, readChildren, readFirstChild, readString, readStringList
     105  from CedarBackup3.config import VALID_COLLECT_MODES, VALID_COMPRESS_MODES
     106  from CedarBackup3.util import isStartOfWeek, buildNormalizedPath
     107  from CedarBackup3.util import resolveCommand, executeCommand
     108  from CedarBackup3.util import ObjectTypeList, UnorderedList, RegexList, encodePath, changeOwnership
     109
     110
     111  ########################################################################
     112  # Module-wide constants and variables
     113  ########################################################################
     114
     115  logger = logging.getLogger("CedarBackup3.log.extend.mbox")
     116
     117  GREPMAIL_COMMAND = [ "grepmail", ]
     118  REVISION_PATH_EXTENSION = "mboxlast"
     119
     120
     121  ########################################################################
     122  # MboxFile class definition
     123  ########################################################################
     124
     125  @total_ordering
     126 -class MboxFile(object):
     127
     128     """
     129     Class representing mbox file configuration..
     130
     131     The following restrictions exist on data in this class:
     132
     133        - The absolute path must be absolute.
     134        - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
     135        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
     136
     137     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
     138           absolutePath, collectMode, compressMode
     139     """
     140
     141 -   def __init__(self, absolutePath=None, collectMode=None, compressMode=None):
     142        """
     143        Constructor for the C{MboxFile} class.
     144
     145        You should never directly instantiate this class.
     146
     147        @param absolutePath: Absolute path to an mbox file on disk.
     148        @param collectMode: Overridden collect mode for this directory.
     149        @param compressMode: Overridden compression mode for this directory.
     150        """
     151        self._absolutePath = None
     152        self._collectMode = None
     153        self._compressMode = None
     154        self.absolutePath = absolutePath
     155        self.collectMode = collectMode
     156        self.compressMode = compressMode
     157
     158 -   def __repr__(self):
     159        """
     160        Official string representation for class instance.
     161        """
     162        return "MboxFile(%s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode)
     163
     164 -   def __str__(self):
     165        """
     166        Informal string representation for class instance.
     167        """
     168        return self.__repr__()
     169
     170 -   def __eq__(self, other):
     171        """Equals operator, iplemented in terms of original Python 2 compare operator."""
     172        return self.__cmp__(other) == 0
     173
     174 -   def __lt__(self, other):
     175        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
     176        return self.__cmp__(other) < 0
     177
     178 -   def __gt__(self, other):
     179        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
     180        return self.__cmp__(other) > 0
     181
     182 -   def __cmp__(self, other):
     183        """
     184        Original Python 2 comparison operator.
     185        @param other: Other object to compare to.
     186        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     187        """
     188        if other is None:
     189           return 1
     190        if self.absolutePath != other.absolutePath:
     191           if str(self.absolutePath or "") < str(other.absolutePath or ""):
     192              return -1
     193           else:
     194              return 1
     195        if self.collectMode != other.collectMode:
     196           if str(self.collectMode or "") < str(other.collectMode or ""):
     197              return -1
     198           else:
     199              return 1
     200        if self.compressMode != other.compressMode:
     201           if str(self.compressMode or "") < str(other.compressMode or ""):
     202              return -1
     203           else:
     204              return 1
     205        return 0
     206
     207 -   def _setAbsolutePath(self, value):
     208        """
     209        Property target used to set the absolute path.
     210        The value must be an absolute path if it is not C{None}.
     211        It does not have to exist on disk at the time of assignment.
     212        @raise ValueError: If the value is not an absolute path.
     213        @raise ValueError: If the value cannot be encoded properly.
     214        """
     215        if value is not None:
     216           if not os.path.isabs(value):
     217              raise ValueError("Absolute path must be, er, an absolute path.")
     218        self._absolutePath = encodePath(value)
     219
     220 -   def _getAbsolutePath(self):
     221        """
     222        Property target used to get the absolute path.
     223        """
     224        return self._absolutePath
     225
     226 -   def _setCollectMode(self, value):
     227        """
     228        Property target used to set the collect mode.
     229        If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
     230        @raise ValueError: If the value is not valid.
     231        """
     232        if value is not None:
     233           if value not in VALID_COLLECT_MODES:
     234              raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
     235        self._collectMode = value
     236
     237 -   def _getCollectMode(self):
     238        """
     239        Property target used to get the collect mode.
     240        """
     241        return self._collectMode
     242
     243 -   def _setCompressMode(self, value):
     244        """
     245        Property target used to set the compress mode.
     246        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
     247        @raise ValueError: If the value is not valid.
     248        """
     249        if value is not None:
     250           if value not in VALID_COMPRESS_MODES:
     251              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
     252        self._compressMode = value
     253
     254 -   def _getCompressMode(self):
     255        """
     256        Property target used to get the compress mode.
     257        """
     258        return self._compressMode
     259
     260     absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox file.")
     261     collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox file.")
     262     compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox file.")
     263
     264
     265  ########################################################################
     266  # MboxDir class definition
     267  ########################################################################
     268
     269  @total_ordering
     270 -class MboxDir(object):
     271
     272     """
     273     Class representing mbox directory configuration..
     274
     275     The following restrictions exist on data in this class:
     276
     277        - The absolute path must be absolute.
     278        - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
     279        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
     280
     281     Unlike collect directory configuration, this is the only place exclusions
     282     are allowed (no global exclusions at the <mbox> configuration level).  Also,
     283     we only allow relative exclusions and there is no configured ignore file.
     284     This is because mbox directory backups are not recursive.
     285
     286     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
     287           absolutePath, collectMode, compressMode, relativeExcludePaths,
     288           excludePatterns
     289     """
     290
     291 -   def __init__(self, absolutePath=None, collectMode=None, compressMode=None,
     292                  relativeExcludePaths=None, excludePatterns=None):
     293        """
     294        Constructor for the C{MboxDir} class.
     295
     296        You should never directly instantiate this class.
     297
     298        @param absolutePath: Absolute path to a mbox file on disk.
     299        @param collectMode: Overridden collect mode for this directory.
     300        @param compressMode: Overridden compression mode for this directory.
     301        @param relativeExcludePaths: List of relative paths to exclude.
     302        @param excludePatterns: List of regular expression patterns to exclude
     303        """
     304        self._absolutePath = None
     305        self._collectMode = None
     306        self._compressMode = None
     307        self._relativeExcludePaths = None
     308        self._excludePatterns = None
     309        self.absolutePath = absolutePath
     310        self.collectMode = collectMode
     311        self.compressMode = compressMode
     312        self.relativeExcludePaths = relativeExcludePaths
     313        self.excludePatterns = excludePatterns
     314
     315 -   def __repr__(self):
     316        """
     317        Official string representation for class instance.
     318        """
     319        return "MboxDir(%s, %s, %s, %s, %s)" % (self.absolutePath, self.collectMode, self.compressMode,
     320                                                self.relativeExcludePaths, self.excludePatterns)
     321
     322 -   def __str__(self):
     323        """
     324        Informal string representation for class instance.
     325        """
     326        return self.__repr__()
     327
     328 -   def __eq__(self, other):
     329        """Equals operator, iplemented in terms of original Python 2 compare operator."""
     330        return self.__cmp__(other) == 0
     331
     332 -   def __lt__(self, other):
     333        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
     334        return self.__cmp__(other) < 0
     335
     336 -   def __gt__(self, other):
     337        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
     338        return self.__cmp__(other) > 0
     339
     340 -   def __cmp__(self, other):
     341        """
     342        Original Python 2 comparison operator.
     343        @param other: Other object to compare to.
     344        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     345        """
     346        if other is None:
     347           return 1
     348        if self.absolutePath != other.absolutePath:
     349           if str(self.absolutePath or "") < str(other.absolutePath or ""):
     350              return -1
     351           else:
     352              return 1
     353        if self.collectMode != other.collectMode:
     354           if str(self.collectMode or "") < str(other.collectMode or ""):
     355              return -1
     356           else:
     357              return 1
     358        if self.compressMode != other.compressMode:
     359           if str(self.compressMode or "") < str(other.compressMode or ""):
     360              return -1
     361           else:
     362              return 1
     363        if self.relativeExcludePaths != other.relativeExcludePaths:
     364           if self.relativeExcludePaths < other.relativeExcludePaths:
     365              return -1
     366           else:
     367              return 1
     368        if self.excludePatterns != other.excludePatterns:
     369           if self.excludePatterns < other.excludePatterns:
     370              return -1
     371           else:
     372              return 1
     373        return 0
     374
     375 -   def _setAbsolutePath(self, value):
     376        """
     377        Property target used to set the absolute path.
     378        The value must be an absolute path if it is not C{None}.
     379        It does not have to exist on disk at the time of assignment.
     380        @raise ValueError: If the value is not an absolute path.
     381        @raise ValueError: If the value cannot be encoded properly.
     382        """
     383        if value is not None:
     384           if not os.path.isabs(value):
     385              raise ValueError("Absolute path must be, er, an absolute path.")
     386        self._absolutePath = encodePath(value)
     387
     388 -   def _getAbsolutePath(self):
     389        """
     390        Property target used to get the absolute path.
     391        """
     392        return self._absolutePath
     393
     394 -   def _setCollectMode(self, value):
     395        """
     396        Property target used to set the collect mode.
     397        If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
     398        @raise ValueError: If the value is not valid.
     399        """
     400        if value is not None:
     401           if value not in VALID_COLLECT_MODES:
     402              raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
     403        self._collectMode = value
     404
     405 -   def _getCollectMode(self):
     406        """
     407        Property target used to get the collect mode.
     408        """
     409        return self._collectMode
     410
     411 -   def _setCompressMode(self, value):
     412        """
     413        Property target used to set the compress mode.
     414        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
     415        @raise ValueError: If the value is not valid.
     416        """
     417        if value is not None:
     418           if value not in VALID_COMPRESS_MODES:
     419              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
     420        self._compressMode = value
     421
     422 -   def _getCompressMode(self):
     423        """
     424        Property target used to get the compress mode.
     425        """
     426        return self._compressMode
     427
     428 -   def _setRelativeExcludePaths(self, value):
     429        """
     430        Property target used to set the relative exclude paths list.
     431        Elements do not have to exist on disk at the time of assignment.
     432        """
     433        if value is None:
     434           self._relativeExcludePaths = None
     435        else:
     436           try:
     437              saved = self._relativeExcludePaths
     438              self._relativeExcludePaths = UnorderedList()
     439              self._relativeExcludePaths.extend(value)
     440           except Exception as e:
     441              self._relativeExcludePaths = saved
     442              raise e
     443
     444 -   def _getRelativeExcludePaths(self):
     445        """
     446        Property target used to get the relative exclude paths list.
     447        """
     448        return self._relativeExcludePaths
     449
     450 -   def _setExcludePatterns(self, value):
     451        """
     452        Property target used to set the exclude patterns list.
     453        """
     454        if value is None:
     455           self._excludePatterns = None
     456        else:
     457           try:
     458              saved = self._excludePatterns
     459              self._excludePatterns = RegexList()
     460              self._excludePatterns.extend(value)
     461           except Exception as e:
     462              self._excludePatterns = saved
     463              raise e
     464
     465 -   def _getExcludePatterns(self):
     466        """
     467        Property target used to get the exclude patterns list.
     468        """
     469        return self._excludePatterns
     470
     471     absolutePath = property(_getAbsolutePath, _setAbsolutePath, None, doc="Absolute path to the mbox directory.")
     472     collectMode = property(_getCollectMode, _setCollectMode, None, doc="Overridden collect mode for this mbox directory.")
     473     compressMode = property(_getCompressMode, _setCompressMode, None, doc="Overridden compress mode for this mbox directory.")
     474     relativeExcludePaths = property(_getRelativeExcludePaths, _setRelativeExcludePaths, None, "List of relative paths to exclude.")
     475     excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expression patterns to exclude.")
     476
     477
     478  ########################################################################
     479  # MboxConfig class definition
     480  ########################################################################
     481
     482  @total_ordering
     483 -class MboxConfig(object):
     484
     485     """
     486     Class representing mbox configuration.
     487
     488     Mbox configuration is used for backing up mbox email files.
     489
     490     The following restrictions exist on data in this class:
     491
     492        - The collect mode must be one of the values in L{VALID_COLLECT_MODES}.
     493        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
     494        - The C{mboxFiles} list must be a list of C{MboxFile} objects
     495        - The C{mboxDirs} list must be a list of C{MboxDir} objects
     496
     497     For the C{mboxFiles} and C{mboxDirs} lists, validation is accomplished
     498     through the L{util.ObjectTypeList} list implementation that overrides common
     499     list methods and transparently ensures that each element is of the proper
     500     type.
     501
     502     Unlike collect configuration, no global exclusions are allowed on this
     503     level.  We only allow relative exclusions at the mbox directory level.
     504     Also, there is no configured ignore file.  This is because mbox directory
     505     backups are not recursive.
     506
     507     @note: Lists within this class are "unordered" for equality comparisons.
     508
     509     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
     510           collectMode, compressMode, mboxFiles, mboxDirs
     511     """
     512
     513 -   def __init__(self, collectMode=None, compressMode=None, mboxFiles=None, mboxDirs=None):
     514        """
     515        Constructor for the C{MboxConfig} class.
     516
     517        @param collectMode: Default collect mode.
     518        @param compressMode: Default compress mode.
     519        @param mboxFiles: List of mbox files to back up
     520        @param mboxDirs: List of mbox directories to back up
     521
     522        @raise ValueError: If one of the values is invalid.
     523        """
     524        self._collectMode = None
     525        self._compressMode = None
     526        self._mboxFiles = None
     527        self._mboxDirs = None
     528        self.collectMode = collectMode
     529        self.compressMode = compressMode
     530        self.mboxFiles = mboxFiles
     531        self.mboxDirs = mboxDirs
     532
     533 -   def __repr__(self):
     534        """
     535        Official string representation for class instance.
     536        """
     537        return "MboxConfig(%s, %s, %s, %s)" % (self.collectMode, self.compressMode, self.mboxFiles, self.mboxDirs)
     538
     539 -   def __str__(self):
     540        """
     541        Informal string representation for class instance.
     542        """
     543        return self.__repr__()
     544
     545 -   def __eq__(self, other):
     546        """Equals operator, iplemented in terms of original Python 2 compare operator."""
     547        return self.__cmp__(other) == 0
     548
     549 -   def __lt__(self, other):
     550        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
     551        return self.__cmp__(other) < 0
     552
     553 -   def __gt__(self, other):
     554        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
     555        return self.__cmp__(other) > 0
     556
     557 -   def __cmp__(self, other):
     558        """
     559        Original Python 2 comparison operator.
     560        Lists within this class are "unordered" for equality comparisons.
     561        @param other: Other object to compare to.
     562        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     563        """
     564        if other is None:
     565           return 1
     566        if self.collectMode != other.collectMode:
     567           if str(self.collectMode or "") < str(other.collectMode or ""):
     568              return -1
     569           else:
     570              return 1
     571        if self.compressMode != other.compressMode:
     572           if str(self.compressMode or "") < str(other.compressMode or ""):
     573              return -1
     574           else:
     575              return 1
     576        if self.mboxFiles != other.mboxFiles:
     577           if self.mboxFiles < other.mboxFiles:
     578              return -1
     579           else:
     580              return 1
     581        if self.mboxDirs != other.mboxDirs:
     582           if self.mboxDirs < other.mboxDirs:
     583              return -1
     584           else:
     585              return 1
     586        return 0
     587
     588 -   def _setCollectMode(self, value):
     589        """
     590        Property target used to set the collect mode.
     591        If not C{None}, the mode must be one of the values in L{VALID_COLLECT_MODES}.
     592        @raise ValueError: If the value is not valid.
     593        """
     594        if value is not None:
     595           if value not in VALID_COLLECT_MODES:
     596              raise ValueError("Collect mode must be one of %s." % VALID_COLLECT_MODES)
     597        self._collectMode = value
     598
     599 -   def _getCollectMode(self):
     600        """
     601        Property target used to get the collect mode.
     602        """
     603        return self._collectMode
     604
     605 -   def _setCompressMode(self, value):
     606        """
     607        Property target used to set the compress mode.
     608        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
     609        @raise ValueError: If the value is not valid.
     610        """
     611        if value is not None:
     612           if value not in VALID_COMPRESS_MODES:
     613              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
     614        self._compressMode = value
     615
     616 -   def _getCompressMode(self):
     617        """
     618        Property target used to get the compress mode.
     619        """
     620        return self._compressMode
     621
     622 -   def _setMboxFiles(self, value):
     623        """
     624        Property target used to set the mboxFiles list.
     625        Either the value must be C{None} or each element must be an C{MboxFile}.
     626        @raise ValueError: If the value is not an C{MboxFile}
     627        """
     628        if value is None:
     629           self._mboxFiles = None
     630        else:
     631           try:
     632              saved = self._mboxFiles
     633              self._mboxFiles = ObjectTypeList(MboxFile, "MboxFile")
     634              self._mboxFiles.extend(value)
     635           except Exception as e:
     636              self._mboxFiles = saved
     637              raise e
     638
     639 -   def _getMboxFiles(self):
     640        """
     641        Property target used to get the mboxFiles list.
     642        """
     643        return self._mboxFiles
     644
     645 -   def _setMboxDirs(self, value):
     646        """
     647        Property target used to set the mboxDirs list.
     648        Either the value must be C{None} or each element must be an C{MboxDir}.
     649        @raise ValueError: If the value is not an C{MboxDir}
     650        """
     651        if value is None:
     652           self._mboxDirs = None
     653        else:
     654           try:
     655              saved = self._mboxDirs
     656              self._mboxDirs = ObjectTypeList(MboxDir, "MboxDir")
     657              self._mboxDirs.extend(value)
     658           except Exception as e:
     659              self._mboxDirs = saved
     660              raise e
     661
     662 -   def _getMboxDirs(self):
     663        """
     664        Property target used to get the mboxDirs list.
     665        """
     666        return self._mboxDirs
     667
     668     collectMode = property(_getCollectMode, _setCollectMode, None, doc="Default collect mode.")
     669     compressMode = property(_getCompressMode, _setCompressMode, None, doc="Default compress mode.")
     670     mboxFiles = property(_getMboxFiles, _setMboxFiles, None, doc="List of mbox files to back up.")
     671     mboxDirs = property(_getMboxDirs, _setMboxDirs, None, doc="List of mbox directories to back up.")
     672
     673
     674  ########################################################################
     675  # LocalConfig class definition
     676  ########################################################################
     677
     678  @total_ordering
     679 -class LocalConfig(object):
     680
     681     """
     682     Class representing this extension's configuration document.
     683
     684     This is not a general-purpose configuration object like the main Cedar
     685     Backup configuration object.  Instead, it just knows how to parse and emit
     686     Mbox-specific configuration values.  Third parties who need to read and
     687     write configuration related to this extension should access it through the
     688     constructor, C{validate} and C{addConfig} methods.
     689
     690     @note: Lists within this class are "unordered" for equality comparisons.
     691
     692     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, mbox,
     693           validate, addConfig
     694     """
     695
     696 -   def __init__(self, xmlData=None, xmlPath=None, validate=True):
     697        """
     698        Initializes a configuration object.
     699
     700        If you initialize the object without passing either C{xmlData} or
     701        C{xmlPath} then configuration will be empty and will be invalid until it
     702        is filled in properly.
     703
     704        No reference to the original XML data or original path is saved off by
     705        this class.  Once the data has been parsed (successfully or not) this
     706        original information is discarded.
     707
     708        Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
     709        method will be called (with its default arguments) against configuration
     710        after successfully parsing any passed-in XML.  Keep in mind that even if
     711        C{validate} is C{False}, it might not be possible to parse the passed-in
     712        XML document if lower-level validations fail.
     713
     714        @note: It is strongly suggested that the C{validate} option always be set
     715        to C{True} (the default) unless there is a specific need to read in
     716        invalid configuration from disk.
     717
     718        @param xmlData: XML data representing configuration.
     719        @type xmlData: String data.
     720
     721        @param xmlPath: Path to an XML file on disk.
     722        @type xmlPath: Absolute path to a file on disk.
     723
     724        @param validate: Validate the document after parsing it.
     725        @type validate: Boolean true/false.
     726
     727        @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
     728        @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
     729        @raise ValueError: If the parsed configuration document is not valid.
     730        """
     731        self._mbox = None
     732        self.mbox = None
     733        if xmlData is not None and xmlPath is not None:
     734           raise ValueError("Use either xmlData or xmlPath, but not both.")
     735        if xmlData is not None:
     736           self._parseXmlData(xmlData)
     737           if validate:
     738              self.validate()
     739        elif xmlPath is not None:
     740           with open(xmlPath) as f:
     741              xmlData = f.read()
     742           self._parseXmlData(xmlData)
     743           if validate:
     744              self.validate()
     745
     746 -   def __repr__(self):
     747        """
     748        Official string representation for class instance.
     749        """
     750        return "LocalConfig(%s)" % (self.mbox)
     751
     752 -   def __str__(self):
     753        """
     754        Informal string representation for class instance.
     755        """
     756        return self.__repr__()
     757
     758 -   def __eq__(self, other):
     759        """Equals operator, iplemented in terms of original Python 2 compare operator."""
     760        return self.__cmp__(other) == 0
     761
     762 -   def __lt__(self, other):
     763        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
     764        return self.__cmp__(other) < 0
     765
     766 -   def __gt__(self, other):
     767        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
     768        return self.__cmp__(other) > 0
     769
     770 -   def __cmp__(self, other):
     771        """
     772        Original Python 2 comparison operator.
     773        Lists within this class are "unordered" for equality comparisons.
     774        @param other: Other object to compare to.
     775        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
     776        """
     777        if other is None:
     778           return 1
     779        if self.mbox != other.mbox:
     780           if self.mbox < other.mbox:
     781              return -1
     782           else:
     783              return 1
     784        return 0
     785
     786 -   def _setMbox(self, value):
     787        """
     788        Property target used to set the mbox configuration value.
     789        If not C{None}, the value must be a C{MboxConfig} object.
     790        @raise ValueError: If the value is not a C{MboxConfig}
     791        """
     792        if value is None:
     793           self._mbox = None
     794        else:
     795           if not isinstance(value, MboxConfig):
     796              raise ValueError("Value must be a C{MboxConfig} object.")
     797           self._mbox = value
     798
     799 -   def _getMbox(self):
     800        """
     801        Property target used to get the mbox configuration value.
     802        """
     803        return self._mbox
     804
     805     mbox = property(_getMbox, _setMbox, None, "Mbox configuration in terms of a C{MboxConfig} object.")
     806
     807 -   def validate(self):
     808        """
     809        Validates configuration represented by the object.
     810
     811        Mbox configuration must be filled in.  Within that, the collect mode and
     812        compress mode are both optional, but the list of repositories must
     813        contain at least one entry.
     814
     815        Each configured file or directory must contain an absolute path, and then
     816        must be either able to take collect mode and compress mode configuration
     817        from the parent C{MboxConfig} object, or must set each value on its own.
     818
     819        @raise ValueError: If one of the validations fails.
     820        """
     821        if self.mbox is None:
     822           raise ValueError("Mbox section is required.")
     823        if (self.mbox.mboxFiles is None or len(self.mbox.mboxFiles) < 1) and \
     824            (self.mbox.mboxDirs is None or len(self.mbox.mboxDirs) < 1):
     825           raise ValueError("At least one mbox file or directory must be configured.")
     826        if self.mbox.mboxFiles is not None:
     827           for mboxFile in self.mbox.mboxFiles:
     828              if mboxFile.absolutePath is None:
     829                 raise ValueError("Each mbox file must set an absolute path.")
     830              if self.mbox.collectMode is None and mboxFile.collectMode is None:
     831                 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox file.")
     832              if self.mbox.compressMode is None and mboxFile.compressMode is None:
     833                 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox file.")
     834        if self.mbox.mboxDirs is not None:
     835           for mboxDir in self.mbox.mboxDirs:
     836              if mboxDir.absolutePath is None:
     837                 raise ValueError("Each mbox directory must set an absolute path.")
     838              if self.mbox.collectMode is None and mboxDir.collectMode is None:
     839                 raise ValueError("Collect mode must either be set in parent mbox section or individual mbox directory.")
     840              if self.mbox.compressMode is None and mboxDir.compressMode is None:
     841                 raise ValueError("Compress mode must either be set in parent mbox section or individual mbox directory.")
     842
     843 -   def addConfig(self, xmlDom, parentNode):
     844        """
     845        Adds an <mbox> configuration section as the next child of a parent.
     846
     847        Third parties should use this function to write configuration related to
     848        this extension.
     849
     850        We add the following fields to the document::
     851
     852           collectMode    //cb_config/mbox/collectMode
     853           compressMode   //cb_config/mbox/compressMode
     854
     855        We also add groups of the following items, one list element per
     856        item::
     857
     858           mboxFiles      //cb_config/mbox/file
     859           mboxDirs       //cb_config/mbox/dir
     860
     861        The mbox files and mbox directories are added by L{_addMboxFile} and
     862        L{_addMboxDir}.
     863
     864        @param xmlDom: DOM tree as from C{impl.createDocument()}.
     865        @param parentNode: Parent that the section should be appended to.
     866        """
     867        if self.mbox is not None:
     868           sectionNode = addContainerNode(xmlDom, parentNode, "mbox")
     869           addStringNode(xmlDom, sectionNode, "collect_mode", self.mbox.collectMode)
     870           addStringNode(xmlDom, sectionNode, "compress_mode", self.mbox.compressMode)
     871           if self.mbox.mboxFiles is not None:
     872              for mboxFile in self.mbox.mboxFiles:
     873                 LocalConfig._addMboxFile(xmlDom, sectionNode, mboxFile)
     874           if self.mbox.mboxDirs is not None:
     875              for mboxDir in self.mbox.mboxDirs:
     876                 LocalConfig._addMboxDir(xmlDom, sectionNode, mboxDir)
     877
     878 -   def _parseXmlData(self, xmlData):
     879        """
     880        Internal method to parse an XML string into the object.
     881
     882        This method parses the XML document into a DOM tree (C{xmlDom}) and then
     883        calls a static method to parse the mbox configuration section.
     884
     885        @param xmlData: XML data to be parsed
     886        @type xmlData: String data
     887
     888        @raise ValueError: If the XML cannot be successfully parsed.
     889        """
     890        (xmlDom, parentNode) = createInputDom(xmlData)
     891        self._mbox = LocalConfig._parseMbox(parentNode)
     892
     893     @staticmethod
     894 -   def _parseMbox(parent):
     895        """
     896        Parses an mbox configuration section.
     897
     898        We read the following individual fields::
     899
     900           collectMode    //cb_config/mbox/collect_mode
     901           compressMode   //cb_config/mbox/compress_mode
     902
     903        We also read groups of the following item, one list element per
     904        item::
     905
     906           mboxFiles      //cb_config/mbox/file
     907           mboxDirs       //cb_config/mbox/dir
     908
     909        The mbox files are parsed by L{_parseMboxFiles} and the mbox
     910        directories are parsed by L{_parseMboxDirs}.
     911
     912        @param parent: Parent node to search beneath.
     913
     914        @return: C{MboxConfig} object or C{None} if the section does not exist.
     915        @raise ValueError: If some filled-in value is invalid.
     916        """
     917        mbox = None
     918        section = readFirstChild(parent, "mbox")
     919        if section is not None:
     920           mbox = MboxConfig()
     921           mbox.collectMode = readString(section, "collect_mode")
     922           mbox.compressMode = readString(section, "compress_mode")
     923           mbox.mboxFiles = LocalConfig._parseMboxFiles(section)
     924           mbox.mboxDirs = LocalConfig._parseMboxDirs(section)
     925        return mbox
     926
     927     @staticmethod
     928 -   def _parseMboxFiles(parent):
     929        """
     930        Reads a list of C{MboxFile} objects from immediately beneath the parent.
     931
     932        We read the following individual fields::
     933
     934           absolutePath            abs_path
     935           collectMode             collect_mode
     936           compressMode            compess_mode
     937
     938        @param parent: Parent node to search beneath.
     939
     940        @return: List of C{MboxFile} objects or C{None} if none are found.
     941        @raise ValueError: If some filled-in value is invalid.
     942        """
     943        lst = []
     944        for entry in readChildren(parent, "file"):
     945           if isElement(entry):
     946              mboxFile = MboxFile()
     947              mboxFile.absolutePath = readString(entry, "abs_path")
     948              mboxFile.collectMode = readString(entry, "collect_mode")
     949              mboxFile.compressMode = readString(entry, "compress_mode")
     950              lst.append(mboxFile)
     951        if lst == []:
     952           lst = None
     953        return lst
     954
     955     @staticmethod
     956 -   def _parseMboxDirs(parent):
     957        """
     958        Reads a list of C{MboxDir} objects from immediately beneath the parent.
     959
     960        We read the following individual fields::
     961
     962           absolutePath            abs_path
     963           collectMode             collect_mode
     964           compressMode            compess_mode
     965
     966        We also read groups of the following items, one list element per
     967        item::
     968
     969           relativeExcludePaths    exclude/rel_path
     970           excludePatterns         exclude/pattern
     971
     972        The exclusions are parsed by L{_parseExclusions}.
     973
     974        @param parent: Parent node to search beneath.
     975
     976        @return: List of C{MboxDir} objects or C{None} if none are found.
     977        @raise ValueError: If some filled-in value is invalid.
     978        """
     979        lst = []
     980        for entry in readChildren(parent, "dir"):
     981           if isElement(entry):
     982              mboxDir = MboxDir()
     983              mboxDir.absolutePath = readString(entry, "abs_path")
     984              mboxDir.collectMode = readString(entry, "collect_mode")
     985              mboxDir.compressMode = readString(entry, "compress_mode")
     986              (mboxDir.relativeExcludePaths, mboxDir.excludePatterns) = LocalConfig._parseExclusions(entry)
     987              lst.append(mboxDir)
     988        if lst == []:
     989           lst = None
     990        return lst
     991
     992     @staticmethod
     993 -   def _parseExclusions(parentNode):
     994        """
     995        Reads exclusions data from immediately beneath the parent.
     996
     997        We read groups of the following items, one list element per item::
     998
     999           relative    exclude/rel_path
    1000           patterns    exclude/pattern
    1001
    1002        If there are none of some pattern (i.e. no relative path items) then
    1003        C{None} will be returned for that item in the tuple.
    1004
    1005        @param parentNode: Parent node to search beneath.
    1006
    1007        @return: Tuple of (relative, patterns) exclusions.
    1008        """
    1009        section = readFirstChild(parentNode, "exclude")
    1010        if section is None:
    1011           return (None, None)
    1012        else:
    1013           relative = readStringList(section, "rel_path")
    1014           patterns = readStringList(section, "pattern")
    1015           return (relative, patterns)
    1016
    1017     @staticmethod
    1018 -   def _addMboxFile(xmlDom, parentNode, mboxFile):
    1019        """
    1020        Adds an mbox file container as the next child of a parent.
    1021
    1022        We add the following fields to the document::
    1023
    1024           absolutePath            file/abs_path
    1025           collectMode             file/collect_mode
    1026           compressMode            file/compress_mode
    1027
    1028        The <file> node itself is created as the next child of the parent node.
    1029        This method only adds one mbox file node.  The parent must loop for each
    1030        mbox file in the C{MboxConfig} object.
    1031
    1032        If C{mboxFile} is C{None}, this method call will be a no-op.
    1033
    1034        @param xmlDom: DOM tree as from C{impl.createDocument()}.
    1035        @param parentNode: Parent that the section should be appended to.
    1036        @param mboxFile: MboxFile to be added to the document.
    1037        """
    1038        if mboxFile is not None:
    1039           sectionNode = addContainerNode(xmlDom, parentNode, "file")
    1040           addStringNode(xmlDom, sectionNode, "abs_path", mboxFile.absolutePath)
    1041           addStringNode(xmlDom, sectionNode, "collect_mode", mboxFile.collectMode)
    1042           addStringNode(xmlDom, sectionNode, "compress_mode", mboxFile.compressMode)
    1043
    1044     @staticmethod
    1045 -   def _addMboxDir(xmlDom, parentNode, mboxDir):
    1046        """
    1047        Adds an mbox directory container as the next child of a parent.
    1048
    1049        We add the following fields to the document::
    1050
    1051           absolutePath            dir/abs_path
    1052           collectMode             dir/collect_mode
    1053           compressMode            dir/compress_mode
    1054
    1055        We also add groups of the following items, one list element per item::
    1056
    1057           relativeExcludePaths    dir/exclude/rel_path
    1058           excludePatterns         dir/exclude/pattern
    1059
    1060        The <dir> node itself is created as the next child of the parent node.
    1061        This method only adds one mbox directory node.  The parent must loop for
    1062        each mbox directory in the C{MboxConfig} object.
    1063
    1064        If C{mboxDir} is C{None}, this method call will be a no-op.
    1065
    1066        @param xmlDom: DOM tree as from C{impl.createDocument()}.
    1067        @param parentNode: Parent that the section should be appended to.
    1068        @param mboxDir: MboxDir to be added to the document.
    1069        """
    1070        if mboxDir is not None:
    1071           sectionNode = addContainerNode(xmlDom, parentNode, "dir")
    1072           addStringNode(xmlDom, sectionNode, "abs_path", mboxDir.absolutePath)
    1073           addStringNode(xmlDom, sectionNode, "collect_mode", mboxDir.collectMode)
    1074           addStringNode(xmlDom, sectionNode, "compress_mode", mboxDir.compressMode)
    1075           if ((mboxDir.relativeExcludePaths is not None and mboxDir.relativeExcludePaths != []) or
    1076               (mboxDir.excludePatterns is not None and mboxDir.excludePatterns != [])):
    1077              excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
    1078              if mboxDir.relativeExcludePaths is not None:
    1079                 for relativePath in mboxDir.relativeExcludePaths:
    1080                    addStringNode(xmlDom, excludeNode, "rel_path", relativePath)
    1081              if mboxDir.excludePatterns is not None:
    1082                 for pattern in mboxDir.excludePatterns:
    1083                    addStringNode(xmlDom, excludeNode, "pattern", pattern)
    1084
    1085
    1086  ########################################################################
    1087  # Public functions
    1088  ########################################################################
    1089
    1090  ###########################
    1091  # executeAction() function
    1092  ###########################
    1093
    1094 -def executeAction(configPath, options, config):
    1095     """
    1096     Executes the mbox backup action.
    1097
    1098     @param configPath: Path to configuration file on disk.
    1099     @type configPath: String representing a path on disk.
    1100
    1101     @param options: Program command-line options.
    1102     @type options: Options object.
    1103
    1104     @param config: Program configuration.
    1105     @type config: Config object.
    1106
    1107     @raise ValueError: Under many generic error conditions
    1108     @raise IOError: If a backup could not be written for some reason.
    1109     """
    1110     logger.debug("Executing mbox extended action.")
    1111     newRevision = datetime.datetime.today()  # mark here so all actions are after this date/time
    1112     if config.options is None or config.collect is None:
    1113        raise ValueError("Cedar Backup configuration is not properly filled in.")
    1114     local = LocalConfig(xmlPath=configPath)
    1115     todayIsStart = isStartOfWeek(config.options.startingDay)
    1116     fullBackup = options.full or todayIsStart
    1117     logger.debug("Full backup flag is [%s]", fullBackup)
    1118     if local.mbox.mboxFiles is not None:
    1119        for mboxFile in local.mbox.mboxFiles:
    1120           logger.debug("Working with mbox file [%s]", mboxFile.absolutePath)
    1121           collectMode = _getCollectMode(local, mboxFile)
    1122           compressMode = _getCompressMode(local, mboxFile)
    1123           lastRevision = _loadLastRevision(config, mboxFile, fullBackup, collectMode)
    1124           if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
    1125              logger.debug("Mbox file meets criteria to be backed up today.")
    1126              _backupMboxFile(config, mboxFile.absolutePath, fullBackup,
    1127                              collectMode, compressMode, lastRevision, newRevision)
    1128           else:
    1129              logger.debug("Mbox file will not be backed up, per collect mode.")
    1130           if collectMode == 'incr':
    1131              _writeNewRevision(config, mboxFile, newRevision)
    1132     if local.mbox.mboxDirs is not None:
    1133        for mboxDir in local.mbox.mboxDirs:
    1134           logger.debug("Working with mbox directory [%s]", mboxDir.absolutePath)
    1135           collectMode = _getCollectMode(local, mboxDir)
    1136           compressMode = _getCompressMode(local, mboxDir)
    1137           lastRevision = _loadLastRevision(config, mboxDir, fullBackup, collectMode)
    1138           (excludePaths, excludePatterns) = _getExclusions(mboxDir)
    1139           if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
    1140              logger.debug("Mbox directory meets criteria to be backed up today.")
    1141              _backupMboxDir(config, mboxDir.absolutePath,
    1142                             fullBackup, collectMode, compressMode,
    1143                             lastRevision, newRevision,
    1144                             excludePaths, excludePatterns)
    1145           else:
    1146              logger.debug("Mbox directory will not be backed up, per collect mode.")
    1147           if collectMode == 'incr':
    1148              _writeNewRevision(config, mboxDir, newRevision)
    1149     logger.info("Executed the mbox extended action successfully.")
    1150
    1151 -def _getCollectMode(local, item):
    1152     """
    1153     Gets the collect mode that should be used for an mbox file or directory.
    1154     Use file- or directory-specific value if possible, otherwise take from mbox section.
    1155     @param local: LocalConfig object.
    1156     @param item: Mbox file or directory
    1157     @return: Collect mode to use.
    1158     """
    1159     if item.collectMode is None:
    1160        collectMode = local.mbox.collectMode
    1161     else:
    1162        collectMode = item.collectMode
    1163     logger.debug("Collect mode is [%s]", collectMode)
    1164     return collectMode
    1165
    1166 -def _getCompressMode(local, item):
    1167     """
    1168     Gets the compress mode that should be used for an mbox file or directory.
    1169     Use file- or directory-specific value if possible, otherwise take from mbox section.
    1170     @param local: LocalConfig object.
    1171     @param item: Mbox file or directory
    1172     @return: Compress mode to use.
    1173     """
    1174     if item.compressMode is None:
    1175        compressMode = local.mbox.compressMode
    1176     else:
    1177        compressMode = item.compressMode
    1178     logger.debug("Compress mode is [%s]", compressMode)
    1179     return compressMode
    1180
    1181 -def _getRevisionPath(config, item):
    1182     """
    1183     Gets the path to the revision file associated with a repository.
    1184     @param config: Cedar Backup configuration.
    1185     @param item: Mbox file or directory
    1186     @return: Absolute path to the revision file associated with the repository.
    1187     """
    1188     normalized = buildNormalizedPath(item.absolutePath)
    1189     filename = "%s.%s" % (normalized, REVISION_PATH_EXTENSION)
    1190     revisionPath = os.path.join(config.options.workingDir, filename)
    1191     logger.debug("Revision file path is [%s]", revisionPath)
    1192     return revisionPath
    1193
    1194 -def _loadLastRevision(config, item, fullBackup, collectMode):
    1195     """
    1196     Loads the last revision date for this item from disk and returns it.
    1197
    1198     If this is a full backup, or if the revision file cannot be loaded for some
    1199     reason, then C{None} is returned.  This indicates that there is no previous
    1200     revision, so the entire mail file or directory should be backed up.
    1201
    1202     @note: We write the actual revision object to disk via pickle, so we don't
    1203     deal with the datetime precision or format at all.  Whatever's in the object
    1204     is what we write.
    1205
    1206     @param config: Cedar Backup configuration.
    1207     @param item: Mbox file or directory
    1208     @param fullBackup: Indicates whether this is a full backup
    1209     @param collectMode: Indicates the collect mode for this item
    1210
    1211     @return: Revision date as a datetime.datetime object or C{None}.
    1212     """
    1213     revisionPath = _getRevisionPath(config, item)
    1214     if fullBackup:
    1215        revisionDate = None
    1216        logger.debug("Revision file ignored because this is a full backup.")
    1217     elif collectMode in ['weekly', 'daily']:
    1218        revisionDate = None
    1219        logger.debug("No revision file based on collect mode [%s].", collectMode)
    1220     else:
    1221        logger.debug("Revision file will be used for non-full incremental backup.")
    1222        if not os.path.isfile(revisionPath):
    1223           revisionDate = None
    1224           logger.debug("Revision file [%s] does not exist on disk.", revisionPath)
    1225        else:
    1226           try:
    1227              with open(revisionPath, "rb") as f:
    1228                 revisionDate = pickle.load(f, fix_imports=True)  # be compatible with Python 2
    1229              logger.debug("Loaded revision file [%s] from disk: [%s]", revisionPath, revisionDate)
    1230           except Exception as e:
    1231              revisionDate = None
    1232              logger.error("Failed loading revision file [%s] from disk: %s", revisionPath, e)
    1233     return revisionDate
    1234
    1235 -def _writeNewRevision(config, item, newRevision):
    1236     """
    1237     Writes new revision information to disk.
    1238
    1239     If we can't write the revision file successfully for any reason, we'll log
    1240     the condition but won't throw an exception.
    1241
    1242     @note: We write the actual revision object to disk via pickle, so we don't
    1243     deal with the datetime precision or format at all.  Whatever's in the object
    1244     is what we write.
    1245
    1246     @param config: Cedar Backup configuration.
    1247     @param item: Mbox file or directory
    1248     @param newRevision: Revision date as a datetime.datetime object.
    1249     """
    1250     revisionPath = _getRevisionPath(config, item)
    1251     try:
    1252        with open(revisionPath, "wb") as f:
    1253           pickle.dump(newRevision, f, 0, fix_imports=True)  # be compatible with Python 2
    1254        changeOwnership(revisionPath, config.options.backupUser, config.options.backupGroup)
    1255        logger.debug("Wrote new revision file [%s] to disk: [%s]", revisionPath, newRevision)
    1256     except Exception as e:
    1257        logger.error("Failed to write revision file [%s] to disk: %s", revisionPath, e)
    1258
    1259 -def _getExclusions(mboxDir):
    1260     """
    1261     Gets exclusions (file and patterns) associated with an mbox directory.
    1262
    1263     The returned files value is a list of absolute paths to be excluded from the
    1264     backup for a given directory.  It is derived from the mbox directory's
    1265     relative exclude paths.
    1266
    1267     The returned patterns value is a list of patterns to be excluded from the
    1268     backup for a given directory.  It is derived from the mbox directory's list
    1269     of patterns.
    1270
    1271     @param mboxDir: Mbox directory object.
    1272
    1273     @return: Tuple (files, patterns) indicating what to exclude.
    1274     """
    1275     paths = []
    1276     if mboxDir.relativeExcludePaths is not None:
    1277        for relativePath in mboxDir.relativeExcludePaths:
    1278           paths.append(os.path.join(mboxDir.absolutePath, relativePath))
    1279     patterns = []
    1280     if mboxDir.excludePatterns is not None:
    1281        patterns.extend(mboxDir.excludePatterns)
    1282     logger.debug("Exclude paths: %s", paths)
    1283     logger.debug("Exclude patterns: %s", patterns)
    1284     return(paths, patterns)
    1285
    1286 -def _getBackupPath(config, mboxPath, compressMode, newRevision, targetDir=None):
    1287     """
    1288     Gets the backup file path (including correct extension) associated with an mbox path.
    1289
    1290     We assume that if the target directory is passed in, that we're backing up a
    1291     directory.  Under these circumstances, we'll just use the basename of the
    1292     individual path as the output file.
    1293
    1294     @note: The backup path only contains the current date in YYYYMMDD format,
    1295     but that's OK because the index information (stored elsewhere) is the actual
    1296     date object.
    1297
    1298     @param config: Cedar Backup configuration.
    1299     @param mboxPath: Path to the indicated mbox file or directory
    1300     @param compressMode: Compress mode to use for this mbox path
    1301     @param newRevision: Revision this backup path represents
    1302     @param targetDir: Target directory in which the path should exist
    1303
    1304     @return: Absolute path to the backup file associated with the repository.
    1305     """
    1306     if targetDir is None:
    1307        normalizedPath = buildNormalizedPath(mboxPath)
    1308        revisionDate = newRevision.strftime("%Y%m%d")
    1309        filename = "mbox-%s-%s" % (revisionDate, normalizedPath)
    1310     else:
    1311        filename = os.path.basename(mboxPath)
    1312     if compressMode == 'gzip':
    1313        filename = "%s.gz" % filename
    1314     elif compressMode == 'bzip2':
    1315        filename = "%s.bz2" % filename
    1316     if targetDir is None:
    1317        backupPath = os.path.join(config.collect.targetDir, filename)
    1318     else:
    1319        backupPath = os.path.join(targetDir, filename)
    1320     logger.debug("Backup file path is [%s]", backupPath)
    1321     return backupPath
    1322
    1323 -def _getTarfilePath(config, mboxPath, compressMode, newRevision):
    1324     """
    1325     Gets the tarfile backup file path (including correct extension) associated
    1326     with an mbox path.
    1327
    1328     Along with the path, the tar archive mode is returned in a form that can
    1329     be used with L{BackupFileList.generateTarfile}.
    1330
    1331     @note: The tarfile path only contains the current date in YYYYMMDD format,
    1332     but that's OK because the index information (stored elsewhere) is the actual
    1333     date object.
    1334
    1335     @param config: Cedar Backup configuration.
    1336     @param mboxPath: Path to the indicated mbox file or directory
    1337     @param compressMode: Compress mode to use for this mbox path
    1338     @param newRevision: Revision this backup path represents
    1339
    1340     @return: Tuple of (absolute path to tarfile, tar archive mode)
    1341     """
    1342     normalizedPath = buildNormalizedPath(mboxPath)
    1343     revisionDate = newRevision.strftime("%Y%m%d")
    1344     filename = "mbox-%s-%s.tar" % (revisionDate, normalizedPath)
    1345     if compressMode == 'gzip':
    1346        filename = "%s.gz" % filename
    1347        archiveMode = "targz"
    1348     elif compressMode == 'bzip2':
    1349        filename = "%s.bz2" % filename
    1350        archiveMode = "tarbz2"
    1351     else:
    1352        archiveMode = "tar"
    1353     tarfilePath = os.path.join(config.collect.targetDir, filename)
    1354     logger.debug("Tarfile path is [%s]", tarfilePath)
    1355     return (tarfilePath, archiveMode)
    1356
    1357 -def _getOutputFile(backupPath, compressMode):
    1358     """
    1359     Opens the output file used for saving backup information.
    1360
    1361     If the compress mode is "gzip", we'll open a C{GzipFile}, and if the
    1362     compress mode is "bzip2", we'll open a C{BZ2File}.  Otherwise, we'll just
    1363     return an object from the normal C{open()} method.
    1364
    1365     @param backupPath: Path to file to open.
    1366     @param compressMode: Compress mode of file ("none", "gzip", "bzip").
    1367
    1368     @return: Output file object, opened in binary mode for use with executeCommand()
    1369     """
    1370     if compressMode == "gzip":
    1371        return GzipFile(backupPath, "wb")
    1372     elif compressMode == "bzip2":
    1373        return BZ2File(backupPath, "wb")
    1374     else:
    1375        return open(backupPath, "wb")
    1376
    1377 -def _backupMboxFile(config, absolutePath,
    1378                      fullBackup, collectMode, compressMode,
    1379                      lastRevision, newRevision, targetDir=None):
    1380     """
    1381     Backs up an individual mbox file.
    1382
    1383     @param config: Cedar Backup configuration.
    1384     @param absolutePath: Path to mbox file to back up.
    1385     @param fullBackup: Indicates whether this should be a full backup.
    1386     @param collectMode: Indicates the collect mode for this item
    1387     @param compressMode: Compress mode of file ("none", "gzip", "bzip")
    1388     @param lastRevision: Date of last backup as datetime.datetime
    1389     @param newRevision: Date of new (current) backup as datetime.datetime
    1390     @param targetDir: Target directory to write the backed-up file into
    1391
    1392     @raise ValueError: If some value is missing or invalid.
    1393     @raise IOError: If there is a problem backing up the mbox file.
    1394     """
    1395     if fullBackup or collectMode != "incr" or lastRevision is None:
    1396        args = [ "-a", "-u", absolutePath, ]  # remove duplicates but fetch entire mailbox
    1397     else:
    1398        revisionDate = lastRevision.strftime("%Y-%m-%dT%H:%M:%S")  # ISO-8601 format; grepmail calls Date::Parse::str2time()
    1399        args = [ "-a", "-u", "-d", "since %s" % revisionDate, absolutePath, ]
    1400     command = resolveCommand(GREPMAIL_COMMAND)
    1401     backupPath = _getBackupPath(config, absolutePath, compressMode, newRevision, targetDir=targetDir)
    1402     with _getOutputFile(backupPath, compressMode) as outputFile:
    1403        result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=outputFile)[0]
    1404        if result != 0:
    1405           raise IOError("Error [%d] executing grepmail on [%s]." % (result, absolutePath))
    1406     logger.debug("Completed backing up mailbox [%s].", absolutePath)
    1407     return backupPath
    1408
    1409 -def _backupMboxDir(config, absolutePath,
    1410                     fullBackup, collectMode, compressMode,
    1411                     lastRevision, newRevision,
    1412                     excludePaths, excludePatterns):
    1413     """
    1414     Backs up a directory containing mbox files.
    1415
    1416     @param config: Cedar Backup configuration.
    1417     @param absolutePath: Path to mbox directory to back up.
    1418     @param fullBackup: Indicates whether this should be a full backup.
    1419     @param collectMode: Indicates the collect mode for this item
    1420     @param compressMode: Compress mode of file ("none", "gzip", "bzip")
    1421     @param lastRevision: Date of last backup as datetime.datetime
    1422     @param newRevision: Date of new (current) backup as datetime.datetime
    1423     @param excludePaths: List of absolute paths to exclude.
    1424     @param excludePatterns: List of patterns to exclude.
    1425
    1426     @raise ValueError: If some value is missing or invalid.
    1427     @raise IOError: If there is a problem backing up the mbox file.
    1428     """
    1429     try:
    1430        tmpdir = tempfile.mkdtemp(dir=config.options.workingDir)
    1431        mboxList = FilesystemList()
    1432        mboxList.excludeDirs = True
    1433        mboxList.excludePaths = excludePaths
    1434        mboxList.excludePatterns = excludePatterns
    1435        mboxList.addDirContents(absolutePath, recursive=False)
    1436        tarList = BackupFileList()
    1437        for item in mboxList:
    1438           backupPath = _backupMboxFile(config, item, fullBackup,
    1439                                        collectMode, "none",  # no need to compress inside compressed tar
    1440                                        lastRevision, newRevision,
    1441                                        targetDir=tmpdir)
    1442           tarList.addFile(backupPath)
    1443        (tarfilePath, archiveMode) = _getTarfilePath(config, absolutePath, compressMode, newRevision)
    1444        tarList.generateTarfile(tarfilePath, archiveMode, ignore=True, flat=True)
    1445        changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
    1446        logger.debug("Completed backing up directory [%s].", absolutePath)
    1447     finally:
    1448        try:
    1449           for cleanitem in tarList:
    1450              if os.path.exists(cleanitem):
    1451                 try:
    1452                    os.remove(cleanitem)
    1453                 except: pass
    1454        except: pass
    1455        try:
    1456           os.rmdir(tmpdir)
    1457        except: pass
    1458