CedarBackup3.extend.split

Package CedarBackup3 :: Package extend :: Module split |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup3.extend.split

      1  # -*- coding: iso-8859-1 -*-
      2  # vim: set ft=python ts=3 sw=3 expandtab:
      3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      4  #
      5  #              C E D A R
      6  #          S O L U T I O N S       "Software done right."
      7  #           S O F T W A R E
      8  #
      9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     10  #
     11  # Copyright (c) 2007,2010,2013,2015 Kenneth J. Pronovici.
     12  # All rights reserved.
     13  #
     14  # This program is free software; you can redistribute it and/or
     15  # modify it under the terms of the GNU General Public License,
     16  # Version 2, as published by the Free Software Foundation.
     17  #
     18  # This program is distributed in the hope that it will be useful,
     19  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     21  #
     22  # Copies of the GNU General Public License are available from
     23  # the Free Software Foundation website, http://www.gnu.org/.
     24  #
     25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     26  #
     27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
     28  # Language : Python 3 (>= 3.4)
     29  # Project  : Official Cedar Backup Extensions
     30  # Purpose  : Provides an extension to split up large files in staging directories.
     31  #
     32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     33
     34  ########################################################################
     35  # Module documentation
     36  ########################################################################
     37
     38  """
     39  Provides an extension to split up large files in staging directories.
     40
     41  When this extension is executed, it will look through the configured Cedar
     42  Backup staging directory for files exceeding a specified size limit, and split
     43  them down into smaller files using the 'split' utility.  Any directory which
     44  has already been split (as indicated by the C{cback.split} file) will be
     45  ignored.
     46
     47  This extension requires a new configuration section <split> and is intended
     48  to be run immediately after the standard stage action or immediately before the
     49  standard store action.  Aside from its own configuration, it requires the
     50  options and staging configuration sections in the standard Cedar Backup
     51  configuration file.
     52
     53  @author: Kenneth J. Pronovici <pronovic@ieee.org>
     54  """
     55
     56  ########################################################################
     57  # Imported modules
     58  ########################################################################
     59
     60  # System modules
     61  import os
     62  import re
     63  import logging
     64  from functools import total_ordering
     65
     66  # Cedar Backup modules
     67  from CedarBackup3.util import resolveCommand, executeCommand, changeOwnership
     68  from CedarBackup3.xmlutil import createInputDom, addContainerNode
     69  from CedarBackup3.xmlutil import readFirstChild
     70  from CedarBackup3.actions.util import findDailyDirs, writeIndicatorFile, getBackupFiles
     71  from CedarBackup3.config import ByteQuantity, readByteQuantity, addByteQuantityNode
     72
     73
     74  ########################################################################
     75  # Module-wide constants and variables
     76  ########################################################################
     77
     78  logger = logging.getLogger("CedarBackup3.log.extend.split")
     79
     80  SPLIT_COMMAND = [ "split", ]
     81  SPLIT_INDICATOR = "cback.split"
     82
     83
     84  ########################################################################
     85  # SplitConfig class definition
     86  ########################################################################
     87
     88  @total_ordering
     89 -class SplitConfig(object):
     90
     91     """
     92     Class representing split configuration.
     93
     94     Split configuration is used for splitting staging directories.
     95
     96     The following restrictions exist on data in this class:
     97
     98        - The size limit must be a ByteQuantity
     99        - The split size must be a ByteQuantity
    100
    101     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
    102           sizeLimit, splitSize
    103     """
    104
    105 -   def __init__(self, sizeLimit=None, splitSize=None):
    106        """
    107        Constructor for the C{SplitCOnfig} class.
    108
    109        @param sizeLimit: Size limit of the files, in bytes
    110        @param splitSize: Size that files exceeding the limit will be split into, in bytes
    111
    112        @raise ValueError: If one of the values is invalid.
    113        """
    114        self._sizeLimit = None
    115        self._splitSize = None
    116        self.sizeLimit = sizeLimit
    117        self.splitSize = splitSize
    118
    119 -   def __repr__(self):
    120        """
    121        Official string representation for class instance.
    122        """
    123        return "SplitConfig(%s, %s)" % (self.sizeLimit, self.splitSize)
    124
    125 -   def __str__(self):
    126        """
    127        Informal string representation for class instance.
    128        """
    129        return self.__repr__()
    130
    131 -   def __eq__(self, other):
    132        """Equals operator, iplemented in terms of original Python 2 compare operator."""
    133        return self.__cmp__(other) == 0
    134
    135 -   def __lt__(self, other):
    136        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
    137        return self.__cmp__(other) < 0
    138
    139 -   def __gt__(self, other):
    140        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
    141        return self.__cmp__(other) > 0
    142
    143 -   def __cmp__(self, other):
    144        """
    145        Original Python 2 comparison operator.
    146        Lists within this class are "unordered" for equality comparisons.
    147        @param other: Other object to compare to.
    148        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    149        """
    150        if other is None:
    151           return 1
    152        if self.sizeLimit != other.sizeLimit:
    153           if (self.sizeLimit or ByteQuantity()) < (other.sizeLimit or ByteQuantity()):
    154              return -1
    155           else:
    156              return 1
    157        if self.splitSize != other.splitSize:
    158           if (self.splitSize or ByteQuantity()) < (other.splitSize or ByteQuantity()):
    159              return -1
    160           else:
    161              return 1
    162        return 0
    163
    164 -   def _setSizeLimit(self, value):
    165        """
    166        Property target used to set the size limit.
    167        If not C{None}, the value must be a C{ByteQuantity} object.
    168        @raise ValueError: If the value is not a C{ByteQuantity}
    169        """
    170        if value is None:
    171           self._sizeLimit = None
    172        else:
    173           if not isinstance(value, ByteQuantity):
    174              raise ValueError("Value must be a C{ByteQuantity} object.")
    175           self._sizeLimit = value
    176
    177 -   def _getSizeLimit(self):
    178        """
    179        Property target used to get the size limit.
    180        """
    181        return self._sizeLimit
    182
    183 -   def _setSplitSize(self, value):
    184        """
    185        Property target used to set the split size.
    186        If not C{None}, the value must be a C{ByteQuantity} object.
    187        @raise ValueError: If the value is not a C{ByteQuantity}
    188        """
    189        if value is None:
    190           self._splitSize = None
    191        else:
    192           if not isinstance(value, ByteQuantity):
    193              raise ValueError("Value must be a C{ByteQuantity} object.")
    194           self._splitSize = value
    195
    196 -   def _getSplitSize(self):
    197        """
    198        Property target used to get the split size.
    199        """
    200        return self._splitSize
    201
    202     sizeLimit = property(_getSizeLimit, _setSizeLimit, None, doc="Size limit, as a ByteQuantity")
    203     splitSize = property(_getSplitSize, _setSplitSize, None, doc="Split size, as a ByteQuantity")
    204
    205
    206  ########################################################################
    207  # LocalConfig class definition
    208  ########################################################################
    209
    210  @total_ordering
    211 -class LocalConfig(object):
    212
    213     """
    214     Class representing this extension's configuration document.
    215
    216     This is not a general-purpose configuration object like the main Cedar
    217     Backup configuration object.  Instead, it just knows how to parse and emit
    218     split-specific configuration values.  Third parties who need to read and
    219     write configuration related to this extension should access it through the
    220     constructor, C{validate} and C{addConfig} methods.
    221
    222     @note: Lists within this class are "unordered" for equality comparisons.
    223
    224     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, split,
    225           validate, addConfig
    226     """
    227
    228 -   def __init__(self, xmlData=None, xmlPath=None, validate=True):
    229        """
    230        Initializes a configuration object.
    231
    232        If you initialize the object without passing either C{xmlData} or
    233        C{xmlPath} then configuration will be empty and will be invalid until it
    234        is filled in properly.
    235
    236        No reference to the original XML data or original path is saved off by
    237        this class.  Once the data has been parsed (successfully or not) this
    238        original information is discarded.
    239
    240        Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
    241        method will be called (with its default arguments) against configuration
    242        after successfully parsing any passed-in XML.  Keep in mind that even if
    243        C{validate} is C{False}, it might not be possible to parse the passed-in
    244        XML document if lower-level validations fail.
    245
    246        @note: It is strongly suggested that the C{validate} option always be set
    247        to C{True} (the default) unless there is a specific need to read in
    248        invalid configuration from disk.
    249
    250        @param xmlData: XML data representing configuration.
    251        @type xmlData: String data.
    252
    253        @param xmlPath: Path to an XML file on disk.
    254        @type xmlPath: Absolute path to a file on disk.
    255
    256        @param validate: Validate the document after parsing it.
    257        @type validate: Boolean true/false.
    258
    259        @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
    260        @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
    261        @raise ValueError: If the parsed configuration document is not valid.
    262        """
    263        self._split = None
    264        self.split = None
    265        if xmlData is not None and xmlPath is not None:
    266           raise ValueError("Use either xmlData or xmlPath, but not both.")
    267        if xmlData is not None:
    268           self._parseXmlData(xmlData)
    269           if validate:
    270              self.validate()
    271        elif xmlPath is not None:
    272           with open(xmlPath) as f:
    273              xmlData = f.read()
    274           self._parseXmlData(xmlData)
    275           if validate:
    276              self.validate()
    277
    278 -   def __repr__(self):
    279        """
    280        Official string representation for class instance.
    281        """
    282        return "LocalConfig(%s)" % (self.split)
    283
    284 -   def __str__(self):
    285        """
    286        Informal string representation for class instance.
    287        """
    288        return self.__repr__()
    289
    290 -   def __eq__(self, other):
    291        """Equals operator, iplemented in terms of original Python 2 compare operator."""
    292        return self.__cmp__(other) == 0
    293
    294 -   def __lt__(self, other):
    295        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
    296        return self.__cmp__(other) < 0
    297
    298 -   def __gt__(self, other):
    299        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
    300        return self.__cmp__(other) > 0
    301
    302 -   def __cmp__(self, other):
    303        """
    304        Original Python 2 comparison operator.
    305        Lists within this class are "unordered" for equality comparisons.
    306        @param other: Other object to compare to.
    307        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    308        """
    309        if other is None:
    310           return 1
    311        if self.split != other.split:
    312           if self.split < other.split:
    313              return -1
    314           else:
    315              return 1
    316        return 0
    317
    318 -   def _setSplit(self, value):
    319        """
    320        Property target used to set the split configuration value.
    321        If not C{None}, the value must be a C{SplitConfig} object.
    322        @raise ValueError: If the value is not a C{SplitConfig}
    323        """
    324        if value is None:
    325           self._split = None
    326        else:
    327           if not isinstance(value, SplitConfig):
    328              raise ValueError("Value must be a C{SplitConfig} object.")
    329           self._split = value
    330
    331 -   def _getSplit(self):
    332        """
    333        Property target used to get the split configuration value.
    334        """
    335        return self._split
    336
    337     split = property(_getSplit, _setSplit, None, "Split configuration in terms of a C{SplitConfig} object.")
    338
    339 -   def validate(self):
    340        """
    341        Validates configuration represented by the object.
    342
    343        Split configuration must be filled in.  Within that, both the size limit
    344        and split size must be filled in.
    345
    346        @raise ValueError: If one of the validations fails.
    347        """
    348        if self.split is None:
    349           raise ValueError("Split section is required.")
    350        if self.split.sizeLimit is None:
    351           raise ValueError("Size limit must be set.")
    352        if self.split.splitSize is None:
    353           raise ValueError("Split size must be set.")
    354
    355 -   def addConfig(self, xmlDom, parentNode):
    356        """
    357        Adds a <split> configuration section as the next child of a parent.
    358
    359        Third parties should use this function to write configuration related to
    360        this extension.
    361
    362        We add the following fields to the document::
    363
    364           sizeLimit      //cb_config/split/size_limit
    365           splitSize      //cb_config/split/split_size
    366
    367        @param xmlDom: DOM tree as from C{impl.createDocument()}.
    368        @param parentNode: Parent that the section should be appended to.
    369        """
    370        if self.split is not None:
    371           sectionNode = addContainerNode(xmlDom, parentNode, "split")
    372           addByteQuantityNode(xmlDom, sectionNode, "size_limit", self.split.sizeLimit)
    373           addByteQuantityNode(xmlDom, sectionNode, "split_size", self.split.splitSize)
    374
    375 -   def _parseXmlData(self, xmlData):
    376        """
    377        Internal method to parse an XML string into the object.
    378
    379        This method parses the XML document into a DOM tree (C{xmlDom}) and then
    380        calls a static method to parse the split configuration section.
    381
    382        @param xmlData: XML data to be parsed
    383        @type xmlData: String data
    384
    385        @raise ValueError: If the XML cannot be successfully parsed.
    386        """
    387        (xmlDom, parentNode) = createInputDom(xmlData)
    388        self._split = LocalConfig._parseSplit(parentNode)
    389
    390     @staticmethod
    391 -   def _parseSplit(parent):
    392        """
    393        Parses an split configuration section.
    394
    395        We read the following individual fields::
    396
    397           sizeLimit      //cb_config/split/size_limit
    398           splitSize      //cb_config/split/split_size
    399
    400        @param parent: Parent node to search beneath.
    401
    402        @return: C{EncryptConfig} object or C{None} if the section does not exist.
    403        @raise ValueError: If some filled-in value is invalid.
    404        """
    405        split = None
    406        section = readFirstChild(parent, "split")
    407        if section is not None:
    408           split = SplitConfig()
    409           split.sizeLimit = readByteQuantity(section, "size_limit")
    410           split.splitSize = readByteQuantity(section, "split_size")
    411        return split
    412
    413
    414  ########################################################################
    415  # Public functions
    416  ########################################################################
    417
    418  ###########################
    419  # executeAction() function
    420  ###########################
    421
    422 -def executeAction(configPath, options, config):
    423     """
    424     Executes the split backup action.
    425
    426     @param configPath: Path to configuration file on disk.
    427     @type configPath: String representing a path on disk.
    428
    429     @param options: Program command-line options.
    430     @type options: Options object.
    431
    432     @param config: Program configuration.
    433     @type config: Config object.
    434
    435     @raise ValueError: Under many generic error conditions
    436     @raise IOError: If there are I/O problems reading or writing files
    437     """
    438     logger.debug("Executing split extended action.")
    439     if config.options is None or config.stage is None:
    440        raise ValueError("Cedar Backup configuration is not properly filled in.")
    441     local = LocalConfig(xmlPath=configPath)
    442     dailyDirs = findDailyDirs(config.stage.targetDir, SPLIT_INDICATOR)
    443     for dailyDir in dailyDirs:
    444        _splitDailyDir(dailyDir, local.split.sizeLimit, local.split.splitSize,
    445                       config.options.backupUser, config.options.backupGroup)
    446        writeIndicatorFile(dailyDir, SPLIT_INDICATOR, config.options.backupUser, config.options.backupGroup)
    447     logger.info("Executed the split extended action successfully.")
    448
    449
    450  ##############################
    451  # _splitDailyDir() function
    452  ##############################
    453
    454 -def _splitDailyDir(dailyDir, sizeLimit, splitSize, backupUser, backupGroup):
    455     """
    456     Splits large files in a daily staging directory.
    457
    458     Files that match INDICATOR_PATTERNS (i.e. C{"cback.store"},
    459     C{"cback.stage"}, etc.) are assumed to be indicator files and are ignored.
    460     All other files are split.
    461
    462     @param dailyDir: Daily directory to encrypt
    463     @param sizeLimit: Size limit, in bytes
    464     @param splitSize: Split size, in bytes
    465     @param backupUser: User that target files should be owned by
    466     @param backupGroup: Group that target files should be owned by
    467
    468     @raise ValueError: If the encrypt mode is not supported.
    469     @raise ValueError: If the daily staging directory does not exist.
    470     """
    471     logger.debug("Begin splitting contents of [%s].", dailyDir)
    472     fileList = getBackupFiles(dailyDir)  # ignores indicator files
    473     for path in fileList:
    474        size = float(os.stat(path).st_size)
    475        if size > sizeLimit:
    476           _splitFile(path, splitSize, backupUser, backupGroup, removeSource=True)
    477     logger.debug("Completed splitting contents of [%s].", dailyDir)
    478
    479
    480  ########################
    481  # _splitFile() function
    482  ########################
    483
    484 -def _splitFile(sourcePath, splitSize, backupUser, backupGroup, removeSource=False):
    485     """
    486     Splits the source file into chunks of the indicated size.
    487
    488     The split files will be owned by the indicated backup user and group.  If
    489     C{removeSource} is C{True}, then the source file will be removed after it is
    490     successfully split.
    491
    492     @param sourcePath: Absolute path of the source file to split
    493     @param splitSize: Encryption mode (only "gpg" is allowed)
    494     @param backupUser: User that target files should be owned by
    495     @param backupGroup: Group that target files should be owned by
    496     @param removeSource: Indicates whether to remove the source file
    497
    498     @raise IOError: If there is a problem accessing, splitting or removing the source file.
    499     """
    500     cwd = os.getcwd()
    501     try:
    502        if not os.path.exists(sourcePath):
    503           raise ValueError("Source path [%s] does not exist." % sourcePath)
    504        dirname = os.path.dirname(sourcePath)
    505        filename = os.path.basename(sourcePath)
    506        prefix = "%s_" % filename
    507        bytes = int(splitSize.bytes) # pylint: disable=W0622
    508        os.chdir(dirname) # need to operate from directory that we want files written to
    509        command = resolveCommand(SPLIT_COMMAND)
    510        args = [ "--verbose", "--numeric-suffixes", "--suffix-length=5", "--bytes=%d" % bytes, filename, prefix, ]
    511        (result, output) = executeCommand(command, args, returnOutput=True, ignoreStderr=False)
    512        if result != 0:
    513           raise IOError("Error [%d] calling split for [%s]." % (result, sourcePath))
    514        pattern = re.compile(r"(creating file [`'])(%s)(.*)(')" % prefix)
    515        match = pattern.search(output[-1:][0])
    516        if match is None:
    517           raise IOError("Unable to parse output from split command.")
    518        value = int(match.group(3).strip())
    519        for index in range(0, value):
    520           path = "%s%05d" % (prefix, index)
    521           if not os.path.exists(path):
    522              raise IOError("After call to split, expected file [%s] does not exist." % path)
    523           changeOwnership(path, backupUser, backupGroup)
    524        if removeSource:
    525           if os.path.exists(sourcePath):
    526              try:
    527                 os.remove(sourcePath)
    528                 logger.debug("Completed removing old file [%s].", sourcePath)
    529              except:
    530                 raise IOError("Failed to remove file [%s] after splitting it." % (sourcePath))
    531     finally:
    532        os.chdir(cwd)
    533