CedarBackup2.actions.store

Package CedarBackup2 :: Package actions :: Module store |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup2.actions.store

      1  # -*- coding: iso-8859-1 -*-
      2  # vim: set ft=python ts=3 sw=3 expandtab:
      3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      4  #
      5  #              C E D A R
      6  #          S O L U T I O N S       "Software done right."
      7  #           S O F T W A R E
      8  #
      9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     10  #
     11  # Copyright (c) 2004-2007,2010 Kenneth J. Pronovici.
     12  # All rights reserved.
     13  #
     14  # This program is free software; you can redistribute it and/or
     15  # modify it under the terms of the GNU General Public License,
     16  # Version 2, as published by the Free Software Foundation.
     17  #
     18  # This program is distributed in the hope that it will be useful,
     19  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     21  #
     22  # Copies of the GNU General Public License are available from
     23  # the Free Software Foundation website, http://www.gnu.org/.
     24  #
     25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     26  #
     27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
     28  # Language : Python 2 (>= 2.7)
     29  # Project  : Cedar Backup, release 2
     30  # Purpose  : Implements the standard 'store' action.
     31  #
     32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     33
     34  ########################################################################
     35  # Module documentation
     36  ########################################################################
     37
     38  """
     39  Implements the standard 'store' action.
     40  @sort: executeStore, writeImage, writeStoreIndicator, consistencyCheck
     41  @author: Kenneth J. Pronovici <pronovic@ieee.org>
     42  @author: Dmitry Rutsky <rutsky@inbox.ru>
     43  """
     44
     45
     46  ########################################################################
     47  # Imported modules
     48  ########################################################################
     49
     50  # System modules
     51  import sys
     52  import os
     53  import logging
     54  import datetime
     55  import tempfile
     56
     57  # Cedar Backup modules
     58  from CedarBackup2.filesystem import compareContents
     59  from CedarBackup2.util import isStartOfWeek
     60  from CedarBackup2.util import mount, unmount, displayBytes
     61  from CedarBackup2.actions.util import createWriter, checkMediaState, buildMediaLabel, writeIndicatorFile
     62  from CedarBackup2.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR, STORE_INDICATOR
     63
     64
     65  ########################################################################
     66  # Module-wide constants and variables
     67  ########################################################################
     68
     69  logger = logging.getLogger("CedarBackup2.log.actions.store")
     70
     71
     72  ########################################################################
     73  # Public functions
     74  ########################################################################
     75
     76  ##########################
     77  # executeStore() function
     78  ##########################
     79
     80 -def executeStore(configPath, options, config):
     81     """
     82     Executes the store backup action.
     83
     84     @note: The rebuild action and the store action are very similar.  The
     85     main difference is that while store only stores a single day's staging
     86     directory, the rebuild action operates on multiple staging directories.
     87
     88     @note: When the store action is complete, we will write a store indicator to
     89     the daily staging directory we used, so it's obvious that the store action
     90     has completed.
     91
     92     @param configPath: Path to configuration file on disk.
     93     @type configPath: String representing a path on disk.
     94
     95     @param options: Program command-line options.
     96     @type options: Options object.
     97
     98     @param config: Program configuration.
     99     @type config: Config object.
    100
    101     @raise ValueError: Under many generic error conditions
    102     @raise IOError: If there are problems reading or writing files.
    103     """
    104     logger.debug("Executing the 'store' action.")
    105     if sys.platform == "darwin":
    106        logger.warn("Warning: the store action is not fully supported on Mac OS X.")
    107        logger.warn("See the Cedar Backup software manual for further information.")
    108     if config.options is None or config.store is None:
    109        raise ValueError("Store configuration is not properly filled in.")
    110     if config.store.checkMedia:
    111        checkMediaState(config.store)  # raises exception if media is not initialized
    112     rebuildMedia = options.full
    113     logger.debug("Rebuild media flag [%s]", rebuildMedia)
    114     todayIsStart = isStartOfWeek(config.options.startingDay)
    115     stagingDirs = _findCorrectDailyDir(options, config)
    116     writeImageBlankSafe(config, rebuildMedia, todayIsStart, config.store.blankBehavior, stagingDirs)
    117     if config.store.checkData:
    118        if sys.platform == "darwin":
    119           logger.warn("Warning: consistency check cannot be run successfully on Mac OS X.")
    120           logger.warn("See the Cedar Backup software manual for further information.")
    121        else:
    122           logger.debug("Running consistency check of media.")
    123           consistencyCheck(config, stagingDirs)
    124     writeStoreIndicator(config, stagingDirs)
    125     logger.info("Executed the 'store' action successfully.")
    126
    127
    128  ########################
    129  # writeImage() function
    130  ########################
    131
    132 -def writeImage(config, newDisc, stagingDirs):
    133     """
    134     Builds and writes an ISO image containing the indicated stage directories.
    135
    136     The generated image will contain each of the staging directories listed in
    137     C{stagingDirs}.  The directories will be placed into the image at the root by
    138     date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
    139     disc at C{/2005/02/10}.
    140
    141     @note: This function is implemented in terms of L{writeImageBlankSafe}.  The
    142     C{newDisc} flag is passed in for both C{rebuildMedia} and C{todayIsStart}.
    143
    144     @param config: Config object.
    145     @param newDisc: Indicates whether the disc should be re-initialized
    146     @param stagingDirs: Dictionary mapping directory path to date suffix.
    147
    148     @raise ValueError: Under many generic error conditions
    149     @raise IOError: If there is a problem writing the image to disc.
    150     """
    151     writeImageBlankSafe(config, newDisc, newDisc, None, stagingDirs)
    152
    153
    154  #################################
    155  # writeImageBlankSafe() function
    156  #################################
    157
    158 -def writeImageBlankSafe(config, rebuildMedia, todayIsStart, blankBehavior, stagingDirs):
    159     """
    160     Builds and writes an ISO image containing the indicated stage directories.
    161
    162     The generated image will contain each of the staging directories listed in
    163     C{stagingDirs}.  The directories will be placed into the image at the root by
    164     date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
    165     disc at C{/2005/02/10}.  The media will always be written with a media
    166     label specific to Cedar Backup.
    167
    168     This function is similar to L{writeImage}, but tries to implement a smarter
    169     blanking strategy.
    170
    171     First, the media is always blanked if the C{rebuildMedia} flag is true.
    172     Then, if C{rebuildMedia} is false, blanking behavior and C{todayIsStart}
    173     come into effect::
    174
    175        If no blanking behavior is specified, and it is the start of the week,
    176        the disc will be blanked
    177
    178        If blanking behavior is specified, and either the blank mode is "daily"
    179        or the blank mode is "weekly" and it is the start of the week, then
    180        the disc will be blanked if it looks like the weekly backup will not
    181        fit onto the media.
    182
    183        Otherwise, the disc will not be blanked
    184
    185     How do we decide whether the weekly backup will fit onto the media?  That is
    186     what the blanking factor is used for.  The following formula is used::
    187
    188        will backup fit? = (bytes available / (1 + bytes required) <= blankFactor
    189
    190     The blanking factor will vary from setup to setup, and will probably
    191     require some experimentation to get it right.
    192
    193     @param config: Config object.
    194     @param rebuildMedia: Indicates whether media should be rebuilt
    195     @param todayIsStart: Indicates whether today is the starting day of the week
    196     @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
    197     @param stagingDirs: Dictionary mapping directory path to date suffix.
    198
    199     @raise ValueError: Under many generic error conditions
    200     @raise IOError: If there is a problem writing the image to disc.
    201     """
    202     mediaLabel = buildMediaLabel()
    203     writer = createWriter(config)
    204     writer.initializeImage(True, config.options.workingDir, mediaLabel)  # default value for newDisc
    205     for stageDir in stagingDirs.keys():
    206        logger.debug("Adding stage directory [%s].", stageDir)
    207        dateSuffix = stagingDirs[stageDir]
    208        writer.addImageEntry(stageDir, dateSuffix)
    209     newDisc = _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior)
    210     writer.setImageNewDisc(newDisc)
    211     writer.writeImage()
    212
    213 -def _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior):
    214     """
    215     Gets a value for the newDisc flag based on blanking factor rules.
    216
    217     The blanking factor rules are described above by L{writeImageBlankSafe}.
    218
    219     @param writer: Previously configured image writer containing image entries
    220     @param rebuildMedia: Indicates whether media should be rebuilt
    221     @param todayIsStart: Indicates whether today is the starting day of the week
    222     @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
    223
    224     @return: newDisc flag to be set on writer.
    225     """
    226     newDisc = False
    227     if rebuildMedia:
    228        newDisc = True
    229        logger.debug("Setting new disc flag based on rebuildMedia flag.")
    230     else:
    231        if blankBehavior is None:
    232           logger.debug("Default media blanking behavior is in effect.")
    233           if todayIsStart:
    234              newDisc = True
    235              logger.debug("Setting new disc flag based on todayIsStart.")
    236        else:
    237           # note: validation says we can assume that behavior is fully filled in if it exists at all
    238           logger.debug("Optimized media blanking behavior is in effect based on configuration.")
    239           if blankBehavior.blankMode == "daily" or (blankBehavior.blankMode == "weekly" and todayIsStart):
    240              logger.debug("New disc flag will be set based on blank factor calculation.")
    241              blankFactor = float(blankBehavior.blankFactor)
    242              logger.debug("Configured blanking factor: %.2f", blankFactor)
    243              available = writer.retrieveCapacity().bytesAvailable
    244              logger.debug("Bytes available: %s", displayBytes(available))
    245              required = writer.getEstimatedImageSize()
    246              logger.debug("Bytes required: %s", displayBytes(required))
    247              ratio = available / (1.0 + required)
    248              logger.debug("Calculated ratio: %.2f", ratio)
    249              newDisc = (ratio <= blankFactor)
    250              logger.debug("%.2f <= %.2f ? %s", ratio, blankFactor, newDisc)
    251           else:
    252              logger.debug("No blank factor calculation is required based on configuration.")
    253     logger.debug("New disc flag [%s].", newDisc)
    254     return newDisc
    255
    256
    257  #################################
    258  # writeStoreIndicator() function
    259  #################################
    260
    261 -def writeStoreIndicator(config, stagingDirs):
    262     """
    263     Writes a store indicator file into staging directories.
    264
    265     The store indicator is written into each of the staging directories when
    266     either a store or rebuild action has written the staging directory to disc.
    267
    268     @param config: Config object.
    269     @param stagingDirs: Dictionary mapping directory path to date suffix.
    270     """
    271     for stagingDir in stagingDirs.keys():
    272        writeIndicatorFile(stagingDir, STORE_INDICATOR,
    273                           config.options.backupUser,
    274                           config.options.backupGroup)
    275
    276
    277  ##############################
    278  # consistencyCheck() function
    279  ##############################
    280
    281 -def consistencyCheck(config, stagingDirs):
    282     """
    283     Runs a consistency check against media in the backup device.
    284
    285     It seems that sometimes, it's possible to create a corrupted multisession
    286     disc (i.e. one that cannot be read) although no errors were encountered
    287     while writing the disc.  This consistency check makes sure that the data
    288     read from disc matches the data that was used to create the disc.
    289
    290     The function mounts the device at a temporary mount point in the working
    291     directory, and then compares the indicated staging directories in the
    292     staging directory and on the media.  The comparison is done via
    293     functionality in C{filesystem.py}.
    294
    295     If no exceptions are thrown, there were no problems with the consistency
    296     check.  A positive confirmation of "no problems" is also written to the log
    297     with C{info} priority.
    298
    299     @warning: The implementation of this function is very UNIX-specific.
    300
    301     @param config: Config object.
    302     @param stagingDirs: Dictionary mapping directory path to date suffix.
    303
    304     @raise ValueError: If the two directories are not equivalent.
    305     @raise IOError: If there is a problem working with the media.
    306     """
    307     logger.debug("Running consistency check.")
    308     mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
    309     try:
    310        mount(config.store.devicePath, mountPoint, "iso9660")
    311        for stagingDir in stagingDirs.keys():
    312           discDir = os.path.join(mountPoint, stagingDirs[stagingDir])
    313           logger.debug("Checking [%s] vs. [%s].", stagingDir, discDir)
    314           compareContents(stagingDir, discDir, verbose=True)
    315           logger.info("Consistency check completed for [%s].  No problems found.", stagingDir)
    316     finally:
    317        unmount(mountPoint, True, 5, 1)  # try 5 times, and remove mount point when done
    318
    319
    320  ########################################################################
    321  # Private utility functions
    322  ########################################################################
    323
    324  #########################
    325  # _findCorrectDailyDir()
    326  #########################
    327
    328 -def _findCorrectDailyDir(options, config):
    329     """
    330     Finds the correct daily staging directory to be written to disk.
    331
    332     In Cedar Backup v1.0, we assumed that the correct staging directory matched
    333     the current date.  However, that has problems.  In particular, it breaks
    334     down if collect is on one side of midnite and stage is on the other, or if
    335     certain processes span midnite.
    336
    337     For v2.0, I'm trying to be smarter.  I'll first check the current day.  If
    338     that directory is found, it's good enough.  If it's not found, I'll look for
    339     a valid directory from the day before or day after I{which has not yet been
    340     staged, according to the stage indicator file}.  The first one I find, I'll
    341     use.  If I use a directory other than for the current day I{and}
    342     C{config.store.warnMidnite} is set, a warning will be put in the log.
    343
    344     There is one exception to this rule.  If the C{options.full} flag is set,
    345     then the special "span midnite" logic will be disabled and any existing
    346     store indicator will be ignored.  I did this because I think that most users
    347     who run C{cback --full store} twice in a row expect the command to generate
    348     two identical discs.  With the other rule in place, running that command
    349     twice in a row could result in an error ("no unstored directory exists") or
    350     could even cause a completely unexpected directory to be written to disc (if
    351     some previous day's contents had not yet been written).
    352
    353     @note: This code is probably longer and more verbose than it needs to be,
    354     but at least it's straightforward.
    355
    356     @param options: Options object.
    357     @param config: Config object.
    358
    359     @return: Correct staging dir, as a dict mapping directory to date suffix.
    360     @raise IOError: If the staging directory cannot be found.
    361     """
    362     oneDay = datetime.timedelta(days=1)
    363     today = datetime.date.today()
    364     yesterday = today - oneDay
    365     tomorrow = today + oneDay
    366     todayDate = today.strftime(DIR_TIME_FORMAT)
    367     yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT)
    368     tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT)
    369     todayPath = os.path.join(config.stage.targetDir, todayDate)
    370     yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate)
    371     tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate)
    372     todayStageInd = os.path.join(todayPath, STAGE_INDICATOR)
    373     yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR)
    374     tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR)
    375     todayStoreInd = os.path.join(todayPath, STORE_INDICATOR)
    376     yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR)
    377     tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR)
    378     if options.full:
    379        if os.path.isdir(todayPath) and os.path.exists(todayStageInd):
    380           logger.info("Store process will use current day's stage directory [%s]", todayPath)
    381           return { todayPath:todayDate }
    382        raise IOError("Unable to find staging directory to store (only tried today due to full option).")
    383     else:
    384        if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd):
    385           logger.info("Store process will use current day's stage directory [%s]", todayPath)
    386           return { todayPath:todayDate }
    387        elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd):
    388           logger.info("Store process will use previous day's stage directory [%s]", yesterdayPath)
    389           if config.store.warnMidnite:
    390              logger.warn("Warning: store process crossed midnite boundary to find data.")
    391           return { yesterdayPath:yesterdayDate }
    392        elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd):
    393           logger.info("Store process will use next day's stage directory [%s]", tomorrowPath)
    394           if config.store.warnMidnite:
    395              logger.warn("Warning: store process crossed midnite boundary to find data.")
    396           return { tomorrowPath:tomorrowDate }
    397        raise IOError("Unable to find unused staging directory to store (tried today, yesterday, tomorrow).")
    398