CedarBackup2.actions.store

Package CedarBackup2 :: Package actions :: Module store |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup2.actions.store

      1  # -*- coding: iso-8859-1 -*-
      2  # vim: set ft=python ts=3 sw=3 expandtab:
      3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      4  #
      5  #              C E D A R
      6  #          S O L U T I O N S       "Software done right."
      7  #           S O F T W A R E
      8  #
      9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     10  #
     11  # Copyright (c) 2004-2007,2010 Kenneth J. Pronovici.
     12  # All rights reserved.
     13  #
     14  # This program is free software; you can redistribute it and/or
     15  # modify it under the terms of the GNU General Public License,
     16  # Version 2, as published by the Free Software Foundation.
     17  #
     18  # This program is distributed in the hope that it will be useful,
     19  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     21  #
     22  # Copies of the GNU General Public License are available from
     23  # the Free Software Foundation website, http://www.gnu.org/.
     24  #
     25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     26  #
     27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
     28  # Language : Python 2 (>= 2.7)
     29  # Project  : Cedar Backup, release 2
     30  # Purpose  : Implements the standard 'store' action.
     31  #
     32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     33
     34  ########################################################################
     35  # Module documentation
     36  ########################################################################
     37
     38  """
     39  Implements the standard 'store' action.
     40  @sort: executeStore, writeImage, writeStoreIndicator, consistencyCheck
     41  @author: Kenneth J. Pronovici <pronovic@ieee.org>
     42  @author: Dmitry Rutsky <rutsky@inbox.ru>
     43  """
     44
     45
     46  ########################################################################
     47  # Imported modules
     48  ########################################################################
     49
     50  # System modules
     51  import sys
     52  import os
     53  import logging
     54  import datetime
     55  import tempfile
     56
     57  # Cedar Backup modules
     58  from CedarBackup2.filesystem import compareContents
     59  from CedarBackup2.util import isStartOfWeek
     60  from CedarBackup2.util import mount, unmount, displayBytes
     61  from CedarBackup2.actions.util import createWriter, checkMediaState, buildMediaLabel, writeIndicatorFile
     62  from CedarBackup2.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR, STORE_INDICATOR
     63
     64
     65  ########################################################################
     66  # Module-wide constants and variables
     67  ########################################################################
     68
     69  logger = logging.getLogger("CedarBackup2.log.actions.store")
     70
     71
     72  ########################################################################
     73  # Public functions
     74  ########################################################################
     75
     76  ##########################
     77  # executeStore() function
     78  ##########################
     79
     80  # pylint: disable=W0613
     81 -def executeStore(configPath, options, config):
     82     """
     83     Executes the store backup action.
     84
     85     @note: The rebuild action and the store action are very similar.  The
     86     main difference is that while store only stores a single day's staging
     87     directory, the rebuild action operates on multiple staging directories.
     88
     89     @note: When the store action is complete, we will write a store indicator to
     90     the daily staging directory we used, so it's obvious that the store action
     91     has completed.
     92
     93     @param configPath: Path to configuration file on disk.
     94     @type configPath: String representing a path on disk.
     95
     96     @param options: Program command-line options.
     97     @type options: Options object.
     98
     99     @param config: Program configuration.
    100     @type config: Config object.
    101
    102     @raise ValueError: Under many generic error conditions
    103     @raise IOError: If there are problems reading or writing files.
    104     """
    105     logger.debug("Executing the 'store' action.")
    106     if sys.platform == "darwin":
    107        logger.warn("Warning: the store action is not fully supported on Mac OS X.")
    108        logger.warn("See the Cedar Backup software manual for further information.")
    109     if config.options is None or config.store is None:
    110        raise ValueError("Store configuration is not properly filled in.")
    111     if config.store.checkMedia:
    112        checkMediaState(config.store)  # raises exception if media is not initialized
    113     rebuildMedia = options.full
    114     logger.debug("Rebuild media flag [%s]", rebuildMedia)
    115     todayIsStart = isStartOfWeek(config.options.startingDay)
    116     stagingDirs = _findCorrectDailyDir(options, config)
    117     writeImageBlankSafe(config, rebuildMedia, todayIsStart, config.store.blankBehavior, stagingDirs)
    118     if config.store.checkData:
    119        if sys.platform == "darwin":
    120           logger.warn("Warning: consistency check cannot be run successfully on Mac OS X.")
    121           logger.warn("See the Cedar Backup software manual for further information.")
    122        else:
    123           logger.debug("Running consistency check of media.")
    124           consistencyCheck(config, stagingDirs)
    125     writeStoreIndicator(config, stagingDirs)
    126     logger.info("Executed the 'store' action successfully.")
    127
    128
    129  ########################
    130  # writeImage() function
    131  ########################
    132
    133 -def writeImage(config, newDisc, stagingDirs):
    134     """
    135     Builds and writes an ISO image containing the indicated stage directories.
    136
    137     The generated image will contain each of the staging directories listed in
    138     C{stagingDirs}.  The directories will be placed into the image at the root by
    139     date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
    140     disc at C{/2005/02/10}.
    141
    142     @note: This function is implemented in terms of L{writeImageBlankSafe}.  The
    143     C{newDisc} flag is passed in for both C{rebuildMedia} and C{todayIsStart}.
    144
    145     @param config: Config object.
    146     @param newDisc: Indicates whether the disc should be re-initialized
    147     @param stagingDirs: Dictionary mapping directory path to date suffix.
    148
    149     @raise ValueError: Under many generic error conditions
    150     @raise IOError: If there is a problem writing the image to disc.
    151     """
    152     writeImageBlankSafe(config, newDisc, newDisc, None, stagingDirs)
    153
    154
    155  #################################
    156  # writeImageBlankSafe() function
    157  #################################
    158
    159 -def writeImageBlankSafe(config, rebuildMedia, todayIsStart, blankBehavior, stagingDirs):
    160     """
    161     Builds and writes an ISO image containing the indicated stage directories.
    162
    163     The generated image will contain each of the staging directories listed in
    164     C{stagingDirs}.  The directories will be placed into the image at the root by
    165     date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
    166     disc at C{/2005/02/10}.  The media will always be written with a media
    167     label specific to Cedar Backup.
    168
    169     This function is similar to L{writeImage}, but tries to implement a smarter
    170     blanking strategy.
    171
    172     First, the media is always blanked if the C{rebuildMedia} flag is true.
    173     Then, if C{rebuildMedia} is false, blanking behavior and C{todayIsStart}
    174     come into effect::
    175
    176        If no blanking behavior is specified, and it is the start of the week,
    177        the disc will be blanked
    178
    179        If blanking behavior is specified, and either the blank mode is "daily"
    180        or the blank mode is "weekly" and it is the start of the week, then
    181        the disc will be blanked if it looks like the weekly backup will not
    182        fit onto the media.
    183
    184        Otherwise, the disc will not be blanked
    185
    186     How do we decide whether the weekly backup will fit onto the media?  That is
    187     what the blanking factor is used for.  The following formula is used::
    188
    189        will backup fit? = (bytes available / (1 + bytes required) <= blankFactor
    190
    191     The blanking factor will vary from setup to setup, and will probably
    192     require some experimentation to get it right.
    193
    194     @param config: Config object.
    195     @param rebuildMedia: Indicates whether media should be rebuilt
    196     @param todayIsStart: Indicates whether today is the starting day of the week
    197     @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
    198     @param stagingDirs: Dictionary mapping directory path to date suffix.
    199
    200     @raise ValueError: Under many generic error conditions
    201     @raise IOError: If there is a problem writing the image to disc.
    202     """
    203     mediaLabel = buildMediaLabel()
    204     writer = createWriter(config)
    205     writer.initializeImage(True, config.options.workingDir, mediaLabel)  # default value for newDisc
    206     for stageDir in stagingDirs.keys():
    207        logger.debug("Adding stage directory [%s].", stageDir)
    208        dateSuffix = stagingDirs[stageDir]
    209        writer.addImageEntry(stageDir, dateSuffix)
    210     newDisc = _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior)
    211     writer.setImageNewDisc(newDisc)
    212     writer.writeImage()
    213
    214 -def _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior):
    215     """
    216     Gets a value for the newDisc flag based on blanking factor rules.
    217
    218     The blanking factor rules are described above by L{writeImageBlankSafe}.
    219
    220     @param writer: Previously configured image writer containing image entries
    221     @param rebuildMedia: Indicates whether media should be rebuilt
    222     @param todayIsStart: Indicates whether today is the starting day of the week
    223     @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
    224
    225     @return: newDisc flag to be set on writer.
    226     """
    227     newDisc = False
    228     if rebuildMedia:
    229        newDisc = True
    230        logger.debug("Setting new disc flag based on rebuildMedia flag.")
    231     else:
    232        if blankBehavior is None:
    233           logger.debug("Default media blanking behavior is in effect.")
    234           if todayIsStart:
    235              newDisc = True
    236              logger.debug("Setting new disc flag based on todayIsStart.")
    237        else:
    238           # note: validation says we can assume that behavior is fully filled in if it exists at all
    239           logger.debug("Optimized media blanking behavior is in effect based on configuration.")
    240           if blankBehavior.blankMode == "daily" or (blankBehavior.blankMode == "weekly" and todayIsStart):
    241              logger.debug("New disc flag will be set based on blank factor calculation.")
    242              blankFactor = float(blankBehavior.blankFactor)
    243              logger.debug("Configured blanking factor: %.2f", blankFactor)
    244              available = writer.retrieveCapacity().bytesAvailable
    245              logger.debug("Bytes available: %s", displayBytes(available))
    246              required = writer.getEstimatedImageSize()
    247              logger.debug("Bytes required: %s", displayBytes(required))
    248              ratio = available / (1.0 + required)
    249              logger.debug("Calculated ratio: %.2f", ratio)
    250              newDisc = (ratio <= blankFactor)
    251              logger.debug("%.2f <= %.2f ? %s", ratio, blankFactor, newDisc)
    252           else:
    253              logger.debug("No blank factor calculation is required based on configuration.")
    254     logger.debug("New disc flag [%s].", newDisc)
    255     return newDisc
    256
    257
    258  #################################
    259  # writeStoreIndicator() function
    260  #################################
    261
    262 -def writeStoreIndicator(config, stagingDirs):
    263     """
    264     Writes a store indicator file into staging directories.
    265
    266     The store indicator is written into each of the staging directories when
    267     either a store or rebuild action has written the staging directory to disc.
    268
    269     @param config: Config object.
    270     @param stagingDirs: Dictionary mapping directory path to date suffix.
    271     """
    272     for stagingDir in stagingDirs.keys():
    273        writeIndicatorFile(stagingDir, STORE_INDICATOR,
    274                           config.options.backupUser,
    275                           config.options.backupGroup)
    276
    277
    278  ##############################
    279  # consistencyCheck() function
    280  ##############################
    281
    282 -def consistencyCheck(config, stagingDirs):
    283     """
    284     Runs a consistency check against media in the backup device.
    285
    286     It seems that sometimes, it's possible to create a corrupted multisession
    287     disc (i.e. one that cannot be read) although no errors were encountered
    288     while writing the disc.  This consistency check makes sure that the data
    289     read from disc matches the data that was used to create the disc.
    290
    291     The function mounts the device at a temporary mount point in the working
    292     directory, and then compares the indicated staging directories in the
    293     staging directory and on the media.  The comparison is done via
    294     functionality in C{filesystem.py}.
    295
    296     If no exceptions are thrown, there were no problems with the consistency
    297     check.  A positive confirmation of "no problems" is also written to the log
    298     with C{info} priority.
    299
    300     @warning: The implementation of this function is very UNIX-specific.
    301
    302     @param config: Config object.
    303     @param stagingDirs: Dictionary mapping directory path to date suffix.
    304
    305     @raise ValueError: If the two directories are not equivalent.
    306     @raise IOError: If there is a problem working with the media.
    307     """
    308     logger.debug("Running consistency check.")
    309     mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
    310     try:
    311        mount(config.store.devicePath, mountPoint, "iso9660")
    312        for stagingDir in stagingDirs.keys():
    313           discDir = os.path.join(mountPoint, stagingDirs[stagingDir])
    314           logger.debug("Checking [%s] vs. [%s].", stagingDir, discDir)
    315           compareContents(stagingDir, discDir, verbose=True)
    316           logger.info("Consistency check completed for [%s].  No problems found.", stagingDir)
    317     finally:
    318        unmount(mountPoint, True, 5, 1)  # try 5 times, and remove mount point when done
    319
    320
    321  ########################################################################
    322  # Private utility functions
    323  ########################################################################
    324
    325  #########################
    326  # _findCorrectDailyDir()
    327  #########################
    328
    329 -def _findCorrectDailyDir(options, config):
    330     """
    331     Finds the correct daily staging directory to be written to disk.
    332
    333     In Cedar Backup v1.0, we assumed that the correct staging directory matched
    334     the current date.  However, that has problems.  In particular, it breaks
    335     down if collect is on one side of midnite and stage is on the other, or if
    336     certain processes span midnite.
    337
    338     For v2.0, I'm trying to be smarter.  I'll first check the current day.  If
    339     that directory is found, it's good enough.  If it's not found, I'll look for
    340     a valid directory from the day before or day after I{which has not yet been
    341     staged, according to the stage indicator file}.  The first one I find, I'll
    342     use.  If I use a directory other than for the current day I{and}
    343     C{config.store.warnMidnite} is set, a warning will be put in the log.
    344
    345     There is one exception to this rule.  If the C{options.full} flag is set,
    346     then the special "span midnite" logic will be disabled and any existing
    347     store indicator will be ignored.  I did this because I think that most users
    348     who run C{cback --full store} twice in a row expect the command to generate
    349     two identical discs.  With the other rule in place, running that command
    350     twice in a row could result in an error ("no unstored directory exists") or
    351     could even cause a completely unexpected directory to be written to disc (if
    352     some previous day's contents had not yet been written).
    353
    354     @note: This code is probably longer and more verbose than it needs to be,
    355     but at least it's straightforward.
    356
    357     @param options: Options object.
    358     @param config: Config object.
    359
    360     @return: Correct staging dir, as a dict mapping directory to date suffix.
    361     @raise IOError: If the staging directory cannot be found.
    362     """
    363     oneDay = datetime.timedelta(days=1)
    364     today = datetime.date.today()
    365     yesterday = today - oneDay
    366     tomorrow = today + oneDay
    367     todayDate = today.strftime(DIR_TIME_FORMAT)
    368     yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT)
    369     tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT)
    370     todayPath = os.path.join(config.stage.targetDir, todayDate)
    371     yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate)
    372     tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate)
    373     todayStageInd = os.path.join(todayPath, STAGE_INDICATOR)
    374     yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR)
    375     tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR)
    376     todayStoreInd = os.path.join(todayPath, STORE_INDICATOR)
    377     yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR)
    378     tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR)
    379     if options.full:
    380        if os.path.isdir(todayPath) and os.path.exists(todayStageInd):
    381           logger.info("Store process will use current day's stage directory [%s]", todayPath)
    382           return { todayPath:todayDate }
    383        raise IOError("Unable to find staging directory to store (only tried today due to full option).")
    384     else:
    385        if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd):
    386           logger.info("Store process will use current day's stage directory [%s]", todayPath)
    387           return { todayPath:todayDate }
    388        elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd):
    389           logger.info("Store process will use previous day's stage directory [%s]", yesterdayPath)
    390           if config.store.warnMidnite:
    391              logger.warn("Warning: store process crossed midnite boundary to find data.")
    392           return { yesterdayPath:yesterdayDate }
    393        elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd):
    394           logger.info("Store process will use next day's stage directory [%s]", tomorrowPath)
    395           if config.store.warnMidnite:
    396              logger.warn("Warning: store process crossed midnite boundary to find data.")
    397           return { tomorrowPath:tomorrowDate }
    398        raise IOError("Unable to find unused staging directory to store (tried today, yesterday, tomorrow).")
    399