CedarBackup2.actions.stage

Package CedarBackup2 :: Package actions :: Module stage |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup2.actions.stage

      1  # -*- coding: iso-8859-1 -*-
      2  # vim: set ft=python ts=3 sw=3 expandtab:
      3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      4  #
      5  #              C E D A R
      6  #          S O L U T I O N S       "Software done right."
      7  #           S O F T W A R E
      8  #
      9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     10  #
     11  # Copyright (c) 2004-2008,2010 Kenneth J. Pronovici.
     12  # All rights reserved.
     13  #
     14  # This program is free software; you can redistribute it and/or
     15  # modify it under the terms of the GNU General Public License,
     16  # Version 2, as published by the Free Software Foundation.
     17  #
     18  # This program is distributed in the hope that it will be useful,
     19  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     20  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     21  #
     22  # Copies of the GNU General Public License are available from
     23  # the Free Software Foundation website, http://www.gnu.org/.
     24  #
     25  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     26  #
     27  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
     28  # Language : Python 2 (>= 2.7)
     29  # Project  : Cedar Backup, release 2
     30  # Purpose  : Implements the standard 'stage' action.
     31  #
     32  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     33
     34  ########################################################################
     35  # Module documentation
     36  ########################################################################
     37
     38  """
     39  Implements the standard 'stage' action.
     40  @sort: executeStage
     41  @author: Kenneth J. Pronovici <pronovic@ieee.org>
     42  """
     43
     44
     45  ########################################################################
     46  # Imported modules
     47  ########################################################################
     48
     49  # System modules
     50  import os
     51  import time
     52  import logging
     53
     54  # Cedar Backup modules
     55  from CedarBackup2.peer import RemotePeer, LocalPeer
     56  from CedarBackup2.util import getUidGid, changeOwnership, isStartOfWeek, isRunningAsRoot
     57  from CedarBackup2.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR
     58  from CedarBackup2.actions.util import writeIndicatorFile
     59
     60
     61  ########################################################################
     62  # Module-wide constants and variables
     63  ########################################################################
     64
     65  logger = logging.getLogger("CedarBackup2.log.actions.stage")
     66
     67
     68  ########################################################################
     69  # Public functions
     70  ########################################################################
     71
     72  ##########################
     73  # executeStage() function
     74  ##########################
     75
     76  # pylint: disable=W0613
     77 -def executeStage(configPath, options, config):
     78     """
     79     Executes the stage backup action.
     80
     81     @note: The daily directory is derived once and then we stick with it, just
     82     in case a backup happens to span midnite.
     83
     84     @note: As portions of the stage action is complete, we will write various
     85     indicator files so that it's obvious what actions have been completed.  Each
     86     peer gets a stage indicator in its collect directory, and then the master
     87     gets a stage indicator in its daily staging directory.  The store process
     88     uses the master's stage indicator to decide whether a directory is ready to
     89     be stored.  Currently, nothing uses the indicator at each peer, and it
     90     exists for reference only.
     91
     92     @param configPath: Path to configuration file on disk.
     93     @type configPath: String representing a path on disk.
     94
     95     @param options: Program command-line options.
     96     @type options: Options object.
     97
     98     @param config: Program configuration.
     99     @type config: Config object.
    100
    101     @raise ValueError: Under many generic error conditions
    102     @raise IOError: If there are problems reading or writing files.
    103     """
    104     logger.debug("Executing the 'stage' action.")
    105     if config.options is None or config.stage is None:
    106        raise ValueError("Stage configuration is not properly filled in.")
    107     dailyDir = _getDailyDir(config)
    108     localPeers = _getLocalPeers(config)
    109     remotePeers = _getRemotePeers(config)
    110     allPeers = localPeers + remotePeers
    111     stagingDirs = _createStagingDirs(config, dailyDir, allPeers)
    112     for peer in allPeers:
    113        logger.info("Staging peer [%s].", peer.name)
    114        ignoreFailures = _getIgnoreFailuresFlag(options, config, peer)
    115        if not peer.checkCollectIndicator():
    116           if not ignoreFailures:
    117              logger.error("Peer [%s] was not ready to be staged.", peer.name)
    118           else:
    119              logger.info("Peer [%s] was not ready to be staged.", peer.name)
    120           continue
    121        logger.debug("Found collect indicator.")
    122        targetDir = stagingDirs[peer.name]
    123        if isRunningAsRoot():
    124           # Since we're running as root, we can change ownership
    125           ownership = getUidGid(config.options.backupUser,  config.options.backupGroup)
    126           logger.debug("Using target dir [%s], ownership [%d:%d].", targetDir, ownership[0], ownership[1])
    127        else:
    128           # Non-root cannot change ownership, so don't set it
    129           ownership = None
    130           logger.debug("Using target dir [%s], ownership [None].", targetDir)
    131        try:
    132           count = peer.stagePeer(targetDir=targetDir, ownership=ownership)  # note: utilize effective user's default umask
    133           logger.info("Staged %d files for peer [%s].", count, peer.name)
    134           peer.writeStageIndicator()
    135        except (ValueError, IOError, OSError), e:
    136           logger.error("Error staging [%s]: %s", peer.name, e)
    137     writeIndicatorFile(dailyDir, STAGE_INDICATOR, config.options.backupUser, config.options.backupGroup)
    138     logger.info("Executed the 'stage' action successfully.")
    139
    140
    141  ########################################################################
    142  # Private utility functions
    143  ########################################################################
    144
    145  ################################
    146  # _createStagingDirs() function
    147  ################################
    148
    149 -def _createStagingDirs(config, dailyDir, peers):
    150     """
    151     Creates staging directories as required.
    152
    153     The main staging directory is the passed in daily directory, something like
    154     C{staging/2002/05/23}.  Then, individual peers get their own directories,
    155     i.e. C{staging/2002/05/23/host}.
    156
    157     @param config: Config object.
    158     @param dailyDir: Daily staging directory.
    159     @param peers: List of all configured peers.
    160
    161     @return: Dictionary mapping peer name to staging directory.
    162     """
    163     mapping = {}
    164     if os.path.isdir(dailyDir):
    165        logger.warn("Staging directory [%s] already existed.", dailyDir)
    166     else:
    167        try:
    168           logger.debug("Creating staging directory [%s].", dailyDir)
    169           os.makedirs(dailyDir)
    170           for path in [ dailyDir, os.path.join(dailyDir, ".."), os.path.join(dailyDir, "..", ".."), ]:
    171              changeOwnership(path, config.options.backupUser, config.options.backupGroup)
    172        except Exception, e:
    173           raise Exception("Unable to create staging directory: %s" % e)
    174     for peer in peers:
    175        peerDir = os.path.join(dailyDir, peer.name)
    176        mapping[peer.name] = peerDir
    177        if os.path.isdir(peerDir):
    178           logger.warn("Peer staging directory [%s] already existed.", peerDir)
    179        else:
    180           try:
    181              logger.debug("Creating peer staging directory [%s].", peerDir)
    182              os.makedirs(peerDir)
    183              changeOwnership(peerDir, config.options.backupUser, config.options.backupGroup)
    184           except Exception, e:
    185              raise Exception("Unable to create staging directory: %s" % e)
    186     return mapping
    187
    188
    189  ########################################################################
    190  # Private attribute "getter" functions
    191  ########################################################################
    192
    193  ####################################
    194  # _getIgnoreFailuresFlag() function
    195  ####################################
    196
    197 -def _getIgnoreFailuresFlag(options, config, peer):
    198     """
    199     Gets the ignore failures flag based on options, configuration, and peer.
    200     @param options: Options object
    201     @param config: Configuration object
    202     @param peer: Peer to check
    203     @return: Whether to ignore stage failures for this peer
    204     """
    205     logger.debug("Ignore failure mode for this peer: %s", peer.ignoreFailureMode)
    206     if peer.ignoreFailureMode is None or peer.ignoreFailureMode == "none":
    207        return False
    208     elif peer.ignoreFailureMode == "all":
    209        return True
    210     else:
    211        if options.full or isStartOfWeek(config.options.startingDay):
    212           return peer.ignoreFailureMode == "weekly"
    213        else:
    214           return peer.ignoreFailureMode == "daily"
    215
    216
    217  ##########################
    218  # _getDailyDir() function
    219  ##########################
    220
    221 -def _getDailyDir(config):
    222     """
    223     Gets the daily staging directory.
    224
    225     This is just a directory in the form C{staging/YYYY/MM/DD}, i.e.
    226     C{staging/2000/10/07}, except it will be an absolute path based on
    227     C{config.stage.targetDir}.
    228
    229     @param config: Config object
    230
    231     @return: Path of daily staging directory.
    232     """
    233     dailyDir = os.path.join(config.stage.targetDir, time.strftime(DIR_TIME_FORMAT))
    234     logger.debug("Daily staging directory is [%s].", dailyDir)
    235     return dailyDir
    236
    237
    238  ############################
    239  # _getLocalPeers() function
    240  ############################
    241
    242 -def _getLocalPeers(config):
    243     """
    244     Return a list of L{LocalPeer} objects based on configuration.
    245     @param config: Config object.
    246     @return: List of L{LocalPeer} objects.
    247     """
    248     localPeers = []
    249     configPeers = None
    250     if config.stage.hasPeers():
    251        logger.debug("Using list of local peers from stage configuration.")
    252        configPeers = config.stage.localPeers
    253     elif config.peers is not None and config.peers.hasPeers():
    254        logger.debug("Using list of local peers from peers configuration.")
    255        configPeers = config.peers.localPeers
    256     if configPeers is not None:
    257        for peer in configPeers:
    258           localPeer = LocalPeer(peer.name, peer.collectDir, peer.ignoreFailureMode)
    259           localPeers.append(localPeer)
    260           logger.debug("Found local peer: [%s]", localPeer.name)
    261     return localPeers
    262
    263
    264  #############################
    265  # _getRemotePeers() function
    266  #############################
    267
    268 -def _getRemotePeers(config):
    269     """
    270     Return a list of L{RemotePeer} objects based on configuration.
    271     @param config: Config object.
    272     @return: List of L{RemotePeer} objects.
    273     """
    274     remotePeers = []
    275     configPeers = None
    276     if config.stage.hasPeers():
    277        logger.debug("Using list of remote peers from stage configuration.")
    278        configPeers = config.stage.remotePeers
    279     elif config.peers is not None and config.peers.hasPeers():
    280        logger.debug("Using list of remote peers from peers configuration.")
    281        configPeers = config.peers.remotePeers
    282     if configPeers is not None:
    283        for peer in configPeers:
    284           remoteUser = _getRemoteUser(config, peer)
    285           localUser = _getLocalUser(config)
    286           rcpCommand = _getRcpCommand(config, peer)
    287           remotePeer = RemotePeer(peer.name, peer.collectDir, config.options.workingDir,
    288                                   remoteUser, rcpCommand, localUser,
    289                                   ignoreFailureMode=peer.ignoreFailureMode)
    290           remotePeers.append(remotePeer)
    291           logger.debug("Found remote peer: [%s]", remotePeer.name)
    292     return remotePeers
    293
    294
    295  ############################
    296  # _getRemoteUser() function
    297  ############################
    298
    299 -def _getRemoteUser(config, remotePeer):
    300     """
    301     Gets the remote user associated with a remote peer.
    302     Use peer's if possible, otherwise take from options section.
    303     @param config: Config object.
    304     @param remotePeer: Configuration-style remote peer object.
    305     @return: Name of remote user associated with remote peer.
    306     """
    307     if remotePeer.remoteUser is None:
    308        return config.options.backupUser
    309     return remotePeer.remoteUser
    310
    311
    312  ###########################
    313  # _getLocalUser() function
    314  ###########################
    315
    316 -def _getLocalUser(config):
    317     """
    318     Gets the remote user associated with a remote peer.
    319     @param config: Config object.
    320     @return: Name of local user that should be used
    321     """
    322     if not isRunningAsRoot():
    323        return None
    324     return config.options.backupUser
    325
    326
    327  ############################
    328  # _getRcpCommand() function
    329  ############################
    330
    331 -def _getRcpCommand(config, remotePeer):
    332     """
    333     Gets the RCP command associated with a remote peer.
    334     Use peer's if possible, otherwise take from options section.
    335     @param config: Config object.
    336     @param remotePeer: Configuration-style remote peer object.
    337     @return: RCP command associated with remote peer.
    338     """
    339     if remotePeer.rcpCommand is None:
    340        return config.options.rcpCommand
    341     return remotePeer.rcpCommand
    342