CedarBackup2.extend.postgresql

Package CedarBackup2 :: Package extend :: Module postgresql |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup2.extend.postgresql

      1  # -*- coding: iso-8859-1 -*-
      2  # vim: set ft=python ts=3 sw=3 expandtab:
      3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      4  #
      5  #              C E D A R
      6  #          S O L U T I O N S       "Software done right."
      7  #           S O F T W A R E
      8  #
      9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     10  #
     11  # Copyright (c) 2006,2010 Kenneth J. Pronovici.
     12  # Copyright (c) 2006 Antoine Beaupre.
     13  # All rights reserved.
     14  #
     15  # This program is free software; you can redistribute it and/or
     16  # modify it under the terms of the GNU General Public License,
     17  # Version 2, as published by the Free Software Foundation.
     18  #
     19  # This program is distributed in the hope that it will be useful,
     20  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     21  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     22  #
     23  # Copies of the GNU General Public License are available from
     24  # the Free Software Foundation website, http://www.gnu.org/.
     25  #
     26  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     27  #
     28  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
     29  #            Antoine Beaupre <anarcat@koumbit.org>
     30  # Language : Python (>= 2.5)
     31  # Project  : Official Cedar Backup Extensions
     32  # Revision : $Id: postgresql.py 1022 2011-10-11 23:27:49Z pronovic $
     33  # Purpose  : Provides an extension to back up PostgreSQL databases.
     34  #
     35  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     36  # This file was created with a width of 132 characters, and NO tabs.
     37
     38  ########################################################################
     39  # Module documentation
     40  ########################################################################
     41
     42  """
     43  Provides an extension to back up PostgreSQL databases.
     44
     45  This is a Cedar Backup extension used to back up PostgreSQL databases via the
     46  Cedar Backup command line.  It requires a new configurations section
     47  <postgresql> and is intended to be run either immediately before or immediately
     48  after the standard collect action.  Aside from its own configuration, it
     49  requires the options and collect configuration sections in the standard Cedar
     50  Backup configuration file.
     51
     52  The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with
     53  the PostgreSQL product.  Output can be compressed using C{gzip} or C{bzip2}.
     54  Administrators can configure the extension either to back up all databases or
     55  to back up only specific databases.  The extension assumes that the current
     56  user has passwordless access to the database since there is no easy way to pass
     57  a password to the C{pg_dump} client. This can be accomplished using appropriate
     58  voodoo in the C{pg_hda.conf} file.
     59
     60  Note that this code always produces a full backup.  There is currently no
     61  facility for making incremental backups.
     62
     63  You should always make C{/etc/cback.conf} unreadble to non-root users once you
     64  place postgresql configuration into it, since postgresql configuration will
     65  contain information about available PostgreSQL databases and usernames.
     66
     67  Use of this extension I{may} expose usernames in the process listing (via
     68  C{ps}) when the backup is running if the username is specified in the
     69  configuration.
     70
     71  @author: Kenneth J. Pronovici <pronovic@ieee.org>
     72  @author: Antoine Beaupre <anarcat@koumbit.org>
     73  """
     74
     75  ########################################################################
     76  # Imported modules
     77  ########################################################################
     78
     79  # System modules
     80  import os
     81  import logging
     82  from gzip import GzipFile
     83  from bz2 import BZ2File
     84
     85  # Cedar Backup modules
     86  from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
     87  from CedarBackup2.xmlutil import readFirstChild, readString, readStringList, readBoolean
     88  from CedarBackup2.config import VALID_COMPRESS_MODES
     89  from CedarBackup2.util import resolveCommand, executeCommand
     90  from CedarBackup2.util import ObjectTypeList, changeOwnership
     91
     92
     93  ########################################################################
     94  # Module-wide constants and variables
     95  ########################################################################
     96
     97  logger = logging.getLogger("CedarBackup2.log.extend.postgresql")
     98  POSTGRESQLDUMP_COMMAND = [ "pg_dump", ]
     99  POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ]
    100
    101
    102  ########################################################################
    103  # PostgresqlConfig class definition
    104  ########################################################################
    105
    106 -class PostgresqlConfig(object):
    107
    108     """
    109     Class representing PostgreSQL configuration.
    110
    111     The PostgreSQL configuration information is used for backing up PostgreSQL databases.
    112
    113     The following restrictions exist on data in this class:
    114
    115        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
    116        - The 'all' flag must be 'Y' if no databases are defined.
    117        - The 'all' flag must be 'N' if any databases are defined.
    118        - Any values in the databases list must be strings.
    119
    120     @sort: __init__, __repr__, __str__, __cmp__, user, all, databases
    121     """
    122
    123 -   def __init__(self, user=None, compressMode=None, all=None, databases=None):  # pylint: disable=W0622
    124        """
    125        Constructor for the C{PostgresqlConfig} class.
    126
    127        @param user: User to execute backup as.
    128        @param compressMode: Compress mode for backed-up files.
    129        @param all: Indicates whether to back up all databases.
    130        @param databases: List of databases to back up.
    131        """
    132        self._user = None
    133        self._compressMode = None
    134        self._all = None
    135        self._databases = None
    136        self.user = user
    137        self.compressMode = compressMode
    138        self.all = all
    139        self.databases = databases
    140
    141 -   def __repr__(self):
    142        """
    143        Official string representation for class instance.
    144        """
    145        return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
    146
    147 -   def __str__(self):
    148        """
    149        Informal string representation for class instance.
    150        """
    151        return self.__repr__()
    152
    153 -   def __cmp__(self, other):
    154        """
    155        Definition of equals operator for this class.
    156        @param other: Other object to compare to.
    157        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    158        """
    159        if other is None:
    160           return 1
    161        if self.user != other.user:
    162           if self.user < other.user:
    163              return -1
    164           else:
    165              return 1
    166        if self.compressMode != other.compressMode:
    167           if self.compressMode < other.compressMode:
    168              return -1
    169           else:
    170              return 1
    171        if self.all != other.all:
    172           if self.all < other.all:
    173              return -1
    174           else:
    175              return 1
    176        if self.databases != other.databases:
    177           if self.databases < other.databases:
    178              return -1
    179           else:
    180              return 1
    181        return 0
    182
    183 -   def _setUser(self, value):
    184        """
    185        Property target used to set the user value.
    186        """
    187        if value is not None:
    188           if len(value) < 1:
    189              raise ValueError("User must be non-empty string.")
    190        self._user = value
    191
    192 -   def _getUser(self):
    193        """
    194        Property target used to get the user value.
    195        """
    196        return self._user
    197
    198 -   def _setCompressMode(self, value):
    199        """
    200        Property target used to set the compress mode.
    201        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
    202        @raise ValueError: If the value is not valid.
    203        """
    204        if value is not None:
    205           if value not in VALID_COMPRESS_MODES:
    206              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
    207        self._compressMode = value
    208
    209 -   def _getCompressMode(self):
    210        """
    211        Property target used to get the compress mode.
    212        """
    213        return self._compressMode
    214
    215 -   def _setAll(self, value):
    216        """
    217        Property target used to set the 'all' flag.
    218        No validations, but we normalize the value to C{True} or C{False}.
    219        """
    220        if value:
    221           self._all = True
    222        else:
    223           self._all = False
    224
    225 -   def _getAll(self):
    226        """
    227        Property target used to get the 'all' flag.
    228        """
    229        return self._all
    230
    231 -   def _setDatabases(self, value):
    232        """
    233        Property target used to set the databases list.
    234        Either the value must be C{None} or each element must be a string.
    235        @raise ValueError: If the value is not a string.
    236        """
    237        if value is None:
    238           self._databases = None
    239        else:
    240           for database in value:
    241              if len(database) < 1:
    242                 raise ValueError("Each database must be a non-empty string.")
    243           try:
    244              saved = self._databases
    245              self._databases = ObjectTypeList(basestring, "string")
    246              self._databases.extend(value)
    247           except Exception, e:
    248              self._databases = saved
    249              raise e
    250
    251 -   def _getDatabases(self):
    252        """
    253        Property target used to get the databases list.
    254        """
    255        return self._databases
    256
    257     user = property(_getUser, _setUser, None, "User to execute backup as.")
    258     compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
    259     all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
    260     databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
    261
    262
    263  ########################################################################
    264  # LocalConfig class definition
    265  ########################################################################
    266
    267 -class LocalConfig(object):
    268
    269     """
    270     Class representing this extension's configuration document.
    271
    272     This is not a general-purpose configuration object like the main Cedar
    273     Backup configuration object.  Instead, it just knows how to parse and emit
    274     PostgreSQL-specific configuration values.  Third parties who need to read and
    275     write configuration related to this extension should access it through the
    276     constructor, C{validate} and C{addConfig} methods.
    277
    278     @note: Lists within this class are "unordered" for equality comparisons.
    279
    280     @sort: __init__, __repr__, __str__, __cmp__, postgresql, validate, addConfig
    281     """
    282
    283 -   def __init__(self, xmlData=None, xmlPath=None, validate=True):
    284        """
    285        Initializes a configuration object.
    286
    287        If you initialize the object without passing either C{xmlData} or
    288        C{xmlPath} then configuration will be empty and will be invalid until it
    289        is filled in properly.
    290
    291        No reference to the original XML data or original path is saved off by
    292        this class.  Once the data has been parsed (successfully or not) this
    293        original information is discarded.
    294
    295        Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
    296        method will be called (with its default arguments) against configuration
    297        after successfully parsing any passed-in XML.  Keep in mind that even if
    298        C{validate} is C{False}, it might not be possible to parse the passed-in
    299        XML document if lower-level validations fail.
    300
    301        @note: It is strongly suggested that the C{validate} option always be set
    302        to C{True} (the default) unless there is a specific need to read in
    303        invalid configuration from disk.
    304
    305        @param xmlData: XML data representing configuration.
    306        @type xmlData: String data.
    307
    308        @param xmlPath: Path to an XML file on disk.
    309        @type xmlPath: Absolute path to a file on disk.
    310
    311        @param validate: Validate the document after parsing it.
    312        @type validate: Boolean true/false.
    313
    314        @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
    315        @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
    316        @raise ValueError: If the parsed configuration document is not valid.
    317        """
    318        self._postgresql = None
    319        self.postgresql = None
    320        if xmlData is not None and xmlPath is not None:
    321           raise ValueError("Use either xmlData or xmlPath, but not both.")
    322        if xmlData is not None:
    323           self._parseXmlData(xmlData)
    324           if validate:
    325              self.validate()
    326        elif xmlPath is not None:
    327           xmlData = open(xmlPath).read()
    328           self._parseXmlData(xmlData)
    329           if validate:
    330              self.validate()
    331
    332 -   def __repr__(self):
    333        """
    334        Official string representation for class instance.
    335        """
    336        return "LocalConfig(%s)" % (self.postgresql)
    337
    338 -   def __str__(self):
    339        """
    340        Informal string representation for class instance.
    341        """
    342        return self.__repr__()
    343
    344 -   def __cmp__(self, other):
    345        """
    346        Definition of equals operator for this class.
    347        Lists within this class are "unordered" for equality comparisons.
    348        @param other: Other object to compare to.
    349        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    350        """
    351        if other is None:
    352           return 1
    353        if self.postgresql != other.postgresql:
    354           if self.postgresql < other.postgresql:
    355              return -1
    356           else:
    357              return 1
    358        return 0
    359
    360 -   def _setPostgresql(self, value):
    361        """
    362        Property target used to set the postgresql configuration value.
    363        If not C{None}, the value must be a C{PostgresqlConfig} object.
    364        @raise ValueError: If the value is not a C{PostgresqlConfig}
    365        """
    366        if value is None:
    367           self._postgresql = None
    368        else:
    369           if not isinstance(value, PostgresqlConfig):
    370              raise ValueError("Value must be a C{PostgresqlConfig} object.")
    371           self._postgresql = value
    372
    373 -   def _getPostgresql(self):
    374        """
    375        Property target used to get the postgresql configuration value.
    376        """
    377        return self._postgresql
    378
    379     postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.")
    380
    381 -   def validate(self):
    382        """
    383        Validates configuration represented by the object.
    384
    385        The compress mode must be filled in.  Then, if the 'all' flag
    386        I{is} set, no databases are allowed, and if the 'all' flag is
    387        I{not} set, at least one database is required.
    388
    389        @raise ValueError: If one of the validations fails.
    390        """
    391        if self.postgresql is None:
    392           raise ValueError("PostgreSQL section is required.")
    393        if self.postgresql.compressMode is None:
    394           raise ValueError("Compress mode value is required.")
    395        if self.postgresql.all:
    396           if self.postgresql.databases is not None and self.postgresql.databases != []:
    397              raise ValueError("Databases cannot be specified if 'all' flag is set.")
    398        else:
    399           if self.postgresql.databases is None or len(self.postgresql.databases) < 1:
    400              raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
    401
    402 -   def addConfig(self, xmlDom, parentNode):
    403        """
    404        Adds a <postgresql> configuration section as the next child of a parent.
    405
    406        Third parties should use this function to write configuration related to
    407        this extension.
    408
    409        We add the following fields to the document::
    410
    411           user           //cb_config/postgresql/user
    412           compressMode   //cb_config/postgresql/compress_mode
    413           all            //cb_config/postgresql/all
    414
    415        We also add groups of the following items, one list element per
    416        item::
    417
    418           database       //cb_config/postgresql/database
    419
    420        @param xmlDom: DOM tree as from C{impl.createDocument()}.
    421        @param parentNode: Parent that the section should be appended to.
    422        """
    423        if self.postgresql is not None:
    424           sectionNode = addContainerNode(xmlDom, parentNode, "postgresql")
    425           addStringNode(xmlDom, sectionNode, "user", self.postgresql.user)
    426           addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode)
    427           addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all)
    428           if self.postgresql.databases is not None:
    429              for database in self.postgresql.databases:
    430                 addStringNode(xmlDom, sectionNode, "database", database)
    431
    432 -   def _parseXmlData(self, xmlData):
    433        """
    434        Internal method to parse an XML string into the object.
    435
    436        This method parses the XML document into a DOM tree (C{xmlDom}) and then
    437        calls a static method to parse the postgresql configuration section.
    438
    439        @param xmlData: XML data to be parsed
    440        @type xmlData: String data
    441
    442        @raise ValueError: If the XML cannot be successfully parsed.
    443        """
    444        (xmlDom, parentNode) = createInputDom(xmlData)
    445        self._postgresql = LocalConfig._parsePostgresql(parentNode)
    446
    447     @staticmethod
    448 -   def _parsePostgresql(parent):
    449        """
    450        Parses a postgresql configuration section.
    451
    452        We read the following fields::
    453
    454           user           //cb_config/postgresql/user
    455           compressMode   //cb_config/postgresql/compress_mode
    456           all            //cb_config/postgresql/all
    457
    458        We also read groups of the following item, one list element per
    459        item::
    460
    461           databases      //cb_config/postgresql/database
    462
    463        @param parent: Parent node to search beneath.
    464
    465        @return: C{PostgresqlConfig} object or C{None} if the section does not exist.
    466        @raise ValueError: If some filled-in value is invalid.
    467        """
    468        postgresql = None
    469        section = readFirstChild(parent, "postgresql")
    470        if section is not None:
    471           postgresql = PostgresqlConfig()
    472           postgresql.user = readString(section, "user")
    473           postgresql.compressMode = readString(section, "compress_mode")
    474           postgresql.all = readBoolean(section, "all")
    475           postgresql.databases = readStringList(section, "database")
    476        return postgresql
    477
    478
    479  ########################################################################
    480  # Public functions
    481  ########################################################################
    482
    483  ###########################
    484  # executeAction() function
    485  ###########################
    486
    487 -def executeAction(configPath, options, config):
    488     """
    489     Executes the PostgreSQL backup action.
    490
    491     @param configPath: Path to configuration file on disk.
    492     @type configPath: String representing a path on disk.
    493
    494     @param options: Program command-line options.
    495     @type options: Options object.
    496
    497     @param config: Program configuration.
    498     @type config: Config object.
    499
    500     @raise ValueError: Under many generic error conditions
    501     @raise IOError: If a backup could not be written for some reason.
    502     """
    503     logger.debug("Executing PostgreSQL extended action.")
    504     if config.options is None or config.collect is None:
    505        raise ValueError("Cedar Backup configuration is not properly filled in.")
    506     local = LocalConfig(xmlPath=configPath)
    507     if local.postgresql.all:
    508        logger.info("Backing up all databases.")
    509        _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
    510                        config.options.backupUser, config.options.backupGroup, None)
    511     if local.postgresql.databases is not None and local.postgresql.databases != []:
    512        logger.debug("Backing up %d individual databases." % len(local.postgresql.databases))
    513        for database in local.postgresql.databases:
    514           logger.info("Backing up database [%s]." % database)
    515           _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
    516                           config.options.backupUser, config.options.backupGroup, database)
    517     logger.info("Executed the PostgreSQL extended action successfully.")
    518
    519 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
    520     """
    521     Backs up an individual PostgreSQL database, or all databases.
    522
    523     This internal method wraps the public method and adds some functionality,
    524     like figuring out a filename, etc.
    525
    526     @param targetDir:  Directory into which backups should be written.
    527     @param compressMode: Compress mode to be used for backed-up files.
    528     @param user: User to use for connecting to the database.
    529     @param backupUser: User to own resulting file.
    530     @param backupGroup: Group to own resulting file.
    531     @param database: Name of database, or C{None} for all databases.
    532
    533     @return: Name of the generated backup file.
    534
    535     @raise ValueError: If some value is missing or invalid.
    536     @raise IOError: If there is a problem executing the PostgreSQL dump.
    537     """
    538     (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
    539     try:
    540        backupDatabase(user, outputFile, database)
    541     finally:
    542        outputFile.close()
    543     if not os.path.exists(filename):
    544        raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
    545     changeOwnership(filename, backupUser, backupGroup)
    546
    547 -def _getOutputFile(targetDir, database, compressMode):
    548     """
    549     Opens the output file used for saving the PostgreSQL dump.
    550
    551     The filename is either C{"postgresqldump.txt"} or
    552     C{"postgresqldump-<database>.txt"}.  The C{".gz"} or C{".bz2"} extension is
    553     added if C{compress} is C{True}.
    554
    555     @param targetDir: Target directory to write file in.
    556     @param database: Name of the database (if any)
    557     @param compressMode: Compress mode to be used for backed-up files.
    558
    559     @return: Tuple of (Output file object, filename)
    560     """
    561     if database is None:
    562        filename = os.path.join(targetDir, "postgresqldump.txt")
    563     else:
    564        filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database)
    565     if compressMode == "gzip":
    566        filename = "%s.gz" % filename
    567        outputFile = GzipFile(filename, "w")
    568     elif compressMode == "bzip2":
    569        filename = "%s.bz2" % filename
    570        outputFile = BZ2File(filename, "w")
    571     else:
    572        outputFile = open(filename, "w")
    573     logger.debug("PostgreSQL dump file will be [%s]." % filename)
    574     return (outputFile, filename)
    575
    576
    577  ############################
    578  # backupDatabase() function
    579  ############################
    580
    581 -def backupDatabase(user, backupFile, database=None):
    582     """
    583     Backs up an individual PostgreSQL database, or all databases.
    584
    585     This function backs up either a named local PostgreSQL database or all local
    586     PostgreSQL databases, using the passed in user for connectivity.
    587     This is I{always} a full backup.  There is no facility for incremental
    588     backups.
    589
    590     The backup data will be written into the passed-in back file.  Normally,
    591     this would be an object as returned from C{open()}, but it is possible to
    592     use something like a C{GzipFile} to write compressed output.  The caller is
    593     responsible for closing the passed-in backup file.
    594
    595     @note: Typically, you would use the C{root} user to back up all databases.
    596
    597     @param user: User to use for connecting to the database.
    598     @type user: String representing PostgreSQL username.
    599
    600     @param backupFile: File use for writing backup.
    601     @type backupFile: Python file object as from C{open()} or C{file()}.
    602
    603     @param database: Name of the database to be backed up.
    604     @type database: String representing database name, or C{None} for all databases.
    605
    606     @raise ValueError: If some value is missing or invalid.
    607     @raise IOError: If there is a problem executing the PostgreSQL dump.
    608     """
    609     args = []
    610     if user is not None:
    611        args.append('-U')
    612        args.append(user)
    613
    614     if database is None:
    615        command = resolveCommand(POSTGRESQLDUMPALL_COMMAND)
    616     else:
    617        command = resolveCommand(POSTGRESQLDUMP_COMMAND)
    618        args.append(database)
    619
    620     result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
    621     if result != 0:
    622        if database is None:
    623           raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result)
    624        else:
    625           raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
    626