CedarBackup2.extend.postgresql

Package CedarBackup2 :: Package extend :: Module postgresql |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup2.extend.postgresql

      1  # -*- coding: iso-8859-1 -*-
      2  # vim: set ft=python ts=3 sw=3 expandtab:
      3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      4  #
      5  #              C E D A R
      6  #          S O L U T I O N S       "Software done right."
      7  #           S O F T W A R E
      8  #
      9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     10  #
     11  # Copyright (c) 2006,2010 Kenneth J. Pronovici.
     12  # Copyright (c) 2006 Antoine Beaupre.
     13  # All rights reserved.
     14  #
     15  # This program is free software; you can redistribute it and/or
     16  # modify it under the terms of the GNU General Public License,
     17  # Version 2, as published by the Free Software Foundation.
     18  #
     19  # This program is distributed in the hope that it will be useful,
     20  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     21  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     22  #
     23  # Copies of the GNU General Public License are available from
     24  # the Free Software Foundation website, http://www.gnu.org/.
     25  #
     26  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     27  #
     28  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
     29  #            Antoine Beaupre <anarcat@koumbit.org>
     30  # Language : Python 2 (>= 2.7)
     31  # Project  : Official Cedar Backup Extensions
     32  # Purpose  : Provides an extension to back up PostgreSQL databases.
     33  #
     34  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     35  # This file was created with a width of 132 characters, and NO tabs.
     36
     37  ########################################################################
     38  # Module documentation
     39  ########################################################################
     40
     41  """
     42  Provides an extension to back up PostgreSQL databases.
     43
     44  This is a Cedar Backup extension used to back up PostgreSQL databases via the
     45  Cedar Backup command line.  It requires a new configurations section
     46  <postgresql> and is intended to be run either immediately before or immediately
     47  after the standard collect action.  Aside from its own configuration, it
     48  requires the options and collect configuration sections in the standard Cedar
     49  Backup configuration file.
     50
     51  The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with
     52  the PostgreSQL product.  Output can be compressed using C{gzip} or C{bzip2}.
     53  Administrators can configure the extension either to back up all databases or
     54  to back up only specific databases.  The extension assumes that the current
     55  user has passwordless access to the database since there is no easy way to pass
     56  a password to the C{pg_dump} client. This can be accomplished using appropriate
     57  voodoo in the C{pg_hda.conf} file.
     58
     59  Note that this code always produces a full backup.  There is currently no
     60  facility for making incremental backups.
     61
     62  You should always make C{/etc/cback.conf} unreadble to non-root users once you
     63  place postgresql configuration into it, since postgresql configuration will
     64  contain information about available PostgreSQL databases and usernames.
     65
     66  Use of this extension I{may} expose usernames in the process listing (via
     67  C{ps}) when the backup is running if the username is specified in the
     68  configuration.
     69
     70  @author: Kenneth J. Pronovici <pronovic@ieee.org>
     71  @author: Antoine Beaupre <anarcat@koumbit.org>
     72  """
     73
     74  ########################################################################
     75  # Imported modules
     76  ########################################################################
     77
     78  # System modules
     79  import os
     80  import logging
     81  from gzip import GzipFile
     82  from bz2 import BZ2File
     83
     84  # Cedar Backup modules
     85  from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
     86  from CedarBackup2.xmlutil import readFirstChild, readString, readStringList, readBoolean
     87  from CedarBackup2.config import VALID_COMPRESS_MODES
     88  from CedarBackup2.util import resolveCommand, executeCommand
     89  from CedarBackup2.util import ObjectTypeList, changeOwnership
     90
     91
     92  ########################################################################
     93  # Module-wide constants and variables
     94  ########################################################################
     95
     96  logger = logging.getLogger("CedarBackup2.log.extend.postgresql")
     97  POSTGRESQLDUMP_COMMAND = [ "pg_dump", ]
     98  POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ]
     99
    100
    101  ########################################################################
    102  # PostgresqlConfig class definition
    103  ########################################################################
    104
    105 -class PostgresqlConfig(object):
    106
    107     """
    108     Class representing PostgreSQL configuration.
    109
    110     The PostgreSQL configuration information is used for backing up PostgreSQL databases.
    111
    112     The following restrictions exist on data in this class:
    113
    114        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
    115        - The 'all' flag must be 'Y' if no databases are defined.
    116        - The 'all' flag must be 'N' if any databases are defined.
    117        - Any values in the databases list must be strings.
    118
    119     @sort: __init__, __repr__, __str__, __cmp__, user, all, databases
    120     """
    121
    122 -   def __init__(self, user=None, compressMode=None, all=None, databases=None):  # pylint: disable=W0622
    123        """
    124        Constructor for the C{PostgresqlConfig} class.
    125
    126        @param user: User to execute backup as.
    127        @param compressMode: Compress mode for backed-up files.
    128        @param all: Indicates whether to back up all databases.
    129        @param databases: List of databases to back up.
    130        """
    131        self._user = None
    132        self._compressMode = None
    133        self._all = None
    134        self._databases = None
    135        self.user = user
    136        self.compressMode = compressMode
    137        self.all = all
    138        self.databases = databases
    139
    140 -   def __repr__(self):
    141        """
    142        Official string representation for class instance.
    143        """
    144        return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
    145
    146 -   def __str__(self):
    147        """
    148        Informal string representation for class instance.
    149        """
    150        return self.__repr__()
    151
    152 -   def __cmp__(self, other):
    153        """
    154        Definition of equals operator for this class.
    155        @param other: Other object to compare to.
    156        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    157        """
    158        if other is None:
    159           return 1
    160        if self.user != other.user:
    161           if self.user < other.user:
    162              return -1
    163           else:
    164              return 1
    165        if self.compressMode != other.compressMode:
    166           if self.compressMode < other.compressMode:
    167              return -1
    168           else:
    169              return 1
    170        if self.all != other.all:
    171           if self.all < other.all:
    172              return -1
    173           else:
    174              return 1
    175        if self.databases != other.databases:
    176           if self.databases < other.databases:
    177              return -1
    178           else:
    179              return 1
    180        return 0
    181
    182 -   def _setUser(self, value):
    183        """
    184        Property target used to set the user value.
    185        """
    186        if value is not None:
    187           if len(value) < 1:
    188              raise ValueError("User must be non-empty string.")
    189        self._user = value
    190
    191 -   def _getUser(self):
    192        """
    193        Property target used to get the user value.
    194        """
    195        return self._user
    196
    197 -   def _setCompressMode(self, value):
    198        """
    199        Property target used to set the compress mode.
    200        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
    201        @raise ValueError: If the value is not valid.
    202        """
    203        if value is not None:
    204           if value not in VALID_COMPRESS_MODES:
    205              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
    206        self._compressMode = value
    207
    208 -   def _getCompressMode(self):
    209        """
    210        Property target used to get the compress mode.
    211        """
    212        return self._compressMode
    213
    214 -   def _setAll(self, value):
    215        """
    216        Property target used to set the 'all' flag.
    217        No validations, but we normalize the value to C{True} or C{False}.
    218        """
    219        if value:
    220           self._all = True
    221        else:
    222           self._all = False
    223
    224 -   def _getAll(self):
    225        """
    226        Property target used to get the 'all' flag.
    227        """
    228        return self._all
    229
    230 -   def _setDatabases(self, value):
    231        """
    232        Property target used to set the databases list.
    233        Either the value must be C{None} or each element must be a string.
    234        @raise ValueError: If the value is not a string.
    235        """
    236        if value is None:
    237           self._databases = None
    238        else:
    239           for database in value:
    240              if len(database) < 1:
    241                 raise ValueError("Each database must be a non-empty string.")
    242           try:
    243              saved = self._databases
    244              self._databases = ObjectTypeList(basestring, "string")
    245              self._databases.extend(value)
    246           except Exception, e:
    247              self._databases = saved
    248              raise e
    249
    250 -   def _getDatabases(self):
    251        """
    252        Property target used to get the databases list.
    253        """
    254        return self._databases
    255
    256     user = property(_getUser, _setUser, None, "User to execute backup as.")
    257     compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
    258     all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
    259     databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
    260
    261
    262  ########################################################################
    263  # LocalConfig class definition
    264  ########################################################################
    265
    266 -class LocalConfig(object):
    267
    268     """
    269     Class representing this extension's configuration document.
    270
    271     This is not a general-purpose configuration object like the main Cedar
    272     Backup configuration object.  Instead, it just knows how to parse and emit
    273     PostgreSQL-specific configuration values.  Third parties who need to read and
    274     write configuration related to this extension should access it through the
    275     constructor, C{validate} and C{addConfig} methods.
    276
    277     @note: Lists within this class are "unordered" for equality comparisons.
    278
    279     @sort: __init__, __repr__, __str__, __cmp__, postgresql, validate, addConfig
    280     """
    281
    282 -   def __init__(self, xmlData=None, xmlPath=None, validate=True):
    283        """
    284        Initializes a configuration object.
    285
    286        If you initialize the object without passing either C{xmlData} or
    287        C{xmlPath} then configuration will be empty and will be invalid until it
    288        is filled in properly.
    289
    290        No reference to the original XML data or original path is saved off by
    291        this class.  Once the data has been parsed (successfully or not) this
    292        original information is discarded.
    293
    294        Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
    295        method will be called (with its default arguments) against configuration
    296        after successfully parsing any passed-in XML.  Keep in mind that even if
    297        C{validate} is C{False}, it might not be possible to parse the passed-in
    298        XML document if lower-level validations fail.
    299
    300        @note: It is strongly suggested that the C{validate} option always be set
    301        to C{True} (the default) unless there is a specific need to read in
    302        invalid configuration from disk.
    303
    304        @param xmlData: XML data representing configuration.
    305        @type xmlData: String data.
    306
    307        @param xmlPath: Path to an XML file on disk.
    308        @type xmlPath: Absolute path to a file on disk.
    309
    310        @param validate: Validate the document after parsing it.
    311        @type validate: Boolean true/false.
    312
    313        @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
    314        @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
    315        @raise ValueError: If the parsed configuration document is not valid.
    316        """
    317        self._postgresql = None
    318        self.postgresql = None
    319        if xmlData is not None and xmlPath is not None:
    320           raise ValueError("Use either xmlData or xmlPath, but not both.")
    321        if xmlData is not None:
    322           self._parseXmlData(xmlData)
    323           if validate:
    324              self.validate()
    325        elif xmlPath is not None:
    326           xmlData = open(xmlPath).read()
    327           self._parseXmlData(xmlData)
    328           if validate:
    329              self.validate()
    330
    331 -   def __repr__(self):
    332        """
    333        Official string representation for class instance.
    334        """
    335        return "LocalConfig(%s)" % (self.postgresql)
    336
    337 -   def __str__(self):
    338        """
    339        Informal string representation for class instance.
    340        """
    341        return self.__repr__()
    342
    343 -   def __cmp__(self, other):
    344        """
    345        Definition of equals operator for this class.
    346        Lists within this class are "unordered" for equality comparisons.
    347        @param other: Other object to compare to.
    348        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    349        """
    350        if other is None:
    351           return 1
    352        if self.postgresql != other.postgresql:
    353           if self.postgresql < other.postgresql:
    354              return -1
    355           else:
    356              return 1
    357        return 0
    358
    359 -   def _setPostgresql(self, value):
    360        """
    361        Property target used to set the postgresql configuration value.
    362        If not C{None}, the value must be a C{PostgresqlConfig} object.
    363        @raise ValueError: If the value is not a C{PostgresqlConfig}
    364        """
    365        if value is None:
    366           self._postgresql = None
    367        else:
    368           if not isinstance(value, PostgresqlConfig):
    369              raise ValueError("Value must be a C{PostgresqlConfig} object.")
    370           self._postgresql = value
    371
    372 -   def _getPostgresql(self):
    373        """
    374        Property target used to get the postgresql configuration value.
    375        """
    376        return self._postgresql
    377
    378     postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.")
    379
    380 -   def validate(self):
    381        """
    382        Validates configuration represented by the object.
    383
    384        The compress mode must be filled in.  Then, if the 'all' flag
    385        I{is} set, no databases are allowed, and if the 'all' flag is
    386        I{not} set, at least one database is required.
    387
    388        @raise ValueError: If one of the validations fails.
    389        """
    390        if self.postgresql is None:
    391           raise ValueError("PostgreSQL section is required.")
    392        if self.postgresql.compressMode is None:
    393           raise ValueError("Compress mode value is required.")
    394        if self.postgresql.all:
    395           if self.postgresql.databases is not None and self.postgresql.databases != []:
    396              raise ValueError("Databases cannot be specified if 'all' flag is set.")
    397        else:
    398           if self.postgresql.databases is None or len(self.postgresql.databases) < 1:
    399              raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
    400
    401 -   def addConfig(self, xmlDom, parentNode):
    402        """
    403        Adds a <postgresql> configuration section as the next child of a parent.
    404
    405        Third parties should use this function to write configuration related to
    406        this extension.
    407
    408        We add the following fields to the document::
    409
    410           user           //cb_config/postgresql/user
    411           compressMode   //cb_config/postgresql/compress_mode
    412           all            //cb_config/postgresql/all
    413
    414        We also add groups of the following items, one list element per
    415        item::
    416
    417           database       //cb_config/postgresql/database
    418
    419        @param xmlDom: DOM tree as from C{impl.createDocument()}.
    420        @param parentNode: Parent that the section should be appended to.
    421        """
    422        if self.postgresql is not None:
    423           sectionNode = addContainerNode(xmlDom, parentNode, "postgresql")
    424           addStringNode(xmlDom, sectionNode, "user", self.postgresql.user)
    425           addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode)
    426           addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all)
    427           if self.postgresql.databases is not None:
    428              for database in self.postgresql.databases:
    429                 addStringNode(xmlDom, sectionNode, "database", database)
    430
    431 -   def _parseXmlData(self, xmlData):
    432        """
    433        Internal method to parse an XML string into the object.
    434
    435        This method parses the XML document into a DOM tree (C{xmlDom}) and then
    436        calls a static method to parse the postgresql configuration section.
    437
    438        @param xmlData: XML data to be parsed
    439        @type xmlData: String data
    440
    441        @raise ValueError: If the XML cannot be successfully parsed.
    442        """
    443        (xmlDom, parentNode) = createInputDom(xmlData)
    444        self._postgresql = LocalConfig._parsePostgresql(parentNode)
    445
    446     @staticmethod
    447 -   def _parsePostgresql(parent):
    448        """
    449        Parses a postgresql configuration section.
    450
    451        We read the following fields::
    452
    453           user           //cb_config/postgresql/user
    454           compressMode   //cb_config/postgresql/compress_mode
    455           all            //cb_config/postgresql/all
    456
    457        We also read groups of the following item, one list element per
    458        item::
    459
    460           databases      //cb_config/postgresql/database
    461
    462        @param parent: Parent node to search beneath.
    463
    464        @return: C{PostgresqlConfig} object or C{None} if the section does not exist.
    465        @raise ValueError: If some filled-in value is invalid.
    466        """
    467        postgresql = None
    468        section = readFirstChild(parent, "postgresql")
    469        if section is not None:
    470           postgresql = PostgresqlConfig()
    471           postgresql.user = readString(section, "user")
    472           postgresql.compressMode = readString(section, "compress_mode")
    473           postgresql.all = readBoolean(section, "all")
    474           postgresql.databases = readStringList(section, "database")
    475        return postgresql
    476
    477
    478  ########################################################################
    479  # Public functions
    480  ########################################################################
    481
    482  ###########################
    483  # executeAction() function
    484  ###########################
    485
    486  # pylint: disable=W0613
    487 -def executeAction(configPath, options, config):
    488     """
    489     Executes the PostgreSQL backup action.
    490
    491     @param configPath: Path to configuration file on disk.
    492     @type configPath: String representing a path on disk.
    493
    494     @param options: Program command-line options.
    495     @type options: Options object.
    496
    497     @param config: Program configuration.
    498     @type config: Config object.
    499
    500     @raise ValueError: Under many generic error conditions
    501     @raise IOError: If a backup could not be written for some reason.
    502     """
    503     logger.debug("Executing PostgreSQL extended action.")
    504     if config.options is None or config.collect is None:
    505        raise ValueError("Cedar Backup configuration is not properly filled in.")
    506     local = LocalConfig(xmlPath=configPath)
    507     if local.postgresql.all:
    508        logger.info("Backing up all databases.")
    509        _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
    510                        config.options.backupUser, config.options.backupGroup, None)
    511     if local.postgresql.databases is not None and local.postgresql.databases != []:
    512        logger.debug("Backing up %d individual databases.", len(local.postgresql.databases))
    513        for database in local.postgresql.databases:
    514           logger.info("Backing up database [%s].", database)
    515           _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
    516                           config.options.backupUser, config.options.backupGroup, database)
    517     logger.info("Executed the PostgreSQL extended action successfully.")
    518
    519 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
    520     """
    521     Backs up an individual PostgreSQL database, or all databases.
    522
    523     This internal method wraps the public method and adds some functionality,
    524     like figuring out a filename, etc.
    525
    526     @param targetDir:  Directory into which backups should be written.
    527     @param compressMode: Compress mode to be used for backed-up files.
    528     @param user: User to use for connecting to the database.
    529     @param backupUser: User to own resulting file.
    530     @param backupGroup: Group to own resulting file.
    531     @param database: Name of database, or C{None} for all databases.
    532
    533     @return: Name of the generated backup file.
    534
    535     @raise ValueError: If some value is missing or invalid.
    536     @raise IOError: If there is a problem executing the PostgreSQL dump.
    537     """
    538     (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
    539     try:
    540        backupDatabase(user, outputFile, database)
    541     finally:
    542        outputFile.close()
    543     if not os.path.exists(filename):
    544        raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
    545     changeOwnership(filename, backupUser, backupGroup)
    546
    547 -def _getOutputFile(targetDir, database, compressMode):
    548     """
    549     Opens the output file used for saving the PostgreSQL dump.
    550
    551     The filename is either C{"postgresqldump.txt"} or
    552     C{"postgresqldump-<database>.txt"}.  The C{".gz"} or C{".bz2"} extension is
    553     added if C{compress} is C{True}.
    554
    555     @param targetDir: Target directory to write file in.
    556     @param database: Name of the database (if any)
    557     @param compressMode: Compress mode to be used for backed-up files.
    558
    559     @return: Tuple of (Output file object, filename)
    560     """
    561     if database is None:
    562        filename = os.path.join(targetDir, "postgresqldump.txt")
    563     else:
    564        filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database)
    565     if compressMode == "gzip":
    566        filename = "%s.gz" % filename
    567        outputFile = GzipFile(filename, "w")
    568     elif compressMode == "bzip2":
    569        filename = "%s.bz2" % filename
    570        outputFile = BZ2File(filename, "w")
    571     else:
    572        outputFile = open(filename, "w")
    573     logger.debug("PostgreSQL dump file will be [%s].", filename)
    574     return (outputFile, filename)
    575
    576
    577  ############################
    578  # backupDatabase() function
    579  ############################
    580
    581 -def backupDatabase(user, backupFile, database=None):
    582     """
    583     Backs up an individual PostgreSQL database, or all databases.
    584
    585     This function backs up either a named local PostgreSQL database or all local
    586     PostgreSQL databases, using the passed in user for connectivity.
    587     This is I{always} a full backup.  There is no facility for incremental
    588     backups.
    589
    590     The backup data will be written into the passed-in back file.  Normally,
    591     this would be an object as returned from C{open()}, but it is possible to
    592     use something like a C{GzipFile} to write compressed output.  The caller is
    593     responsible for closing the passed-in backup file.
    594
    595     @note: Typically, you would use the C{root} user to back up all databases.
    596
    597     @param user: User to use for connecting to the database.
    598     @type user: String representing PostgreSQL username.
    599
    600     @param backupFile: File use for writing backup.
    601     @type backupFile: Python file object as from C{open()} or C{file()}.
    602
    603     @param database: Name of the database to be backed up.
    604     @type database: String representing database name, or C{None} for all databases.
    605
    606     @raise ValueError: If some value is missing or invalid.
    607     @raise IOError: If there is a problem executing the PostgreSQL dump.
    608     """
    609     args = []
    610     if user is not None:
    611        args.append('-U')
    612        args.append(user)
    613
    614     if database is None:
    615        command = resolveCommand(POSTGRESQLDUMPALL_COMMAND)
    616     else:
    617        command = resolveCommand(POSTGRESQLDUMP_COMMAND)
    618        args.append(database)
    619
    620     result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
    621     if result != 0:
    622        if database is None:
    623           raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result)
    624        else:
    625           raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
    626