CedarBackup3.extend.postgresql

Package CedarBackup3 :: Package extend :: Module postgresql |
[hide private]
[frames] | no frames]

# Source Code for Module CedarBackup3.extend.postgresql

      1  # -*- coding: iso-8859-1 -*-
      2  # vim: set ft=python ts=3 sw=3 expandtab:
      3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
      4  #
      5  #              C E D A R
      6  #          S O L U T I O N S       "Software done right."
      7  #           S O F T W A R E
      8  #
      9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     10  #
     11  # Copyright (c) 2006,2010,2015 Kenneth J. Pronovici.
     12  # Copyright (c) 2006 Antoine Beaupre.
     13  # All rights reserved.
     14  #
     15  # This program is free software; you can redistribute it and/or
     16  # modify it under the terms of the GNU General Public License,
     17  # Version 2, as published by the Free Software Foundation.
     18  #
     19  # This program is distributed in the hope that it will be useful,
     20  # but WITHOUT ANY WARRANTY; without even the implied warranty of
     21  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
     22  #
     23  # Copies of the GNU General Public License are available from
     24  # the Free Software Foundation website, http://www.gnu.org/.
     25  #
     26  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     27  #
     28  # Author   : Kenneth J. Pronovici <pronovic@ieee.org>
     29  #            Antoine Beaupre <anarcat@koumbit.org>
     30  # Language : Python 3 (>= 3.4)
     31  # Project  : Official Cedar Backup Extensions
     32  # Purpose  : Provides an extension to back up PostgreSQL databases.
     33  #
     34  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
     35  # This file was created with a width of 132 characters, and NO tabs.
     36
     37  ########################################################################
     38  # Module documentation
     39  ########################################################################
     40
     41  """
     42  Provides an extension to back up PostgreSQL databases.
     43
     44  This is a Cedar Backup extension used to back up PostgreSQL databases via the
     45  Cedar Backup command line.  It requires a new configurations section
     46  <postgresql> and is intended to be run either immediately before or immediately
     47  after the standard collect action.  Aside from its own configuration, it
     48  requires the options and collect configuration sections in the standard Cedar
     49  Backup configuration file.
     50
     51  The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with
     52  the PostgreSQL product.  Output can be compressed using C{gzip} or C{bzip2}.
     53  Administrators can configure the extension either to back up all databases or
     54  to back up only specific databases.  The extension assumes that the current
     55  user has passwordless access to the database since there is no easy way to pass
     56  a password to the C{pg_dump} client. This can be accomplished using appropriate
     57  voodoo in the C{pg_hda.conf} file.
     58
     59  Note that this code always produces a full backup.  There is currently no
     60  facility for making incremental backups.
     61
     62  You should always make C{/etc/cback3.conf} unreadble to non-root users once you
     63  place postgresql configuration into it, since postgresql configuration will
     64  contain information about available PostgreSQL databases and usernames.
     65
     66  Use of this extension I{may} expose usernames in the process listing (via
     67  C{ps}) when the backup is running if the username is specified in the
     68  configuration.
     69
     70  @author: Kenneth J. Pronovici <pronovic@ieee.org>
     71  @author: Antoine Beaupre <anarcat@koumbit.org>
     72  """
     73
     74  ########################################################################
     75  # Imported modules
     76  ########################################################################
     77
     78  # System modules
     79  import os
     80  import logging
     81  from gzip import GzipFile
     82  from bz2 import BZ2File
     83  from functools import total_ordering
     84
     85  # Cedar Backup modules
     86  from CedarBackup3.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
     87  from CedarBackup3.xmlutil import readFirstChild, readString, readStringList, readBoolean
     88  from CedarBackup3.config import VALID_COMPRESS_MODES
     89  from CedarBackup3.util import resolveCommand, executeCommand
     90  from CedarBackup3.util import ObjectTypeList, changeOwnership
     91
     92
     93  ########################################################################
     94  # Module-wide constants and variables
     95  ########################################################################
     96
     97  logger = logging.getLogger("CedarBackup3.log.extend.postgresql")
     98  POSTGRESQLDUMP_COMMAND = [ "pg_dump", ]
     99  POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ]
    100
    101
    102  ########################################################################
    103  # PostgresqlConfig class definition
    104  ########################################################################
    105
    106  @total_ordering
    107 -class PostgresqlConfig(object):
    108
    109     """
    110     Class representing PostgreSQL configuration.
    111
    112     The PostgreSQL configuration information is used for backing up PostgreSQL databases.
    113
    114     The following restrictions exist on data in this class:
    115
    116        - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
    117        - The 'all' flag must be 'Y' if no databases are defined.
    118        - The 'all' flag must be 'N' if any databases are defined.
    119        - Any values in the databases list must be strings.
    120
    121     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__, user,
    122           all, databases
    123     """
    124
    125 -   def __init__(self, user=None, compressMode=None, all=None, databases=None):  # pylint: disable=W0622
    126        """
    127        Constructor for the C{PostgresqlConfig} class.
    128
    129        @param user: User to execute backup as.
    130        @param compressMode: Compress mode for backed-up files.
    131        @param all: Indicates whether to back up all databases.
    132        @param databases: List of databases to back up.
    133        """
    134        self._user = None
    135        self._compressMode = None
    136        self._all = None
    137        self._databases = None
    138        self.user = user
    139        self.compressMode = compressMode
    140        self.all = all
    141        self.databases = databases
    142
    143 -   def __repr__(self):
    144        """
    145        Official string representation for class instance.
    146        """
    147        return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
    148
    149 -   def __str__(self):
    150        """
    151        Informal string representation for class instance.
    152        """
    153        return self.__repr__()
    154
    155 -   def __eq__(self, other):
    156        """Equals operator, iplemented in terms of original Python 2 compare operator."""
    157        return self.__cmp__(other) == 0
    158
    159 -   def __lt__(self, other):
    160        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
    161        return self.__cmp__(other) < 0
    162
    163 -   def __gt__(self, other):
    164        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
    165        return self.__cmp__(other) > 0
    166
    167 -   def __cmp__(self, other):
    168        """
    169        Original Python 2 comparison operator.
    170        @param other: Other object to compare to.
    171        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    172        """
    173        if other is None:
    174           return 1
    175        if self.user != other.user:
    176           if str(self.user or "") < str(other.user or ""):
    177              return -1
    178           else:
    179              return 1
    180        if self.compressMode != other.compressMode:
    181           if str(self.compressMode or "") < str(other.compressMode or ""):
    182              return -1
    183           else:
    184              return 1
    185        if self.all != other.all:
    186           if self.all < other.all:
    187              return -1
    188           else:
    189              return 1
    190        if self.databases != other.databases:
    191           if self.databases < other.databases:
    192              return -1
    193           else:
    194              return 1
    195        return 0
    196
    197 -   def _setUser(self, value):
    198        """
    199        Property target used to set the user value.
    200        """
    201        if value is not None:
    202           if len(value) < 1:
    203              raise ValueError("User must be non-empty string.")
    204        self._user = value
    205
    206 -   def _getUser(self):
    207        """
    208        Property target used to get the user value.
    209        """
    210        return self._user
    211
    212 -   def _setCompressMode(self, value):
    213        """
    214        Property target used to set the compress mode.
    215        If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
    216        @raise ValueError: If the value is not valid.
    217        """
    218        if value is not None:
    219           if value not in VALID_COMPRESS_MODES:
    220              raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
    221        self._compressMode = value
    222
    223 -   def _getCompressMode(self):
    224        """
    225        Property target used to get the compress mode.
    226        """
    227        return self._compressMode
    228
    229 -   def _setAll(self, value):
    230        """
    231        Property target used to set the 'all' flag.
    232        No validations, but we normalize the value to C{True} or C{False}.
    233        """
    234        if value:
    235           self._all = True
    236        else:
    237           self._all = False
    238
    239 -   def _getAll(self):
    240        """
    241        Property target used to get the 'all' flag.
    242        """
    243        return self._all
    244
    245 -   def _setDatabases(self, value):
    246        """
    247        Property target used to set the databases list.
    248        Either the value must be C{None} or each element must be a string.
    249        @raise ValueError: If the value is not a string.
    250        """
    251        if value is None:
    252           self._databases = None
    253        else:
    254           for database in value:
    255              if len(database) < 1:
    256                 raise ValueError("Each database must be a non-empty string.")
    257           try:
    258              saved = self._databases
    259              self._databases = ObjectTypeList(str, "string")
    260              self._databases.extend(value)
    261           except Exception as e:
    262              self._databases = saved
    263              raise e
    264
    265 -   def _getDatabases(self):
    266        """
    267        Property target used to get the databases list.
    268        """
    269        return self._databases
    270
    271     user = property(_getUser, _setUser, None, "User to execute backup as.")
    272     compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
    273     all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
    274     databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
    275
    276
    277  ########################################################################
    278  # LocalConfig class definition
    279  ########################################################################
    280
    281  @total_ordering
    282 -class LocalConfig(object):
    283
    284     """
    285     Class representing this extension's configuration document.
    286
    287     This is not a general-purpose configuration object like the main Cedar
    288     Backup configuration object.  Instead, it just knows how to parse and emit
    289     PostgreSQL-specific configuration values.  Third parties who need to read and
    290     write configuration related to this extension should access it through the
    291     constructor, C{validate} and C{addConfig} methods.
    292
    293     @note: Lists within this class are "unordered" for equality comparisons.
    294
    295     @sort: __init__, __repr__, __str__, __cmp__, __eq__, __lt__, __gt__,
    296           postgresql, validate, addConfig
    297     """
    298
    299 -   def __init__(self, xmlData=None, xmlPath=None, validate=True):
    300        """
    301        Initializes a configuration object.
    302
    303        If you initialize the object without passing either C{xmlData} or
    304        C{xmlPath} then configuration will be empty and will be invalid until it
    305        is filled in properly.
    306
    307        No reference to the original XML data or original path is saved off by
    308        this class.  Once the data has been parsed (successfully or not) this
    309        original information is discarded.
    310
    311        Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
    312        method will be called (with its default arguments) against configuration
    313        after successfully parsing any passed-in XML.  Keep in mind that even if
    314        C{validate} is C{False}, it might not be possible to parse the passed-in
    315        XML document if lower-level validations fail.
    316
    317        @note: It is strongly suggested that the C{validate} option always be set
    318        to C{True} (the default) unless there is a specific need to read in
    319        invalid configuration from disk.
    320
    321        @param xmlData: XML data representing configuration.
    322        @type xmlData: String data.
    323
    324        @param xmlPath: Path to an XML file on disk.
    325        @type xmlPath: Absolute path to a file on disk.
    326
    327        @param validate: Validate the document after parsing it.
    328        @type validate: Boolean true/false.
    329
    330        @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
    331        @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
    332        @raise ValueError: If the parsed configuration document is not valid.
    333        """
    334        self._postgresql = None
    335        self.postgresql = None
    336        if xmlData is not None and xmlPath is not None:
    337           raise ValueError("Use either xmlData or xmlPath, but not both.")
    338        if xmlData is not None:
    339           self._parseXmlData(xmlData)
    340           if validate:
    341              self.validate()
    342        elif xmlPath is not None:
    343           with open(xmlPath) as f:
    344              xmlData = f.read()
    345           self._parseXmlData(xmlData)
    346           if validate:
    347              self.validate()
    348
    349 -   def __repr__(self):
    350        """
    351        Official string representation for class instance.
    352        """
    353        return "LocalConfig(%s)" % (self.postgresql)
    354
    355 -   def __str__(self):
    356        """
    357        Informal string representation for class instance.
    358        """
    359        return self.__repr__()
    360
    361 -   def __eq__(self, other):
    362        """Equals operator, iplemented in terms of original Python 2 compare operator."""
    363        return self.__cmp__(other) == 0
    364
    365 -   def __lt__(self, other):
    366        """Less-than operator, iplemented in terms of original Python 2 compare operator."""
    367        return self.__cmp__(other) < 0
    368
    369 -   def __gt__(self, other):
    370        """Greater-than operator, iplemented in terms of original Python 2 compare operator."""
    371        return self.__cmp__(other) > 0
    372
    373 -   def __cmp__(self, other):
    374        """
    375        Original Python 2 comparison operator.
    376        Lists within this class are "unordered" for equality comparisons.
    377        @param other: Other object to compare to.
    378        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
    379        """
    380        if other is None:
    381           return 1
    382        if self.postgresql != other.postgresql:
    383           if self.postgresql < other.postgresql:
    384              return -1
    385           else:
    386              return 1
    387        return 0
    388
    389 -   def _setPostgresql(self, value):
    390        """
    391        Property target used to set the postgresql configuration value.
    392        If not C{None}, the value must be a C{PostgresqlConfig} object.
    393        @raise ValueError: If the value is not a C{PostgresqlConfig}
    394        """
    395        if value is None:
    396           self._postgresql = None
    397        else:
    398           if not isinstance(value, PostgresqlConfig):
    399              raise ValueError("Value must be a C{PostgresqlConfig} object.")
    400           self._postgresql = value
    401
    402 -   def _getPostgresql(self):
    403        """
    404        Property target used to get the postgresql configuration value.
    405        """
    406        return self._postgresql
    407
    408     postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.")
    409
    410 -   def validate(self):
    411        """
    412        Validates configuration represented by the object.
    413
    414        The compress mode must be filled in.  Then, if the 'all' flag
    415        I{is} set, no databases are allowed, and if the 'all' flag is
    416        I{not} set, at least one database is required.
    417
    418        @raise ValueError: If one of the validations fails.
    419        """
    420        if self.postgresql is None:
    421           raise ValueError("PostgreSQL section is required.")
    422        if self.postgresql.compressMode is None:
    423           raise ValueError("Compress mode value is required.")
    424        if self.postgresql.all:
    425           if self.postgresql.databases is not None and self.postgresql.databases != []:
    426              raise ValueError("Databases cannot be specified if 'all' flag is set.")
    427        else:
    428           if self.postgresql.databases is None or len(self.postgresql.databases) < 1:
    429              raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
    430
    431 -   def addConfig(self, xmlDom, parentNode):
    432        """
    433        Adds a <postgresql> configuration section as the next child of a parent.
    434
    435        Third parties should use this function to write configuration related to
    436        this extension.
    437
    438        We add the following fields to the document::
    439
    440           user           //cb_config/postgresql/user
    441           compressMode   //cb_config/postgresql/compress_mode
    442           all            //cb_config/postgresql/all
    443
    444        We also add groups of the following items, one list element per
    445        item::
    446
    447           database       //cb_config/postgresql/database
    448
    449        @param xmlDom: DOM tree as from C{impl.createDocument()}.
    450        @param parentNode: Parent that the section should be appended to.
    451        """
    452        if self.postgresql is not None:
    453           sectionNode = addContainerNode(xmlDom, parentNode, "postgresql")
    454           addStringNode(xmlDom, sectionNode, "user", self.postgresql.user)
    455           addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode)
    456           addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all)
    457           if self.postgresql.databases is not None:
    458              for database in self.postgresql.databases:
    459                 addStringNode(xmlDom, sectionNode, "database", database)
    460
    461 -   def _parseXmlData(self, xmlData):
    462        """
    463        Internal method to parse an XML string into the object.
    464
    465        This method parses the XML document into a DOM tree (C{xmlDom}) and then
    466        calls a static method to parse the postgresql configuration section.
    467
    468        @param xmlData: XML data to be parsed
    469        @type xmlData: String data
    470
    471        @raise ValueError: If the XML cannot be successfully parsed.
    472        """
    473        (xmlDom, parentNode) = createInputDom(xmlData)
    474        self._postgresql = LocalConfig._parsePostgresql(parentNode)
    475
    476     @staticmethod
    477 -   def _parsePostgresql(parent):
    478        """
    479        Parses a postgresql configuration section.
    480
    481        We read the following fields::
    482
    483           user           //cb_config/postgresql/user
    484           compressMode   //cb_config/postgresql/compress_mode
    485           all            //cb_config/postgresql/all
    486
    487        We also read groups of the following item, one list element per
    488        item::
    489
    490           databases      //cb_config/postgresql/database
    491
    492        @param parent: Parent node to search beneath.
    493
    494        @return: C{PostgresqlConfig} object or C{None} if the section does not exist.
    495        @raise ValueError: If some filled-in value is invalid.
    496        """
    497        postgresql = None
    498        section = readFirstChild(parent, "postgresql")
    499        if section is not None:
    500           postgresql = PostgresqlConfig()
    501           postgresql.user = readString(section, "user")
    502           postgresql.compressMode = readString(section, "compress_mode")
    503           postgresql.all = readBoolean(section, "all")
    504           postgresql.databases = readStringList(section, "database")
    505        return postgresql
    506
    507
    508  ########################################################################
    509  # Public functions
    510  ########################################################################
    511
    512  ###########################
    513  # executeAction() function
    514  ###########################
    515
    516 -def executeAction(configPath, options, config):
    517     """
    518     Executes the PostgreSQL backup action.
    519
    520     @param configPath: Path to configuration file on disk.
    521     @type configPath: String representing a path on disk.
    522
    523     @param options: Program command-line options.
    524     @type options: Options object.
    525
    526     @param config: Program configuration.
    527     @type config: Config object.
    528
    529     @raise ValueError: Under many generic error conditions
    530     @raise IOError: If a backup could not be written for some reason.
    531     """
    532     logger.debug("Executing PostgreSQL extended action.")
    533     if config.options is None or config.collect is None:
    534        raise ValueError("Cedar Backup configuration is not properly filled in.")
    535     local = LocalConfig(xmlPath=configPath)
    536     if local.postgresql.all:
    537        logger.info("Backing up all databases.")
    538        _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
    539                        config.options.backupUser, config.options.backupGroup, None)
    540     if local.postgresql.databases is not None and local.postgresql.databases != []:
    541        logger.debug("Backing up %d individual databases.", len(local.postgresql.databases))
    542        for database in local.postgresql.databases:
    543           logger.info("Backing up database [%s].", database)
    544           _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
    545                           config.options.backupUser, config.options.backupGroup, database)
    546     logger.info("Executed the PostgreSQL extended action successfully.")
    547
    548 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
    549     """
    550     Backs up an individual PostgreSQL database, or all databases.
    551
    552     This internal method wraps the public method and adds some functionality,
    553     like figuring out a filename, etc.
    554
    555     @param targetDir:  Directory into which backups should be written.
    556     @param compressMode: Compress mode to be used for backed-up files.
    557     @param user: User to use for connecting to the database.
    558     @param backupUser: User to own resulting file.
    559     @param backupGroup: Group to own resulting file.
    560     @param database: Name of database, or C{None} for all databases.
    561
    562     @return: Name of the generated backup file.
    563
    564     @raise ValueError: If some value is missing or invalid.
    565     @raise IOError: If there is a problem executing the PostgreSQL dump.
    566     """
    567     (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
    568     with outputFile:
    569        backupDatabase(user, outputFile, database)
    570     if not os.path.exists(filename):
    571        raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
    572     changeOwnership(filename, backupUser, backupGroup)
    573
    574  #pylint: disable=R0204
    575 -def _getOutputFile(targetDir, database, compressMode):
    576     """
    577     Opens the output file used for saving the PostgreSQL dump.
    578
    579     The filename is either C{"postgresqldump.txt"} or
    580     C{"postgresqldump-<database>.txt"}.  The C{".gz"} or C{".bz2"} extension is
    581     added if C{compress} is C{True}.
    582
    583     @param targetDir: Target directory to write file in.
    584     @param database: Name of the database (if any)
    585     @param compressMode: Compress mode to be used for backed-up files.
    586
    587     @return: Tuple of (Output file object, filename), file opened in binary mode for use with executeCommand()
    588     """
    589     if database is None:
    590        filename = os.path.join(targetDir, "postgresqldump.txt")
    591     else:
    592        filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database)
    593     if compressMode == "gzip":
    594        filename = "%s.gz" % filename
    595        outputFile = GzipFile(filename, "wb")
    596     elif compressMode == "bzip2":
    597        filename = "%s.bz2" % filename
    598        outputFile = BZ2File(filename, "wb")
    599     else:
    600        outputFile = open(filename, "wb")
    601     logger.debug("PostgreSQL dump file will be [%s].", filename)
    602     return (outputFile, filename)
    603
    604
    605  ############################
    606  # backupDatabase() function
    607  ############################
    608
    609 -def backupDatabase(user, backupFile, database=None):
    610     """
    611     Backs up an individual PostgreSQL database, or all databases.
    612
    613     This function backs up either a named local PostgreSQL database or all local
    614     PostgreSQL databases, using the passed in user for connectivity.
    615     This is I{always} a full backup.  There is no facility for incremental
    616     backups.
    617
    618     The backup data will be written into the passed-in back file.  Normally,
    619     this would be an object as returned from C{open()}, but it is possible to
    620     use something like a C{GzipFile} to write compressed output.  The caller is
    621     responsible for closing the passed-in backup file.
    622
    623     @note: Typically, you would use the C{root} user to back up all databases.
    624
    625     @param user: User to use for connecting to the database.
    626     @type user: String representing PostgreSQL username.
    627
    628     @param backupFile: File use for writing backup.
    629     @type backupFile: Python file object as from C{open()} or C{file()}.
    630
    631     @param database: Name of the database to be backed up.
    632     @type database: String representing database name, or C{None} for all databases.
    633
    634     @raise ValueError: If some value is missing or invalid.
    635     @raise IOError: If there is a problem executing the PostgreSQL dump.
    636     """
    637     args = []
    638     if user is not None:
    639        args.append('-U')
    640        args.append(user)
    641
    642     if database is None:
    643        command = resolveCommand(POSTGRESQLDUMPALL_COMMAND)
    644     else:
    645        command = resolveCommand(POSTGRESQLDUMP_COMMAND)
    646        args.append(database)
    647
    648     result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
    649     if result != 0:
    650        if database is None:
    651           raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result)
    652        else:
    653           raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
    654