pygccxml.parser.declarations_cache

Package pygccxml :: Package parser :: Module declarations_cache |

# Source Code for Module pygccxml.parser.declarations_cache

      1  #! /usr/bin/python
      2  # Copyright 2004-2008 Roman Yakovenko.
      3  # Distributed under the Boost Software License, Version 1.0. (See
      4  # accompanying file LICENSE_1_0.txt or copy at
      5  # http://www.boost.org/LICENSE_1_0.txt)
      6
      7  import os
      8  import md5
      9  import time
     10  import cPickle
     11  from pygccxml import utils
     12  import config as cxx_parsers_cfg
     13
     14 -def file_signature( filename ):
     15      if not os.path.isfile( filename ):
     16          return None
     17      if not os.path.exists( filename ):
     18          return None
     19      # Extend here to use md5 hash for signature
     20      # - This change allows duplicate autogenerated files to be recognized
     21      #return os.path.getmtime( source )
     22      sig = md5.new()
     23      f = file(filename,'r')
     24      sig.update(f.read())
     25      f.close()
     26      return sig.hexdigest()
     27
     28 -def configuration_signature( config ):
     29      """ Return a signature for a configuration (config_t)
     30          object.  This can then be used as a key in the cache.
     31          This method must take into account anything about
     32          a configuration that could cause the declarations generated
     33          to be different between runs.
     34      """
     35      sig = md5.new()
     36      if isinstance( config, cxx_parsers_cfg.gccxml_configuration_t ):
     37          sig.update(str(config.gccxml_path))
     38      sig.update(str(config.working_directory))
     39      if isinstance( config, cxx_parsers_cfg.gccxml_configuration_t ):
     40          sig.update(str(config.cflags))
     41      for p in config.include_paths:
     42          sig.update(str(p))
     43      for s in config.define_symbols:
     44          sig.update(str(s))
     45      for u in config.undefine_symbols:
     46          sig.update(str(u))
     47      return sig.hexdigest()
     48
     49 -class cache_base_t( object ):
     50      logger = utils.loggers.declarations_cache
     51
     52 -    def __init__( self ):
     53          object.__init__(self)
     54
     55 -    def flush(self):
     56          """ Flush (write out) the cache to disk if needed. """
     57          raise NotImplementedError()
     58
     59 -    def update(self, source_file, configuration, declarations, included_files):
     60          """ Update cache entry.
     61          @param source_file: path to the C++ source file being parsed
     62          @param configuration: configuration used in parsing (config_t)
     63          @param declarations: declaration tree found when parsing
     64          @param included_files: files included by parsing.
     65          """
     66          raise NotImplementedError()
     67
     68 -    def cached_value(self, source_file, configuration):
     69          """ Return declarations we have cached for the source_file and configuration
     70              given.
     71          @param source_file: path to the C++ source file being parsed.
     72          @param configuration: configuration to use for parsing (config_t)
     73          """
     74          raise NotImplementedError()
     75
     76 -class record_t( object ):
     77 -    def __init__( self
     78                    , source_signature
     79                    , config_signature
     80                    , included_files
     81                    , included_files_signature
     82                    , declarations ):
     83          self.__source_signature = source_signature
     84          self.__config_signature = config_signature
     85          self.__included_files = included_files
     86          self.__included_files_signature = included_files_signature
     87          self.__declarations = declarations
     88          self.__was_hit = True # Track if there was a cache hit
     89
     90 -    def _get_was_hit(self):
     91          return self.__was_hit
     92 -    def _set_was_hit(self, was_hit):
     93          self.__was_hit = was_hit
     94      was_hit = property( _get_was_hit, _set_was_hit )
     95
     96 -    def key(self):
     97          return ( self.__source_signature, self.__config_signature)
     98
     99      @staticmethod
    100 -    def create_key( source_file, configuration ):
    101          return ( file_signature(source_file)
    102                   , configuration_signature(configuration))
    103
    104 -    def __source_signature(self):
    105          return self.__source_signature
    106      source_signature = property( __source_signature )
    107
    108 -    def __config_signature(self):
    109          return self.__config_signature
    110      config_signature = property( __config_signature )
    111
    112 -    def __included_files(self):
    113          return self.__included_files
    114      included_files = property( __included_files )
    115
    116 -    def __included_files_signature(self):
    117          return self.__included_files_signature
    118      included_files_signature = property( __included_files_signature )
    119
    120 -    def __declarations(self):
    121          return self.__declarations
    122      declarations = property( __declarations )
    123
    124 -class file_cache_t( cache_base_t ):
    125      """ Cache implementation to store data in a pickled form in a file.
    126          This class contains some cache logic that keeps track of which entries
    127          have been 'hit' in the cache and if an entry has not been hit then
    128          it is deleted at the time of the flush().  This keeps the cache from
    129          growing larger when files change and are not used again.
    130      """
    131
    132 -    def __init__( self, name ):
    133          """
    134          @param name: name of the cache file.
    135          """
    136          cache_base_t.__init__( self )
    137          self.__name = name                              # Name of cache file
    138          self.__cache = self.__load( self.__name )       # Map record_key to record_t
    139          self.__needs_flushed = not bool( self.__cache ) # If empty then we need to flush
    140          for entry in self.__cache.itervalues(): # Clear hit flags
    141              entry.was_hit = False
    142
    143      @staticmethod
    144 -    def __load( file_name ):
    145          " Load pickled cache from file and return the object. "
    146          cache = None
    147          if os.path.exists( file_name ) and not os.path.isfile( file_name ):
    148              raise RuntimeError( 'Cache should be initialized with valid full file name' )
    149          if not os.path.exists( file_name ):
    150              file( file_name, 'w+b' ).close()
    151              return {}
    152          cache_file_obj = file( file_name, 'rb' )
    153          try:
    154              file_cache_t.logger.info( 'Loading cache file "%s".' % file_name )
    155              start_time = time.clock()
    156              cache = cPickle.load( cache_file_obj )
    157              file_cache_t.logger.debug( "Cache file has been loaded in %.1f secs"%( time.clock() - start_time ) )
    158              file_cache_t.logger.debug( "Found cache in file: [%s]  entries: %s"
    159                                 % ( file_name, len( cache.keys() ) ) )
    160          except Exception, error:
    161              file_cache_t.logger.exception( "Error occured while reading cache file: %s", error )
    162              cache_file_obj.close()
    163              file_cache_t.logger.info( "Invalid cache file: [%s]  Regenerating." % file_name )
    164              file(file_name, 'w+b').close()   # Create empty file
    165              cache = {}                       # Empty cache
    166          return cache
    167
    168 -    def flush(self):
    169          # If not marked as needing flushed, then return immediately
    170          if not self.__needs_flushed:
    171              self.logger.debug("Cache did not change, ignoring flush.")
    172              return
    173
    174          # Remove entries that did not get a cache hit
    175          num_removed = 0
    176          for key in self.__cache.keys():
    177              if not self.__cache[key].was_hit:
    178                  num_removed += 1
    179                  del self.__cache[key]
    180          if num_removed > 0:
    181              self.logger.debug(  "There are %s removed entries from cache." % num_removed )
    182          # Save out the cache to disk
    183          cache_file = file( self.__name, 'w+b' )
    184          cPickle.dump( self.__cache, cache_file, cPickle.HIGHEST_PROTOCOL )
    185          cache_file.close()
    186
    187 -    def update(self, source_file, configuration, declarations, included_files):
    188          """ Update a cached record with the current key and value contents. """
    189          record = record_t( source_signature=file_signature(source_file)
    190                             , config_signature=configuration_signature(configuration)
    191                             , included_files=included_files
    192                             , included_files_signature=map( file_signature, included_files)
    193                             , declarations=declarations
    194                          )
    195          # Switched over to holding full record in cache so we don't have
    196          # to keep creating records in the next method.
    197          self.__cache[ record.key() ] = record
    198          self.__cache[ record.key() ].was_hit = True
    199          self.__needs_flushed = True
    200
    201 -    def cached_value(self, source_file, configuration):
    202          """ Attempt to lookup the cached decls for the given file and configuration.
    203              If not found or signature check fails, returns None.
    204          """
    205          key = record_t.create_key(source_file, configuration)
    206          if not self.__cache.has_key( key ):
    207              return None
    208          record = self.__cache[key]
    209          if self.__is_valid_signature( record ):
    210              record.was_hit = True                  # Record cache hit
    211              return record.declarations
    212          else: #some file has been changed
    213              del self.__cache[key]
    214              return None
    215
    216 -    def __is_valid_signature( self, record ):
    217          # This is now part of key
    218          #if self.__signature( record.source_file ) != record.source_file_signature:
    219          #    return False
    220          for index, included_file in enumerate( record.included_files ):
    221              if file_signature( included_file ) != record.included_files_signature[index]:
    222                  return False
    223          return True
    224
    225
    226 -class dummy_cache_t( cache_base_t ):
    227 -    def __init__( self ):
    228          cache_base_t.__init__(self)
    229
    230 -    def flush(self):
    231          pass
    232
    233 -    def update(self, source_file, configuration, declarations, included_files):
    234          pass
    235
    236 -    def cached_value(self, source_file, configuration):
    237          return None
    238