dbf.tables

Package dbf :: Module tables |
[hide private]

# Source Code for Module dbf.tables

       1  "table definitions"
       2  import os
       3  import sys
       4  import csv
       5  import codecs
       6  import locale
       7  import unicodedata
       8  import weakref
       9  from array import array
      10  from bisect import bisect_left, bisect_right
      11  from decimal import Decimal
      12  from shutil import copyfileobj
      13  import dbf
      14  from dbf import _io as io
      15  from dbf.dates import Date, DateTime, Time
      16  from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode, DoNotIndex
      17
      18  input_decoding = locale.getdefaultlocale()[1]    # treat non-unicode data as ...
      19  default_codepage = 'cp1252'  # if no codepage specified on dbf creation, use this
      20  return_ascii = False         # if True -- convert back to icky ascii, losing chars if no mapping
      21
      22  version_map = {
      23          '\x02' : 'FoxBASE',
      24          '\x03' : 'dBase III Plus',
      25          '\x04' : 'dBase IV',
      26          '\x05' : 'dBase V',
      27          '\x30' : 'Visual FoxPro',
      28          '\x31' : 'Visual FoxPro (auto increment field)',
      29          '\x43' : 'dBase IV SQL',
      30          '\x7b' : 'dBase IV w/memos',
      31          '\x83' : 'dBase III Plus w/memos',
      32          '\x8b' : 'dBase IV w/memos',
      33          '\x8e' : 'dBase IV w/SQL table',
      34          '\xf5' : 'FoxPro w/memos'}
      35
      36  code_pages = {
      37          '\x00' : ('ascii', "plain ol' ascii"),
      38          '\x01' : ('cp437', 'U.S. MS-DOS'),
      39          '\x02' : ('cp850', 'International MS-DOS'),
      40          '\x03' : ('cp1252', 'Windows ANSI'),
      41          '\x04' : ('mac_roman', 'Standard Macintosh'),
      42          '\x08' : ('cp865', 'Danish OEM'),
      43          '\x09' : ('cp437', 'Dutch OEM'),
      44          '\x0A' : ('cp850', 'Dutch OEM (secondary)'),
      45          '\x0B' : ('cp437', 'Finnish OEM'),
      46          '\x0D' : ('cp437', 'French OEM'),
      47          '\x0E' : ('cp850', 'French OEM (secondary)'),
      48          '\x0F' : ('cp437', 'German OEM'),
      49          '\x10' : ('cp850', 'German OEM (secondary)'),
      50          '\x11' : ('cp437', 'Italian OEM'),
      51          '\x12' : ('cp850', 'Italian OEM (secondary)'),
      52          '\x13' : ('cp932', 'Japanese Shift-JIS'),
      53          '\x14' : ('cp850', 'Spanish OEM (secondary)'),
      54          '\x15' : ('cp437', 'Swedish OEM'),
      55          '\x16' : ('cp850', 'Swedish OEM (secondary)'),
      56          '\x17' : ('cp865', 'Norwegian OEM'),
      57          '\x18' : ('cp437', 'Spanish OEM'),
      58          '\x19' : ('cp437', 'English OEM (Britain)'),
      59          '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'),
      60          '\x1B' : ('cp437', 'English OEM (U.S.)'),
      61          '\x1C' : ('cp863', 'French OEM (Canada)'),
      62          '\x1D' : ('cp850', 'French OEM (secondary)'),
      63          '\x1F' : ('cp852', 'Czech OEM'),
      64          '\x22' : ('cp852', 'Hungarian OEM'),
      65          '\x23' : ('cp852', 'Polish OEM'),
      66          '\x24' : ('cp860', 'Portugese OEM'),
      67          '\x25' : ('cp850', 'Potugese OEM (secondary)'),
      68          '\x26' : ('cp866', 'Russian OEM'),
      69          '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'),
      70          '\x40' : ('cp852', 'Romanian OEM'),
      71          '\x4D' : ('cp936', 'Chinese GBK (PRC)'),
      72          '\x4E' : ('cp949', 'Korean (ANSI/OEM)'),
      73          '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'),
      74          '\x50' : ('cp874', 'Thai (ANSI/OEM)'),
      75          '\x57' : ('cp1252', 'ANSI'),
      76          '\x58' : ('cp1252', 'Western European ANSI'),
      77          '\x59' : ('cp1252', 'Spanish ANSI'),
      78          '\x64' : ('cp852', 'Eastern European MS-DOS'),
      79          '\x65' : ('cp866', 'Russian MS-DOS'),
      80          '\x66' : ('cp865', 'Nordic MS-DOS'),
      81          '\x67' : ('cp861', 'Icelandic MS-DOS'),
      82          '\x68' : (None, 'Kamenicky (Czech) MS-DOS'),
      83          '\x69' : (None, 'Mazovia (Polish) MS-DOS'),
      84          '\x6a' : ('cp737', 'Greek MS-DOS (437G)'),
      85          '\x6b' : ('cp857', 'Turkish MS-DOS'),
      86          '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'),
      87          '\x79' : ('cp949', 'Korean Windows'),
      88          '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'),
      89          '\x7b' : ('cp932', 'Japanese Windows'),
      90          '\x7c' : ('cp874', 'Thai Windows'),
      91          '\x7d' : ('cp1255', 'Hebrew Windows'),
      92          '\x7e' : ('cp1256', 'Arabic Windows'),
      93          '\xc8' : ('cp1250', 'Eastern European Windows'),
      94          '\xc9' : ('cp1251', 'Russian Windows'),
      95          '\xca' : ('cp1254', 'Turkish Windows'),
      96          '\xcb' : ('cp1253', 'Greek Windows'),
      97          '\x96' : ('mac_cyrillic', 'Russian Macintosh'),
      98          '\x97' : ('mac_latin2', 'Macintosh EE'),
      99          '\x98' : ('mac_greek', 'Greek Macintosh') }
     100
     101  if sys.version_info[:2] < (2, 6):
     102      # define our own property type
     103 -    class property(object):
     104          "Emulate PyProperty_Type() in Objects/descrobject.c"
     105
     106 -        def __init__(self, fget=None, fset=None, fdel=None, doc=None):
     107              self.fget = fget
     108              self.fset = fset
     109              self.fdel = fdel
     110              self.__doc__ = doc or fget.__doc__
     111 -        def __call__(self, func):
     112              self.fget = func
     113              if not self.__doc__:
     114                  self.__doc__ = fget.__doc__
     115 -        def __get__(self, obj, objtype=None):
     116              if obj is None:
     117                  return self
     118              if self.fget is None:
     119                  raise AttributeError, "unreadable attribute"
     120              return self.fget(obj)
     121 -        def __set__(self, obj, value):
     122              if self.fset is None:
     123                  raise AttributeError, "can't set attribute"
     124              self.fset(obj, value)
     125 -        def __delete__(self, obj):
     126              if self.fdel is None:
     127                  raise AttributeError, "can't delete attribute"
     128              self.fdel(obj)
     129 -        def setter(self, func):
     130              self.fset = func
     131              return self
     132 -        def deleter(self, func):
     133              self.fdel = func
     134              return self
     135  # Internal classes
     136 -class _DbfRecord(object):
     137      """Provides routines to extract and save data within the fields of a dbf record."""
     138      __slots__ = ['_recnum', '_layout', '_data', '_dirty', '__weakref__']
     139 -    def _retrieveFieldValue(yo, record_data, fielddef):
     140          """calls appropriate routine to fetch value stored in field from array
     141          @param record_data: the data portion of the record
     142          @type record_data: array of characters
     143          @param fielddef: description of the field definition
     144          @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags'
     145          @returns: python data stored in field"""
     146
     147          field_type = fielddef['type']
     148          retrieve = yo._layout.fieldtypes[field_type]['Retrieve']
     149          datum = retrieve(record_data, fielddef, yo._layout.memo)
     150          if field_type in yo._layout.character_fields:
     151              datum = yo._layout.decoder(datum)[0]
     152              if yo._layout.return_ascii:
     153                  try:
     154                      datum = yo._layout.output_encoder(datum)[0]
     155                  except UnicodeEncodeError:
     156                      datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore')
     157          return datum
     158 -    def _updateFieldValue(yo, fielddef, value):
     159          "calls appropriate routine to convert value to ascii bytes, and save it in record"
     160          field_type = fielddef['type']
     161          update = yo._layout.fieldtypes[field_type]['Update']
     162          if field_type in yo._layout.character_fields:
     163              if not isinstance(value, unicode):
     164                  if yo._layout.input_decoder is None:
     165                      raise NonUnicode("String not in unicode format, no default encoding specified")
     166                  value = yo._layout.input_decoder(value)[0]     # input ascii => unicode
     167              value = yo._layout.encoder(value)[0]           # unicode => table ascii
     168          bytes = array('c', update(value, fielddef, yo._layout.memo))
     169          size = fielddef['length']
     170          if len(bytes) > size:
     171              raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size))
     172          blank = array('c', ' ' * size)
     173          start = fielddef['start']
     174          end = start + size
     175          blank[:len(bytes)] = bytes[:]
     176          yo._data[start:end] = blank[:]
     177          yo._dirty = True
     178 -    def _update_disk(yo, location='', data=None):
     179          if not yo._layout.inmemory:
     180              if yo._recnum < 0:
     181                  raise DbfError("Attempted to update record that has been packed")
     182              if location == '':
     183                  location = yo._recnum * yo._layout.header.record_length + yo._layout.header.start
     184              if data is None:
     185                  data = yo._data
     186              yo._layout.dfd.seek(location)
     187              yo._layout.dfd.write(data)
     188              yo._dirty = False
     189          for index in yo.record_table._indexen:
     190              index(yo)
     191 -    def __call__(yo, *specs):
     192          results = []
     193          if not specs:
     194              specs = yo._layout.index
     195          specs = _normalize_tuples(tuples=specs, length=2, filler=[_nop])
     196          for field, func in specs:
     197              results.append(func(yo[field]))
     198          return tuple(results)
     199
     200 -    def __contains__(yo, key):
     201          return key in yo._layout.fields or key in ['record_number','delete_flag']
     202 -    def __iter__(yo):
     203          return (yo[field] for field in yo._layout.fields)
     204 -    def __getattr__(yo, name):
     205          if name[0:2] == '__' and name[-2:] == '__':
     206              raise AttributeError, 'Method %s is not implemented.' % name
     207          elif name == 'record_number':
     208              return yo._recnum
     209          elif name == 'delete_flag':
     210              return yo._data[0] != ' '
     211          elif not name in yo._layout.fields:
     212              raise FieldMissing(name)
     213          try:
     214              fielddef = yo._layout[name]
     215              value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef)
     216              return value
     217          except DbfError, error:
     218              error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message)
     219              raise
     220 -    def __getitem__(yo, item):
     221          if type(item) in (int, long):
     222              if not -yo._layout.header.field_count <= item < yo._layout.header.field_count:
     223                  raise IndexError("Field offset %d is not in record" % item)
     224              return yo[yo._layout.fields[item]]
     225          elif type(item) == slice:
     226              sequence = []
     227              for index in yo._layout.fields[item]:
     228                  sequence.append(yo[index])
     229              return sequence
     230          elif type(item) == str:
     231              return yo.__getattr__(item)
     232          else:
     233              raise TypeError("%s is not a field name" % item)
     234 -    def __len__(yo):
     235          return yo._layout.header.field_count
     236 -    def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
     237          """record = ascii array of entire record; layout=record specification; memo = memo object for table"""
     238          record = object.__new__(cls)
     239          record._dirty = False
     240          record._recnum = recnum
     241          record._layout = layout
     242          if layout.blankrecord is None and not _fromdisk:
     243              record._createBlankRecord()
     244          record._data = layout.blankrecord
     245          if recnum == -1:                    # not a disk-backed record
     246              return record
     247          elif type(kamikaze) == array:
     248              record._data = kamikaze[:]
     249          elif type(kamikaze) == str:
     250              record._data = array('c', kamikaze)
     251          else:
     252              record._data = kamikaze._data[:]
     253          datalen = len(record._data)
     254          if datalen < layout.header.record_length:
     255              record._data.extend(layout.blankrecord[datalen:])
     256          elif datalen > layout.header.record_length:
     257              record._data = record._data[:layout.header.record_length]
     258          if not _fromdisk and not layout.inmemory:
     259              record._update_disk()
     260          return record
     261 -    def __setattr__(yo, name, value):
     262          if name in yo.__slots__:
     263              object.__setattr__(yo, name, value)
     264              return
     265          elif not name in yo._layout.fields:
     266              raise FieldMissing(name)
     267          fielddef = yo._layout[name]
     268          try:
     269              yo._updateFieldValue(fielddef, value)
     270          except DbfError, error:
     271              error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message)
     272              error.data = name
     273              raise
     274 -    def __setitem__(yo, name, value):
     275          if type(name) == str:
     276              yo.__setattr__(name, value)
     277          elif type(name) in (int, long):
     278              yo.__setattr__(yo._layout.fields[name], value)
     279          elif type(name) == slice:
     280              sequence = []
     281              for field in yo._layout.fields[name]:
     282                  sequence.append(field)
     283              if len(sequence) != len(value):
     284                  raise DbfError("length of slices not equal")
     285              for field, val in zip(sequence, value):
     286                  yo[field] = val
     287          else:
     288              raise TypeError("%s is not a field name" % name)
     289 -    def __str__(yo):
     290          result = []
     291          for seq, field in enumerate(yo.field_names):
     292              result.append("%3d - %-10s: %s" % (seq, field, yo[field]))
     293          return '\n'.join(result)
     294 -    def __repr__(yo):
     295          return yo._data.tostring()
     296 -    def _createBlankRecord(yo):
     297          "creates a blank record data chunk"
     298          layout = yo._layout
     299          ondisk = layout.ondisk
     300          layout.ondisk = False
     301          yo._data = array('c', ' ' * layout.header.record_length)
     302          layout.memofields = []
     303          for field in layout.fields:
     304              yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']())
     305              if layout[field]['type'] in layout.memotypes:
     306                  layout.memofields.append(field)
     307          layout.blankrecord = yo._data[:]
     308          layout.ondisk = ondisk
     309 -    def delete_record(yo):
     310          "marks record as deleted"
     311          yo._data[0] = '*'
     312          yo._dirty = True
     313          return yo
     314      @property
     315 -    def field_names(yo):
     316          "fields in table/record"
     317          return yo._layout.fields[:]
     318 -    def gather_fields(yo, dictionary, drop=False):        # dict, drop_missing=False):
     319          "saves a dictionary into a record's fields\nkeys with no matching field will raise a FieldMissing exception unless drop_missing = True"
     320          old_data = yo._data[:]
     321          try:
     322              for key in dictionary:
     323                  if not key in yo.field_names:
     324                      if drop:
     325                          continue
     326                      raise FieldMissing(key)
     327                  yo.__setattr__(key, dictionary[key])
     328          except:
     329              yo._data[:] = old_data
     330              raise
     331          return yo
     332      @property
     333 -    def has_been_deleted(yo):
     334          "marked for deletion?"
     335          return yo._data[0] == '*'
     336 -    def read_record(yo):
     337          "refresh record data from disk"
     338          size = yo._layout.header.record_length
     339          location = yo._recnum * size + yo._layout.header.start
     340          yo._layout.dfd.seek(location)
     341          yo._data[:] = yo._meta.dfd.read(size)
     342          yo._dirty = False
     343          return yo
     344      @property
     345 -    def record_number(yo):
     346          "physical record number"
     347          return yo._recnum
     348      @property
     349 -    def record_table(yo):
     350          table = yo._layout.table()
     351          if table is None:
     352              raise DbfError("table is no longer available")
     353          return table
     354 -    def check_index(yo):
     355          for dbfindex in yo._layout.table()._indexen:
     356              dbfindex(yo)
     357 -    def reset_record(yo, keep_fields=None):
     358          "blanks record"
     359          if keep_fields is None:
     360              keep_fields = []
     361          keep = {}
     362          for field in keep_fields:
     363              keep[field] = yo[field]
     364          if yo._layout.blankrecord == None:
     365              yo._createBlankRecord()
     366          yo._data[:] = yo._layout.blankrecord[:]
     367          for field in keep_fields:
     368              yo[field] = keep[field]
     369          yo._dirty = True
     370          return yo
     371 -    def scatter_fields(yo, blank=False):
     372          "returns a dictionary of fieldnames and values which can be used with gather_fields().  if blank is True, values are empty."
     373          keys = yo._layout.fields
     374          if blank:
     375              values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys]
     376          else:
     377              values = [yo[field] for field in keys]
     378          return dict(zip(keys, values))
     379 -    def undelete_record(yo):
     380          "marks record as active"
     381          yo._data[0] = ' '
     382          yo._dirty = True
     383          return yo
     384 -    def write_record(yo, **kwargs):
     385          "write record data to disk"
     386          if kwargs:
     387              yo.gather_fields(kwargs)
     388          if yo._dirty:
     389              yo._update_disk()
     390              return 1
     391          return 0
     392 -class _DbfMemo(object):
     393      """Provides access to memo fields as dictionaries
     394         must override _init, _get_memo, and _put_memo to
     395         store memo contents to disk"""
     396 -    def _init(yo):
     397          "initialize disk file usage"
     398 -    def _get_memo(yo, block):
     399          "retrieve memo contents from disk"
     400 -    def _put_memo(yo, data):
     401          "store memo contents to disk"
     402 -    def __init__(yo, meta):
     403          ""
     404          yo.meta = meta
     405          yo.memory = {}
     406          yo.nextmemo = 1
     407          yo._init()
     408          yo.meta.newmemofile = False
     409 -    def get_memo(yo, block, field):
     410          "gets the memo in block"
     411          if yo.meta.ignorememos or not block:
     412              return ''
     413          if yo.meta.ondisk:
     414              return yo._get_memo(block)
     415          else:
     416              return yo.memory[block]
     417 -    def put_memo(yo, data):
     418          "stores data in memo file, returns block number"
     419          if yo.meta.ignorememos or data == '':
     420              return 0
     421          if yo.meta.inmemory:
     422              thismemo = yo.nextmemo
     423              yo.nextmemo += 1
     424              yo.memory[thismemo] = data
     425          else:
     426              thismemo = yo._put_memo(data)
     427          return thismemo
     428 -class _Db3Memo(_DbfMemo):
     429 -    def _init(yo):
     430          "dBase III specific"
     431          yo.meta.memo_size= 512
     432          yo.record_header_length = 2
     433          if yo.meta.ondisk and not yo.meta.ignorememos:
     434              if yo.meta.newmemofile:
     435                  yo.meta.mfd = open(yo.meta.memoname, 'w+b')
     436                  yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508)
     437              else:
     438                  try:
     439                      yo.meta.mfd = open(yo.meta.memoname, 'r+b')
     440                      yo.meta.mfd.seek(0)
     441                      yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4))
     442                  except:
     443                      raise DbfError("memo file appears to be corrupt")
     444 -    def _get_memo(yo, block):
     445          block = int(block)
     446          yo.meta.mfd.seek(block * yo.meta.memo_size)
     447          eom = -1
     448          data = ''
     449          while eom == -1:
     450              newdata = yo.meta.mfd.read(yo.meta.memo_size)
     451              if not newdata:
     452                  return data
     453              data += newdata
     454              eom = data.find('\x1a\x1a')
     455          return data[:eom].rstrip()
     456 -    def _put_memo(yo, data):
     457          data = data.rstrip()
     458          length = len(data) + yo.record_header_length  # room for two ^Z at end of memo
     459          blocks = length // yo.meta.memo_size
     460          if length % yo.meta.memo_size:
     461              blocks += 1
     462          thismemo = yo.nextmemo
     463          yo.nextmemo = thismemo + blocks
     464          yo.meta.mfd.seek(0)
     465          yo.meta.mfd.write(io.packLongInt(yo.nextmemo))
     466          yo.meta.mfd.seek(thismemo * yo.meta.memo_size)
     467          yo.meta.mfd.write(data)
     468          yo.meta.mfd.write('\x1a\x1a')
     469          double_check = yo._get_memo(thismemo)
     470          if len(double_check) != len(data):
     471              uhoh = open('dbf_memo_dump.err','wb')
     472              uhoh.write('thismemo: %d' % thismemo)
     473              uhoh.write('nextmemo: %d' % yo.nextmemo)
     474              uhoh.write('saved: %d bytes' % len(data))
     475              uhoh.write(data)
     476              uhoh.write('retrieved: %d bytes' % len(double_check))
     477              uhoh.write(double_check)
     478              uhoh.close()
     479              raise DbfError("unknown error: memo not saved")
     480          return thismemo
     481 -class _VfpMemo(_DbfMemo):
     482 -    def _init(yo):
     483          "Visual Foxpro 6 specific"
     484          if yo.meta.ondisk and not yo.meta.ignorememos:
     485              yo.record_header_length = 8
     486              if yo.meta.newmemofile:
     487                  if yo.meta.memo_size == 0:
     488                      yo.meta.memo_size = 1
     489                  elif 1 < yo.meta.memo_size < 33:
     490                      yo.meta.memo_size *= 512
     491                  yo.meta.mfd = open(yo.meta.memoname, 'w+b')
     492                  nextmemo = 512 // yo.meta.memo_size
     493                  if nextmemo * yo.meta.memo_size < 512:
     494                      nextmemo += 1
     495                  yo.nextmemo = nextmemo
     496                  yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \
     497                          io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504)
     498              else:
     499                  try:
     500                      yo.meta.mfd = open(yo.meta.memoname, 'r+b')
     501                      yo.meta.mfd.seek(0)
     502                      header = yo.meta.mfd.read(512)
     503                      yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True)
     504                      yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True)
     505                  except:
     506                      raise DbfError("memo file appears to be corrupt")
     507 -    def _get_memo(yo, block):
     508          yo.meta.mfd.seek(block * yo.meta.memo_size)
     509          header = yo.meta.mfd.read(8)
     510          length = io.unpackLongInt(header[4:], bigendian=True)
     511          return yo.meta.mfd.read(length)
     512 -    def _put_memo(yo, data):
     513          data = data.rstrip()     # no trailing whitespace
     514          yo.meta.mfd.seek(0)
     515          thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True)
     516          yo.meta.mfd.seek(0)
     517          length = len(data) + yo.record_header_length  # room for two ^Z at end of memo
     518          blocks = length // yo.meta.memo_size
     519          if length % yo.meta.memo_size:
     520              blocks += 1
     521          yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True))
     522          yo.meta.mfd.seek(thismemo*yo.meta.memo_size)
     523          yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data)
     524          return thismemo
     525  # Public classes
     526 -class DbfTable(object):
     527      """Provides a framework for dbf style tables."""
     528      _version = 'basic memory table'
     529      _versionabbv = 'dbf'
     530      _fieldtypes = {
     531              'D' : { 'Type':'Date',    'Init':io.addDate,    'Blank':Date.today, 'Retrieve':io.retrieveDate,    'Update':io.updateDate, },
     532              'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool,       'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, },
     533              'M' : { 'Type':'Memo',    'Init':io.addMemo,    'Blank':str,        'Retrieve':io.retrieveMemo,    'Update':io.updateMemo, } }
     534      _memoext = ''
     535      _memotypes = tuple('M', )
     536      _memoClass = _DbfMemo
     537      _yesMemoMask = ''
     538      _noMemoMask = ''
     539      _fixed_fields = ('M','D','L')           # always same length in table
     540      _variable_fields = tuple()              # variable length in table
     541      _character_fields = tuple('M', )        # field representing character data
     542      _decimal_fields = tuple()               # text-based numeric fields
     543      _numeric_fields = tuple()               # fields representing a number
     544      _dbfTableHeader = array('c', '\x00' * 32)
     545      _dbfTableHeader[0] = '\x00'             # table type - none
     546      _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
     547      _dbfTableHeader[10] = '\x01'            # record length -- one for delete flag
     548      _dbfTableHeader[29] = '\x00'            # code page -- none, using plain ascii
     549      _dbfTableHeader = _dbfTableHeader.tostring()
     550      _dbfTableHeaderExtra = ''
     551      _supported_tables = []
     552      _read_only = False
     553      _meta_only = False
     554      _use_deleted = True
     555      backup = False
     556 -    class _DbfLists(object):
     557          "implements the weakref structure for DbfLists"
     558 -        def __init__(yo):
     559              yo._lists = set()
     560 -        def __iter__(yo):
     561              yo._lists = set([s for s in yo._lists if s() is not None])
     562              return (s() for s in yo._lists if s() is not None)
     563 -        def __len__(yo):
     564              yo._lists = set([s for s in yo._lists if s() is not None])
     565              return len(yo._lists)
     566 -        def add(yo, new_list):
     567              yo._lists.add(weakref.ref(new_list))
     568              yo._lists = set([s for s in yo._lists if s() is not None])
     569 -    class _Indexen(object):
     570          "implements the weakref structure for seperate indexes"
     571 -        def __init__(yo):
     572              yo._indexen = set()
     573 -        def __iter__(yo):
     574              yo._indexen = set([s for s in yo._indexen if s() is not None])
     575              return (s() for s in yo._indexen if s() is not None)
     576 -        def __len__(yo):
     577              yo._indexen = set([s for s in yo._indexen if s() is not None])
     578              return len(yo._indexen)
     579 -        def add(yo, new_list):
     580              yo._indexen.add(weakref.ref(new_list))
     581              yo._indexen = set([s for s in yo._indexen if s() is not None])
     582 -    class _MetaData(dict):
     583          blankrecord = None
     584          fields = None
     585          filename = None
     586          dfd = None
     587          memoname = None
     588          newmemofile = False
     589          memo = None
     590          mfd = None
     591          ignorememos = False
     592          memofields = None
     593          current = -1
     594 -    class _TableHeader(object):
     595 -        def __init__(yo, data):
     596              if len(data) != 32:
     597                  raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data))
     598              yo._data = array('c', data + '\x0d')
     599 -        def codepage(yo, cp=None):
     600              "get/set code page of table"
     601              if cp is None:
     602                  return yo._data[29]
     603              else:
     604                  cp, sd, ld = _codepage_lookup(cp)
     605                  yo._data[29] = cp
     606                  return cp
     607          @property
     608 -        def data(yo):
     609              "main data structure"
     610              date = io.packDate(Date.today())
     611              yo._data[1:4] = array('c', date)
     612              return yo._data.tostring()
     613          @data.setter
     614 -        def data(yo, bytes):
     615              if len(bytes) < 32:
     616                  raise DbfError("length for data of %d is less than 32" % len(bytes))
     617              yo._data[:] = array('c', bytes)
     618          @property
     619 -        def extra(yo):
     620              "extra dbf info (located after headers, before data records)"
     621              fieldblock = yo._data[32:]
     622              for i in range(len(fieldblock)//32+1):
     623                  cr = i * 32
     624                  if fieldblock[cr] == '\x0d':
     625                      break
     626              else:
     627                  raise DbfError("corrupt field structure")
     628              cr += 33    # skip past CR
     629              return yo._data[cr:].tostring()
     630          @extra.setter
     631 -        def extra(yo, data):
     632              fieldblock = yo._data[32:]
     633              for i in range(len(fieldblock)//32+1):
     634                  cr = i * 32
     635                  if fieldblock[cr] == '\x0d':
     636                      break
     637              else:
     638                  raise DbfError("corrupt field structure")
     639              cr += 33    # skip past CR
     640              yo._data[cr:] = array('c', data)                             # extra
     641              yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))  # start
     642          @property
     643 -        def field_count(yo):
     644              "number of fields (read-only)"
     645              fieldblock = yo._data[32:]
     646              for i in range(len(fieldblock)//32+1):
     647                  cr = i * 32
     648                  if fieldblock[cr] == '\x0d':
     649                      break
     650              else:
     651                  raise DbfError("corrupt field structure")
     652              return len(fieldblock[:cr]) // 32
     653          @property
     654 -        def fields(yo):
     655              "field block structure"
     656              fieldblock = yo._data[32:]
     657              for i in range(len(fieldblock)//32+1):
     658                  cr = i * 32
     659                  if fieldblock[cr] == '\x0d':
     660                      break
     661              else:
     662                  raise DbfError("corrupt field structure")
     663              return fieldblock[:cr].tostring()
     664          @fields.setter
     665 -        def fields(yo, block):
     666              fieldblock = yo._data[32:]
     667              for i in range(len(fieldblock)//32+1):
     668                  cr = i * 32
     669                  if fieldblock[cr] == '\x0d':
     670                      break
     671              else:
     672                  raise DbfError("corrupt field structure")
     673              cr += 32    # convert to indexing main structure
     674              fieldlen = len(block)
     675              if fieldlen % 32 != 0:
     676                  raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen)
     677              yo._data[32:cr] = array('c', block)                           # fields
     678              yo._data[8:10] = array('c', io.packShortInt(len(yo._data)))   # start
     679              fieldlen = fieldlen // 32
     680              recordlen = 1                                     # deleted flag
     681              for i in range(fieldlen):
     682                  recordlen += ord(block[i*32+16])
     683              yo._data[10:12] = array('c', io.packShortInt(recordlen))
     684          @property
     685 -        def record_count(yo):
     686              "number of records (maximum 16,777,215)"
     687              return io.unpackLongInt(yo._data[4:8].tostring())
     688          @record_count.setter
     689 -        def record_count(yo, count):
     690              yo._data[4:8] = array('c', io.packLongInt(count))
     691          @property
     692 -        def record_length(yo):
     693              "length of a record (read_only) (max of 65,535)"
     694              return io.unpackShortInt(yo._data[10:12].tostring())
     695          @property
     696 -        def start(yo):
     697              "starting position of first record in file (must be within first 64K)"
     698              return io.unpackShortInt(yo._data[8:10].tostring())
     699          @start.setter
     700 -        def start(yo, pos):
     701              yo._data[8:10] = array('c', io.packShortInt(pos))
     702          @property
     703 -        def update(yo):
     704              "date of last table modification (read-only)"
     705              return io.unpackDate(yo._data[1:4].tostring())
     706          @property
     707 -        def version(yo):
     708              "dbf version"
     709              return yo._data[0]
     710          @version.setter
     711 -        def version(yo, ver):
     712              yo._data[0] = ver
     713 -    class _Table(object):
     714          "implements the weakref table for records"
     715 -        def __init__(yo, count, meta):
     716              yo._meta = meta
     717              yo._weakref_list = [weakref.ref(lambda x: None)] * count
     718 -        def __getitem__(yo, index):
     719              maybe = yo._weakref_list[index]()
     720              if maybe is None:
     721                  if index < 0:
     722                      index += yo._meta.header.record_count
     723                  size = yo._meta.header.record_length
     724                  location = index * size + yo._meta.header.start
     725                  yo._meta.dfd.seek(location)
     726                  if yo._meta.dfd.tell() != location:
     727                      raise ValueError("unable to seek to offset %d in file" % location)
     728                  bytes = yo._meta.dfd.read(size)
     729                  if not bytes:
     730                      raise ValueError("unable to read record data from %s at location %d" % (yo._meta.filename, location))
     731                  maybe = _DbfRecord(recnum=index, layout=yo._meta, kamikaze=bytes, _fromdisk=True)
     732                  yo._weakref_list[index] = weakref.ref(maybe)
     733              return maybe
     734 -        def append(yo, record):
     735              yo._weakref_list.append(weakref.ref(record))
     736 -        def clear(yo):
     737              yo._weakref_list[:] = []
     738 -    class DbfIterator(object):
     739          "returns records using current index"
     740 -        def __init__(yo, table):
     741              yo._table = table
     742              yo._index = -1
     743              yo._more_records = True
     744 -        def __iter__(yo):
     745              return yo
     746 -        def next(yo):
     747              while yo._more_records:
     748                  yo._index += 1
     749                  if yo._index >= len(yo._table):
     750                      yo._more_records = False
     751                      continue
     752                  record = yo._table[yo._index]
     753                  if not yo._table.use_deleted and record.has_been_deleted:
     754                      continue
     755                  return record
     756              else:
     757                  raise StopIteration
     758 -    def _buildHeaderFields(yo):
     759          "constructs fieldblock for disk table"
     760          fieldblock = array('c', '')
     761          memo = False
     762          yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask))
     763          for field in yo._meta.fields:
     764              if yo._meta.fields.count(field) > 1:
     765                  raise DbfError("corrupted field structure (noticed in _buildHeaderFields)")
     766              fielddef = array('c', '\x00' * 32)
     767              fielddef[:11] = array('c', io.packStr(field))
     768              fielddef[11] = yo._meta[field]['type']
     769              fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start']))
     770              fielddef[16] = chr(yo._meta[field]['length'])
     771              fielddef[17] = chr(yo._meta[field]['decimals'])
     772              fielddef[18] = chr(yo._meta[field]['flags'])
     773              fieldblock.extend(fielddef)
     774              if yo._meta[field]['type'] in yo._meta.memotypes:
     775                  memo = True
     776          yo._meta.header.fields = fieldblock.tostring()
     777          if memo:
     778              yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask))
     779              if yo._meta.memo is None:
     780                  yo._meta.memo = yo._memoClass(yo._meta)
     781 -    def _checkMemoIntegrity(yo):
     782          "dBase III specific"
     783          if yo._meta.header.version == '\x83':
     784              try:
     785                  yo._meta.memo = yo._memoClass(yo._meta)
     786              except:
     787                  yo._meta.dfd.close()
     788                  yo._meta.dfd = None
     789                  raise
     790          if not yo._meta.ignorememos:
     791              for field in yo._meta.fields:
     792                  if yo._meta[field]['type'] in yo._memotypes:
     793                      if yo._meta.header.version != '\x83':
     794                          yo._meta.dfd.close()
     795                          yo._meta.dfd = None
     796                          raise DbfError("Table structure corrupt:  memo fields exist, header declares no memos")
     797                      elif not os.path.exists(yo._meta.memoname):
     798                          yo._meta.dfd.close()
     799                          yo._meta.dfd = None
     800                          raise DbfError("Table structure corrupt:  memo fields exist without memo file")
     801                      break
     802 -    def _initializeFields(yo):
     803          "builds the FieldList of names, types, and descriptions from the disk file"
     804          yo._meta.fields[:] = []
     805          offset = 1
     806          fieldsdef = yo._meta.header.fields
     807          if len(fieldsdef) % 32 != 0:
     808              raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
     809          if len(fieldsdef) // 32 != yo.field_count:
     810              raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
     811          for i in range(yo.field_count):
     812              fieldblock = fieldsdef[i*32:(i+1)*32]
     813              name = io.unpackStr(fieldblock[:11])
     814              type = fieldblock[11]
     815              if not type in yo._meta.fieldtypes:
     816                  raise DbfError("Unknown field type: %s" % type)
     817              start = offset
     818              length = ord(fieldblock[16])
     819              offset += length
     820              end = start + length
     821              decimals = ord(fieldblock[17])
     822              flags = ord(fieldblock[18])
     823              if name in yo._meta.fields:
     824                  raise DbfError('Duplicate field name found: %s' % name)
     825              yo._meta.fields.append(name)
     826              yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
     827 -    def _fieldLayout(yo, i):
     828          "Returns field information Name Type(Length[,Decimals])"
     829          name = yo._meta.fields[i]
     830          type = yo._meta[name]['type']
     831          length = yo._meta[name]['length']
     832          decimals = yo._meta[name]['decimals']
     833          if type in yo._decimal_fields:
     834              description = "%s %s(%d,%d)" % (name, type, length, decimals)
     835          elif type in yo._fixed_fields:
     836              description = "%s %s" % (name, type)
     837          else:
     838              description = "%s %s(%d)" % (name, type, length)
     839          return description
     840 -    def _loadtable(yo):
     841          "loads the records from disk to memory"
     842          if yo._meta_only:
     843              raise DbfError("%s has been closed, records are unavailable" % yo.filename)
     844          dfd = yo._meta.dfd
     845          header = yo._meta.header
     846          dfd.seek(header.start)
     847          allrecords = dfd.read()                     # kludge to get around mysterious errno 0 problems
     848          dfd.seek(0)
     849          length = header.record_length
     850          for i in range(header.record_count):
     851              record_data = allrecords[length*i:length*i+length]
     852              yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True))
     853          dfd.seek(0)
     854 -    def _list_fields(yo, specs, sep=','):
     855          if specs is None:
     856              specs = yo.field_names
     857          elif isinstance(specs, str):
     858              specs = specs.split(sep)
     859          else:
     860              specs = list(specs)
     861          specs = [s.strip() for s in specs]
     862          return specs
     863 -    def _update_disk(yo, headeronly=False):
     864          "synchronizes the disk file with current data"
     865          if yo._meta.inmemory:
     866              return
     867          fd = yo._meta.dfd
     868          fd.seek(0)
     869          fd.write(yo._meta.header.data)
     870          if not headeronly:
     871              for record in yo._table:
     872                  record._update_disk()
     873                  fd.flush()
     874                  fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length)
     875          if 'db3' in yo._versionabbv:
     876              fd.seek(0, os.SEEK_END)
     877              fd.write('\x1a')        # required for dBase III
     878              fd.flush()
     879              fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length + 1)
     880
     881 -    def __contains__(yo, key):
     882          return key in yo.field_names
     883 -    def __enter__(yo):
     884          return yo
     885 -    def __exit__(yo, *exc_info):
     886          yo.close()
     887 -    def __getattr__(yo, name):
     888          if name in ('_table'):
     889                  if yo._meta.ondisk:
     890                      yo._table = yo._Table(len(yo), yo._meta)
     891                  else:
     892                      yo._table = []
     893                      yo._loadtable()
     894          return object.__getattribute__(yo, name)
     895 -    def __getitem__(yo, value):
     896          if type(value) == int:
     897              if not -yo._meta.header.record_count <= value < yo._meta.header.record_count:
     898                  raise IndexError("Record %d is not in table." % value)
     899              return yo._table[value]
     900          elif type(value) == slice:
     901              sequence = List(desc='%s -->  %s' % (yo.filename, value), field_names=yo.field_names)
     902              yo._dbflists.add(sequence)
     903              for index in range(len(yo))[value]:
     904                  record = yo._table[index]
     905                  if yo.use_deleted is True or not record.has_been_deleted:
     906                      sequence.append(record)
     907              return sequence
     908          else:
     909              raise TypeError('type <%s> not valid for indexing' % type(value))
     910 -    def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False,
     911                   read_only=False, keep_memos=False, meta_only=False, codepage=None):
     912          """open/create dbf file
     913          filename should include path if needed
     914          field_specs can be either a ;-delimited string or a list of strings
     915          memo_size is always 512 for db3 memos
     916          ignore_memos is useful if the memo file is missing or corrupt
     917          read_only will load records into memory, then close the disk file
     918          keep_memos will also load any memo fields into memory
     919          meta_only will ignore all records, keeping only basic table information
     920          codepage will override whatever is set in the table itself"""
     921          if filename[0] == filename[-1] == ':':
     922              if field_specs is None:
     923                  raise DbfError("field list must be specified for memory tables")
     924          elif type(yo) is DbfTable:
     925              raise DbfError("only memory tables supported")
     926          yo._dbflists = yo._DbfLists()
     927          yo._indexen = yo._Indexen()
     928          yo._meta = meta = yo._MetaData()
     929          meta.table = weakref.ref(yo)
     930          meta.filename = filename
     931          meta.fields = []
     932          meta.fieldtypes = yo._fieldtypes
     933          meta.fixed_fields = yo._fixed_fields
     934          meta.variable_fields = yo._variable_fields
     935          meta.character_fields = yo._character_fields
     936          meta.decimal_fields = yo._decimal_fields
     937          meta.numeric_fields = yo._numeric_fields
     938          meta.memotypes = yo._memotypes
     939          meta.ignorememos = ignore_memos
     940          meta.memo_size = memo_size
     941          meta.input_decoder = codecs.getdecoder(input_decoding)      # from ascii to unicode
     942          meta.output_encoder = codecs.getencoder(input_decoding)     # and back to ascii
     943          meta.return_ascii = return_ascii
     944          meta.header = header = yo._TableHeader(yo._dbfTableHeader)
     945          header.extra = yo._dbfTableHeaderExtra
     946          header.data        #force update of date
     947          if filename[0] == filename[-1] == ':':
     948              yo._table = []
     949              meta.ondisk = False
     950              meta.inmemory = True
     951              meta.memoname = filename
     952          else:
     953              base, ext = os.path.splitext(filename)
     954              if ext == '':
     955                  meta.filename =  base + '.dbf'
     956              meta.memoname = base + yo._memoext
     957              meta.ondisk = True
     958              meta.inmemory = False
     959          if field_specs:
     960              if meta.ondisk:
     961                  meta.dfd = open(meta.filename, 'w+b')
     962                  meta.newmemofile = True
     963              yo.add_fields(field_specs)
     964              header.codepage(codepage or default_codepage)
     965              cp, sd, ld = _codepage_lookup(meta.header.codepage())
     966              meta.decoder = codecs.getdecoder(sd)
     967              meta.encoder = codecs.getencoder(sd)
     968              return
     969          try:
     970              dfd = meta.dfd = open(meta.filename, 'r+b')
     971          except IOError, e:
     972              raise DbfError(str(e))
     973          dfd.seek(0)
     974          meta.header = header = yo._TableHeader(dfd.read(32))
     975          if not header.version in yo._supported_tables:
     976              dfd.close()
     977              dfd = None
     978              raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version)))
     979          cp, sd, ld = _codepage_lookup(meta.header.codepage())
     980          yo._meta.decoder = codecs.getdecoder(sd)
     981          yo._meta.encoder = codecs.getencoder(sd)
     982          fieldblock = dfd.read(header.start - 32)
     983          for i in range(len(fieldblock)//32+1):
     984              fieldend = i * 32
     985              if fieldblock[fieldend] == '\x0d':
     986                  break
     987          else:
     988              raise DbfError("corrupt field structure in header")
     989          if len(fieldblock[:fieldend]) % 32 != 0:
     990              raise DbfError("corrupt field structure in header")
     991          header.fields = fieldblock[:fieldend]
     992          header.extra = fieldblock[fieldend+1:]  # skip trailing \r
     993          yo._initializeFields()
     994          yo._checkMemoIntegrity()
     995          meta.current = -1
     996          if len(yo) > 0:
     997              meta.current = 0
     998          dfd.seek(0)
     999          if meta_only:
    1000              yo.close(keep_table=False, keep_memos=False)
    1001          elif read_only:
    1002              yo.close(keep_table=True, keep_memos=keep_memos)
    1003          if codepage is not None:
    1004              cp, sd, ld = _codepage_lookup(codepage)
    1005              yo._meta.decoder = codecs.getdecoder(sd)
    1006              yo._meta.encoder = codecs.getencoder(sd)
    1007
    1008 -    def __iter__(yo):
    1009          return yo.DbfIterator(yo)
    1010 -    def __len__(yo):
    1011          return yo._meta.header.record_count
    1012 -    def __nonzero__(yo):
    1013          return yo._meta.header.record_count != 0
    1014 -    def __repr__(yo):
    1015          if yo._read_only:
    1016              return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename
    1017          elif yo._meta_only:
    1018              return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename
    1019          else:
    1020              return __name__ + ".Table('%s')" % yo._meta.filename
    1021 -    def __str__(yo):
    1022          if yo._read_only:
    1023              status = "read-only"
    1024          elif yo._meta_only:
    1025              status = "meta-only"
    1026          else:
    1027              status = "read/write"
    1028          str =  """
    1029          Table:         %s
    1030          Type:          %s
    1031          Codepage:      %s
    1032          Status:        %s
    1033          Last updated:  %s
    1034          Record count:  %d
    1035          Field count:   %d
    1036          Record length: %d """ % (yo.filename, version_map.get(yo._meta.header.version,
    1037              'unknown - ' + hex(ord(yo._meta.header.version))), yo.codepage, status,
    1038              yo.last_update, len(yo), yo.field_count, yo.record_length)
    1039          str += "\n        --Fields--\n"
    1040          for i in range(len(yo._meta.fields)):
    1041              str += "%11d) %s\n" % (i, yo._fieldLayout(i))
    1042          return str
    1043      @property
    1044 -    def codepage(yo):
    1045          return "%s (%s)" % code_pages[yo._meta.header.codepage()]
    1046      @codepage.setter
    1047 -    def codepage(yo, cp):
    1048          cp = code_pages[yo._meta.header.codepage(cp)][0]
    1049          yo._meta.decoder = codecs.getdecoder(cp)
    1050          yo._meta.encoder = codecs.getencoder(cp)
    1051          yo._update_disk(headeronly=True)
    1052      @property
    1053 -    def field_count(yo):
    1054          "the number of fields in the table"
    1055          return yo._meta.header.field_count
    1056      @property
    1057 -    def field_names(yo):
    1058          "a list of the fields in the table"
    1059          return yo._meta.fields[:]
    1060      @property
    1061 -    def filename(yo):
    1062          "table's file name, including path (if specified on open)"
    1063          return yo._meta.filename
    1064      @property
    1065 -    def last_update(yo):
    1066          "date of last update"
    1067          return yo._meta.header.update
    1068      @property
    1069 -    def memoname(yo):
    1070          "table's memo name (if path included in filename on open)"
    1071          return yo._meta.memoname
    1072      @property
    1073 -    def record_length(yo):
    1074          "number of bytes in a record"
    1075          return yo._meta.header.record_length
    1076      @property
    1077 -    def record_number(yo):
    1078          "index number of the current record"
    1079          return yo._meta.current
    1080      @property
    1081 -    def supported_tables(yo):
    1082          "allowable table types"
    1083          return yo._supported_tables
    1084      @property
    1085 -    def use_deleted(yo):
    1086          "process or ignore deleted records"
    1087          return yo._use_deleted
    1088      @use_deleted.setter
    1089 -    def use_deleted(yo, new_setting):
    1090          yo._use_deleted = new_setting
    1091      @property
    1092 -    def version(yo):
    1093          "returns the dbf type of the table"
    1094          return yo._version
    1095 -    def add_fields(yo, field_specs):
    1096          """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]]
    1097          backup table is created with _backup appended to name
    1098          then modifies current structure"""
    1099          all_records = [record for record in yo]
    1100          if yo:
    1101              yo.create_backup()
    1102          yo._meta.blankrecord = None
    1103          meta = yo._meta
    1104          offset = meta.header.record_length
    1105          fields = yo._list_fields(field_specs, sep=';')
    1106          for field in fields:
    1107              try:
    1108                  name, format = field.split()
    1109                  if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum():
    1110                      raise DbfError("%s invalid:  field names must start with a letter, and can only contain letters, digits, and _" % name)
    1111                  name = name.lower()
    1112                  if name in meta.fields:
    1113                      raise DbfError("Field '%s' already exists" % name)
    1114                  field_type = format[0].upper()
    1115                  if len(name) > 10:
    1116                      raise DbfError("Maximum field name length is 10.  '%s' is %d characters long." % (name, len(name)))
    1117                  if not field_type in meta.fieldtypes.keys():
    1118                      raise DbfError("Unknown field type:  %s" % field_type)
    1119                  length, decimals = yo._meta.fieldtypes[field_type]['Init'](format)
    1120              except ValueError:
    1121                  raise DbfError("invalid field specifier: %s" % field)
    1122              start = offset
    1123              end = offset + length
    1124              offset = end
    1125              meta.fields.append(name)
    1126              meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0}
    1127              if meta[name]['type'] in yo._memotypes and meta.memo is None:
    1128                  meta.memo = yo._memoClass(meta)
    1129              for record in yo:
    1130                  record[name] = meta.fieldtypes[field_type]['Blank']()
    1131          yo._buildHeaderFields()
    1132          yo._update_disk()
    1133 -    def append(yo, kamikaze='', drop=False, multiple=1):
    1134          "adds <multiple> blank records, and fills fields with dict/tuple values if present"
    1135          if not yo.field_count:
    1136              raise DbfError("No fields defined, cannot append")
    1137          empty_table = len(yo) == 0
    1138          dictdata = False
    1139          tupledata = False
    1140          if not isinstance(kamikaze, _DbfRecord):
    1141              if isinstance(kamikaze, dict):
    1142                  dictdata = kamikaze
    1143                  kamikaze = ''
    1144              elif isinstance(kamikaze, tuple):
    1145                  tupledata = kamikaze
    1146                  kamikaze = ''
    1147          newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze)
    1148          yo._table.append(newrecord)
    1149          yo._meta.header.record_count += 1
    1150          if dictdata:
    1151              newrecord.gather_fields(dictdata, drop=drop)
    1152          elif tupledata:
    1153              for index, item in enumerate(tupledata):
    1154                  newrecord[index] = item
    1155          elif kamikaze == str:
    1156              for field in yo._meta.memofields:
    1157                  newrecord[field] = ''
    1158          elif kamikaze:
    1159              for field in yo._meta.memofields:
    1160                  newrecord[field] = kamikaze[field]
    1161          newrecord.write_record()
    1162          multiple -= 1
    1163          if multiple:
    1164              data = newrecord._data
    1165              single = yo._meta.header.record_count
    1166              total = single + multiple
    1167              while single < total:
    1168                  multi_record = _DbfRecord(single, yo._meta, kamikaze=data)
    1169                  yo._table.append(multi_record)
    1170                  for field in yo._meta.memofields:
    1171                      multi_record[field] = newrecord[field]
    1172                  single += 1
    1173                  multi_record.write_record()
    1174              yo._meta.header.record_count = total   # += multiple
    1175              yo._meta.current = yo._meta.header.record_count - 1
    1176              newrecord = multi_record
    1177          yo._update_disk(headeronly=True)
    1178          if empty_table:
    1179              yo._meta.current = 0
    1180          return newrecord
    1181 -    def bof(yo, _move=False):
    1182          "moves record pointer to previous usable record; returns True if no more usable records"
    1183          current = yo._meta.current
    1184          try:
    1185              while yo._meta.current > 0:
    1186                  yo._meta.current -= 1
    1187                  if yo.use_deleted or not yo.current().has_been_deleted:
    1188                      break
    1189              else:
    1190                  yo._meta.current = -1
    1191                  return True
    1192              return False
    1193          finally:
    1194              if not _move:
    1195                  yo._meta.current = current
    1196 -    def bottom(yo, get_record=False):
    1197          """sets record pointer to bottom of table
    1198          if get_record, seeks to and returns last (non-deleted) record
    1199          DbfError if table is empty
    1200          Bof if all records deleted and use_deleted is False"""
    1201          yo._meta.current = yo._meta.header.record_count
    1202          if get_record:
    1203              try:
    1204                  return yo.prev()
    1205              except Bof:
    1206                  yo._meta.current = yo._meta.header.record_count
    1207                  raise Eof()
    1208 -    def close(yo, keep_table=False, keep_memos=False):
    1209          """closes disk files
    1210          ensures table data is available if keep_table
    1211          ensures memo data is available if keep_memos"""
    1212          yo._meta.inmemory = True
    1213          if keep_table:
    1214              replacement_table = []
    1215              for record in yo._table:
    1216                  replacement_table.append(record)
    1217              yo._table = replacement_table
    1218          else:
    1219              if yo._meta.ondisk:
    1220                  yo._meta_only = True
    1221          if yo._meta.mfd is not None:
    1222              if not keep_memos:
    1223                  yo._meta.ignorememos = True
    1224              else:
    1225                  memo_fields = []
    1226                  for field in yo.field_names:
    1227                      if yo.is_memotype(field):
    1228                          memo_fields.append(field)
    1229                  for record in yo:
    1230                      for field in memo_fields:
    1231                          record[field] = record[field]
    1232              yo._meta.mfd.close()
    1233              yo._meta.mfd = None
    1234          if yo._meta.ondisk:
    1235              yo._meta.dfd.close()
    1236              yo._meta.dfd = None
    1237          if keep_table:
    1238              yo._read_only = True
    1239          yo._meta.ondisk = False
    1240 -    def create_backup(yo, new_name=None, overwrite=False):
    1241          "creates a backup table -- ignored if memory table"
    1242          if yo.filename[0] == yo.filename[-1] == ':':
    1243              return
    1244          if new_name is None:
    1245              new_name = os.path.splitext(yo.filename)[0] + '_backup.dbf'
    1246          else:
    1247              overwrite = True
    1248          if overwrite or not yo.backup:
    1249              bkup = open(new_name, 'wb')
    1250              try:
    1251                  yo._meta.dfd.seek(0)
    1252                  copyfileobj(yo._meta.dfd, bkup)
    1253                  yo.backup = new_name
    1254              finally:
    1255                  bkup.close()
    1256 -    def create_index(yo, key):
    1257          return Index(yo, key)
    1258 -    def current(yo, index=False):
    1259          "returns current logical record, or its index"
    1260          if yo._meta.current < 0:
    1261              raise Bof()
    1262          elif yo._meta.current >= yo._meta.header.record_count:
    1263              raise Eof()
    1264          if index:
    1265              return yo._meta.current
    1266          return yo._table[yo._meta.current]
    1267 -    def delete_fields(yo, doomed):
    1268          """removes field(s) from the table
    1269          creates backup files with _backup appended to the file name,
    1270          then modifies current structure"""
    1271          doomed = yo._list_fields(doomed)
    1272          for victim in doomed:
    1273              if victim not in yo._meta.fields:
    1274                  raise DbfError("field %s not in table -- delete aborted" % victim)
    1275          all_records = [record for record in yo]
    1276          yo.create_backup()
    1277          for victim in doomed:
    1278              yo._meta.fields.pop(yo._meta.fields.index(victim))
    1279              start = yo._meta[victim]['start']
    1280              end = yo._meta[victim]['end']
    1281              for record in yo:
    1282                  record._data = record._data[:start] + record._data[end:]
    1283              for field in yo._meta.fields:
    1284                  if yo._meta[field]['start'] == end:
    1285                      end = yo._meta[field]['end']
    1286                      yo._meta[field]['start'] = start
    1287                      yo._meta[field]['end'] = start + yo._meta[field]['length']
    1288                      start = yo._meta[field]['end']
    1289              yo._buildHeaderFields()
    1290          yo._update_disk()
    1291 -    def eof(yo, _move=False):
    1292          "moves record pointer to next usable record; returns True if no more usable records"
    1293          current = yo._meta.current
    1294          try:
    1295              while yo._meta.current < yo._meta.header.record_count - 1:
    1296                  yo._meta.current += 1
    1297                  if yo.use_deleted or not yo.current().has_been_deleted:
    1298                      break
    1299              else:
    1300                  yo._meta.current = yo._meta.header.record_count
    1301                  return True
    1302              return False
    1303          finally:
    1304              if not _move:
    1305                  yo._meta.current = current
    1306 -    def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
    1307          """writes the table using CSV or tab-delimited format, using the filename
    1308          given if specified, otherwise the table name"""
    1309          if filename is not None:
    1310              path, filename = os.path.split(filename)
    1311          else:
    1312              path, filename = os.path.split(yo.filename)
    1313          filename = os.path.join(path, filename)
    1314          field_specs = yo._list_fields(field_specs)
    1315          if records is None:
    1316              records = yo
    1317          format = format.lower()
    1318          if format not in ('csv', 'tab', 'fixed'):
    1319              raise DbfError("export format: csv, tab, or fixed -- not %s" % format)
    1320          if format == 'fixed':
    1321              format = 'txt'
    1322          base, ext = os.path.splitext(filename)
    1323          if ext.lower() in ('', '.dbf'):
    1324              filename = base + "." + format[:3]
    1325          fd = open(filename, 'w')
    1326          try:
    1327              if format == 'csv':
    1328                  csvfile = csv.writer(fd, dialect='dbf')
    1329                  if header:
    1330                      csvfile.writerow(field_specs)
    1331                  for record in records:
    1332                      fields = []
    1333                      for fieldname in field_specs:
    1334                          fields.append(record[fieldname])
    1335                      csvfile.writerow(fields)
    1336              elif format == 'tab':
    1337                  if header:
    1338                      fd.write('\t'.join(field_specs) + '\n')
    1339                  for record in records:
    1340                      fields = []
    1341                      for fieldname in field_specs:
    1342                          fields.append(str(record[fieldname]))
    1343                      fd.write('\t'.join(fields) + '\n')
    1344              else: # format == 'fixed'
    1345                  header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w')
    1346                  header.write("%-15s  Size\n" % "Field Name")
    1347                  header.write("%-15s  ----\n" % ("-" * 15))
    1348                  sizes = []
    1349                  for field in field_specs:
    1350                      size = yo.size(field)[0]
    1351                      sizes.append(size)
    1352                      header.write("%-15s  %3d\n" % (field, size))
    1353                  header.write('\nTotal Records in file: %d\n' % len(records))
    1354                  header.close()
    1355                  for record in records:
    1356                      fields = []
    1357                      for i, field_name in enumerate(field_specs):
    1358                          fields.append("%-*s" % (sizes[i], record[field_name]))
    1359                      fd.write(''.join(fields) + '\n')
    1360          finally:
    1361              fd.close()
    1362              fd = None
    1363          return len(records)
    1364 -    def get_record(yo, recno):
    1365          "returns record at physical_index[recno]"
    1366          return yo._table[recno]
    1367 -    def goto(yo, criteria):
    1368          """changes the record pointer to the first matching (non-deleted) record
    1369          criteria should be either a tuple of tuple(value, field, func) triples,
    1370          or an integer to go to"""
    1371          if isinstance(criteria, int):
    1372              if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count:
    1373                  raise IndexError("Record %d does not exist" % criteria)
    1374              if criteria < 0:
    1375                  criteria += yo._meta.header.record_count
    1376              yo._meta.current = criteria
    1377              return yo.current()
    1378          criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop])
    1379          specs = tuple([(field, func) for value, field, func in criteria])
    1380          match = tuple([value for value, field, func in criteria])
    1381          current = yo.current(index=True)
    1382          matchlen = len(match)
    1383          while not yo.Eof():
    1384              record = yo.current()
    1385              results = record(*specs)
    1386              if results == match:
    1387                  return record
    1388          return yo.goto(current)
    1389 -    def is_decimal(yo, name):
    1390          "returns True if name is a variable-length field type"
    1391          return yo._meta[name]['type'] in yo._decimal_fields
    1392 -    def is_memotype(yo, name):
    1393          "returns True if name is a memo type field"
    1394          return yo._meta[name]['type'] in yo._memotypes
    1395 -    def new(yo, filename, field_specs=None, codepage=None):
    1396          "returns a new table of the same type"
    1397          if field_specs is None:
    1398              field_specs = yo.structure()
    1399          if not (filename[0] == filename[-1] == ':'):
    1400              path, name = os.path.split(filename)
    1401              if path == "":
    1402                  filename = os.path.join(os.path.split(yo.filename)[0], filename)
    1403              elif name == "":
    1404                  filename = os.path.join(path, os.path.split(yo.filename)[1])
    1405          if codepage is None:
    1406              codepage = yo._meta.header.codepage()[0]
    1407          return yo.__class__(filename, field_specs, codepage=codepage)
    1408 -    def next(yo):
    1409          "set record pointer to next (non-deleted) record, and return it"
    1410          if yo.eof(_move=True):
    1411              raise Eof()
    1412          return yo.current()
    1413 -    def open(yo):
    1414          meta = yo._meta
    1415          meta.inmemory = False
    1416          meta.ondisk = True
    1417          yo._read_only = False
    1418          yo._meta_only = False
    1419          if '_table' in dir(yo):
    1420              del yo._table
    1421          dfd = meta.dfd = open(meta.filename, 'r+b')
    1422          dfd.seek(0)
    1423          meta.header = header = yo._TableHeader(dfd.read(32))
    1424          if not header.version in yo._supported_tables:
    1425              dfd.close()
    1426              dfd = None
    1427              raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version)))
    1428          cp, sd, ld = _codepage_lookup(meta.header.codepage())
    1429          meta.decoder = codecs.getdecoder(sd)
    1430          meta.encoder = codecs.getencoder(sd)
    1431          fieldblock = dfd.read(header.start - 32)
    1432          for i in range(len(fieldblock)//32+1):
    1433              fieldend = i * 32
    1434              if fieldblock[fieldend] == '\x0d':
    1435                  break
    1436          else:
    1437              raise DbfError("corrupt field structure in header")
    1438          if len(fieldblock[:fieldend]) % 32 != 0:
    1439              raise DbfError("corrupt field structure in header")
    1440          header.fields = fieldblock[:fieldend]
    1441          header.extra = fieldblock[fieldend+1:]  # skip trailing \r
    1442          yo._initializeFields()
    1443          yo._checkMemoIntegrity()
    1444          meta.current = -1
    1445          if len(yo) > 0:
    1446              meta.current = 0
    1447          dfd.seek(0)
    1448
    1449 -    def pack(yo, _pack=True):
    1450          "physically removes all deleted records"
    1451          for dbfindex in yo._indexen:
    1452              dbfindex.clear()
    1453          newtable = []
    1454          index = 0
    1455          offset = 0 # +1 for each purged record
    1456          for record in yo._table:
    1457              found = False
    1458              if record.has_been_deleted and _pack:
    1459                  for dbflist in yo._dbflists:
    1460                      if dbflist._purge(record, record.record_number - offset, 1):
    1461                          found = True
    1462                  record._recnum = -1
    1463              else:
    1464                  record._recnum = index
    1465                  newtable.append(record)
    1466                  index += 1
    1467              if found:
    1468                  offset += 1
    1469                  found = False
    1470          yo._table.clear()
    1471          for record in newtable:
    1472              yo._table.append(record)
    1473          yo._meta.header.record_count = index
    1474          yo._current = -1
    1475          yo._update_disk()
    1476          yo.reindex()
    1477 -    def prev(yo):
    1478          "set record pointer to previous (non-deleted) record, and return it"
    1479          if yo.bof(_move=True):
    1480              raise Bof
    1481          return yo.current()
    1482 -    def query(yo, sql_command=None, python=None):
    1483          "uses exec to perform queries on the table"
    1484          if sql_command:
    1485              return sql(yo, sql_command)
    1486          elif python is None:
    1487              raise DbfError("query: python parameter must be specified")
    1488          possible = List(desc="%s -->  %s" % (yo.filename, python), field_names=yo.field_names)
    1489          yo._dbflists.add(possible)
    1490          query_result = {}
    1491          select = 'query_result["keep"] = %s' % python
    1492          g = {}
    1493          use_deleted = yo.use_deleted
    1494          for record in yo:
    1495              query_result['keep'] = False
    1496              g['query_result'] = query_result
    1497              exec select in g, record
    1498              if query_result['keep']:
    1499                  possible.append(record)
    1500              record.write_record()
    1501          return possible
    1502 -    def reindex(yo):
    1503          for dbfindex in yo._indexen:
    1504              dbfindex.reindex()
    1505 -    def rename_field(yo, oldname, newname):
    1506          "renames an existing field"
    1507          if yo:
    1508              yo.create_backup()
    1509          if not oldname in yo._meta.fields:
    1510              raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname)
    1511          if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum():
    1512              raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits")
    1513          newname = newname.lower()
    1514          if newname in yo._meta.fields:
    1515              raise DbfError("field --%s-- already exists" % newname)
    1516          if len(newname) > 10:
    1517              raise DbfError("maximum field name length is 10.  '%s' is %d characters long." % (newname, len(newname)))
    1518          yo._meta[newname] = yo._meta[oldname]
    1519          yo._meta.fields[yo._meta.fields.index(oldname)] = newname
    1520          yo._buildHeaderFields()
    1521          yo._update_disk(headeronly=True)
    1522 -    def size(yo, field):
    1523          "returns size of field as a tuple of (length, decimals)"
    1524          if field in yo:
    1525              return (yo._meta[field]['length'], yo._meta[field]['decimals'])
    1526          raise DbfError("%s is not a field in %s" % (field, yo.filename))
    1527 -    def structure(yo, fields=None):
    1528          """return list of fields suitable for creating same table layout
    1529          @param fields: list of fields or None for all fields"""
    1530          field_specs = []
    1531          fields = yo._list_fields(fields)
    1532          try:
    1533              for name in fields:
    1534                  field_specs.append(yo._fieldLayout(yo.field_names.index(name)))
    1535          except ValueError:
    1536              raise DbfError("field --%s-- does not exist" % name)
    1537          return field_specs
    1538 -    def top(yo, get_record=False):
    1539          """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record
    1540          DbfError if table is empty
    1541          Eof if all records are deleted and use_deleted is False"""
    1542          yo._meta.current = -1
    1543          if get_record:
    1544              try:
    1545                  return yo.next()
    1546              except Eof:
    1547                  yo._meta.current = -1
    1548                  raise Bof()
    1549 -    def type(yo, field):
    1550          "returns type of field"
    1551          if field in yo:
    1552              return yo._meta[field]['type']
    1553          raise DbfError("%s is not a field in %s" % (field, yo.filename))
    1554 -    def zap(yo, areyousure=False):
    1555          """removes all records from table -- this cannot be undone!
    1556          areyousure must be True, else error is raised"""
    1557          if areyousure:
    1558              if yo._meta.inmemory:
    1559                  yo._table = []
    1560              else:
    1561                  yo._table.clear()
    1562              yo._meta.header.record_count = 0
    1563              yo._current = -1
    1564              yo._update_disk()
    1565          else:
    1566              raise DbfError("You must say you are sure to wipe the table")
    1567 -class Db3Table(DbfTable):
    1568      """Provides an interface for working with dBase III tables."""
    1569      _version = 'dBase III Plus'
    1570      _versionabbv = 'db3'
    1571      _fieldtypes = {
    1572              'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
    1573              'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
    1574              'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
    1575              'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
    1576              'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric} }
    1577      _memoext = '.dbt'
    1578      _memotypes = ('M',)
    1579      _memoClass = _Db3Memo
    1580      _yesMemoMask = '\x80'
    1581      _noMemoMask = '\x7f'
    1582      _fixed_fields = ('D','L','M')
    1583      _variable_fields = ('C','N')
    1584      _character_fields = ('C','M')
    1585      _decimal_fields = ('N',)
    1586      _numeric_fields = ('N',)
    1587      _dbfTableHeader = array('c', '\x00' * 32)
    1588      _dbfTableHeader[0] = '\x03'         # version - dBase III w/o memo's
    1589      _dbfTableHeader[8:10] = array('c', io.packShortInt(33))
    1590      _dbfTableHeader[10] = '\x01'        # record length -- one for delete flag
    1591      _dbfTableHeader[29] = '\x03'        # code page -- 437 US-MS DOS
    1592      _dbfTableHeader = _dbfTableHeader.tostring()
    1593      _dbfTableHeaderExtra = ''
    1594      _supported_tables = ['\x03', '\x83']
    1595      _read_only = False
    1596      _meta_only = False
    1597      _use_deleted = True
    1598 -    def _checkMemoIntegrity(yo):
    1599          "dBase III specific"
    1600          if yo._meta.header.version == '\x83':
    1601              try:
    1602                  yo._meta.memo = yo._memoClass(yo._meta)
    1603              except:
    1604                  yo._meta.dfd.close()
    1605                  yo._meta.dfd = None
    1606                  raise
    1607          if not yo._meta.ignorememos:
    1608              for field in yo._meta.fields:
    1609                  if yo._meta[field]['type'] in yo._memotypes:
    1610                      if yo._meta.header.version != '\x83':
    1611                          yo._meta.dfd.close()
    1612                          yo._meta.dfd = None
    1613                          raise DbfError("Table structure corrupt:  memo fields exist, header declares no memos")
    1614                      elif not os.path.exists(yo._meta.memoname):
    1615                          yo._meta.dfd.close()
    1616                          yo._meta.dfd = None
    1617                          raise DbfError("Table structure corrupt:  memo fields exist without memo file")
    1618                      break
    1619 -    def _initializeFields(yo):
    1620          "builds the FieldList of names, types, and descriptions"
    1621          yo._meta.fields[:] = []
    1622          offset = 1
    1623          fieldsdef = yo._meta.header.fields
    1624          if len(fieldsdef) % 32 != 0:
    1625              raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
    1626          if len(fieldsdef) // 32 != yo.field_count:
    1627              raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
    1628          for i in range(yo.field_count):
    1629              fieldblock = fieldsdef[i*32:(i+1)*32]
    1630              name = io.unpackStr(fieldblock[:11])
    1631              type = fieldblock[11]
    1632              if not type in yo._meta.fieldtypes:
    1633                  raise DbfError("Unknown field type: %s" % type)
    1634              start = offset
    1635              length = ord(fieldblock[16])
    1636              offset += length
    1637              end = start + length
    1638              decimals = ord(fieldblock[17])
    1639              flags = ord(fieldblock[18])
    1640              yo._meta.fields.append(name)
    1641              yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
    1642 -class FpTable(DbfTable):
    1643      'Provides an interface for working with FoxPro 2 tables'
    1644      _version = 'Foxpro'
    1645      _versionabbv = 'fp'
    1646      _fieldtypes = {
    1647              'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
    1648              'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
    1649              'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
    1650              'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
    1651              'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
    1652              'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo},
    1653              'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
    1654              'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
    1655              '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
    1656      _memoext = '.fpt'
    1657      _memotypes = ('G','M','P')
    1658      _memoClass = _VfpMemo
    1659      _yesMemoMask = '\xf5'               # 1111 0101
    1660      _noMemoMask = '\x03'                # 0000 0011
    1661      _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
    1662      _variable_fields = ('C','F','N')
    1663      _character_fields = ('C','M')       # field representing character data
    1664      _decimal_fields = ('F','N')
    1665      _numeric_fields = ('B','F','I','N','Y')
    1666      _supported_tables = ('\x03', '\xf5')
    1667      _dbfTableHeader = array('c', '\x00' * 32)
    1668      _dbfTableHeader[0] = '\x30'         # version - Foxpro 6  0011 0000
    1669      _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
    1670      _dbfTableHeader[10] = '\x01'        # record length -- one for delete flag
    1671      _dbfTableHeader[29] = '\x03'        # code page -- 437 US-MS DOS
    1672      _dbfTableHeader = _dbfTableHeader.tostring()
    1673      _dbfTableHeaderExtra = '\x00' * 263
    1674      _use_deleted = True
    1675 -    def _checkMemoIntegrity(yo):
    1676          if os.path.exists(yo._meta.memoname):
    1677              try:
    1678                  yo._meta.memo = yo._memoClass(yo._meta)
    1679              except:
    1680                  yo._meta.dfd.close()
    1681                  yo._meta.dfd = None
    1682                  raise
    1683          if not yo._meta.ignorememos:
    1684              for field in yo._meta.fields:
    1685                  if yo._meta[field]['type'] in yo._memotypes:
    1686                      if not os.path.exists(yo._meta.memoname):
    1687                          yo._meta.dfd.close()
    1688                          yo._meta.dfd = None
    1689                          raise DbfError("Table structure corrupt:  memo fields exist without memo file")
    1690                      break
    1691 -    def _initializeFields(yo):
    1692          "builds the FieldList of names, types, and descriptions"
    1693          yo._meta.fields[:] = []
    1694          offset = 1
    1695          fieldsdef = yo._meta.header.fields
    1696          if len(fieldsdef) % 32 != 0:
    1697              raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef))
    1698          if len(fieldsdef) // 32 != yo.field_count:
    1699              raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32))
    1700          for i in range(yo.field_count):
    1701              fieldblock = fieldsdef[i*32:(i+1)*32]
    1702              name = io.unpackStr(fieldblock[:11])
    1703              type = fieldblock[11]
    1704              if not type in yo._meta.fieldtypes:
    1705                  raise DbfError("Unknown field type: %s" % type)
    1706              elif type == '0':
    1707                  return          # ignore nullflags
    1708              start = offset
    1709              length = ord(fieldblock[16])
    1710              offset += length
    1711              end = start + length
    1712              decimals = ord(fieldblock[17])
    1713              flags = ord(fieldblock[18])
    1714              yo._meta.fields.append(name)
    1715              yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
    1716
    1717 -class VfpTable(DbfTable):
    1718      'Provides an interface for working with Visual FoxPro 6 tables'
    1719      _version = 'Visual Foxpro v6'
    1720      _versionabbv = 'vfp'
    1721      _fieldtypes = {
    1722              'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
    1723              'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
    1724              'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
    1725              'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
    1726              'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
    1727              'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
    1728              'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
    1729              'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
    1730              'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
    1731              'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
    1732              'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
    1733              'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo},
    1734              '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
    1735      _memoext = '.fpt'
    1736      _memotypes = ('G','M','P')
    1737      _memoClass = _VfpMemo
    1738      _yesMemoMask = '\x30'               # 0011 0000
    1739      _noMemoMask = '\x30'                # 0011 0000
    1740      _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
    1741      _variable_fields = ('C','F','N')
    1742      _character_fields = ('C','M')       # field representing character data
    1743      _decimal_fields = ('F','N')
    1744      _numeric_fields = ('B','F','I','N','Y')
    1745      _supported_tables = ('\x30',)
    1746      _dbfTableHeader = array('c', '\x00' * 32)
    1747      _dbfTableHeader[0] = '\x30'         # version - Foxpro 6  0011 0000
    1748      _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263))
    1749      _dbfTableHeader[10] = '\x01'        # record length -- one for delete flag
    1750      _dbfTableHeader[29] = '\x03'        # code page -- 437 US-MS DOS
    1751      _dbfTableHeader = _dbfTableHeader.tostring()
    1752      _dbfTableHeaderExtra = '\x00' * 263
    1753      _use_deleted = True
    1754 -    def _checkMemoIntegrity(yo):
    1755          if os.path.exists(yo._meta.memoname):
    1756              try:
    1757                  yo._meta.memo = yo._memoClass(yo._meta)
    1758              except:
    1759                  yo._meta.dfd.close()
    1760                  yo._meta.dfd = None
    1761                  raise
    1762          if not yo._meta.ignorememos:
    1763              for field in yo._meta.fields:
    1764                  if yo._meta[field]['type'] in yo._memotypes:
    1765                      if not os.path.exists(yo._meta.memoname):
    1766                          yo._meta.dfd.close()
    1767                          yo._meta.dfd = None
    1768                          raise DbfError("Table structure corrupt:  memo fields exist without memo file")
    1769                      break
    1770 -    def _initializeFields(yo):
    1771          "builds the FieldList of names, types, and descriptions"
    1772          yo._meta.fields[:] = []
    1773          offset = 1
    1774          fieldsdef = yo._meta.header.fields
    1775          for i in range(yo.field_count):
    1776              fieldblock = fieldsdef[i*32:(i+1)*32]
    1777              name = io.unpackStr(fieldblock[:11])
    1778              type = fieldblock[11]
    1779              if not type in yo._meta.fieldtypes:
    1780                  raise DbfError("Unknown field type: %s" % type)
    1781              elif type == '0':
    1782                  return          # ignore nullflags
    1783              start = io.unpackLongInt(fieldblock[12:16])
    1784              length = ord(fieldblock[16])
    1785              offset += length
    1786              end = start + length
    1787              decimals = ord(fieldblock[17])
    1788              flags = ord(fieldblock[18])
    1789              yo._meta.fields.append(name)
    1790              yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
    1791 -class List(object):
    1792      "list of Dbf records, with set-like behavior"
    1793      _desc = ''
    1794 -    def __init__(yo, new_records=None, desc=None, key=None, field_names=None):
    1795          yo.field_names = field_names
    1796          yo._list = []
    1797          yo._set = set()
    1798          if key is not None:
    1799              yo.key = key
    1800              if key.__doc__ is None:
    1801                  key.__doc__ = 'unknown'
    1802          key = yo.key
    1803          yo._current = -1
    1804          if isinstance(new_records, yo.__class__) and key is new_records.key:
    1805                  yo._list = new_records._list[:]
    1806                  yo._set = new_records._set.copy()
    1807                  yo._current = 0
    1808          elif new_records is not None:
    1809              for record in new_records:
    1810                  value = key(record)
    1811                  item = (record.record_table, record.record_number, value)
    1812                  if value not in yo._set:
    1813                      yo._set.add(value)
    1814                      yo._list.append(item)
    1815              yo._current = 0
    1816          if desc is not None:
    1817              yo._desc = desc
    1818 -    def __add__(yo, other):
    1819          key = yo.key
    1820          if isinstance(other, (DbfTable, list)):
    1821              other = yo.__class__(other, key=key)
    1822          if isinstance(other, yo.__class__):
    1823              result = yo.__class__()
    1824              result._set = yo._set.copy()
    1825              result._list[:] = yo._list[:]
    1826              result.key = yo.key
    1827              if key is other.key:   # same key?  just compare key values
    1828                  for item in other._list:
    1829                      if item[2] not in result._set:
    1830                          result._set.add(item[2])
    1831                          result._list.append(item)
    1832              else:                   # different keys, use this list's key on other's records
    1833                  for rec in other:
    1834                      value = key(rec)
    1835                      if value not in result._set:
    1836                          result._set.add(value)
    1837                          result._list.append((rec.record_table, rec.record_number, value))
    1838              result._current = 0 if result else -1
    1839              return result
    1840          return NotImplemented
    1841 -    def __contains__(yo, record):
    1842          if isinstance(record, tuple):
    1843              item = record
    1844          else:
    1845              item = yo.key(record)
    1846          return item in yo._set
    1847 -    def __delitem__(yo, key):
    1848          if isinstance(key, int):
    1849              item = yo._list.pop[key]
    1850              yo._set.remove(item[2])
    1851          elif isinstance(key, slice):
    1852              yo._set.difference_update([item[2] for item in yo._list[key]])
    1853              yo._list.__delitem__(key)
    1854          else:
    1855              raise TypeError
    1856 -    def __getitem__(yo, key):
    1857          if isinstance(key, int):
    1858              count = len(yo._list)
    1859              if not -count <= key < count:
    1860                  raise IndexError("Record %d is not in list." % key)
    1861              return yo._get_record(*yo._list[key])
    1862          elif isinstance(key, slice):
    1863              result = yo.__class__()
    1864              result._list[:] = yo._list[key]
    1865              result._set = set(result._list)
    1866              result.key = yo.key
    1867              result._current = 0 if result else -1
    1868              return result
    1869          else:
    1870              raise TypeError('indices must be integers')
    1871 -    def __iter__(yo):
    1872          return (table.get_record(recno) for table, recno, value in yo._list)
    1873 -    def __len__(yo):
    1874          return len(yo._list)
    1875 -    def __nonzero__(yo):
    1876          return len(yo) > 0
    1877 -    def __radd__(yo, other):
    1878          return yo.__add__(other)
    1879 -    def __repr__(yo):
    1880          if yo._desc:
    1881              return "%s(key=%s - %s - %d records)" % (yo.__class__, yo.key.__doc__, yo._desc, len(yo._list))
    1882          else:
    1883              return "%s(key=%s - %d records)" % (yo.__class__, yo.key.__doc__, len(yo._list))
    1884 -    def __rsub__(yo, other):
    1885          key = yo.key
    1886          if isinstance(other, (DbfTable, list)):
    1887              other = yo.__class__(other, key=key)
    1888          if isinstance(other, yo.__class__):
    1889              result = yo.__class__()
    1890              result._list[:] = other._list[:]
    1891              result._set = other._set.copy()
    1892              result.key = key
    1893              lost = set()
    1894              if key is other.key:
    1895                  for item in yo._list:
    1896                      if item[2] in result._list:
    1897                          result._set.remove(item[2])
    1898                          lost.add(item)
    1899              else:
    1900                  for rec in other:
    1901                      value = key(rec)
    1902                      if value in result._set:
    1903                          result._set.remove(value)
    1904                          lost.add((rec.record_table, rec.record_number, value))
    1905              result._list = [item for item in result._list if item not in lost]
    1906              result._current = 0 if result else -1
    1907              return result
    1908          return NotImplemented
    1909 -    def __sub__(yo, other):
    1910          key = yo.key
    1911          if isinstance(other, (DbfTable, list)):
    1912              other = yo.__class__(other, key=key)
    1913          if isinstance(other, yo.__class__):
    1914              result = yo.__class__()
    1915              result._list[:] = yo._list[:]
    1916              result._set = yo._set.copy()
    1917              result.key = key
    1918              lost = set()
    1919              if key is other.key:
    1920                  for item in other._list:
    1921                      if item[2] in result._set:
    1922                          result._set.remove(item[2])
    1923                          lost.add(item[2])
    1924              else:
    1925                  for rec in other:
    1926                      value = key(rec)
    1927                      if value in result._set:
    1928                          result._set.remove(value)
    1929                          lost.add(value)
    1930              result._list = [item for item in result._list if item[2] not in lost]
    1931              result._current = 0 if result else -1
    1932              return result
    1933          return NotImplemented
    1934 -    def _maybe_add(yo, item):
    1935          if item[2] not in yo._set:
    1936              yo._set.add(item[2])
    1937              yo._list.append(item)
    1938 -    def _get_record(yo, table=None, rec_no=None, value=None):
    1939          if table is rec_no is None:
    1940              table, rec_no, value = yo._list[yo._current]
    1941          return table.get_record(rec_no)
    1942 -    def _purge(yo, record, old_record_number, offset):
    1943          partial = record.record_table, old_record_number
    1944          records = sorted(yo._list, key=lambda item: (item[0], item[1]))
    1945          for item in records:
    1946              if partial == item[:2]:
    1947                  found = True
    1948                  break
    1949              elif partial[0] is item[0] and partial[1] < item[1]:
    1950                  found = False
    1951                  break
    1952          else:
    1953              found = False
    1954          if found:
    1955              yo._list.pop(yo._list.index(item))
    1956              yo._set.remove(item[2])
    1957          start = records.index(item) + found
    1958          for item in records[start:]:
    1959              if item[0] is not partial[0]:       # into other table's records
    1960                  break
    1961              i = yo._list.index(item)
    1962              yo._set.remove(item[2])
    1963              item = item[0], (item[1] - offset), item[2]
    1964              yo._list[i] = item
    1965              yo._set.add(item[2])
    1966          return found
    1967 -    def append(yo, new_record):
    1968          yo._maybe_add((new_record.record_table, new_record.record_number, yo.key(new_record)))
    1969          if yo._current == -1 and yo._list:
    1970              yo._current = 0
    1971          return new_record
    1972 -    def bottom(yo):
    1973          if yo._list:
    1974              yo._current = len(yo._list) - 1
    1975              return yo._get_record()
    1976          raise DbfError("dbf.List is empty")
    1977 -    def clear(yo):
    1978          yo._list = []
    1979          yo._set = set()
    1980          yo._current = -1
    1981 -    def current(yo):
    1982          if yo._current < 0:
    1983              raise Bof()
    1984          elif yo._current == len(yo._list):
    1985              raise Eof()
    1986          return yo._get_record()
    1987 -    def extend(yo, new_records):
    1988          key = yo.key
    1989          if isinstance(new_records, yo.__class__):
    1990              if key is new_records.key:   # same key?  just compare key values
    1991                  for item in new_records._list:
    1992                      yo._maybe_add(item)
    1993              else:                   # different keys, use this list's key on other's records
    1994                  for rec in new_records:
    1995                      value = key(rec)
    1996                      yo._maybe_add((rec.record_table, rec.record_number, value))
    1997          else:
    1998              for record in new_records:
    1999                  value = key(rec)
    2000                  yo._maybe_add((rec.record_table, rec.record_number, value))
    2001          if yo._current == -1 and yo._list:
    2002              yo._current = 0
    2003 -    def goto(yo, index_number):
    2004          if yo._list:
    2005              if 0 <= index_number <= len(yo._list):
    2006                  yo._current = index_number
    2007                  return yo._get_record()
    2008              raise DbfError("index %d not in dbf.List of %d records" % (index_number, len(yo._list)))
    2009          raise DbfError("dbf.List is empty")
    2010 -    def index(yo, sort=None, reverse=False):
    2011          "sort= ((field_name, func), (field_name, func),) | 'ORIGINAL'"
    2012          if sort is None:
    2013              results = []
    2014              for field, func in yo._meta.index:
    2015                  results.append("%s(%s)" % (func.__name__, field))
    2016              return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed])
    2017          yo._meta.index_reversed = reverse
    2018          if sort == 'ORIGINAL':
    2019              yo._index = range(yo._meta.header.record_count)
    2020              yo._meta.index = 'ORIGINAL'
    2021              if reverse:
    2022                  yo._index.reverse()
    2023              return
    2024          new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop])
    2025          yo._meta.index = tuple(new_sort)
    2026          yo._meta.orderresults = [''] * len(yo)
    2027          for record in yo:
    2028              yo._meta.orderresults[record.record_number] = record()
    2029          yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
    2030 -    def index(yo, record, start=None, stop=None):
    2031          item = record.record_table, record.record_number, yo.key(record)
    2032          if start is None:
    2033              start = 0
    2034          if stop is None:
    2035              stop = len(yo._list)
    2036          return yo._list.index(item, start, stop)
    2037 -    def insert(yo, i, record):
    2038          item = record.record_table, record.record_number, yo.key(record)
    2039          if item not in yo._set:
    2040              yo._set.add(item[2])
    2041              yo._list.insert(i, item)
    2042 -    def key(yo, record):
    2043          "table_name, record_number"
    2044          return record.record_table, record.record_number
    2045 -    def next(yo):
    2046          if yo._current < len(yo._list):
    2047              yo._current += 1
    2048              if yo._current < len(yo._list):
    2049                  return yo._get_record()
    2050          raise Eof()
    2051 -    def pop(yo, index=None):
    2052          if index is None:
    2053              table, recno, value = yo._list.pop()
    2054          else:
    2055              table, recno, value = yo._list.pop(index)
    2056          yo._set.remove(value)
    2057          return yo._get_record(table, recno, value)
    2058 -    def prev(yo):
    2059          if yo._current >= 0:
    2060              yo._current -= 1
    2061              if yo._current > -1:
    2062                  return yo._get_record()
    2063          raise Bof()
    2064 -    def remove(yo, record):
    2065          item = record.record_table, record.record_number, yo.key(record)
    2066          yo._list.remove(item)
    2067          yo._set.remove(item[2])
    2068 -    def reverse(yo):
    2069          return yo._list.reverse()
    2070 -    def top(yo):
    2071          if yo._list:
    2072              yo._current = 0
    2073              return yo._get_record()
    2074          raise DbfError("dbf.List is empty")
    2075 -    def sort(yo, key=None, reverse=False):
    2076          if key is None:
    2077              return yo._list.sort(reverse=reverse)
    2078          return yo._list.sort(key=lambda item: key(item[0].get_record(item[1])), reverse=reverse)
    2079
    2080 -class DbfCsv(csv.Dialect):
    2081      "csv format for exporting tables"
    2082      delimiter = ','
    2083      doublequote = True
    2084      escapechar = None
    2085      lineterminator = '\n'
    2086      quotechar = '"'
    2087      skipinitialspace = True
    2088      quoting = csv.QUOTE_NONNUMERIC
    2089 -class Index(object):
    2090 -    class IndexIterator(object):
    2091          "returns records using this index"
    2092 -        def __init__(yo, table, records):
    2093              yo.table = table
    2094              yo.records = records
    2095              yo.index = 0
    2096 -        def __iter__(yo):
    2097              return yo
    2098 -        def next(yo):
    2099              while yo.index < len(yo.records):
    2100                  record = yo.table.get_record(yo.records[yo.index])
    2101                  yo.index += 1
    2102                  if not yo.table.use_deleted and record.has_been_deleted:
    2103                      continue
    2104                  return record
    2105              else:
    2106                  raise StopIteration
    2107 -    def __init__(yo, table, key, field_names=None):
    2108          yo._table = table
    2109          yo._values = []             # ordered list of values
    2110          yo._rec_by_val = []         # matching record numbers
    2111          yo._records = {}            # record numbers:values
    2112          yo.__doc__ = key.__doc__ or 'unknown'
    2113          yo.key = key
    2114          yo.field_names = field_names or table.field_names
    2115          for record in table:
    2116              value = key(record)
    2117              if value is DoNotIndex:
    2118                  continue
    2119              rec_num = record.record_number
    2120              if not isinstance(value, tuple):
    2121                  value = (value, )
    2122              vindex = bisect_right(yo._values, value)
    2123              yo._values.insert(vindex, value)
    2124              yo._rec_by_val.insert(vindex, rec_num)
    2125              yo._records[rec_num] = value
    2126          table._indexen.add(yo)
    2127 -    def __call__(yo, record):
    2128          rec_num = record.record_number
    2129          if rec_num in yo._records:
    2130              value = yo._records[rec_num]
    2131              vindex = bisect_left(yo._values, value)
    2132              yo._values.pop(vindex)
    2133              yo._rec_by_val.pop(vindex)
    2134          value = yo.key(record)
    2135          if value is DoNotIndex:
    2136              return
    2137          if not isinstance(value, tuple):
    2138              value = (value, )
    2139          vindex = bisect_right(yo._values, value)
    2140          yo._values.insert(vindex, value)
    2141          yo._rec_by_val.insert(vindex, rec_num)
    2142          yo._records[rec_num] = value
    2143 -    def __contains__(yo, match):
    2144          if isinstance(match, _DbfRecord):
    2145              if match.record_table is yo._table:
    2146                  return match.record_number in yo._records
    2147              match = yo.key(match)
    2148          elif not isinstance(match, tuple):
    2149              match = (match, )
    2150          return yo.find(match) != -1
    2151 -    def __getitem__(yo, key):
    2152          if isinstance(key, int):
    2153              count = len(yo._values)
    2154              if not -count <= key < count:
    2155                  raise IndexError("Record %d is not in list." % key)
    2156              rec_num = yo._rec_by_val[key]
    2157              return yo._table.get_record(rec_num)
    2158          elif isinstance(key, slice):
    2159              result = List(field_names=yo._table.field_names)
    2160              yo._table._dbflists.add(result)
    2161              start, stop, step = key.start, key.stop, key.step
    2162              if start is None: start = 0
    2163              if stop is None: stop = len(yo._rec_by_val)
    2164              if step is None: step = 1
    2165              for loc in range(start, stop, step):
    2166                  record = yo._table.get_record(yo._rec_by_val[loc])
    2167                  result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
    2168              result._current = 0 if result else -1
    2169              return result
    2170          elif isinstance (key, (str, unicode, tuple, _DbfRecord)):
    2171              if isinstance(key, _DbfRecord):
    2172                  key = yo.key(key)
    2173              elif not isinstance(key, tuple):
    2174                  key = (key, )
    2175              loc = yo.find(key)
    2176              if loc == -1:
    2177                  raise KeyError(key)
    2178              return yo._table.get_record(yo._rec_by_val[loc])
    2179          else:
    2180              raise TypeError('indices must be integers, match objects must by strings or tuples')
    2181 -    def __enter__(yo):
    2182          return yo
    2183 -    def __exit__(yo, *exc_info):
    2184          yo._table.close()
    2185          yo._values[:] = []
    2186          yo._rec_by_val[:] = []
    2187          yo._records.clear()
    2188          return False
    2189 -    def __iter__(yo):
    2190          return yo.IndexIterator(yo._table, yo._rec_by_val)
    2191 -    def __len__(yo):
    2192          return len(yo._records)
    2193 -    def _partial_match(yo, target, match):
    2194          target = target[:len(match)]
    2195          if isinstance(match[-1], (str, unicode)):
    2196              target = list(target)
    2197              target[-1] = target[-1][:len(match[-1])]
    2198              target = tuple(target)
    2199          return target == match
    2200 -    def _purge(yo, rec_num):
    2201          value = yo._records.get(rec_num)
    2202          if value is not None:
    2203              vindex = bisect_left(yo._values, value)
    2204              del yo._records[rec_num]
    2205              yo._values.pop(vindex)
    2206              yo._rec_by_val.pop(vindex)
    2207 -    def _search(yo, match, lo=0, hi=None):
    2208          if hi is None:
    2209              hi = len(yo._values)
    2210          return bisect_left(yo._values, match, lo, hi)
    2211 -    def clear(yo):
    2212          "removes all entries from index"
    2213          yo._values[:] = []
    2214          yo._rec_by_val[:] = []
    2215          yo._records.clear()
    2216 -    def close(yo):
    2217          yo._table.close()
    2218 -    def find(yo, match, partial=False):
    2219          "returns numeric index of (partial) match, or -1"
    2220          if isinstance(match, _DbfRecord):
    2221              if match.record_number in yo._records:
    2222                  return yo._values.index(yo._records[match.record_number])
    2223              else:
    2224                  return -1
    2225          if not isinstance(match, tuple):
    2226              match = (match, )
    2227          loc = yo._search(match)
    2228          while loc < len(yo._values) and yo._values[loc] == match:
    2229              if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
    2230                  loc += 1
    2231                  continue
    2232              return loc
    2233          if partial:
    2234              while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
    2235                  if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted:
    2236                      loc += 1
    2237                      continue
    2238                  return loc
    2239          return -1
    2240 -    def find_index(yo, match):
    2241          "returns numeric index of either (partial) match, or position of where match would be"
    2242          if isinstance(match, _DbfRecord):
    2243              if match.record_number in yo._records:
    2244                  return yo._values.index(yo._records[match.record_number])
    2245              else:
    2246                  match = yo.key(match)
    2247          if not isinstance(match, tuple):
    2248              match = (match, )
    2249          loc = yo._search(match)
    2250          return loc
    2251      @classmethod
    2252 -    def from_file(cls, table, index_file):
    2253
    2254          def get_idx_records(data, length, howmany):
    2255              ptr = 0
    2256              current = 0
    2257              while current < howmany:
    2258                  key = data[ptr:ptr+length].replace('\x00','')
    2259                  rec = io.unpackLongInt(data[ptr+length:ptr+length+4], bigendian=True)
    2260                  yield key, recnum
    2261                  ptr += length + 4
    2262                  current += 1
    2263
    2264          def next_item(idx_file, node_loc, keylen):
    2265              idx_file.seek(node_loc)
    2266              data_chunk = idx_file.read(512)
    2267              attributes = io.unpackShortInt(data_chunk[:2])
    2268              howmany = io.unpackShortInt(data_chunk[2:4])
    2269              if attributes in (2, 3):
    2270                  for key, recnum in get_idx_records(data_chunk[12:512], keylen, howmany):
    2271                      yield key, recnum
    2272              else:
    2273                  for ignore, next_node in get_idx_records(data_chunk[12:512], keylen, howmany):
    2274                      print ignore, next_node
    2275                      for key, recnum in next_item(idx_file, next_node, keylen):
    2276                          yield key, recnum
    2277
    2278
    2279          idx = object.__new__(cls)
    2280          #- idx.key = lambda rec: DoNotIndex
    2281          data = open(index_file, 'rb')
    2282          header = data.read(512)
    2283          rootnode = io.unpackLongInt(header[:4])
    2284          keylen = io.unpackShortInt(header[12:14])
    2285          idx.__doc__ = header[16:236].replace('\x00','')
    2286          for_expr = header[236:456].replace('\x00','')
    2287          if for_expr:
    2288              idx.__doc__ += ' for ' + for_expr.replace('=','==')
    2289          for rec in next_item(data, rootnode, keylen):
    2290              print rec
    2291
    2292 -    def index(yo, match, partial=False):
    2293          "returns numeric index of (partial) match, or raises ValueError"
    2294          loc = yo.find(match, partial)
    2295          if loc == -1:
    2296              if isinstance(match, _DbfRecord):
    2297                  raise ValueError("table <%s> record [%d] not in index <%s>" % (yo._table.filename, match.record_number, yo.__doc__))
    2298              else:
    2299                  raise ValueError("match criteria <%s> not in index" % (match, ))
    2300          return loc
    2301 -    def reindex(yo):
    2302          "reindexes all records"
    2303          for record in yo._table:
    2304              yo(record)
    2305 -    def query(yo, sql_command=None, python=None):
    2306          """recognized sql commands are SELECT, UPDATE, REPLACE, INSERT, DELETE, and RECALL"""
    2307          if sql_command:
    2308              return sql(yo, sql_command)
    2309          elif python is None:
    2310              raise DbfError("query: python parameter must be specified")
    2311          possible = List(desc="%s -->  %s" % (yo._table.filename, python), field_names=yo._table.field_names)
    2312          yo._table._dbflists.add(possible)
    2313          query_result = {}
    2314          select = 'query_result["keep"] = %s' % python
    2315          g = {}
    2316          for record in yo:
    2317              query_result['keep'] = False
    2318              g['query_result'] = query_result
    2319              exec select in g, record
    2320              if query_result['keep']:
    2321                  possible.append(record)
    2322              record.write_record()
    2323          return possible
    2324 -    def search(yo, match, partial=False):
    2325          "returns dbf.List of all (partially) matching records"
    2326          result = List(field_names=yo._table.field_names)
    2327          yo._table._dbflists.add(result)
    2328          if not isinstance(match, tuple):
    2329              match = (match, )
    2330          loc = yo._search(match)
    2331          if loc == len(yo._values):
    2332              return result
    2333          while loc < len(yo._values) and yo._values[loc] == match:
    2334              record = yo._table.get_record(yo._rec_by_val[loc])
    2335              if not yo._table.use_deleted and record.has_been_deleted:
    2336                  loc += 1
    2337                  continue
    2338              result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
    2339              loc += 1
    2340          if partial:
    2341              while loc < len(yo._values) and yo._partial_match(yo._values[loc], match):
    2342                  record = yo._table.get_record(yo._rec_by_val[loc])
    2343                  if not yo._table.use_deleted and record.has_been_deleted:
    2344                      loc += 1
    2345                      continue
    2346                  result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record)))
    2347                  loc += 1
    2348          return result
    2349
    2350  csv.register_dialect('dbf', DbfCsv)
    2351
    2352  sql_functions = {
    2353          'select':None,
    2354          'update':None,
    2355          'insert':None,
    2356          'delete':None,
    2357          'count': None}
    2358 -def sql_criteria(records, criteria):
    2359      "creates a function matching the sql criteria"
    2360      function = """def func(records):
    2361      \"\"\"%s\"\"\"
    2362      matched = List(field_names=records[0].field_names)
    2363      for rec in records:
    2364          %s
    2365
    2366          if %s:
    2367              matched.append(rec)
    2368      return matched"""
    2369      fields = []
    2370      for field in records[0].field_names:
    2371          if field in criteria:
    2372              fields.append(field)
    2373      fields = '\n        '.join(['%s = rec.%s' % (field, field) for field in fields])
    2374      g = {'List':List}
    2375      function %= (criteria, fields, criteria)
    2376  #-     print function
    2377      exec function in g
    2378      return g['func']
    2379
    2380 -def sql_cmd(records, command):
    2381      "creates a function matching to apply command to each record in records"
    2382      function = """def func(records):
    2383      \"\"\"%s\"\"\"
    2384      changed = 0
    2385      for rec in records:
    2386          %s
    2387
    2388          %s
    2389
    2390          %s
    2391          changed += rec.write_record()
    2392      return changed"""
    2393      fields = []
    2394      for field in records[0].field_names:
    2395          if field in command:
    2396              fields.append(field)
    2397      pre_fields = '\n        '.join(['%s = rec.%s' % (field, field) for field in fields])
    2398      post_fields = '\n        '.join(['rec.%s = %s' % (field, field) for field in fields])
    2399      g = dbf.sql_user_functions.copy()
    2400      if '=' not in command and ' with ' in command.lower():
    2401          offset = command.lower().index(' with ')
    2402          command = command[:offset] + ' = ' + command[offset+6:]
    2403      function %= (command, pre_fields, command, post_fields)
    2404  #-     print function
    2405      exec function in g
    2406      return g['func']
    2407
    2408 -def sql(records, command):
    2409      """recognized sql commands are SELECT, UPDATE, INSERT, DELETE, and RECALL"""
    2410      table = records[0].record_table
    2411      sql_command = command
    2412      no_condition = False
    2413      if ' for ' in command:
    2414          command, condition = command.split(' for ')
    2415          condition = sql_criteria(records, condition)
    2416      else:
    2417          def condition(records):
    2418              return records[:]
    2419          no_condition = True
    2420      name, command = command.split(' ', 1)
    2421      name = name.lower()
    2422      field_names = table.field_names
    2423      if name == 'select':
    2424          if command.strip() != '*':
    2425              field_names = command.replace(' ','').split(',')
    2426          def command(records):
    2427              return
    2428      else:
    2429          command = sql_cmd(records, command)
    2430      if name not in ('delete','insert','recall','select','update','replace'):
    2431          raise DbfError("unrecognized sql command: %s" % name.upper())
    2432      if name == 'insert' and not no_condition:
    2433          raise DbfError("FOR clause not allowed with INSERT")
    2434      possible = List(desc=sql_command, field_names=field_names)
    2435      tables = set()
    2436      if name == 'insert':
    2437          raise DbfError("INSERT not currently implemented")
    2438          record = table.append()
    2439          command(record)
    2440          record.write_record()
    2441          record.check_index()
    2442          possible.append(record)
    2443          changed = 0
    2444      else:
    2445          possible = condition(records)
    2446          possible.field_names = field_names
    2447          changed = command(possible)
    2448          for record in possible:
    2449              tables.add(record.record_table)
    2450              if name == 'delete':
    2451                  record.delete_record()
    2452              elif name == 'recall':
    2453                  record.undelete_record()
    2454              elif name == 'select':
    2455                  pass
    2456              elif name == 'update' or name == 'replace':
    2457                  pass
    2458                  #command(record)
    2459              else:
    2460                  raise DbfError("unrecognized sql command: %s" % sql.upper)
    2461              record.write_record()
    2462      for list_table in tables:
    2463          list_table._dbflists.add(possible)
    2464      possible.modified = changed
    2465      return possible
    2466 -def _nop(value):
    2467      "returns parameter unchanged"
    2468      return value
    2469 -def _normalize_tuples(tuples, length, filler):
    2470      "ensures each tuple is the same length, using filler[-missing] for the gaps"
    2471      final = []
    2472      for t in tuples:
    2473          if len(t) < length:
    2474              final.append( tuple([item for item in t] + filler[len(t)-length:]) )
    2475          else:
    2476              final.append(t)
    2477      return tuple(final)
    2478 -def _codepage_lookup(cp):
    2479      if cp not in code_pages:
    2480          for code_page in sorted(code_pages.keys()):
    2481              sd, ld = code_pages[code_page]
    2482              if cp == sd or cp == ld:
    2483                  if sd is None:
    2484                      raise DbfError("Unsupported codepage: %s" % ld)
    2485                  cp = code_page
    2486                  break
    2487          else:
    2488              raise DbfError("Unsupported codepage: %s" % cp)
    2489      sd, ld = code_pages[cp]
    2490      return cp, sd, ld
    2491 -def ascii(new_setting=None):
    2492      "get/set return_ascii setting"
    2493      global return_ascii
    2494      if new_setting is None:
    2495          return return_ascii
    2496      else:
    2497          return_ascii = new_setting
    2498 -def codepage(cp=None):
    2499      "get/set default codepage for any new tables"
    2500      global default_codepage
    2501      cp, sd, ld = _codepage_lookup(cp or default_codepage)
    2502      default_codepage = sd
    2503      return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
    2504 -def encoding(cp=None):
    2505      "get/set default encoding for non-unicode strings passed into a table"
    2506      global input_decoding
    2507      cp, sd, ld = _codepage_lookup(cp or input_decoding)
    2508      default_codepage = sd
    2509      return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
    2510 -class _Db4Table(DbfTable):
    2511      version = 'dBase IV w/memos (non-functional)'
    2512      _versionabbv = 'db4'
    2513      _fieldtypes = {
    2514              'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter},
    2515              'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency},
    2516              'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble},
    2517              'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric},
    2518              'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric},
    2519              'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger},
    2520              'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical},
    2521              'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate},
    2522              'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime},
    2523              'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
    2524              'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
    2525              'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo},
    2526              '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} }
    2527      _memoext = '.dbt'
    2528      _memotypes = ('G','M','P')
    2529      _memoClass = _VfpMemo
    2530      _yesMemoMask = '\x8b'               # 0011 0000
    2531      _noMemoMask = '\x04'                # 0011 0000
    2532      _fixed_fields = ('B','D','G','I','L','M','P','T','Y')
    2533      _variable_fields = ('C','F','N')
    2534      _character_fields = ('C','M')       # field representing character data
    2535      _decimal_fields = ('F','N')
    2536      _numeric_fields = ('B','F','I','N','Y')
    2537      _supported_tables = ('\x04', '\x8b')
    2538      _dbfTableHeader = ['\x00'] * 32
    2539      _dbfTableHeader[0] = '\x8b'         # version - Foxpro 6  0011 0000
    2540      _dbfTableHeader[10] = '\x01'        # record length -- one for delete flag
    2541      _dbfTableHeader[29] = '\x03'        # code page -- 437 US-MS DOS
    2542      _dbfTableHeader = ''.join(_dbfTableHeader)
    2543      _dbfTableHeaderExtra = ''
    2544      _use_deleted = True
    2545 -    def _checkMemoIntegrity(yo):
    2546          "dBase III specific"
    2547          if yo._meta.header.version == '\x8b':
    2548              try:
    2549                  yo._meta.memo = yo._memoClass(yo._meta)
    2550              except:
    2551                  yo._meta.dfd.close()
    2552                  yo._meta.dfd = None
    2553                  raise
    2554          if not yo._meta.ignorememos:
    2555              for field in yo._meta.fields:
    2556                  if yo._meta[field]['type'] in yo._memotypes:
    2557                      if yo._meta.header.version != '\x8b':
    2558                          yo._meta.dfd.close()
    2559                          yo._meta.dfd = None
    2560                          raise DbfError("Table structure corrupt:  memo fields exist, header declares no memos")
    2561                      elif not os.path.exists(yo._meta.memoname):
    2562                          yo._meta.dfd.close()
    2563                          yo._meta.dfd = None
    2564                          raise DbfError("Table structure corrupt:  memo fields exist without memo file")
    2565                      break
    2566