dumbdbm.py revision 0320464583957258031ebee1b8897343211e4339 
1"""A dumb and slow but simple dbm clone.
2
3For database spam, spam.dir contains the index (a text file),
4spam.bak *may* contain a backup of the index (also a text file),
5while spam.dat contains the data (a binary file).
6
7XXX TO DO:
8
9- seems to contain a bug when updating...
10
11- reclaim free space (currently, space once occupied by deleted or expanded
12items is never reused)
13
14- support concurrent access (currently, if two processes take turns making
15updates, they can mess up the index)
16
17- support efficient access to large databases (currently, the whole index
18is read when the database is opened, and some updates rewrite the whole index)
19
20- support opening for read-only (flag = 'm')
21
22"""
23
24import os as _os
25import __builtin__
26import UserDict
27
28_open = __builtin__.open
29
30_BLOCKSIZE = 512
31
32error = IOError                         # For anydbm
33
34class _Database(UserDict.DictMixin):
35
36    # The on-disk directory and data files can remain in mutually
37    # inconsistent states for an arbitrarily long time (see comments
38    # at the end of __setitem__).  This is only repaired when _commit()
39    # gets called.  One place _commit() gets called is from __del__(),
40    # and if that occurs at program shutdown time, module globals may
41    # already have gotten rebound to None.  Since it's crucial that
42    # _commit() finish successfully, we can't ignore shutdown races
43    # here, and _commit() must not reference any globals.
44    _os = _os       # for _commit()
45    _open = _open   # for _commit()
46
47    def __init__(self, filebasename, mode):
48        self._mode = mode
49
50        # The directory file is a text file.  Each line looks like
51        #    "%r, (%d, %d)\n" % (key, pos, siz)
52        # where key is the string key, pos is the offset into the dat
53        # file of the associated value's first byte, and siz is the number
54        # of bytes in the associated value.
55        self._dirfile = filebasename + _os.extsep + 'dir'
56
57        # The data file is a binary file pointed into by the directory
58        # file, and holds the values associated with keys.  Each value
59        # begins at a _BLOCKSIZE-aligned byte offset, and is a raw
60        # binary 8-bit string value.
61        self._datfile = filebasename + _os.extsep + 'dat'
62        self._bakfile = filebasename + _os.extsep + 'bak'
63
64        # The index is an in-memory dict, mirroring the directory file.
65        self._index = None  # maps keys to (pos, siz) pairs
66
67        # Mod by Jack: create data file if needed
68        try:
69            f = _open(self._datfile, 'r')
70        except IOError:
71            f = _open(self._datfile, 'w', self._mode)
72        f.close()
73        self._update()
74
75    # Read directory file into the in-memory index dict.
76    def _update(self):
77        self._index = {}
78        try:
79            f = _open(self._dirfile)
80        except IOError:
81            pass
82        else:
83            for line in f:
84                key, pos_and_siz_pair = eval(line)
85                self._index[key] = pos_and_siz_pair
86            f.close()
87
88    # Write the index dict to the directory file.  The original directory
89    # file (if any) is renamed with a .bak extension first.  If a .bak
90    # file currently exists, it's deleted.
91    def _commit(self):
92        # CAUTION:  It's vital that _commit() succeed, and _commit() can
93        # be called from __del__().  Therefore we must never reference a
94        # global in this routine.
95        try:
96            self._os.unlink(self._bakfile)
97        except self._os.error:
98            pass
99
100        try:
101            self._os.rename(self._dirfile, self._bakfile)
102        except self._os.error:
103            pass
104
105        f = self._open(self._dirfile, 'w', self._mode)
106        for key, pos_and_siz_pair in self._index.iteritems():
107            f.write("%r, %r\n" % (key, pos_and_siz_pair))
108        f.close()
109
110    def __getitem__(self, key):
111        pos, siz = self._index[key]     # may raise KeyError
112        f = _open(self._datfile, 'rb')
113        f.seek(pos)
114        dat = f.read(siz)
115        f.close()
116        return dat
117
118    # Append val to the data file, starting at a _BLOCKSIZE-aligned
119    # offset.  The data file is first padded with NUL bytes (if needed)
120    # to get to an aligned offset.  Return pair
121    #     (starting offset of val, len(val))
122    def _addval(self, val):
123        f = _open(self._datfile, 'rb+')
124        f.seek(0, 2)
125        pos = int(f.tell())
126        npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
127        f.write('\0'*(npos-pos))
128        pos = npos
129        f.write(val)
130        f.close()
131        return (pos, len(val))
132
133    # Write val to the data file, starting at offset pos.  The caller
134    # is responsible for ensuring that there's enough room starting at
135    # pos to hold val, without overwriting some other value.  Return
136    # pair (pos, len(val)).
137    def _setval(self, pos, val):
138        f = _open(self._datfile, 'rb+')
139        f.seek(pos)
140        f.write(val)
141        f.close()
142        return (pos, len(val))
143
144    # key is a new key whose associated value starts in the data file
145    # at offset pos and with length siz.  Add an index record to
146    # the in-memory index dict, and append one to the directory file.
147    def _addkey(self, key, pos_and_siz_pair):
148        self._index[key] = pos_and_siz_pair
149        f = _open(self._dirfile, 'a', self._mode)
150        f.write("%r, %r\n" % (key, pos_and_siz_pair))
151        f.close()
152
153    def __setitem__(self, key, val):
154        if not type(key) == type('') == type(val):
155            raise TypeError, "keys and values must be strings"
156        if key not in self._index:
157            self._addkey(key, self._addval(val))
158        else:
159            # See whether the new value is small enough to fit in the
160            # (padded) space currently occupied by the old value.
161            pos, siz = self._index[key]
162            oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE
163            newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE
164            if newblocks <= oldblocks:
165                self._index[key] = self._setval(pos, val)
166            else:
167                # The new value doesn't fit in the (padded) space used
168                # by the old value.  The blocks used by the old value are
169                # forever lost.
170                self._index[key] = self._addval(val)
171
172            # Note that _index may be out of synch with the directory
173            # file now:  _setval() and _addval() don't update the directory
174            # file.  This also means that the on-disk directory and data
175            # files are in a mutually inconsistent state, and they'll
176            # remain that way until _commit() is called.  Note that this
177            # is a disaster (for the database) if the program crashes
178            # (so that _commit() never gets called).
179
180    def __delitem__(self, key):
181        # The blocks used by the associated value are lost.
182        del self._index[key]
183        # XXX It's unclear why we do a _commit() here (the code always
184        # XXX has, so I'm not changing it).  _setitem__ doesn't try to
185        # XXX keep the directory file in synch.  Why should we?  Or
186        # XXX why shouldn't __setitem__?
187        self._commit()
188
189    def keys(self):
190        return self._index.keys()
191
192    def has_key(self, key):
193        return key in self._index
194
195    def __contains__(self, key):
196        return key in self._index
197
198    def iterkeys(self):
199        return self._index.iterkeys()
200    __iter__ = iterkeys
201
202    def __len__(self):
203        return len(self._index)
204
205    def close(self):
206        self._commit()
207        self._index = None
208        self._datfile = self._dirfile = self._bakfile = None
209
210    def __del__(self):
211        if self._index is not None:
212            self._commit()
213
214
215
216def open(file, flag=None, mode=0666):
217    """Open the database file, filename, and return corresponding object.
218
219    The flag argument, used to control how the database is opened in the
220    other DBM implementations, is ignored in the dumbdbm module; the
221    database is always opened for update, and will be created if it does
222    not exist.
223
224    The optional mode argument is the UNIX mode of the file, used only when
225    the database has to be created.  It defaults to octal code 0666 (and
226    will be modified by the prevailing umask).
227
228    """
229    # flag argument is currently ignored
230    return _Database(file, mode)
231