casacore
BucketCache.h
Go to the documentation of this file.
1 //# BucketCache.h: Cache for buckets in a part of a file
2 //# Copyright (C) 1994,1995,1996,1999,2000,2001
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef CASA_BUCKETCACHE_H
29 #define CASA_BUCKETCACHE_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
33 #include <casacore/casa/IO/BucketFile.h>
34 #include <casacore/casa/Containers/Block.h>
35 #include <casacore/casa/OS/CanonicalConversion.h>
36 
37 //# Forward clarations
38 #include <casacore/casa/iosfwd.h>
39 
40 
41 namespace casacore { //# NAMESPACE CASACORE - BEGIN
42 
43 // <summary>
44 // Define the type of the static read and write function.
45 // </summary>
46 // <use visibility=export>
47 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
48 // </reviewed>
49 
50 // <synopsis>
51 // The BucketCache class needs a way to convert its data from local
52 // to canonical format and vice-versa. This is done by callback
53 // functions defined at construction time.
54 // <p>
55 // The ToLocal callback function has to allocate a buffer of the correct
56 // size and to copy/convert the canonical data in the input buffer to
57 // this buffer. The pointer this newly allocated buffer has to be returned.
58 // The BucketCache class keeps this pointer in the cache block.
59 // <p>
60 // The FromLocal callback function has to copy/convert the data from the
61 // buffer in local format to the buffer in canonical format. It should
62 // NOT delete the buffer; that has to be done by the DeleteBuffer function.
63 // <p>
64 // The AddBuffer callback function has to create (and initialize) a
65 // buffer to be added to the file and cache.
66 // When the file gets extended, BucketCache only registers the new size,
67 // but does not werite anything. When a bucket is read between the
68 // actual file size and the new file size, the AddBuffer callback function
69 // is called to create a buffer and possibly initialize it.
70 // <p>
71 // The DeleteBuffer callback function has to delete the buffer
72 // allocated by the ToLocal function.
73 // <p>
74 // The functions get a pointer to the owner object, which was provided
75 // at construction time. The callback function has to cast this to the
76 // correct type and can use it thereafter.
77 // <br>
78 // C++ supports pointers to members, but it is a bit hard. Therefore pointers
79 // to static members are used (which are simple pointers to functions).
80 // A pointer to the owner object is also passed to let the static function
81 // call the correct member function (when needed).
82 // </synopsis>
83 //
84 // <example>
85 // See class <linkto class=BucketCache>BucketCache</linkto>.
86 // </example>
87 
88 // <group name=BucketCache_CallBack>
89 typedef char* (*BucketCacheToLocal) (void* ownerObject, const char* canonical);
90 typedef void (*BucketCacheFromLocal) (void* ownerObject, char* canonical,
91  const char* local);
92 typedef char* (*BucketCacheAddBuffer) (void* ownerObject);
93 typedef void (*BucketCacheDeleteBuffer) (void* ownerObject, char* buffer);
94 // </group>
95 
96 
97 
98 // <summary>
99 // Cache for buckets in a part of a file
100 // </summary>
101 
102 // <use visibility=export>
103 
104 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
105 // </reviewed>
106 
107 // <prerequisite>
108 //# Classes you should understand before using this one.
109 // <li> <linkto class=BucketFile>BucketFile</linkto>
110 // </prerequisite>
111 
112 // <etymology>
113 // BucketCache implements a cache for buckets in (a part of) a file.
114 // </etymology>
115 
116 // <synopsis>
117 // A cache may allow more efficient quasi-random IO.
118 // It can, for instance, be used when a limited number of blocks
119 // in a file have to be accessed again and again.
120 // <p>
121 // The class BucketCache provides such a cache. It can be used on a
122 // consecutive part of a file as long as that part is not simultaneously
123 // accessed in another way (including another BucketCache object).
124 // <p>
125 // BucketCache stores the data as given.
126 // It uses <linkto group=BucketCache_CallBack>callback functions</linkto>
127 // to allocate/delete buffers and to convert the data to/from local format.
128 // <p>
129 // When a new bucket is needed and all slots in the cache are used,
130 // BucketCache will remove the least recently used bucket from the
131 // cache. When the dirty flag is set, it will first be written.
132 // <p>
133 // BucketCache maintains a list of free buckets. Initially this list is
134 // empty. When a bucket is removed, it is added to the free list.
135 // AddBucket will take buckets from the free list before extending the file.
136 // <p>
137 // Since it is possible to handle only a part of a file by a BucketCache
138 // object, it is also possible to have multiple BucketCache objects on
139 // the same file (as long as they access disjoint parts of the file).
140 // Each BucketCache object can have its own bucket size. This can,
141 // for example, be used to have tiled arrays with different tile shapes
142 // in the same file.
143 // <p>
144 // Statistics are kept to know how efficient the cache is working.
145 // It is possible to initialize and show the statistics.
146 // </synopsis>
147 
148 // <motivation>
149 // A cache may reduce IO traffix considerably.
150 // Furthermore it is more efficient to keep a cache in local format.
151 // In that way conversion to/from local only have to be done when
152 // data gets read/written. It also allows for precalculations.
153 // </motivation>
154 
155 // <example>
156 // <srcblock>
157 // // Define the callback function for reading a bucket.
158 // char* bToLocal (void*, const char* data)
159 // {
160 // char* ptr = new char[32768];
161 // memcpy (ptr, data, 32768);
162 // return ptr;
163 // }
164 // // Define the callback function for writing a bucket.
165 // void bFromLocal (void*, char* data, const char* local)
166 // {
167 // memcpy (data, local, 32768);
168 // }
169 // // Define the callback function for initializing a new bucket.
170 // char* bAddBuffer (void*)
171 // {
172 // char* ptr = new char[32768];
173 // for (uInt i=0; i++; i<32768) {
174 // ptr[i] = 0;
175 // }
176 // return ptr;
177 // }
178 // // Define the callback function for deleting a bucket.
179 // void bDeleteBuffer (void*, char* buffer)
180 // {
181 // delete [] buffer;
182 // }
183 //
184 // void someFunc()
185 // {
186 // // Open the filebuf.
187 // BucketFile file(...);
188 // file.open();
189 // uInt i;
190 // // Create a cache for the part of the file starting at offset 512
191 // // consisting of 1000 buckets. The cache consists of 10 buckets.
192 // // Each bucket is 32768 bytes.
193 // BucketCache cache (&file, 512, 32768, 1000, 10, 0,
194 // bToLocal, bFromLocal, bAddBuffer, bDeleteBuffer);
195 // // Write all buckets into the file.
196 // for (i=0; i<100; i++) {
197 // char* buf = new char[32768];
198 // cache.addBucket (buf);
199 // }
200 // Flush the cache to write all buckets in it.
201 // cache.flush();
202 // // Read all buckets from the file.
203 // for (i=0; i<1000; i++) {
204 // char* buf = cache.getBucket(i);
205 // ...
206 // }
207 // cout << cache.nBucket() << endl;
208 // }
209 // </srcblock>
210 // </example>
211 
212 // <todo asof="$DATE:$">
213 // <li> When ready, use HashMap for the internal maps.
214 // </todo>
215 
216 
218 {
219 public:
220 
221  // Create the cache for (a part of) a file.
222  // The file part used starts at startOffset. Its length is
223  // bucketSize*nrOfBuckets bytes.
224  // When the file is smaller, the remainder is indicated as an extension
225  // similarly to the behaviour of function extend.
226  BucketCache (BucketFile* file, Int64 startOffset, uInt bucketSize,
227  uInt nrOfBuckets, uInt cacheSize,
228  void* ownerObject,
229  BucketCacheToLocal readCallBack,
230  BucketCacheFromLocal writeCallBack,
231  BucketCacheAddBuffer addCallBack,
232  BucketCacheDeleteBuffer deleteCallBack);
233 
235 
236  // Flush the cache from the given slot on.
237  // By default the entire cache is flushed.
238  // When the entire cache is flushed, possible remaining uninitialized
239  // buckets will be initialized first.
240  // A True status is returned when buckets had to be written.
241  Bool flush (uInt fromSlot = 0);
242 
243  // Clear the cache from the given slot on.
244  // By default the entire cache is cleared.
245  // It will remove the buckets in the cleared part.
246  // If wanted and needed, the buckets are flushed to the file
247  // before removing them.
248  // It can be used to enforce rereading buckets from the file.
249  void clear (uInt fromSlot = 0, Bool doFlush = True);
250 
251  // Resize the cache.
252  // When the cache gets smaller, the latter buckets are cached out.
253  // It does not take "least recently used" into account.
255 
256  // Resynchronize the object (after another process updated the file).
257  // It clears the cache (so all data will be reread) and sets
258  // the new sizes.
259  void resync (uInt nrBucket, uInt nrOfFreeBucket, Int firstFreeBucket);
260 
261  // Get the current nr of buckets in the file.
262  uInt nBucket() const;
263 
264  // Get the current cache size (in buckets).
265  uInt cacheSize() const;
266 
267  // Set the dirty bit for the current bucket.
268  void setDirty();
269 
270  // Make another bucket current.
271  // When no more cache slots are available, the one least recently
272  // used is flushed.
273  // The data in the bucket is converted using the ToLocal callback
274  // function. When the bucket does not exist yet in the file, it
275  // gets added and initialized using the AddBuffer callback function.
276  // A pointer to the data in converted format is returned.
277  char* getBucket (uInt bucketNr);
278 
279  // Extend the file with the given number of buckets.
280  // The buckets get initialized when they are acquired
281  // (using getBucket) for the first time.
282  void extend (uInt nrBucket);
283 
284  // Add a bucket to the file and make it the current one.
285  // When no more cache slots are available, the one least recently
286  // used is flushed.
287  // <br> When no free buckets are available, the file will be
288  // extended with one bucket. It returns the new bucket number.
289  // The buffer must have been allocated on the heap.
290  // It will get part of the cache; its contents are not copied.
291  // Thus the buffer should hereafter NOT be used for other purposes.
292  // It will be deleted later via the DeleteBuffer callback function.
293  // The data is copied into the bucket. A pointer to the data in
294  // local format is returned.
295  uInt addBucket (char* data);
296 
297  // Remove the current bucket; i.e. add it to the beginning of the
298  // free bucket list.
299  void removeBucket();
300 
301  // Get a part from the file outside the cached area.
302  // It is checked if that part is indeed outside the cached file area.
303  void get (char* buf, uInt length, Int64 offset);
304 
305  // Put a part from the file outside the cached area.
306  // It is checked if that part is indeed outside the cached file area.
307  void put (const char* buf, uInt length, Int64 offset);
308 
309  // Get the bucket number of the first free bucket.
310  // -1 = no free buckets.
311  Int firstFreeBucket() const;
312 
313  // Get the number of free buckets.
314  uInt nFreeBucket() const;
315 
316  // (Re)initialize the cache statistics.
318 
319  // Show the statistics.
320  void showStatistics (ostream& os) const;
321 
322 private:
323  // The file used.
325  // The owner object.
326  void* its_Owner;
327  // The read callback function.
328  BucketCacheToLocal its_ReadCallBack;
329  // The write callback function.
330  BucketCacheFromLocal its_WriteCallBack;
331  // The add bucket callback function.
332  BucketCacheAddBuffer its_InitCallBack;
333  // The delete callback function.
334  BucketCacheDeleteBuffer its_DeleteCallBack;
335  // The starting offsets of the buckets in the file.
337  // The bucket size.
339  // The current nr of buckets in the file.
341  // The new nr of buckets in the file (after extension).
343  // The size of the cache (i.e. #buckets fitting in it).
345  // The nr of slots used in the cache.
347  // The cache itself.
349  // The cache slot actually used.
351  // The slot numbers of the buckets in the cache (-1 = not in cache).
353  // The buckets in the cache.
355  // Determine if a block is dirty (i.e. changed) (1=dirty).
357  // Determine when a block is used for the last time.
359  // The Least Recently Used counter.
361  // The internal buffer.
362  char* its_Buffer;
363  // The number of free buckets.
365  // The first free bucket (-1 = no free buckets).
367  // The statistics.
372 
373 
374  // Copy constructor is not possible.
376 
377  // Assignment is not possible.
379 
380  // Set the LRU information for the current slot.
381  void setLRU();
382 
383  // Get a cache slot for the bucket.
384  void getSlot (uInt bucketNr);
385 
386  // Write a bucket.
387  void writeBucket (uInt slotNr);
388 
389  // Read a bucket.
390  void readBucket (uInt slotNr);
391 
392  // Initialize the bucket buffer.
393  // The uninitialized buckets before this bucket are also initialized.
394  // It returns a pointer to the buffer.
395  void initializeBuckets (uInt bucketNr);
396 
397  // Check if the offset of a non-cached part is correct.
398  void checkOffset (uInt length, Int64 offset) const;
399 };
400 
401 
402 
404  { return its_CacheSize; }
405 
407  { return its_FirstFree; }
408 
410  { return its_NrOfFree; }
411 
412 
413 
414 
415 } //# NAMESPACE CASACORE - END
416 
417 #endif
Cache for buckets in a part of a file.
Definition: BucketCache.h:218
uInt its_ActualSlot
The cache slot actually used.
Definition: BucketCache.h:350
Int64 its_StartOffset
The starting offsets of the buckets in the file.
Definition: BucketCache.h:336
BucketCache & operator=(const BucketCache &)
Assignment is not possible.
uInt its_LRUCounter
The Least Recently Used counter.
Definition: BucketCache.h:360
BucketCache(BucketFile *file, Int64 startOffset, uInt bucketSize, uInt nrOfBuckets, uInt cacheSize, void *ownerObject, BucketCacheToLocal readCallBack, BucketCacheFromLocal writeCallBack, BucketCacheAddBuffer addCallBack, BucketCacheDeleteBuffer deleteCallBack)
Create the cache for (a part of) a file.
Block< uInt > its_LRU
Determine when a block is used for the last time.
Definition: BucketCache.h:358
Block< uInt > its_Dirty
Determine if a block is dirty (i.e.
Definition: BucketCache.h:356
void initializeBuckets(uInt bucketNr)
Initialize the bucket buffer.
void setDirty()
Set the dirty bit for the current bucket.
Bool flush(uInt fromSlot=0)
Flush the cache from the given slot on.
uInt its_NrOfFree
The number of free buckets.
Definition: BucketCache.h:364
Block< uInt > its_BucketNr
The buckets in the cache.
Definition: BucketCache.h:354
void resize(uInt cacheSize)
Resize the cache.
char * getBucket(uInt bucketNr)
Make another bucket current.
void removeBucket()
Remove the current bucket; i.e.
uInt naccess_p
The statistics.
Definition: BucketCache.h:368
uInt nBucket() const
Get the current nr of buckets in the file.
void resync(uInt nrBucket, uInt nrOfFreeBucket, Int firstFreeBucket)
Resynchronize the object (after another process updated the file).
void showStatistics(ostream &os) const
Show the statistics.
void writeBucket(uInt slotNr)
Write a bucket.
BucketCacheAddBuffer its_InitCallBack
The add bucket callback function.
Definition: BucketCache.h:332
uInt its_NewNrOfBuckets
The new nr of buckets in the file (after extension).
Definition: BucketCache.h:342
Int firstFreeBucket() const
Get the bucket number of the first free bucket.
Definition: BucketCache.h:406
void readBucket(uInt slotNr)
Read a bucket.
BucketCache(const BucketCache &)
Copy constructor is not possible.
void initStatistics()
(Re)initialize the cache statistics.
BucketCacheDeleteBuffer its_DeleteCallBack
The delete callback function.
Definition: BucketCache.h:334
void extend(uInt nrBucket)
Extend the file with the given number of buckets.
BucketCacheToLocal its_ReadCallBack
The read callback function.
Definition: BucketCache.h:328
Block< Int > its_SlotNr
The slot numbers of the buckets in the cache (-1 = not in cache).
Definition: BucketCache.h:352
uInt cacheSize() const
Get the current cache size (in buckets).
Definition: BucketCache.h:403
void checkOffset(uInt length, Int64 offset) const
Check if the offset of a non-cached part is correct.
uInt its_CacheSize
The size of the cache (i.e.
Definition: BucketCache.h:344
BucketCacheFromLocal its_WriteCallBack
The write callback function.
Definition: BucketCache.h:330
PtrBlock< char * > its_Cache
The cache itself.
Definition: BucketCache.h:348
void setLRU()
Set the LRU information for the current slot.
void * its_Owner
The owner object.
Definition: BucketCache.h:326
char * its_Buffer
The internal buffer.
Definition: BucketCache.h:362
Int its_FirstFree
The first free bucket (-1 = no free buckets).
Definition: BucketCache.h:366
uInt its_CurNrOfBuckets
The current nr of buckets in the file.
Definition: BucketCache.h:340
uInt nFreeBucket() const
Get the number of free buckets.
Definition: BucketCache.h:409
void clear(uInt fromSlot=0, Bool doFlush=True)
Clear the cache from the given slot on.
void getSlot(uInt bucketNr)
Get a cache slot for the bucket.
BucketFile * its_file
The file used.
Definition: BucketCache.h:324
uInt addBucket(char *data)
Add a bucket to the file and make it the current one.
uInt its_BucketSize
The bucket size.
Definition: BucketCache.h:338
void put(const char *buf, uInt length, Int64 offset)
Put a part from the file outside the cached area.
uInt its_CacheSizeUsed
The nr of slots used in the cache.
Definition: BucketCache.h:346
void get(char *buf, uInt length, Int64 offset)
Get a part from the file outside the cached area.
this file contains all the compiler specific defines
Definition: mainpage.dox:28
unsigned int uInt
Definition: aipstype.h:51
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
LatticeExprNode length(const LatticeExprNode &expr, const LatticeExprNode &axis)
2-argument function to get the length of an axis.
int Int
Definition: aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
const Bool True
Definition: aipstype.h:43