casacore
ISMBase.h
Go to the documentation of this file.
1 //# ISMBase.h: Base class of the Incremental Storage Manager
2 //# Copyright (C) 1996,1997,1999,2000,2001,2002
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_ISMBASE_H
29 #define TABLES_ISMBASE_H
30 
31 
32 //# Includes
33 #include <casacore/casa/aips.h>
34 #include <casacore/tables/DataMan/DataManager.h>
35 #include <casacore/casa/Containers/Block.h>
36 #include <casacore/casa/iosfwd.h>
37 
38 namespace casacore { //# NAMESPACE CASACORE - BEGIN
39 
40 //# Forward declarations
41 class BucketCache;
42 class BucketFile;
43 class ISMBucket;
44 class ISMIndex;
45 class ISMColumn;
46 class StManArrayFile;
47 
48 // <summary>
49 // Base class of the Incremental Storage Manager
50 // </summary>
51 
52 // <use visibility=local>
53 
54 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tIncrementalStMan.cc">
55 // </reviewed>
56 
57 // <prerequisite>
58 //# Classes you should understand before using this one.
59 // <li> <linkto class=IncrementalStMan>IncrementalStMan</linkto>
60 // <li> <linkto class=ISMColumn>ISMColumn</linkto>
61 // </prerequisite>
62 
63 // <etymology>
64 // ISMBase is the base class of the Incremental Storage Manager.
65 // </etymology>
66 
67 // <synopsis>
68 // The behaviour of this class is described in
69 // <linkto class="IncrementalStMan:description">IncrementalStMan</linkto>.
70 
71 // <motivation>
72 // The public interface of ISMBase is quite large, because the other
73 // internal ISM classes need these functions. To have a class with a
74 // minimal interface for the normal user, class <src>IncrementalStMan</src>
75 // is derived from it.
76 // <br>IncrementalStMan needs an isA- instead of hasA-relation to be
77 // able to bind columns to it in class <linkto class=SetupNewTable>
78 // SetupNewTable</linkto>.
79 // </motivation>
80 
81 // <todo asof="$DATE:$">
82 //# A List of bugs, limitations, extensions or planned refinements.
83 // <li> Removed AipsIO argument from open and close.
84 // </todo>
85 
86 
87 class ISMBase: public DataManager
88 {
89 public:
90  // Create an incremental storage manager without a name.
91  // The bucket size has to be given in bytes and the cache size in buckets.
92  // The bucket size is checked or calculated (if 0) as described in
93  // IncrementalStMan.h.
94  explicit ISMBase (uInt bucketSize = 0, Bool checkBucketSize = True,
95  uInt cacheSize = 1);
96 
97  // Create an incremental storage manager with the given name.
98  // The bucket size has to be given in bytes and the cache size in buckets.
99  // The bucket size is checked or calculated (if 0) as described in
100  // IncrementalStMan.h.
102  uInt bucketSize, Bool checkBucketSize, uInt cacheSize);
103 
104  // Create an incremental storage manager with the given name.
105  // The specifications are in the record (as created by dataManagerSpec).
106  ISMBase (const String& aDataManName,
107  const Record& spec);
108 
110 
111  // Clone this object.
112  // It does not clone ISMColumn objects possibly used.
113  // The caller has to delete the newly created object.
114  virtual DataManager* clone() const;
115 
116  // Get the type name of the data manager (i.e. IncrementalStMan).
117  virtual String dataManagerType() const;
118 
119  // Get the name given to the storage manager (in the constructor).
120  virtual String dataManagerName() const;
121 
122  // Record a record containing data manager specifications.
123  virtual Record dataManagerSpec() const;
124 
125  // Get data manager properties that can be modified.
126  // It is only ActualCacheSize (the actual cache size in buckets).
127  // It is a subset of the data manager specification.
128  virtual Record getProperties() const;
129 
130  // Modify data manager properties.
131  // Only MaxCacheSize can be used. It is similar to function setCacheSize
132  // with <src>canExceedNrBuckets=False</src>.
133  virtual void setProperties (const Record& spec);
134 
135  // Get the version of the class.
136  uInt version() const;
137 
138  // Set the cache size (in buckets).
139  // If <src>canExceedNrBuckets=True</src>, the given cache size can be
140  // larger than the nr of buckets in the file. In this way the cache can
141  // be made large enough for a future file extnsion.
142  // Otherwise, it is limited to the actual number of buckets. This is useful
143  // if one wants the entire file to be cached.
144  void setCacheSize (uInt cacheSize, Bool canExceedNrBuckets);
145 
146  // Get the current cache size (in buckets).
147  uInt cacheSize() const;
148 
149  // Clear the cache used by this storage manager.
150  // It will flush the cache as needed and remove all buckets from it.
151  void clearCache();
152 
153  // Show the statistics of all caches used.
154  virtual void showCacheStatistics (ostream& os) const;
155 
156  // Show the index statistics.
157  void showIndexStatistics (ostream& os);
158 
159  // Show the layout of the buckets
160  void showBucketLayout (ostream& os);
161 
162  // Get the bucket size (in bytes).
163  uInt bucketSize() const;
164 
165  // Get the size of a uInt in external format (can be canonical or local).
166  uInt uIntSize() const;
167 
168  // Get the size of a rownr in external format (can be canonical or local).
169  uInt rownrSize() const;
170 
171  // Get the bucket containing the given row.
172  // Also return the first and last row of that bucket.
173  // The bucket object is created and deleted by the caching mechanism.
174  ISMBucket* getBucket (rownr_t rownr, rownr_t& bucketStartRow,
175  rownr_t& bucketNrrow);
176 
177  // Get the next bucket.
178  // cursor=0 indicates the start of the iteration.
179  // The first bucket returned is the bucket containing the rownr
180  // given in bucketStartRow.
181  // After each iteration BucketStartRow and bucketNrrow are set.
182  // A 0 is returned when no more buckets.
183  // The bucket object is created and deleted by the caching mechanism.
184  ISMBucket* nextBucket (uInt& cursor, rownr_t& bucketStartRow,
185  rownr_t& bucketNrrow);
186 
187  // Get access to the temporary buffer.
188  char* tempBuffer() const;
189 
190  // Get a unique column number for the column
191  // (it is only unique for this storage manager).
192  // This is used by ISMColumnIndArr to create a unique file name.
193  uInt uniqueNr();
194 
195  // Get the number of rows in this storage manager.
196  rownr_t nrow() const;
197 
198  // Can the storage manager add rows? (yes)
199  virtual Bool canAddRow() const;
200 
201  // Can the storage manager delete rows? (yes)
202  virtual Bool canRemoveRow() const;
203 
204  // Can the storage manager add columns? (not yet)
205  virtual Bool canAddColumn() const;
206 
207  // Can the storage manager delete columns? (not yet)
208  virtual Bool canRemoveColumn() const;
209 
210  // Make the object from the type name string.
211  // This function gets registered in the DataManager "constructor" map.
212  // The caller has to delete the object.
214  const Record& spec);
215 
216  // Get access to the given column.
217  ISMColumn& getColumn (uInt colnr);
218 
219  // Add a bucket to the storage manager (i.e. to the cache).
220  // The pointer is taken over.
221  void addBucket (rownr_t rownr, ISMBucket* bucket);
222 
223  // Make the current bucket in the cache dirty (i.e. something has been
224  // changed in it and it needs to be written when removed from the cache).
225  // (used by ISMColumn::putValue).
227 
228  // Open (if needed) the file for indirect arrays with the given mode.
229  // Return a pointer to the object.
231 
232  // Check that there are no repeated rowIds in the buckets comprising this ISM.
233  Bool checkBucketLayout (uInt& offendingCursor,
234  rownr_t& offendingBucketStartRow,
235  uInt& offendingBucketNrow,
236  uInt& offendingBucketNr,
237  uInt& offendingCol,
238  uInt& ffendingIndex,
239  rownr_t& offendingRow,
240  rownr_t& offendingPrevRow);
241 
242 private:
243  // Copy constructor (only meant for clone function).
244  ISMBase (const ISMBase& that);
245 
246  // Assignment cannot be used.
247  ISMBase& operator= (const ISMBase& that);
248 
249  // (Re)create the index, file, and cache object.
250  void recreate();
251 
252  // The data manager supports use of MultiFile.
253  virtual Bool hasMultiFileSupport() const;
254 
255  // Flush and optionally fsync the data.
256  // It returns a True status if it had to flush (i.e. if data have changed).
257  virtual Bool flush (AipsIO&, Bool fsync);
258 
259  // Let the storage manager create files as needed for a new table.
260  // This allows a column with an indirect array to create its file.
261  virtual void create64 (rownr_t nrrow);
262 
263  // Open the storage manager file for an existing table, read in
264  // the data, and let the ISMColumn objects read their data.
265  virtual rownr_t open64 (rownr_t nrrow, AipsIO&);
266 
267  // Resync the storage manager with the new file contents.
268  // This is done by clearing the cache.
269  virtual rownr_t resync64 (rownr_t nrrow);
270 
271  // Reopen the storage manager files for read/write.
272  virtual void reopenRW();
273 
274  // The data manager will be deleted (because all its columns are
275  // requested to be deleted).
276  // So clean up the things needed (e.g. delete files).
277  virtual void deleteManager();
278 
279  // Let the storage manager initialize itself.
280  // It is used by create and open.
281  void init();
282 
283  // Add rows to the storage manager.
284  // Per column it extends the interval for which the last value written
285  // is valid.
286  virtual void addRow64 (rownr_t nrrow);
287 
288  // Delete a row from all columns.
289  virtual void removeRow64 (rownr_t rownr);
290 
291  // Do the final addition of a column.
292  // The <src>DataManagerColumn</src> object has already been created
293  // (by the <src>makeXXColumn</src> function) and added to
294  // <src>colSet_p</src>. However, it still has to be added to the
295  // data files, which is done by this function. It uses the
296  // pointer to find the correct column in the <src>colSet_p</src>.
297  virtual void addColumn (DataManagerColumn*);
298 
299  // Remove a column from the data file and the <src>colSet_p</src>.
300  // The <src>DataManagerColumn</src> object gets deleted..
302 
303  // Create a column in the storage manager on behalf of a table column.
304  // The caller has to delete the newly created object.
305  // <group>
306  // Create a scalar column.
308  int dataType,
309  const String& dataTypeID);
310  // Create a direct array column.
312  int dataType,
313  const String& dataTypeID);
314  // Create an indirect array column.
316  int dataType,
317  const String& dataTypeID);
318  // </group>
319 
320  // Get the cache object.
321  // This will construct the cache object if not present yet.
322  // The cache object will be deleted by the destructor.
324 
325  // Get the index object.
326  // This will construct the index object if not present yet.
327  // The index object will be deleted by the destructor.
328  ISMIndex& getIndex();
329 
330  // Construct the cache object (if not constructed yet).
331  void makeCache();
332 
333  // Construct the index object (if not constructed yet) and read it.
334  void makeIndex();
335 
336  // Read the index (at the end of the file).
337  void readIndex();
338 
339  // Write the index (at the end of the file).
340  void writeIndex();
341 
342 
343  //# Declare member variables.
344  // Name of data manager.
346  // The version of the class.
348  // The file containing the indirect arrays.
350  // Unique nr for column in this storage manager.
352  // The number of rows in the columns.
354  // The assembly of all columns.
356  // The cache with the ISM buckets.
358  // The file containing all data.
360  // The ISM bucket index.
362  // The persistent cache size.
364  // The actual cache size.
366  // The initial number of buckets in the cache.
368  // The nr of free buckets.
370  // The first free bucket.
372  // The bucket size.
374  // Check a positive bucketsize?
376  // Has the data changed since the last flush?
378  // The size of a uInt in external format (local or canonical).
380  // The size of a rownr in external format (local or canonical).
382  // A temporary read/write buffer (also for other classes).
384 };
385 
386 
387 inline uInt ISMBase::version() const
388 {
389  return version_p;
390 }
391 
392 inline uInt ISMBase::cacheSize() const
393 {
394  return cacheSize_p;
395 }
396 
398 {
399  return uniqnr_p++;
400 }
401 
402 inline rownr_t ISMBase::nrow() const
403 {
404  return nrrow_p;
405 }
406 
407 inline uInt ISMBase::bucketSize() const
408 {
409  return bucketSize_p;
410 }
411 
412 inline uInt ISMBase::uIntSize() const
413 {
414  return uIntSize_p;
415 }
416 
417 inline uInt ISMBase::rownrSize() const
418 {
419  return rownrSize_p;
420 }
421 
422 inline char* ISMBase::tempBuffer() const
423 {
424  return tempBuffer_p;
425 }
426 
428 {
429  if (cache_p == 0) {
430  makeCache();
431  }
432  return *cache_p;
433 }
434 
436 {
437  if (index_p == 0) {
438  makeIndex();
439  }
440  return *index_p;
441 }
442 
444 {
445  return *(colSet_p[colnr]);
446 }
447 
448 
449 
450 } //# NAMESPACE CASACORE - END
451 
452 #endif
Cache for buckets in a part of a file.
Definition: BucketCache.h:218
OpenOption
Define the possible ByteIO open options.
Definition: ByteIO.h:65
Abstract base class for a data manager.
Definition: DataManager.h:221
rownr_t nrrow_p
The number of rows in the columns.
Definition: ISMBase.h:353
virtual Record getProperties() const
Get data manager properties that can be modified.
virtual void create64(rownr_t nrrow)
Let the storage manager create files as needed for a new table.
PtrBlock< ISMColumn * > colSet_p
The assembly of all columns.
Definition: ISMBase.h:355
void readIndex()
Read the index (at the end of the file).
virtual void setProperties(const Record &spec)
Modify data manager properties.
ISMBase(const ISMBase &that)
Copy constructor (only meant for clone function).
virtual rownr_t open64(rownr_t nrrow, AipsIO &)
Open the storage manager file for an existing table, read in the data, and let the ISMColumn objects ...
virtual void addColumn(DataManagerColumn *)
Do the final addition of a column.
char * tempBuffer() const
Get access to the temporary buffer.
Definition: ISMBase.h:422
char * tempBuffer_p
A temporary read/write buffer (also for other classes).
Definition: ISMBase.h:383
uInt uniqnr_p
Unique nr for column in this storage manager.
Definition: ISMBase.h:351
ISMIndex * index_p
The ISM bucket index.
Definition: ISMBase.h:361
virtual Bool flush(AipsIO &, Bool fsync)
Flush and optionally fsync the data.
Bool checkBucketLayout(uInt &offendingCursor, rownr_t &offendingBucketStartRow, uInt &offendingBucketNrow, uInt &offendingBucketNr, uInt &offendingCol, uInt &ffendingIndex, rownr_t &offendingRow, rownr_t &offendingPrevRow)
Check that there are no repeated rowIds in the buckets comprising this ISM.
uInt bucketSize_p
The bucket size.
Definition: ISMBase.h:373
ISMBase & operator=(const ISMBase &that)
Assignment cannot be used.
virtual Bool canAddRow() const
Can the storage manager add rows? (yes)
uInt bucketSize() const
Get the bucket size (in bytes).
Definition: ISMBase.h:407
rownr_t nrow() const
Get the number of rows in this storage manager.
Definition: ISMBase.h:402
virtual String dataManagerName() const
Get the name given to the storage manager (in the constructor).
ISMColumn & getColumn(uInt colnr)
Get access to the given column.
Definition: ISMBase.h:443
ISMIndex & getIndex()
Get the index object.
Definition: ISMBase.h:435
void setCacheSize(uInt cacheSize, Bool canExceedNrBuckets)
Set the cache size (in buckets).
void showIndexStatistics(ostream &os)
Show the index statistics.
void setBucketDirty()
Make the current bucket in the cache dirty (i.e.
uInt rownrSize() const
Get the size of a rownr in external format (can be canonical or local).
Definition: ISMBase.h:417
virtual String dataManagerType() const
Get the type name of the data manager (i.e.
uInt nbucketInit_p
The initial number of buckets in the cache.
Definition: ISMBase.h:367
uInt persCacheSize_p
The persistent cache size.
Definition: ISMBase.h:363
virtual Record dataManagerSpec() const
Record a record containing data manager specifications.
uInt version_p
The version of the class.
Definition: ISMBase.h:347
virtual void reopenRW()
Reopen the storage manager files for read/write.
void recreate()
(Re)create the index, file, and cache object.
Bool checkBucketSize_p
Check a positive bucketsize?
Definition: ISMBase.h:375
virtual void removeColumn(DataManagerColumn *)
Remove a column from the data file and the colSet_p.
StManArrayFile * iosfile_p
The file containing the indirect arrays.
Definition: ISMBase.h:349
void init()
Let the storage manager initialize itself.
BucketCache & getCache()
Get the cache object.
Definition: ISMBase.h:427
virtual void showCacheStatistics(ostream &os) const
Show the statistics of all caches used.
uInt uniqueNr()
Get a unique column number for the column (it is only unique for this storage manager).
Definition: ISMBase.h:397
virtual DataManagerColumn * makeDirArrColumn(const String &name, int dataType, const String &dataTypeID)
Create a direct array column.
void makeIndex()
Construct the index object (if not constructed yet) and read it.
uInt nFreeBucket_p
The nr of free buckets.
Definition: ISMBase.h:369
uInt uIntSize_p
The size of a uInt in external format (local or canonical).
Definition: ISMBase.h:379
Bool dataChanged_p
Has the data changed since the last flush?
Definition: ISMBase.h:377
String dataManName_p
Name of data manager.
Definition: ISMBase.h:345
virtual rownr_t resync64(rownr_t nrrow)
Resync the storage manager with the new file contents.
virtual Bool canRemoveColumn() const
Can the storage manager delete columns? (not yet)
virtual DataManagerColumn * makeScalarColumn(const String &name, int dataType, const String &dataTypeID)
Create a column in the storage manager on behalf of a table column.
ISMBase(const String &aDataManName, const Record &spec)
Create an incremental storage manager with the given name.
uInt rownrSize_p
The size of a rownr in external format (local or canonical).
Definition: ISMBase.h:381
virtual DataManager * clone() const
Clone this object.
virtual void deleteManager()
The data manager will be deleted (because all its columns are requested to be deleted).
uInt cacheSize_p
The actual cache size.
Definition: ISMBase.h:365
void clearCache()
Clear the cache used by this storage manager.
uInt version() const
Get the version of the class.
Definition: ISMBase.h:387
void addBucket(rownr_t rownr, ISMBucket *bucket)
Add a bucket to the storage manager (i.e.
virtual Bool hasMultiFileSupport() const
The data manager supports use of MultiFile.
uInt cacheSize() const
Get the current cache size (in buckets).
Definition: ISMBase.h:392
ISMBucket * getBucket(rownr_t rownr, rownr_t &bucketStartRow, rownr_t &bucketNrrow)
Get the bucket containing the given row.
void writeIndex()
Write the index (at the end of the file).
Int firstFree_p
The first free bucket.
Definition: ISMBase.h:371
ISMBucket * nextBucket(uInt &cursor, rownr_t &bucketStartRow, rownr_t &bucketNrrow)
Get the next bucket.
BucketFile * file_p
The file containing all data.
Definition: ISMBase.h:359
virtual Bool canAddColumn() const
Can the storage manager add columns? (not yet)
BucketCache * cache_p
The cache with the ISM buckets.
Definition: ISMBase.h:357
virtual DataManagerColumn * makeIndArrColumn(const String &name, int dataType, const String &dataTypeID)
Create an indirect array column.
static DataManager * makeObject(const String &dataManagerType, const Record &spec)
Make the object from the type name string.
ISMBase(const String &dataManagerName, uInt bucketSize, Bool checkBucketSize, uInt cacheSize)
Create an incremental storage manager with the given name.
void makeCache()
Construct the cache object (if not constructed yet).
uInt uIntSize() const
Get the size of a uInt in external format (can be canonical or local).
Definition: ISMBase.h:412
virtual void addRow64(rownr_t nrrow)
Add rows to the storage manager.
ISMBase(uInt bucketSize=0, Bool checkBucketSize=True, uInt cacheSize=1)
Create an incremental storage manager without a name.
StManArrayFile * openArrayFile(ByteIO::OpenOption opt)
Open (if needed) the file for indirect arrays with the given mode.
virtual Bool canRemoveRow() const
Can the storage manager delete rows? (yes)
void showBucketLayout(ostream &os)
Show the layout of the buckets.
virtual void removeRow64(rownr_t rownr)
Delete a row from all columns.
A drop-in replacement for Block<T*>.
Definition: Block.h:814
String: the storage and methods of handling collections of characters.
Definition: String.h:225
this file contains all the compiler specific defines
Definition: mainpage.dox:28
unsigned int uInt
Definition: aipstype.h:51
int Int
Definition: aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
const Bool True
Definition: aipstype.h:43
uInt64 rownr_t
Define the type of a row number in a table.
Definition: aipsxtype.h:46