casacore
ISMColumn.h
Go to the documentation of this file.
1 //# ISMColumn.h: A Column in the Incremental Storage Manager
2 //# Copyright (C) 1996,1997,1998,1999,2002
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_ISMCOLUMN_H
29 #define TABLES_ISMCOLUMN_H
30 
31 
32 //# Includes
33 #include <casacore/casa/aips.h>
34 #include <casacore/tables/DataMan/StManColumnBase.h>
35 #include <casacore/tables/DataMan/ISMBase.h>
36 #include <casacore/casa/Arrays/IPosition.h>
37 #include <casacore/casa/Containers/Block.h>
38 #include <casacore/casa/Utilities/Compare.h>
39 #include <casacore/casa/OS/Conversion.h>
40 
41 namespace casacore { //# NAMESPACE CASACORE - BEGIN
42 
43 //# Forward declarations
44 class ISMBucket;
45 
46 
47 // <summary>
48 // A Column in the Incremental Storage Manager.
49 // </summary>
50 
51 // <use visibility=local>
52 
53 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
54 // </reviewed>
55 
56 // <prerequisite>
57 //# Classes you should understand before using this one.
58 // <li> <linkto class=ISMBase>ISMBase</linkto>
59 // </prerequisite>
60 
61 // <etymology>
62 // ISMColumn represents a Column in the Incremental Storage Manager.
63 // </etymology>
64 
65 // <synopsis>
66 // ISMColumn handles the access to a column containing scalars or direct
67 // arrays of the various data types. It uses class <linkto class=ISMBucket>
68 // ISMBucket</linkto> to get and put the data into the correct bucket.
69 // When the value does not fit in the bucket, the bucket is split
70 // and the new bucket is added to the storage manager.
71 // <p>
72 // The object maintains a variable indicating the last row ever put.
73 // This is used to decide if a put of a value is valid until the
74 // end of the table or for that one row only. In this way it does not
75 // make any difference if rows are added before or after a value is put
76 // <br>
77 // A value put before or at the last row ever put will only affect that
78 // one row. The rows before and after it keep their original value. If
79 // needed that value is copied.
80 // <p>
81 // To optimize (especially sequential) access to the column, ISMColumn
82 // maintains the last value gotten and the rows for which it is valid.
83 // In this way a get does not need to access the data in the bucket.
84 // <p>
85 // ISMColumn use the static conversion functions in the
86 // <linkto class=Conversion>Conversion</linkto> framework to
87 // get/put the data in external format (be it canonical or local).
88 // Most data types are fixed length, but some are variable length
89 // (e.g. String). In external format variable length data is preceeded
90 // by its total length (which includes the length itself). This makes
91 // it possible to get the length of a data value without having to
92 // interpret it, which is easy when (re)moving a value. For this reason
93 // ISMColumn contains its own conversion functions for Strings.
94 // <p>
95 // ISMColumn also acts as the base class for more specialized ISM
96 // column classes (i.e. <linkto class=ISMIndColumn>ISMIndColumn</linkto>
97 // for indirect columns).
98 // In this way <linkto class=ISMBase>ISMBase</linkto> can hold a
99 // block of <src>ISMColumn*</src> for any column. Furthermore
100 // <src>ISMColumn</src> contains the hooks to allow a derived class
101 // to use other ISMColumn functions (e.g. there are "action" functions
102 // for a derived class to react on the duplication or removal of
103 // a data value (e.g. due to a bucket split).
104 // </synopsis>
105 
106 // <motivation>
107 // ISMColumn encapsulates all operations on an ISM Column.
108 // </motivation>
109 
110 //# <todo asof="$DATE:$">
111 //# A List of bugs, limitations, extensions or planned refinements.
112 //# </todo>
113 
114 
116 {
117 public:
118  // Create a ISMColumn object with the given parent.
119  // It initializes the various variables.
120  // It keeps the pointer to its parent (but does not own it).
121  ISMColumn (ISMBase* parent, int dataType, uInt colnr);
122 
124 
125  // Set the shape of an array in the column.
126  virtual void setShapeColumn (const IPosition& shape);
127 
128  // Get the dimensionality of the item in the given row.
129  // This is the same for all rows.
130  virtual uInt ndim (rownr_t rownr);
131 
132  // Get the shape of the array in the given row.
133  // This is the same for all rows.
134  virtual IPosition shape (rownr_t rownr);
135 
136  // Let the column object initialize itself for a newly created table.
137  // This is meant for a derived class.
138  virtual void doCreate (ISMBucket*);
139 
140  // Let the column object initialize itself for an existing table.
141  virtual void getFile (rownr_t nrrow);
142 
143  // Flush and optionally fsync the data.
144  // This is meant for a derived class.
145  virtual Bool flush (rownr_t nrrow, Bool fsync);
146 
147  // Resync the storage manager with the new file contents.
148  // It resets the last rownr put.
149  void resync (rownr_t nrrow);
150 
151  // Let the column reopen its data files for read/write access.
152  virtual void reopenRW();
153 
154  // Get a scalar value in the given row.
155  // <group>
156  virtual void getBool (rownr_t rownr, Bool* dataPtr);
157  virtual void getuChar (rownr_t rownr, uChar* dataPtr);
158  virtual void getShort (rownr_t rownr, Short* dataPtr);
159  virtual void getuShort (rownr_t rownr, uShort* dataPtr);
160  virtual void getInt (rownr_t rownr, Int* dataPtr);
161  virtual void getuInt (rownr_t rownr, uInt* dataPtr);
162  virtual void getInt64 (rownr_t rownr, Int64* dataPtr);
163  virtual void getfloat (rownr_t rownr, float* dataPtr);
164  virtual void getdouble (rownr_t rownr, double* dataPtr);
165  virtual void getComplex (rownr_t rownr, Complex* dataPtr);
166  virtual void getDComplex (rownr_t rownr, DComplex* dataPtr);
167  virtual void getString (rownr_t rownr, String* dataPtr);
168  // </group>
169 
170  // Put a scalar value in the given row.
171  // <group>
172  virtual void putBool (rownr_t rownr, const Bool* dataPtr);
173  virtual void putuChar (rownr_t rownr, const uChar* dataPtr);
174  virtual void putShort (rownr_t rownr, const Short* dataPtr);
175  virtual void putuShort (rownr_t rownr, const uShort* dataPtr);
176  virtual void putInt (rownr_t rownr, const Int* dataPtr);
177  virtual void putuInt (rownr_t rownr, const uInt* dataPtr);
178  virtual void putInt64 (rownr_t rownr, const Int64* dataPtr);
179  virtual void putfloat (rownr_t rownr, const float* dataPtr);
180  virtual void putdouble (rownr_t rownr, const double* dataPtr);
181  virtual void putComplex (rownr_t rownr, const Complex* dataPtr);
182  virtual void putDComplex (rownr_t rownr, const DComplex* dataPtr);
183  virtual void putString (rownr_t rownr, const String* dataPtr);
184  // </group>
185 
186  // Get the scalar values in the entire column.
187  // The buffer pointed to by dataPtr has to have the correct length.
188  // (which is guaranteed by the ScalarColumn getColumn function).
189  virtual void getScalarColumnV (ArrayBase& dataPtr);
190 
191  // Put the scalar values into the entire column.
192  // The buffer pointed to by dataPtr has to have the correct length.
193  // (which is guaranteed by the ScalarColumn putColumn function).
194  virtual void putScalarColumnV (const ArrayBase& dataPtr);
195 
196  // Get the scalar values in some cells of the column.
197  // The buffer pointed to by dataPtr has to have the correct length.
198  // (which is guaranteed by the ScalarColumn getColumnCells function).
199  virtual void getScalarColumnCellsV (const RefRows& rownrs,
200  ArrayBase& dataPtr);
201 
202  // Get an array value in the given row.
203  virtual void getArrayV (rownr_t rownr, ArrayBase& dataPtr);
204 
205  // Put an array value in the given row.
206  virtual void putArrayV (rownr_t rownr, const ArrayBase& dataPtr);
207 
208  // Add (newNrrow-oldNrrow) rows to the column and initialize
209  // the new rows when needed.
210  virtual void addRow (rownr_t newNrrow, rownr_t oldNrrow);
211 
212  // Remove the given row in the bucket from the column.
213  void remove (rownr_t bucketRownr, ISMBucket* bucket, rownr_t bucketNrrow,
214  rownr_t newNrrow);
215 
216  // Get the function needed to read/write a uInt and rownr from/to
217  // external format. This is used by other classes to read the length
218  // of a variable data value.
219  // <group>
224  // </group>
225 
226  // Give a derived class the opportunity to react on the duplication
227  // of a value. It is used by ISMIndColumn.
228  virtual void handleCopy (rownr_t rownr, const char* value);
229 
230  // Give a derived class the opportunity to react on the removal
231  // of a value. It is used by ISMIndColumn.
232  virtual void handleRemove (rownr_t rownr, const char* value);
233 
234  // Get the fixed length of the data value in a cell of this column
235  // (0 = variable length).
236  uInt getFixedLength() const;
237 
238  // Get the nr of elements in this data value.
239  uInt nelements() const;
240 
241 
242 protected:
243  // Test if the last value is invalid for this row.
245 
246  // Get the value for this row.
247  // Set the cache if the flag is set.
248  void getValue (rownr_t rownr, void* value, Bool setCache);
249 
250  // Put the value for this row.
251  void putValue (rownr_t rownr, const void* value);
252 
253  //# Declare member variables.
254  // Pointer to the parent storage manager.
256  // Length of column cell value in storage format (0 = variable length).
257  // If 0, the value is always preceeded by a uInt giving the length.
259  // Column sequence number of this column.
261  // The shape of the column.
263  // Number of elements in a value for this column.
265  // Number of values to be copied.
266  // Normally this is nrelem_p, but for complex types it is 2*nrelem_p.
267  // When local format is used, it is the number of bytes.
269  // Cache for interval for which last value read is valid.
270  // The last value is valid for startRow_p till endRow_p (inclusive).
273  void* lastValue_p;
274  // The last row for which a value has been put.
276  // The size of the data type in local format.
278  // Pointer to a convert function for writing.
280  // Pointer to a convert function for reading.
282  // Pointer to a compare function.
283  ObjCompareFunc* compareFunc_p;
284 
285 
286 private:
287  // Forbid copy constructor.
289 
290  // Forbid assignment.
292 
293  // Initialize part of the object.
294  // It is used by doCreate and getFile.
295  void init();
296 
297  // Clear the object (used by destructor and init).
298  void clear();
299 
300  // Put the value in all buckets from the given row on.
301  void putFromRow (rownr_t rownr, const char* data, uInt lenData);
302 
303  // Put a data value into the bucket.
304  // When it is at the first row of the bucket, it replaces the value.
305  // Otherwise it is added.
306  void putData (ISMBucket* bucket, rownr_t bucketStartRow,
307  rownr_t bucketNrrow, rownr_t bucketRownr,
308  const char* data, uInt lenData,
309  Bool afterLastRow, Bool canSplit);
310 
311  // Replace a value at the given offset in the bucket.
312  // If the bucket is too small, it will be split (if allowed).
313  void replaceData (ISMBucket* bucket, rownr_t bucketStartRow,
314  rownr_t bucketNrrow, rownr_t bucketRownr, uInt& offset,
315  const char* data, uInt lenData, Bool canSplit = True);
316 
317  // Add a value at the given index in the bucket.
318  // If the bucket is too small, it will be split (if allowed).
319  Bool addData (ISMBucket* bucket, rownr_t bucketStartRow,
320  rownr_t bucketNrrow, rownr_t bucketRownr, uInt inx,
321  const char* data, uInt lenData,
322  Bool afterLastRow = False, Bool canSplit = True);
323 
324  // Handle the duplicated values after a bucket split.
325  void handleSplit (ISMBucket& bucket, const Block<Bool>& duplicated);
326 
327  // Compare the values.
328  virtual Bool compareValue (const void* val1, const void* val2) const;
329 
330  // Handle a String in copying to/from external format.
331  // <group>
332  static size_t fromString (void* out, const void* in, size_t n,
333  Conversion::ValueFunction* writeLeng);
334  static size_t toString (void* out, const void* in, size_t n,
335  Conversion::ValueFunction* readLeng);
336  static size_t writeStringBE (void* out, const void* in, size_t n);
337  static size_t readStringBE (void* out, const void* in, size_t n);
338  static size_t writeStringLE (void* out, const void* in, size_t n);
339  static size_t readStringLE (void* out, const void* in, size_t n);
340  // </group>
341 
354 
367 
368  void putScaCol (const Vector<Bool>&);
369  void putScaCol (const Vector<uChar>&);
370  void putScaCol (const Vector<Short>&);
371  void putScaCol (const Vector<uShort>&);
372  void putScaCol (const Vector<Int>&);
373  void putScaCol (const Vector<uInt>&);
374  void putScaCol (const Vector<Int64>&);
375  void putScaCol (const Vector<float>&);
376  void putScaCol (const Vector<double>&);
377  void putScaCol (const Vector<Complex>&);
379  void putScaCol (const Vector<String>&);
380 };
381 
382 
384 {
385  return rownr < startRow_p || rownr > endRow_p;
386 }
387 
389 {
390  return fixedLength_p;
391 }
392 
394 {
395  return nrelem_p;
396 }
397 
398 
399 
400 } //# NAMESPACE CASACORE - END
401 
402 #endif
Non-templated base class for templated Array class.
Definition: ArrayBase.h:73
size_t ValueFunction(void *to, const void *from, size_t nvalues)
Define the signature of a function converting nvalues values from internal to external format or vice...
Definition: Conversion.h:100
virtual void putDComplex(rownr_t rownr, const DComplex *dataPtr)
ObjCompareFunc * compareFunc_p
Pointer to a compare function.
Definition: ISMColumn.h:283
static size_t writeStringLE(void *out, const void *in, size_t n)
void resync(rownr_t nrrow)
Resync the storage manager with the new file contents.
void getScaCol(Vector< uInt > &)
IPosition shape_p
The shape of the column.
Definition: ISMColumn.h:262
virtual void getShort(rownr_t rownr, Short *dataPtr)
uInt getFixedLength() const
Get the fixed length of the data value in a cell of this column (0 = variable length).
Definition: ISMColumn.h:388
void getScaCol(Vector< Short > &)
virtual Bool compareValue(const void *val1, const void *val2) const
Compare the values.
void getScaColCells(const RefRows &, Vector< uInt > &)
static size_t writeStringBE(void *out, const void *in, size_t n)
static Conversion::ValueFunction * getReaduInt(Bool asCanonical)
Get the function needed to read/write a uInt and rownr from/to external format.
void getScaCol(Vector< uChar > &)
void getScaCol(Vector< Int > &)
void getScaCol(Vector< double > &)
void getScaColCells(const RefRows &, Vector< Complex > &)
ISMColumn(const ISMColumn &)
Forbid copy constructor.
virtual void putfloat(rownr_t rownr, const float *dataPtr)
virtual void putScalarColumnV(const ArrayBase &dataPtr)
Put the scalar values into the entire column.
void getScaCol(Vector< float > &)
void getScaCol(Vector< String > &)
void putScaCol(const Vector< Int > &)
void putValue(rownr_t rownr, const void *value)
Put the value for this row.
virtual void getArrayV(rownr_t rownr, ArrayBase &dataPtr)
Get an array value in the given row.
Conversion::ValueFunction * writeFunc_p
Pointer to a convert function for writing.
Definition: ISMColumn.h:279
virtual void handleRemove(rownr_t rownr, const char *value)
Give a derived class the opportunity to react on the removal of a value.
static Conversion::ValueFunction * getWriteuInt(Bool asCanonical)
ISMColumn(ISMBase *parent, int dataType, uInt colnr)
Create a ISMColumn object with the given parent.
virtual void putArrayV(rownr_t rownr, const ArrayBase &dataPtr)
Put an array value in the given row.
rownr_t lastRowPut_p
The last row for which a value has been put.
Definition: ISMColumn.h:275
virtual void getScalarColumnCellsV(const RefRows &rownrs, ArrayBase &dataPtr)
Get the scalar values in some cells of the column.
void getScaColCells(const RefRows &, Vector< Short > &)
void replaceData(ISMBucket *bucket, rownr_t bucketStartRow, rownr_t bucketNrrow, rownr_t bucketRownr, uInt &offset, const char *data, uInt lenData, Bool canSplit=True)
Replace a value at the given offset in the bucket.
virtual void putBool(rownr_t rownr, const Bool *dataPtr)
Put a scalar value in the given row.
void putScaCol(const Vector< Int64 > &)
void getScaCol(Vector< uShort > &)
virtual Bool flush(rownr_t nrrow, Bool fsync)
Flush and optionally fsync the data.
virtual void getScalarColumnV(ArrayBase &dataPtr)
Get the scalar values in the entire column.
void getScaColCells(const RefRows &, Vector< double > &)
virtual void putuShort(rownr_t rownr, const uShort *dataPtr)
virtual void getBool(rownr_t rownr, Bool *dataPtr)
Get a scalar value in the given row.
virtual void putShort(rownr_t rownr, const Short *dataPtr)
void putScaCol(const Vector< uShort > &)
void getScaCol(Vector< Int64 > &)
uInt nrelem_p
Number of elements in a value for this column.
Definition: ISMColumn.h:264
void putScaCol(const Vector< Bool > &)
void getScaColCells(const RefRows &, Vector< Bool > &)
void remove(rownr_t bucketRownr, ISMBucket *bucket, rownr_t bucketNrrow, rownr_t newNrrow)
Remove the given row in the bucket from the column.
void getScaCol(Vector< Complex > &)
void putScaCol(const Vector< DComplex > &)
void getScaColCells(const RefRows &, Vector< uChar > &)
virtual void getString(rownr_t rownr, String *dataPtr)
virtual void doCreate(ISMBucket *)
Let the column object initialize itself for a newly created table.
Bool addData(ISMBucket *bucket, rownr_t bucketStartRow, rownr_t bucketNrrow, rownr_t bucketRownr, uInt inx, const char *data, uInt lenData, Bool afterLastRow=False, Bool canSplit=True)
Add a value at the given index in the bucket.
static size_t fromString(void *out, const void *in, size_t n, Conversion::ValueFunction *writeLeng)
Handle a String in copying to/from external format.
virtual void getuShort(rownr_t rownr, uShort *dataPtr)
static size_t readStringLE(void *out, const void *in, size_t n)
ISMColumn & operator=(const ISMColumn &)
Forbid assignment.
uInt colnr_p
Column sequence number of this column.
Definition: ISMColumn.h:260
virtual void getComplex(rownr_t rownr, Complex *dataPtr)
virtual void getdouble(rownr_t rownr, double *dataPtr)
void getScaColCells(const RefRows &, Vector< float > &)
void putData(ISMBucket *bucket, rownr_t bucketStartRow, rownr_t bucketNrrow, rownr_t bucketRownr, const char *data, uInt lenData, Bool afterLastRow, Bool canSplit)
Put a data value into the bucket.
virtual void getuChar(rownr_t rownr, uChar *dataPtr)
static Conversion::ValueFunction * getWriteRownr(Bool asCanonical)
void handleSplit(ISMBucket &bucket, const Block< Bool > &duplicated)
Handle the duplicated values after a bucket split.
void getScaColCells(const RefRows &, Vector< String > &)
void putScaCol(const Vector< uChar > &)
void init()
Initialize part of the object.
void putScaCol(const Vector< float > &)
void putScaCol(const Vector< Short > &)
static Conversion::ValueFunction * getReadRownr(Bool asCanonical)
virtual void putInt(rownr_t rownr, const Int *dataPtr)
void getScaColCells(const RefRows &, Vector< DComplex > &)
void getScaColCells(const RefRows &, Vector< Int64 > &)
uInt nelements() const
Get the nr of elements in this data value.
Definition: ISMColumn.h:393
void putScaCol(const Vector< String > &)
void clear()
Clear the object (used by destructor and init).
virtual IPosition shape(rownr_t rownr)
Get the shape of the array in the given row.
rownr_t startRow_p
Cache for interval for which last value read is valid.
Definition: ISMColumn.h:271
virtual void getfloat(rownr_t rownr, float *dataPtr)
Conversion::ValueFunction * readFunc_p
Pointer to a convert function for reading.
Definition: ISMColumn.h:281
void putScaCol(const Vector< double > &)
virtual void putString(rownr_t rownr, const String *dataPtr)
virtual void getFile(rownr_t nrrow)
Let the column object initialize itself for an existing table.
static size_t toString(void *out, const void *in, size_t n, Conversion::ValueFunction *readLeng)
Bool isLastValueInvalid(rownr_t rownr)
Test if the last value is invalid for this row.
Definition: ISMColumn.h:383
ISMBase * stmanPtr_p
Pointer to the parent storage manager.
Definition: ISMColumn.h:255
void getValue(rownr_t rownr, void *value, Bool setCache)
Get the value for this row.
virtual void getuInt(rownr_t rownr, uInt *dataPtr)
uInt fixedLength_p
Length of column cell value in storage format (0 = variable length).
Definition: ISMColumn.h:258
virtual void setShapeColumn(const IPosition &shape)
Set the shape of an array in the column.
virtual void getInt(rownr_t rownr, Int *dataPtr)
virtual void putInt64(rownr_t rownr, const Int64 *dataPtr)
void getScaCol(Vector< DComplex > &)
virtual void putuChar(rownr_t rownr, const uChar *dataPtr)
virtual void reopenRW()
Let the column reopen its data files for read/write access.
void getScaColCells(const RefRows &, Vector< Int > &)
void putScaCol(const Vector< uInt > &)
void putFromRow(rownr_t rownr, const char *data, uInt lenData)
Put the value in all buckets from the given row on.
virtual void getInt64(rownr_t rownr, Int64 *dataPtr)
virtual uInt ndim(rownr_t rownr)
Get the dimensionality of the item in the given row.
virtual void getDComplex(rownr_t rownr, DComplex *dataPtr)
virtual void addRow(rownr_t newNrrow, rownr_t oldNrrow)
Add (newNrrow-oldNrrow) rows to the column and initialize the new rows when needed.
static size_t readStringBE(void *out, const void *in, size_t n)
void getScaColCells(const RefRows &, Vector< uShort > &)
void getScaCol(Vector< Bool > &)
uInt typeSize_p
The size of the data type in local format.
Definition: ISMColumn.h:277
uInt nrcopy_p
Number of values to be copied.
Definition: ISMColumn.h:268
virtual void putdouble(rownr_t rownr, const double *dataPtr)
virtual void putuInt(rownr_t rownr, const uInt *dataPtr)
void putScaCol(const Vector< Complex > &)
virtual void handleCopy(rownr_t rownr, const char *value)
Give a derived class the opportunity to react on the duplication of a value.
virtual void putComplex(rownr_t rownr, const Complex *dataPtr)
virtual int dataType() const
Return the data type of the column.
String: the storage and methods of handling collections of characters.
Definition: String.h:225
this file contains all the compiler specific defines
Definition: mainpage.dox:28
unsigned char uChar
Definition: aipstype.h:47
const Bool False
Definition: aipstype.h:44
short Short
Definition: aipstype.h:48
unsigned int uInt
Definition: aipstype.h:51
unsigned short uShort
Definition: aipstype.h:49
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
int Int
Definition: aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
LatticeExprNode value(const LatticeExprNode &expr)
This function returns the value of the expression without a mask.
const Bool True
Definition: aipstype.h:43
uInt64 rownr_t
Define the type of a row number in a table.
Definition: aipsxtype.h:46