casacore
TiledDataStMan.h
Go to the documentation of this file.
1 //# TiledDataStMan.h: Tiled Data Storage Manager
2 //# Copyright (C) 1995,1996,1997,1999,2001
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_TILEDDATASTMAN_H
29 #define TABLES_TILEDDATASTMAN_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
33 #include <casacore/tables/DataMan/TiledStMan.h>
34 #include <casacore/casa/Containers/Block.h>
35 #include <casacore/casa/BasicSL/String.h>
36 #include <vector>
37 
38 namespace casacore { //# NAMESPACE CASACORE - BEGIN
39 
40 //# Forward Declarations
41 
42 
43 // <summary>
44 // Tiled Data Storage Manager.
45 // </summary>
46 
47 // <use visibility=export>
48 
49 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
50 // </reviewed>
51 
52 // <prerequisite>
53 //# Classes you should understand before using this one.
54 // <li> <linkto class=TiledStMan>TiledStMan</linkto>
55 // <li> <linkto class=TSMCube>TSMCube</linkto>
56 // <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
57 // for a discussion of the maximum cache size
58 // <li> <linkto class=Record>Record</linkto>
59 // </prerequisite>
60 
61 // <etymology>
62 // TiledDataStMan is the Tiled Storage Manager for general
63 // data arrays.
64 // </etymology>
65 
66 // <synopsis>
67 // TiledDataStMan is a derivation from TiledStMan, the abstract
68 // tiled storage manager class. A description of the basics
69 // of tiled storage managers is given in the
70 // <linkto module=Tables:TiledStMan>Tables module</linkto> description.
71 // <p>
72 // TiledDataStMan allows the user explicit control over the
73 // definition and extension of hypercubes by means of the accessor
74 // class <linkto class=TiledDataStManAccessor>TiledDataStManAccessor</linkto>.
75 // The user can determine which row should be put in which hypercube,
76 // so it is possible to put row 0-9 in hypercube A, row 10-29 in B,
77 // row 30-39 in A again, etc.. This makes it possible to use a tiled
78 // storage manager for a data column containing data with
79 // different shapes (e.g. line and continuum data). Actually,
80 // this storage manager is developed for irregularly shaped
81 // UV-data, but can be used for any purpose.
82 // <br>
83 // Each extensible hypercube uses a file of its own. This means that there
84 // shouldn't be too many of them, otherwise the number of files may
85 // get too high.
86 // <p>
87 // The TiledDataStMan has the following (extra) properties:
88 // <ul>
89 // <li> When multiple hypercubes are used, one or more id columns have
90 // to be used to differentiate between them. The id values must
91 // be defined when the hypercube gets added; they cannot be put
92 // explicitly.
93 // <li> A hypercube can be extensible in its last dimension by setting
94 // its last dimension to zero. In that case extendHypercube can
95 // be used to extend the hypercube when needed.
96 // All fixed sized hypercubes are stored in one file, while there
97 // is one file per extensible hypercube.
98 // <li> The table must be large enough to accommodate the addition
99 // or extension of a hypercube. This means that a sufficient
100 // number of rows must be added to the table before a hypercube
101 // can be added or extended. It is the responsibility of the user
102 // to "synchronize" addition of rows and hypercubes.
103 // <li> It is possible to define coordinates for the hypercube axes
104 // in several ways:
105 // <ul>
106 // <li> Use the TiledDataStMan storage manager to hold their values
107 // and define the coordinates when adding or extending the
108 // hypercube. This is the preferred way.
109 // <li> As above, but use explicit puts to write their values.
110 // This has to be used when coordinates are defined after
111 // the hypercube has been added or extended.
112 // Note that several rows may share the same value, so
113 // overwriting a value may affect multiple rows.
114 // <li> Use another storage manager to hold their values.
115 // This is useful when their values depend on other axes,
116 // because that cannot be handled by TiledDataStMan.
117 // </ul>
118 // Note that it is possible to store one coordinate column with
119 // TiledDataStMan and another with another storage manager.
120 // </ul>
121 // </synopsis>
122 
123 // <motivation>
124 // This tiled storage manager allows one to create and extend hypercubes
125 // as needed. One has complete control over which row is stored in which
126 // hypercube.
127 // </motivation>
128 
129 // <example>
130 // The following example shows how to create a TiledDataStMan tiled
131 // storage manager using the hypercolumn as defined in the table description.
132 // Furthermore it shows how to use TiledDataStManAccessor
133 // to add a hypercube, while defining its tile shape, coordinates,
134 // and id-value.
135 // The example shows that reading the data back does not require any knowledge
136 // of the data manager. It's exactly the same if another data manager was used.
137 // <br>
138 // The table created contains the equally shaped data columns "Data" and
139 // "Weight".
140 // Each cell in those columns contains a 2D array with shape [12,20]. The
141 // coordinates of those arrays are "Pol" and "Freq".
142 // The tiled storage manager superimposes two more axes ("Baseline"and "Time")
143 // on the data resulting in a 4D hypercube with shape [12,20,30,42].
144 // The table contains 42*30 rows (which has to be equal to the number of
145 // elements in the superimposed axes).
146 // <br>
147 // The tile shape of the hypercube is (arbitrarily) set to [4,5,6,7].
148 // Of course, any tile shape could be chosen. This tile shape results
149 // in a tile size of 6720 bytes (4*5*6*7 *(4+4) bytes), which is not
150 // that large (32768 as tile size is very reasonable). The number of tiles
151 // is integral in each dimension, so no space is wasted.
152 // Finally it makes access along the various axes about equally efficient.
153 // <br>
154 // Although in this example only one hypercube is added, multiple hypercubes
155 // are possible, because an id column has been defined.
156 // <note role=caution>
157 // The example uses the global Array function indgen to fill the data
158 // and coordinate arrays with arbitrary values.
159 // </note>
160 // Note that the description of class
161 // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
162 // contains a discussion about the effect of setting the maximum cache size.
163 //
164 // <srcblock>
165 // // Define the table description and the columns in it.
166 // TableDesc td ("", "1", TableDesc::Scratch);
167 // td.addColumn (ScalarColumnDesc<float> ("Time"));
168 // td.addColumn (ScalarColumnDesc<float> ("Baseline"));
169 // td.addColumn (ArrayColumnDesc<float> ("Pol", 1));
170 // td.addColumn (ArrayColumnDesc<float> ("Freq", 1));
171 // td.addColumn (ScalarColumnDesc<String> ("Id"));
172 // td.addColumn (ArrayColumnDesc<float> ("Data", 2));
173 // td.addColumn (ArrayColumnDesc<float> ("Weight", 2));
174 // // Define the 4-dim hypercolumn with its data, coordinate and id columns.
175 // td.defineHypercolumn ("TSMExample",
176 // 4,
177 // stringToVector ("Data,Weight"),
178 // stringToVector ("Pol,Freq,Baseline,Time"),
179 // stringToVector ("Id"));
180 //
181 // // Now create a new table from the description.
182 // SetupNewTable newtab("tTiledDataStMan_tmp.data", td, Table::New);
183 // // Create a TiledDataStMan storage manager for the hypercolumn
184 // // and bind the columns to it.
185 // TiledDataStMan sm1 ("TSMExample");
186 // newtab.bindAll (sm1);
187 // // Create the table with 42*30 rows.
188 // Table table(newtab, 42*30);
189 // // Create the accessor to be able to add a hypercube to this
190 // // storage manager.
191 // TiledDataStManAccessor accessor(table, "TSMExample");
192 // // Define the values for the coordinates of the hypercube
193 // // and put them into the record.
194 // Vector<float> timeValues(42);
195 // Vector<float> baselineValues(30);
196 // Vector<float> freqValues(20);
197 // Vector<float> polValues(12);
198 // indgen (timeValues);
199 // indgen (baselineValues, float(100));
200 // indgen (freqValues, float(200));
201 // indgen (polValues, float(300));
202 // Record hyperDef;
203 // hyperDef.define ("Time", timeValues);
204 // hyperDef.define ("Baseline", baselineValues);
205 // hyperDef.define ("Freq", freqValues);
206 // hyperDef.define ("Pol", polValues);
207 // // Define the id value as well.
208 // hyperDef.define ("Id", "");
209 // // Now add the hypercube with the given shape, tile shape,
210 // // and coordinate and id values.
211 // accessor.addHypercube (IPosition(4,12,20,30,42),
212 // IPosition(4,4,5,6,7), hyperDef);
213 // ArrayColumn<float> data (table, "Data");
214 // ArrayColumn<float> weight (table, "Weight");
215 // Matrix<float> array(IPosition(2,12,20));
216 // indgen (array);
217 // // Write some data into the data columns.
218 // for (uInt i=0; i<30*42; i++) {
219 // data.put (i, array);
220 // weight.put (i, array+float(100));
221 // array += float(200);
222 // }
223 // // Prepare for reading the data back.
224 // // Note that time and baseline are in fact scalar columns. They are
225 // // superimposed dimensions on the hypercube.
226 // ScalarColumn<float> time (table, "Time");
227 // ScalarColumn<float> baseline (table, "Baseline");
228 // ArrayColumn<float> freq (table, "Freq");
229 // ArrayColumn<float> pol (table, "Pol");
230 // ScalarColumn<String> id (table, "Id");
231 // float fValue;
232 // String sValue;
233 // for (rownr_t i=0; i<table.nrow(); i++) {
234 // data.get (i, array);
235 // weight.get (i, array);
236 // pol.get (i, polValues);
237 // freq.get (i, freqValues);
238 // baseline.get (i, fValue);
239 // time.get (i, fValue);
240 // id.get (i, sValue);
241 // }
242 // </srcblock>
243 // Note that in this example an id column was not necessary, because
244 // there is only one hypercube.
245 // <p>
246 // The following example is more advanced. Two (extensible) hypercubes
247 // are used for line and continuum data. Writing such a data set
248 // could be done as shown. Reading it back is the same as above.
249 // <br>
250 // In this example the data columns contain line and continuum data.
251 // So there are two types of data, each with their own shape and
252 // stored in their own (extensible) hypercube. Note that the last
253 // dimension of the hypercube shape is set to zero (to make extensible),
254 // but the last tile shape dimension has been filled in,
255 // because the exact tile shape must be known.
256 // <br>
257 // Before each put of the data the appropriate hypercube is extended.
258 // Also the time has to be put, which is done (as an example) in
259 // two different ways (using an explicit put and using the extendHypercube).
260 //
261 // <srcblock>
262 // // Defining TableDesc and storage manager is same as in first example.
263 // // Create the table.
264 // Table table(newtab);
265 // // Create the accessor to be able to add the hypercubes to this
266 // // storage manager.
267 // TiledDataStManAccessor accessor(table, "TSMExample");
268 // // Fill the coordinate values.
269 // // Note that the time axis of the hypercube will have length 0 to
270 // // make it extensible. Therefore the time coordinate can only be
271 // // filled in when the hypercube is extended.
272 // Vector<float> baselineValues(30);
273 // Vector<float> freqValuesCont(1);
274 // Vector<float> freqValuesLine(20);
275 // Vector<float> polValues(4);
276 // indgen (baselineValues, float(100));
277 // indgen (freqValuesLine, float(200));
278 // indgen (freqValuesCont, float(200));
279 // indgen (polValues, float(300));
280 // Record hyperDefLine;
281 // hyperDefLine.define ("Baseline", baselineValues);
282 // hyperDefLine.define ("Pol", polValues);
283 // // Make similar record for line data.
284 // // Fill the correct id and frequency values for each type.
285 // // Add the 2 hypercubes.
286 // Record hyperDefCont (hyperDefLine);
287 // hyperDefLine.define ("Id", "L");
288 // hyperDefLine.define ("Freq", freqValuesLine);
289 // hyperDefCont.define ("Id", "C");
290 // hyperDefCont.define ("Freq", freqValuesCont);
291 // // Add the hypercubes.
292 // // Define their last dimension as zero to make them extensible.
293 // accessor.addHypercube (IPosition(4,4,20,30,0),
294 // IPosition(4,4,5,6,7), hyperDefLine);
295 // accessor.addHypercube (IPosition(4,4,1,30,0),
296 // IPosition(4,4,1,6,7), hyperDefCont);
297 // ScalarColumn<float> time (table, "Time");
298 // ScalarColumn<float> baseline (table, "Baseline");
299 // ArrayColumn<float> freq (table, "Freq");
300 // ArrayColumn<float> pol (table, "Pol");
301 // ArrayColumn<float> data (table, "Data");
302 // ArrayColumn<float> weight (table, "Weight");
303 // Matrix<float> arrayLine(IPosition(2,4,20));
304 // Matrix<float> arrayCont(IPosition(2,4,1));
305 // indgen (arrayLine);
306 // indgen (arrayCont);
307 // // Write some data into the data columns.
308 // // Alternately line and continuum is written.
309 // // Each hypercube requires 30 rows to be added (i.e. nr of baselines).
310 // // The last dimension of each hypercube is extended with 1.
311 // rownr_t rownr = 0;
312 // for (uInt i=0; i<42; i++) {
313 // if (i%2 == 0) {
314 // table.addRow (30);
315 // accessor.extendHypercube (1, hyperDefLine);
316 // time.put (rownr, float(i));
317 // for (uInt j=0; j<30; j++) {
318 // data.put (rownr, arrayLine);
319 // weight.put (rownr, arrayLine);
320 // rownr++;
321 // }
322 // }else{
323 // table.addRow (30);
324 // Vector<float> timeValue(1);
325 // timeValue(0) = float(i);
326 // hyperDefCont.define ("Time", timeValue);
327 // accessor.extendHypercube (1, hyperDefCont);
328 // time.put (rownr, float(i));
329 // for (uInt j=0; j<30; j++) {
330 // data.put (rownr, arrayCont);
331 // weight.put (rownr, arrayCont);
332 // rownr++;
333 // }
334 // }
335 // }
336 // </srcblock>
337 // Note that in this example the time is defined in 2 different ways.
338 // The first one by an explicit put, the second one as a record in
339 // the extendHypercube call. The second way if the preferred one,
340 // although it requires a bit more coding.
341 // </example>
342 
343 //# <todo asof="$DATE:$">
344 //# A List of bugs, limitations, extensions or planned refinements.
345 //# </todo>
346 
347 
349 {
351 
352 public:
353  // Create a TiledDataStMan storage manager for the hypercolumn
354  // with the given name.
355  // The hypercolumn name is also the name of the storage manager.
356  // The given maximum cache size (default is unlimited) is persistent,
357  // thus will be reused when the table is read back. Note that the class
358  // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
359  // allows one to overwrite the maximum cache size temporarily.
360  // <br>The constructor taking a Record expects fields in the record with
361  // the name of the arguments in uppercase. If not defined, their
362  // default value is used.
363  // <group>
364  TiledDataStMan (const String& hypercolumnName,
366  TiledDataStMan (const String& hypercolumnName,
367  const Record& spec);
368  // </group>
369 
371 
372  // Clone this object.
373  // It does not clone TSMColumn objects possibly used.
374  DataManager* clone() const;
375 
376  // Get the type name of the data manager (i.e. TiledDataStMan).
378 
379  // Make the object from the type name string.
380  // This function gets registered in the DataManager "constructor" map.
382  const Record& spec);
383 
384 private:
385  // Create a TiledDataStMan.
386  // This constructor is private, because it should only be used
387  // by makeObject.
389 
390  // Forbid copy constructor.
392 
393  // Forbid assignment.
395 
396  // Add rows to the storage manager.
397  // This will only increase the number of rows. When a hypercube is
398  // added or extended, it will be checked whether the number of rows
399  // is sufficient.
400  void addRow64 (rownr_t nrrow);
401 
402  // Add a hypercube.
403  // The number of rows in the table must be large enough to
404  // accommodate this hypercube.
405  // The possible id values must be given in the record, while
406  // coordinate values are optional. The field names in the record
407  // should match the coordinate and id column names.
408  // The last dimension in the cube shape can be zero, indicating that
409  // the hypercube is extensible.
410  void addHypercube (const IPosition& cubeShape,
411  const IPosition& tileShape,
412  const Record& values);
413 
414  // Extend the hypercube with the given number of elements in
415  // the last dimension.
416  // The record should contain the id values (to get the correct
417  // hypercube) and optionally coordinate values for the elements added.
418  void extendHypercube (uInt64 incrInLastDim, const Record& values);
419 
420  // Get the hypercube in which the given row is stored.
421  virtual TSMCube* getHypercube (rownr_t rownr);
422 
423  // Get the hypercube in which the given row is stored.
424  // It also returns the position of the row in that hypercube.
425  virtual TSMCube* getHypercube (rownr_t rownr, IPosition& position);
426 
427  // Flush and optionally fsync the data.
428  // It returns a True status if it had to flush (i.e. if data have changed).
429  virtual Bool flush (AipsIO&, Bool fsync);
430 
431  // Let the storage manager create files as needed for a new table.
432  // This allows a column with an indirect array to create its file.
433  virtual void create64 (rownr_t nrrow);
434 
435  // Read the header info.
436  virtual void readHeader (rownr_t nrrow, Bool firstTime);
437 
438  // Update the map of row numbers to cube number plus offset.
439  void updateRowMap (uInt cubeNr, uInt64 incrInLastDim);
440 
441  // Check if the table is large enough to hold this
442  // hypercube extension.
443  void checkNrrow (const IPosition& cubeShape,
444  uInt64 incrInLastDim) const;
445 
446 
447  //# Declare the data members.
448  // The map of row number to cube and position in cube.
449  std::vector<rownr_t> rowMap_p;
450  std::vector<uInt> cubeMap_p;
451  std::vector<uInt> posMap_p;
452  // The row number since the last hypercube extension.
454 };
455 
456 
457 
458 
459 } //# NAMESPACE CASACORE - END
460 
461 #endif
Abstract base class for a data manager.
Definition: DataManager.h:221
String: the storage and methods of handling collections of characters.
Definition: String.h:225
std::vector< uInt > cubeMap_p
virtual TSMCube * getHypercube(rownr_t rownr)
Get the hypercube in which the given row is stored.
TiledDataStMan(const String &hypercolumnName, uInt64 maximumCacheSize=0)
Create a TiledDataStMan storage manager for the hypercolumn with the given name.
TiledDataStMan()
Create a TiledDataStMan.
void checkNrrow(const IPosition &cubeShape, uInt64 incrInLastDim) const
Check if the table is large enough to hold this hypercube extension.
virtual void readHeader(rownr_t nrrow, Bool firstTime)
Read the header info.
std::vector< rownr_t > rowMap_p
The map of row number to cube and position in cube.
static DataManager * makeObject(const String &dataManagerType, const Record &spec)
Make the object from the type name string.
std::vector< uInt > posMap_p
virtual Bool flush(AipsIO &, Bool fsync)
Flush and optionally fsync the data.
void addHypercube(const IPosition &cubeShape, const IPosition &tileShape, const Record &values)
Add a hypercube.
String dataManagerType() const
Get the type name of the data manager (i.e.
TiledDataStMan(const TiledDataStMan &)
Forbid copy constructor.
void updateRowMap(uInt cubeNr, uInt64 incrInLastDim)
Update the map of row numbers to cube number plus offset.
virtual TSMCube * getHypercube(rownr_t rownr, IPosition &position)
Get the hypercube in which the given row is stored.
TiledDataStMan(const String &hypercolumnName, const Record &spec)
void addRow64(rownr_t nrrow)
Add rows to the storage manager.
void extendHypercube(uInt64 incrInLastDim, const Record &values)
Extend the hypercube with the given number of elements in the last dimension.
virtual void create64(rownr_t nrrow)
Let the storage manager create files as needed for a new table.
DataManager * clone() const
Clone this object.
TiledDataStMan & operator=(const TiledDataStMan &)
Forbid assignment.
rownr_t nrrowLast_p
The row number since the last hypercube extension.
uInt maximumCacheSize() const
Get the current maximum cache size (in MiB (MibiByte)).
Definition: TiledStMan.h:532
const IPosition & tileShape(rownr_t rownr) const
Get the tile shape of the data in the given row.
this file contains all the compiler specific defines
Definition: mainpage.dox:28
unsigned int uInt
Definition: aipstype.h:51
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
uInt64 rownr_t
Define the type of a row number in a table.
Definition: aipsxtype.h:46
unsigned long long uInt64
Definition: aipsxtype.h:39