casacore
StArrayFile.h
Go to the documentation of this file.
1 //# StArrayFile.h: Read/write array in external format for a storage manager
2 //# Copyright (C) 1994,1995,1996,1997,1999,2001,2002
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_STARRAYFILE_H
29 #define TABLES_STARRAYFILE_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
33 #include <casacore/casa/IO/RegularFileIO.h>
34 #include <casacore/casa/IO/TypeIO.h>
35 #include <casacore/casa/BasicSL/String.h>
36 #include <casacore/casa/BasicSL/Complex.h>
37 
38 namespace casacore { //# NAMESPACE CASACORE - BEGIN
39 
40 //# Forward Declarations
41 class MultiFileBase;
42 class IPosition;
43 
44 
45 // <summary>
46 // Read/write array in external format for a storage manager
47 // </summary>
48 
49 // <use visibility=local>
50 
51 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
52 // </reviewed>
53 
54 // <prerequisite>
55 //# Classes you should understand before using this one.
56 // <li> ToLocal
57 // <li> FromLocal
58 // </prerequisite>
59 
60 // <etymology>
61 // StManArrayFile is a class used by table storage managers
62 // to store indirect arrays in a file.
63 // </etymology>
64 
65 // <synopsis>
66 // StManArrayFile is for use by the table storage manager, in particular
67 // to read/write indirectly stored arrays.
68 // Instead of holding the data in memory, they are written directly
69 // into a file. It also allows to access a part of an array, which
70 // is needed for the table system to access an array section.
71 // It does not use a cache of its own, but it is relying on the
72 // underlying system routines to cache and buffer adequately.
73 //
74 // This class could in principle also be used for other array purposes,
75 // for example, to implement a paged array class for really huge arrays.
76 //
77 // An StManArrayFile object is connected to one file. It is possible
78 // to hold multiple arrays in the file, each with its own shape.
79 // An array is stored as its shape followed by the actual data
80 // (all in little or big endian format). An array of strings is written as
81 // an array of offsets pointing to the actual strings.
82 // When a string gets a new value, the new value is written at the
83 // end of the file and the file space with the old value is lost.
84 //
85 // Currently only the basic types are supported, but arbitrary types
86 // could also be supported by writing/reading an element in the normal
87 // way into the AipsIO buffer. It would only require that AipsIO
88 // would contain a function to get its buffers and to restart them.
89 // </synopsis>
90 
91 // <example>
92 // <srcblock>
93 // void writeArray (const Array<Bool>& array)
94 // {
95 // // Construct object and update file StArray.dat.
96 // StManArrayFile arrayFile("StArray.dat, ByteIO::New);
97 // // Reserve space for an array with the given shape and data type.
98 // // This writes the shape at the end of the file and reserves
99 // // space the hold the entire Bool array.
100 // // It fills in the file offset where the shape is stored
101 // // and returns the length of the shape in the file.
102 // Int64 offset;
103 // uInt shapeLength = arrayFile.putShape (array.shape(), offset, static_cast<Bool*>(0));
104 // // Now put the actual array.
105 // // This has to be put at the returned file offset plus the length
106 // // of the shape in the file.
107 // Bool deleteIt;
108 // const Bool* dataPtr = array.getStorage (deleteIt);
109 // arrayFile.put (offset+shapeLength, 0, array.nelements(), dataPtr);
110 // array.freeStorage (dataPtr, deleteIt);
111 // }
112 // </srcblock>
113 // </example>
114 
115 // <motivation>
116 // The AipsIO class was not suitable for indirect table arrays,
117 // because it uses memory to hold the data. Furthermore it is
118 // not possible to access part of the data in AipsIO.
119 // </motivation>
120 
121 // <todo asof="$DATE:$">
122 // <li> implement long double
123 // <li> support arbitrary types
124 // <li> when rewriting a string value, use the current file
125 // space if it fits
126 // </todo>
127 
128 
130 {
131 public:
132 
133  // Construct the object and attach it to the give file.
134  // The OpenOption determines how the file is opened
135  // (e.g. ByteIO::New for a new file).
136  // The buffersize is used to allocate a buffer of a proper size
137  // for the underlying filebuf object (see iostream package).
138  // A bufferSize 0 means using the default size (currently 65536).
140  uInt version=0, Bool bigEndian=True,
141  uInt bufferSize=0,
142  MultiFileBase* mfile=0);
143 
144  // Close the possibly opened file.
146 
147  // Flush and optionally fsync the data.
148  // It returns True when any data was written since the last flush.
149  Bool flush (Bool fsync);
150 
151  // Reopen the file for read/write access.
152  void reopenRW();
153 
154  // Resync the file (i.e. clear possible cache information).
155  void resync();
156 
157  // Return the current file length (merely a debug tool).
159  { return leng_p; }
160 
161  // Put the array shape and store its file offset into the offset argument.
162  // Reserve file space for the associated array.
163  // The length of the shape part in the file is returned.
164  // The file offset plus the shape length is the starting offset of the
165  // actual array data (which can be used by get and put).
166  // Space is reserved to store the reference count.
167  // <group>
168  uInt putShape (const IPosition& shape, Int64& fileOffset,
169  const Bool* dummy);
170  uInt putShape (const IPosition& shape, Int64& fileOffset,
171  const Char* dummy);
172  uInt putShape (const IPosition& shape, Int64& fileOffset,
173  const uChar* dummy);
174  uInt putShape (const IPosition& shape, Int64& fileOffset,
175  const Short* dummy);
176  uInt putShape (const IPosition& shape, Int64& fileOffset,
177  const uShort* dummy);
178  uInt putShape (const IPosition& shape, Int64& fileOffset,
179  const Int* dummy);
180  uInt putShape (const IPosition& shape, Int64& fileOffset,
181  const uInt* dummy);
182  uInt putShape (const IPosition& shape, Int64& fileOffset,
183  const Int64* dummy);
184  uInt putShape (const IPosition& shape, Int64& fileOffset,
185  const uInt64* dummy);
186  uInt putShape (const IPosition& shape, Int64& fileOffset,
187  const Float* dummy);
188  uInt putShape (const IPosition& shape, Int64& fileOffset,
189  const Double* dummy);
190  uInt putShape (const IPosition& shape, Int64& fileOffset,
191  const Complex* dummy);
192  uInt putShape (const IPosition& shape, Int64& fileOffset,
193  const DComplex* dummy);
194  uInt putShape (const IPosition& shape, Int64& fileOffset,
195  const String* dummy);
196  // </group>
197 
198  // Get the reference count.
200 
201  // Put the reference count.
202  // An exception is thrown if a value other than 1 is put for version 0.
203  void putRefCount (uInt refCount, Int64 offset);
204 
205  // Put nr elements at the given file offset and array offset.
206  // The file offset of the first array element is the file offset
207  // of the shape plus the length of the shape in the file.
208  // The array offset is counted in number of elements. It can be
209  // used to put only a (contiguous) section of the array.
210  // <group>
211  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Bool*);
212  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Char*);
213  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uChar*);
214  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Short*);
215  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uShort*);
216  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int*);
217  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt*);
218  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int64*);
219  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt64*);
220  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Float*);
221  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Double*);
222 //#// void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const long double*);
223  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Complex*);
224  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const DComplex*);
225  void put (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const String*);
226  // </group>
227 
228  // Get the shape at the given file offset.
229  // It will reshape the IPosition vector when needed.
230  // It returns the length of the shape in the file.
232 
233  // Get nr elements at the given file offset and array offset.
234  // The file offset of the first array element is the file offset
235  // of the shape plus the length of the shape in the file.
236  // The array offset is counted in number of elements. It can be
237  // used to get only a (contiguous) section of the array.
238  // <group>
239  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Bool*);
240  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Char*);
241  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uChar*);
242  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Short*);
243  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uShort*);
244  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int*);
245  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt*);
246  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int64*);
247  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt64*);
248  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Float*);
249  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Double*);
250 //#// void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, long double*);
251  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Complex*);
252  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, DComplex*);
253  void get (Int64 fileOffset, Int64 arrayOffset, uInt64 nr, String*);
254  // </group>
255 
256  // Copy the array with <src>nr</src> elements from one file offset
257  // to another.
258  // <group>
259  void copyArrayBool (Int64 to, Int64 from, uInt64 nr);
260  void copyArrayChar (Int64 to, Int64 from, uInt64 nr);
261  void copyArrayuChar (Int64 to, Int64 from, uInt64 nr);
262  void copyArrayShort (Int64 to, Int64 from, uInt64 nr);
263  void copyArrayuShort (Int64 to, Int64 from, uInt64 nr);
264  void copyArrayInt (Int64 to, Int64 from, uInt64 nr);
265  void copyArrayuInt (Int64 to, Int64 from, uInt64 nr);
266  void copyArrayInt64 (Int64 to, Int64 from, uInt64 nr);
267  void copyArrayuInt64 (Int64 to, Int64 from, uInt64 nr);
268  void copyArrayFloat (Int64 to, Int64 from, uInt64 nr);
269  void copyArrayDouble (Int64 to, Int64 from, uInt64 nr);
270 //#// void copyArrayLDouble (Int64 to, Int64 from, uInt64 nr);
271  void copyArrayComplex (Int64 to, Int64 from, uInt64 nr);
272  void copyArrayDComplex (Int64 to, Int64 from, uInt64 nr);
273  void copyArrayString (Int64 to, Int64 from, uInt64 nr);
274  // </group>
275 
276 private:
277  ByteIO* file_p; //# File object
278  TypeIO* iofil_p; //# IO object
279  Int64 leng_p; //# File length
280  uInt version_p; //# Version of StArrayFile file
281  Bool swput_p; //# True = put is possible
282  Bool hasPut_p; //# True = put since last flush
293 
294  // Put a single value at the current file offset.
295  // It returns the length of the value in the file.
296  // <group>
297  uInt put (const Int&);
298  uInt put (const uInt&);
299  // </group>
300 
301  // Put the array shape at the end of the file and reserve
302  // space for nr elements (each lenElem bytes long).
303  // It fills the file offset of the shape.
304  // It returns the length of the shape in the file.
305  uInt putRes (const IPosition& shape, Int64& fileOffset, float lenElem);
306 
307  // Get a single value at the current file offset.
308  // It returns the length of the value in the file.
309  // <group>
310  uInt get (Int&);
311  uInt get (uInt&);
312  // </group>
313 
314  // Copy data with the given length from one file offset to another.
315  void copyData (Int64 to, Int64 from, uInt64 length);
316 
317  // Position the file on the given offset.
318  void setpos (Int64 offset);
319 };
320 
321 
323 {
324  file_p->reopenRW();
325 }
327 {
328  hasPut_p = True;
329  return iofil_p->write (1, &value);
330 }
332 {
333  hasPut_p = True;
334  return iofil_p->write (1, &value);
335 }
337 {
338  return iofil_p->read (1, &value);
339 }
341 {
342  return iofil_p->read (1, &value);
343 }
344 
345 
346 
347 } //# NAMESPACE CASACORE - END
348 
349 #endif
virtual void reopenRW()
Reopen the underlying IO stream for read/write access.
OpenOption
Define the possible ByteIO open options.
Definition: ByteIO.h:65
Abstract base class to combine multiple files in a single one.
void copyArrayInt64(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Char *)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Char *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Bool *)
Put nr elements at the given file offset and array offset.
void copyArrayComplex(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, DComplex *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const String *dummy)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uChar *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Bool *dummy)
Put the array shape and store its file offset into the offset argument.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Short *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Bool *)
Get nr elements at the given file offset and array offset.
void copyArrayChar(Int64 to, Int64 from, uInt64 nr)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Float *)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int64 *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Complex *dummy)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const DComplex *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uShort *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Double *dummy)
void copyArrayFloat(Int64 to, Int64 from, uInt64 nr)
void resync()
Resync the file (i.e.
void copyData(Int64 to, Int64 from, uInt64 length)
Copy data with the given length from one file offset to another.
void copyArrayDouble(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Double *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Short *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Char *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Complex *)
void putRefCount(uInt refCount, Int64 offset)
Put the reference count.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Complex *)
void copyArrayInt(Int64 to, Int64 from, uInt64 nr)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uChar *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Int *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Float *dummy)
void setpos(Int64 offset)
Position the file on the given offset.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const String *)
void copyArrayBool(Int64 to, Int64 from, uInt64 nr)
Copy the array with nr elements from one file offset to another.
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Short *)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Int *dummy)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uShort *)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uChar *)
Bool flush(Bool fsync)
Flush and optionally fsync the data.
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uInt *dummy)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const Int64 *dummy)
uInt putRes(const IPosition &shape, Int64 &fileOffset, float lenElem)
Put the array shape at the end of the file and reserve space for nr elements (each lenElem bytes long...
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uShort *dummy)
void copyArrayuInt(Int64 to, Int64 from, uInt64 nr)
StManArrayFile(const String &name, ByteIO::OpenOption, uInt version=0, Bool bigEndian=True, uInt bufferSize=0, MultiFileBase *mfile=0)
Construct the object and attach it to the give file.
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const Double *)
~StManArrayFile()
Close the possibly opened file.
uInt getShape(Int64 fileOffset, IPosition &shape)
Get the shape at the given file offset.
uInt getRefCount(Int64 offset)
Get the reference count.
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, uInt64 *)
void reopenRW()
Reopen the file for read/write access.
Definition: StArrayFile.h:322
Int64 length()
Return the current file length (merely a debug tool).
Definition: StArrayFile.h:158
void copyArrayuInt64(Int64 to, Int64 from, uInt64 nr)
void copyArrayuChar(Int64 to, Int64 from, uInt64 nr)
void copyArrayShort(Int64 to, Int64 from, uInt64 nr)
uInt putShape(const IPosition &shape, Int64 &fileOffset, const uInt64 *dummy)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const uInt64 *)
void copyArrayDComplex(Int64 to, Int64 from, uInt64 nr)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Float *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, Int64 *)
void get(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, String *)
void copyArrayString(Int64 to, Int64 from, uInt64 nr)
void copyArrayuShort(Int64 to, Int64 from, uInt64 nr)
void put(Int64 fileOffset, Int64 arrayOffset, uInt64 nr, const DComplex *)
String: the storage and methods of handling collections of characters.
Definition: String.h:225
virtual size_t write(size_t nvalues, const Bool *value)
Convert the values and write them to the ByteIO object.
virtual size_t read(size_t nvalues, Bool *value)
Read the values from the ByteIO object and convert them.
this file contains all the compiler specific defines
Definition: mainpage.dox:28
unsigned char uChar
Definition: aipstype.h:47
short Short
Definition: aipstype.h:48
unsigned int uInt
Definition: aipstype.h:51
unsigned short uShort
Definition: aipstype.h:49
TableExprNode shape(const TableExprNode &array)
Function operating on any scalar or array resulting in a Double array containing the shape.
Definition: ExprNode.h:1987
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
float Float
Definition: aipstype.h:54
int Int
Definition: aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
LatticeExprNode value(const LatticeExprNode &expr)
This function returns the value of the expression without a mask.
const Bool True
Definition: aipstype.h:43
double Double
Definition: aipstype.h:55
char Char
Definition: aipstype.h:46
unsigned long long uInt64
Definition: aipsxtype.h:39