casacore
ReadAsciiTable.h
Go to the documentation of this file.
1 //# ReadAsciiTable.h: Filling a table from an Ascii file
2 //# Copyright (C) 1993,1994,1995,1999,2001,2002
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_READASCIITABLE_H
29 #define TABLES_READASCIITABLE_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
33 #include <casacore/casa/BasicSL/String.h>
34 #include <casacore/casa/Arrays/IPosition.h>
35 #include <casacore/tables/Tables/Table.h>
36 
37 //# Forward Declarations
38 #include <casacore/casa/iosfwd.h>
39 
40 namespace casacore { //# NAMESPACE CASACORE - BEGIN
41 
42 class Regex;
43 class IPosition;
44 class LogIO;
45 class TableRecord;
46 class TableColumn;
47 
48 
49 // <summary>
50 // Filling a table from an Ascii file.
51 // </summary>
52 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
53 // </reviewed>
54 
55 // <use visibility=export>
56 
57 // <prerequisite>
58 // <li> <linkto class="Table:description">Table</linkto>
59 // </prerequisite>
60 
61 // <synopsis>
62 // Global functions to fill a table from an Ascii file.
63 //
64 // The table columns are filled from a file containing the data values
65 // separated by a separator (optionally followed by whitespace). The
66 // default separator is a comma. Non-given values default to 0, False, or
67 // blank string (depending on data type). A value is not given between 2
68 // consecutive separators or if less values are given than needed.
69 // One line per table row should be given.
70 // The following two header lines define the columns in the table:
71 // <ol>
72 // <li> The first line contains the names of the variables in each column.
73 // These names may be enclosed in double quotes.
74 // <li> The second line contains the data types of each column.
75 // Valid types are:
76 // <ul>
77 // <li> S for Short Integer data
78 // <li> I for Integer data
79 // <li> R for Real data
80 // <li> D for Double Precision data
81 // <li> X for Complex data (Real, Imaginary)
82 // <li> DX for Double Precision Complex data (R,I)
83 // <li> Z for Complex data (Amplitude, Phase)
84 // <li> DZ for Double Precision Complex data (A,P)
85 // <li> A for ASCII data (must be enclosed in double
86 // quotes if it contains one or more blanks)
87 // <li> DMS for MVAngle-format position in DMS (converted to radians)
88 // In this case a colon separated position is seen as
89 // degrees and not as hours.
90 // Blanks instead of : can be used as separator.
91 // <li> HMS for MVAngle-format position in HMS (converted to radians)
92 // Blanks instead of : can be used as separator.
93 // </ul>
94 // The type can optionally be followed by one or more positive numbers
95 // (separated by commas without whitespace) indicating that the column
96 // contains an array. The numbers give the shape of the array.
97 // E.g. <src>D2,4</src> defines a column containing arrays with
98 // shape [2,4]. It "consumes" 8 numbers in each input data line.
99 // The last column can contain a 0 in one of the shape numbers.
100 // It indicates that the arrays are variable shaped; it "consumes"
101 // all remaining numbers in each input data line. If needed,
102 // the arrays are filled with default values (0, False, or blank).
103 // E.g. <src>I0</src> indicates a variable shaped vector.
104 // <src>I0,4</src> with a line with remaining input
105 // <src>1 2 3 4 5 6 7 8 9</src> results in an array with shape [3,4]
106 // (filled with with 3 zeroes).
107 // </ol>
108 // If the <src>autoHeader</src> argument is True, the column definition
109 // lines should not be given. It recognizes the types from the first data
110 // line. It gives the names 'column0', etc. to the columns.
111 // It can recognize integer, double, and string types.
112 // It is possible to give a shape argument which has the same function
113 // as the shape values discussed above.
114 // <p>
115 // There are two forms of the readAsciiTable function:
116 // <ol>
117 // <li> The simplest form has two input files.
118 // The second input file contains the column data.
119 // The first input file contains the keywords (if any)
120 // and the column definitions.
121 // The keywords in the first file, if there are any, must be enclosed
122 // between a line that starts with ".keywords" and a line that starts
123 // with ".endkeywords". To define column keywords, .keywords should be
124 // followed by whitespace and the column name.
125 // Between these two lines each line should contain the following:
126 // <ul>
127 // <li> The keyword name, e.g., ANYKEY
128 // <li> The datatype of the keyword (cf. list of valid types above)
129 // <li> The value or values for the keyword (the keyword may contain a
130 // scalar or a vector of values). e.g., 3.14159 21.78945
131 // </ul>
132 // After the keywords definitions, the two column definition lines
133 // should follow (unless <src>autoHeader=True</src> is given).
134 // <br>For example:
135 // <srcblock>
136 // .keywords
137 // KEYI I 10
138 // KEYIV I 11 12 13 14
139 // KEYF R 1.2
140 // KEYFV R -3.2 0 5.6
141 // KEYD D 1.23456789
142 // KEYDV D 1 2 3 4 5 6 7 8 9
143 // KEYX X -1.5 -3
144 // KEYXC X 0 1 2 3 4 5 6 7 8 9
145 // KEYZ Z -3 -1.5
146 // KEYZV Z 0 0.1 0.2 0.3 0.4 0.5
147 // KEYS A "1 2 3 4 5"
148 // KEYSV A " 1 2 " "AAA" BBB bbb CCc C "@#$%^&*()"
149 // .endkeywords
150 // .keywords COLDX
151 // IKEYS A "coldx ikey"
152 // DKEYS A "coldx dkey"
153 // .endkeywords
154 // COLI COLF COLD COLX COLZ COLS
155 // I R D X Z A
156 // </srcblock>
157 // defines a table with 12 table keywords (of which 6 contain vector
158 // values), 2 keywords for column COLDX, and and 6 columns.
159 // The number of rows is determined by the number of
160 // lines in the second input file.
161 // <li> The other form is to combine the two files in one file.
162 // In that case the data lines must be preceeded by the optional
163 // keyword and column definitions (without an intermediate blank line).
164 // </ol>
165 // </synopsis>
166 
167 // <example>
168 // <srcblock>
169 // readAsciiTable ("file.in", "", "table.test");
170 // </srcblock>
171 // creates a table with name <src>table.test</src> from the text file
172 // <src>file.in</src>. The text file could look like:
173 // <srcblock>
174 // COLI COLF COLD COLX COLZ COLS
175 // I R D X Z A
176 // 1 1.1 1.11 1.12 1.13 1.14 1.15 Str1
177 // 10 11 12 13 14 15 16 String17
178 // </srcblock>
179 // resulting in a table with 6 columns and 2 rows.
180 // </example>
181 
182 // <group name=readAsciiTable>
183 
184 
185 // Create a table with name as given by tableName.
186 // If autoHeader==True, the format is automatically derived from the
187 // first data line. It can recognize integer, double, and String types.
188 // The columns will be named column1, column2, etc..
189 // If the autoShape argument is given with 1 or more axes, all values are
190 // treated as a single column with the given shape. Note that one of the
191 // can have length 0 indicating a variable shaped array.
192 // If autoHeader==False, the layout of the table has to be defined in
193 // the first 2 lines of the input file. The remaining lines in the
194 // input file contain the data.
195 //
196 // When the tableDescName is not blank, the table description will
197 // be stored in a table description file with the given name.
198 // <br>It returns a string containing the format of the columns in
199 // the form COL1=R, COL2=D, ...
200 //
201 // The separator gives the character separating the values. The default
202 // is a blank. Note that irrespective of the separator, blanks between
203 // values are always ignored. A string value has to be enclosed in
204 // double quotes if it has to contain blanks or the separator value.
205 //
206 // Header and data lines starting with the regular expression given in the
207 // commentMarker are ignored. By default no comment marker is present.
208 // E.g. "#" ignores all lines starting with the #-sign.
209 // " *#" does the same, but the lines to ignore can start with whitespace.
210 //
211 // The first and last line argument give the 1-relative number of the
212 // first and last line to read from the file. firstLine <= 0 is the
213 // same as 1. lastLine <= 0 means until end-of-file.
214 // Note that lines matching the comment marker are also counted.
215 String readAsciiTable (const String& filein, const String& tableDescName,
216  const String& tableName, Bool autoHeader = False,
217  Char separator = ' ',
218  const String& commentMarkerRegex = "",
219  Int firstLine = 1, Int lastLine = -1,
220  const IPosition& autoShape = IPosition());
221 
222 // This form gets the header info in the given vectors.
223 // Each element in the dataTypes vector has to be of the form as would
224 // be given in a header line.
225 String readAsciiTable (const String& filein, const String& tableproto,
226  const String& tablename,
227  const Vector<String>& columnNames,
228  const Vector<String>& dataTypes,
229  Char separator, const String& commentMarkerRegex,
230  Int firstLine, Int lastLine);
231 
232 // This form reads TWO Ascii files. The first file may contain
233 // keywords and their values as well as the two lines described above for
234 // the names and type of variables. The second file is intended for data only.
235 //
236 // When the tableDescName is not blank, the table description will
237 // be stored in a table description file with the given name.
238 // <br>It returns a string containing the format of the columns in
239 // the form COL1=R, COL2=D, ...
240 //
241 // The separator gives the character separating the values. The default
242 // is a blank. Note that irrespective of the separator, blanks between
243 // values are always ignored. A string value has to be enclosed in
244 // double quotes if it has to contain blanks or the separator value.
245 //
246 // Header and data lines starting with the regular expression given in the
247 // commentMarker are ignored. By default no comment marker is present.
248 // E.g. "#" ignores all lines starting with the #-sign.
249 // " *#" does the same, but the lines to ignore can start with whitespace.
250 //
251 // The first and last line argument give the 1-relative number of the
252 // first and last line to read from the data file. firstLine <= 0 is the
253 // same as 1. lastLine <= 0 means until end-of-file.
254 // Note that lines matching the comment marker are also counted.
255 // <group>
256 String readAsciiTable (const String& headerFile, const String& dataFile,
257  const String& tableDescName, const String& tablename,
258  Char separator = ' ',
259  const String& commentMarkerRegex = "",
260  Int firstLine = 1, Int lastLine = -1);
261 //# Note that this char* version is needed, because of the first version
262 //# Taking a Bool as the 4th argument.
263 String readAsciiTable (const String& headerFile, const String& dataFile,
264  const String& tableDescName, const char* tablename,
265  Char separator = ' ',
266  const String& commentMarkerRegex = "",
267  Int firstLine = 1, Int lastLine = -1);
268 // </group>
269 
270 // Similar versions as above, but returning a Table object.
271 // The format string is returned in the first argument.
272 // The type of Table can be given (Plain or Memory).
273 // <group>
274 Table readAsciiTable (String& formatString, Table::TableType tableType,
275  const String& filein, const String& tableDescName,
276  const String& tableName, Bool autoHeader = False,
277  Char separator = ' ',
278  const String& commentMarkerRegex = "",
279  Int firstLine = 1, Int lastLine = -1,
280  const IPosition& autoShape = IPosition());
281 Table readAsciiTable (String& formatString, Table::TableType tableType,
282  const String& filein, const String& tableproto,
283  const String& tablename,
284  const Vector<String>& columnNames,
285  const Vector<String>& dataTypes,
286  Char separator, const String& commentMarkerRegex,
287  Int firstLine, Int lastLine);
288 Table readAsciiTable (String& formatString, Table::TableType tableType,
289  const String& headerFile, const String& dataFile,
290  const String& tableDescName, const String& tablename,
291  Char separator = ' ',
292  const String& commentMarkerRegex = "",
293  Int firstLine = 1, Int lastLine = -1);
294 Table readAsciiTable (String& formatString, Table::TableType tableType,
295  const String& headerFile, const String& dataFile,
296  const String& tableDescName, const char* tablename,
297  Char separator = ' ',
298  const String& commentMarkerRegex = "",
299  Int firstLine = 1, Int lastLine = -1);
300 // </group>
301 
302 // </group>
303 
304 
305 
306 
307 // <summary>
308 // Helper class for readAsciiTable
309 // </summary>
310 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
311 // </reviewed>
312 
313 // <use visibility=local>
314 
315 // <synopsis>
316 // This class contains static functions as helpers for readAsciiTable.
317 // </synopsis>
318 
320 {
321 public:
322  // Run the readAsciiTable.
323  static String run (const String& headerfile, const String& filein,
324  const String& tableproto, const String& tablename,
325  Bool autoHeader, const IPosition& autoShape,
326  const Vector<String>& columnNames,
327  const Vector<String>& dataTypes,
328  Char separator,
329  const String& commentMarkerRegex,
330  Int firstLine, Int lastLine);
331  static Table runt (String& formatString, Table::TableType tableType,
332  const String& headerfile, const String& filein,
333  const String& tableproto, const String& tablename,
334  Bool autoHeader, const IPosition& autoShape,
335  const Vector<String>& columnNames,
336  const Vector<String>& dataTypes,
337  Char separator,
338  const String& commentMarkerRegex,
339  Int firstLine, Int lastLine);
340 
341  // Read a position using MVAngle.
342  // If isDMS is True, a position with : is treated as DMS instead of HMS.
343  // This function is a bit more relaxed than MVAngle::read.
344  // It allows whitespace. Furthermore it allows whitespace as separator :.
345  static double stringToPos (const String& pos, Bool isDMS);
346 
347 private:
348  // Define types.
351 
352 
353  // Do the actual run.
354  static String doRun (const String& headerfile, const String& filein,
355  const String& tableproto, const String& tablename,
356  Bool autoHeader, const IPosition& autoShape,
357  const Vector<String>& columnNames,
358  const Vector<String>& dataTypes,
359  Char separator,
360  Bool testComment, const Regex& commentMarker,
361  Int firstLine, Int lastLine);
362 
363  // Do the actual work of making and filling the table.
364  static Table makeTab (String& formatString, Table::TableType tableType,
365  const String& headerfile, const String& filein,
366  const String& tableproto,
367  const String& tablename,
368  Bool autoHeader, const IPosition& autoShape,
369  const Vector<String>& columnNames,
370  const Vector<String>& dataTypes,
371  Char separator,
372  Bool testComment, const Regex& commentMarker,
373  Int firstLine, Int lastLine);
374 
375  // Get the next line. Skip lines to be ignored.
376  // It returns False when no more lines are available.
377  static Bool getLine (ifstream& file, Int& lineNumber,
378  char* line, Int lineSize,
379  Bool testComment, const Regex& commentMarker,
380  Int firstLine, Int lastLine);
381 
382  // Get the next part of the line using the separator as delimiter.
383  // Leading blanks are ignored.
384  static Int getNext (const Char* string, Int strlen, Char* result,
385  Int& at, Char separator);
386 
387  // Derive the types from the values in the first data line.
388  static void getTypes (const IPosition& shape,
389  const Char* in, Int leng,
390  Char* string1, Char* string2, Char separator);
391 
392  // Turn the string into a Bool value.
393  // Empty string, value 0 and any value starting with f, F, n or N are False.
394  static Bool makeBool (const String& str);
395 
396  // Handle a keyword set.
397  static void handleKeyset (Int lineSize, char* string1,
398  char* first, char* second,
399  TableRecord& keysets,
400  LogIO& logger,
401  const String& fileName,
402  ifstream& jFile,
403  Int& lineNumber,
404  Char separator,
405  Bool testComment,
406  const Regex& commentMarker,
407  Int firstLine, Int lastLine);
408 
409  // Get the shape and type from the type string.
410  static Int getTypeShape (const String& typestr,
411  IPosition& shape, Int& type);
412 
413  // Get the next scalar value with the given type from string1.
414  static Bool getValue (char* string1, Int lineSize, char* first,
415  Int& at1, Char separator,
416  Int type, void* value);
417 
418  // Handle the next scalar with the given type from the data line and
419  // put it into the table column.
420  static void handleScalar (char* string1, Int lineSize, char* first,
421  Int& at1, Char separator,
422  Int type,
423  TableColumn& tabcol, rownr_t rownr);
424 
425  // Get the next array with the given type from string1.
426  // It returns the shape (for variable shaped arrays).
427  static IPosition getArray (char* string1, Int lineSize, char* first,
428  Int& at1, Char separator,
429  const IPosition& shape, Int varAxis,
430  Int type, void* valueBlock);
431 
432  // Get the next array with the given type from the data line and
433  // put it into the table column.
434  static void handleArray (char* string1, Int lineSize, char* first,
435  Int& at1, Char separator,
436  const IPosition& shape, Int varAxis,
437  Int type,
438  TableColumn& tabcol, rownr_t rownr);
439 };
440 
441 
442 
443 } //# NAMESPACE CASACORE - END
444 
445 #endif
Helper class for readAsciiTable.
static Int getNext(const Char *string, Int strlen, Char *result, Int &at, Char separator)
Get the next part of the line using the separator as delimiter.
static IPosition getArray(char *string1, Int lineSize, char *first, Int &at1, Char separator, const IPosition &shape, Int varAxis, Int type, void *valueBlock)
Get the next array with the given type from string1.
static Bool getLine(ifstream &file, Int &lineNumber, char *line, Int lineSize, Bool testComment, const Regex &commentMarker, Int firstLine, Int lastLine)
Get the next line.
static void getTypes(const IPosition &shape, const Char *in, Int leng, Char *string1, Char *string2, Char separator)
Derive the types from the values in the first data line.
static Table runt(String &formatString, Table::TableType tableType, const String &headerfile, const String &filein, const String &tableproto, const String &tablename, Bool autoHeader, const IPosition &autoShape, const Vector< String > &columnNames, const Vector< String > &dataTypes, Char separator, const String &commentMarkerRegex, Int firstLine, Int lastLine)
static Table makeTab(String &formatString, Table::TableType tableType, const String &headerfile, const String &filein, const String &tableproto, const String &tablename, Bool autoHeader, const IPosition &autoShape, const Vector< String > &columnNames, const Vector< String > &dataTypes, Char separator, Bool testComment, const Regex &commentMarker, Int firstLine, Int lastLine)
Do the actual work of making and filling the table.
static void handleKeyset(Int lineSize, char *string1, char *first, char *second, TableRecord &keysets, LogIO &logger, const String &fileName, ifstream &jFile, Int &lineNumber, Char separator, Bool testComment, const Regex &commentMarker, Int firstLine, Int lastLine)
Handle a keyword set.
static String doRun(const String &headerfile, const String &filein, const String &tableproto, const String &tablename, Bool autoHeader, const IPosition &autoShape, const Vector< String > &columnNames, const Vector< String > &dataTypes, Char separator, Bool testComment, const Regex &commentMarker, Int firstLine, Int lastLine)
Do the actual run.
static void handleScalar(char *string1, Int lineSize, char *first, Int &at1, Char separator, Int type, TableColumn &tabcol, rownr_t rownr)
Handle the next scalar with the given type from the data line and put it into the table column.
static double stringToPos(const String &pos, Bool isDMS)
Read a position using MVAngle.
static Bool getValue(char *string1, Int lineSize, char *first, Int &at1, Char separator, Int type, void *value)
Get the next scalar value with the given type from string1.
static void handleArray(char *string1, Int lineSize, char *first, Int &at1, Char separator, const IPosition &shape, Int varAxis, Int type, TableColumn &tabcol, rownr_t rownr)
Get the next array with the given type from the data line and put it into the table column.
static Int getTypeShape(const String &typestr, IPosition &shape, Int &type)
Get the shape and type from the type string.
static Bool makeBool(const String &str)
Turn the string into a Bool value.
static String run(const String &headerfile, const String &filein, const String &tableproto, const String &tablename, Bool autoHeader, const IPosition &autoShape, const Vector< String > &columnNames, const Vector< String > &dataTypes, Char separator, const String &commentMarkerRegex, Int firstLine, Int lastLine)
Run the readAsciiTable.
String: the storage and methods of handling collections of characters.
Definition: String.h:225
TableType
Define the possible table types.
Definition: Table.h:188
struct Node * first
Definition: malloc.h:330
const Double second
Time interval [T]:
this file contains all the compiler specific defines
Definition: mainpage.dox:28
const Bool False
Definition: aipstype.h:44
TableExprNode shape(const TableExprNode &array)
Function operating on any scalar or array resulting in a Double array containing the shape.
Definition: ExprNode.h:1987
int Int
Definition: aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
LatticeExprNode value(const LatticeExprNode &expr)
This function returns the value of the expression without a mask.
uInt64 rownr_t
Define the type of a row number in a table.
Definition: aipsxtype.h:46
char Char
Definition: aipstype.h:46
String readAsciiTable(const String &headerFile, const String &dataFile, const String &tableDescName, const char *tablename, Char separator=' ', const String &commentMarkerRegex="", Int firstLine=1, Int lastLine=-1)
String readAsciiTable(const String &headerFile, const String &dataFile, const String &tableDescName, const String &tablename, Char separator=' ', const String &commentMarkerRegex="", Int firstLine=1, Int lastLine=-1)
This form reads TWO Ascii files.
Table readAsciiTable(String &formatString, Table::TableType tableType, const String &filein, const String &tableproto, const String &tablename, const Vector< String > &columnNames, const Vector< String > &dataTypes, Char separator, const String &commentMarkerRegex, Int firstLine, Int lastLine)
String readAsciiTable(const String &filein, const String &tableproto, const String &tablename, const Vector< String > &columnNames, const Vector< String > &dataTypes, Char separator, const String &commentMarkerRegex, Int firstLine, Int lastLine)
This form gets the header info in the given vectors.
String readAsciiTable(const String &filein, const String &tableDescName, const String &tableName, Bool autoHeader=False, Char separator=' ', const String &commentMarkerRegex="", Int firstLine=1, Int lastLine=-1, const IPosition &autoShape=IPosition())
Create a table with name as given by tableName.
Table readAsciiTable(String &formatString, Table::TableType tableType, const String &headerFile, const String &dataFile, const String &tableDescName, const String &tablename, Char separator=' ', const String &commentMarkerRegex="", Int firstLine=1, Int lastLine=-1)
Table readAsciiTable(String &formatString, Table::TableType tableType, const String &filein, const String &tableDescName, const String &tableName, Bool autoHeader=False, Char separator=' ', const String &commentMarkerRegex="", Int firstLine=1, Int lastLine=-1, const IPosition &autoShape=IPosition())
Similar versions as above, but returning a Table object.
Table readAsciiTable(String &formatString, Table::TableType tableType, const String &headerFile, const String &dataFile, const String &tableDescName, const char *tablename, Char separator=' ', const String &commentMarkerRegex="", Int firstLine=1, Int lastLine=-1)