casacore
|
Class that provides binary table I/O. More...
#include <BufferedColumnarFile.h>
Public Member Functions | |
VarBufferedColumnarFile () noexcept=default | |
VarBufferedColumnarFile (const VarBufferedColumnarFile &rhs)=delete | |
VarBufferedColumnarFile (VarBufferedColumnarFile &&rhs) noexcept | |
~VarBufferedColumnarFile () noexcept | |
VarBufferedColumnarFile & | operator= (VarBufferedColumnarFile &&rhs) |
void | Close () |
Close the file. | |
void | Read (uint64_t row, uint64_t column_offset, float *data, uint64_t n) |
Read one cell containing an array of floats. | |
void | Read (uint64_t row, uint64_t column_offset, double *data, uint64_t n) |
Read array of doubles. | |
void | Read (uint64_t row, uint64_t column_offset, int32_t *data, uint64_t n) |
Read array of int32_t. | |
void | Read (uint64_t row, uint64_t column_offset, std::complex< float > *data, uint64_t n) |
Read array of complex floats. | |
void | Read (uint64_t row, uint64_t column_offset, bool *data, uint64_t n) |
Read an array of bools. | |
void | Write (uint64_t row, uint64_t column_offset, const float *data, uint64_t n) |
Write one cell containing an array of floats. | |
void | Write (uint64_t row, uint64_t column_offset, const double *data, uint64_t n) |
Write an array of doubles. | |
void | Write (uint64_t row, uint64_t column_offset, const int32_t *data, uint64_t n) |
Write an array of int32_t. | |
void | Write (uint64_t row, uint64_t column_offset, const std::complex< float > *data, uint64_t n) |
Write an array of complex floats. | |
void | Write (uint64_t row, uint64_t column_offset, const std::complex< double > *data, uint64_t n) |
Write an array of complex doubles. | |
void | Write (uint64_t row, uint64_t column_offset, const bool *data, uint64_t n) |
Write an array of bools. | |
void | SetStride (uint64_t new_stride) |
Set the number of bytes per row for this file. | |
void | AddRows (uint64_t n_rows) |
Adds a given number of rows to the back of the file. | |
void | DeleteRow () |
Deletes the last row. | |
const std::string & | Filename () const |
bool | IsOpen () const |
uint64_t | NRows () const |
Total number of rows stored in this file. | |
void | ReadHeader (unsigned char *data) |
Read an optional extra header to the file. | |
uint64_t | Stride () const |
Total number of bytes in one row. | |
void | WriteHeader (const unsigned char *data) |
Write an optional extra header to the file. | |
Static Public Member Functions | |
static VarBufferedColumnarFile | CreateNew (const std::string &filename, uint64_t header_size, uint64_t stride) |
Create a new file on disk. | |
static VarBufferedColumnarFile | OpenExisting (const std::string &filename, size_t header_size) |
Open an existing file from disk. | |
Private Member Functions | |
VarBufferedColumnarFile (const std::string &filename, uint64_t header_size, uint64_t stride) | |
Create or overwrite a new columnar file on disk. | |
VarBufferedColumnarFile (const std::string &filename, size_t header_size) | |
Open an existing columnar file. | |
void | ActivateBlock (uint64_t row) |
template<typename ValueType > | |
void | ReadImplementation (uint64_t row, uint64_t column_offset, ValueType *data, uint64_t n) |
template<typename ValueType > | |
void | WriteImplementation (uint64_t row, uint64_t column_offset, const ValueType *data, uint64_t n) |
void | WriteActiveBlock () |
![]() | |
RowBasedFile ()=default | |
RowBasedFile (const RowBasedFile &rhs)=delete | |
RowBasedFile (RowBasedFile &&rhs) noexcept | |
RowBasedFile (const std::string &filename, uint64_t header_size, uint64_t stride) | |
Create or overwrite a new columnar file on disk. | |
RowBasedFile (const std::string &filename, size_t header_size) | |
Open an existing columnar file. | |
~RowBasedFile () noexcept | |
RowBasedFile & | operator= (RowBasedFile &&rhs) |
void | Close () |
Close the file. | |
void | Truncate (uint64_t n_rows) |
void | Seek (off_t pos, int seek_direction) |
void | ReadData (unsigned char *data, uint64_t size) |
void | WriteData (const unsigned char *data, uint64_t size) |
bool | IsOpen () const |
uint64_t | DataLocation () const |
Offset of the first row in the file. | |
const std::string & | Filename () const |
uint64_t | HeaderSize () const |
Number of bytes reserved for an optional header. | |
void | WriteHeader (const unsigned char *data) |
Write an optional extra header to the file. | |
void | ReadHeader (unsigned char *data) |
Read an optional extra header to the file. | |
uint64_t | NRows () const |
Total number of rows stored in this file. | |
void | SetNRows (uint64_t new_n_rows) |
uint64_t | Stride () const |
Total number of bytes in one row. | |
void | SetStride (uint64_t new_stride) |
Set the number of bytes per row for this file. | |
void | AddRows (uint64_t n_rows) |
Adds a given number of rows to the back of the file. | |
void | DeleteRow () |
Deletes the last row. | |
Private Attributes | |
std::vector< unsigned char > | packed_buffer_ |
This buffer is used temporarily for (un)packing booleans. | |
bool | block_changed_ |
uint64_t | active_block_ |
uint64_t | rows_per_block_ |
std::vector< unsigned char > | block_buffer_ |
Class that provides binary table I/O.
It is rather low-level, and requires the user to keep track of the columns and their datatype. It uses a simple binary format where the columns are interleaved into the output file. Booleans are written with bit-packing.
This class limits calls to the read/write functions by caching a block of data around the last accessed row. The size of the buffer is templated to make testing of the caching easier. Production code can normally use the alias BufferedColumnarFile, which uses a reasonable buffer size.
This class aims to be as fast as possible for large data files that are read from or written to consecutively. It does not try to optimize random access, partial data access and access is not transactional.
The class uses exceptions to handle any I/O errors.
Definition at line 33 of file BufferedColumnarFile.h.
|
defaultnoexcept |
|
delete |
|
inlinenoexcept |
Definition at line 47 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::block_changed_.
|
inlinenoexcept |
Definition at line 58 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::block_changed_, casacore::RowBasedFile::DataLocation(), casacore::VarBufferedColumnarFile< BufferSize >::IsOpen(), casacore::VarBufferedColumnarFile< BufferSize >::NRows(), casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_, casacore::RowBasedFile::Seek(), casacore::VarBufferedColumnarFile< BufferSize >::Stride(), and casacore::RowBasedFile::WriteData().
|
inlineprivate |
Create or overwrite a new columnar file on disk.
Definition at line 248 of file BufferedColumnarFile.h.
|
inlineprivate |
Open an existing columnar file.
Definition at line 257 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::packed_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_, and casacore::VarBufferedColumnarFile< BufferSize >::Stride().
|
inlineprivate |
Fill the remainder of block_buffer_ with zeroes. Doing it here makes the code robust and avoids the need for inserting zeroes when adding rows out-of-order, e.g., when adding row 5 while NRows() is 2."
Definition at line 267 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::block_changed_, casacore::RowBasedFile::DataLocation(), casacore::VarBufferedColumnarFile< BufferSize >::NRows(), casacore::RowBasedFile::ReadData(), casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_, casacore::RowBasedFile::Seek(), casacore::VarBufferedColumnarFile< BufferSize >::Stride(), and casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock().
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::Read(), casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation(), casacore::VarBufferedColumnarFile< BufferSize >::Write(), and casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation().
|
inline |
Adds a given number of rows to the back of the file.
Definition at line 232 of file RowBasedFile.h.
|
inline |
Close the file.
After closing, all calls to I/O functions cause undefined behaviour, untill the class is assigned to a new instance.
Even if writing fails, the file should still be closed to prevent the file to remain open after destruction.
Definition at line 87 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::block_changed_, casacore::RowBasedFile::Close(), casacore::VarBufferedColumnarFile< BufferSize >::IsOpen(), and casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock().
Referenced by casacore::AntennaPairFile::Close(), casacore::UvwFile::Close(), and casacore::VarBufferedColumnarFile< BufferSize >::operator=().
|
inlinestatic |
Create a new file on disk.
If the file exists, it is overwritten.
header | Optional header. |
stride | The number of bytes in one row (total over all columns). |
Definition at line 108 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile().
|
inline |
Deletes the last row.
Definition at line 237 of file RowBasedFile.h.
|
inline |
Definition at line 178 of file RowBasedFile.h.
Referenced by casacore::AntennaPairFile::Filename(), and casacore::UvwFile::Filename().
|
inline |
Definition at line 172 of file RowBasedFile.h.
Referenced by casacore::AntennaPairFile::Close(), casacore::VarBufferedColumnarFile< BufferSize >::Close(), casacore::UvwFile::Close(), casacore::UvwFile::ReadUvw(), casacore::UvwFile::WriteUvw(), and casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile().
|
inline |
Total number of rows stored in this file.
Definition at line 203 of file RowBasedFile.h.
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::Read(), casacore::UvwFile::ReadActiveBlock(), casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation(), casacore::UvwFile::UvwFile(), casacore::VarBufferedColumnarFile< BufferSize >::Write(), casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation(), and casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile().
|
inlinestatic |
Open an existing file from disk.
If the file does not exists, an exception is thrown.
header | Optional header. |
stride | The number of bytes in one row (total over all columns). |
Definition at line 120 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile().
|
inline |
Definition at line 71 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::block_changed_, casacore::VarBufferedColumnarFile< BufferSize >::Close(), casacore::RowBasedFile::operator=(), casacore::VarBufferedColumnarFile< BufferSize >::packed_buffer_, and casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_.
|
inline |
Read an array of bools.
See float version for documentation. Booleans are stored with bit-packing.
Definition at line 164 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::RowBasedFile::DataLocation(), casacore::VarBufferedColumnarFile< BufferSize >::NRows(), casacore::VarBufferedColumnarFile< BufferSize >::packed_buffer_, casacore::RowBasedFile::ReadData(), casacore::RowBasedFile::Seek(), casacore::VarBufferedColumnarFile< BufferSize >::Stride(), and UnpackBoolArray().
|
inline |
Read array of doubles.
See float version for documentation.
Definition at line 141 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation().
|
inline |
Read one cell containing an array of floats.
If the row was not written yet, zeros are returned.
row | The cell's row index. |
column_offset | The position of this column counted from the start of the row, in bytes. |
data | Buffer in which the data will be stored. |
n | Size of the column in number of elements (NOT in bytes!). |
Definition at line 134 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation().
Referenced by casacore::StokesIStManColumn::getArrayGeneric(), and casacore::UvwFile::ReadActiveBlock().
|
inline |
Read array of int32_t.
See float version for documentation.
Definition at line 148 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation().
|
inline |
Read array of complex floats.
See float version for documentation.
Definition at line 155 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation().
|
inline |
Read an optional extra header to the file.
Definition at line 196 of file RowBasedFile.h.
Referenced by casacore::UvwFile::ReadHeader().
|
inlineprivate |
Definition at line 290 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::NRows(), casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_, and casacore::VarBufferedColumnarFile< BufferSize >::Stride().
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::Read(), casacore::VarBufferedColumnarFile< BufferSize >::Read(), casacore::VarBufferedColumnarFile< BufferSize >::Read(), and casacore::VarBufferedColumnarFile< BufferSize >::Read().
|
inline |
Set the number of bytes per row for this file.
This changes the format of the file, and because of this the file is emptied.
Definition at line 236 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::block_changed_, casacore::VarBufferedColumnarFile< BufferSize >::packed_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_, and casacore::RowBasedFile::SetStride().
|
inline |
Total number of bytes in one row.
This value is also stored in the file, and is read from the file in OpenExisting().
Definition at line 214 of file RowBasedFile.h.
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::Read(), casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation(), casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile(), casacore::VarBufferedColumnarFile< BufferSize >::Write(), casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation(), and casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile().
|
inline |
Write an array of bools.
Bools are stored with bit-packing. See float version for documentation.
Definition at line 221 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::RowBasedFile::DataLocation(), casacore::VarBufferedColumnarFile< BufferSize >::NRows(), PackBoolArray(), casacore::VarBufferedColumnarFile< BufferSize >::packed_buffer_, casacore::RowBasedFile::Seek(), casacore::RowBasedFile::SetNRows(), casacore::VarBufferedColumnarFile< BufferSize >::Stride(), and casacore::RowBasedFile::WriteData().
|
inline |
Write an array of doubles.
See float version for documentation.
Definition at line 188 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation().
|
inline |
Write one cell containing an array of floats.
If the row is past the end of the file, the file is enlarged (making NRows() = row + 1).
Definition at line 180 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation().
Referenced by casacore::StokesIStManColumn::putArrayGeneric(), and casacore::UvwFile::WriteActiveBlock().
|
inline |
Write an array of int32_t.
See float version for documentation.
Definition at line 196 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation().
|
inline |
Write an array of complex doubles.
See float version for documentation.
Definition at line 212 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation().
|
inline |
Write an array of complex floats.
See float version for documentation.
Definition at line 204 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation().
|
inlineprivate |
Definition at line 319 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::block_changed_, casacore::RowBasedFile::DataLocation(), casacore::VarBufferedColumnarFile< BufferSize >::NRows(), casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_, casacore::RowBasedFile::Seek(), casacore::VarBufferedColumnarFile< BufferSize >::Stride(), and casacore::RowBasedFile::WriteData().
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), and casacore::VarBufferedColumnarFile< BufferSize >::Close().
|
inline |
Write an optional extra header to the file.
When creating the file, the requested space is saved to store this header.
data | An array equal to the size of the header given in the CreateNew() and OpenExisting() calls. |
Definition at line 189 of file RowBasedFile.h.
Referenced by casacore::UvwFile::WriteHeader().
|
inlineprivate |
Definition at line 306 of file BufferedColumnarFile.h.
References casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::active_block_, casacore::VarBufferedColumnarFile< BufferSize >::block_buffer_, casacore::VarBufferedColumnarFile< BufferSize >::block_changed_, casacore::VarBufferedColumnarFile< BufferSize >::NRows(), casacore::VarBufferedColumnarFile< BufferSize >::rows_per_block_, casacore::RowBasedFile::SetNRows(), and casacore::VarBufferedColumnarFile< BufferSize >::Stride().
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::Write(), casacore::VarBufferedColumnarFile< BufferSize >::Write(), casacore::VarBufferedColumnarFile< BufferSize >::Write(), casacore::VarBufferedColumnarFile< BufferSize >::Write(), and casacore::VarBufferedColumnarFile< BufferSize >::Write().
|
private |
Definition at line 335 of file BufferedColumnarFile.h.
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::operator=(), casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation(), casacore::VarBufferedColumnarFile< BufferSize >::SetStride(), casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile(), casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation(), and casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile().
|
private |
Definition at line 337 of file BufferedColumnarFile.h.
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::operator=(), casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation(), casacore::VarBufferedColumnarFile< BufferSize >::SetStride(), casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile(), casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation(), and casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile().
|
private |
Definition at line 334 of file BufferedColumnarFile.h.
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::Close(), casacore::VarBufferedColumnarFile< BufferSize >::operator=(), casacore::VarBufferedColumnarFile< BufferSize >::SetStride(), casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile(), casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation(), and casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile().
|
private |
This buffer is used temporarily for (un)packing booleans.
Storing it as a member avoids memory allocations.
Definition at line 332 of file BufferedColumnarFile.h.
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::operator=(), casacore::VarBufferedColumnarFile< BufferSize >::Read(), casacore::VarBufferedColumnarFile< BufferSize >::SetStride(), casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile(), and casacore::VarBufferedColumnarFile< BufferSize >::Write().
|
private |
Definition at line 336 of file BufferedColumnarFile.h.
Referenced by casacore::VarBufferedColumnarFile< BufferSize >::ActivateBlock(), casacore::VarBufferedColumnarFile< BufferSize >::operator=(), casacore::VarBufferedColumnarFile< BufferSize >::ReadImplementation(), casacore::VarBufferedColumnarFile< BufferSize >::SetStride(), casacore::VarBufferedColumnarFile< BufferSize >::VarBufferedColumnarFile(), casacore::VarBufferedColumnarFile< BufferSize >::WriteActiveBlock(), casacore::VarBufferedColumnarFile< BufferSize >::WriteImplementation(), and casacore::VarBufferedColumnarFile< BufferSize >::~VarBufferedColumnarFile().