#include <MultiHDF5.h>

Public Member Functions
	MultiHDF5 (const String &name, ByteIO::OpenOption, Int blockSize=0)
	Open or create a MultiHDF5 with the given name.

	MultiHDF5 (const String &name, const std::shared_ptr< MultiFileBase > &parent, ByteIO::OpenOption, Int blockSize=0)
	Open or create a MultiHDF5 which is nested in the given parent.

	~MultiHDF5 () override
	The destructor flushes and closes the file.

	MultiHDF5 (const MultiHDF5 &)=delete
	Copy constructor and assignment not possible.

MultiHDF5 &	operator= (const MultiHDF5 &)=delete

std::shared_ptr< MultiFileBase >	makeNested (const std::shared_ptr< MultiFileBase > &parent, const String &name, ByteIO::OpenOption, Int blockSize) const override
	Make the correct MultiFileBase object for a nested file.

void	reopenRW () override
	Open the given logical file and return its file id.

void	fsync () override
	Fsync the file (i.e., force the data to be physically written).

const std::shared_ptr< HDF5File > &	getHDF5File () const
	Get the HDF5File object.

const HDF5Object &	getHDF5Object () const

Public Member Functions inherited from casacore::MultiFileBase
	MultiFileBase (const String &name, Int blockSize, Bool useODirect)
	Create a MultiFileBase object with the given name.

virtual	~MultiFileBase ()
	The destructor flushes dirty blocks and closes the container file.

	MultiFileBase (const MultiFileBase &)=delete
	Forbid copy constructor.

MultiFileBase &	operator= (const MultiFileBase &)=delete
	Forbid assignment.

String	fileName () const
	Get the file name of the MultiFileBase container file.

Bool	isWritable () const
	Is the container file writable?

Int	openFile (const String &name)
	Open the given logical file and return its file id.

Int	createFile (const String &name, ByteIO::OpenOption=ByteIO::New)
	Create a new logical file and return its file id.

void	flushFile (Int fileId)
	Flush the possible dirty buffer of the given logical file.

void	closeFile (Int fileId)
	Close a logical file.

void	deleteFile (Int fileId)
	Delete a logical file.

Int64	fileSize (Int fileId) const
	Get the size of a logical file.

Int64	read (Int fileId, void *buffer, Int64 size, Int64 offset)
	Read a block at the given offset in the logical file.

Int64	write (Int fileId, const void *buffer, Int64 size, Int64 offset)
	Write a block at the given offset in the logical file.

void	truncate (Int fileId, Int64 size)
	Truncate the logical file to the given size.

void	flush ()
	Flush the file by writing all dirty data and all header info.

Int64	blockSize () const
	Get the block size used.

uInt	nfile () const
	Get the nr of logical files.

Int64	nblock () const
	Get the total nr of data blocks used.

const std::vector< MultiFileInfo > &	info () const
	Get the info object (for test purposes mainly).

const std::vector< Int64 > &	freeBlocks () const
	Get the free blocks (for test purposes mainly).

Int	fileId (const String &name, Bool throwExcp=True) const
	Return the file id of a file in the MultiFileBase object.

Bool	useODirect () const
	Is O_DIRECT used?

Private Member Functions
void	init (ByteIO::OpenOption option)
	Initialize the MultiFile object.

void	doOpenFile (MultiFileInfo &) override
	Do the class-specific actions on opening a file.

void	doCloseFile (MultiFileInfo &) override
	Do the class-specific actions on closing a file.

void	doAddFile (MultiFileInfo &) override
	Do the class-specific actions on adding a file.

void	doDeleteFile (MultiFileInfo &) override
	Do the class-specific actions on deleting a file.

void	doTruncateFile (MultiFileInfo &info, uInt64 nrblk) override
	Truncate the file to `nrblk` blocks (does nothing).

void	doFlushFile () override
	Flush the file itself.

void	close () override
	Flush and close the file.

void	writeHeader () override
	Write the header info.

void	readHeader (Bool always=True) override
	Read the header info.

void	extend (MultiFileInfo &info, Int64 lastblk) override
	Extend the virtual file to fit lastblk.

void	readBlock (MultiFileInfo &info, Int64 blknr, void *buffer) override
	Read a data block.

void	writeBlock (MultiFileInfo &info, Int64 blknr, const void *buffer) override
	Write a data block.

Private Attributes
std::shared_ptr< HDF5File >	itsFile

std::shared_ptr< HDF5Group >	itsGroup

HDF5Object *	itsHDF5
	This points to the HDF5Group if present, otherwise to the HDF5File.

Additional Inherited Members
Static Public Member Functions inherited from casacore::MultiFileBase
static std::shared_ptr< MultiFileBase >	openMF (const String &fileName)
	Open the correct MultiFileBase (as plain or HDF5).

Protected Member Functions inherited from casacore::MultiFileBase
void	resync ()
	Resync with another process by clearing the buffers and rereading the header.

void	setNewFile ()
	Set the flags and blockSize for a new MultiFile/HDF5.

Protected Attributes inherited from casacore::MultiFileBase
String	itsName

Int64	itsBlockSize

Int64	itsNrBlock

Int64	itsHdrCounter

std::vector< MultiFileInfo >	itsInfo

std::shared_ptr< MultiFileBuffer >	itsBuffer

Bool	itsUseODirect

Bool	itsWritable

Bool	itsChanged

std::vector< Int64 >	itsFreeBlocks

Detailed Description

Class to combine multiple files in a single HDF5 file.

Intended use:

Public interface

Review Status

Test programs:: tMultiHDF5

Synopsis

This class is a container file holding multiple virtual files. It is primarily meant as a container file for the storage manager files of a table to reduce the number of files used (especially for Lustre) and to reduce the number of open files (especially when concatenating tables).
A secondary goal is offering the ability to use an IO buffer size that matches the file system well (large buffer size for e.g. ZFS).

The SetupNewTable constructor has a StorageOption argument to define if a MultiFile has to be used and if so, the buffer size to use. It is also possible to specify that through aipsrc variables.

A virtual file is spread over multiple (fixed size) data blocks in the MultiFile. A data block is never shared by multiple files. For each virtual file MultiFile keeps a MultiFileInfo object telling the file size and the blocks numbers used for the file. When flushing the MultiFile, this meta info is written into a header block and, if needed, continuation blocks. On open and resync, it is read back.

A virtual file is represented by an MFFileIO object, which is derived from ByteIO and as such part of the casacore IO framework. It makes it possible for applications to access a virtual file in the same way as a regular file.

It is possible to delete a virtual file. Its blocks will be added to the free block list (which is also stored in the meta info).

Example

In principle it is possible to use the MultiFile functions directly. However, in general it is much easier to use an MFFileIO object per virtual file as shown below.

// Create a new MultiFile using a block size of 1 MB.
MultiFile mfile("file.mf', ByteIO::New, 1048576);
// Create a virtual file in it.
MFFileIO mf1(mfile, "mf1", ByteIO::New);
// Use it (for example) as the sink of AipsIO.
AipsIO stream (&mf1);
// Write values.
stream << (Int)10;
stream << True;
// Seek to beginning of file and read data in.
stream.setpos (0);
Int vali;
Bool valb;
stream >> vali >> valb;

Definition at line 100 of file MultiHDF5.h.

Constructor & Destructor Documentation

◆ MultiHDF5() [1/3]

casacore::MultiHDF5::MultiHDF5	(	const String &	name,
		ByteIO::OpenOption	,
		Int	blockSize = `0`
	)

explicit

Open or create a MultiHDF5 with the given name.

Upon creation the block size can be given. If 0, it uses the block size of the file system the file is on.

◆ MultiHDF5() [2/3]

casacore::MultiHDF5::MultiHDF5	(	const String &	name,
		const std::shared_ptr< MultiFileBase > &	parent,
		ByteIO::OpenOption	,
		Int	blockSize = `0`
	)

explicit

Open or create a MultiHDF5 which is nested in the given parent.

The data are read/written in a group with the given name in the parent. Upon creation the block size can be given. If 0, it uses the block size of the parent.

◆ ~MultiHDF5()

casacore::MultiHDF5::~MultiHDF5 ( )

override

The destructor flushes and closes the file.

◆ MultiHDF5() [3/3]

casacore::MultiHDF5::MultiHDF5 ( const MultiHDF5 & )

delete

Copy constructor and assignment not possible.

Member Function Documentation

◆ close()

void casacore::MultiHDF5::close ( )

overrideprivatevirtual

Flush and close the file.

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Intended use:

Review Status

Synopsis

Example

Constructor & Destructor Documentation

◆ MultiHDF5() [1/3]

◆ MultiHDF5() [2/3]

◆ ~MultiHDF5()

◆ MultiHDF5() [3/3]

Member Function Documentation

◆ close()

◆ doAddFile()

◆ doCloseFile()

◆ doDeleteFile()

◆ doFlushFile()

◆ doOpenFile()

◆ doTruncateFile()

◆ extend()

◆ fsync()

◆ getHDF5File()

◆ getHDF5Object()

◆ init()

◆ makeNested()

◆ operator=()

◆ readBlock()

◆ readHeader()

◆ reopenRW()

◆ writeBlock()

◆ writeHeader()

Member Data Documentation

◆ itsFile

◆ itsGroup

◆ itsHDF5