MultiHDF5.h

Go to the documentation of this file.

 
//# MultiHDF5.h: Class to combine multiple files in a single HDF5 file
//# Copyright (C) 2015
//# Associated Universities, Inc. Washington DC, USA.
//#
//# This library is free software; you can redistribute it and/or modify it
//# under the terms of the GNU Library General Public License as published by
//# the Free Software Foundation; either version 2 of the License, or (at your
//# option) any later version.
//#
//# This library is distributed in the hope that it will be useful, but WITHOUT
//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
//# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
//# License for more details.
//#
//# You should have received a copy of the GNU Library General Public License
//# along with this library; if not, write to the Free Software Foundation,
//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
//#
//# Correspondence concerning AIPS++ should be addressed as follows:
//#        Internet email: casa-feedback@nrao.edu.
//#        Postal address: AIPS++ Project Office
//#                        National Radio Astronomy Observatory
//#                        520 Edgemont Road
//#                        Charlottesville, VA 22903-2475 USA
 
#ifndef CASA_MULTIHDF5_H
#define CASA_MULTIHDF5_H
 
//# Includes
#include <casacore/casa/aips.h>
#include <casacore/casa/IO/MultiFile.h>
#include <casacore/casa/HDF5/HDF5File.h>
#include <casacore/casa/HDF5/HDF5Group.h>
#include <memory>
 
 
namespace casacore { //# NAMESPACE CASACORE - BEGIN
 
  // <summary> 
  // Class to combine multiple files in a single HDF5 file.
  // </summary>
 
  // <use visibility=export>
 
  // <reviewed reviewer="" date="" tests="tMultiHDF5" demos="">
  // </reviewed>
 
  // <synopsis> 
  // This class is a container file holding multiple virtual files. It is
  // primarily meant as a container file for the storage manager files of a
  // table to reduce the number of files used (especially for Lustre) and to
  // reduce the number of open files (especially when concatenating tables).
  // <br>A secondary goal is offering the ability to use an IO buffer size
  // that matches the file system well (large buffer size for e.g. ZFS).
  //
  // The SetupNewTable constructor has a StorageOption argument to define
  // if a MultiFile has to be used and if so, the buffer size to use.
  // It is also possible to specify that through aipsrc variables.
  //
  // A virtual file is spread over multiple (fixed size) data blocks in the
  // MultiFile. A data block is never shared by multiple files.
  // For each virtual file MultiFile keeps a MultiFileInfo object telling
  // the file size and the blocks numbers used for the file. When flushing
  // the MultiFile, this meta info is written into a header block and,
  // if needed, continuation blocks. On open and resync, it is read back.
  // <br>
  //
  // A virtual file is represented by an MFFileIO object, which is derived
  // from ByteIO and as such part of the casacore IO framework. It makes it
  // possible for applications to access a virtual file in the same way as
  // a regular file.
  //
  // It is possible to delete a virtual file. Its blocks will be added to
  // the free block list (which is also stored in the meta info).
  // </synopsis>
 
  // <example>
  // In principle it is possible to use the MultiFile functions directly.
  // However, in general it is much easier to use an MFFileIO object
  // per virtual file as shown below.
  // <srcblock>
  //    // Create a new MultiFile using a block size of 1 MB.
  //    MultiFile mfile("file.mf', ByteIO::New, 1048576);
  //    // Create a virtual file in it.
  //    MFFileIO mf1(mfile, "mf1", ByteIO::New);
  //    // Use it (for example) as the sink of AipsIO.
  //    AipsIO stream (&mf1);
  //    // Write values.
  //    stream << (Int)10;
  //    stream << True;
  //    // Seek to beginning of file and read data in.
  //    stream.setpos (0);
  //    Int vali;
  //    Bool valb;
  //    stream >> vali >> valb;
  // </srcblock>
  // </example>
 
  class MultiHDF5 : public MultiFileBase
  {
  public:
    // Open or create a MultiHDF5 with the given name.
    // Upon creation the block size can be given. If 0, it uses the block size
    // of the file system the file is on.
    explicit MultiHDF5 (const String& name, ByteIO::OpenOption, Int blockSize=0);
 
 
    // Open or create a MultiHDF5 which is nested in the given parent.
    // The data are read/written in a group with the given name in the parent.
    // Upon creation the block size can be given. If 0, it uses the block size
    // of the parent.
    explicit MultiHDF5 (const String& name,
                        const std::shared_ptr<MultiFileBase>& parent,
                        ByteIO::OpenOption, Int blockSize=0);
 
    // The destructor flushes and closes the file.
    ~MultiHDF5() override;
 
    // Copy constructor and assignment not possible.
    MultiHDF5 (const MultiHDF5&) = delete;
    MultiHDF5& operator= (const MultiHDF5&) = delete;
 
    // Make the correct MultiFileBase object for a nested file.
    // It creates a new group under which the virtual files are created.
    std::shared_ptr<MultiFileBase> makeNested
    (const std::shared_ptr<MultiFileBase>& parent, const String& name,
     ByteIO::OpenOption, Int blockSize) const override;
                                                  
    // Open the given logical file and return its file id.
    // If the name is unknown, an exception is thrown.
    // It creates the HDF5 group and dataset opbject.
    // Reopen the underlying file for read/write access.
    // Nothing will be done if the file is writable already.
    // Otherwise it will be reopened and an exception will be thrown
    // if it is not possible to reopen it for read/write access.
    void reopenRW() override;
 
    // Fsync the file (i.e., force the data to be physically written).
    void fsync() override;
 
    // Get the HDF5File object.
    const std::shared_ptr<HDF5File>& getHDF5File() const
      { return itsFile; }
    const HDF5Object& getHDF5Object() const
      { return *itsHDF5; }
 
  private:
    // Initialize the MultiFile object.
    void init (ByteIO::OpenOption option);
    // Do the class-specific actions on opening a file.
    void doOpenFile (MultiFileInfo&) override;
    // Do the class-specific actions on closing a file.
    void doCloseFile (MultiFileInfo&) override;
    // Do the class-specific actions on adding a file.
    void doAddFile (MultiFileInfo&) override;
    // Do the class-specific actions on deleting a file.
    void doDeleteFile (MultiFileInfo&) override;
    // Truncate the file to <src>nrblk</src> blocks (does nothing).
    void doTruncateFile (MultiFileInfo& info, uInt64 nrblk) override;
    // Flush the file itself.
    void doFlushFile() override;
    // Flush and close the file.
    void close() override;
    // Write the header info.
    void writeHeader() override;
    // Read the header info. If always==False, the info is only read if the
    // header counter has changed.
    void readHeader (Bool always=True) override;
    // Extend the virtual file to fit lastblk.
    void extend (MultiFileInfo& info, Int64 lastblk) override;
    // Read a data block.
    void readBlock (MultiFileInfo& info, Int64 blknr,
                    void* buffer) override;
    // Write a data block.
    void writeBlock (MultiFileInfo& info, Int64 blknr,
                     const void* buffer) override;
 
    //# Data members
    std::shared_ptr<HDF5File>  itsFile;
    std::shared_ptr<HDF5Group> itsGroup;
    // This points to the HDF5Group if present, otherwise to the HDF5File.
    // It tells where the HDF5 data are accessed.
    HDF5Object*                itsHDF5;
  };
 
 
} //# NAMESPACE CASACORE - END
 
#endif