MultiFile.h

Go to the documentation of this file.

//# MultiFile.h: Class to combine multiple files in a single one
//# Copyright (C) 2014
//# Associated Universities, Inc. Washington DC, USA.
//#
//# This library is free software; you can redistribute it and/or modify it
//# under the terms of the GNU Library General Public License as published by
//# the Free Software Foundation; either version 2 of the License, or (at your
//# option) any later version.
//#
//# This library is distributed in the hope that it will be useful, but WITHOUT
//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
//# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
//# License for more details.
//#
//# You should have received a copy of the GNU Library General Public License
//# along with this library; if not, write to the Free Software Foundation,
//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
//#
//# Correspondence concerning AIPS++ should be addressed as follows:
//#        Internet email: casa-feedback@nrao.edu.
//#        Postal address: AIPS++ Project Office
//#                        National Radio Astronomy Observatory
//#                        520 Edgemont Road
//#                        Charlottesville, VA 22903-2475 USA
 
#ifndef CASA_MULTIFILE_H
#define CASA_MULTIFILE_H
 
//# Includes
#include <casacore/casa/aips.h>
#include <casacore/casa/IO/MultiFileBase.h>
#include <memory>
 
namespace casacore { //# NAMESPACE CASACORE - BEGIN
 
  //# Forward declarations.
  class ByteIO;
  class CanonicalIO;
  class MemoryIO;
 
 
  // <summary> 
  // Class to combine multiple files in a single one.
  // </summary>
 
  // <use visibility=export>
 
  // <reviewed reviewer="" date="" tests="tMultiFile" demos="">
  // </reviewed>
 
  // <synopsis> 
  // This class (derived from MultiFileBase) is a container file holding
  // multiple virtual files in a regular file.
  // It is primarily meant as a container file for the storage manager files
  // of a table to reduce the number of files used (especially for Lustre) and
  // to reduce the number of open files (especially when concatenating tables).
  // <br> MultiFile has the following properties:
  // <ul>
  //  <li> It can choose an IO buffer size that matches the file system well
  //       (e.g., to support a large buffer size on ZFS or Lustre).
  //  <li> O_DIRECT (if supported by the OS) can be used to tell the OS kernel
  //       to bypass its file cache. It does not speed up the I/O, but it makes
  //       I/O behaviour more predictable which a real-time system might need.
  //  <li> Often the data to be read from MultiFile will not exactly match the
  //       block size and offset. MultiFile will buffer the data and copy the
  //       part that is needed (similar to stdio). However, when matching
  //       block size and offset are used, data will directly be read into the
  //       user's buffer to achieve zero-copy behaviour.
  //  <li> It is possible to nest MultiFile's. Thus a MultiFile can be a file
  //       in a parent MultiFile. In this way it is easily possible to store
  //       a main table and its subtables (such as an MS) in a single file. 
  //  <li> Optionally each block is stored with a 32-bit CRC to check if the
  //       data in a block are correctly read. The CRC values are stored as
  //       part of the header, thus not in each individual block. This is done
  //       to make the zero-copy behaviour possible (as described above).
  //  <li> The header and the index are stored in the first block. If too large,
  //       continuation blocks are used. There are two sets of continuation
  //       blocks between which is alternated. This is done for robustness
  //       purposes; there is always a valid one in case of a crash in the
  //       middle of writing the continuation blocks. Note that the first
  //       header block is written after the continuation blocks, so it always
  //       points to a valid set of continuation blocks.
  // </ul>
  //
  // The SetupNewTable constructor has a StorageOption argument to define
  // if a MultiFile has to be used and if so, the buffer size to use.
  // It is also possible to specify that through aipsrc variables.
  //
  // A virtual file is spread over multiple (fixed size) data blocks in the
  // MultiFile. A data block is never shared by multiple files.
  // For each virtual file MultiFile keeps a MultiFileInfo object telling
  // the file size and the block numbers used for the file. When flushing
  // the MultiFile, this meta info is written into the header block. If it
  // does not fit in the header block, the rest is written in continuation blocks.
  // On open and resync, it is read back. There are two sets of continuation
  // blocks which are alternately used when the header is written. This is done
  // to have a valid header in case of a crash in the middle of writing the header.
  //
  // A virtual file is represented by an MFFileIO object, which is derived
  // from ByteIO and as such part of the casacore IO framework. It makes it
  // possible for applications to access a virtual file in the same way as
  // a regular file.
  //
  // It is possible to delete a virtual file. Its blocks will be added to
  // the free block list (which is also stored in the meta info).
  // The MultiFile is truncated when blocks are deleted at the end of the file.
  // </synopsis>
 
  // <example>
  // In principle it is possible to use the MultiFile functions directly.
  // However, in general it is much easier to use an MFFileIO object
  // per virtual file as shown below.
  // <srcblock>
  //    // Create a new MultiFile using a block size of 1 MB.
  //    std::shared_ptr<MultiFileBase> mfile
  //           (new MultiFile("file.mf", ByteIO::New, 1048576));
  //    // Create a virtual file in it.
  //    MFFileIO mf1(mfile, "mf1", ByteIO::New);
  //    // Use it (for example) as the sink of AipsIO.
  //    AipsIO stream (&mf1);
  //    // Write values.
  //    stream << (Int)10;
  //    stream << True;
  //    // Seek to beginning of file and read data in.
  //    stream.setpos (0);
  //    Int vali;
  //    Bool valb;
  //    stream >> vali >> valb;
  // </srcblock>
  // </example>
 
  // <todo>
  //  <li> MultiFile can be optimized how cont.blocks are used. In case of
  //       file truncation, it could check if only cont.blocks are present
  //       after the blocks to be removed. In such a case they can be moved
  //       backwards. Also the nr of cont.blocks can shrink. In such a case
  //       the unused blocks are not added to the free list. Only the nr of
  //       actually used cont.blocks is decremented. They could be added to
  //       the free list later.
  //       The reason for above is that the free list is written into the
  //       header blocks before the required nr of continuation blocks is known.
  //  <li> Keep a journal file telling which files are created and which
  //       blocks are allocated for a virtual file.
  // </todo>
 
  class MultiFile: public MultiFileBase
  {
  public:
    // Open or create a MultiFile with the given name.
    // Upon creation the block size can be given. If 0, it uses the block size
    // of the file system the file is on.
    // <br>If useODirect=True, the O_DIRECT flag is used (if supported).
    // It tells the kernel to bypass its file cache to have more predictable
    // I/O behaviour.
    // <br>If useCRC=True, 32-bit CRC values are calculated and stored for
    // each data block. Note that useCRC is only used for new files.
    explicit MultiFile (const String& name, ByteIO::OpenOption, Int blockSize=0,
                        Bool useODirect=False, Bool useCRC=False);
 
    // Open or create a MultiFile with the given name which is nested in the
    // given parent. Thus data are read/written in the parent file.
    // Upon creation the block size can be given. If 0, it uses the block size
    // of the parent.
    explicit MultiFile (const String& name,
                        const std::shared_ptr<MultiFileBase>& parent,
                        ByteIO::OpenOption, Int blockSize=0);
 
    // The destructor flushes and closes the file.
    ~MultiFile() override;
 
    // Copy constructor and assignment not possible.
    MultiFile (const MultiFile&) = delete;
    MultiFile& operator= (const MultiFile&) = delete;
 
    // Make a nested MultiFile.
    std::shared_ptr<MultiFileBase> makeNested
    (const std::shared_ptr<MultiFileBase>& parent, const String& name,
     ByteIO::OpenOption, Int blockSize) const override;
 
    // Reopen the underlying file for read/write access.
    // Nothing will be done if the file is writable already.
    // Otherwise it will be reopened and an exception will be thrown
    // if it is not possible to reopen it for read/write access.
    void reopenRW() override;
 
    // Fsync the file (i.e., force the data to be physically written).
    void fsync() override;
 
    // Show some info.
    void show (std::ostream&) const;
 
    // Compress a block index by looking for subsequent block numbers.
    static std::vector<Int64> packIndex (const std::vector<Int64>& blockNrs);
 
    // Decompress a block index by inserting subsequent block numbers.
    static std::vector<Int64> unpackIndex (const std::vector<Int64>& blockNrs);
 
  private:
    // Initialize the MultiFile object.
    void init (ByteIO::OpenOption option);
    // Read the file info for the new version 2.
    void getInfoVersion2 (Int64 contBlockNr, CanonicalIO& aio);
    // Write a vector of Int64.
    void writeVector (CanonicalIO& cio, const std::vector<Int64>& index);
    void writeVector (CanonicalIO& cio, const std::vector<uInt>& index);
    // Read a vector of Int64.
    void readVector (CanonicalIO& cio, std::vector<Int64>& index);
    void readVector (CanonicalIO& cio, std::vector<uInt>& index);
    // Write the remainder of the header (in case exceeding 1 block).
    // <src>iobuf</src> should be large enough
    void writeRemainder (MemoryIO& mio, CanonicalIO&, MultiFileBuffer& mfbuf);
    // Read the remainder of the header into the buffer.
    void readRemainder (Int64 headerSize, Int64 blockNr, std::vector<char>& buf);
    // Truncate the file if blocks are freed at the end.
    void truncateIfNeeded();
    // Header writing hooks (meant for derived test classes).
    virtual void writeHeaderShow (Int64 ncont, Int64 todo) const;
    virtual void writeHeaderTest();
    // </group>
    
    // Do the class-specific actions on opening a file.
    void doOpenFile (MultiFileInfo&) override;
    // Do the class-specific actions on closing a file.
    void doCloseFile (MultiFileInfo&) override;
    // Do the class-specific actions on adding a file.
    void doAddFile (MultiFileInfo&) override;
    // Do the class-specific actions on deleting a file.
    void doDeleteFile (MultiFileInfo&) override;
    // Truncate the file to <src>nrblk</src> blocks.
    void doTruncateFile (MultiFileInfo& info, uInt64 nrblk) override;
    // Flush the file itself.
    void doFlushFile() override;
    // Flush and close the file.
    void close() override;
    // Write the header info.
    void writeHeader() override;
    // Read the header info. If always==False, the info is only read if the
    // header counter has changed.
    void readHeader (Bool always=True) override;
    // Extend the virtual file to fit lastblk.
    void extend (MultiFileInfo& info, Int64 lastblk) override;
 
  protected:
    // Store the CRC of a data block in the index.
    void storeCRC (const void* buffer, Int64 blknr);
    // Check the CRC of a data block read.
    void checkCRC (const void* buffer, Int64 blknr) const;
    // Calculate the CRC of a data block.
    uInt calcCRC (const void* buffer, Int64 size) const;
    // Extend the virtual file to fit lastblk.
    // Optionally the free blocks are not used.
    virtual void extendVF (MultiFileInfo& info, Int64 lastblk, Bool useFreeBlocks);
    // Write a data block.
    void writeBlock (MultiFileInfo& info, Int64 blknr,
                     const void* buffer) override;
    // Read a data block.
    void readBlock (MultiFileInfo& info, Int64 blknr,
                    void* buffer) override;
    // Read the version 1 header.
    void readHeaderVersion1 (Int64 headerSize, std::vector<char>& buf);
    // Read the version 2 and higher header.
    void readHeaderVersion2 (std::vector<char>& buf);
 
    //# Data members
    // Define two continuation sets where the header overflow can be stored
    MultiFileInfo itsHdrCont[2];
    uInt  itsNrContUsed[2];     // nr of cont.blocks actually used
    uInt  itsHdrContInx;        // Continuation set last used (0 or 1)
    Bool  itsUseCRC;
    std::vector<uInt> itsCRC;   // CRC value per block (empty if useCRC=False)
    std::unique_ptr<ByteIO> itsIO;   // A regular file or nested MFFileIO
  };
 
 
 
} //# NAMESPACE CASACORE - END
 
#endif