The main class for the Dysco storage manager. More...

#include <dyscostman.h>

Public Member Functions
	DyscoStMan (unsigned dataBitRate, unsigned weightBitRate, const casacore::String &name="DyscoStMan")
	Convenience constructor to create a new storage manager with some settings without having to fill a 'spec' Record. More...

void	SetGaussianDistribution ()
	Initialize the storage manager to use a Gaussian distribution for the quantization. More...

void	SetUniformDistribution ()
	Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer). More...

void	SetStudentsTDistribution (double nu)
	Initialize the storage manager to use a Student T distribution for the quantization (i.e., use a linear quantizer). More...

void	SetTruncatedGaussianDistribution (double truncationSigma)
	Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer). More...

void	SetNormalization (Normalization normalization)
	Set the type of normalization. More...

void	SetStaticSeed (bool staticSeed)

	DyscoStMan (const casacore::String &name, const casacore::Record &spec)
	This constructor is called by Casa when it needs to create a DyscoStMan. More...

	DyscoStMan (const DyscoStMan &source)
	Copy constructor that initializes a storage manager with similar specs. More...

	~DyscoStMan ()
	Destructor. More...

DyscoStMan &	operator= (const DyscoStMan &source)=delete
	Assignment – new dyscostman takes the settings of the source (but not the columns and/or data). More...

virtual casacore::DataManager *	clone () const final override
	Polymorphical copy constructor, equal to DyscoStMan(const DyscoStMan&). More...

virtual casacore::String	dataManagerType () const final override
	Type of manager. More...

virtual casacore::String	dataManagerName () const final override
	Returns the name of this manager as specified during construction. More...

virtual casacore::Record	dataManagerSpec () const final override
	Get manager specifications. More...

uint	getNRow () const
	Get the number of rows in the measurement set. More...

virtual casacore::Bool	canAddRow () const final override
	Whether rows can be added. More...

virtual casacore::Bool	canRemoveRow () const final override
	Whether rows can be removed. More...

virtual casacore::Bool	canAddColumn () const final override
	Whether columns can be added. More...

virtual casacore::Bool	canRemoveColumn () const final override
	Whether columns can be removed. More...

Public Member Functions inherited from casacore::DataManager
	DataManager ()
	Default constructor. More...

virtual	~DataManager ()

void	dataManagerInfo (Record &info) const
	Add SEQNR and SPEC (the DataManagerSpec subrecord) to the info. More...

virtual Record	getProperties () const
	Get data manager properties that can be modified. More...

virtual void	setProperties (const Record &spec)
	Modify data manager properties given in record fields. More...

virtual Bool	isStorageManager () const
	Is the data manager a storage manager? The default is yes. More...

virtual Bool	canReallocateColumns () const
	Tell if the data manager wants to reallocate the data manager column objects. More...

virtual DataManagerColumn *	reallocateColumn (DataManagerColumn *column)
	Reallocate the column object if it is part of this data manager. More...

uInt	sequenceNr () const
	Get the (unique) sequence nr of this data manager. More...

uInt	ncolumn () const
	Get the nr of columns in this data manager (can be zero). More...

Bool	asBigEndian () const
	Have the data to be stored in big or little endian canonical format? More...

const TSMOption &	tsmOption () const
	Get the TSM option. More...

MultiFileBase *	multiFile ()
	Get the MultiFile pointer (can be 0). More...

String	keywordName (const String &keyword) const
	Compose a keyword name from the given keyword appended with the sequence number (e.g. More...

String	fileName () const
	Compose a unique filename from the table name and sequence number. More...

ByteIO::OpenOption	fileOption () const
	Get the AipsIO option of the underlying file. More...

virtual Bool	isRegular () const
	Is this a regular storage manager? It is regular if it allows addition of rows and writing data in them. More...

Table &	table () const
	Get the table this object is associated with. More...

virtual Bool	canRenameColumn () const
	Does the data manager allow to rename columns? (default yes) More...

virtual void	setMaximumCacheSize (uInt nMiB)
	Set the maximum cache size (in bytes) to be used by a storage manager. More...

virtual void	showCacheStatistics (std::ostream &) const
	Show the data manager's IO statistics. More...

DataManagerColumn *	createScalarColumn (const String &columnName, int dataType, const String &dataTypeId)
	Create a column in the data manager on behalf of a table column. More...

DataManagerColumn *	createDirArrColumn (const String &columnName, int dataType, const String &dataTypeId)
	Create a direct array column. More...

DataManagerColumn *	createIndArrColumn (const String &columnName, int dataType, const String &dataTypeId)
	Create an indirect array column. More...

DataManager *	getClone () const
	Has the object already been cloned? More...

void	setClone (DataManager *clone) const
	Set the pointer to the clone. More...

Static Public Member Functions
static casacore::DataManager *	makeObject (const casacore::String &name, const casacore::Record &spec)
	Create an object with given name and spec. More...

static void	registerClass ()
	This function makes the DyscoStMan known to casacore. More...

Static Public Member Functions inherited from casacore::DataManager
static void	registerCtor (const String &type, DataManagerCtor func)
	Register a mapping of a data manager type to its static construction function. More...

static DataManagerCtor	getCtor (const String &dataManagerType)
	Get the "constructor" of a data manager (thread-safe). More...

static Bool	isRegistered (const String &dataManagerType)
	Test if a data manager is registered (thread-safe). More...

static DataManager *	unknownDataManager (const String &dataManagerType, const Record &spec)
	Serve as default function for theirRegisterMap, which catches all unknown data manager types. More...

Protected Member Functions
uint64_t	nBlocksInFile () const
	The number of rows that are actually stored in the file. More...

size_t	nRowsInBlock () const
	Number of rows in one "time-block", i.e. More...

size_t	nAntennae () const
	Number of antennae used in a time block. More...

size_t	getBlockIndex (uint64_t row) const
	Return index of block that contains the given measurement set row. More...

size_t	getRowWithinBlock (uint64_t row) const
	Return the offset of the row within the block. More...

uint64_t	getRowIndex (size_t block) const
	Calculate first measurement set row index of a given block index. More...

bool	areOffsetsInitialized () const
	This method returns `true` when the number of rows per block and the number of antennae per block are known. More...

void	initializeRowsPerBlock (size_t rowsPerBlock, size_t antennaCount, bool writeToHeader)
	To be called by a column once it determines rowsPerBlock and antennaCount. More...

Protected Member Functions inherited from casacore::DataManager
void	decrementNcolumn ()
	Decrement number of columns (in case a column is deleted). More...

void	setEndian (Bool bigEndian)
	Tell the data manager if big or little endian format is needed. More...

void	setTsmOption (const TSMOption &tsmOption)
	Tell the data manager which TSM option to use. More...

void	setMultiFile (MultiFileBase *mfile)
	Tell the data manager that MultiFile can be used. More...

virtual Bool	hasMultiFileSupport () const
	Does the data manager support use of MultiFile? A derived class has to return True if it can use the MultiFile. More...

void	throwDataTypeOther (const String &columnName, int dataType) const
	Throw an exception in case data type is TpOther, because the storage managers (and maybe other data managers) do not support such columns. More...

Private Member Functions
void	readCompressedData (size_t blockIndex, const DyscoStManColumn column, unsigned char dest, size_t size)

void	writeCompressedData (size_t blockIndex, const DyscoStManColumn column, const unsigned char data, size_t size)

void	readHeader ()

void	writeHeader ()

void	makeEmpty ()

void	setFromSpec (const casacore::Record &spec)

size_t	getFileOffset (size_t blockIndex) const

virtual casacore::Bool	flush (casacore::AipsIO &, casacore::Bool doFsync) final override
	Flush and optionally fsync the data. More...

virtual void	create64 (casacore::rownr_t nRow) final override
	Let the storage manager create files as needed for a new table. More...

virtual casacore::rownr_t	open64 (casacore::rownr_t nRow, casacore::AipsIO &) final override
	Open the storage manager file for an existing table. More...

virtual casacore::DataManagerColumn *	makeScalarColumn (const casacore::String &name, int dataType, const casacore::String &dataTypeID) final override
	Create a column in the storage manager on behalf of a table column. More...

virtual casacore::DataManagerColumn *	makeDirArrColumn (const casacore::String &name, int dataType, const casacore::String &dataTypeID) final override
	Create a direct array column. More...

virtual casacore::DataManagerColumn *	makeIndArrColumn (const casacore::String &name, int dataType, const casacore::String &dataTypeID) final override
	Create an indirect array column. More...

virtual casacore::rownr_t	resync64 (casacore::rownr_t nRow) final override

virtual void	deleteManager () final override
	The data manager will be deleted (because all its columns are requested to be deleted). More...

virtual void	prepare () final override
	Prepare the columns, let the data manager initialize itself further. More...

virtual void	reopenRW () final override
	Reopen the storage manager files for read/write. More...

virtual void	addRow64 (casacore::rownr_t nrrow) final override
	Add rows to the storage manager. More...

virtual void	removeRow64 (casacore::rownr_t rowNr) final override
	Delete a row from all columns. More...

virtual void	addColumn (casacore::DataManagerColumn *) final override
	Do the final addition of a column. More...

virtual void	removeColumn (casacore::DataManagerColumn *) final override
	Remove a column from the data file. More...

Private Attributes
uint64_t	_nRow

uint64_t	_nBlocksInFile

uint32_t	_rowsPerBlock

uint32_t	_antennaCount

uint32_t	_blockSize

unsigned	_headerSize

std::mutex	_mutex

std::unique_ptr< std::fstream >	_fStream

std::string	_name

unsigned	_dataBitCount

unsigned	_weightBitCount

DyscoDistribution	_distribution

Normalization	_normalization

double	_studentTNu

double	_distributionTruncation

bool	_staticSeed

std::vector< std::unique_ptr< DyscoStManColumn > >	_columns

Static Private Attributes
static const unsigned short	VERSION_MAJOR

static const unsigned short	VERSION_MINOR

Friends
class	DyscoStManColumn

Additional Inherited Members
Static Public Attributes inherited from casacore::DataManager
static rownr_t	MAXROWNR32
	Define the highest row number that can be represented as signed 32-bit. More...

Detailed Description

The main class for the Dysco storage manager.

Definition at line 46 of file dyscostman.h.

Constructor & Destructor Documentation

◆ DyscoStMan() [1/3]

dyscostman::DyscoStMan::DyscoStMan	(	unsigned	dataBitRate,
		unsigned	weightBitRate,
		const casacore::String &	name = `"DyscoStMan"`
	)

Convenience constructor to create a new storage manager with some settings without having to fill a 'spec' Record.

The storage manager will be initialized to AF normalization with a truncated Gaussian distribution for the quantization, and a truncation of sigma = 2.5. To change the settings, use one of the Set...Distribution() methods and SetNormalization().

Parameters

dataBitRate	The number of bits per float used for visibilities.
weightBitRate	The number of bits per float used for the weight column.
name	Storage manager name.

Referenced by clone(), and makeObject().

◆ DyscoStMan() [2/3]

dyscostman::DyscoStMan::DyscoStMan	(	const casacore::String &	name,
		const casacore::Record &	spec
	)

This constructor is called by Casa when it needs to create a DyscoStMan.

Casa will call makeObject() that will call this constructor. When it loads an DyscoStMan for an existing MS, the "spec" parameter will be empty, thus the class should initialize its properties by reading them from the file. The spec is used to make a new storage manager with specs similar to another one.

Parameters

name	Name of this storage manager.
spec	Specs to initialize this class with.

◆ DyscoStMan() [3/3]

dyscostman::DyscoStMan::DyscoStMan ( const DyscoStMan & source )

Copy constructor that initializes a storage manager with similar specs.

The columns are not copied: the new manager will be empty.

◆ ~DyscoStMan()

dyscostman::DyscoStMan::~DyscoStMan ( )

Destructor.

Member Function Documentation

◆ addColumn()

virtual void dyscostman::DyscoStMan::addColumn ( casacore::DataManagerColumn * )

finaloverrideprivatevirtual

Do the final addition of a column.

Reimplemented from casacore::DataManager.

◆ addRow64()

virtual void dyscostman::DyscoStMan::addRow64 ( casacore::rownr_t nrrow )

finaloverrideprivatevirtual

Add rows to the storage manager.

◆ areOffsetsInitialized()

bool dyscostman::DyscoStMan::areOffsetsInitialized ( ) const

inlineprotected

This method returns true when the number of rows per block and the number of antennae per block are known.

This is only the case once the first time- block was written to the file.

Returns: True when the nr of rows per block and antennae are available.

Definition at line 296 of file dyscostman.h.

References _rowsPerBlock.

Referenced by dyscostman::DyscoStManColumn::areOffsetsInitialized().

◆ canAddColumn()

virtual casacore::Bool dyscostman::DyscoStMan::canAddColumn ( ) const

inlinefinaloverridevirtual

Whether columns can be added.

Returns: true (but restrictions apply; columns can only be added as long as no writes have been performed on the set).

Reimplemented from casacore::DataManager.

Definition at line 202 of file dyscostman.h.

◆ canAddRow()

virtual casacore::Bool dyscostman::DyscoStMan::canAddRow ( ) const

inlinefinaloverridevirtual

Whether rows can be added.

Returns: true

Reimplemented from casacore::DataManager.

Definition at line 189 of file dyscostman.h.

◆ canRemoveColumn()

virtual casacore::Bool dyscostman::DyscoStMan::canRemoveColumn ( ) const

inlinefinaloverridevirtual

Whether columns can be removed.

Returns: true (but restrictions apply – still to be checked)

Todo:: Describe restrictons

Reimplemented from casacore::DataManager.

Definition at line 209 of file dyscostman.h.

◆ canRemoveRow()

virtual casacore::Bool dyscostman::DyscoStMan::canRemoveRow ( ) const

inlinefinaloverridevirtual

Whether rows can be removed.

Returns: true (but only rows at the end can actually be removed)

Reimplemented from casacore::DataManager.

Definition at line 195 of file dyscostman.h.

◆ clone()

virtual casacore::DataManager* dyscostman::DyscoStMan::clone ( ) const

inlinefinaloverridevirtual

Polymorphical copy constructor, equal to DyscoStMan(const DyscoStMan&).

Returns: Empty manager with specs as the source.

Implements casacore::DataManager.

Definition at line 158 of file dyscostman.h.

References DyscoStMan().

◆ create64()

virtual void dyscostman::DyscoStMan::create64 ( casacore::rownr_t nRow )

finaloverrideprivatevirtual

Let the storage manager create files as needed for a new table.

This allows a column with an indirect array to create its file.

◆ dataManagerName()

virtual casacore::String dyscostman::DyscoStMan::dataManagerName ( ) const

inlinefinaloverridevirtual

Returns the name of this manager as specified during construction.

Reimplemented from casacore::DataManager.

Definition at line 169 of file dyscostman.h.

References _name.

◆ dataManagerSpec()

virtual casacore::Record dyscostman::DyscoStMan::dataManagerSpec ( ) const

finaloverridevirtual

Get manager specifications.

Includes method settings, etc. Can be used to make a second storage manager with

Returns: Record containing data manager specifications.

Reimplemented from casacore::DataManager.

◆ dataManagerType()

virtual casacore::String dyscostman::DyscoStMan::dataManagerType ( ) const

inlinefinaloverridevirtual

Type of manager.

Returns: "DyscoStMan".

Implements casacore::DataManager.

Definition at line 164 of file dyscostman.h.

◆ deleteManager()

virtual void dyscostman::DyscoStMan::deleteManager ( )

finaloverrideprivatevirtual

The data manager will be deleted (because all its columns are requested to be deleted).

So clean up the things needed (e.g. delete files).

Implements casacore::DataManager.

◆ flush()

virtual casacore::Bool dyscostman::DyscoStMan::flush	(	casacore::AipsIO &	,
		casacore::Bool	doFsync
	)

finaloverrideprivatevirtual

Flush and optionally fsync the data.

The AipsIO stream represents the main table file and can be used by virtual column engines to store SMALL amounts of data.

Implements casacore::DataManager.

◆ getBlockIndex()

size_t dyscostman::DyscoStMan::getBlockIndex ( uint64_t row ) const

inlineprotected

Return index of block that contains the given measurement set row.

This can only be calculated after a first time block was written (see areOffsetsInitialized()).

Parameters

row	A measurement set row.

Returns: Block index.

Definition at line 269 of file dyscostman.h.

References _rowsPerBlock.

Referenced by dyscostman::DyscoStManColumn::getBlockIndex().

◆ getFileOffset()

size_t dyscostman::DyscoStMan::getFileOffset ( size_t blockIndex ) const

inlineprivate

Definition at line 326 of file dyscostman.h.

References _blockSize, and _headerSize.

◆ getNRow()

uint dyscostman::DyscoStMan::getNRow ( ) const

inline

Get the number of rows in the measurement set.

Returns: Number of rows in the measurement set.

Definition at line 183 of file dyscostman.h.

References _nRow.

◆ getRowIndex()

uint64_t dyscostman::DyscoStMan::getRowIndex ( size_t block ) const

inlineprotected

Calculate first measurement set row index of a given block index.

Parameters

block A block index

Returns: First measurement set row index of given block.

Definition at line 286 of file dyscostman.h.

References _rowsPerBlock.

Referenced by dyscostman::DyscoStManColumn::getRowIndex().

◆ getRowWithinBlock()

size_t dyscostman::DyscoStMan::getRowWithinBlock ( uint64_t row ) const

inlineprotected

Return the offset of the row within the block.

This can only be calculated after a first time block was written (see areOffsetsInitialized()).

See also: getBlockIndex().

Parameters

row	A measurement set row.

Returns: offset of row within block.

Definition at line 279 of file dyscostman.h.

References _rowsPerBlock.

Referenced by dyscostman::DyscoStManColumn::getRowWithinBlock().

◆ initializeRowsPerBlock()

void dyscostman::DyscoStMan::initializeRowsPerBlock	(	size_t	rowsPerBlock,
		size_t	antennaCount,
		bool	writeToHeader
	)

protected

To be called by a column once it determines rowsPerBlock and antennaCount.

Parameters

rowsPerBlock	Number of measurement set rows in one time block.
antennaCount	Highest antenna index+1 used in a time block.
writeToHeader	Write the header?

Referenced by dyscostman::DyscoStManColumn::initializeRowsPerBlock().

◆ makeDirArrColumn()

virtual casacore::DataManagerColumn* dyscostman::DyscoStMan::makeDirArrColumn	(	const casacore::String &	name,
		int	dataType,
		const casacore::String &	dataTypeID
	)

finaloverrideprivatevirtual

Create a direct array column.

Implements casacore::DataManager.

◆ makeEmpty()

void dyscostman::DyscoStMan::makeEmpty ( )

private

◆ makeIndArrColumn()

virtual casacore::DataManagerColumn* dyscostman::DyscoStMan::makeIndArrColumn	(	const casacore::String &	name,
		int	dataType,
		const casacore::String &	dataTypeID
	)

finaloverrideprivatevirtual

Create an indirect array column.

Implements casacore::DataManager.

◆ makeObject()

static casacore::DataManager* dyscostman::DyscoStMan::makeObject	(	const casacore::String &	name,
		const casacore::Record &	spec
	)

inlinestatic

Create an object with given name and spec.

This methods gets registered in the DataManager "constructor" map. The caller has to delete the object. New class will be initialized via DyscoStMan(const casacore::String& name, const casacore::Record& spec).

Returns: A DyscoStMan with given specs.

Definition at line 219 of file dyscostman.h.

References DyscoStMan().

◆ makeScalarColumn()

virtual casacore::DataManagerColumn* dyscostman::DyscoStMan::makeScalarColumn	(	const casacore::String &	name,
		int	dataType,
		const casacore::String &	dataTypeID
	)

finaloverrideprivatevirtual

Create a column in the storage manager on behalf of a table column.

The caller will NOT delete the newly created object. Create a scalar column.

Implements casacore::DataManager.

◆ nAntennae()

size_t dyscostman::DyscoStMan::nAntennae ( ) const

inlineprotected

Number of antennae used in a time block.

This does not have to be equal to the number of antennae stored in the measurement set. This value is only available after a first time block was written (see areOffsetsInitialized()).

Returns: Number of antennae.

Definition at line 260 of file dyscostman.h.

References _antennaCount.

Referenced by dyscostman::DyscoStManColumn::nAntennae().

◆ nBlocksInFile()

uint64_t dyscostman::DyscoStMan::nBlocksInFile ( ) const

inlineprotected

The number of rows that are actually stored in the file.

This method is synchronized (i.e., thread-safe).

Definition at line 239 of file dyscostman.h.

References _mutex, and _nBlocksInFile.

Referenced by dyscostman::DyscoStManColumn::nBlocksInFile().

◆ nRowsInBlock()

size_t dyscostman::DyscoStMan::nRowsInBlock ( ) const

inlineprotected

Number of rows in one "time-block", i.e.

a sequence of rows that belong to the same timestep, spw and field. This value is only available after a first time block was written (see areOffsetsInitialized()).

Returns: Number of measurement set rows in one time block.

Definition at line 251 of file dyscostman.h.

References _rowsPerBlock.

Referenced by dyscostman::DyscoStManColumn::nRowsInBlock().

◆ open64()

virtual casacore::rownr_t dyscostman::DyscoStMan::open64	(	casacore::rownr_t	nRow,
		casacore::AipsIO &
	)

finaloverrideprivatevirtual

Open the storage manager file for an existing table.

Return the number of rows in the data file.

◆ operator=()

DyscoStMan& dyscostman::DyscoStMan::operator= ( const DyscoStMan & source )

delete

Assignment – new dyscostman takes the settings of the source (but not the columns and/or data).

Parameters

source Source manager.

◆ prepare()

virtual void dyscostman::DyscoStMan::prepare ( )

finaloverrideprivatevirtual

Prepare the columns, let the data manager initialize itself further.

Prepare is called after create/open has been called for all columns. In this way one can be sure that referenced columns are read back and partly initialized.

Reimplemented from casacore::DataManager.

◆ readCompressedData()

void dyscostman::DyscoStMan::readCompressedData	(	size_t	blockIndex,
		const DyscoStManColumn *	column,
		unsigned char *	dest,
		size_t	size
	)

private

Referenced by dyscostman::DyscoStManColumn::readCompressedData().

◆ readHeader()

void dyscostman::DyscoStMan::readHeader ( )

private

◆ registerClass()

static void dyscostman::DyscoStMan::registerClass ( )

static

This function makes the DyscoStMan known to casacore.

The function is necessary for loading the storage manager from a shared library. It should have this specific name ("register_" + storage manager's name in lowercase) to be able to be automatically called when the library is loaded. That function will forward the call here.

◆ removeColumn()

virtual void dyscostman::DyscoStMan::removeColumn ( casacore::DataManagerColumn * )

finaloverrideprivatevirtual

Remove a column from the data file.

Reimplemented from casacore::DataManager.

◆ removeRow64()

virtual void dyscostman::DyscoStMan::removeRow64 ( casacore::rownr_t rowNr )

finaloverrideprivatevirtual

Delete a row from all columns.

◆ reopenRW()

virtual void dyscostman::DyscoStMan::reopenRW ( )

finaloverrideprivatevirtual

Reopen the storage manager files for read/write.

Reimplemented from casacore::DataManager.

◆ resync64()

virtual casacore::rownr_t dyscostman::DyscoStMan::resync64 ( casacore::rownr_t nRow )

finaloverrideprivatevirtual

◆ setFromSpec()

void dyscostman::DyscoStMan::setFromSpec ( const casacore::Record & spec )

private

◆ SetGaussianDistribution()

void dyscostman::DyscoStMan::SetGaussianDistribution ( )

inline

Initialize the storage manager to use a Gaussian distribution for the quantization.

This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.

In tests with MWA and LOFAR data, the Gaussian distribution showed lesser compression accuracy compared to the truncated Gaussian and uniform distributions.

See also: SetUniformDistribution(), SetTruncatedGaussianDistribution()

Definition at line 72 of file dyscostman.h.

References _distribution, and dyscostman::GaussianDistribution.

◆ SetNormalization()

void dyscostman::DyscoStMan::SetNormalization ( Normalization normalization )

inline

Set the type of normalization.

This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.

Definition at line 121 of file dyscostman.h.

References _normalization.

◆ SetStaticSeed()

void dyscostman::DyscoStMan::SetStaticSeed ( bool staticSeed )

inline

Definition at line 125 of file dyscostman.h.

References _staticSeed.

◆ SetStudentsTDistribution()

void dyscostman::DyscoStMan::SetStudentsTDistribution ( double nu )

inline

Initialize the storage manager to use a Student T distribution for the quantization (i.e., use a linear quantizer).

This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.

The Student T distribution performed not very well on test sets, and was mainly added for testing.

Definition at line 95 of file dyscostman.h.

References _distribution, _studentTNu, and dyscostman::StudentsTDistribution.

◆ SetTruncatedGaussianDistribution()

void dyscostman::DyscoStMan::SetTruncatedGaussianDistribution ( double truncationSigma )

inline

Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer).

This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.

In tests with MWA and LOFAR data, the truncated Gaussian distribution with a sigma of 1.5 to 2.5 is the recommended distribution.

Parameters

truncationSigma At which point the distribution is truncated. Good values are 1.5 to 2.5.

Definition at line 111 of file dyscostman.h.

References _distribution, _distributionTruncation, and dyscostman::TruncatedGaussianDistribution.

◆ SetUniformDistribution()

void dyscostman::DyscoStMan::SetUniformDistribution ( )

inline

Initialize the storage manager to use a Uniform distribution for the quantization (i.e., use a linear quantizer).

This method should only be called directly after creating DyscoStMan, before adding columns, and reading/writing data.

In tests with MWA and LOFAR data, the Uniform distribution showed very good results, only the truncated Gaussian distribution showed better results for some cases.

Definition at line 84 of file dyscostman.h.

References _distribution, and dyscostman::UniformDistribution.

◆ writeCompressedData()

void dyscostman::DyscoStMan::writeCompressedData	(	size_t	blockIndex,
		const DyscoStManColumn *	column,
		const unsigned char *	data,
		size_t	size
	)

private

Referenced by dyscostman::DyscoStManColumn::writeCompressedData().

◆ writeHeader()

void dyscostman::DyscoStMan::writeHeader ( )

private

Friends And Related Function Documentation

◆ DyscoStManColumn

friend class DyscoStManColumn

friend

Definition at line 308 of file dyscostman.h.

Member Data Documentation

◆ _antennaCount

uint32_t dyscostman::DyscoStMan::_antennaCount

private

Definition at line 389 of file dyscostman.h.

Referenced by nAntennae().

◆ _blockSize

uint32_t dyscostman::DyscoStMan::_blockSize

private

Definition at line 390 of file dyscostman.h.

Referenced by getFileOffset().

◆ _columns

std::vector<std::unique_ptr<DyscoStManColumn> > dyscostman::DyscoStMan::_columns

private

Definition at line 404 of file dyscostman.h.

◆ _dataBitCount

unsigned dyscostman::DyscoStMan::_dataBitCount

private

Definition at line 397 of file dyscostman.h.

◆ _distribution

DyscoDistribution dyscostman::DyscoStMan::_distribution

private

Definition at line 399 of file dyscostman.h.

Referenced by SetGaussianDistribution(), SetStudentsTDistribution(), SetTruncatedGaussianDistribution(), and SetUniformDistribution().

◆ _distributionTruncation

double dyscostman::DyscoStMan::_distributionTruncation

private

Definition at line 401 of file dyscostman.h.

Referenced by SetTruncatedGaussianDistribution().

◆ _fStream

std::unique_ptr<std::fstream> dyscostman::DyscoStMan::_fStream

private

Definition at line 394 of file dyscostman.h.

◆ _headerSize

unsigned dyscostman::DyscoStMan::_headerSize

private

Definition at line 392 of file dyscostman.h.

Referenced by getFileOffset().

◆ _mutex

std::mutex dyscostman::DyscoStMan::_mutex

mutableprivate

Definition at line 393 of file dyscostman.h.

Referenced by nBlocksInFile().

◆ _name

std::string dyscostman::DyscoStMan::_name

private

Definition at line 396 of file dyscostman.h.

Referenced by dataManagerName().

◆ _nBlocksInFile

uint64_t dyscostman::DyscoStMan::_nBlocksInFile

private

Definition at line 387 of file dyscostman.h.

Referenced by nBlocksInFile().

◆ _normalization

Normalization dyscostman::DyscoStMan::_normalization

private

Definition at line 400 of file dyscostman.h.

Referenced by SetNormalization().

◆ _nRow

uint64_t dyscostman::DyscoStMan::_nRow

private

Definition at line 386 of file dyscostman.h.

Referenced by getNRow().

◆ _rowsPerBlock

uint32_t dyscostman::DyscoStMan::_rowsPerBlock

private

Definition at line 388 of file dyscostman.h.

Referenced by areOffsetsInitialized(), getBlockIndex(), getRowIndex(), getRowWithinBlock(), and nRowsInBlock().

◆ _staticSeed

bool dyscostman::DyscoStMan::_staticSeed

private

Definition at line 402 of file dyscostman.h.

Referenced by SetStaticSeed().

◆ _studentTNu

double dyscostman::DyscoStMan::_studentTNu

private

Definition at line 401 of file dyscostman.h.

Referenced by SetStudentsTDistribution().

◆ _weightBitCount

unsigned dyscostman::DyscoStMan::_weightBitCount

private

Definition at line 398 of file dyscostman.h.

◆ VERSION_MAJOR

const unsigned short dyscostman::DyscoStMan::VERSION_MAJOR

staticprivate

Definition at line 310 of file dyscostman.h.

◆ VERSION_MINOR

const unsigned short dyscostman::DyscoStMan::VERSION_MINOR

staticprivate

Definition at line 310 of file dyscostman.h.

The documentation for this class was generated from the following file:

tables/Dysco/dyscostman.h

Public Member Functions

Static Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Static Private Attributes

Friends

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ DyscoStMan() [1/3]

◆ DyscoStMan() [2/3]

◆ DyscoStMan() [3/3]

◆ ~DyscoStMan()

Member Function Documentation

◆ addColumn()

◆ addRow64()

◆ areOffsetsInitialized()

◆ canAddColumn()

◆ canAddRow()

◆ canRemoveColumn()

◆ canRemoveRow()

◆ clone()

◆ create64()

◆ dataManagerName()

◆ dataManagerSpec()

◆ dataManagerType()

◆ deleteManager()

◆ flush()

◆ getBlockIndex()

◆ getFileOffset()

◆ getNRow()

◆ getRowIndex()

◆ getRowWithinBlock()

◆ initializeRowsPerBlock()

◆ makeDirArrColumn()

◆ makeEmpty()

◆ makeIndArrColumn()

◆ makeObject()

◆ makeScalarColumn()

◆ nAntennae()

◆ nBlocksInFile()

◆ nRowsInBlock()

◆ open64()

◆ operator=()

◆ prepare()

◆ readCompressedData()

◆ readHeader()

◆ registerClass()

◆ removeColumn()

◆ removeRow64()

◆ reopenRW()

◆ resync64()

◆ setFromSpec()

◆ SetGaussianDistribution()

◆ SetNormalization()

◆ SetStaticSeed()

◆ SetStudentsTDistribution()

◆ SetTruncatedGaussianDistribution()

◆ SetUniformDistribution()

◆ writeCompressedData()

◆ writeHeader()

Friends And Related Function Documentation

◆ DyscoStManColumn

Member Data Documentation

◆ _antennaCount

◆ _blockSize

◆ _columns

◆ _dataBitCount

◆ _distribution

◆ _distributionTruncation

◆ _fStream

◆ _headerSize

◆ _mutex

◆ _name

◆ _nBlocksInFile

◆ _normalization

◆ _nRow

◆ _rowsPerBlock

◆ _staticSeed