Implement rules for filtering bad samples. More...

#include <BadSampleFilter.h>

Public Member Functions
	BadSampleFilter ()

void	setFilterBadSamples (bool val)
	Enable filtering of bad samples. More...

bool	filterBadSamples () const
	Return true if bad sample filtering has been enabled. More...

void	setMinDsmId (unsigned int val)
	The default minimum DSM ID for a valid sample is 1. More...

unsigned int	minDsmId () const

void	setMaxDsmId (unsigned int val)
	The default maximum DSM ID for a valid sample is 1024. More...

unsigned int	maxDsmId () const

void	setMinSampleLength (unsigned int val)
	The default minimum sample length is 1. More...

unsigned int	minSampleLength () const

void	setMaxSampleLength (unsigned int val)
	The default maximum sample length is 32768. More...

unsigned int	maxSampleLength () const

void	setMinSampleTime (const nidas::util::UTime &val)
	Set the minimum sample time for a valid sample. More...

UTime	minSampleTime () const

void	setMaxSampleTime (const nidas::util::UTime &val)
	Set the maximum sample time for a valid sample. More...

UTime	maxSampleTime () const

void	setSkipNidasHeader (bool enable)
	Set to true to allow the NIDAS header to be skipped if it cannot be found at the start of a file or sample stream. More...

bool	skipNidasHeader () const

void	setDefaultTimeRange (const UTime &start, const UTime &end)
	Set start and end times as the filter times only if current filter times are unset, and do not change whether the filter is enabled. More...

void	setSampleTypeLimit (nidas::core::sampleType stype)
	Set the single acceptable sample type, all others will be filtered out as bad. More...

nidas::core::sampleType	sampleTypeLimit () const
	Return the current sampleType setting. More...

void	setRules (const std::string &rule)
	Parse the filter rule string and throw NidasAppException if it does not parse. More...

bool	invalidSampleHeader (const SampleHeader &sheader)
	Return true if matches any of the criteria for an invalid sample. More...

bool	operator== (const BadSampleFilter &right) const

Private Types
typedef nidas::util::UTime	UTime

Private Member Functions
void	setRule (const std::string &rule)
	Set a single filter setting as a string in the form 'on', 'off', or <field>=. More...

Private Attributes
bool	_filterBadSamples

unsigned int	_minDsmId

unsigned int	_maxDsmId

size_t	_minSampleLength

size_t	_maxSampleLength

dsm_time_t	_minSampleTime

dsm_time_t	_maxSampleTime

bool	_skipNidasHeader

nidas::core::sampleType	_sampleType

Detailed Description

Implement rules for filtering bad samples.

Member Typedef Documentation

typedef nidas::util::UTime nidas::core::BadSampleFilter::UTime

private

Constructor & Destructor Documentation

BadSampleFilter::BadSampleFilter ( )

Member Function Documentation

bool nidas::core::BadSampleFilter::filterBadSamples ( ) const

inline

Return true if bad sample filtering has been enabled.

See setFilterBadSamples(). Enable inline to make it faster to check over and over.

References _filterBadSamples.

Referenced by nidas::dynld::SampleInputStream::closeBlocks(), nidas::core::operator<<(), and nidas::dynld::SampleInputStream::sampleFromHeader().

bool BadSampleFilter::invalidSampleHeader ( const SampleHeader & sheader )

Return true if matches any of the criteria for an invalid sample.

References _filterBadSamples, _maxDsmId, _maxSampleLength, _maxSampleTime, _minDsmId, _minSampleLength, _minSampleTime, _sampleType, nidas::util::LogContext::active(), nidas::util::LogMessage::format(), GET_DSM_ID, nidas::core::SampleHeader::getDataByteLength(), nidas::core::SampleHeader::getId(), nidas::core::SampleHeader::getTimeTag(), nidas::core::SampleHeader::getType(), nidas::util::LogContext::log(), LOG_VERBOSE, and nidas::core::UNKNOWN_ST.

Referenced by nidas::dynld::SampleInputStream::sampleFromHeader().

unsigned int nidas::core::BadSampleFilter::maxDsmId ( ) const

inline

References _maxDsmId.

Referenced by nidas::core::operator<<().

unsigned int nidas::core::BadSampleFilter::maxSampleLength ( ) const

inline

References _maxSampleLength.

Referenced by nidas::core::operator<<().

UTime nidas::core::BadSampleFilter::maxSampleTime ( ) const

inline

References _maxSampleTime.

Referenced by nidas::core::operator<<().

unsigned int nidas::core::BadSampleFilter::minDsmId ( ) const

inline

References _minDsmId.

Referenced by nidas::core::operator<<().

unsigned int nidas::core::BadSampleFilter::minSampleLength ( ) const

inline

References _minSampleLength.

Referenced by nidas::core::operator<<().

UTime nidas::core::BadSampleFilter::minSampleTime ( ) const

inline

References _minSampleTime.

Referenced by nidas::core::operator<<().

bool BadSampleFilter::operator== ( const BadSampleFilter & right ) const

References _filterBadSamples, _maxDsmId, _maxSampleLength, _maxSampleTime, _minDsmId, _minSampleLength, _minSampleTime, _sampleType, and _skipNidasHeader.

nidas::core::sampleType nidas::core::BadSampleFilter::sampleTypeLimit ( ) const

inline

Return the current sampleType setting.

References _sampleType.

Referenced by nidas::core::operator<<().

void BadSampleFilter::setDefaultTimeRange	(	const UTime &	start,
		const UTime &	end
	)

Set start and end times as the filter times only if current filter times are unset, and do not change whether the filter is enabled.

References _maxSampleTime, _minSampleTime, nidas::util::UTime::toUsecs(), and USECS_PER_DAY.

Referenced by NidsMerge::run(), StatsProcess::run(), DataPrep::run(), and DataDump::run().

void BadSampleFilter::setFilterBadSamples ( bool val )

Enable filtering of bad samples.

A sample is invalid if the sample type is out of range, or if the DSM ID, sample length, or sample time fall out their specified valid ranges. Other methods change the valid ranges. As a convenience, calling any of the methods to change the range filters automatically enables filtering, same as calling setFilterBadSamples() directly. Once filtering is enabled by any of those methods, it can only be disabled by calling setFilterBadSamples(false). It is a mistake to use these methods to skip otherwise valid samples. When sample is invalid, the SampleInputStream loses sync with the sample stream and begins looking for another valid sample one byte at a time, possibly discovering a sample which meets the valid criteria but was not an actual sample.

References _filterBadSamples.

Referenced by nidas::core::BadSampleFilterArg::parse(), nidas::dynld::SampleInputStream::setFilterBadSamples(), setMaxDsmId(), setMaxSampleLength(), setMaxSampleTime(), setMinDsmId(), setMinSampleLength(), setMinSampleTime(), setRule(), and setSampleTypeLimit().

void BadSampleFilter::setMaxDsmId ( unsigned int val )

The default maximum DSM ID for a valid sample is 1024.

See setFilterBadSamples().

References _maxDsmId, and setFilterBadSamples().

Referenced by nidas::dynld::SampleInputStream::setMaxDsmId(), and setRule().

void BadSampleFilter::setMaxSampleLength ( unsigned int val )

The default maximum sample length is 32768.

See setFilterBadSamples().

References _maxSampleLength, and setFilterBadSamples().

Referenced by nidas::dynld::SampleInputStream::setMaxSampleLength(), and setRule().

void BadSampleFilter::setMaxSampleTime ( const nidas::util::UTime & val )

Set the maximum sample time for a valid sample.

See setFilterBadSamples().

References _maxSampleTime, setFilterBadSamples(), and nidas::util::UTime::toUsecs().

Referenced by nidas::dynld::SampleInputStream::setMaxSampleTime(), and setRule().

void BadSampleFilter::setMinDsmId ( unsigned int val )

The default minimum DSM ID for a valid sample is 1.

See setFilterBadSamples().

References _minDsmId, and setFilterBadSamples().

Referenced by nidas::dynld::SampleInputStream::setMinDsmId(), and setRule().

void BadSampleFilter::setMinSampleLength ( unsigned int val )

The default minimum sample length is 1.

See setFilterBadSamples().

References _minSampleLength, and setFilterBadSamples().

Referenced by nidas::dynld::SampleInputStream::setMinSampleLength(), and setRule().

void BadSampleFilter::setMinSampleTime ( const nidas::util::UTime & val )

Set the minimum sample time for a valid sample.

See setFilterBadSamples().

References _minSampleTime, setFilterBadSamples(), and nidas::util::UTime::toUsecs().

Referenced by nidas::dynld::SampleInputStream::setMinSampleTime(), and setRule().

void BadSampleFilter::setRule ( const std::string & rule )

private

Set a single filter setting as a string in the form 'on', 'off', or <field>=.

See setRules(). Like setRules(), throw NidasAppException if the rule cannot be parsed. This is an internal implementation method. Anything valid for setRule() can be passed to setRules() instead.

References nidas::core::CHAR_ST, setFilterBadSamples(), setMaxDsmId(), setMaxSampleLength(), setMaxSampleTime(), setMinDsmId(), setMinSampleLength(), setMinSampleTime(), setSampleTypeLimit(), and setSkipNidasHeader().

Referenced by setRules().

void BadSampleFilter::setRules ( const std::string & rule )

Parse the filter rule string and throw NidasAppException if it does not parse.

An empty rule changes nothing.

Rules are a comma-separated list of key=value settings using these keys:

maxdsm,mindsm,maxtime,mintime,maxlen,minlen,on,off,skipnidasheader

Any of the keys may be specified in any order. Once enabled, these are the defaults:

maxdsm=1024,mindsm=1,maxlen=32768,minlen=1,skipnidasheader=off

Times are specified in any format that UTime can parse by default, such as ISO time, YYYY-mm-ddTHH:MM:SS.fff, or condensed time YYYYmmddHHMMSS.fff. The time string cannot contain any commas.

Additionally the keys 'on' and 'off' enable or disable filtering. So a rule of just 'on' enables the default filtering. Setting any of the rules implicitly enables filtering.

References setRule().

Referenced by nidas::core::BadSampleFilterArg::parse().

void BadSampleFilter::setSampleTypeLimit ( nidas::core::sampleType stype )

Set the single acceptable sample type, all others will be filtered out as bad.

If set to UNKNOWN_ST, then all valid sample types are accepted.

References _sampleType, and setFilterBadSamples().

Referenced by setRule().

void BadSampleFilter::setSkipNidasHeader ( bool enable )

Set to true to allow the NIDAS header to be skipped if it cannot be found at the start of a file or sample stream.

This does not also enable bad sample filtering, since it is reasonable to try to skip a missing header in a file which otherwise contains good samples.

References _skipNidasHeader.

Referenced by setRule().

bool nidas::core::BadSampleFilter::skipNidasHeader ( ) const

inline

References _skipNidasHeader.

Referenced by nidas::core::operator<<(), and nidas::dynld::SampleInputStream::parseInputHeader().