A reader for data stored in NetCDF CF format in multiple files. More...

#include <teca_multi_cf_reader.h>

Inheritance diagram for teca_multi_cf_reader:

Collaboration diagram for teca_multi_cf_reader:

Public Member Functions
std::shared_ptr< teca_multi_cf_reader >	shared_from_this ()

std::shared_ptr< teca_multi_cf_reader const >	shared_from_this () const

	teca_multi_cf_reader (const teca_multi_cf_reader &src)=delete

	teca_multi_cf_reader (teca_multi_cf_reader &&src)=delete

teca_multi_cf_reader &	operator= (const teca_multi_cf_reader &src)=delete

teca_multi_cf_reader &	operator= (teca_multi_cf_reader &&src)=delete

const char *	get_class_name () const override

int	set_input_file (const std::string &input_file)

std::string	get_input_file ()

int	add_reader (const std::string &regex, const std::string &key, int provides_time, int provides_geometry, const std::vector< std::string > &variables)

int	set_time_reader (const std::string &key)
	sets the reader that provides the time axis More...

int	set_geometry_reader (const std::string &key)
	sets the reader that provides the mesh geometry More...

int	add_variable_reader (const std::string &key, const std::string &variable)
	adds to the list of variables that a reader will provide More...

int	set_variable_reader (const std::string &key, const std::vector< std::string > &variable)
	sets the list of variable that a reader will provide. More...

void	get_variables (std::vector< std::string > &vars)
	get the list of variables that the reader will serve up More...

periodic_in_x
Set to indicate the presence of a periodic boundary in the x direction. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_periodic_in_x (int flag)

int	get_periodic_in_x () const

x_axis_variable
Set the variable to use for the mesh x-axis. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_x_axis_variable (const std::string &var)

std::string	get_x_axis_variable () const

y_axis_variable
Set the variable to use for the mesh y-axis. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_y_axis_variable (const std::string &var)

std::string	get_y_axis_variable () const

z_axis_variable
Set the variable to use for the mesh z-axis. Leaving the z-axis empty results in a 2D mesh. You must set this to the correct vertical coordinate dimension to produce a 3D mesh. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_z_axis_variable (const std::string &var)

std::string	get_z_axis_variable () const

t_axis_variable_
Set the variable to use for the mesh t-axis. Default "time". Setting this to an empty string disables the time axis. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_t_axis_variable (const std::string &var)

std::string	get_t_axis_variable () const

calendar
Use this to override the calendar, or set one when specifying t_values directly. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_calendar (const std::string &calendar)

std::string	get_calendar () const

t_units
Use this to set or override the time units. This is necessary when specifying time values directly. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_t_units (const std::string &units)

std::string	get_t_units () const

filename_time_template
a way to infer time from the filename if the time axis is not stored in the file itself. If set this will override the corresponding setting from the MCF file for all internal readers. strftime format codes are used. For example for the files: my_file_20170516_00.nc my_file_20170516_03.nc ... the template would be my_file_%Y%m%d_%H.nc
void	set_filename_time_template (const std::string &templ)

std::string	get_filename_time_template () const

t_values
Set the time values to use instead if a time variable doesn't exist or you need to override it. If set this will override the corresponding setting from the MCF file for all internal readers.
size_t	get_number_of_t_values ()

void	append_t_value (const double &v)

void	set_t_value (const double &v)

void	set_t_value (size_t i, const double &v)

void	set_t_values (const std::vector< double > &v)

void	set_t_values (const std::initializer_list< double > &&l)

const double &	get_t_value (size_t i) const

const std::vector< double > &	get_t_values () const

void	clear_t_values ()

max_metadata_ranks
set/get the number of ranks used to read the time axis. If set this will override the corresponding setting from the MCF file for all internal readers.
void	set_max_metadata_ranks (const int &v)

const int &	get_max_metadata_ranks () const

clamp_dimensions_of_one
If set the requested extent will be clamped in a given direction if the coorinate axis in that direction has a length of 1 and the requested extent would be out of bounds. This is a work around to enable loading 2D data with a vertical dimension of 1, into a 3D mesh and should be used with caution.
void	set_clamp_dimensions_of_one (int flag)

int	get_clamp_dimensions_of_one () const

collective_buffer
Enables MPI I/O colective buffering. Collective buffering is only valid when the spatial partitioner is enabled and the number of spatial partitions is equal to the number of MPI ranks, and the code is single threaded. This is an experimental feature.
void	set_collective_buffer (int flag)

int	get_collective_buffer () const

target_bounds
If set a teca_cartesian_mesh_coordinate_transform will be added to the internal pipeline of each managed reader. There must always be 6 values provided in the form "X0, x1, y0, y1, z0, z1" that define the bounds to which each axis will be transformed. Use "1, 0" for axis that should be passed through without applying the transform.
void	set_target_bounds (const std::vector< double > &bounds)

const std::vector< double > &	get_target_bounds () const

target_x_axis_variable
Set the name of the variable to use for the transformed x-coordinate axis. If not set the name is passed through.
void	set_target_x_axis_variable (const std::string &flag)

std::string	get_target_x_axis_variable () const

target_y_axis_variable
Set the name of the variable to use for the transformed y-coordinate axis.. If not set the name is passed through.
void	set_target_y_axis_variable (const std::string &flag)

std::string	get_target_y_axis_variable () const

target_z_axis_variable
Set the name of the variable to use for the transformed z-coordinate axis.. If not set the name is passed through.
void	set_target_z_axis_variable (const std::string &flag)

std::string	get_target_z_axis_variable () const

target_x_axis_units
set/get the units for the transformed x-coordinate axis. If not set the units are passed through.
void	set_target_x_axis_units (const std::string &flag)

std::string	get_target_x_axis_units () const

target_y_axis_units
set/get the units for the transformed y-coordinate axis. If not set the units are passed through.
void	set_target_y_axis_units (const std::string &flag)

std::string	get_target_y_axis_units () const

target_z_axis_units
set/get the units for the transformed z-coordinate axis. If not set the units are passed through.
void	set_target_z_axis_units (const std::string &flag)

std::string	get_target_z_axis_units () const

validate_time_axis
If set consistency checks are made to ensure that time axis from managed readers match each other. Names, calendar, units, and values of each array are verified.
void	set_validate_time_axis (const int &v)

const int &	get_validate_time_axis () const

validate_spatial_coordinates
If set consistency checks are made to ensure that spatial axes from managed readers match each other. Names, units, and values of each array are verified.
void	set_validate_spatial_coordinates (const int &v)

const int &	get_validate_spatial_coordinates () const

Public Member Functions inherited from teca_algorithm
	teca_algorithm (const teca_algorithm &src)=delete

	teca_algorithm (teca_algorithm &&src)=delete

teca_algorithm &	operator= (const teca_algorithm &src)=delete

teca_algorithm &	operator= (teca_algorithm &&src)=delete

void	set_communicator (MPI_Comm comm)

MPI_Comm	get_communicator ()
	get the active communicator More...

virtual teca_algorithm_output_port	get_output_port (unsigned int port=0)

void	set_input_connection (const teca_algorithm_output_port &port)
	set an input to this algorithm More...

virtual void	set_input_connection (unsigned int id, const teca_algorithm_output_port &port)
	set an input to this algorithm More...

virtual void	remove_input_connection (unsigned int id)
	remove input connections More...

void	clear_input_connections ()
	remove all input connections More...

const_p_teca_dataset	get_output_data (unsigned int port=0)

void	pop_cache (unsigned int port=0, int top=0)

void	set_cache_size (unsigned int n)
	set the cache size. the default is 1. (threadsafe) More...

virtual int	update ()
	execute the pipeline from this instance up. More...

virtual int	update (unsigned int port)
	execute the pipeline from this instance up. More...

virtual teca_metadata	update_metadata (unsigned int port=0)
	get meta data considering this instance up. More...

void	set_executive (p_teca_algorithm_executive exe)
	set the executive More...

p_teca_algorithm_executive	get_executive ()
	get the executive More...

virtual void	to_stream (std::ostream &s) const

virtual void	from_stream (std::istream &s)
	deserialize from the stream. More...

void	set_verbose (const int &v)

const int &	get_verbose () const

Static Public Member Functions
static p_teca_multi_cf_reader	New ()

Additional Inherited Members
Protected Member Functions inherited from teca_algorithm
void	set_number_of_input_connections (unsigned int n)

void	set_number_of_output_ports (unsigned int n)

void	set_modified (unsigned int port)
	an overload to set_modified by port More...

virtual std::vector< teca_metadata >	get_upstream_request (unsigned int port, const std::vector< teca_metadata > &input_md, const teca_metadata &request)

virtual teca_metadata	get_cache_key (unsigned int port, const teca_metadata &request) const

virtual teca_metadata	get_output_metadata (teca_algorithm_output_port &current)

virtual const_p_teca_dataset	request_data (teca_algorithm_output_port &port, const teca_metadata &request)

virtual int	validate_cache (teca_algorithm_output_port &current)

virtual void	clear_modified (teca_algorithm_output_port current)

const_p_teca_dataset	get_output_data (unsigned int port, const teca_metadata &request)

int	cache_output_data (unsigned int port, const teca_metadata &request, const_p_teca_dataset &data)

void	clear_cache (unsigned int port)
	clear the cache on the given output port More...

unsigned int	get_number_of_input_connections ()
	get the number of input connections More...

teca_algorithm_output_port &	get_input_connection (unsigned int i)

void	clear_modified (unsigned int port)
	clear the modified flag on the i'th output More...

int	get_modified (unsigned int port) const
	return the output port's modified flag value More...

Protected Attributes inherited from teca_algorithm
int	verbose

Detailed Description

A reader for data stored in NetCDF CF format in multiple files.

The data read is presented to the down stream as a single dataset

use the add_reader method to specify regular expression and corresponding list of variables to read. a reader, not necessarily the same one, must be selected to provide the time and spatial axes.

this reader could handle spatio-temporal interpolations as well, however that is currently not implemented. as a result all data is expected to be on the same coordinate system.

A number of algorithm properties modify run time behavior, most of these are exposed from teca_cf_reader. see the teca_cf_reader for details.

The reader may be initialized via a configuration file. The configuration file consists of name = value pairs and flags organized in sections. Sections are declared using []. There is an optional global section followed by a number of [cf_reader] sections. Each [cf_reader] section consists of a name(optional), a regex, a list of variables, a provides_time flag(optional) and a provides geometry flag(optional). At least one section must contain a provides_time and provides geometry flag. The global section may contain a data_root. Occurrences of the string data_root% in the regex are replaced with the value of data_root.

The following example configures the reader to read hus,ua and va.

# TECA multi_cf_reader config
 
data_root = /opt/TECA_data/HighResMIP/ECMWF-IFS-HR-SST-present
 
[cf_reader]
regex = %data_root%/hus/hus.*\.nc$
variables = hus
provides_time
provides_geometry
 
[cf_reader]
regex = %data_root%/va/va.*\.nc$
variables = va
 
[cf_reader]
regex = %data_root%/ua/ua.*\.nc$
variables = ua

Member Function Documentation

◆ add_reader()

int teca_multi_cf_reader::add_reader	(	const std::string &	regex,
		const std::string &	key,
		int	provides_time,
		int	provides_geometry,
		const std::vector< std::string > &	variables
	)

Adds a reader to the collection and at the same time specifies how it will be used. This is alternative way to configure the multi_cf_reader instead of providing the configuration via an MCF file (see set_input_file).

◆ add_variable_reader()

int teca_multi_cf_reader::add_variable_reader	(	const std::string &	key,
		const std::string &	variable
	)

adds to the list of variables that a reader will provide

◆ append_t_value()

void teca_multi_cf_reader::append_t_value ( const double & v )

inline

append to the t_value algorithm vector property

◆ clear_t_values()

void teca_multi_cf_reader::clear_t_values ( )

inline

clear the t_value algorithm vector property

◆ get_class_name()

const char* teca_multi_cf_reader::get_class_name ( ) const

inlineoverridevirtual

returns the name of the class

Implements teca_algorithm.

◆ get_max_metadata_ranks()

const int& teca_multi_cf_reader::get_max_metadata_ranks ( ) const

inline

Get the value of the max_metadata_ranks algorithm property

◆ get_number_of_t_values()

size_t teca_multi_cf_reader::get_number_of_t_values ( )

inline

get the size of the t_value algorithm vector property

◆ get_t_value()

const double& teca_multi_cf_reader::get_t_value ( size_t i ) const

inline

get the i-th element of the t_value algorithm vector property

◆ get_t_values()

const std::vector< double >& teca_multi_cf_reader::get_t_values ( ) const

inline

get the t_value algorithm vector property

◆ get_validate_spatial_coordinates()

const int& teca_multi_cf_reader::get_validate_spatial_coordinates ( ) const

inline

Get the value of the validate_spatial_coordinates algorithm property

◆ get_validate_time_axis()

const int& teca_multi_cf_reader::get_validate_time_axis ( ) const

inline

Get the value of the validate_time_axis algorithm property

◆ get_variables()

void teca_multi_cf_reader::get_variables ( std::vector< std::string > & vars )

get the list of variables that the reader will serve up

◆ New()

static p_teca_multi_cf_reader teca_multi_cf_reader::New ( )

inlinestatic

Returns an instance of teca_multi_cf_reader

◆ set_geometry_reader()

int teca_multi_cf_reader::set_geometry_reader ( const std::string & key )

sets the reader that provides the mesh geometry

◆ set_input_file()

int teca_multi_cf_reader::set_input_file ( const std::string & input_file )

Set the MCF configuration file that describes the dataset to read. Each section in the MCF file adds an internal reader.

◆ set_max_metadata_ranks()

void teca_multi_cf_reader::set_max_metadata_ranks ( const int & v )

inline

Set the value of the max_metadata_ranks algorithm property

◆ set_t_value() [1/2]

void teca_multi_cf_reader::set_t_value ( const double & v )

inline

set the t_value algorithm vector property to a single value

◆ set_t_value() [2/2]

void teca_multi_cf_reader::set_t_value	(	size_t	i,
		const double &	v
	)

inline

set the i-th element of the t_value algorithm vector property

◆ set_t_values() [1/2]

void teca_multi_cf_reader::set_t_values ( const std::initializer_list< double > && l )

inline

set the t_value algorithm vector property

◆ set_t_values() [2/2]

void teca_multi_cf_reader::set_t_values ( const std::vector< double > & v )

inline

set the t_value algorithm vector property

◆ set_time_reader()

int teca_multi_cf_reader::set_time_reader ( const std::string & key )

sets the reader that provides the time axis

◆ set_validate_spatial_coordinates()

void teca_multi_cf_reader::set_validate_spatial_coordinates ( const int & v )

inline

Set the value of the validate_spatial_coordinates algorithm property

◆ set_validate_time_axis()

void teca_multi_cf_reader::set_validate_time_axis ( const int & v )

inline

Set the value of the validate_time_axis algorithm property

◆ set_variable_reader()

int teca_multi_cf_reader::set_variable_reader	(	const std::string &	key,
		const std::vector< std::string > &	variable
	)

sets the list of variable that a reader will provide.

◆ shared_from_this() [1/2]

std::shared_ptr< teca_multi_cf_reader > teca_multi_cf_reader::shared_from_this ( )

inline

Enables the static constructor

◆ shared_from_this() [2/2]

std::shared_ptr< teca_multi_cf_reader const> teca_multi_cf_reader::shared_from_this ( ) const

inline

Enables the static constructor

The documentation for this class was generated from the following file:

develop/io/teca_multi_cf_reader.h

Public Member Functions

Static Public Member Functions

Additional Inherited Members

Detailed Description

Member Function Documentation

◆ add_reader()

◆ add_variable_reader()

◆ append_t_value()

◆ clear_t_values()

◆ get_class_name()

◆ get_max_metadata_ranks()

◆ get_number_of_t_values()

◆ get_t_value()

◆ get_t_values()

◆ get_validate_spatial_coordinates()

◆ get_validate_time_axis()

◆ get_variables()

◆ New()

◆ set_geometry_reader()

◆ set_input_file()

◆ set_max_metadata_ranks()

◆ set_t_value() [1/2]

◆ set_t_value() [2/2]

◆ set_t_values() [1/2]

◆ set_t_values() [2/2]

◆ set_time_reader()

◆ set_validate_spatial_coordinates()

◆ set_validate_time_axis()

◆ set_variable_reader()

◆ shared_from_this() [1/2]

◆ shared_from_this() [2/2]