teca_cf_writer Class Reference

A writer for Cartesian meshes in NetCDF CF2 format. More...

#include <teca_cf_writer.h>

Inheritance diagram for teca_cf_writer:

Collaboration diagram for teca_cf_writer:

Public Member Functions
std::shared_ptr< teca_cf_writer >	shared_from_this ()
std::shared_ptr< teca_cf_writer const >	shared_from_this () const
	teca_cf_writer (const teca_cf_writer &src)=delete
	teca_cf_writer (teca_cf_writer &&src)=delete
teca_cf_writer &	operator= (const teca_cf_writer &src)=delete
teca_cf_writer &	operator= (teca_cf_writer &&src)=delete
const char *	get_class_name () const override
file_name
Set the output filename. For time series the substring t% is replaced with the current time step or date. See comments on date_format below for info about date formatting.
void	set_file_name (const std::string &v)
const std::string &	get_file_name () const
date_format
set the format for the date to write in the filename. this requires the input dataset to have unit/calendar information if none are available, the time index is used instead. (F-HZ)
void	set_date_format (const std::string &v)
const std::string &	get_date_format () const
first_step
Set the first step in the range of time step to process.
void	set_first_step (const long &v)
const long &	get_first_step () const
last_step
Set the last step in the range of time step to process.
void	set_last_step (const long &v)
const long &	get_last_step () const
steps_per_file
Set how many time steps are written to each file when the layout mode is set to number_of_steps.
void	set_steps_per_file (const unsigned int &v)
const unsigned int &	get_steps_per_file () const
mode_flags
sets the flags passed to NetCDF during file creation. (NC_CLOBBER)
void	set_mode_flags (const int &v)
const int &	get_mode_flags () const
use_unlimited_dim
if set the slowest varying dimension is specified to be NC_UNLIMITED. This has a negative impact on performance when reading the values in a single pass. However, unlimited dimensions are used ubiquitously thus by default it is set. For data being consumed by TECA performance will be better when using fixed dimensions. (1) This feature requires collective writes and is incompatible with out of order execution, and hence currently not supported.
void	set_use_unlimited_dim (const int &v)
const int &	get_use_unlimited_dim () const
compression_level
sets the compression level used for each variable compression is not used if the value is less than or equal to 0. This feature requires collective writes and is incompatible with out of order execution, and hence currently not supported.
void	set_compression_level (const int &v)
const int &	get_compression_level () const
collective_buffer
Enables MPI I/O colective buffering. Collective buffering is only valid when the spatial partitioner is enabled and the number of spatial partitions is equal to the number of MPI ranks. If set to -1 (the default) collective buffering will automatically enabled when it is possible to do so.
void	set_collective_buffer (const int &v)
const int &	get_collective_buffer () const
flush_files
Flush files before closing them, this may be necessary if accessing data immediately.
void	set_flush_files (const int &v)
const int &	get_flush_files () const
move_variables_to_root
Move variables to root instead of creating groups
void	set_move_variables_to_root (const int &v)
const int &	get_move_variables_to_root () const
point_array
Specify the arrays to write. A data array is only written to disk if it is included in this list. It is an error to not specify at least one point centered array to write
size_t	get_number_of_point_arrays ()
void	append_point_array (const std::string &v)
void	set_point_array (const std::string &v)
void	set_point_array (size_t i, const std::string &v)
void	set_point_arrays (const std::vector< std::string > &v)
void	set_point_arrays (const std::initializer_list< std::string > &&l)
const std::string &	get_point_array (size_t i) const
const std::vector< std::string > &	get_point_arrays () const
void	clear_point_arrays ()
information_array
Set the list of non-geometric arrays to write.
size_t	get_number_of_information_arrays ()
void	append_information_array (const std::string &v)
void	set_information_array (const std::string &v)
void	set_information_array (size_t i, const std::string &v)
void	set_information_arrays (const std::vector< std::string > &v)
void	set_information_arrays (const std::initializer_list< std::string > &&l)
const std::string &	get_information_array (size_t i) const
const std::vector< std::string > &	get_information_arrays () const
void	clear_information_arrays ()
number_of_spatial_partitions
Set the number of spatial partitions. If less than one then the number of MPI ranks is used.
void	set_number_of_spatial_partitions (const long &v)
const long &	get_number_of_spatial_partitions () const
partition_x
enables/disables spatial partitioning in the x-direction
void	set_partition_x (const int &v)
const int &	get_partition_x () const
partition_y
enables/disables spatial partitioning in the y-direction
void	set_partition_y (const int &v)
const int &	get_partition_y () const
partition_z
enables/disables spatial partitioning in the z-direction
void	set_partition_z (const int &v)
const int &	get_partition_z () const
minimum_block_size_x
Sets the minimum block size for spatial partitioning in the x-direction
void	set_minimum_block_size_x (const long &v)
const long &	get_minimum_block_size_x () const
minimum_block_size_y
Sets the minimum block size for spatial partitioning in the y-direction
void	set_minimum_block_size_y (const long &v)
const long &	get_minimum_block_size_y () const
minimum_block_size_z
Sets the minimum block size for spatial partitioning in the z-direction
void	set_minimum_block_size_z (const long &v)
const long &	get_minimum_block_size_z () const
number_of_temporal_partitions
Set the number of temporal partitions. If set to less than one then the number of time steps is used. The temporal_partition_size property takes precedence, if it is set then the this property is ignored. The default value is zero.
void	set_number_of_temporal_partitions (const long &v)
const long &	get_number_of_temporal_partitions () const
temporal_partition_size
Set the size of the temporal partitions. If set to less than one then the number_of_temporal_partition property is used instead. The default value is zero.
void	set_temporal_partition_size (const long &v)
const long &	get_temporal_partition_size () const
index_executive_compatibility
If set and spatial partitioner is enabled, the writer will make one request per time step using the index_request_key as the teca_index_executive would. This could be used parallelize existing algorithms over space and time.
void	set_index_executive_compatability (const int &v)
const int &	get_index_executive_compatability () const
Public Member Functions inherited from teca_threaded_algorithm
std::shared_ptr< teca_threaded_algorithm >	shared_from_this ()
std::shared_ptr< teca_threaded_algorithm const >	shared_from_this () const
	teca_threaded_algorithm (const teca_threaded_algorithm &src)=delete
	teca_threaded_algorithm (teca_threaded_algorithm &&src)=delete
teca_threaded_algorithm &	operator= (const teca_threaded_algorithm &src)=delete
teca_threaded_algorithm &	operator= (teca_threaded_algorithm &&src)=delete
void	set_thread_pool_size (int n_threads)
unsigned int	get_thread_pool_size () const noexcept
	Get the number of threads in the pool. More...
void	set_data_request_queue (const p_teca_data_request_queue &queue)
	explicitly set the thread pool to submit requests to More...
void	set_bind_threads (const int &v)
const int &	get_bind_threads () const
void	set_stream_size (const int &v)
const int &	get_stream_size () const
void	set_poll_interval (const long long &v)
const long long &	get_poll_interval () const
void	set_threads_per_device (const int &v)
const int &	get_threads_per_device () const
void	set_ranks_per_device (const int &v)
const int &	get_ranks_per_device () const
void	set_propagate_device_assignment (const int &v)
const int &	get_propagate_device_assignment () const
Public Member Functions inherited from teca_algorithm
	teca_algorithm (const teca_algorithm &src)=delete
	teca_algorithm (teca_algorithm &&src)=delete
teca_algorithm &	operator= (const teca_algorithm &src)=delete
teca_algorithm &	operator= (teca_algorithm &&src)=delete
void	set_communicator (MPI_Comm comm)
MPI_Comm	get_communicator ()
	get the active communicator More...
virtual teca_algorithm_output_port	get_output_port (unsigned int port=0)
void	set_input_connection (const teca_algorithm_output_port &port)
	set an input to this algorithm More...
virtual void	set_input_connection (unsigned int id, const teca_algorithm_output_port &port)
	set an input to this algorithm More...
virtual void	remove_input_connection (unsigned int id)
	remove input connections More...
void	clear_input_connections ()
	remove all input connections More...
const_p_teca_dataset	get_output_data (unsigned int port=0)
void	pop_cache (unsigned int port=0, int top=0)
void	set_cache_size (unsigned int n)
	set the cache size. the default is 1. (threadsafe) More...
virtual int	update ()
	execute the pipeline from this instance up. More...
virtual int	update (unsigned int port)
	execute the pipeline from this instance up. More...
virtual teca_metadata	update_metadata (unsigned int port=0)
	get meta data considering this instance up. More...
void	set_executive (p_teca_algorithm_executive exe)
	set the executive More...
p_teca_algorithm_executive	get_executive ()
	get the executive More...
virtual void	to_stream (std::ostream &s) const
virtual void	from_stream (std::istream &s)
	deserialize from the stream. More...
void	set_verbose (const int &v)
const int &	get_verbose () const

Static Public Member Functions
static p_teca_cf_writer	New ()
Static Public Member Functions inherited from teca_threaded_algorithm
static p_teca_threaded_algorithm	New ()

layout
Set the layout mode to one of : number_of_steps, daily, monthly, seasonal, or yearly. This controls the size of the files written. In daily, monthly, seasonal, and yearly modes each file will contain the steps spanning the given duration. The number_of_steps mode writes a fixed number of steps per file which can be set using the steps_per_file property.
enum	{   invalid =0 , number_of_steps =1 , daily =2 , monthly =3 ,   seasonal =4 , yearly =5 }
void	set_layout (const int &v)
const int &	get_layout () const
void	set_layout_to_number_of_steps ()
void	set_layout_to_daily ()
void	set_layout_to_monthly ()
void	set_layout_to_seasonal ()
void	set_layout_to_yearly ()
int	set_layout (const std::string &layout)
	set the layout mode from a string. More...
int	validate_layout (int mode)
const char *	get_layout_name () const

spatial_partitioner
Enable spatial partitioner, When spatial partitioner is enabled both temporal and spatial dimensions of input data are partitioned for load balancing. The partitioner is controled by the number_of_spatial_partitions, number_of_temporal_partitions and, temporal_partition_size properties.
enum	{ temporal , spatial , space_time }
void	set_partitioner (const int &v)
const int &	get_partitioner () const
void	set_partitioner (const std::string &part)
	set the partitioner from a string More...
void	set_partitioner_to_temporal ()
	enables temporal partitioner More...
void	set_partitioner_to_spatial ()
	enables spatial partitioner More...
void	set_partitioner_to_space_time ()
	enables space-time partitioner More...
int	validate_partitioner (int mode)
const char *	get_partitioner_name () const

Additional Inherited Members
Protected Member Functions inherited from teca_threaded_algorithm
const_p_teca_dataset	execute (unsigned int port, const std::vector< const_p_teca_dataset > &input_data, const teca_metadata &request) override
const_p_teca_dataset	request_data (teca_algorithm_output_port &port, const teca_metadata &request) override
Protected Member Functions inherited from teca_algorithm
void	set_number_of_input_connections (unsigned int n)
void	set_number_of_output_ports (unsigned int n)
virtual void	set_modified ()
void	set_modified (unsigned int port)
	an overload to set_modified by port More...
virtual teca_metadata	get_cache_key (unsigned int port, const teca_metadata &request) const
virtual teca_metadata	get_output_metadata (teca_algorithm_output_port &current)
virtual int	validate_cache (teca_algorithm_output_port &current)
virtual void	clear_modified (teca_algorithm_output_port current)
const_p_teca_dataset	get_output_data (unsigned int port, const teca_metadata &request)
int	cache_output_data (unsigned int port, const teca_metadata &request, const_p_teca_dataset &data)
void	clear_cache (unsigned int port)
	clear the cache on the given output port More...
unsigned int	get_number_of_input_connections ()
	get the number of input connections More...
teca_algorithm_output_port &	get_input_connection (unsigned int i)
void	clear_modified (unsigned int port)
	clear the modified flag on the i'th output More...
int	get_modified (unsigned int port) const
	return the output port's modified flag value More...
Protected Attributes inherited from teca_algorithm
int	verbose

Detailed Description

A writer for Cartesian meshes in NetCDF CF2 format.

Writes data to NetCDF CF2 format. This algorithm is conceptually an execution engine capable of driving the above pipeline with our without threads and stream results in the order that they are generated placing them in the correct location in the output dataset. The output dataset is a collection of files each with a user specified number of time steps per file. The output dataset may be arranged using a fixed number of steps per file or daily, monthly, seasonal, or yearly file layouts. The total number of time steps in the output dataset is determined by the combination of the number of time steps in the input dataset and user defined subsetting if any. The writer uses MPI collective I/O to produce the files. In parallel time steps are mapped to ranks such that each rank has approximately the same number of time steps. Incoming steps are mapped to files. A given MPI rank may be writing to multiple files. The use of MPI collectives implies care must be taken in its use to avoid deadlocks.

Due to the use of MPI collectives I/O certain information must be known during the report phase of pipeline execution, before the execute phase of pipeline execution begins. The information that is needed is:

number of time steps

specified by the pipeline control index_initializer key found in metadata
produced by the source (e.g CF reader)

extent

6, 64 bit integers defining the 3 spatial dimensions of each timestep found
in metadata produced by the source (e.g CF reader)

point arrays

list of strings naming the point centered arrays that will be written. set
by the user prior to execution by writer properties.

information arrays

list of strings naming the non-geometric arrays that will written. set by
the user prior to execution by writer properties. See also size attribute
below.

type_code

the teca_variant_array_code naming the type of each array. this will be in
the array attributes metadata generated by the producer of the array (e.g
any algorithm that adds an array should provide this metadata).

size

a 64 bit integer declaring the size of each information array. this will be
in the array attributes metadata generated by the producer of the array (e.g
any algorithm that adds an array should provide this metadata).

Member Enumeration Documentation

anonymous enum

anonymous enum

Enumerator
temporal	map time steps to MPI ranks
spatial	map spatial extents to MPI ranks, time is processed sequentially
space_time	both spatial and temporal extents to MPI ranks

Member Function Documentation

append_information_array()

void teca_cf_writer::append_information_array ( const std::string &  v )

inline

append to the information_array algorithm vector property

append_point_array()

void teca_cf_writer::append_point_array ( const std::string &  v )

inline

append to the point_array algorithm vector property

clear_information_arrays()

void teca_cf_writer::clear_information_arrays ( )

inline

clear the information_array algorithm vector property

clear_point_arrays()

void teca_cf_writer::clear_point_arrays ( )

inline

clear the point_array algorithm vector property

get_class_name()

const char* teca_cf_writer::get_class_name ( ) const

inlineoverridevirtual

returns the name of the class

Reimplemented from teca_threaded_algorithm.

get_collective_buffer()

const int& teca_cf_writer::get_collective_buffer ( ) const

inline

Get the value of the collective_buffer algorithm property

get_compression_level()

const int& teca_cf_writer::get_compression_level ( ) const

inline

Get the value of the compression_level algorithm property

get_date_format()

const std::string& teca_cf_writer::get_date_format ( ) const

inline

Get the value of the date_format algorithm property

get_file_name()

const std::string& teca_cf_writer::get_file_name ( ) const

inline

Get the value of the file_name algorithm property

get_first_step()

const long& teca_cf_writer::get_first_step ( ) const

inline

Get the value of the first_step algorithm property

get_flush_files()

const int& teca_cf_writer::get_flush_files ( ) const

inline

Get the value of the flush_files algorithm property

get_index_executive_compatability()

const int& teca_cf_writer::get_index_executive_compatability ( ) const

inline

Get the value of the index_executive_compatability algorithm property

get_information_array()

const std::string& teca_cf_writer::get_information_array ( size_t  i ) const

inline

get the i-th element of the information_array algorithm vector property

get_information_arrays()

const std::vector< std::string >& teca_cf_writer::get_information_arrays ( ) const

inline

get the information_array algorithm vector property

get_last_step()

const long& teca_cf_writer::get_last_step ( ) const

inline

Get the value of the last_step algorithm property

get_layout()

const int& teca_cf_writer::get_layout ( ) const

inline

Get the value of the layout algorithm property

get_layout_name()

const char* teca_cf_writer::get_layout_name

(

)

const

Returns

a string representation of the current layout

get_minimum_block_size_x()

const long& teca_cf_writer::get_minimum_block_size_x ( ) const

inline

Get the value of the minimum_block_size_x algorithm property

get_minimum_block_size_y()

const long& teca_cf_writer::get_minimum_block_size_y ( ) const

inline

Get the value of the minimum_block_size_y algorithm property

get_minimum_block_size_z()

const long& teca_cf_writer::get_minimum_block_size_z ( ) const

inline

Get the value of the minimum_block_size_z algorithm property

get_mode_flags()

const int& teca_cf_writer::get_mode_flags ( ) const

inline

Get the value of the mode_flags algorithm property

get_move_variables_to_root()

const int& teca_cf_writer::get_move_variables_to_root ( ) const

inline

Get the value of the move_variables_to_root algorithm property

get_number_of_information_arrays()

size_t teca_cf_writer::get_number_of_information_arrays ( )

inline

get the size of the information_array algorithm vector property

get_number_of_point_arrays()

size_t teca_cf_writer::get_number_of_point_arrays ( )

inline

get the size of the point_array algorithm vector property

get_number_of_spatial_partitions()

const long& teca_cf_writer::get_number_of_spatial_partitions ( ) const

inline

Get the value of the number_of_spatial_partitions algorithm property

get_number_of_temporal_partitions()

const long& teca_cf_writer::get_number_of_temporal_partitions ( ) const

inline

Get the value of the number_of_temporal_partitions algorithm property

get_partition_x()

const int& teca_cf_writer::get_partition_x ( ) const

inline

Get the value of the partition_x algorithm property

get_partition_y()

const int& teca_cf_writer::get_partition_y ( ) const

inline

Get the value of the partition_y algorithm property

get_partition_z()

const int& teca_cf_writer::get_partition_z ( ) const

inline

Get the value of the partition_z algorithm property

get_partitioner()

const int& teca_cf_writer::get_partitioner ( ) const

inline

Get the value of the partitioner algorithm property

get_partitioner_name()

const char* teca_cf_writer::get_partitioner_name

(

)

const

Returns

the name of the current partitioner mode

get_point_array()

const std::string& teca_cf_writer::get_point_array ( size_t  i ) const

inline

get the i-th element of the point_array algorithm vector property

get_point_arrays()

const std::vector< std::string >& teca_cf_writer::get_point_arrays ( ) const

inline

get the point_array algorithm vector property

get_steps_per_file()

const unsigned int& teca_cf_writer::get_steps_per_file ( ) const

inline

Get the value of the steps_per_file algorithm property

get_temporal_partition_size()

const long& teca_cf_writer::get_temporal_partition_size ( ) const

inline

Get the value of the temporal_partition_size algorithm property

get_use_unlimited_dim()

const int& teca_cf_writer::get_use_unlimited_dim ( ) const

inline

Get the value of the use_unlimited_dim algorithm property

New()

static p_teca_cf_writer teca_cf_writer::New ( )

inlinestatic

Returns an instance of teca_cf_writer

set_collective_buffer()

void teca_cf_writer::set_collective_buffer ( const int &  v )

inline

Set the value of the collective_buffer algorithm property

set_compression_level()

void teca_cf_writer::set_compression_level ( const int &  v )

inline

Set the value of the compression_level algorithm property

set_date_format()

void teca_cf_writer::set_date_format ( const std::string &  v )

inline

Set the value of the date_format algorithm property

set_file_name()

void teca_cf_writer::set_file_name ( const std::string &  v )

inline

Set the value of the file_name algorithm property

set_first_step()

void teca_cf_writer::set_first_step ( const long &  v )

inline

Set the value of the first_step algorithm property

set_flush_files()

void teca_cf_writer::set_flush_files ( const int &  v )

inline

Set the value of the flush_files algorithm property

set_index_executive_compatability()

void teca_cf_writer::set_index_executive_compatability ( const int &  v )

inline

Set the value of the index_executive_compatability algorithm property

set_information_array() [1/2]

void teca_cf_writer::set_information_array ( const std::string &  v )

inline

set the information_array algorithm vector property to a single value

set_information_array() [2/2]

void teca_cf_writer::set_information_array ( size_t  i, const std::string &  v  )

inline

set the i-th element of the information_array algorithm vector property

set_information_arrays() [1/2]

void teca_cf_writer::set_information_arrays ( const std::initializer_list< std::string > &&  l )

inline

set the information_array algorithm vector property

set_information_arrays() [2/2]

void teca_cf_writer::set_information_arrays ( const std::vector< std::string > &  v )

inline

set the information_array algorithm vector property

set_last_step()

void teca_cf_writer::set_last_step ( const long &  v )

inline

Set the value of the last_step algorithm property

set_layout() [1/2]

void teca_cf_writer::set_layout ( const int &  v )

inline

Set the value of the layout algorithm property

set_layout() [2/2]

int teca_cf_writer::set_layout

(

const std::string &

layout

)

set the layout mode from a string.

set_minimum_block_size_x()

void teca_cf_writer::set_minimum_block_size_x ( const long &  v )

inline

Set the value of the minimum_block_size_x algorithm property

set_minimum_block_size_y()

void teca_cf_writer::set_minimum_block_size_y ( const long &  v )

inline

Set the value of the minimum_block_size_y algorithm property

set_minimum_block_size_z()

void teca_cf_writer::set_minimum_block_size_z ( const long &  v )

inline

Set the value of the minimum_block_size_z algorithm property

set_mode_flags()

void teca_cf_writer::set_mode_flags ( const int &  v )

inline

Set the value of the mode_flags algorithm property

set_move_variables_to_root()

void teca_cf_writer::set_move_variables_to_root ( const int &  v )

inline

Set the value of the move_variables_to_root algorithm property

set_number_of_spatial_partitions()

void teca_cf_writer::set_number_of_spatial_partitions ( const long &  v )

inline

Set the value of the number_of_spatial_partitions algorithm property

set_number_of_temporal_partitions()

void teca_cf_writer::set_number_of_temporal_partitions ( const long &  v )

inline

Set the value of the number_of_temporal_partitions algorithm property

set_partition_x()

void teca_cf_writer::set_partition_x ( const int &  v )

inline

Set the value of the partition_x algorithm property

set_partition_y()

void teca_cf_writer::set_partition_y ( const int &  v )

inline

Set the value of the partition_y algorithm property

set_partition_z()

void teca_cf_writer::set_partition_z ( const int &  v )

inline

Set the value of the partition_z algorithm property

set_partitioner() [1/2]

void teca_cf_writer::set_partitioner ( const int &  v )

inline

Set the value of the partitioner algorithm property

set_partitioner() [2/2]

void teca_cf_writer::set_partitioner

(

const std::string &

part

)

set the partitioner from a string

set_partitioner_to_space_time()

void teca_cf_writer::set_partitioner_to_space_time ( )

inline

enables space-time partitioner

set_partitioner_to_spatial()

void teca_cf_writer::set_partitioner_to_spatial ( )

inline

enables spatial partitioner

set_partitioner_to_temporal()

void teca_cf_writer::set_partitioner_to_temporal ( )

inline

enables temporal partitioner

set_point_array() [1/2]

void teca_cf_writer::set_point_array ( const std::string &  v )

inline

set the point_array algorithm vector property to a single value

set_point_array() [2/2]

void teca_cf_writer::set_point_array ( size_t  i, const std::string &  v  )

inline

set the i-th element of the point_array algorithm vector property

set_point_arrays() [1/2]

void teca_cf_writer::set_point_arrays ( const std::initializer_list< std::string > &&  l )

inline

set the point_array algorithm vector property

set_point_arrays() [2/2]

void teca_cf_writer::set_point_arrays ( const std::vector< std::string > &  v )

inline

set the point_array algorithm vector property

set_steps_per_file()

void teca_cf_writer::set_steps_per_file ( const unsigned int &  v )

inline

Set the value of the steps_per_file algorithm property

set_temporal_partition_size()

void teca_cf_writer::set_temporal_partition_size ( const long &  v )

inline

Set the value of the temporal_partition_size algorithm property

set_use_unlimited_dim()

void teca_cf_writer::set_use_unlimited_dim ( const int &  v )

inline

Set the value of the use_unlimited_dim algorithm property

shared_from_this() [1/2]

std::shared_ptr< teca_cf_writer > teca_cf_writer::shared_from_this ( )

inline

Enables the static constructor

shared_from_this() [2/2]

std::shared_ptr< teca_cf_writer const> teca_cf_writer::shared_from_this ( ) const

inline

Enables the static constructor

validate_layout()

int teca_cf_writer::validate_layout ( int  mode )

inline

Returns

0 if the passed value is a valid layout mode

validate_partitioner()

int teca_cf_writer::validate_partitioner ( int  mode )

inline

Returns

0 if the passed value is a valid partitioner mode

---

The documentation for this class was generated from the following file:

• rtdfixjuly2025/io/teca_cf_writer.h