Base class for MPI+threads+GPUs map reduce reduction over a set of indices. More...

#include <teca_index_reduce.h>

Inheritance diagram for teca_index_reduce:

Collaboration diagram for teca_index_reduce:

Public Types
using	extent_type = std::vector< long >

using	bounds_type = std::vector< double >

Public Member Functions
	teca_index_reduce (const teca_index_reduce &src)=delete

	teca_index_reduce (teca_index_reduce &&src)=delete

teca_index_reduce &	operator= (const teca_index_reduce &src)=delete

teca_index_reduce &	operator= (teca_index_reduce &&src)=delete

extent
If provided the extent defines the index space region [i0, i1, j0, j1, k0, k1] of data to request. If not provided the whole_extent is requested when the whole_extent key is present. Otherwise the request is not modified.
void	set_extent (const extent_type &v)

const extent_type &	get_extent () const

bounds
If provided the bounds defines the world coordinate space region [x0, x1, y0, y1, z0, z1] of data to request. bounds take precedence over extents. If bounds are not provided any provided extents will be used (see ::set_extents) otherwise the request is not modified.
void	set_bounds (const bounds_type &v)

const bounds_type &	get_bounds () const

arrays
set the list of arrays to request. If not provided then the request is not modified.
size_t	get_number_of_arrays ()

void	append_array (const std::string &v)

void	set_array (const std::string &v)

void	set_array (size_t i, const std::string &v)

void	set_arrays (const std::vector< std::string > &v)

void	set_arrays (const std::initializer_list< std::string > &&l)

const std::string &	get_array (size_t i) const

const std::vector< std::string > &	get_arrays () const

void	clear_arrays ()

start_index
set the first index to process. Indices go from 0 to n-1. The default start_index is 0.
void	set_start_index (const long &v)

const long &	get_start_index () const

end_index
set the last index to process. Indices go from 0 to n-1. The default end_index is -1, which is used to indicate all indices should be processed.
void	set_end_index (const long &v)

const long &	get_end_index () const

Public Member Functions inherited from teca_threaded_algorithm
std::shared_ptr< teca_threaded_algorithm >	shared_from_this ()

std::shared_ptr< teca_threaded_algorithm const >	shared_from_this () const

	teca_threaded_algorithm (const teca_threaded_algorithm &src)=delete

	teca_threaded_algorithm (teca_threaded_algorithm &&src)=delete

teca_threaded_algorithm &	operator= (const teca_threaded_algorithm &src)=delete

teca_threaded_algorithm &	operator= (teca_threaded_algorithm &&src)=delete

const char *	get_class_name () const override

void	set_thread_pool_size (int n_threads)

unsigned int	get_thread_pool_size () const noexcept
	Get the number of threads in the pool. More...

void	set_data_request_queue (const p_teca_data_request_queue &queue)
	explicitly set the thread pool to submit requests to More...

void	set_bind_threads (const int &v)

const int &	get_bind_threads () const

void	set_stream_size (const int &v)

const int &	get_stream_size () const

void	set_poll_interval (const long long &v)

const long long &	get_poll_interval () const

void	set_threads_per_device (const int &v)

const int &	get_threads_per_device () const

void	set_ranks_per_device (const int &v)

const int &	get_ranks_per_device () const

void	set_propagate_device_assignment (const int &v)

const int &	get_propagate_device_assignment () const

Public Member Functions inherited from teca_algorithm
	teca_algorithm (const teca_algorithm &src)=delete

	teca_algorithm (teca_algorithm &&src)=delete

teca_algorithm &	operator= (const teca_algorithm &src)=delete

teca_algorithm &	operator= (teca_algorithm &&src)=delete

void	set_communicator (MPI_Comm comm)

MPI_Comm	get_communicator ()
	get the active communicator More...

virtual teca_algorithm_output_port	get_output_port (unsigned int port=0)

void	set_input_connection (const teca_algorithm_output_port &port)
	set an input to this algorithm More...

virtual void	set_input_connection (unsigned int id, const teca_algorithm_output_port &port)
	set an input to this algorithm More...

virtual void	remove_input_connection (unsigned int id)
	remove input connections More...

void	clear_input_connections ()
	remove all input connections More...

const_p_teca_dataset	get_output_data (unsigned int port=0)

void	pop_cache (unsigned int port=0, int top=0)

void	set_cache_size (unsigned int n)
	set the cache size. the default is 1. (threadsafe) More...

virtual int	update ()
	execute the pipeline from this instance up. More...

virtual int	update (unsigned int port)
	execute the pipeline from this instance up. More...

virtual teca_metadata	update_metadata (unsigned int port=0)
	get meta data considering this instance up. More...

void	set_executive (p_teca_algorithm_executive exe)
	set the executive More...

p_teca_algorithm_executive	get_executive ()
	get the executive More...

virtual void	to_stream (std::ostream &s) const

virtual void	from_stream (std::istream &s)
	deserialize from the stream. More...

void	set_verbose (const int &v)

const int &	get_verbose () const

Protected Member Functions
virtual p_teca_dataset	reduce (int device_id, const const_p_teca_dataset &left, const const_p_teca_dataset &right)=0

virtual p_teca_dataset	finalize (int device_id, const const_p_teca_dataset &ds)

virtual std::vector< teca_metadata >	initialize_upstream_request (unsigned int port, const std::vector< teca_metadata > &input_md, const teca_metadata &request)

virtual teca_metadata	initialize_output_metadata (unsigned int port, const std::vector< teca_metadata > &input_md)

std::vector< teca_metadata >	get_upstream_request (unsigned int port, const std::vector< teca_metadata > &input_md, const teca_metadata &request) override

const_p_teca_dataset	execute (unsigned int port, const std::vector< const_p_teca_dataset > &input_data, const teca_metadata &request, int streaming) override

teca_metadata	get_output_metadata (unsigned int port, const std::vector< teca_metadata > &input_md) override
	consumes index metadata, and partitions indices across MPI ranks. More...

virtual const_p_teca_dataset	execute (unsigned int port, const std::vector< const_p_teca_dataset > &input_data, const teca_metadata &request, int streaming)

const_p_teca_dataset	execute (unsigned int port, const std::vector< const_p_teca_dataset > &input_data, const teca_metadata &request) override

Protected Member Functions inherited from teca_threaded_algorithm
const_p_teca_dataset	execute (unsigned int port, const std::vector< const_p_teca_dataset > &input_data, const teca_metadata &request) override

const_p_teca_dataset	request_data (teca_algorithm_output_port &port, const teca_metadata &request) override

Protected Member Functions inherited from teca_algorithm
void	set_number_of_input_connections (unsigned int n)

void	set_number_of_output_ports (unsigned int n)

virtual void	set_modified ()

void	set_modified (unsigned int port)
	an overload to set_modified by port More...

virtual teca_metadata	get_cache_key (unsigned int port, const teca_metadata &request) const

virtual teca_metadata	get_output_metadata (teca_algorithm_output_port &current)

virtual int	validate_cache (teca_algorithm_output_port &current)

virtual void	clear_modified (teca_algorithm_output_port current)

const_p_teca_dataset	get_output_data (unsigned int port, const teca_metadata &request)

int	cache_output_data (unsigned int port, const teca_metadata &request, const_p_teca_dataset &data)

void	clear_cache (unsigned int port)
	clear the cache on the given output port More...

unsigned int	get_number_of_input_connections ()
	get the number of input connections More...

teca_algorithm_output_port &	get_input_connection (unsigned int i)

void	clear_modified (unsigned int port)
	clear the modified flag on the i'th output More...

int	get_modified (unsigned int port) const
	return the output port's modified flag value More...

Additional Inherited Members
Static Public Member Functions inherited from teca_threaded_algorithm
static p_teca_threaded_algorithm	New ()

Protected Attributes inherited from teca_algorithm
int	verbose

Detailed Description

Base class for MPI+threads+GPUs map reduce reduction over a set of indices.

The available indices are partitioned across MPI ranks and threads. Threads are assigned to service GPUs or CPU cores. One can restrict operation to a range of time steps by setting first and last indices to process.

metadata keys:

Requires:

key	description
index_initializer_key	Holds the name of the key that tells how many
	indices are available. The named key must also
	be present and should contain the number of
	indices available.
index_request_key	Holds the name of the key used to request a
	specific index. Requests are generated with this
	name set to a specific index to be processed
	some upstream algorithm is expected to produce
	the data associated with the given index.

Exports:

Key	Description
index_request_key	The name of the key holding the requested index
<index_request_key>	the requested index
device_id	the CPU (-1) or CUDA device (0 - n-1 devices) to
	use for calculations
bounds	the [x0 x1 y0 y1 z0 z1] spatial bounds requested
	(optional)
extent	the [i0 i1 j0 j1 k0 k1] index space grid extent
	requested (optional)
arrays	a list of arrays requested (optional)

Member Function Documentation

◆ append_array()

void teca_index_reduce::append_array ( const std::string & v )

inline

append to the array algorithm vector property

◆ clear_arrays()

void teca_index_reduce::clear_arrays ( )

inline

clear the array algorithm vector property

◆ execute()

const_p_teca_dataset teca_index_reduce::execute	(	unsigned int	port,
		const std::vector< const_p_teca_dataset > &	input_data,
		const teca_metadata &	request,
		int	streaming
	)

overrideprotectedvirtual

Uses MPI communication to collect remote data for required for the reduction. Calls ::reduce with each pair of datasets until the datasets across all threads and ranks are reduced into a single dataset, which is returned.

Reimplemented from teca_threaded_algorithm.

◆ finalize()

virtual p_teca_dataset teca_index_reduce::finalize	(	int	device_id,
		const const_p_teca_dataset &	ds
	)

inlineprotectedvirtual

An override that is called when the reduction is complete. The default implementation passes data through. This might be used for instance to complete an averaging operation where the ::reduce override sums the data and the ::finalize override scales by 1/N, where N is the number of datasets summed.

Parameters

[in]	device_id	The device that should be used for the reduction. A value of -1 indicates the CPU should be used.
[in]	ds	the reduced dataset

Returns: a dataset that has been finalized.

Reimplemented in teca_programmable_reduce.

◆ get_array()

const std::string& teca_index_reduce::get_array ( size_t i ) const

inline

get the i-th element of the array algorithm vector property

◆ get_arrays()

const std::vector< std::string >& teca_index_reduce::get_arrays ( ) const

inline

get the array algorithm vector property

◆ get_bounds()

const bounds_type& teca_index_reduce::get_bounds ( ) const

inline

Get the value of the bounds algorithm property

◆ get_end_index()

const long& teca_index_reduce::get_end_index ( ) const

inline

Get the value of the end_index algorithm property

◆ get_extent()

const extent_type& teca_index_reduce::get_extent ( ) const

inline

Get the value of the extent algorithm property

◆ get_number_of_arrays()

size_t teca_index_reduce::get_number_of_arrays ( )

inline

get the size of the array algorithm vector property

◆ get_output_metadata()

teca_metadata teca_index_reduce::get_output_metadata	(	unsigned int	port,
		const std::vector< teca_metadata > &	input_md
	)

overrideprotectedvirtual

consumes index metadata, and partitions indices across MPI ranks.

Reimplemented from teca_algorithm.

◆ get_start_index()

const long& teca_index_reduce::get_start_index ( ) const

inline

Get the value of the start_index algorithm property

◆ get_upstream_request()

std::vector<teca_metadata> teca_index_reduce::get_upstream_request	(	unsigned int	port,
		const std::vector< teca_metadata > &	input_md,
		const teca_metadata &	request
	)

overrideprotectedvirtual

Generates an upstream request for each index. will call initialize_upstream_request and apply the results to all time steps.

Reimplemented from teca_algorithm.

◆ initialize_output_metadata()

virtual teca_metadata teca_index_reduce::initialize_output_metadata	(	unsigned int	port,
		const std::vector< teca_metadata > &	input_md
	)

protectedvirtual

An override that allows derived classes to report what they can produce. this will be called from ::get_output_metadata which will strip out time and partition time across MPI ranks. The default implementation passes the incoming metadata through.

Reimplemented in teca_cf_time_axis_data_reduce, and teca_programmable_reduce.

◆ initialize_upstream_request()

virtual std::vector<teca_metadata> teca_index_reduce::initialize_upstream_request	(	unsigned int	port,
		const std::vector< teca_metadata > &	input_md,
		const teca_metadata &	request
	)

protectedvirtual

An override that allows derived classes to generate upstream requests that will be applied over all time steps. derived classes implement this method instead of ::get_upstream_request, which here is already implemented to handle the application of requests over the index set. The default implementation creates an empty request that is then populated with extent, bounds, and arrays if these have been provided.

Reimplemented in teca_cf_time_axis_data_reduce, and teca_programmable_reduce.

◆ reduce()

virtual p_teca_dataset teca_index_reduce::reduce	(	int	device_id,
		const const_p_teca_dataset &	left,
		const const_p_teca_dataset &	right
	)

protectedpure virtual

An override that implements the reduction. given two datasets a left and right, reduce into a single dataset and return.

Parameters

[in]	device_id	The device that should be used for the reduction. A value of -1 indicates the CPU should be used.
[in]	left	a dataset to reduce
[in]	right	a dataset to reduce

Returns: the reduced dataset

Implemented in teca_cf_time_axis_data_reduce, teca_programmable_reduce, and teca_table_reduce.

◆ set_array() [1/2]

void teca_index_reduce::set_array ( const std::string & v )

inline

set the array algorithm vector property to a single value

◆ set_array() [2/2]

void teca_index_reduce::set_array	(	size_t	i,
		const std::string &	v
	)

inline

set the i-th element of the array algorithm vector property

◆ set_arrays() [1/2]

void teca_index_reduce::set_arrays ( const std::initializer_list< std::string > && l )

inline

set the array algorithm vector property

◆ set_arrays() [2/2]

void teca_index_reduce::set_arrays ( const std::vector< std::string > & v )

inline

set the array algorithm vector property

◆ set_bounds()

void teca_index_reduce::set_bounds ( const bounds_type & v )

inline

Set the value of the bounds algorithm property

◆ set_end_index()

void teca_index_reduce::set_end_index ( const long & v )

inline

Set the value of the end_index algorithm property

◆ set_extent()

void teca_index_reduce::set_extent ( const extent_type & v )

inline

Set the value of the extent algorithm property

◆ set_start_index()

void teca_index_reduce::set_start_index ( const long & v )

inline

Set the value of the start_index algorithm property

The documentation for this class was generated from the following file:

develop/core/teca_index_reduce.h

Public Types

Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

metadata keys:

Requires:

Exports:

Member Function Documentation

◆ append_array()

◆ clear_arrays()

◆ execute()

◆ finalize()

◆ get_array()

◆ get_arrays()

◆ get_bounds()

◆ get_end_index()

◆ get_extent()

◆ get_number_of_arrays()

◆ get_output_metadata()

◆ get_start_index()

◆ get_upstream_request()

◆ initialize_output_metadata()

◆ initialize_upstream_request()

◆ reduce()

◆ set_array() [1/2]

◆ set_array() [2/2]

◆ set_arrays() [1/2]

◆ set_arrays() [2/2]

◆ set_bounds()

◆ set_end_index()

◆ set_extent()

◆ set_start_index()