X-Git-Url: http://bilbo.iut-bm.univ-fcomte.fr/pub/gitweb/simgrid.git/blobdiff_plain/0e4e8c06b0e29faa6be8391dfa8a1fe6fe5dd300..8bf7ffc43ad5507982e924a7f05bbb13c89965cb:/include/smpi/smpi.h diff --git a/include/smpi/smpi.h b/include/smpi/smpi.h index da78005458..08120350e4 100644 --- a/include/smpi/smpi.h +++ b/include/smpi/smpi.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2021. The SimGrid Team. All rights reserved. */ +/* Copyright (c) 2007-2023. The SimGrid Team. All rights reserved. */ /* This program is free software; you can redistribute it and/or modify it * under the terms of the license (GNU LGPL) which comes with this package. */ @@ -17,18 +17,14 @@ #include #ifdef __cplusplus +#include #include #endif -#ifdef _WIN32 -#define MPI_CALL(type, name, args) \ - type name args; \ - type _XBT_CONCAT(P, name) args -#else #define MPI_CALL(type, name, args) \ type name args __attribute__((weak)); \ - type _XBT_CONCAT(P, name) args -#endif + type _XBT_CONCAT(P, name) \ + args SG_BEGIN_DECL #define MPI_THREAD_SINGLE 0 @@ -39,7 +35,7 @@ SG_BEGIN_DECL #define MPI_MAX_PROCESSOR_NAME 100 #define MPI_MAX_NAME_STRING 100 #define MPI_MAX_ERROR_STRING 100 -#define MPI_MAX_DATAREP_STRIN 100 +#define MPI_MAX_DATAREP_STRING 128 #define MPI_MAX_INFO_KEY 100 #define MPI_MAX_INFO_VAL 100 #define MPI_MAX_OBJECT_NAME 100 @@ -153,6 +149,7 @@ enum ERROR_ENUM { #define MPI_TAG_LB -6 #define MPI_UNIVERSE_SIZE -7 #define MPI_LASTUSEDCODE -8 +#define MPI_KEYVAL_INVALID -9 #define MPI_MODE_NOSTORE 0x1 #define MPI_MODE_NOPUT 0x2 @@ -160,15 +157,10 @@ enum ERROR_ENUM { #define MPI_MODE_NOSUCCEED 0x8 #define MPI_MODE_NOCHECK 0x10 -#define MPI_KEYVAL_INVALID 0 #define MPI_NULL_COPY_FN NULL #define MPI_NULL_DELETE_FN NULL #define MPI_ERR_LASTCODE 74 -#define MPI_CXX_BOOL MPI_DATATYPE_NULL -#define MPI_CXX_FLOAT_COMPLEX MPI_DATATYPE_NULL -#define MPI_CXX_DOUBLE_COMPLEX MPI_DATATYPE_NULL -#define MPI_CXX_LONG_DOUBLE_COMPLEX MPI_DATATYPE_NULL #define MPI_REAL2 MPI_DATATYPE_NULL #define MPI_COMPLEX4 MPI_DATATYPE_NULL @@ -210,7 +202,6 @@ enum ERROR_ENUM { #define MPI_SEEK_SET 600 #define MPI_SEEK_CUR 602 #define MPI_SEEK_END 604 -#define MPI_MAX_DATAREP_STRING 128 #define MPI_WIN_BASE -1 #define MPI_WIN_SIZE -2 @@ -243,15 +234,11 @@ typedef SMPI_Info* MPI_Info; #define MPI_STATUSES_IGNORE ((MPI_Status*)NULL) #define MPI_STATUS_SIZE 5 -#if !defined(DLL_EXPORT) #if defined(c_plusplus) || defined(__cplusplus) #define SMPI_PREDEFINED_POINTER(type, internal) (static_cast (static_cast (&(internal)))) #else #define SMPI_PREDEFINED_POINTER(type, internal) ((type) ((void *) &(internal))) #endif -#else -#define SMPI_PREDEFINED_POINTER(type, internal) ((type) &(internal)) -#endif extern SMPI_Datatype smpi_MPI_DATATYPE_NULL; extern SMPI_Datatype smpi_MPI_CHAR; @@ -308,6 +295,10 @@ extern SMPI_Datatype smpi_MPI_INTEGER4; extern SMPI_Datatype smpi_MPI_INTEGER8; extern SMPI_Datatype smpi_MPI_INTEGER16; extern SMPI_Datatype smpi_MPI_COUNT; +extern SMPI_Datatype smpi_MPI_CXX_BOOL; +extern SMPI_Datatype smpi_MPI_MPI_CXX_FLOAT_COMPLEX; +extern SMPI_Datatype smpi_MPI_MPI_CXX_DOULE_COMPLEX; +extern SMPI_Datatype smpi_MPI_MPI_CXX_LONG_DOUBLE_COMPLEX; #define MPI_DATATYPE_NULL SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_DATATYPE_NULL) #define MPI_CHAR SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_CHAR) @@ -369,6 +360,18 @@ extern SMPI_Datatype smpi_MPI_COUNT; #define MPI_INTEGER16 SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_INTEGER16) #define MPI_COUNT SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_COUNT) +#if defined(c_plusplus) || defined(__cplusplus) +#define MPI_CXX_BOOL SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_CXX_BOOL) +#define MPI_CXX_FLOAT_COMPLEX SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_CXX_FLOAT_COMPLEX) +#define MPI_CXX_DOUBLE_COMPLEX SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_CXX_DOUBLE_COMPLEX) +#define MPI_CXX_LONG_DOUBLE_COMPLEX SMPI_PREDEFINED_POINTER(MPI_Datatype, smpi_MPI_CXX_LONG_DOUBLE_COMPLEX) +#else +#define MPI_CXX_BOOL MPI_DATATYPE_NULL +#define MPI_CXX_FLOAT_COMPLEX MPI_DATATYPE_NULL +#define MPI_CXX_DOUBLE_COMPLEX MPI_DATATYPE_NULL +#define MPI_CXX_LONG_DOUBLE_COMPLEX MPI_DATATYPE_NULL +#endif + //defines for fortran compatibility #if defined(__alpha__) || defined(__sparc64__) || defined(__x86_64__) || defined(__ia64__) || defined(__aarch64__) #define MPI_INTEGER MPI_INT @@ -402,27 +405,45 @@ typedef void MPI_User_function(void *invec, void *inoutvec, int *len, MPI_Dataty typedef SMPI_Op *MPI_Op; #define MPI_OP_NULL ((MPI_Op)NULL) -XBT_PUBLIC_DATA MPI_Op MPI_MAX; -XBT_PUBLIC_DATA MPI_Op MPI_MIN; -XBT_PUBLIC_DATA MPI_Op MPI_MAXLOC; -XBT_PUBLIC_DATA MPI_Op MPI_MINLOC; -XBT_PUBLIC_DATA MPI_Op MPI_SUM; -XBT_PUBLIC_DATA MPI_Op MPI_PROD; -XBT_PUBLIC_DATA MPI_Op MPI_LAND; -XBT_PUBLIC_DATA MPI_Op MPI_LOR; -XBT_PUBLIC_DATA MPI_Op MPI_LXOR; -XBT_PUBLIC_DATA MPI_Op MPI_BAND; -XBT_PUBLIC_DATA MPI_Op MPI_BOR; -XBT_PUBLIC_DATA MPI_Op MPI_BXOR; +extern SMPI_Op smpi_MPI_MAX; +extern SMPI_Op smpi_MPI_MIN; +extern SMPI_Op smpi_MPI_MAXLOC; +extern SMPI_Op smpi_MPI_MINLOC; +extern SMPI_Op smpi_MPI_SUM; +extern SMPI_Op smpi_MPI_PROD; +extern SMPI_Op smpi_MPI_LAND; +extern SMPI_Op smpi_MPI_LOR; +extern SMPI_Op smpi_MPI_LXOR; +extern SMPI_Op smpi_MPI_BAND; +extern SMPI_Op smpi_MPI_BOR; +extern SMPI_Op smpi_MPI_BXOR; +extern SMPI_Op smpi_MPI_REPLACE; +extern SMPI_Op smpi_MPI_NO_OP; + +#define MPI_MAX SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_MAX) +#define MPI_MIN SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_MIN) +#define MPI_MAXLOC SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_MAXLOC) +#define MPI_MINLOC SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_MINLOC) +#define MPI_SUM SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_SUM) +#define MPI_PROD SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_PROD) +#define MPI_LAND SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_LAND) +#define MPI_LOR SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_LOR) +#define MPI_LXOR SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_LXOR) +#define MPI_BAND SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_BAND) +#define MPI_BOR SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_BOR) +#define MPI_BXOR SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_BXOR) + //For accumulate -XBT_PUBLIC_DATA MPI_Op MPI_REPLACE; -XBT_PUBLIC_DATA MPI_Op MPI_NO_OP; +#define MPI_REPLACE SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_REPLACE) +#define MPI_NO_OP SMPI_PREDEFINED_POINTER(MPI_Op, smpi_MPI_NO_OP) + typedef SMPI_Group* MPI_Group; -#define MPI_GROUP_NULL ((MPI_Group)NULL) +extern SMPI_Group smpi_MPI_GROUP_EMPTY; +#define MPI_GROUP_EMPTY SMPI_PREDEFINED_POINTER(MPI_Group, smpi_MPI_GROUP_EMPTY) -XBT_PUBLIC_DATA MPI_Group MPI_GROUP_EMPTY; +#define MPI_GROUP_NULL ((MPI_Group)NULL) typedef SMPI_Comm* MPI_Comm; @@ -438,8 +459,10 @@ typedef SMPI_Request* MPI_Request; typedef SMPI_Errhandler* MPI_Errhandler; #define MPI_ERRHANDLER_NULL ((MPI_Errhandler)NULL) -XBT_PUBLIC_DATA MPI_Errhandler MPI_ERRORS_RETURN; -XBT_PUBLIC_DATA MPI_Errhandler MPI_ERRORS_ARE_FATAL; +extern SMPI_Errhandler smpi_MPI_ERRORS_RETURN; +#define MPI_ERRORS_RETURN SMPI_PREDEFINED_POINTER(MPI_Errhandler, smpi_MPI_ERRORS_RETURN) +extern SMPI_Errhandler smpi_MPI_ERRORS_ARE_FATAL; +#define MPI_ERRORS_ARE_FATAL SMPI_PREDEFINED_POINTER(MPI_Errhandler, smpi_MPI_ERRORS_ARE_FATAL) typedef enum SMPI_Combiner_enum{ MPI_COMBINER_NAMED, @@ -682,8 +705,13 @@ MPI_CALL(XBT_PUBLIC int, MPI_Irsend, MPI_CALL(XBT_PUBLIC int, MPI_Sendrecv, (const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dst, int sendtag, void* recvbuf, int recvcount, MPI_Datatype recvtype, int src, int recvtag, MPI_Comm comm, MPI_Status* status)); +MPI_CALL(XBT_PUBLIC int, MPI_Isendrecv, + (const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dst, int sendtag, void* recvbuf, int recvcount, + MPI_Datatype recvtype, int src, int recvtag, MPI_Comm comm, MPI_Request* req)); MPI_CALL(XBT_PUBLIC int, MPI_Sendrecv_replace, (void* buf, int count, MPI_Datatype datatype, int dst, int sendtag, int src, int recvtag, MPI_Comm comm, MPI_Status* status)); +MPI_CALL(XBT_PUBLIC int, MPI_Isendrecv_replace, (void* buf, int count, MPI_Datatype datatype, int dst, int sendtag, + int src, int recvtag, MPI_Comm comm, MPI_Request* req)); MPI_CALL(XBT_PUBLIC int, MPI_Test, (MPI_Request * request, int* flag, MPI_Status* status)); MPI_CALL(XBT_PUBLIC int, MPI_Testany, (int count, MPI_Request requests[], int* index, int* flag, MPI_Status* status)); @@ -1017,8 +1045,9 @@ MPI_CALL(XBT_PUBLIC int, MPI_Dist_graph_create, (MPI_Comm comm_old, int n, const const int* weights, MPI_Info info, int reorder, MPI_Comm* comm_dist_graph)); MPI_CALL(XBT_PUBLIC int, MPI_Dist_graph_create_adjacent, (MPI_Comm comm_old, int indegree, const int* sources, const int* sourceweights, int outdegree, const int* destinations, const int* destweights, MPI_Info info, int reorder, MPI_Comm* comm_dist_graph)); -MPI_CALL(XBT_PUBLIC int, MPI_Dist_graph_neighbors, (MPI_Comm comm, int maxindegree, int* sources, int* sourceweights, - int maxoutdegree, int* destinations, int* destweights)); +MPI_CALL(XBT_PUBLIC int, MPI_Dist_graph_neighbors, + (MPI_Comm comm, int maxindegree, int* sources, int* sourceweights, int maxoutdegree, int* destinations, + int* destweights)); MPI_CALL(XBT_PUBLIC int, MPI_Dist_graph_neighbors_count, (MPI_Comm comm, int *indegree, int *outdegree, int *weighted)); MPI_CALL(XBT_PUBLIC int, MPI_Win_test, (MPI_Win win, int* flag)); @@ -1099,13 +1128,21 @@ MPI_CALL(XBT_PUBLIC int, MPI_Ineighbor_alltoallw, const MPI_Aint* recvdisps, const MPI_Datatype* recvtypes, MPI_Comm comm, MPI_Request *request)); MPI_CALL(XBT_PUBLIC int, MPI_Status_f2c, (MPI_Fint *f_status, MPI_Status *c_status)); MPI_CALL(XBT_PUBLIC int, MPI_Status_c2f, (MPI_Status *c_status, MPI_Fint *f_status)); - +MPI_CALL(XBT_PUBLIC int, MPI_Parrived, (MPI_Request request, int partition, int *flag)); +MPI_CALL(XBT_PUBLIC int, MPI_Pready, (int partitions, MPI_Request request)); +MPI_CALL(XBT_PUBLIC int, MPI_Pready_range, (int partition_low, int partition_high, MPI_Request request)); +MPI_CALL(XBT_PUBLIC int, MPI_Pready_list, (int length, int partition_list[], MPI_Request request)); +MPI_CALL(XBT_PUBLIC int, MPI_Precv_init, (void* buf, int partitions, MPI_Count count, + MPI_Datatype datatype, int source, int tag, MPI_Comm comm, + MPI_Info info, MPI_Request *request)); +MPI_CALL(XBT_PUBLIC int, MPI_Psend_init, (const void* buf, int partitions, MPI_Count count, + MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, + MPI_Info info, MPI_Request *request)); //FIXME: End of all the not yet implemented stuff // smpi functions -XBT_PUBLIC int smpi_global_size(); XBT_PUBLIC MPI_Comm smpi_process_comm_self(); XBT_PUBLIC MPI_Info smpi_process_info_env(); XBT_PUBLIC void* smpi_process_get_user_data(); @@ -1117,15 +1154,18 @@ XBT_PUBLIC void smpi_execute_flops_benched(double flops); XBT_PUBLIC void smpi_execute(double duration); XBT_PUBLIC void smpi_execute_benched(double duration); +XBT_PUBLIC void smpi_bench_begin(); +XBT_PUBLIC void smpi_bench_end(); + XBT_PUBLIC unsigned long long smpi_rastro_resolution(); XBT_PUBLIC unsigned long long smpi_rastro_timestamp(); -XBT_PUBLIC void smpi_sample_1(int global, const char* file, int line, int iters, double threshold); -XBT_PUBLIC int smpi_sample_2(int global, const char* file, int line, int iter_count); -XBT_PUBLIC void smpi_sample_3(int global, const char* file, int line); -XBT_PUBLIC int smpi_sample_exit(int global, const char* file, int line, int iter_count); +XBT_PUBLIC int smpi_sample_cond(int global, const char* file, const char* tag, int iters, double threshold, + int iter_count); +XBT_PUBLIC void smpi_sample_iter(int global, const char* file, const char* tag); +XBT_PUBLIC int smpi_sample_exit(int global, const char* file, const char* tag, int iter_count); /** * Need a public setter for SMPI copy_callback function, so users can define - * their own while still using default SIMIX_copy_callback for S4U copies. + * their own while still using default copy callback for S4U copies. */ XBT_PUBLIC void smpi_comm_set_copy_data_callback(void (*callback)(smx_activity_t, void*, size_t)); @@ -1134,15 +1174,20 @@ XBT_PUBLIC void smpi_comm_set_copy_data_callback(void (*callback)(smx_activity_t * called from the user's application! (With the __FILE__ and __LINE__ values * passed as parameters.) */ -XBT_PUBLIC void smpi_trace_set_call_location(const char* file, int line); +XBT_PUBLIC void smpi_trace_set_call_location(const char* file, int line, const char* call_name); /** Fortran binding **/ -XBT_PUBLIC void smpi_trace_set_call_location_(const char* file, const int* line); +XBT_PUBLIC void smpi_trace_set_call_location_(const char* file, const int* line, const char* call_name); /** Fortran binding + -fsecond-underscore **/ -XBT_PUBLIC void smpi_trace_set_call_location__(const char* file, const int* line); +XBT_PUBLIC void smpi_trace_set_call_location__(const char* file, const int* line, const char* call_name); #define SMPI_ITER_NAME1(line) _XBT_CONCAT(iter_count, line) #define SMPI_ITER_NAME(line) SMPI_ITER_NAME1(line) -#define SMPI_SAMPLE_LOOP(loop_init, loop_end, loop_iter, global, iters, thres) \ +#define SMPI_CTAG_NAME1(line) _XBT_CONCAT(ctag, line) +#define SMPI_CTAG_NAME(line) SMPI_CTAG_NAME1(line) + +#define SMPI_SAMPLE_LOOP(loop_init, loop_end, loop_iter, global, iters, thres, tag) \ + char SMPI_CTAG_NAME(__LINE__)[132]; \ + snprintf(SMPI_CTAG_NAME(__LINE__), 132, "%s%d", tag, __LINE__); \ int SMPI_ITER_NAME(__LINE__) = 0; \ { \ loop_init; \ @@ -1151,14 +1196,20 @@ XBT_PUBLIC void smpi_trace_set_call_location__(const char* file, const int* line (loop_iter); \ } \ } \ - for (loop_init; (loop_end) ? (smpi_sample_1((global), __FILE__, __LINE__, (iters), (thres)), \ - (smpi_sample_2((global), __FILE__, __LINE__, SMPI_ITER_NAME(__LINE__)))) \ - : smpi_sample_exit((global), __FILE__, __LINE__, SMPI_ITER_NAME(__LINE__)); \ - smpi_sample_3((global), __FILE__, __LINE__), (loop_iter)) + for (loop_init; \ + (loop_end) ? smpi_sample_cond((global), __FILE__, SMPI_CTAG_NAME(__LINE__), (iters), (thres), \ + SMPI_ITER_NAME(__LINE__)) \ + : smpi_sample_exit((global), __FILE__, SMPI_CTAG_NAME(__LINE__), SMPI_ITER_NAME(__LINE__)); \ + smpi_sample_iter((global), __FILE__, SMPI_CTAG_NAME(__LINE__)), (loop_iter)) + #define SMPI_SAMPLE_LOCAL(loop_init, loop_end, loop_iter, iters, thres) \ - SMPI_SAMPLE_LOOP(loop_init, (loop_end), (loop_iter), 0, (iters), (thres)) + SMPI_SAMPLE_LOOP(loop_init, (loop_end), (loop_iter), 0, (iters), (thres), "") +#define SMPI_SAMPLE_LOCAL_TAG(loop_init, loop_end, loop_iter, iters, thres, tag) \ + SMPI_SAMPLE_LOOP(loop_init, (loop_end), (loop_iter), 0, (iters), (thres), tag) #define SMPI_SAMPLE_GLOBAL(loop_init, loop_end, loop_iter, iters, thres) \ - SMPI_SAMPLE_LOOP(loop_init, (loop_end), (loop_iter), 1, (iters), (thres)) + SMPI_SAMPLE_LOOP(loop_init, (loop_end), (loop_iter), 1, (iters), (thres), "") +#define SMPI_SAMPLE_GLOBAL_TAG(loop_init, loop_end, loop_iter, iters, thres, tag) \ + SMPI_SAMPLE_LOOP(loop_init, (loop_end), (loop_iter), 1, (iters), (thres), tag) #define SMPI_SAMPLE_DELAY(duration) for(smpi_execute(duration); 0; ) #define SMPI_SAMPLE_FLOPS(flops) for(smpi_execute_flops(flops); 0; ) XBT_PUBLIC void* smpi_shared_malloc(size_t size, const char* file, int line); @@ -1184,9 +1235,9 @@ XBT_PUBLIC int smpi_main(const char* program, int argc, char* argv[]); /* Trace replay specific stuff */ XBT_PUBLIC void smpi_replay_init(const char* instance_id, int rank, double start_delay_flops); // Only initialization -XBT_PUBLIC void smpi_replay_main(int rank, const char* trace_filename); // Launch the replay once init is done +XBT_PUBLIC void smpi_replay_main(int rank, const char* private_trace_filename); // Launch the replay once init is done XBT_PUBLIC void smpi_replay_run(const char* instance_id, int rank, double start_delay_flops, - const char* trace_filename); // Both init and start + const char* private_trace_filename); // Both init and start XBT_PUBLIC void SMPI_app_instance_register(const char* name, xbt_main_func_t code, int num_processes); XBT_PUBLIC void SMPI_init(); @@ -1195,14 +1246,46 @@ XBT_PUBLIC void SMPI_thread_create(); SG_END_DECL -/* C++ declarations for shared_malloc */ #ifdef __cplusplus +XBT_PUBLIC void SMPI_app_instance_start(const char* name, std::function const& code, + std::vector const& hosts); +XBT_PUBLIC void SMPI_app_instance_join(const std::string& instance_id); + +/* This version without parameter is nice to use with SMPI_app_instance_start() */ +XBT_PUBLIC void MPI_Init(); + +/* C++ declarations for shared_malloc and default copy buffer callback */ XBT_PUBLIC int smpi_is_shared(const void* ptr, std::vector>& private_blocks, size_t* offset); std::vector> shift_and_frame_private_blocks(const std::vector>& vec, size_t offset, size_t buff_size); std::vector> merge_private_blocks(const std::vector>& src, const std::vector>& dst); + +/* May be used by S4U simulations to manually initialize SMPI */ +XBT_PUBLIC void smpi_comm_copy_buffer_callback(simgrid::kernel::activity::CommImpl* comm, void* buff, + size_t buff_size); + +/** + * @brief Callback to set cost for SMPI operations (send, recv, isend) + * + * This callback replaces the configuration parameters smpi/or, smpi/os, smpi/ois. + * It offers more flexibility for cost functions. + * + * @param size Size of message being received/sent + * @param source Source host + * @param dst Destination host + */ +using SmpiOpCostCb = std::function; +/** @brief SMPI functions that accept cost functions */ +enum class SmpiOperation { RECV = 2, SEND = 1, ISEND = 0 }; +/** + * @brief Register a cost callback for some SMPI function (MPI_Send, MPI_ISend or MPI_Recv) + * + * @param op SMPI function + * @param cb User's callback + */ +XBT_PUBLIC void smpi_register_op_cost_callback(SmpiOperation op, const SmpiOpCostCb& cb); #endif -#endif +#endif