-:    0:Source:/home/MPI/testing/mpich2/mpich2/src/include/mpiimpl.h
        -:    0:Graph:bsendutil.gcno
        -:    0:Data:bsendutil.gcda
        -:    0:Runs:856
        -:    0:Programs:280
        -:    1:/* -*- Mode: C; c-basic-offset:4 ; -*- */
        -:    2:/*  
        -:    3: *  (C) 2001 by Argonne National Laboratory.
        -:    4: *      See COPYRIGHT in top-level directory.
        -:    5: */
        -:    6:#ifndef MPIIMPL_H_INCLUDED
        -:    7:#define MPIIMPL_H_INCLUDED
        -:    8:
        -:    9:/*
        -:   10: * This file is the temporary home of most of the definitions used to 
        -:   11: * implement MPICH.  We will eventually divide this file into logical
        -:   12: * pieces once we are certain of the relationships between the components.
        -:   13: */
        -:   14:
        -:   15:/* style: define:vsnprintf:1 sig:0 */
        -:   16:/* style: allow:printf:3 sig:0 */
        -:   17:
        -:   18:/* Include the mpi definitions */
        -:   19:#include "mpi.h"
        -:   20:
        -:   21:/* Include nested mpi (NMPI) definitions */
        -:   22:#include "nmpi.h"
        -:   23:
        -:   24:/* There are a few definitions that must be made *before* the mpichconf.h
        -:   25:   file is included.  These include the definitions of the error levels and some
        -:   26:   thread granularity constants */
        -:   27:#include "mpichconfconst.h"
        -:   28:
        -:   29:/* Data computed by configure.  This is included *after* mpi.h because we
        -:   30:   do not want mpi.h to depend on any other files or configure flags */
        -:   31:#include "mpichconf.h"
        -:   32:
        -:   33:/* if we are defining this, we must define it before including mpl.h */
        -:   34:#if defined(MPICH_DEBUG_MEMINIT)
        -:   35:#define MPL_VG_ENABLED 1
        -:   36:#endif
        -:   37:#include "mpl.h"
        -:   38:
        -:   39:#include <stdio.h>
        -:   40:#ifdef STDC_HEADERS
        -:   41:#include <stdlib.h>
        -:   42:#include <stdarg.h>
        -:   43:#include <string.h>
        -:   44:#else
        -:   45:#ifdef HAVE_STDLIB_H
        -:   46:#include <stdlib.h>
        -:   47:#endif
        -:   48:#ifdef HAVE_STDARG_H
        -:   49:#include <stdarg.h>
        -:   50:#endif
        -:   51:#ifdef HAVE_STRING_H
        -:   52:#include <string.h>
        -:   53:#endif
        -:   54:#endif
        -:   55:
        -:   56:#ifdef HAVE_SYS_TYPES_H
        -:   57:#include <sys/types.h>
        -:   58:#endif
        -:   59:
        -:   60:/* for MAXHOSTNAMELEN under Linux and OSX */
        -:   61:#ifdef HAVE_SYS_PARAM_H
        -:   62:#include <sys/param.h>
        -:   63:#endif
        -:   64:
        -:   65:#if (!defined MAXHOSTNAMELEN) && (!defined MAX_HOSTNAME_LEN)
        -:   66:#define MAX_HOSTNAME_LEN 256
        -:   67:#elif !defined MAX_HOSTNAME_LEN
        -:   68:#define MAX_HOSTNAME_LEN MAXHOSTNAMELEN
        -:   69:#endif
        -:   70:
        -:   71:/* Default PMI version to use */
        -:   72:#define MPIU_DEFAULT_PMI_VERSION 1
        -:   73:#define MPIU_DEFAULT_PMI_SUBVERSION 1
        -:   74:
        -:   75:/* This allows us to keep names local to a single file when we can use
        -:   76:   weak symbols */
        -:   77:#ifdef  USE_WEAK_SYMBOLS
        -:   78:#define PMPI_LOCAL static
        -:   79:#else
        -:   80:#define PMPI_LOCAL 
        -:   81:#endif
        -:   82:
        -:   83:/* Fix for universal endianess added in autoconf 2.62 */
        -:   84:#ifdef WORDS_UNIVERSAL_ENDIAN
        -:   85:#if defined(__BIG_ENDIAN__)
        -:   86:#elif defined(__LITTLE_ENDIAN__)
        -:   87:#define WORDS_LITTLEENDIAN
        -:   88:#else
        -:   89:#error 'Universal endianess defined without __BIG_ENDIAN__ or __LITTLE_ENDIAN__'
        -:   90:#endif
        -:   91:#endif
        -:   92:
        -:   93:/* Include some basic (and easily shared) definitions */
        -:   94:#include "mpibase.h"
        -:   95:
        -:   96:/* FIXME: The code base should not define two of these */
        -:   97:/* This is used to quote a name in a definition (see FUNCNAME/FCNAME below) */
        -:   98:#ifndef MPIDI_QUOTE
        -:   99:#define MPIDI_QUOTE(A) MPIDI_QUOTE2(A)
        -:  100:#define MPIDI_QUOTE2(A) #A
        -:  101:#endif
        -:  102:
        -:  103:/* 
        -:  104:   Include the implementation definitions (e.g., error reporting, thread
        -:  105:   portability)
        -:  106:   More detailed documentation is contained in the MPICH2 and ADI3 manuals.
        -:  107: */
        -:  108:/* FIXME: ... to do ... */
        -:  109:#include "mpitypedefs.h"
        -:  110:
        -:  111:/* This is the default implementation of MPIU_Memcpy.  We define this
        -:  112:   before including mpidpre.h so that it can be used when a device or
        -:  113:   channel can use it if it's overriding MPIU_Memcpy.  */
        -:  114:MPIU_DBG_ATTRIBUTE_NOINLINE
        -:  115:ATTRIBUTE((unused))
        -:  116:static MPIU_DBG_INLINE_KEYWORD void MPIUI_Memcpy(void * dst, const void * src, size_t len)
    #####:  117:{
    #####:  118:    memcpy(dst, src, len);
    #####:  119:}
        -:  120:
        -:  121:/* Include definitions from the device which must exist before items in this
        -:  122:   file (mpiimpl.h) can be defined. mpidpre.h must be included before any
        -:  123:   files that allow the device to override or extend any terms; this includes
        -:  124:   mpiimplthread.h and mpiutil.h */
        -:  125:/* ------------------------------------------------------------------------- */
        -:  126:#include "mpidpre.h"
        -:  127:/* ------------------------------------------------------------------------- */
        -:  128:
        -:  129:/* Overriding memcpy:
        -:  130:   Devices and channels can override the default implementation of
        -:  131:   MPIU_Memcpy by defining the MPIU_Memcpy macro.  The implementation
        -:  132:   can call MPIUI_Memcpy for the default memcpy implementation.   
        -:  133:   Note that MPIU_Memcpy and MPIUI_Memcpy return void rather than a
        -:  134:   pointer to the destination buffer.  This is different from C89
        -:  135:   memcpy.
        -:  136:*/
        -:  137:#ifndef MPIU_Memcpy
        -:  138:#define MPIU_Memcpy(dst, src, len)                \
        -:  139:    do {                                          \
        -:  140:        MPIU_MEM_CHECK_MEMCPY((dst),(src),(len)); \
        -:  141:        MPIUI_Memcpy((dst), (src), (len));        \
        -:  142:    } while (0)
        -:  143:#endif
        -:  144:
        -:  145:#include "mpiimplthread.h"
        -:  146:/* #include "mpiu_monitors.h" */
        -:  147:
        -:  148:#include "mpiutil.h"
        -:  149:
        -:  150:/* ------------------------------------------------------------------------- */
        -:  151:/* mpidebug.h */
        -:  152:/* ------------------------------------------------------------------------- */
        -:  153:/* Debugging and printf control */
        -:  154:/* Use these *only* for debugging output intended for the implementors
        -:  155:   and maintainers of MPICH.  Do *not* use these for any output that
        -:  156:   general users may normally see.  Use either the error code creation
        -:  157:   routines for error messages or MPIU_msg_printf etc. for general messages 
        -:  158:   (MPIU_msg_printf will go through gettext).  
        -:  159:
        -:  160:   FIXME: Document all of these macros
        -:  161:
        -:  162:   NOTE: These macros and values are deprecated.  See 
        -:  163:   www.mcs.anl.gov/mpi/mpich2/developer/design/debugmsg.htm for 
        -:  164:   the new design (only partially implemented at this time).
        -:  165:   
        -:  166:   The implementation is in mpidbg.h
        -:  167:*/
        -:  168:#include "mpidbg.h"
        -:  169:
        -:  170:#if defined(MPICH_DBG_OUTPUT)
        -:  171:#define MPIU_DBG_PRINTF(e)			\
        -:  172:{						\
        -:  173:    if (MPIU_dbg_state != MPIU_DBG_STATE_NONE)	\
        -:  174:    {						\
        -:  175:	MPIU_dbg_printf e;			\
        -:  176:    }						\
        -:  177:}
        -:  178:/* The first argument is a place holder to allow the selection of a subset
        -:  179:   of debugging events.  The second is a placeholder to allow a numeric
        -:  180:   level of debugging within that class.  The third is the debugging text */
        -:  181:#define MPIU_DBG_PRINTF_CLASS(_c,_l,_e) MPIU_DBG_PRINTF(_e)
        -:  182:#else
        -:  183:#define MPIU_DBG_PRINTF(e)
        -:  184:#define MPIU_DBG_PRINTF_CLASS(_c,_l,_e)
        -:  185:#endif
        -:  186:/* FIXME: These should use the MPIU_DBG_STMT macros and not be defined
        -:  187:   as macros themselves (to make it clear that they are macros, and not
        -:  188:   always called) */
        -:  189:#ifdef USE_MPIU_DBG_PRINT_VC
        -:  190:void MPIU_DBG_PrintVC(MPIDI_VC_t *vc);
        -:  191:void MPIU_DBG_PrintVCState2(MPIDI_VC_t *vc, MPIDI_VC_State_t new_state);
        -:  192:void MPIU_DBG_PrintVCState(MPIDI_VC_t *vc);
        -:  193:#else
        -:  194:#define MPIU_DBG_PrintVC(vc)
        -:  195:#define MPIU_DBG_PrintVCState2(vc, new_state)
        -:  196:#define MPIU_DBG_PrintVCState(vc)
        -:  197:#endif
        -:  198:
        -:  199:
        -:  200:/* The follow is temporarily provided for backward compatibility.  Any code
        -:  201:   using dbg_printf should be updated to use MPIU_DBG_PRINTF. */
        -:  202:#define dbg_printf MPIU_dbg_printf
        -:  203:
        -:  204:/* MPIR_IDebug withdrawn because the MPIU_DBG_MSG interface provides 
        -:  205:   a more flexible, integrated, and documented mechanism */
        -:  206:
        -:  207:/* ------------------------------------------------------------------------- */
        -:  208:/* end of mpidebug.h */
        -:  209:/* ------------------------------------------------------------------------- */
        -:  210:
        -:  211:/* Routines for memory management */
        -:  212:#include "mpimem.h"
        -:  213:
        -:  214:/*
        -:  215: * Use MPIU_SYSCALL to wrap system calls; this provides a convenient point
        -:  216: * for timing the calls and keeping track of the use of system calls.
        -:  217: * This macro simply invokes the system call and does not even handle
        -:  218: * EINTR.
        -:  219: * To use, 
        -:  220: *    MPIU_SYSCALL( return-value, name-of-call, args-in-parenthesis )
        -:  221: * e.g., change "n = read(fd,buf,maxn);" into
        -:  222: *    MPIU_SYSCALL( n,read,(fd,buf,maxn) );
        -:  223: * An example that prints each syscall to stdout is shown below. 
        -:  224: */
        -:  225:#ifdef USE_LOG_SYSCALLS
        -:  226:#define MPIU_SYSCALL(a_,b_,c_) { \
        -:  227:    printf( "[%d]about to call %s\n", MPIR_Process.comm_world->rank,#b_);\
        -:  228:          fflush(stdout); errno = 0;\
        -:  229:    a_ = b_ c_; \
        -:  230:    if ((a_)>=0 || errno==0) {\
        -:  231:    printf( "[%d]%s returned %d\n", \
        -:  232:          MPIR_Process.comm_world->rank, #b_, a_ );\
        -:  233:    } \
        -:  234: else { \
        -:  235:    printf( "[%d]%s returned %d (errno = %d,%s)\n", \
        -:  236:          MPIR_Process.comm_world->rank, \
        -:  237:          #b_, a_, errno, strerror(errno));\
        -:  238:    };           fflush(stdout);}
        -:  239:#else
        -:  240:#define MPIU_SYSCALL(a_,b_,c_) a_ = b_ c_
        -:  241:#endif
        -:  242:
        -:  243:/*TDSOverview.tex
        -:  244:  
        -:  245:  MPI has a number of data structures, most of which are represented by 
        -:  246:  an opaque handle in an MPI program.  In the MPICH implementation of MPI, 
        -:  247:  these handles are represented
        -:  248:  as integers; this makes implementation of the C/Fortran handle transfer 
        -:  249:  calls (part of MPI-2) easy.  
        -:  250: 
        -:  251:  MPID objects (again with the possible exception of 'MPI_Request's) 
        -:  252:  are allocated by a common set of object allocation functions.
        -:  253:  These are 
        -:  254:.vb
        -:  255:    void *MPIU_Handle_obj_create( MPIU_Object_alloc_t *objmem )
        -:  256:    void MPIU_Handle_obj_destroy( MPIU_Object_alloc_t *objmem, void *object )
        -:  257:.ve
        -:  258:  where 'objmem' is a pointer to a memory allocation object that knows 
        -:  259:  enough to allocate objects, including the
        -:  260:  size of the object and the location of preallocated memory, as well 
        -:  261:  as the type of memory allocator.  By providing the routines to allocate and
        -:  262:  free the memory, we make it easy to use the same interface to allocate both
        -:  263:  local and shared memory for objects (always using the same kind for each 
        -:  264:  type of object).
        -:  265:
        -:  266:  The names create/destroy were chosen because they are different from 
        -:  267:  new/delete (C++ operations) and malloc/free.  
        -:  268:  Any name choice will have some conflicts with other uses, of course.
        -:  269:
        -:  270:  Reference Counts:
        -:  271:  Many MPI objects have reference count semantics.  
        -:  272:  The semantics of MPI require that many objects that have been freed by the 
        -:  273:  user 
        -:  274:  (e.g., with 'MPI_Type_free' or 'MPI_Comm_free') remain valid until all 
        -:  275:  pending
        -:  276:  references to that object (e.g., by an 'MPI_Irecv') are complete.  There
        -:  277:  are several ways to implement this; MPICH uses `reference counts` in the
        -:  278:  objects.  To support the 'MPI_THREAD_MULTIPLE' level of thread-safety, these
        -:  279:  reference counts must be accessed and updated atomically.  
        -:  280:  A reference count for
        -:  281:  `any` object can be incremented (atomically) 
        -:  282:  with 'MPIU_Object_add_ref(objptr)'
        -:  283:  and decremented with 'MPIU_Object_release_ref(objptr,newval_ptr)'.  
        -:  284:  These have been designed so that then can be implemented as inlined 
        -:  285:  macros rather than function calls, even in the multithreaded case, and
        -:  286:  can use special processor instructions that guarantee atomicity to 
        -:  287:  avoid thread locks.
        -:  288:  The decrement routine sets the value pointed at by 'inuse_ptr' to 0 if 
        -:  289:  the postdecrement value of the reference counter is zero, and to a non-zero
        -:  290:  value otherwise.  If this value is zero, then the routine that decremented 
        -:  291:  the
        -:  292:  reference count should free the object.  This may be as simple as 
        -:  293:  calling 'MPIU_Handle_obj_destroy' (for simple objects with no other allocated
        -:  294:  storage) or may require calling a separate routine to destroy the object.
        -:  295:  Because MPI uses 'MPI_xxx_free' to both decrement the reference count and 
        -:  296:  free the object if the reference count is zero, we avoid the use of 'free'
        -:  297:  in the MPID routines.
        -:  298:
        -:  299:  The 'inuse_ptr' approach is used rather than requiring the post-decrement
        -:  300:  value because, for reference-count semantics, all that is necessary is
        -:  301:  to know when the reference count reaches zero, and this can sometimes
        -:  302:  be implemented more cheaply that requiring the post-decrement value (e.g.,
        -:  303:  on IA32, there is an instruction for this operation).
        -:  304:
        -:  305:  Question:
        -:  306:  Should we state that this is a macro so that we can use a register for
        -:  307:  the output value?  That avoids a store.  Alternately, have the macro 
        -:  308:  return the value as if it was a function?
        -:  309:
        -:  310:  Structure Definitions:
        -:  311:  The structure definitions in this document define `only` that part of
        -:  312:  a structure that may be used by code that is making use of the ADI.
        -:  313:  Thus, some structures, such as 'MPID_Comm', have many defined fields;
        -:  314:  these are used to support MPI routines such as 'MPI_Comm_size' and
        -:  315:  'MPI_Comm_remote_group'.  Other structures may have few or no defined
        -:  316:  members; these structures have no fields used outside of the ADI.  
        -:  317:  In C++ terms,  all members of these structures are 'private'.  
        -:  318:
        -:  319:  For the initial implementation, we expect that the structure definitions 
        -:  320:  will be designed for the multimethod device.  However, all items that are
        -:  321:  specific to a particular device (including the multi-method device) 
        -:  322:  will be placed at the end of the structure;
        -:  323:  the document will clearly identify the members that all implementations
        -:  324:  will provide.  This simplifies much of the code in both the ADI and the 
        -:  325:  implementation of the MPI routines because structure member can be directly
        -:  326:  accessed rather than using some macro or C++ style method interface.
        -:  327:  
        -:  328: T*/
        -:  329:
        -:  330:/* mpi_lang.h - Prototypes for language specific routines. Currently used to
        -:  331: * set keyval attribute callbacks
        -:  332: */
        -:  333:#include "mpi_lang.h"
        -:  334:/* Known language bindings */
        -:  335:/*E
        -:  336:  MPID_Lang_t - Known language bindings for MPI
        -:  337:
        -:  338:  A few operations in MPI need to know what language they were called from
        -:  339:  or created by.  This type enumerates the possible languages so that
        -:  340:  the MPI implementation can choose the correct behavior.  An example of this
        -:  341:  are the keyval attribute copy and delete functions.
        -:  342:
        -:  343:  Module:
        -:  344:  Attribute-DS
        -:  345:  E*/
        -:  346:typedef enum MPID_Lang_t { MPID_LANG_C 
        -:  347:#ifdef HAVE_FORTRAN_BINDING
        -:  348:			   , MPID_LANG_FORTRAN
        -:  349:			   , MPID_LANG_FORTRAN90
        -:  350:#endif
        -:  351:#ifdef HAVE_CXX_BINDING
        -:  352:			   , MPID_LANG_CXX
        -:  353:#endif
        -:  354:} MPID_Lang_t;
        -:  355:
        -:  356:/* Macros for the MPI handles (e.g., the object that encodes an
        -:  357:   MPI_Datatype) */
        -:  358:#include "mpihandlemem.h"
        -:  359:
        -:  360:/* This routine is used to install an attribute free routine for datatypes
        -:  361:   at finalize-time */
        -:  362:void MPIR_DatatypeAttrFinalize( void );
        -:  363:
        -:  364:/* ------------------------------------------------------------------------- */
        -:  365:/* Should the following be moved into mpihandlemem.h ?*/
        -:  366:/* ------------------------------------------------------------------------- */
        -:  367:
        -:  368:/* Routines to initialize handle allocations */
        -:  369:/* These are now internal to the handlemem package
        -:  370:void *MPIU_Handle_direct_init( void *, int, int, int );
        -:  371:void *MPIU_Handle_indirect_init( void *(**)[], int *, int, int, int, int );
        -:  372:int MPIU_Handle_free( void *((*)[]), int );
        -:  373:*/
        -:  374:/* Convert Handles to objects for MPI types that have predefined objects */
        -:  375:#define MPID_Getb_ptr(kind,a,bmsk,ptr)                                  \
        -:  376:{                                                                       \
        -:  377:   switch (HANDLE_GET_KIND(a)) {                                        \
        -:  378:      case HANDLE_KIND_BUILTIN:                                         \
        -:  379:          ptr=MPID_##kind##_builtin+((a)&(bmsk));                       \
        -:  380:          break;                                                        \
        -:  381:      case HANDLE_KIND_DIRECT:                                          \
        -:  382:          ptr=MPID_##kind##_direct+HANDLE_INDEX(a);                     \
        -:  383:          break;                                                        \
        -:  384:      case HANDLE_KIND_INDIRECT:                                        \
        -:  385:          ptr=((MPID_##kind*)                                           \
        -:  386:               MPIU_Handle_get_ptr_indirect(a,&MPID_##kind##_mem));     \
        -:  387:          break;                                                        \
        -:  388:      case HANDLE_KIND_INVALID:                                         \
        -:  389:      default:								\
        -:  390:          ptr=0;							\
        -:  391:          break;							\
        -:  392:    }                                                                   \
        -:  393:}
        -:  394:
        -:  395:/* Convert handles to objects for MPI types that do _not_ have any predefined
        -:  396:   objects */
        -:  397:#define MPID_Get_ptr(kind,a,ptr)					\
        -:  398:{									\
        -:  399:   switch (HANDLE_GET_KIND(a)) {					\
        -:  400:      case HANDLE_KIND_DIRECT:						\
        -:  401:          ptr=MPID_##kind##_direct+HANDLE_INDEX(a);			\
        -:  402:          break;							\
        -:  403:      case HANDLE_KIND_INDIRECT:					\
        -:  404:          ptr=((MPID_##kind*)						\
        -:  405:               MPIU_Handle_get_ptr_indirect(a,&MPID_##kind##_mem));	\
        -:  406:          break;							\
        -:  407:      case HANDLE_KIND_INVALID:						\
        -:  408:      case HANDLE_KIND_BUILTIN:						\
        -:  409:      default:								\
        -:  410:          ptr=0;							\
        -:  411:          break;							\
        -:  412:     }									\
        -:  413:}
        -:  414:
        -:  415:/* FIXME: the masks should be defined with the handle definitions instead
        -:  416:   of inserted here as literals */
        -:  417:#define MPID_Comm_get_ptr(a,ptr)       MPID_Getb_ptr(Comm,a,0x03ffffff,ptr)
        -:  418:#define MPID_Group_get_ptr(a,ptr)      MPID_Getb_ptr(Group,a,0x03ffffff,ptr)
        -:  419:#define MPID_File_get_ptr(a,ptr)       MPID_Get_ptr(File,a,ptr)
        -:  420:#define MPID_Errhandler_get_ptr(a,ptr) MPID_Getb_ptr(Errhandler,a,0x3,ptr)
        -:  421:#define MPID_Op_get_ptr(a,ptr)         MPID_Getb_ptr(Op,a,0x000000ff,ptr)
        -:  422:#define MPID_Info_get_ptr(a,ptr)       MPID_Get_ptr(Info,a,ptr)
        -:  423:#define MPID_Win_get_ptr(a,ptr)        MPID_Get_ptr(Win,a,ptr)
        -:  424:#define MPID_Request_get_ptr(a,ptr)    MPID_Get_ptr(Request,a,ptr)
        -:  425:#define MPID_Grequest_class_get_ptr(a,ptr) MPID_Get_ptr(Grequest_class,a,ptr)
        -:  426:/* Keyvals have a special format. This is roughly MPID_Get_ptrb, but
        -:  427:   the handle index is in a smaller bit field.  In addition, 
        -:  428:   there is no storage for the builtin keyvals.  
        -:  429:   For the indirect case, we mask off the part of the keyval that is
        -:  430:   in the bits normally used for the indirect block index.
        -:  431:*/
        -:  432:#define MPID_Keyval_get_ptr(a,ptr)     \
        -:  433:{                                                                       \
        -:  434:   switch (HANDLE_GET_KIND(a)) {                                        \
        -:  435:      case HANDLE_KIND_BUILTIN:                                         \
        -:  436:          ptr=0;                                                        \
        -:  437:          break;                                                        \
        -:  438:      case HANDLE_KIND_DIRECT:                                          \
        -:  439:          ptr=MPID_Keyval_direct+((a)&0x3fffff);                        \
        -:  440:          break;                                                        \
        -:  441:      case HANDLE_KIND_INDIRECT:                                        \
        -:  442:          ptr=((MPID_Keyval*)                                           \
        -:  443:             MPIU_Handle_get_ptr_indirect((a)&0xfc3fffff,&MPID_Keyval_mem)); \
        -:  444:          break;                                                        \
        -:  445:      case HANDLE_KIND_INVALID:                                         \
        -:  446:      default:								\
        -:  447:          ptr=0;							\
        -:  448:          break;							\
        -:  449:    }                                                                   \
        -:  450:}
        -:  451:
        -:  452:/* Valid pointer checks */
        -:  453:/* This test is lame.  Should eventually include cookie test 
        -:  454:   and in-range addresses */
        -:  455:#define MPID_Valid_ptr(kind,ptr,err) \
        -:  456:  {if (!(ptr)) { err = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, \
        -:  457:                                             "**nullptrtype", "**nullptrtype %s", #kind ); } }
        -:  458:#define MPID_Valid_ptr_class(kind,ptr,errclass,err) \
        -:  459:  {if (!(ptr)) { err = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, errclass, \
        -:  460:                                             "**nullptrtype", "**nullptrtype %s", #kind ); } }
        -:  461:
        -:  462:#define MPID_Info_valid_ptr(ptr,err) MPID_Valid_ptr_class(Info,ptr,MPI_ERR_INFO,err)
        -:  463:/* Check not only for a null pointer but for an invalid communicator,
        -:  464:   such as one that has been freed.  Let's try the ref_count as the test
        -:  465:   for now */
        -:  466:#define MPID_Comm_valid_ptr(ptr,err) {                \
        -:  467:     MPID_Valid_ptr_class(Comm,ptr,MPI_ERR_COMM,err); \
        -:  468:     if ((ptr) && MPIU_Object_get_ref(ptr) == 0) {    \
        -:  469:         MPIU_ERR_SET(err,MPI_ERR_COMM,"**comm");     \
        -:  470:         ptr = 0;                                     \
        -:  471:     }                                                \
        -:  472:}
        -:  473:#define MPID_Group_valid_ptr(ptr,err) MPID_Valid_ptr_class(Group,ptr,MPI_ERR_GROUP,err)
        -:  474:#define MPID_Win_valid_ptr(ptr,err) MPID_Valid_ptr_class(Win,ptr,MPI_ERR_WIN,err)
        -:  475:#define MPID_Op_valid_ptr(ptr,err) MPID_Valid_ptr_class(Op,ptr,MPI_ERR_OP,err)
        -:  476:#define MPID_Errhandler_valid_ptr(ptr,err) MPID_Valid_ptr_class(Errhandler,ptr,MPI_ERR_ARG,err)
        -:  477:#define MPID_File_valid_ptr(ptr,err) MPID_Valid_ptr_class(File,ptr,MPI_ERR_FILE,err)
        -:  478:#define MPID_Request_valid_ptr(ptr,err) MPID_Valid_ptr_class(Request,ptr,MPI_ERR_REQUEST,err)
        -:  479:#define MPID_Keyval_valid_ptr(ptr,err) MPID_Valid_ptr_class(Keyval,ptr,MPI_ERR_KEYVAL,err)
        -:  480:
        -:  481:/* FIXME: 
        -:  482:   Generic pointer test.  This is applied to any address, not just one from
        -:  483:   an MPI object.
        -:  484:   Currently unimplemented (returns success except for null pointers.
        -:  485:   With a little work, could check that the pointer is properly aligned,
        -:  486:   using something like 
        -:  487:   ((p) == 0 || ((char *)(p) & MPID_Alignbits[alignment] != 0)
        -:  488:   where MPID_Alignbits is set with a mask whose bits must be zero in a 
        -:  489:   properly aligned quantity.  For systems with no alignment rules, 
        -:  490:   all of these masks are zero, and this part of test can be eliminated.
        -:  491: */
        -:  492:#define MPID_Pointer_is_invalid(p,alignment) ((p) == 0)
        -:  493:/* Fixme: The following MPID_ALIGNED_xxx values are temporary.  They 
        -:  494:   need to be computed by configure and included in the mpichconf.h file.
        -:  495:   Note that they cannot be set conservatively (i.e., as sizeof(object)),
        -:  496:   since the runtime system may generate objects with lesser alignment
        -:  497:   rules if the processor allows them.
        -:  498: */
        -:  499:#define MPID_ALIGNED_PTR_INT   1
        -:  500:#define MPID_ALIGNED_PTR_LONG  1
        -:  501:#define MPID_ALIGNED_PTR_VOIDP 1
        -:  502:/* ------------------------------------------------------------------------- */
        -:  503:/* end of code that should the following be moved into mpihandlemem.h ?*/
        -:  504:/* ------------------------------------------------------------------------- */
        -:  505:
        -:  506:#include "mpiparam.h"
        -:  507:
        -:  508:/* ------------------------------------------------------------------------- */
        -:  509:/* Info */
        -:  510:/*TInfoOverview.tex
        -:  511:
        -:  512:  'MPI_Info' provides a way to create a list of '(key,value)' pairs
        -:  513:  where the 'key' and 'value' are both strings.  Because many routines, both
        -:  514:  in the MPI implementation and in related APIs such as the PMI process
        -:  515:  management interface, require 'MPI_Info' arguments, we define a simple 
        -:  516:  structure for each 'MPI_Info' element.  Elements are allocated by the 
        -:  517:  generic object allocator; the head element is always empty (no 'key'
        -:  518:  or 'value' is defined on the head element).  
        -:  519:  
        -:  520:  For simplicity, we have not abstracted the info data structures;
        -:  521:  routines that want to work with the linked list may do so directly.
        -:  522:  Because the 'MPI_Info' type is a handle and not a pointer, an MPIU
        -:  523:  (utility) routine is provided to handle the 
        -:  524:  deallocation of 'MPID_Info' elements.  See the implementation of
        -:  525:  'MPI_Info_create' for how an Info type is allocated.
        -:  526:
        -:  527:  Thread Safety:
        -:  528:
        -:  529:  The info interface itself is not thread-robust.  In particular, the routines
        -:  530:  'MPI_INFO_GET_NKEYS' and 'MPI_INFO_GET_NTHKEY' assume that no other 
        -:  531:  thread modifies the info key.  (If the info routines had the concept
        -:  532:  of a next value, they would not be thread safe.  As it stands, a user
        -:  533:  must be careful if several threads have access to the same info object.) 
        -:  534:  Further, 'MPI_INFO_DUP', while not 
        -:  535:  explicitly advising implementers to be careful of one thread modifying the
        -:  536:  'MPI_Info' structure while 'MPI_INFO_DUP' is copying it, requires that the
        -:  537:  operation take place in a thread-safe manner.
        -:  538:  There isn'' much that we can do about these cases.  There are other cases
        -:  539:  that must be handled.  In particular, multiple threads are allowed to 
        -:  540:  update the same info value.  Thus, all of the update routines must be thread
        -:  541:  safe; the simple implementation used in the MPICH implementation uses locks.
        -:  542:  Note that the 'MPI_Info_delete' call does not need a lock; the defintion of
        -:  543:  thread-safety means that any order of the calls functions correctly; since
        -:  544:  it invalid either to delete the same 'MPI_Info' twice or to modify an
        -:  545:  'MPI_Info' that has been deleted, only one thread at a time can call 
        -:  546:  'MPI_Info_free' on any particular 'MPI_Info' value.  
        -:  547:
        -:  548:  T*/
        -:  549:/*S
        -:  550:  MPID_Info - Structure of an MPID info
        -:  551:
        -:  552:  Notes:
        -:  553:  There is no reference count because 'MPI_Info' values, unlike other MPI 
        -:  554:  objects, may be changed after they are passed to a routine without 
        -:  555:  changing the routine''s behavior.  In other words, any routine that uses
        -:  556:  an 'MPI_Info' object must make a copy or otherwise act on any info value
        -:  557:  that it needs.
        -:  558:
        -:  559:  A linked list is used because the typical 'MPI_Info' list will be short
        -:  560:  and a simple linked list is easy to implement and to maintain.  Similarly,
        -:  561:  a single structure rather than separate header and element structures are
        -:  562:  defined for simplicity.  No separate thread lock is provided because
        -:  563:  info routines are not performance critical; they may use the single
        -:  564:  critical section lock in the 'MPIR_Process' structure when they need a
        -:  565:  thread lock.
        -:  566:  
        -:  567:  This particular form of linked list (in particular, with this particular
        -:  568:  choice of the first two members) is used because it allows us to use 
        -:  569:  the same routines to manage this list as are used to manage the 
        -:  570:  list of free objects (in the file 'src/util/mem/handlemem.c').  In 
        -:  571:  particular, if lock-free routines for updating a linked list are 
        -:  572:  provided, they can be used for managing the 'MPID_Info' structure as well.
        -:  573:
        -:  574:  The MPI standard requires that keys can be no less that 32 characters and
        -:  575:  no more than 255 characters.  There is no mandated limit on the size 
        -:  576:  of values.
        -:  577:
        -:  578:  Module:
        -:  579:  Info-DS
        -:  580:  S*/
        -:  581:typedef struct MPID_Info {
        -:  582:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  583:    struct MPID_Info   *next;
        -:  584:    char               *key;
        -:  585:    char               *value;
        -:  586:} MPID_Info;
        -:  587:extern MPIU_Object_alloc_t MPID_Info_mem;
        -:  588:/* Preallocated info objects */
        -:  589:extern MPID_Info MPID_Info_direct[];
        -:  590:/* ------------------------------------------------------------------------- */
        -:  591:
        -:  592:/* ------------------------------------------------------------------------- */
        -:  593:/* Error Handlers */
        -:  594:/*E
        -:  595:  MPID_Errhandler_fn - MPID Structure to hold an error handler function
        -:  596:
        -:  597:  Notes:
        -:  598:  The MPI-1 Standard declared only the C version of this, implicitly 
        -:  599:  assuming that 'int' and 'MPI_Fint' were the same. 
        -:  600:
        -:  601:  Since Fortran does not have a C-style variable number of arguments 
        -:  602:  interface, the Fortran interface simply accepts two arguments.  Some
        -:  603:  calling conventions for Fortran (particularly under Windows) require
        -:  604:  this.
        -:  605:
        -:  606:  Module:
        -:  607:  ErrHand-DS
        -:  608:  
        -:  609:  Questions:
        -:  610:  What do we want to do about C++?  Do we want a hook for a routine that can
        -:  611:  be called to throw an exception in C++, particularly if we give C++ access
        -:  612:  to this structure?  Does the C++ handler need to be different (not part
        -:  613:  of the union)?
        -:  614:
        -:  615:  E*/
        -:  616:typedef union MPID_Errhandler_fn {
        -:  617:   void (*C_Comm_Handler_function) ( MPI_Comm *, int *, ... );
        -:  618:   void (*F77_Handler_function) ( MPI_Fint *, MPI_Fint * );
        -:  619:   void (*C_Win_Handler_function) ( MPI_Win *, int *, ... );
        -:  620:   void (*C_File_Handler_function) ( MPI_File *, int *, ... );
        -:  621:} MPID_Errhandler_fn;
        -:  622:
        -:  623:/*S
        -:  624:  MPID_Errhandler - Description of the error handler structure
        -:  625:
        -:  626:  Notes:
        -:  627:  Device-specific information may indicate whether the error handler is active;
        -:  628:  this can help prevent infinite recursion in error handlers caused by 
        -:  629:  user-error without requiring the user to be as careful.  We might want to 
        -:  630:  make this part of the interface so that the 'MPI_xxx_call_errhandler' 
        -:  631:  routines would check.
        -:  632:
        -:  633:  It is useful to have a way to indicate that the errhandler is no longer
        -:  634:  valid, to help catch the case where the user has freed the errhandler but
        -:  635:  is still using a copy of the 'MPI_Errhandler' value.  We may want to 
        -:  636:  define the 'id' value for deleted errhandlers.
        -:  637:
        -:  638:  Module:
        -:  639:  ErrHand-DS
        -:  640:  S*/
        -:  641:typedef struct MPID_Errhandler {
        -:  642:  MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  643:  MPID_Lang_t        language;
        -:  644:  MPID_Object_kind   kind;
        -:  645:  MPID_Errhandler_fn errfn;
        -:  646:  /* Other, device-specific information */
        -:  647:#ifdef MPID_DEV_ERRHANDLER_DECL
        -:  648:    MPID_DEV_ERRHANDLER_DECL
        -:  649:#endif
        -:  650:} MPID_Errhandler;
        -:  651:extern MPIU_Object_alloc_t MPID_Errhandler_mem;
        -:  652:/* Preallocated errhandler objects */
        -:  653:extern MPID_Errhandler MPID_Errhandler_builtin[];
        -:  654:extern MPID_Errhandler MPID_Errhandler_direct[];
        -:  655:
        -:  656:/* We never reference count the builtin error handler objects, regardless of how
        -:  657: * we decide to reference count the other predefined objects.  If we get to the
        -:  658: * point where we never reference count *any* of the builtin objects then we
        -:  659: * should probably remove these checks and let them fall through to the checks
        -:  660: * for BUILTIN down in the MPIU_Object_* routines. */
        -:  661:#define MPIR_Errhandler_add_ref( _errhand )                               \
        -:  662:    do {                                                                  \
        -:  663:        if (HANDLE_GET_KIND((_errhand)->handle) != HANDLE_KIND_BUILTIN) { \
        -:  664:            MPIU_Object_add_ref( _errhand );                              \
        -:  665:        }                                                                 \
        -:  666:    } while (0)
        -:  667:#define MPIR_Errhandler_release_ref( _errhand, _inuse )                   \
        -:  668:    do {                                                                  \
        -:  669:        if (HANDLE_GET_KIND((_errhand)->handle) != HANDLE_KIND_BUILTIN) { \
        -:  670:            MPIU_Object_release_ref( (_errhand), (_inuse) );              \
        -:  671:        }                                                                 \
        -:  672:        else {                                                            \
        -:  673:            *(_inuse) = 1;                                                \
        -:  674:        }                                                                 \
        -:  675:    } while (0)
        -:  676:/* ------------------------------------------------------------------------- */
        -:  677:
        -:  678:/* ------------------------------------------------------------------------- */
        -:  679:/* Keyvals and attributes */
        -:  680:/*TKyOverview.tex
        -:  681:
        -:  682:  Keyvals are MPI objects that, unlike most MPI objects, are defined to be
        -:  683:  integers rather than a handle (e.g., 'MPI_Comm').  However, they really
        -:  684:  `are` MPI opaque objects and are handled by the MPICH implementation in
        -:  685:  the same way as all other MPI opaque objects.  The only difference is that
        -:  686:  there is no 'typedef int MPI_Keyval;' in 'mpi.h'.  In particular, keyvals
        -:  687:  are encoded (for direct and indirect references) in the same way that 
        -:  688:  other MPI opaque objects are
        -:  689:
        -:  690:  Each keyval has a copy and a delete function associated with it.
        -:  691:  Unfortunately, these have a slightly different calling sequence for
        -:  692:  each language, particularly when the size of a pointer is 
        -:  693:  different from the size of a Fortran integer.  The unions 
        -:  694:  'MPID_Copy_function' and 'MPID_Delete_function' capture the differences
        -:  695:  in a single union type.
        -:  696:
        -:  697:  Notes:
        -:  698:  One potential user error is to access an attribute in one language (say
        -:  699:  Fortran) that was created in another (say C).  We could add a check and
        -:  700:  generate an error message in this case; note that this would have to 
        -:  701:  be an option, because (particularly when accessing the attribute from C), 
        -:  702:  it may be what the user intended, and in any case, it is a valid operation.
        -:  703:
        -:  704:  T*/
        -:  705:/*TAttrOverview.tex
        -:  706: *
        -:  707: * The MPI standard allows `attributes`, essentially an '(integer,pointer)'
        -:  708: * pair, to be attached to communicators, windows, and datatypes.  
        -:  709: * The integer is a `keyval`, which is allocated by a call (at the MPI level)
        -:  710: * to 'MPI_Comm/Type/Win_create_keyval'.  The pointer is the value of 
        -:  711: * the attribute.
        -:  712: * Attributes are primarily intended for use by the user, for example, to save
        -:  713: * information on a communicator, but can also be used to pass data to the
        -:  714: * MPI implementation.  For example, an attribute may be used to pass 
        -:  715: * Quality of Service information to an implementation to be used with 
        -:  716: * communication on a particular communicator.  
        -:  717: * To provide the most general access by the ADI to all attributes, the
        -:  718: * ADI defines a collection of routines that are used by the implementation
        -:  719: * of the MPI attribute routines (such as 'MPI_Comm_get_attr').
        -:  720: * In addition, the MPI routines involving attributes will invoke the 
        -:  721: * corresponding 'hook' functions (e.g., 'MPID_Dev_comm_attr_set_hook') 
        -:  722: * should the device define them.
        -:  723: *
        -:  724: * Attributes on windows and datatypes are defined by MPI but not of 
        -:  725: * interest (as yet) to the device.
        -:  726: *
        -:  727: * In addition, there are seven predefined attributes that the device must
        -:  728: * supply to the implementation.  This is accomplished through 
        -:  729: * data values that are part of the 'MPIR_Process' data block.
        -:  730: *  The predefined keyvals on 'MPI_COMM_WORLD' are\:
        -:  731: *.vb
        -:  732: * Keyval                     Related Module
        -:  733: * MPI_APPNUM                 Dynamic
        -:  734: * MPI_HOST                   Core
        -:  735: * MPI_IO                     Core
        -:  736: * MPI_LASTUSEDCODE           Error
        -:  737: * MPI_TAG_UB                 Communication
        -:  738: * MPI_UNIVERSE_SIZE          Dynamic
        -:  739: * MPI_WTIME_IS_GLOBAL        Timer
        -:  740: *.ve
        -:  741: * The values stored in the 'MPIR_Process' block are the actual values.  For 
        -:  742: * example, the value of 'MPI_TAG_UB' is the integer value of the largest tag.
        -:  743: * The
        -:  744: * value of 'MPI_WTIME_IS_GLOBAL' is a '1' for true and '0' for false.  Likely
        -:  745: * values for 'MPI_IO' and 'MPI_HOST' are 'MPI_ANY_SOURCE' and 'MPI_PROC_NULL'
        -:  746: * respectively.
        -:  747: *
        -:  748: T*/
        -:  749:
        -:  750:/* Include the attribute access routines that permit access to the 
        -:  751:   attribute or its pointer, needed for cross-language access to attributes */
        -:  752:#include "mpi_attr.h"
        -:  753:
        -:  754:/* Because Comm, Datatype, and File handles are all ints, and because
        -:  755:   attributes are otherwise identical between the three types, we
        -:  756:   only store generic copy and delete functions.  This allows us to use
        -:  757:   common code for the attribute set, delete, and dup functions */
        -:  758:/*E
        -:  759:  MPID_Copy_function - MPID Structure to hold an attribute copy function
        -:  760:
        -:  761:  Notes:
        -:  762:  The appropriate element of this union is selected by using the language
        -:  763:  field of the 'keyval'.
        -:  764:
        -:  765:  Because 'MPI_Comm', 'MPI_Win', and 'MPI_Datatype' are all 'int's in 
        -:  766:  MPICH2, we use a single C copy function rather than have separate
        -:  767:  ones for the Communicator, Window, and Datatype attributes.
        -:  768:
        -:  769:  There are no corresponding typedefs for the Fortran functions.  The 
        -:  770:  F77 function corresponds to the Fortran 77 binding used in MPI-1 and the
        -:  771:  F90 function corresponds to the Fortran 90 binding used in MPI-2.
        -:  772:
        -:  773:  Module:
        -:  774:  Attribute-DS
        -:  775:
        -:  776:  E*/
        -:  777:int
        -:  778:MPIR_Attr_copy_c_proxy(
        -:  779:    MPI_Comm_copy_attr_function* user_function,
        -:  780:    int handle,
        -:  781:    int keyval,
        -:  782:    void* extra_state,
        -:  783:    MPIR_AttrType attrib_type,
        -:  784:    void* attrib,
        -:  785:    void** attrib_copy,
        -:  786:    int* flag
        -:  787:    );
        -:  788:
        -:  789:typedef struct MPID_Copy_function {
        -:  790:  int  (*C_CopyFunction)( int, int, void *, void *, void *, int * );
        -:  791:  void (*F77_CopyFunction)  ( MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, 
        -:  792:                              MPI_Fint *, MPI_Fint *, MPI_Fint * );
        -:  793:  void (*F90_CopyFunction)  ( MPI_Fint *, MPI_Fint *, MPI_Aint *, MPI_Aint *,
        -:  794:                              MPI_Aint *, MPI_Fint *, MPI_Fint * );
        -:  795:  /* The generic lang-independent user_function and proxy will
        -:  796:   * replace the lang dependent copy funcs above
        -:  797:   * Currently the lang-indpendent funcs are used only for keyvals
        -:  798:   */
        -:  799:  MPI_Comm_copy_attr_function *user_function;
        -:  800:  MPID_Attr_copy_proxy *proxy;
        -:  801:  /* The C++ function is the same as the C function */
        -:  802:} MPID_Copy_function;
        -:  803:
        -:  804:/*E
        -:  805:  MPID_Delete_function - MPID Structure to hold an attribute delete function
        -:  806:
        -:  807:  Notes:
        -:  808:  The appropriate element of this union is selected by using the language
        -:  809:  field of the 'keyval'.
        -:  810:
        -:  811:  Because 'MPI_Comm', 'MPI_Win', and 'MPI_Datatype' are all 'int's in 
        -:  812:  MPICH2, we use a single C delete function rather than have separate
        -:  813:  ones for the Communicator, Window, and Datatype attributes.
        -:  814:
        -:  815:  There are no corresponding typedefs for the Fortran functions.  The 
        -:  816:  F77 function corresponds to the Fortran 77 binding used in MPI-1 and the
        -:  817:  F90 function corresponds to the Fortran 90 binding used in MPI-2.
        -:  818:
        -:  819:  Module:
        -:  820:  Attribute-DS
        -:  821:
        -:  822:  E*/
        -:  823:int
        -:  824:MPIR_Attr_delete_c_proxy(
        -:  825:    MPI_Comm_delete_attr_function* user_function,
        -:  826:    int handle,
        -:  827:    int keyval,
        -:  828:    MPIR_AttrType attrib_type,
        -:  829:    void* attrib,
        -:  830:    void* extra_state
        -:  831:    );
        -:  832:
        -:  833:typedef struct MPID_Delete_function {
        -:  834:  int  (*C_DeleteFunction)  ( int, int, void *, void * );
        -:  835:  void (*F77_DeleteFunction)( MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, 
        -:  836:                              MPI_Fint * );
        -:  837:  void (*F90_DeleteFunction)( MPI_Fint *, MPI_Fint *, MPI_Aint *, MPI_Aint *, 
        -:  838:                              MPI_Fint * );
        -:  839:  /* The generic lang-independent user_function and proxy will
        -:  840:   * replace the lang dependent copy funcs above
        -:  841:   * Currently the lang-indpendent funcs are used only for keyvals
        -:  842:   */
        -:  843:  MPI_Comm_delete_attr_function *user_function;
        -:  844:  MPID_Attr_delete_proxy *proxy;
        -:  845:} MPID_Delete_function;
        -:  846:
        -:  847:/*S
        -:  848:  MPID_Keyval - Structure of an MPID keyval
        -:  849:
        -:  850:  Module:
        -:  851:  Attribute-DS
        -:  852:
        -:  853:  S*/
        -:  854:typedef struct MPID_Keyval {
        -:  855:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  856:    MPID_Object_kind     kind;
        -:  857:    int                  was_freed;
        -:  858:    void                 *extra_state;
        -:  859:    MPID_Copy_function   copyfn;
        -:  860:    MPID_Delete_function delfn;
        -:  861:  /* other, device-specific information */
        -:  862:#ifdef MPID_DEV_KEYVAL_DECL
        -:  863:    MPID_DEV_KEYVAL_DECL
        -:  864:#endif
        -:  865:} MPID_Keyval;
        -:  866:
        -:  867:#define MPIR_Keyval_add_ref( _keyval )                                  \
        -:  868:    do {                                                                \
        -:  869:        MPIU_Object_add_ref( _keyval );                                 \
        -:  870:    } while(0)
        -:  871:
        -:  872:#define MPIR_Keyval_release_ref( _keyval, _inuse )                      \
        -:  873:    do {                                                                \
        -:  874:        MPIU_Object_release_ref( _keyval, _inuse );                     \
        -:  875:    } while(0)
        -:  876:
        -:  877:
        -:  878:/* Attribute values in C/C++ are void * and in Fortran are ADDRESS_SIZED
        -:  879:   integers.  Normally, these are the same size, but in at least one 
        -:  880:   case, the address-sized integers was selected as longer than void *
        -:  881:   to work with the datatype code used in the I/O library.  While this
        -:  882:   is really a limitation in the current Datatype implementation. */
        -:  883:#ifdef USE_AINT_FOR_ATTRVAL
        -:  884:typedef MPI_Aint MPID_AttrVal_t;
        -:  885:#else
        -:  886:typedef void * MPID_AttrVal_t;
        -:  887:#endif
        -:  888:
        -:  889:/* Attributes need no ref count or handle, but since we want to use the
        -:  890:   common block allocator for them, we must provide those elements 
        -:  891:*/
        -:  892:/*S
        -:  893:  MPID_Attribute - Structure of an MPID attribute
        -:  894:
        -:  895:  Notes:
        -:  896:  Attributes don''t have 'ref_count's because they don''t have reference
        -:  897:  count semantics.  That is, there are no shallow copies or duplicates
        -:  898:  of an attibute.  An attribute is copied when the communicator that
        -:  899:  it is attached to is duplicated.  Subsequent operations, such as
        -:  900:  'MPI_Comm_attr_free', can change the attribute list for one of the
        -:  901:  communicators but not the other, making it impractical to keep the
        -:  902:  same list.  (We could defer making the copy until the list is changed,
        -:  903:  but even then, there would be no reference count on the individual
        -:  904:  attributes.)
        -:  905: 
        -:  906:  A pointer to the keyval, rather than the (integer) keyval itself is
        -:  907:  used since there is no need within the attribute structure to make
        -:  908:  it any harder to find the keyval structure.
        -:  909:
        -:  910:  The attribute value is a 'void *'.  If 'sizeof(MPI_Fint)' > 'sizeof(void*)',
        -:  911:  then this must be changed (no such system has been encountered yet).
        -:  912:  For the Fortran 77 routines in the case where 'sizeof(MPI_Fint)' < 
        -:  913:  'sizeof(void*)', the high end of the 'void *' value is used.  That is,
        -:  914:  we cast it to 'MPI_Fint *' and use that value.
        -:  915:
        -:  916:  MPI defines three kinds of attributes (see MPI 2.1, Section 16.3, pages 
        -:  917:  487-488 (the standard says two, but there are really three, as discussed
        -:  918:  below).  These are pointer-valued attributes and two types of integer-valued
        -:  919:  attributes.  
        -:  920:  Pointer-valued attributes are used in C.
        -:  921:  Integer-valued attributes are used in Fortran.  These are of type either
        -:  922:  INTEGER or INTEGER(KIND=MPI_ADDRESS_KIND).
        -:  923:
        -:  924:  The predefined attributes are a combination of INTEGER and pointers.
        -:  925: 
        -:  926:  Module:
        -:  927:  Attribute-DS
        -:  928:
        -:  929: S*/
        -:  930:typedef struct MPID_Attribute {
        -:  931:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  932:    MPID_Keyval  *keyval;           /* Keyval structure for this attribute */
        -:  933:
        -:  934:    struct MPID_Attribute *next;    /* Pointer to next in the list */
        -:  935:    MPIR_AttrType attrType;         /* Type of the attribute */
        -:  936:    long        pre_sentinal;       /* Used to detect user errors in accessing
        -:  937:				       the value */
        -:  938:    MPID_AttrVal_t value;           /* Stored value. An Aint must be at least
        -:  939:				       as large as an address - some builds
        -:  940:				       may make an Aint larger than a void * */
        -:  941:    long        post_sentinal;      /* Like pre_sentinal */
        -:  942:    /* other, device-specific information */
        -:  943:#ifdef MPID_DEV_ATTR_DECL
        -:  944:    MPID_DEV_ATTR_DECL
        -:  945:#endif
        -:  946:} MPID_Attribute;
        -:  947:/* ------------------------------------------------------------------------- */
        -:  948:
        -:  949:/*---------------------------------------------------------------------------
        -:  950: * Groups are *not* a major data structure in MPICH-2.  They are provided
        -:  951: * only because they are required for the group operations (e.g., 
        -:  952: * MPI_Group_intersection) and for the scalable RMA synchronization
        -:  953: *---------------------------------------------------------------------------*/
        -:  954:/* This structure is used to implement the group operations such as 
        -:  955:   MPI_Group_translate_ranks */
        -:  956:typedef struct MPID_Group_pmap_t {
        -:  957:    int          lrank;     /* Local rank in group (between 0 and size-1) */
        -:  958:    int          lpid;      /* local process id, from VCONN */
        -:  959:    int          next_lpid; /* Index of next lpid (in lpid order) */
        -:  960:    int          flag;      /* marker, used to implement group operations */
        -:  961:} MPID_Group_pmap_t;
        -:  962:
        -:  963:/* Any changes in the MPID_Group structure must be made to the
        -:  964:   predefined value in MPID_Group_builtin for MPI_GROUP_EMPTY in 
        -:  965:   src/mpi/group/grouputil.c */
        -:  966:/*S
        -:  967: MPID_Group - Description of the Group data structure
        -:  968:
        -:  969: The processes in the group of 'MPI_COMM_WORLD' have lpid values 0 to 'size'-1,
        -:  970: where 'size' is the size of 'MPI_COMM_WORLD'.  Processes created by 
        -:  971: 'MPI_Comm_spawn' or 'MPI_Comm_spawn_multiple' or added by 'MPI_Comm_attach' 
        -:  972: or  
        -:  973: 'MPI_Comm_connect'
        -:  974: are numbered greater than 'size - 1' (on the calling process). See the 
        -:  975: discussion of LocalPID values.
        -:  976:
        -:  977: Note that when dynamic process creation is used, the pids are `not` unique
        -:  978: across the universe of connected MPI processes.  This is ok, as long as
        -:  979: pids are interpreted `only` on the process that owns them.
        -:  980:
        -:  981: Only for MPI-1 are the lpid''s equal to the `global` pids.  The local pids
        -:  982: can be thought of as a reference not to the remote process itself, but
        -:  983: how the remote process can be reached from this process.  We may want to 
        -:  984: have a structure 'MPID_Lpid_t' that contains information on the remote
        -:  985: process, such as (for TCP) the hostname, ip address (it may be different if
        -:  986: multiple interfaces are supported; we may even want plural ip addresses for
        -:  987: stripping communication), and port (or ports).  For shared memory connected
        -:  988: processes, it might have the address of a remote queue.  The lpid number 
        -:  989: is an index into a table of 'MPID_Lpid_t'''s that contain this (device- and
        -:  990: method-specific) information.
        -:  991:
        -:  992: Module:
        -:  993: Group-DS
        -:  994:
        -:  995: S*/
        -:  996:typedef struct MPID_Group {
        -:  997:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  998:    int          size;           /* Size of a group */
        -:  999:    int          rank;           /* rank of this process relative to this 
        -: 1000:				    group */
        -: 1001:    int          idx_of_first_lpid;
        -: 1002:    MPID_Group_pmap_t *lrank_to_lpid; /* Array mapping a local rank to local 
        -: 1003:					 process number */
        -: 1004:    /* We may want some additional data for the RMA syncrhonization calls */
        -: 1005:  /* Other, device-specific information */
        -: 1006:#ifdef MPID_DEV_GROUP_DECL
        -: 1007:    MPID_DEV_GROUP_DECL
        -: 1008:#endif
        -: 1009:} MPID_Group;
        -: 1010:
        -: 1011:extern MPIU_Object_alloc_t MPID_Group_mem;
        -: 1012:/* Preallocated group objects */
        -: 1013:#define MPID_GROUP_N_BUILTIN 1
        -: 1014:extern MPID_Group MPID_Group_builtin[MPID_GROUP_N_BUILTIN];
        -: 1015:extern MPID_Group MPID_Group_direct[];
        -: 1016:
        -: 1017:#define MPIR_Group_add_ref( _group ) \
        -: 1018:    do { MPIU_Object_add_ref( _group ); } while (0)
        -: 1019:
        -: 1020:#define MPIR_Group_release_ref( _group, _inuse ) \
        -: 1021:     do { MPIU_Object_release_ref( _group, _inuse ); } while (0)
        -: 1022:
        -: 1023:void MPIR_Group_setup_lpid_list( MPID_Group * );
        -: 1024:int MPIR_GroupCheckVCRSubset( MPID_Group *group_ptr, int vsize, MPID_VCR *vcr, int *idx );
        -: 1025:
        -: 1026:/* ------------------------------------------------------------------------- */
        -: 1027:
        -: 1028:/*E
        -: 1029:  MPID_Comm_kind_t - Name the two types of communicators
        -: 1030:  E*/
        -: 1031:typedef enum MPID_Comm_kind_t { 
        -: 1032:    MPID_INTRACOMM = 0, 
        -: 1033:    MPID_INTERCOMM = 1 } MPID_Comm_kind_t;
        -: 1034:/* Communicators */
        -: 1035:
        -: 1036:/*S
        -: 1037:  MPID_Comm - Description of the Communicator data structure
        -: 1038:
        -: 1039:  Notes:
        -: 1040:  Note that the size and rank duplicate data in the groups that
        -: 1041:  make up this communicator.  These are used often enough that this
        -: 1042:  optimization is valuable.  
        -: 1043:
        -: 1044:  This definition provides only a 16-bit integer for context id''s .
        -: 1045:  This should be sufficient for most applications.  However, extending
        -: 1046:  this to a 32-bit (or longer) integer should be easy.
        -: 1047:
        -: 1048:  There are two context ids.  One is used for sending and one for 
        -: 1049:  receiving.  In the case of an Intracommunicator, they are the same
        -: 1050:  context id.  They differ in the case of intercommunicators, where 
        -: 1051:  they may come from processes in different comm worlds (in the
        -: 1052:  case of MPI-2 dynamic process intercomms).  
        -: 1053:
        -: 1054:  The virtual connection table is an explicit member of this structure.
        -: 1055:  This contains the information used to contact a particular process,
        -: 1056:  indexed by the rank relative to this communicator.
        -: 1057:
        -: 1058:  Groups are allocated lazily.  That is, the group pointers may be
        -: 1059:  null, created only when needed by a routine such as 'MPI_Comm_group'.
        -: 1060:  The local process ids needed to form the group are available within
        -: 1061:  the virtual connection table.
        -: 1062:  For intercommunicators, we may want to always have the groups.  If not, 
        -: 1063:  we either need the 'local_group' or we need a virtual connection table
        -: 1064:  corresponding to the 'local_group' (we may want this anyway to simplify
        -: 1065:  the implementation of the intercommunicator collective routines).
        -: 1066:
        -: 1067:  The pointer to the structure 'MPID_Collops' containing pointers to the 
        -: 1068:  collective  
        -: 1069:  routines allows an implementation to replace each routine on a 
        -: 1070:  routine-by-routine basis.  By default, this pointer is null, as are the 
        -: 1071:  pointers within the structure.  If either pointer is null, the implementation
        -: 1072:  uses the generic provided implementation.  This choice, rather than
        -: 1073:  initializing the table with pointers to all of the collective routines,
        -: 1074:  is made to reduce the space used in the communicators and to eliminate the
        -: 1075:  need to include the implementation of all collective routines in all MPI 
        -: 1076:  executables, even if the routines are not used.
        -: 1077:
        -: 1078:  The macro 'MPID_HAS_HETERO' may be defined by a device to indicate that
        -: 1079:  the device supports MPI programs that must communicate between processes with
        -: 1080:  different data representations (e.g., different sized integers or different
        -: 1081:  byte orderings).  If the device does need to define this value, it should
        -: 1082:  be defined in the file 'mpidpre.h'. 
        -: 1083:
        -: 1084:  Please note that the local_size and remote_size fields can be confusing.  For
        -: 1085:  intracommunicators both fields are always equal to the size of the
        -: 1086:  communicator.  For intercommunicators local_size is equal to the size of
        -: 1087:  local_group while remote_size is equal to the size of remote_group.
        -: 1088:
        -: 1089:  Module:
        -: 1090:  Communicator-DS
        -: 1091:
        -: 1092:  Question:
        -: 1093:  For fault tolerance, do we want to have a standard field for communicator 
        -: 1094:  health?  For example, ok, failure detected, all (live) members of failed 
        -: 1095:  communicator have acked.
        -: 1096:  S*/
        -: 1097:typedef struct MPID_Comm {
        -: 1098:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1099:    MPIR_Context_id_t context_id; /* Send context id.  See notes */
        -: 1100:    MPIR_Context_id_t recvcontext_id; /* Send context id.  See notes */
        -: 1101:    int           remote_size;   /* Value of MPI_Comm_(remote)_size */
        -: 1102:    int           rank;          /* Value of MPI_Comm_rank */
        -: 1103:    MPID_VCRT     vcrt;          /* virtual connecton reference table */
        -: 1104:    MPID_VCR *    vcr;           /* alias to the array of virtual connections
        -: 1105:				    in vcrt */
        -: 1106:    MPID_VCRT     local_vcrt;    /* local virtual connecton reference table */
        -: 1107:    MPID_VCR *    local_vcr;     /* alias to the array of local virtual
        -: 1108:				    connections in local vcrt */
        -: 1109:    MPID_Attribute *attributes;  /* List of attributes */
        -: 1110:    int           local_size;    /* Value of MPI_Comm_size for local group */
        -: 1111:    MPID_Group   *local_group,   /* Groups in communicator. */
        -: 1112:                 *remote_group;  /* The local and remote groups are the
        -: 1113:                                    same for intra communicators */
        -: 1114:    MPID_Comm_kind_t comm_kind;  /* MPID_INTRACOMM or MPID_INTERCOMM */
        -: 1115:    char          name[MPI_MAX_OBJECT_NAME];  /* Required for MPI-2 */
        -: 1116:    MPID_Errhandler *errhandler; /* Pointer to the error handler structure */
        -: 1117:    struct MPID_Comm    *local_comm; /* Defined only for intercomms, holds
        -: 1118:				        an intracomm for the local group */
        -: 1119:    int is_node_aware;           /* true if node topology info is available,
        -: 1120:                                    such as node_comm and node_roots_comm */
        -: 1121:    struct MPID_Comm *node_comm; /* Comm of processes in this comm that are on
        -: 1122:                                    the same node as this process. */
        -: 1123:    struct MPID_Comm *node_roots_comm; /* Comm of root processes for other nodes. */
        -: 1124:    int *intranode_table;        /* intranode_table[i] gives the rank in
        -: 1125:                                    node_comm of rank i in this comm or -1 if i
        -: 1126:                                    is not in this process' node_comm.
        -: 1127:                                    It is of size 'local_size'. */
        -: 1128:    int *internode_table;        /* internode_table[i] gives the rank in
        -: 1129:                                    node_roots_comm of rank i in this comm.
        -: 1130:                                    It is of size 'local_size'. */
        -: 1131:    int           is_low_group;  /* For intercomms only, this boolean is
        -: 1132:				    set for all members of one of the 
        -: 1133:				    two groups of processes and clear for 
        -: 1134:				    the other.  It enables certain
        -: 1135:				    intercommunicator collective operations
        -: 1136:				    that wish to use half-duplex operations
        -: 1137:				    to implement a full-duplex operation */
        -: 1138:    struct MPID_Comm     *comm_next;/* Provides a chain through all active 
        -: 1139:				       communicators */
        -: 1140:    struct MPID_Collops  *coll_fns; /* Pointer to a table of functions 
        -: 1141:                                              implementing the collective 
        -: 1142:                                              routines */
        -: 1143:    struct MPID_TopoOps  *topo_fns; /* Pointer to a table of functions
        -: 1144:				       implementting the topology routines
        -: 1145:				    */
        -: 1146:#ifdef MPID_HAS_HETERO
        -: 1147:    int is_hetero;
        -: 1148:#endif
        -: 1149:  /* Other, device-specific information */
        -: 1150:#ifdef MPID_DEV_COMM_DECL
        -: 1151:    MPID_DEV_COMM_DECL
        -: 1152:#endif
        -: 1153:} MPID_Comm;
        -: 1154:extern MPIU_Object_alloc_t MPID_Comm_mem;
        -: 1155:
        -: 1156:/* MPIR_Comm_release is a helper routine that releases references to a comm.
        -: 1157:   The second arg is false unless this is called as part of 
        -: 1158:   MPI_Comm_disconnect .
        -: 1159:*/
        -: 1160:int MPIR_Comm_release(MPID_Comm *, int );
        -: 1161:int MPIR_Comm_release_always(MPID_Comm *comm_ptr, int isDisconnect);
        -: 1162:
        -: 1163:#define MPIR_Comm_add_ref(_comm) \
        -: 1164:    do { MPIU_Object_add_ref((_comm)); } while (0)
        -: 1165:
        -: 1166:#define MPIR_Comm_release_ref( _comm, _inuse ) \
        -: 1167:    do { MPIU_Object_release_ref( _comm, _inuse ); } while (0)
        -: 1168:
        -: 1169:/* Preallocated comm objects.  There are 3: comm_world, comm_self, and 
        -: 1170:   a private (non-user accessible) dup of comm world that is provided 
        -: 1171:   if needed in MPI_Finalize.  Having a separate version of comm_world
        -: 1172:   avoids possible interference with User code */
        -: 1173:#define MPID_COMM_N_BUILTIN 3
        -: 1174:extern MPID_Comm MPID_Comm_builtin[MPID_COMM_N_BUILTIN];
        -: 1175:extern MPID_Comm MPID_Comm_direct[];
        -: 1176:/* This is the handle for the internal MPI_COMM_WORLD .  The "2" at the end
        -: 1177:   of the handle is 3-1 (e.g., the index in the builtin array) */
        -: 1178:#define MPIR_ICOMM_WORLD  ((MPI_Comm)0x44000002)
        -: 1179:
        -: 1180:/* The following preprocessor macros provide bitfield access information for
        -: 1181: * context ID values.  They follow a uniform naming pattern:
        -: 1182: *
        -: 1183: * MPID_CONTEXT_foo_WIDTH - the width in bits of the field
        -: 1184: * MPID_CONTEXT_foo_MASK  - A valid bit mask for bit-wise AND and OR operations
        -: 1185: *                          with exactly all of the bits in the field set.
        -: 1186: * MPID_CONTEXT_foo_SHIFT - The number of bits that the field should be shifted
        -: 1187: *                          rightwards to place it in the least significant bits
        -: 1188: *                          of the ID.  There may still be higher order bits
        -: 1189: *                          from other fields, so the _MASK should be used first
        -: 1190: *                          if you want to reliably retrieve the exact value of
        -: 1191: *                          the field.
        -: 1192: */
        -: 1193:
        -: 1194:/* yields an rvalue that is the value of the field_name_ in the least significant bits */
        -: 1195:#define MPID_CONTEXT_READ_FIELD(field_name_,id_) \
        -: 1196:    (((id_) & MPID_CONTEXT_##field_name_##_MASK) >> MPID_CONTEXT_##field_name_##_SHIFT)
        -: 1197:/* yields an rvalue that is the old_id_ with field_name_ set to field_val_ */
        -: 1198:#define MPID_CONTEXT_SET_FIELD(field_name_,old_id_,field_val_) \
        -: 1199:    ((old_id_ & ~MPID_CONTEXT_##field_name_##_MASK) | ((field_val_) << MPID_CONTEXT_##field_name_##_SHIFT))
        -: 1200:
        -: 1201:/* Context suffixes for separating pt2pt and collective communication */
        -: 1202:#define MPID_CONTEXT_SUFFIX_WIDTH (1)
        -: 1203:#define MPID_CONTEXT_SUFFIX_SHIFT (0)
        -: 1204:#define MPID_CONTEXT_SUFFIX_MASK ((1 << MPID_CONTEXT_SUFFIX_WIDTH) - 1)
        -: 1205:#define MPID_CONTEXT_INTRA_PT2PT (0)
        -: 1206:#define MPID_CONTEXT_INTRA_COLL  (1)
        -: 1207:#define MPID_CONTEXT_INTER_PT2PT (0)
        -: 1208:#define MPID_CONTEXT_INTER_COLL  (1)
        -: 1209:
        -: 1210:/* Used to derive context IDs for sub-communicators from a parent communicator's
        -: 1211:   context ID value.  This field comes after the one bit suffix.
        -: 1212:   values are shifted left by 1. */
        -: 1213:#define MPID_CONTEXT_SUBCOMM_WIDTH (2)
        -: 1214:#define MPID_CONTEXT_SUBCOMM_SHIFT (MPID_CONTEXT_SUFFIX_WIDTH + MPID_CONTEXT_SUFFIX_SHIFT)
        -: 1215:#define MPID_CONTEXT_SUBCOMM_MASK      (((1 << MPID_CONTEXT_SUBCOMM_WIDTH) - 1) << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1216:
        -: 1217:/* these values may be added/subtracted directly to/from an existing context ID
        -: 1218: * in order to determine the context ID of the child/parent */
        -: 1219:#define MPID_CONTEXT_PARENT_OFFSET    (0 << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1220:#define MPID_CONTEXT_INTRANODE_OFFSET (1 << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1221:#define MPID_CONTEXT_INTERNODE_OFFSET (2 << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1222:
        -: 1223:/* this field (IS_LOCALCOM) is used to derive a context ID for local
        -: 1224: * communicators of intercommunicators without communication */
        -: 1225:#define MPID_CONTEXT_IS_LOCALCOMM_WIDTH (1)
        -: 1226:#define MPID_CONTEXT_IS_LOCALCOMM_SHIFT (MPID_CONTEXT_SUBCOMM_SHIFT + MPID_CONTEXT_SUBCOMM_WIDTH)
        -: 1227:#define MPID_CONTEXT_IS_LOCALCOMM_MASK (((1 << MPID_CONTEXT_IS_LOCALCOMM_WIDTH) - 1) << MPID_CONTEXT_IS_LOCALCOMM_SHIFT)
        -: 1228:
        -: 1229:/* MPIR_MAX_CONTEXT_MASK is the number of ints that make up the bit vector that
        -: 1230: * describes the context ID prefix space.
        -: 1231: *
        -: 1232: * The following must hold:
        -: 1233: * (num_bits_in_vector) <= (maximum_context_id_prefix)
        -: 1234: *   which is the following in concrete terms:
        -: 1235: * MPIR_MAX_CONTEXT_MASK*MPIR_CONTEXT_INT_BITS <= 2**(MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_WIDTH))
        -: 1236: *
        -: 1237: * We currently always assume MPIR_CONTEXT_INT_BITS is 32, regardless of the
        -: 1238: * value of sizeof(int)*CHAR_BITS.  We also make the assumption that CHAR_BITS==8.
        -: 1239: *
        -: 1240: * For a 16-bit context id field and CHAR_BITS==8, this implies MPIR_MAX_CONTEXT_MASK <= 256
        -: 1241: */
        -: 1242:
        -: 1243:/* number of bits to shift right by in order to obtain the context ID prefix */
        -: 1244:#define MPID_CONTEXT_PREFIX_SHIFT (MPID_CONTEXT_IS_LOCALCOMM_SHIFT + MPID_CONTEXT_IS_LOCALCOMM_WIDTH)
        -: 1245:#define MPID_CONTEXT_PREFIX_WIDTH (MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_WIDTH))
        -: 1246:#define MPID_CONTEXT_PREFIX_MASK (((1 << MPID_CONTEXT_PREFIX_WIDTH) - 1) << MPID_CONTEXT_PREFIX_SHIFT)
        -: 1247:
        -: 1248:#define MPID_CONTEXT_DYNAMIC_PROC_WIDTH (1) /* the upper half is reserved for dynamic procs */
        -: 1249:#define MPID_CONTEXT_DYNAMIC_PROC_SHIFT (MPIR_CONTEXT_ID_BITS - MPID_CONTEXT_DYNAMIC_PROC_WIDTH) /* the upper half is reserved for dynamic procs */
        -: 1250:#define MPID_CONTEXT_DYNAMIC_PROC_MASK (((1 << MPID_CONTEXT_DYNAMIC_PROC_WIDTH) - 1) << MPID_CONTEXT_DYNAMIC_PROC_SHIFT)
        -: 1251:
        -: 1252:/* should probably be (sizeof(int)*CHAR_BITS) once we make the code CHAR_BITS-clean */
        -: 1253:#define MPIR_CONTEXT_INT_BITS (32)
        -: 1254:#define MPIR_CONTEXT_ID_BITS (sizeof(MPIR_Context_id_t)*8) /* 8 --> CHAR_BITS eventually */
        -: 1255:#define MPIR_MAX_CONTEXT_MASK \
        -: 1256:    ((1 << (MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_WIDTH))) / MPIR_CONTEXT_INT_BITS)
        -: 1257:
        -: 1258:/* Utility routines.  Where possible, these are kept in the source directory
        -: 1259:   with the other comm routines (src/mpi/comm, in mpicomm.h).  However,
        -: 1260:   to create a new communicator after a spawn or connect-accept operation, 
        -: 1261:   the device may need to create a new contextid */
        -: 1262:int MPIR_Get_contextid( MPID_Comm *, MPIR_Context_id_t *context_id );
        -: 1263:
        -: 1264:/* ------------------------------------------------------------------------- */
        -: 1265:
        -: 1266:/* Requests */
        -: 1267:/* This currently defines a single structure type for all requests.  
        -: 1268:   Eventually, we may want a union type, as used in MPICH-1 */
        -: 1269:/*E
        -: 1270:  MPID_Request_kind - Kinds of MPI Requests
        -: 1271:
        -: 1272:  Module:
        -: 1273:  Request-DS
        -: 1274:
        -: 1275:  E*/
        -: 1276:typedef enum MPID_Request_kind_t {
        -: 1277:    MPID_REQUEST_UNDEFINED,
        -: 1278:    MPID_REQUEST_SEND,
        -: 1279:    MPID_REQUEST_RECV,
        -: 1280:    MPID_PREQUEST_SEND,
        -: 1281:    MPID_PREQUEST_RECV,
        -: 1282:    MPID_UREQUEST,
        -: 1283:    MPID_LAST_REQUEST_KIND
        -: 1284:#ifdef MPID_DEV_REQUEST_KIND_DECL
        -: 1285:    , MPID_DEV_REQUEST_KIND_DECL
        -: 1286:#endif
        -: 1287:} MPID_Request_kind_t;
        -: 1288:
        -: 1289:/* Typedefs for Fortran generalized requests */
        -: 1290:typedef void (MPIR_Grequest_f77_cancel_function)(void *, int*, int *); 
        -: 1291:typedef void (MPIR_Grequest_f77_free_function)(void *, int *); 
        -: 1292:typedef void (MPIR_Grequest_f77_query_function)(void *, MPI_Status *, int *); 
        -: 1293:
        -: 1294:/*S
        -: 1295:  MPID_Request - Description of the Request data structure
        -: 1296:
        -: 1297:  Module:
        -: 1298:  Request-DS
        -: 1299:
        -: 1300:  Notes:
        -: 1301:  If it is necessary to remember the MPI datatype, this information is 
        -: 1302:  saved within the device-specific fields provided by 'MPID_DEV_REQUEST_DECL'.
        -: 1303:
        -: 1304:  Requests come in many flavors, as stored in the 'kind' field.  It is 
        -: 1305:  expected that each kind of request will have its own structure type 
        -: 1306:  (e.g., 'MPID_Request_send_t') that extends the 'MPID_Request'.
        -: 1307:  
        -: 1308:  S*/
        -: 1309:typedef struct MPID_Request {
        -: 1310:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1311:    MPID_Request_kind_t kind;
        -: 1312:    /* completion counter */
        -: 1313:    volatile int cc;
        -: 1314:    /* pointer to the completion counter */
        -: 1315:    /* This is necessary for the case when an operation is described by a 
        -: 1316:       list of requests */
        -: 1317:    int volatile *cc_ptr;
        -: 1318:    /* A comm is needed to find the proper error handler */
        -: 1319:    MPID_Comm *comm;
        -: 1320:    /* Status is needed for wait/test/recv */
        -: 1321:    MPI_Status status;
        -: 1322:    /* Persistent requests have their own "real" requests.  Receive requests
        -: 1323:       have partnering send requests when src=dest. etc. */
        -: 1324:    struct MPID_Request *partner_request;
        -: 1325:    /* User-defined request support */
        -: 1326:    MPI_Grequest_cancel_function *cancel_fn;
        -: 1327:    MPI_Grequest_free_function   *free_fn;
        -: 1328:    MPI_Grequest_query_function  *query_fn;
        -: 1329:    MPIX_Grequest_poll_function   *poll_fn;
        -: 1330:    MPIX_Grequest_wait_function   *wait_fn;
        -: 1331:    void             *grequest_extra_state;
        -: 1332:    MPIX_Grequest_class	        greq_class;
        -: 1333:    MPID_Lang_t                  greq_lang;         /* language that defined
        -: 1334:						       the generalize req */
        -: 1335:    
        -: 1336:    /* Other, device-specific information */
        -: 1337:#ifdef MPID_DEV_REQUEST_DECL
        -: 1338:    MPID_DEV_REQUEST_DECL
        -: 1339:#endif
        -: 1340:} MPID_Request;
        -: 1341:
        -: 1342:extern MPIU_Object_alloc_t MPID_Request_mem;
        -: 1343:/* Preallocated request objects */
        -: 1344:extern MPID_Request MPID_Request_direct[];
        -: 1345:
        -: 1346:#define MPIR_Request_add_ref( _req ) \
        -: 1347:    do { MPIU_Object_add_ref( _req ); } while (0)
        -: 1348:
        -: 1349:#define MPIR_Request_release_ref( _req, _inuse ) \
        -: 1350:    do { MPIU_Object_release_ref( _req, _inuse ); } while (0)
        -: 1351:
        -: 1352:/* These macros allow us to implement a sendq when debugger support is
        -: 1353:   selected.  As there is extra overhead for this, we only do this
        -: 1354:   when specifically requested 
        -: 1355:*/
        -: 1356:#ifdef HAVE_DEBUGGER_SUPPORT
        -: 1357:void MPIR_WaitForDebugger( void );
        -: 1358:void MPIR_DebuggerSetAborting( const char * );
        -: 1359:void MPIR_Sendq_remember(MPID_Request *, int, int, int );
        -: 1360:void MPIR_Sendq_forget(MPID_Request *);
        -: 1361:void MPIR_CommL_remember( MPID_Comm * );
        -: 1362:void MPIR_CommL_forget( MPID_Comm * );
        -: 1363:
        -: 1364:#define MPIR_SENDQ_REMEMBER(_a,_b,_c,_d) MPIR_Sendq_remember(_a,_b,_c,_d)
        -: 1365:#define MPIR_SENDQ_FORGET(_a) MPIR_Sendq_forget(_a)
        -: 1366:#define MPIR_COMML_REMEMBER(_a) MPIR_CommL_remember( _a )
        -: 1367:#define MPIR_COMML_FORGET(_a) MPIR_CommL_forget( _a )
        -: 1368:#else
        -: 1369:#define MPIR_SENDQ_REMEMBER(a,b,c,d)
        -: 1370:#define MPIR_SENDQ_FORGET(a)
        -: 1371:#define MPIR_COMML_REMEMBER(_a) 
        -: 1372:#define MPIR_COMML_FORGET(_a) 
        -: 1373:#endif
        -: 1374:
        -: 1375:
        -: 1376:/* ------------------------------------------------------------------------- */
        -: 1377:/* Prototypes and definitions for the node ID code.  This is used to support
        -: 1378:   hierarchical collectives in a (mostly) device-independent way. */
        -: 1379:#if defined(MPID_USE_NODE_IDS)
        -: 1380:/* MPID_Node_id_t is a signed integer type defined by the device in mpidpre.h. */
        -: 1381:int MPID_Get_node_id(MPID_Comm *comm, int rank, MPID_Node_id_t *id_p);
        -: 1382:int MPID_Get_max_node_id(MPID_Comm *comm, MPID_Node_id_t *max_id_p);
        -: 1383:#endif
        -: 1384:
        -: 1385:/* ------------------------------------------------------------------------- */
        -: 1386:/*S
        -: 1387:  MPID_Progress_state - object to hold progress state when using the blocking
        -: 1388:  progress routines.
        -: 1389:
        -: 1390:  Module:
        -: 1391:  Misc
        -: 1392:
        -: 1393:  Notes:
        -: 1394:  The device must define MPID_PROGRESS_STATE_DECL.  It should  include any state
        -: 1395:  that needs to be maintained between calls to MPID_Progress_{start,wait,end}.
        -: 1396:  S*/
        -: 1397:typedef struct MPID_Progress_state
        -: 1398:{
        -: 1399:    MPID_PROGRESS_STATE_DECL
        -: 1400:}
        -: 1401:MPID_Progress_state;
        -: 1402:/* ------------------------------------------------------------------------- */
        -: 1403:
        -: 1404:/* ------------------------------------------------------------------------- */
        -: 1405:/* end of mpirma.h (in src/mpi/rma?) */
        -: 1406:/* ------------------------------------------------------------------------- */
        -: 1407:
        -: 1408:/* Windows */
        -: 1409:#ifdef USE_MPID_RMA_TABLE
        -: 1410:struct MPID_Win;
        -: 1411:typedef struct MPIRI_RMA_Ops {
        -: 1412:    int (*Win_free)(struct MPID_Win **);
        -: 1413:    int (*Put)(void *, int, MPI_Datatype, int, MPI_Aint, int, MPI_Datatype, 
        -: 1414:		struct MPID_Win *);
        -: 1415:    int (*Get)(void *, int, MPI_Datatype, int, MPI_Aint, int, MPI_Datatype, 
        -: 1416:		struct MPID_Win *);
        -: 1417:    int (*Accumulate)(void *, int, MPI_Datatype, int, MPI_Aint, int, 
        -: 1418:		       MPI_Datatype, MPI_Op, struct MPID_Win *);
        -: 1419:    int (*Win_fence)(int, struct MPID_Win *);
        -: 1420:    int (*Win_post)(MPID_Group *, int, struct MPID_Win *);
        -: 1421:    int (*Win_start)(MPID_Group *, int, struct MPID_Win *);
        -: 1422:    int (*Win_complete)(struct MPID_Win *);
        -: 1423:    int (*Win_wait)(struct MPID_Win *);
        -: 1424:    int (*Win_test)(struct MPID_Win *, int *);
        -: 1425:    int (*Win_lock)(int, int, int, struct MPID_Win *);
        -: 1426:    int (*Win_unlock)(int, struct MPID_Win *);
        -: 1427:} MPIRI_RMAFns;
        -: 1428:#define MPIRI_RMAFNS_VERSION 2
        -: 1429:/* Note that the memory allocation/free routines do not take a window, 
        -: 1430:   so they must be initialized separately, and are a per-run, not per-window
        -: 1431:   object.  If the device can manage different kinds of memory allocations,
        -: 1432:   these routines must internally provide that flexibility. */
        -: 1433:/* 
        -: 1434:    void *(*Alloc_mem)(size_t, MPID_Info *);
        -: 1435:    int (*Free_mem)(void *);
        -: 1436:*/
        -: 1437:#endif
        -: 1438:
        -: 1439:/*S
        -: 1440:  MPID_Win - Description of the Window Object data structure.
        -: 1441:
        -: 1442:  Module:
        -: 1443:  Win-DS
        -: 1444:
        -: 1445:  Notes:
        -: 1446:  The following 3 keyvals are defined for attributes on all MPI 
        -: 1447:  Window objects\:
        -: 1448:.vb
        -: 1449: MPI_WIN_SIZE
        -: 1450: MPI_WIN_BASE
        -: 1451: MPI_WIN_DISP_UNIT
        -: 1452:.ve
        -: 1453:  These correspond to the values in 'length', 'start_address', and 
        -: 1454:  'disp_unit'.
        -: 1455:
        -: 1456:  The communicator in the window is the same communicator that the user
        -: 1457:  provided to 'MPI_Win_create' (not a dup).  However, each intracommunicator
        -: 1458:  has a special context id that may be used if MPI communication is used 
        -: 1459:  by the implementation to implement the RMA operations.
        -: 1460:
        -: 1461:  There is no separate window group; the group of the communicator should be
        -: 1462:  used.
        -: 1463:
        -: 1464:  Question:
        -: 1465:  Should a 'MPID_Win' be defined after 'MPID_Segment' in case the device 
        -: 1466:  wants to 
        -: 1467:  store a queue of pending put/get operations, described with 'MPID_Segment'
        -: 1468:  (or 'MPID_Request')s?
        -: 1469:
        -: 1470:  S*/
        -: 1471:typedef struct MPID_Win {
        -: 1472:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1473:    int fence_cnt;     /* 0 = no fence has been called; 
        -: 1474:                          1 = fence has been called */ 
        -: 1475:    MPID_Errhandler *errhandler;  /* Pointer to the error handler structure */
        -: 1476:    void *base;
        -: 1477:    MPI_Aint    size;        
        -: 1478:    int          disp_unit;      /* Displacement unit of *local* window */
        -: 1479:    MPID_Attribute *attributes;
        -: 1480:    MPID_Group *start_group_ptr; /* group passed in MPI_Win_start */
        -: 1481:    int start_assert;            /* assert passed to MPI_Win_start */
        -: 1482:    MPI_Comm    comm;         /* communicator of window (dup) */
        -: 1483:#ifdef USE_THREADED_WINDOW_CODE
        -: 1484:    /* These were causing compilation errors.  We need to figure out how to
        -: 1485:       integrate threads into MPICH2 before including these fields. */
        -: 1486:    /* FIXME: The test here should be within a test for threaded support */
        -: 1487:#ifdef HAVE_PTHREAD_H
        -: 1488:    pthread_t wait_thread_id; /* id of thread handling MPI_Win_wait */
        -: 1489:    pthread_t passive_target_thread_id; /* thread for passive target RMA */
        -: 1490:#elif defined(HAVE_WINTHREADS)
        -: 1491:    HANDLE wait_thread_id;
        -: 1492:    HANDLE passive_target_thread_id;
        -: 1493:#endif
        -: 1494:#endif
        -: 1495:    /* */
        -: 1496:#ifdef USE_MPID_RMA_TABLE
        -: 1497:    MPIRI_RMAFns RMAFns;
        -: 1498:#endif    
        -: 1499:    /* These are COPIES of the values so that addresses to them
        -: 1500:       can be returned as attributes.  They are initialized by the
        -: 1501:       MPI_Win_get_attr function */
        -: 1502:    int  copyDispUnit;
        -: 1503:    MPI_Aint copySize;
        -: 1504:    
        -: 1505:    char          name[MPI_MAX_OBJECT_NAME];  
        -: 1506:  /* Other, device-specific information */
        -: 1507:#ifdef MPID_DEV_WIN_DECL
        -: 1508:    MPID_DEV_WIN_DECL
        -: 1509:#endif
        -: 1510:} MPID_Win;
        -: 1511:extern MPIU_Object_alloc_t MPID_Win_mem;
        -: 1512:/* Preallocated win objects */
        -: 1513:extern MPID_Win MPID_Win_direct[];
        -: 1514:
        -: 1515:
        -: 1516:/* ------------------------------------------------------------------------- */
        -: 1517:/* also in mpirma.h ?*/
        -: 1518:/* ------------------------------------------------------------------------- */
        -: 1519:
        -: 1520:/*
        -: 1521: * Good Memory (may be required for passive target operations on MPI_Win)
        -: 1522: */
        -: 1523:
        -: 1524:/*@
        -: 1525:  MPID_Alloc_mem - Allocate memory suitable for passive target RMA operations
        -: 1526:
        -: 1527:  Input Parameter:
        -: 1528:+ size - Number of types to allocate.
        -: 1529:- info - Info object
        -: 1530:
        -: 1531:  Return value:
        -: 1532:  Pointer to the allocated memory.  If the memory is not available, 
        -: 1533:  returns null.
        -: 1534:
        -: 1535:  Notes:
        -: 1536:  This routine is used to implement 'MPI_Alloc_mem'.  It is for that reason
        -: 1537:  that there is no communicator argument.  
        -: 1538:
        -: 1539:  This memory may `only` be freed with 'MPID_Free_mem'.
        -: 1540:
        -: 1541:  This is a `local`, not a collective operation.  It functions more like a
        -: 1542:  good form of 'malloc' than collective shared-memory allocators such as
        -: 1543:  the 'shmalloc' found on SGI systems.
        -: 1544:
        -: 1545:  Implementations of this routine may wish to use 'MPID_Memory_register'.  
        -: 1546:  However, this routine has slighly different requirements, so a separate
        -: 1547:  entry point is provided.
        -: 1548:
        -: 1549:  Question:
        -: 1550:  Since this takes an info object, should there be an error routine in the 
        -: 1551:  case that the info object contains an error?
        -: 1552:
        -: 1553:  Module:
        -: 1554:  Win
        -: 1555:  @*/
        -: 1556:void *MPID_Alloc_mem( size_t size, MPID_Info *info );
        -: 1557:
        -: 1558:/*@
        -: 1559:  MPID_Free_mem - Frees memory allocated with 'MPID_Alloc_mem'
        -: 1560:
        -: 1561:  Input Parameter:
        -: 1562:. ptr - Pointer to memory allocated by 'MPID_Alloc_mem'.
        -: 1563:
        -: 1564:  Return value:
        -: 1565:  'MPI_SUCCESS' if memory was successfully freed; an MPI error code otherwise.
        -: 1566:
        -: 1567:  Notes:
        -: 1568:  The return value is provided because it may not be easy to validate the
        -: 1569:  value of 'ptr' without attempting to free the memory.
        -: 1570:
        -: 1571:  Module:
        -: 1572:  Win
        -: 1573:  @*/
        -: 1574:int MPID_Free_mem( void *ptr );
        -: 1575:
        -: 1576:/*@
        -: 1577:  MPID_Mem_was_alloced - Return true if this memory was allocated with 
        -: 1578:  'MPID_Alloc_mem'
        -: 1579:
        -: 1580:  Input Parameters:
        -: 1581:+ ptr  - Address of memory
        -: 1582:- size - Size of reqion in bytes.
        -: 1583:
        -: 1584:  Return value:
        -: 1585:  True if the memory was allocated with 'MPID_Alloc_mem', false otherwise.
        -: 1586:
        -: 1587:  Notes:
        -: 1588:  This routine may be needed by 'MPI_Win_create' to ensure that the memory 
        -: 1589:  for passive target RMA operations was allocated with 'MPI_Mem_alloc'.
        -: 1590:  This may be used, for example, for ensuring that memory used with
        -: 1591:  passive target operations was allocated with 'MPID_Alloc_mem'.
        -: 1592:
        -: 1593:  Module:
        -: 1594:  Win
        -: 1595:  @*/
        -: 1596:int MPID_Mem_was_alloced( void *ptr );  /* brad : this isn't used or implemented anywhere */
        -: 1597:
        -: 1598:/* ------------------------------------------------------------------------- */
        -: 1599:/* end of also in mpirma.h ? */
        -: 1600:/* ------------------------------------------------------------------------- */
        -: 1601:
        -: 1602:/* ------------------------------------------------------------------------- */
        -: 1603:/* Reduction and accumulate operations */
        -: 1604:/*E
        -: 1605:  MPID_Op_kind - Enumerates types of MPI_Op types
        -: 1606:
        -: 1607:  Notes:
        -: 1608:  These are needed for implementing 'MPI_Accumulate', since only predefined
        -: 1609:  operations are allowed for that operation.  
        -: 1610:
        -: 1611:  A gap in the enum values was made allow additional predefined operations
        -: 1612:  to be inserted.  This might include future additions to MPI or experimental
        -: 1613:  extensions (such as a Read-Modify-Write operation).
        -: 1614:
        -: 1615:  Module:
        -: 1616:  Collective-DS
        -: 1617:  E*/
        -: 1618:typedef enum MPID_Op_kind { MPID_OP_MAX=1, MPID_OP_MIN=2, 
        -: 1619:			    MPID_OP_SUM=3, MPID_OP_PROD=4, 
        -: 1620:	       MPID_OP_LAND=5, MPID_OP_BAND=6, MPID_OP_LOR=7, MPID_OP_BOR=8,
        -: 1621:	       MPID_OP_LXOR=9, MPID_OP_BXOR=10, MPID_OP_MAXLOC=11, 
        -: 1622:               MPID_OP_MINLOC=12, MPID_OP_REPLACE=13, 
        -: 1623:               MPID_OP_USER_NONCOMMUTE=32, MPID_OP_USER=33 }
        -: 1624:  MPID_Op_kind;
        -: 1625:
        -: 1626:/*S
        -: 1627:  MPID_User_function - Definition of a user function for MPI_Op types.
        -: 1628:
        -: 1629:  Notes:
        -: 1630:  This includes a 'const' to make clear which is the 'in' argument and 
        -: 1631:  which the 'inout' argument, and to indicate that the 'count' and 'datatype'
        -: 1632:  arguments are unchanged (they are addresses in an attempt to allow 
        -: 1633:  interoperation with Fortran).  It includes 'restrict' to emphasize that 
        -: 1634:  no overlapping operations are allowed.
        -: 1635:
        -: 1636:  We need to include a Fortran version, since those arguments will
        -: 1637:  have type 'MPI_Fint *' instead.  We also need to add a test to the
        -: 1638:  test suite for this case; in fact, we need tests for each of the handle
        -: 1639:  types to ensure that the transfered handle works correctly.
        -: 1640:
        -: 1641:  This is part of the collective module because user-defined operations
        -: 1642:  are valid only for the collective computation routines and not for 
        -: 1643:  RMA accumulate.
        -: 1644:
        -: 1645:  Yes, the 'restrict' is in the correct location.  C compilers that 
        -: 1646:  support 'restrict' should be able to generate code that is as good as a
        -: 1647:  Fortran compiler would for these functions.
        -: 1648:
        -: 1649:  We should note on the manual pages for user-defined operations that
        -: 1650:  'restrict' should be used when available, and that a cast may be 
        -: 1651:  required when passing such a function to 'MPI_Op_create'.
        -: 1652:
        -: 1653:  Question:
        -: 1654:  Should each of these function types have an associated typedef?
        -: 1655:
        -: 1656:  Should there be a C++ function here?
        -: 1657:
        -: 1658:  Module:
        -: 1659:  Collective-DS
        -: 1660:  S*/
        -: 1661:typedef union MPID_User_function {
        -: 1662:    void (*c_function) ( const void *, void *, 
        -: 1663:			 const int *, const MPI_Datatype * ); 
        -: 1664:    void (*f77_function) ( const void *, void *,
        -: 1665:			  const MPI_Fint *, const MPI_Fint * );
        -: 1666:} MPID_User_function;
        -: 1667:/* FIXME: Should there be "restrict" in the definitions above, e.g., 
        -: 1668:   (*c_function)( const void restrict * , void restrict *, ... )? */
        -: 1669:
        -: 1670:/*S
        -: 1671:  MPID_Op - MPI_Op structure
        -: 1672:
        -: 1673:  Notes:
        -: 1674:  All of the predefined functions are commutative.  Only user functions may 
        -: 1675:  be noncummutative, so there are two separate op types for commutative and
        -: 1676:  non-commutative user-defined operations.
        -: 1677:
        -: 1678:  Operations do not require reference counts because there are no nonblocking
        -: 1679:  operations that accept user-defined operations.  Thus, there is no way that
        -: 1680:  a valid program can free an 'MPI_Op' while it is in use.
        -: 1681:
        -: 1682:  Module:
        -: 1683:  Collective-DS
        -: 1684:  S*/
        -: 1685:typedef struct MPID_Op {
        -: 1686:     MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1687:     MPID_Op_kind       kind;
        -: 1688:     MPID_Lang_t        language;
        -: 1689:     MPID_User_function function;
        -: 1690:  } MPID_Op;
        -: 1691:#define MPID_OP_N_BUILTIN 14
        -: 1692:extern MPID_Op MPID_Op_builtin[MPID_OP_N_BUILTIN];
        -: 1693:extern MPID_Op MPID_Op_direct[];
        -: 1694:extern MPIU_Object_alloc_t MPID_Op_mem;
        -: 1695:
        -: 1696:#define MPIR_Op_release_ref( _op, _inuse ) \
        -: 1697:    do { MPIU_Object_release_ref( _op, _inuse ); } while (0)
        -: 1698:
        -: 1699:/* ------------------------------------------------------------------------- */
        -: 1700:
        -: 1701:/* ------------------------------------------------------------------------- */
        -: 1702:/* mpicoll.h (in src/mpi/coll?) */
        -: 1703:/* ------------------------------------------------------------------------- */
        -: 1704:
        -: 1705:/* Collective operations */
        -: 1706:typedef struct MPID_Collops {
        -: 1707:    int ref_count;   /* Supports lazy copies */
        -: 1708:    /* Contains pointers to the functions for the MPI collectives */
        -: 1709:    int (*Barrier) (MPID_Comm *);
        -: 1710:    int (*Bcast) (void*, int, MPI_Datatype, int, MPID_Comm * );
        -: 1711:    int (*Gather) (void*, int, MPI_Datatype, void*, int, MPI_Datatype, 
        -: 1712:                   int, MPID_Comm *); 
        -: 1713:    int (*Gatherv) (void*, int, MPI_Datatype, void*, int *, int *, 
        -: 1714:                    MPI_Datatype, int, MPID_Comm *); 
        -: 1715:    int (*Scatter) (void*, int, MPI_Datatype, void*, int, MPI_Datatype, 
        -: 1716:                    int, MPID_Comm *);
        -: 1717:    int (*Scatterv) (void*, int *, int *, MPI_Datatype, void*, int, 
        -: 1718:                    MPI_Datatype, int, MPID_Comm *);
        -: 1719:    int (*Allgather) (void*, int, MPI_Datatype, void*, int, 
        -: 1720:                      MPI_Datatype, MPID_Comm *);
        -: 1721:    int (*Allgatherv) (void*, int, MPI_Datatype, void*, int *, int *, 
        -: 1722:                       MPI_Datatype, MPID_Comm *);
        -: 1723:    int (*Alltoall) (void*, int, MPI_Datatype, void*, int, MPI_Datatype, 
        -: 1724:                               MPID_Comm *);
        -: 1725:    int (*Alltoallv) (void*, int *, int *, MPI_Datatype, void*, int *, 
        -: 1726:                     int *, MPI_Datatype, MPID_Comm *);
        -: 1727:    int (*Alltoallw) (void*, int *, int *, MPI_Datatype *, void*, int *, 
        -: 1728:                     int *, MPI_Datatype *, MPID_Comm *);
        -: 1729:    int (*Reduce) (void*, void*, int, MPI_Datatype, MPI_Op, int, 
        -: 1730:                   MPID_Comm *);
        -: 1731:    int (*Allreduce) (void*, void*, int, MPI_Datatype, MPI_Op, 
        -: 1732:                      MPID_Comm *);
        -: 1733:    int (*Reduce_scatter) (void*, void*, int *, MPI_Datatype, MPI_Op, 
        -: 1734:                           MPID_Comm *);
        -: 1735:    int (*Scan) (void*, void*, int, MPI_Datatype, MPI_Op, MPID_Comm * );
        -: 1736:    int (*Exscan) (void*, void*, int, MPI_Datatype, MPI_Op, MPID_Comm * );
        -: 1737:    
        -: 1738:} MPID_Collops;
        -: 1739:
        -: 1740:#define MPIR_BARRIER_TAG 1
        -: 1741:/* ------------------------------------------------------------------------- */
        -: 1742:/* end of mpicoll.h (in src/mpi/coll? */
        -: 1743:/* ------------------------------------------------------------------------- */
        -: 1744:
        -: 1745:/* ------------------------------------------------------------------------- */
        -: 1746:/* mpitopo.h (in src/mpi/topo? */
        -: 1747:/*
        -: 1748: * The following struture allows the device detailed control over the 
        -: 1749: * functions that are used to implement the topology routines.  If either
        -: 1750: * the pointer to this structure is null or any individual entry is null,
        -: 1751: * the default function is used (this follows exactly the same rules as the
        -: 1752: * collective operations, provided in the MPID_Collops structure).
        -: 1753: */
        -: 1754:/* ------------------------------------------------------------------------- */
        -: 1755:
        -: 1756:typedef struct MPID_TopoOps {
        -: 1757:    int (*cartCreate)( const MPID_Comm *, int, const int[], const int [],
        -: 1758:		       int, MPI_Comm * );
        -: 1759:    int (*cartMap)   ( const MPID_Comm *, int, const int[], const int [], 
        -: 1760:		       int * );
        -: 1761:    int (*graphCreate)( const MPID_Comm *, int, const int[], const int [],
        -: 1762:			int, MPI_Comm * );
        -: 1763:    int (*graphMap)   ( const MPID_Comm *, int, const int[], const int[], 
        -: 1764:			int * );
        -: 1765:} MPID_TopoOps;
        -: 1766:/* ------------------------------------------------------------------------- */
        -: 1767:/* end of mpitopo.h (in src/mpi/topo? */
        -: 1768:/* ------------------------------------------------------------------------- */
        -: 1769:
        -: 1770:/* Time stamps */
        -: 1771:/* Get the timer definitions.  The source file for this include is
        -: 1772:   src/mpi/timer/mpichtimer.h.in */
        -: 1773:#include "mpichtimer.h"
        -: 1774:
        -: 1775:typedef struct MPID_Stateinfo_t {
        -: 1776:    MPID_Time_t stamp;
        -: 1777:    int count;
        -: 1778:} MPID_Stateinfo_t;
        -: 1779:#define MPICH_MAX_STATES 512
        -: 1780:/* Timer state routines (src/util/instrm/states.c) */
        -: 1781:void MPID_TimerStateBegin( int, MPID_Time_t * );
        -: 1782:void MPID_TimerStateEnd( int, MPID_Time_t * );
        -: 1783:
        -: 1784:/* ------------------------------------------------------------------------- */
        -: 1785:/* Thread types */
        -: 1786:/* Temporary; this will include "mpichthread.h" eventually */
        -: 1787:
        -: 1788:#ifdef MPICH_DEBUG_NESTING
        -: 1789:#define MPICH_MAX_NESTFILENAME 256
        -: 1790:typedef struct MPICH_Nestinfo { 
        -: 1791:    char file[MPICH_MAX_NESTFILENAME];
        -: 1792:    int  line;
        -: 1793:} MPICH_Nestinfo_t;
        -: 1794:#define MPICH_MAX_NESTINFO 16
        -: 1795:#endif /* MPICH_DEBUG_NESTING */
        -: 1796:
        -: 1797:typedef struct MPICH_PerThread_t {
        -: 1798:    int              nest_count;   /* For layered MPI implementation */
        -: 1799:    int              op_errno;     /* For errors in predefined MPI_Ops */
        -: 1800:#ifdef MPICH_DEBUG_NESTING
        -: 1801:    MPICH_Nestinfo_t nestinfo[MPICH_MAX_NESTINFO];
        -: 1802:#endif
        -: 1803:    /* FIXME: Is this used anywhere? */
        -: 1804:#ifdef HAVE_TIMING
        -: 1805:    MPID_Stateinfo_t timestamps[MPICH_MAX_STATES];  /* per thread state info */
        -: 1806:#endif
        -: 1807:#if defined(MPID_DEV_PERTHREAD_DECL)
        -: 1808:    MPID_DEV_PERTHREAD_DECL
        -: 1809:#endif    
        -: 1810:} MPICH_PerThread_t;
        -: 1811:
        -: 1812:#if !defined(MPICH_IS_THREADED)
        -: 1813:/* If single threaded, make this point at a pre-allocated segment.
        -: 1814:   This structure is allocated in src/mpi/init/initthread.c */
        -: 1815:extern MPICH_PerThread_t MPIR_Thread;
        -: 1816:
        -: 1817:/* The following three macros define a way to portably access thread-private
        -: 1818:   storage in MPICH2, and avoid extra overhead when MPICH2 is single 
        -: 1819:   threaded
        -: 1820:   INITKEY - Create the key.  Must happen *before* the other threads 
        -: 1821:             are created
        -: 1822:   INIT    - Create the thread-private storage.  Must happen once per thread
        -: 1823:   DECL    - Declare local variables
        -: 1824:   GET     - Access the thread-private storage
        -: 1825:   FIELD   - Access the thread-private field (by name)
        -: 1826:
        -: 1827:   The "DECL" is the extern so that there is always a statement for
        -: 1828:   the declaration.
        -: 1829:*/
        -: 1830:#define MPIU_THREADPRIV_INITKEY
        -: 1831:#define MPIU_THREADPRIV_INIT 
        -: 1832:/* Empty declarations are not allowed in C. However multiple decls are allowed */
        -: 1833:#define MPIU_THREADPRIV_DECL extern MPICH_PerThread_t MPIR_Thread
        -: 1834:#define MPIU_THREADPRIV_GET
        -: 1835:#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread._a)
        -: 1836:
        -: 1837:#elif  defined(HAVE_RUNTIME_THREADCHECK)
        -: 1838:/* In the case where the thread level is set in MPI_Init_thread, we
        -: 1839:   need a blended version of the non-threaded and the thread-multiple
        -: 1840:   definitions.
        -: 1841:   
        -: 1842:   The approach is to have TWO MPICH_PerThread_t pointers.  One is local
        -: 1843:   (The MPIU_THREADPRIV_DECL is used in the routines local definitions), 
        -: 1844:   as in the threaded version of these macros.  This is set by using a routine
        -: 1845:   to get thread-private storage.  The second is a preallocated, extern 
        -: 1846:   MPICH_PerThread_t struct, as in the single threaded case.  Based on
        -: 1847:   MPIR_Process.isThreaded, one or the other is used.
        -: 1848:   
        -: 1849: */
        -: 1850:/* For the single threaded case, we use a preallocated structure 
        -: 1851:   This structure is allocated in src/mpi/init/initthread.c */
        -: 1852:extern MPICH_PerThread_t MPIR_ThreadSingle;
        -: 1853:
        -: 1854:/* We need to provide a function that will cleanup the storage attached
        -: 1855:   to the key.  */
        -: 1856:#define MPIU_THREADPRIV_INITKEY  \
        -: 1857:    {if (MPIR_Process.isThreaded) {\
        -: 1858:	    MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_Process.thread_storage,NULL);}}
        -: 1859:#define MPIU_THREADPRIV_INIT {if (MPIR_Process.isThreaded) {\
        -: 1860:	MPICH_PerThread_t *(pt_) = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t));	\
        -: 1861:	MPID_Thread_tls_set(&MPIR_Process.thread_storage, (void *) (pt_)); \
        -: 1862:        }}
        -: 1863:#define MPIU_THREADPRIV_DECL \
        -: 1864:    MPICH_PerThread_t *MPIR_Thread=0
        -: 1865:#define MPIU_THREADPRIV_GET  \
        -: 1866:    {if (!MPIR_Thread){MPIR_GetPerThread( &MPIR_Thread );}}
        -: 1867:#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread->_a)
        -: 1868:
        -: 1869:#else /* Thread multiple */
        -: 1870:/* The following three macros define a way to portably access thread-private
        -: 1871:   storage in MPICH2, and avoid extra overhead when MPICH2 is single 
        -: 1872:   threaded.  We initialize the MPIR_Thread pointer to null so that
        -: 1873:   we need call the routine to get the thread-private storage only once
        -: 1874:   in an invocation of a routine.  */
        -: 1875:
        -: 1876:#define MPIU_THREADPRIV_INITKEY  \
        -: 1877:    MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_Process.thread_storage,NULL)
        -: 1878:#define MPIU_THREADPRIV_INIT {\
        -: 1879:	MPICH_PerThread_t *(pt_) = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t));	\
        -: 1880:	MPID_Thread_tls_set(&MPIR_Process.thread_storage, (void *) (pt_)); \
        -: 1881:        }
        -: 1882:#define MPIU_THREADPRIV_DECL MPICH_PerThread_t *MPIR_Thread=0
        -: 1883:#define MPIU_THREADPRIV_GET {if (!MPIR_Thread)MPIR_GetPerThread( &MPIR_Thread );}
        -: 1884:#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread->_a)
        -: 1885:#endif
        -: 1886:
        -: 1887:/* Per process data */
        -: 1888:typedef enum MPIR_MPI_State_t { MPICH_PRE_INIT=0, MPICH_WITHIN_MPI=1,
        -: 1889:               MPICH_POST_FINALIZED=2 } MPIR_MPI_State_t;
        -: 1890:
        -: 1891:typedef struct PreDefined_attrs {
        -: 1892:    int appnum;          /* Application number provided by mpiexec (MPI-2) */
        -: 1893:    int host;            /* host */
        -: 1894:    int io;              /* standard io allowed */
        -: 1895:    int lastusedcode;    /* last used error code (MPI-2) */
        -: 1896:    int tag_ub;          /* Maximum message tag */
        -: 1897:    int universe;        /* Universe size from mpiexec (MPI-2) */
        -: 1898:    int wtime_is_global; /* Wtime is global over processes in COMM_WORLD */
        -: 1899:} PreDefined_attrs;
        -: 1900:
        -: 1901:struct MPID_Datatype;
        -: 1902:
        -: 1903:typedef struct MPICH_PerProcess_t {
        -: 1904:    MPIR_MPI_State_t  initialized;      /* Is MPI initalized? */
        -: 1905:    int               do_error_checks;  /* runtime error check control */
        -: 1906:    struct MPID_Comm  *comm_world;      /* Easy access to comm_world for
        -: 1907:                                           error handler */
        -: 1908:    struct MPID_Comm  *comm_self;       /* Easy access to comm_self */
        -: 1909:    struct MPID_Comm  *comm_parent;     /* Easy access to comm_parent */
        -: 1910:    struct MPID_Comm  *icomm_world;     /* An internal version of comm_world
        -: 1911:					   that is separate from user's 
        -: 1912:					   versions */
        -: 1913:    PreDefined_attrs  attrs;            /* Predefined attribute values */
        -: 1914:
        -: 1915:    /* The topology routines dimsCreate is independent of any communicator.
        -: 1916:       If this pointer is null, the default routine is used */
        -: 1917:    int (*dimsCreate)( int, int, int *);
        -: 1918:
        -: 1919:    /* Attribute dup functions.  Here for lazy initialization */
        -: 1920:    int (*attr_dup)( int, MPID_Attribute *, MPID_Attribute ** );
        -: 1921:    int (*attr_free)( int, MPID_Attribute ** );
        -: 1922:    /* There is no win_attr_dup function because there can be no MPI_Win_dup
        -: 1923:       function */
        -: 1924:    /* Routine to get the messages corresponding to dynamically created
        -: 1925:       error messages */
        -: 1926:    const char *(*errcode_to_string)( int );
        -: 1927:#ifdef HAVE_CXX_BINDING
        -: 1928:    /* Routines to call C++ functions from the C implementation of the
        -: 1929:       MPI reduction and attribute routines */
        -: 1930:    void (*cxx_call_op_fn)( void *, void *, int, MPI_Datatype, 
        -: 1931:			    MPI_User_function * );
        -: 1932:    /* Error handling functions.  As for the attribute functions,
        -: 1933:       we pass the integer file/comm/win, the address of the error code, 
        -: 1934:       and the C function to call (itself a function defined by the
        -: 1935:       C++ interface and exported to C).  The first argument is used
        -: 1936:       to specify the kind (comm,file,win) */
        -: 1937:    void  (*cxx_call_errfn) ( int, int *, int *, void (*)(void) );
        -: 1938:#endif /* HAVE_CXX_BINDING */
        -: 1939:} MPICH_PerProcess_t;
        -: 1940:extern MPICH_PerProcess_t MPIR_Process;
        -: 1941:
        -: 1942:/*D
        -: 1943:  MPICH_THREAD_LEVEL - Indicates the maximum level of thread
        -: 1944:  support provided at compile time.
        -: 1945: 
        -: 1946:  Values:
        -: 1947:  Any of the 'MPI_THREAD_xxx' values (these are preprocessor-time constants)
        -: 1948:
        -: 1949:  Notes:
        -: 1950:  The macro 'MPICH_THREAD_LEVEL' defines the maximum level of
        -: 1951:  thread support provided, and may be used at compile time to remove
        -: 1952:  thread locks and other code needed only in a multithreaded environment.
        -: 1953:
        -: 1954:  A typical use is 
        -: 1955:.vb
        -: 1956:  #ifdef MPICH_IS_THREADED
        -: 1957:     lock((r)->lock_ptr);
        -: 1958:     (r)->ref_count++;
        -: 1959:     unlock((r)->lock_ptr);
        -: 1960:  #else
        -: 1961:     (r)->ref_count ++;
        -: 1962:  #fi
        -: 1963:.ve
        -: 1964:
        -: 1965:  Note that 'MPICH_IS_THREADED' is defined as 1 if 
        -: 1966:.vb
        -: 1967:  MPICH_THREAD_LEVEL >= MPI_THREAD_MULTIPLE
        -: 1968:.ve
        -: 1969:  is true.  The test should be used only for special cases (such as 
        -: 1970:  handling 'SERIALIZED').
        -: 1971:
        -: 1972:  Module:
        -: 1973:  Environment-DS
        -: 1974:  D*/
        -: 1975:
        -: 1976:/* ------------------------------------------------------------------------- */
        -: 1977:/* In MPICH2, each function has an "enter" and "exit" macro.  These can be 
        -: 1978: * used to add various features to each function at compile time, or they
        -: 1979: * can be set to empty to provide the fastest possible production version.
        -: 1980: *
        -: 1981: * There are at this time three choices of features (beyond the empty choice)
        -: 1982: * 1. timing (controlled by macros in mpitimerimpl.h)
        -: 1983: *    These collect data on when each function began and finished; the
        -: 1984: *    resulting data can be displayed using special programs
        -: 1985: * 2. nesting (selected with --enable-g=nesting)
        -: 1986: *    Checks that the "NMPI" functions and the Nest_incr/decr calls
        -: 1987: *    are properly nested at runtime.  
        -: 1988: * 3. Debug logging (selected with --enable-g=log)
        -: 1989: *    Invokes MPIU_DBG_MSG at the entry and exit for each routine            
        -: 1990: * 4. Additional memory validation of the memory arena (--enable-g=memarena)
        -: 1991: */
        -: 1992:/* ------------------------------------------------------------------------- */
        -: 1993:/* if fine-grain nest testing is enabled then define the function enter/exit
        -: 1994:   macros to track the nesting level; otherwise, allow the timing module the
        -: 1995:   opportunity to define the macros */
        -: 1996:#if defined(MPICH_DEBUG_FINE_GRAIN_NESTING)
        -: 1997:#   include "mpiu_func_nesting.h"
        -: 1998:#elif defined(MPICH_DEBUG_MEMARENA)
        -: 1999:#   include "mpifuncmem.h"
        -: 2000:#elif defined(USE_DBG_LOGGING)
        -: 2001:#   include "mpifunclog.h"
        -: 2002:#elif !defined(NEEDS_FUNC_ENTER_EXIT_DEFS)
        -: 2003:    /* If no timing choice is selected, this sets the entry/exit macros 
        -: 2004:       to empty */
        -: 2005:#   include "mpitimerimpl.h"
        -: 2006:#endif
        -: 2007:
        -: 2008:#ifdef NEEDS_FUNC_ENTER_EXIT_DEFS
        -: 2009:/* mpich layer definitions */
        -: 2010:#define MPID_MPI_FUNC_ENTER(a)			MPIR_FUNC_ENTER(a)
        -: 2011:#define MPID_MPI_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2012:#define MPID_MPI_PT2PT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2013:#define MPID_MPI_PT2PT_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2014:#define MPID_MPI_PT2PT_FUNC_ENTER_FRONT(a)	MPIR_FUNC_ENTER(a)
        -: 2015:#define MPID_MPI_PT2PT_FUNC_EXIT_FRONT(a)	MPIR_FUNC_EXIT(a)
        -: 2016:#define MPID_MPI_PT2PT_FUNC_ENTER_BACK(a)	MPIR_FUNC_ENTER(a)
        -: 2017:#define MPID_MPI_PT2PT_FUNC_ENTER_BOTH(a)	MPIR_FUNC_ENTER(a)
        -: 2018:#define MPID_MPI_PT2PT_FUNC_EXIT_BACK(a)	MPIR_FUNC_EXIT(a)
        -: 2019:#define MPID_MPI_PT2PT_FUNC_EXIT_BOTH(a)	MPIR_FUNC_EXIT(a)
        -: 2020:#define MPID_MPI_COLL_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2021:#define MPID_MPI_COLL_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2022:#define MPID_MPI_RMA_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2023:#define MPID_MPI_RMA_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2024:#define MPID_MPI_INIT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2025:#define MPID_MPI_INIT_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2026:#define MPID_MPI_FINALIZE_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2027:#define MPID_MPI_FINALIZE_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2028:
        -: 2029:/* device layer definitions */
        -: 2030:#define MPIDI_FUNC_ENTER(a)			MPIR_FUNC_ENTER(a)
        -: 2031:#define MPIDI_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2032:#define MPIDI_PT2PT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2033:#define MPIDI_PT2PT_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2034:#define MPIDI_PT2PT_FUNC_ENTER_FRONT(a)		MPIR_FUNC_ENTER(a)
        -: 2035:#define MPIDI_PT2PT_FUNC_EXIT_FRONT(a)		MPIR_FUNC_EXIT(a)
        -: 2036:#define MPIDI_PT2PT_FUNC_ENTER_BACK(a)		MPIR_FUNC_ENTER(a)
        -: 2037:#define MPIDI_PT2PT_FUNC_ENTER_BOTH(a)		MPIR_FUNC_ENTER(a)
        -: 2038:#define MPIDI_PT2PT_FUNC_EXIT_BACK(a)		MPIR_FUNC_EXIT(a)
        -: 2039:#define MPIDI_PT2PT_FUNC_EXIT_BOTH(a)		MPIR_FUNC_EXIT(a)
        -: 2040:#define MPIDI_COLL_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2041:#define MPIDI_COLL_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2042:#define MPIDI_RMA_FUNC_ENTER(a)			MPIR_FUNC_ENTER(a)
        -: 2043:#define MPIDI_RMA_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2044:#define MPIDI_INIT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2045:#define MPIDI_INIT_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2046:#define MPIDI_FINALIZE_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2047:#define MPIDI_FINALIZE_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2048:
        -: 2049:/* evaporate the timing macros since timing is not selected */
        -: 2050:#define MPIU_Timer_init(rank, size)
        -: 2051:#define MPIU_Timer_finalize()
        -: 2052:#endif /* NEEDS_FUNC_ENTER_EXIT_DEFS */
        -: 2053:
        -: 2054:/* Definitions for error handling and reporting */
        -: 2055:#include "mpierror.h"
        -: 2056:#include "mpierrs.h"
        -: 2057:
        -: 2058:/* FIXME: This routine is only used within mpi/src/err/errutil.c and 
        -: 2059:   smpd.  We may not want to export it.  */
        -: 2060:void MPIR_Err_print_stack(FILE *, int);
        -: 2061:
        -: 2062:
        -: 2063:/* ------------------------------------------------------------------------- */
        -: 2064:/* XXX DJG FIXME delete this? */
        -: 2065:/* FIXME: Merge these with the object refcount update routines (perhaps as
        -: 2066:   part of a general "atomic update" file */
        -: 2067:/*
        -: 2068: * Standardized general-purpose atomic update routines.  Some comments:
        -: 2069: * Setmax atomically implements *a_ptr = max(b,*a_ptr) .  This can
        -: 2070: * be implemented using compare-and-swap (form max, if new max is 
        -: 2071: * larger, compare-and-swap against old max.  if failure, restart).
        -: 2072: * Fetch_and_increment can be implemented in a similar way; for
        -: 2073: * example, in IA32, 
        -: 2074: * loop:
        -: 2075: *   mov eax, valptr
        -: 2076: *   mov oldvalptr, eax
        -: 2077: *   mov ebx, eax
        -: 2078: *   inc ebx
        -: 2079: *   lock: cmpxchg valptr, ebx
        -: 2080: *   jnz loop
        -: 2081: *
        -: 2082: * Implementations using LoadLink/StoreConditional are similar.
        -: 2083: *
        -: 2084: * Question: can we use the simple code for MPI_THREAD_SERIALIZED?
        -: 2085: * If not, do we want a separate set of definitions that can be used
        -: 2086: * in the code where serialized is ok.
        -: 2087: *
        -: 2088: * Currently, these are used only in the routines to create new error classes
        -: 2089: * and codes (src/mpi/errhan/dynerrutil.c and add_error_class.c).  
        -: 2090: * Note that MPI object reference counts are handled with their own routines.
        -: 2091: *
        -: 2092: * Because of the current use of these routines is within the SINGLE_CS
        -: 2093: * thread lock (for the THREAD_MULTIPLE case), we currently
        -: 2094: * do *not* include a separate Critical section for these operations.
        -: 2095: *
        -: 2096: */
        -: 2097:#define MPIR_Setmax(a_ptr,b) if (b>*(a_ptr)) { *(a_ptr) = b; }
        -: 2098:#define MPIR_Fetch_and_increment(count_ptr,value_ptr) \
        -: 2099:    { *value_ptr = *count_ptr; *count_ptr += 1; }
        -: 2100:
        -: 2101:/* ------------------------------------------------------------------------- */
        -: 2102:
        -: 2103:/* FIXME: Move these to the communicator block; make sure that all 
        -: 2104:   objects have such hooks */
        -: 2105:#ifndef HAVE_DEV_COMM_HOOK
        -: 2106:#define MPID_Dev_comm_create_hook( a )
        -: 2107:#define MPID_Dev_comm_destroy_hook( a )
        -: 2108:#endif
        -: 2109:
        -: 2110:/* ------------------------------------------------------------------------- */
        -: 2111:/* FIXME: What is the scope of these functions?  Can they be moved into
        -: 2112:   src/mpi/pt2pt? */
        -: 2113:/* ------------------------------------------------------------------------- */
        -: 2114:
        -: 2115:/* Do not set MPI_ERROR (only set if ERR_IN_STATUS is returned */
        -: 2116:#define MPIR_Status_set_empty(status_)			\
        -: 2117:{							\
        -: 2118:    if ((status_) != MPI_STATUS_IGNORE)			\
        -: 2119:    {							\
        -: 2120:	(status_)->MPI_SOURCE = MPI_ANY_SOURCE;		\
        -: 2121:	(status_)->MPI_TAG = MPI_ANY_TAG;		\
        -: 2122:	(status_)->count = 0;				\
        -: 2123:	(status_)->cancelled = FALSE;			\
        -: 2124:    }							\
        -: 2125:}
        -: 2126:/* See MPI 1.1, section 3.11, Null Processes */
        -: 2127:/* Do not set MPI_ERROR (only set if ERR_IN_STATUS is returned */
        -: 2128:#define MPIR_Status_set_procnull(status_)		\
        -: 2129:{							\
        -: 2130:    if ((status_) != MPI_STATUS_IGNORE)			\
        -: 2131:    {							\
        -: 2132:	(status_)->MPI_SOURCE = MPI_PROC_NULL;		\
        -: 2133:	(status_)->MPI_TAG = MPI_ANY_TAG;		\
        -: 2134:	(status_)->count = 0;				\
        -: 2135:	(status_)->cancelled = FALSE;			\
        -: 2136:    }							\
        -: 2137:}
        -: 2138:
        -: 2139:#define MPIR_Request_extract_status(request_ptr_, status_)								\
        -: 2140:{															\
        -: 2141:    if ((status_) != MPI_STATUS_IGNORE)											\
        -: 2142:    {															\
        -: 2143:	int error__;													\
        -: 2144:															\
        -: 2145:	/* According to the MPI 1.1 standard page 22 lines 9-12, the MPI_ERROR field may not be modified except by the	\
        -: 2146:	   functions in section 3.7.5 which return MPI_ERR_IN_STATUSES (MPI_Wait{all,some} and MPI_Test{all,some}). */	\
        -: 2147:	error__ = (status_)->MPI_ERROR;											\
        -: 2148:	*(status_) = (request_ptr_)->status;										\
        -: 2149:	(status_)->MPI_ERROR = error__;											\
        -: 2150:    }															\
        -: 2151:}
        -: 2152:/* ------------------------------------------------------------------------- */
        -: 2153:
        -: 2154:/* FIXME: The bindings should be divided into three groups:
        -: 2155:   1. ADI3 routines.  These should have structure comment documentation, e.g., 
        -: 2156:   the text from doc/adi3/adi3.c
        -: 2157:   2. General utility routines.  These should have a short description
        -: 2158:   3. Local utility routines, e.g., routines used within a single subdirectory.
        -: 2159:   These should be moved into an include file in that subdirectory 
        -: 2160:*/
        -: 2161:/* Bindings for internal routines */
        -: 2162:/*@ MPIR_Add_finalize - Add a routine to be called when MPI_Finalize is invoked
        -: 2163:
        -: 2164:+ routine - Routine to call
        -: 2165:. extra   - Void pointer to data to pass to the routine
        -: 2166:- priority - Indicates the priority of this callback and controls the order
        -: 2167:  in which callbacks are executed.  Use a priority of zero for most handlers;
        -: 2168:  higher priorities will be executed first.
        -: 2169:
        -: 2170:Notes:
        -: 2171:  The routine 'MPID_Finalize' is executed with priority 
        -: 2172:  'MPIR_FINALIZE_CALLBACK_PRIO' (currently defined as 5).  Handlers with
        -: 2173:  a higher priority execute before 'MPID_Finalize' is called; those with
        -: 2174:  a lower priority after 'MPID_Finalize' is called.  
        -: 2175:@*/
        -: 2176:void MPIR_Add_finalize( int (*routine)( void * ), void *extra, int priority );
        -: 2177:
        -: 2178:#define MPIR_FINALIZE_CALLBACK_PRIO 5
        -: 2179:#define MPIR_FINALIZE_CALLBACK_HANDLE_CHECK_PRIO 1
        -: 2180:#define MPIR_FINALIZE_CALLBACK_DEFAULT_PRIO 0
        -: 2181:#define MPIR_FINALIZE_CALLBACK_MAX_PRIO 10
        -: 2182:
        -: 2183:/* For no error checking, we could define MPIR_Nest_incr/decr as empty */
        -: 2184:
        -: 2185:/* These routines export the nesting controls for use in ROMIO */
        -: 2186:void MPIR_Nest_incr_export(void);
        -: 2187:void MPIR_Nest_decr_export(void);
        -: 2188:
        -: 2189:#ifdef MPICH_DEBUG_NESTING
        -: 2190:/* These two routines export the versions of the nest macros that
        -: 2191:   provide the file/line where the nest value changes, also for use in ROMIO */
        -: 2192:void MPIR_Nest_incr_export_dbg(const char *, int);
        -: 2193:void MPIR_Nest_decr_export_dbg(const char *, int);
        -: 2194:
        -: 2195:/* FIXME: We should move the initialization and error reporting into
        -: 2196:   routines that can be called when necessary */
        -: 2197:#define MPIR_Nest_init() {\
        -: 2198:   int _i;\
        -: 2199:   for (_i=0;_i<MPICH_MAX_NESTINFO;_i++) {\
        -: 2200:      MPIU_THREADPRIV_FIELD(nestinfo)[_i].file[0] = 0;\
        -: 2201:      MPIU_THREADPRIV_FIELD(nestinfo)[_i].line = 0;}}
        -: 2202:#define MPIR_Nest_incr() {\
        -: 2203:     if (MPIU_THREADPRIV_FIELD(nest_count) >= MPICH_MAX_NESTINFO) {\
        -: 2204:     MPIU_Internal_error_printf("nest stack exceeded at %s:%d\n",\
        -: 2205:          __FILE__,__LINE__);\
        -: 2206:     }else{\
        -: 2207:     MPIU_Strncpy(MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,__FILE__,\
        -: 2208:                  MPICH_MAX_NESTFILENAME);\
        -: 2209:     MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line=__LINE__;}\
        -: 2210:     MPIU_THREADPRIV_FIELD(nest_count)++; }
        -: 2211:/* Set the line for the current entry to - the old line - this can help
        -: 2212:   identify increments that did not set the fields */
        -: 2213:#define MPIR_Nest_decr() {\
        -: 2214:    if (MPIU_THREADPRIV_FIELD(nest_count) >= 0) {\
        -: 2215:	MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line=-__LINE__;} \
        -: 2216:     MPIU_THREADPRIV_FIELD(nest_count)--; \
        -: 2217:     if (MPIU_THREADPRIV_FIELD(nest_count) < MPICH_MAX_NESTINFO && \
        -: 2218:    strcmp(MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,__FILE__) != 0) {\
        -: 2219:         MPIU_Msg_printf( "Decremented nest count in file %s:%d but incremented in different file (%s:%d)\n",\
        -: 2220:                          __FILE__,__LINE__,\
        -: 2221:                          MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,\
        -: 2222:                          MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line);\
        -: 2223:     }else if (MPIU_THREADPRIV_FIELD(nest_count) < 0){\
        -: 2224:	 MPIU_Msg_printf("Decremented nest count in file %s:%d is negative\n",\
        -: 2225:			 __FILE__,__LINE__);}\
        -: 2226:}
        -: 2227:#else
        -: 2228:#define MPIR_Nest_init()
        -: 2229:#define MPIR_Nest_incr() {MPIU_THREADPRIV_FIELD(nest_count)++;}
        -: 2230:#define MPIR_Nest_decr() {MPIU_THREADPRIV_FIELD(nest_count)--;}
        -: 2231:#endif /* MPICH_DEBUG_NESTING */
        -: 2232:
        -: 2233:#define MPIR_Nest_value() (MPIU_THREADPRIV_FIELD(nest_count))
        -: 2234:
        -: 2235:
        -: 2236:/*int MPIR_Comm_attr_dup(MPID_Comm *, MPID_Attribute **);
        -: 2237:  int MPIR_Comm_attr_delete(MPID_Comm *, MPID_Attribute *);*/
        -: 2238:int MPIR_Comm_copy( MPID_Comm *, int, MPID_Comm ** );
        -: 2239:/* Fortran keyvals are set with functions in mpi_f77interface.h */
        -: 2240:#ifdef HAVE_CXX_BINDING
        -: 2241:extern void MPIR_Keyval_set_cxx( int, void (*)(void), void (*)(void) );
        -: 2242:extern void MPIR_Op_set_cxx( MPI_Op, void (*)(void) );
        -: 2243:extern void MPIR_Errhandler_set_cxx( MPI_Errhandler, void (*)(void) );
        -: 2244:#endif
        -: 2245:
        -: 2246:int MPIR_Group_create( int, MPID_Group ** );
        -: 2247:int MPIR_Group_release(MPID_Group *group_ptr);
        -: 2248:
        -: 2249:int MPIR_dup_fn ( MPI_Comm, int, void *, void *, void *, int * );
        -: 2250:int MPIR_Request_complete(MPI_Request *, MPID_Request *, MPI_Status *, int *);
        -: 2251:int MPIR_Request_get_error(MPID_Request *);
        -: 2252:
        -: 2253:/* The following routines perform the callouts to the user routines registered
        -: 2254:   as part of a generalized request.  They handle any language binding issues
        -: 2255:   that are necessary. They are used when completing, freeing, cancelling or
        -: 2256:   extracting the status from a generalized request. */
        -: 2257:int MPIR_Grequest_cancel(MPID_Request * request_ptr, int complete);
        -: 2258:int MPIR_Grequest_query(MPID_Request * request_ptr);
        -: 2259:int MPIR_Grequest_free(MPID_Request * request_ptr);
        -: 2260:
        -: 2261:/* this routine was added to support our extension relaxing the progress rules
        -: 2262: * for generalized requests */
        -: 2263:int MPIR_Grequest_progress_poke(int count, MPID_Request **request_ptrs, 
        -: 2264:		MPI_Status array_of_statuses[] );
        -: 2265:int MPIR_Grequest_waitall(int count, MPID_Request * const *  request_ptrs);
        -: 2266:
        -: 2267:/* ------------------------------------------------------------------------- */
        -: 2268:/* Prototypes for language-specific routines, such as routines to set
        -: 2269:   Fortran keyval attributes */
        -: 2270:#ifdef HAVE_FORTRAN_BINDING
        -: 2271:#include "mpi_f77interface.h"
        -: 2272:#endif
        -: 2273:
        -: 2274:/* ADI Bindings */
        -: 2275:/*@
        -: 2276:  MPID_Init - Initialize the device
        -: 2277:
        -: 2278:  Input Parameters:
        -: 2279:+ argc_p - Pointer to the argument count
        -: 2280:. argv_p - Pointer to the argument list
        -: 2281:- requested - Requested level of thread support.  Values are the same as
        -: 2282:  for the 'required' argument to 'MPI_Init_thread', except that we define
        -: 2283:  an enum for these values.
        -: 2284:
        -: 2285:  Output Parameters:
        -: 2286:+ provided - Provided level of thread support.  May be less than the 
        -: 2287:  requested level of support.
        -: 2288:. has_args - Set to true if 'argc_p' and 'argv_p' contain the command
        -: 2289:  line arguments.  See below.
        -: 2290:- has_env  - Set to true if the environment of the process has been 
        -: 2291:  set as the user expects.  See below.
        -: 2292:
        -: 2293:  Return value:
        -: 2294:  Returns 'MPI_SUCCESS' on success and an MPI error code on failure.  Failure
        -: 2295:  can happen when, for example, the device is unable  to start or contact the
        -: 2296:  number of processes specified by the 'mpiexec' command.
        -: 2297:
        -: 2298:  Notes:
        -: 2299:  Null arguments for 'argc_p' and 'argv_p' `must` be valid (see MPI-2, section
        -: 2300:  4.2)
        -: 2301:
        -: 2302:  Multi-method devices should initialize each method within this call.
        -: 2303:  They can use environment variables and/or command-line arguments
        -: 2304:  to decide which methods to initialize (but note that they must not
        -: 2305:  `depend` on using command-line arguments).
        -: 2306:
        -: 2307:  This call also initializes all MPID data needed by the device.  This
        -: 2308:  includes the 'MPID_Request's and any other data structures used by 
        -: 2309:  the device.
        -: 2310:
        -: 2311:  The arguments 'has_args' and 'has_env' indicate whether the process was
        -: 2312:  started with command-line arguments or environment variables.  In some
        -: 2313:  cases, only the root process is started with these values; in others, 
        -: 2314:  the startup environment ensures that each process receives the 
        -: 2315:  command-line arguments and environment variables that the user expects. 
        -: 2316:  While the MPI standard makes no requirements that command line arguments or 
        -: 2317:  environment variables are provided to all processes, most users expect a
        -: 2318:  common environment.  These variables allow an MPI implementation (that is
        -: 2319:  based on ADI-3) to provide both of these by making use of MPI communication
        -: 2320:  after 'MPID_Init' is called but before 'MPI_Init' returns to the user, if
        -: 2321:  the process management environment does not provide this service.
        -: 2322:
        -: 2323:
        -: 2324:  This routine is used to implement both 'MPI_Init' and 'MPI_Init_thread'.
        -: 2325:
        -: 2326:  Setting the environment requires a 'setenv' function.  Some
        -: 2327:  systems may not have this.  In that case, the documentation must make 
        -: 2328:  clear that the environment may not be propagated to the generated processes.
        -: 2329:
        -: 2330:  Module:
        -: 2331:  MPID_CORE
        -: 2332:
        -: 2333:  Questions:
        -: 2334:
        -: 2335:  The values for 'has_args' and 'has_env' are boolean.  
        -: 2336:  They could be more specific.  For 
        -: 2337:  example, the value could indicate the rank in 'MPI_COMM_WORLD' of a 
        -: 2338:  process that has the values; the value 'MPI_ANY_SOURCE' (or a '-1') could
        -: 2339:  indicate that the value is available on all processes (including this one).
        -: 2340:  We may want this since otherwise the processes may need to determine whether
        -: 2341:  any process needs the command line.  Another option would be to use positive 
        -: 2342:  values in the same way that the 'color' argument is used in 'MPI_Comm_split';
        -: 2343:  a negative value indicates the member of the processes with that color that 
        -: 2344:  has the values of the command line arguments (or environment).  This allows
        -: 2345:  for non-SPMD programs.
        -: 2346:
        -: 2347:  Do we require that the startup environment (e.g., whatever 'mpiexec' is 
        -: 2348:  using to start processes) is responsible for delivering
        -: 2349:  the command line arguments and environment variables that the user expects?
        -: 2350:  That is, if the user is running an SPMD program, and expects each process
        -: 2351:  to get the same command line argument, who is responsible for this?  
        -: 2352:  The 'has_args' and 'has_env' values are intended to allow the ADI to 
        -: 2353:  handle this while taking advantage of any support that the process 
        -: 2354:  manager framework may provide.
        -: 2355:
        -: 2356:  Alternately, how do we find out from the process management environment
        -: 2357:  whether it took care of the environment or the command line arguments?  
        -: 2358:  Do we need a 'PMI_Env_query' function that can answer these questions
        -: 2359:  dynamically (in case a different process manager is used through the same
        -: 2360:  interface)?
        -: 2361:
        -: 2362:  Can we fix the Fortran command-line arguments?  That is, can we arrange for
        -: 2363:  'iargc' and 'getarg' (and the POSIX equivalents) to return the correct 
        -: 2364:  values?  See, for example, the Absoft implementations of 'getarg'.  
        -: 2365:  We could also contact PGI about the Portland Group compilers, and of 
        -: 2366:  course the 'g77' source code is available.
        -: 2367:  Does each process have the same values for the environment variables 
        -: 2368:  when this routine returns?
        -: 2369:
        -: 2370:  If we don''t require that all processes get the same argument list, 
        -: 2371:  we need to find out if they did anyway so that 'MPI_Init_thread' can
        -: 2372:  fixup the list for the user.  This argues for another return value that
        -: 2373:  flags how much of the environment the 'MPID_Init' routine set up
        -: 2374:  so that the 'MPI_Init_thread' call can provide the rest.  The reason
        -: 2375:  for this is that, even though the MPI standard does not require it, 
        -: 2376:  a user-friendly implementation should, in the SPMD mode, give each
        -: 2377:  process the same environment and argument lists unless the user 
        -: 2378:  explicitly directed otherwise.
        -: 2379:
        -: 2380:  How does this interface to PMI?  Do we need to know anything?  Should
        -: 2381:  this call have an info argument to support PMI?
        -: 2382:
        -: 2383:  The following questions involve how environment variables and command
        -: 2384:  line arguments are used to control the behavior of the implementation. 
        -: 2385:  Many of these values must be determined at the time that 'MPID_Init' 
        -: 2386:  is called.  These all should be considered in the context of the 
        -: 2387:  parameter routines described in the MPICH2 Design Document.
        -: 2388:
        -: 2389:  Are there recommended environment variable names?  For example, in ADI-2,
        -: 2390:  there are many debugging options that are part of the common device.
        -: 2391:  In MPI-2, we can''t require command line arguments, so any such options
        -: 2392:  must also have environment variables.  E.g., 'MPICH_ADI_DEBUG' or
        -: 2393:  'MPICH_ADI_DB'.
        -: 2394:
        -: 2395:  Names that are explicitly prohibited?  For example, do we want to 
        -: 2396:  reserve any names that 'MPI_Init_thread' (as opposed to 'MPID_Init')
        -: 2397:  might use?  
        -: 2398:
        -: 2399:  How does information on command-line arguments and environment variables
        -: 2400:  recognized by the device get added to the documentation?
        -: 2401:
        -: 2402:  What about control for other impact on the environment?  For example,
        -: 2403:  what signals should the device catch (e.g., 'SIGFPE'? 'SIGTRAP'?)?  
        -: 2404:  Which of these should be optional (e.g., ignore or leave signal alone) 
        -: 2405:  or selectable (e.g., port to listen on)?  For example, catching 'SIGTRAP'
        -: 2406:  causes problems for 'gdb', so we''d like to be able to leave 'SIGTRAP' 
        -: 2407:  unchanged in some cases.
        -: 2408:
        -: 2409:  Another environment variable should control whether fault-tolerance is 
        -: 2410:  desired.  If fault-tolerance is selected, then some collective operations 
        -: 2411:  will need to use different algorithms and most fatal errors detected by the 
        -: 2412:  MPI implementation should abort only the affected process, not all processes.
        -: 2413:  @*/
        -: 2414:int MPID_Init( int *argc_p, char ***argv_p, int requested, 
        -: 2415:	       int *provided, int *has_args, int *has_env );
        -: 2416:
        -: 2417:/* was: 
        -: 2418: int MPID_Init( int *argc_p, char ***argv_p, 
        -: 2419:	       int requested, int *provided,
        -: 2420:	       MPID_Comm **parent_comm, int *has_args, int *has_env ); */
        -: 2421:
        -: 2422:/*@ 
        -: 2423:  MPID_InitCompleted - Notify the device that the MPI_Init or MPI_Initthread
        -: 2424:  call has completed setting up MPI
        -: 2425:
        -: 2426: Notes:
        -: 2427: This call allows the device to complete any setup that it wishes to perform
        -: 2428: and for which it needs to access any of the structures (such as 'MPIR_Process')
        -: 2429: that are initialized after 'MPID_Init' is called.  If the device does not need
        -: 2430: any extra operations, then it may provide either an empty function or even
        -: 2431: define this as a macro with the value 'MPI_SUCCESS'.
        -: 2432:  @*/
        -: 2433:int MPID_InitCompleted( void );
        -: 2434:
        -: 2435:/*@
        -: 2436:  MPID_Finalize - Perform the device-specific termination of an MPI job
        -: 2437:
        -: 2438:  Return Value:
        -: 2439:  'MPI_SUCCESS' or a valid MPI error code.  Normally, this routine will
        -: 2440:  return 'MPI_SUCCESS'.  Only in extrordinary circumstances can this
        -: 2441:  routine fail; for example, if some process stops responding during the
        -: 2442:  finalize step.  In this case, 'MPID_Finalize' should return an MPI 
        -: 2443:  error code indicating the reason that it failed.
        -: 2444:
        -: 2445:  Notes:
        -: 2446:
        -: 2447:  Module:
        -: 2448:  MPID_CORE
        -: 2449:
        -: 2450:  Questions:
        -: 2451:  Need to check the MPI-2 requirements on 'MPI_Finalize' with respect to
        -: 2452:  things like which process must remain after 'MPID_Finalize' is called.
        -: 2453:  @*/
        -: 2454:int MPID_Finalize(void);
        -: 2455:/*@
        -: 2456:  MPID_Abort - Abort at least the processes in the specified communicator.
        -: 2457:
        -: 2458:  Input Parameters:
        -: 2459:+ comm        - Communicator of processes to abort
        -: 2460:. mpi_errno   - MPI error code containing the reason for the abort
        -: 2461:. exit_code   - Exit code to return to the calling environment.  See notes.
        -: 2462:- error_msg   - Optional error message
        -: 2463:
        -: 2464:  Return value:
        -: 2465:  'MPI_SUCCESS' or an MPI error code.  Normally, this routine should not 
        -: 2466:  return, since the calling process must be a member of the communicator.  
        -: 2467:  However, under some circumstances, the 'MPID_Abort' might fail; in this 
        -: 2468:  case, returning an error indication is appropriate.
        -: 2469:
        -: 2470:  Notes:
        -: 2471:
        -: 2472:  In a fault-tolerant MPI implementation, this operation should abort `only` 
        -: 2473:  the processes in the specified communicator.  Any communicator that shares
        -: 2474:  processes with the aborted communicator becomes invalid.  For more 
        -: 2475:  details, see (paper not yet written on fault-tolerant MPI).
        -: 2476:
        -: 2477:  In particular, if the communicator is 'MPI_COMM_SELF', only the calling 
        -: 2478:  process should be aborted.
        -: 2479:
        -: 2480:  The 'exit_code' is the exit code that this particular process will 
        -: 2481:  attempt to provide to the 'mpiexec' or other program invocation 
        -: 2482:  environment.  See 'mpiexec' for a discussion of how exit codes from 
        -: 2483:  many processes may be combined.
        -: 2484:
        -: 2485:  If the error_msg field is non-NULL this string will be used as the message
        -: 2486:  with the abort output.  Otherwise, the output message will be base on the
        -: 2487:  error message associated with the mpi_errno.
        -: 2488:
        -: 2489:  An external agent that is aborting processes can invoke this with either
        -: 2490:  'MPI_COMM_WORLD' or 'MPI_COMM_SELF'.  For example, if the process manager
        -: 2491:  wishes to abort a group of processes, it should cause 'MPID_Abort' to 
        -: 2492:  be invoked with 'MPI_COMM_SELF' on each process in the group.
        -: 2493:
        -: 2494:  Question:
        -: 2495:  An alternative design is to provide an 'MPID_Group' instead of a
        -: 2496:  communicator.  This would allow a process manager to ask the ADI 
        -: 2497:  to kill an entire group of processes without needing a communicator.
        -: 2498:  However, the implementation of 'MPID_Abort' will either do this by
        -: 2499:  communicating with other processes or by requesting the process manager
        -: 2500:  to kill the processes.  That brings up this question: should 
        -: 2501:  'MPID_Abort' use 'PMI' to kill processes?  Should it be required to
        -: 2502:  notify the process manager?  What about persistent resources (such 
        -: 2503:  as SYSV segments or forked processes)?  
        -: 2504:
        -: 2505:  This suggests that for any persistent resource, an exit handler be
        -: 2506:  defined.  These would be executed by 'MPID_Abort' or 'MPID_Finalize'.  
        -: 2507:  See the implementation of 'MPI_Finalize' for an example of exit callbacks.
        -: 2508:  In addition, code that registered persistent resources could use persistent
        -: 2509:  storage (i.e., a file) to record that information, allowing cleanup 
        -: 2510:  utilities (such as 'mpiexec') to remove any resources left after the 
        -: 2511:  process exits.
        -: 2512:
        -: 2513:  'MPI_Finalize' requires that attributes on 'MPI_COMM_SELF' be deleted 
        -: 2514:  before anything else happens; this allows libraries to attach end-of-job
        -: 2515:  actions to 'MPI_Finalize'.  It is valuable to have a similar 
        -: 2516:  capability on 'MPI_Abort', with the caveat that 'MPI_Abort' may not 
        -: 2517:  guarantee that the run-on-abort routines were called.  This provides a
        -: 2518:  consistent way for the MPICH implementation to handle freeing any 
        -: 2519:  persistent resources.  However, such callbacks must be limited since
        -: 2520:  communication may not be possible once 'MPI_Abort' is called.  Further,
        -: 2521:  any callbacks must guarantee that they have finite termination.  
        -: 2522:  
        -: 2523:  One possible extension would be to allow `users` to add actions to be 
        -: 2524:  run when 'MPI_Abort' is called, perhaps through a special attribute value
        -: 2525:  applied to 'MPI_COMM_SELF'.  Note that is is incorrect to call the delete 
        -: 2526:  functions for the normal attributes on 'MPI_COMM_SELF' because MPI
        -: 2527:  only specifies that those are run on 'MPI_Finalize' (i.e., normal 
        -: 2528:  termination). 
        -: 2529:
        -: 2530:  Module:
        -: 2531:  MPID_CORE
        -: 2532:  @*/
        -: 2533:
        -: 2534:/* FIXME: the 4th argument isn't part of the original design and isn't documented */
        -: 2535:
        -: 2536:# if 0
        -: 2537:int MPID_Abort( MPID_Comm *comm, int mpi_errno, int exit_code, const char *error_msg );
        -: 2538:#endif
        -: 2539:/* FIXME: Should we turn off this flag and only declare MPID_Abort in mpiutil.h? */
        -: 2540:/* We want to also declare MPID_Abort in mpiutil.h if mpiimpl.h is not used */
        -: 2541:#define HAS_MPID_ABORT_DECL
        -: 2542:
        -: 2543:int MPID_Open_port(MPID_Info *, char *);
        -: 2544:int MPID_Close_port(const char *);
        -: 2545:
        -: 2546:/*@
        -: 2547:   MPID_Comm_accept - MPID entry point for MPI_Comm_accept
        -: 2548:
        -: 2549:   Input Parameters:
        -: 2550:+  port_name - port name
        -: 2551:.  info - info
        -: 2552:.  root - root
        -: 2553:-  comm - communicator
        -: 2554:
        -: 2555:   Output Parameters:
        -: 2556:.  MPI_Comm *newcomm - new communicator
        -: 2557:
        -: 2558:  Return Value:
        -: 2559:  'MPI_SUCCESS' or a valid MPI error code.
        -: 2560:@*/
        -: 2561:int MPID_Comm_accept(char *, MPID_Info *, int, MPID_Comm *, MPID_Comm **);
        -: 2562:
        -: 2563:/*@
        -: 2564:   MPID_Comm_connect - MPID entry point for MPI_Comm_connect
        -: 2565:
        -: 2566:   Input Parameters:
        -: 2567:+  port_name - port name
        -: 2568:.  info - info
        -: 2569:.  root - root
        -: 2570:-  comm - communicator
        -: 2571:
        -: 2572:   Output Parameters:
        -: 2573:.  newcomm_ptr - new intercommunicator
        -: 2574:
        -: 2575:  Return Value:
        -: 2576:  'MPI_SUCCESS' or a valid MPI error code.
        -: 2577:@*/
        -: 2578:int MPID_Comm_connect(const char *, MPID_Info *, int, MPID_Comm *, MPID_Comm **);
        -: 2579:
        -: 2580:int MPID_Comm_disconnect(MPID_Comm *);
        -: 2581:
        -: 2582:int MPID_Comm_spawn_multiple(int, char *[], char* *[], int [], MPID_Info* [],
        -: 2583:                             int, MPID_Comm *, MPID_Comm **, int []);
        -: 2584:
        -: 2585:/*@
        -: 2586:  MPID_Send - MPID entry point for MPI_Send
        -: 2587:
        -: 2588:  Notes:
        -: 2589:  The only difference between this and 'MPI_Send' is that the basic
        -: 2590:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2591:  have been made, the MPI opaque objects have been replaced by
        -: 2592:  MPID objects, a context id offset is provided in addition to the 
        -: 2593:  communicator, and a request may be returned.  The context offset is 
        -: 2594:  added to the context of the communicator
        -: 2595:  to get the context it used by the message.
        -: 2596:  A request is returned only if the ADI implementation was unable to 
        -: 2597:  complete the send of the message.  In that case, the usual 'MPI_Wait'
        -: 2598:  logic should be used to complete the request.  This approach is used to 
        -: 2599:  allow a simple implementation of the ADI.  The ADI is free to always 
        -: 2600:  complete the message and never return a request.
        -: 2601:
        -: 2602:  Module:
        -: 2603:  Communication
        -: 2604:
        -: 2605:  @*/
        -: 2606:int MPID_Send( const void *buf, int count, MPI_Datatype datatype,
        -: 2607:	       int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2608:	       MPID_Request **request );
        -: 2609:
        -: 2610:/*@
        -: 2611:  MPID_Rsend - MPID entry point for MPI_Rsend
        -: 2612:
        -: 2613:  Notes:
        -: 2614:  The only difference between this and 'MPI_Rsend' is that the basic
        -: 2615:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2616:  have been made, the MPI opaque objects have been replaced by
        -: 2617:  MPID objects, a context id offset is provided in addition to the 
        -: 2618:  communicator, and a request may be returned.  The context offset is 
        -: 2619:  added to the context of the communicator
        -: 2620:  to get the context it used by the message.
        -: 2621:  A request is returned only if the ADI implementation was unable to 
        -: 2622:  complete the send of the message.  In that case, the usual 'MPI_Wait'
        -: 2623:  logic should be used to complete the request.  This approach is used to 
        -: 2624:  allow a simple implementation of the ADI.  The ADI is free to always 
        -: 2625:  complete the message and never return a request.
        -: 2626:
        -: 2627:  Module:
        -: 2628:  Communication
        -: 2629:
        -: 2630:  @*/
        -: 2631:int MPID_Rsend( const void *buf, int count, MPI_Datatype datatype,
        -: 2632:		int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2633:		MPID_Request **request );
        -: 2634:
        -: 2635:/*@
        -: 2636:  MPID_Ssend - MPID entry point for MPI_Ssend
        -: 2637:
        -: 2638:  Notes:
        -: 2639:  The only difference between this and 'MPI_Ssend' is that the basic
        -: 2640:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2641:  have been made, the MPI opaque objects have been replaced by
        -: 2642:  MPID objects, a context id offset is provided in addition to the 
        -: 2643:  communicator, and a request may be returned.  The context offset is 
        -: 2644:  added to the context of the communicator
        -: 2645:  to get the context it used by the message.
        -: 2646:  A request is returned only if the ADI implementation was unable to 
        -: 2647:  complete the send of the message.  In that case, the usual 'MPI_Wait'
        -: 2648:  logic should be used to complete the request.  This approach is used to 
        -: 2649:  allow a simple implementation of the ADI.  The ADI is free to always 
        -: 2650:  complete the message and never return a request.
        -: 2651:
        -: 2652:  Module:
        -: 2653:  Communication
        -: 2654:
        -: 2655:  @*/
        -: 2656:int MPID_Ssend( const void *buf, int count, MPI_Datatype datatype,
        -: 2657:		int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2658:		MPID_Request **request );
        -: 2659:
        -: 2660:/*@
        -: 2661:  MPID_tBsend - Attempt a send and return if it would block
        -: 2662:
        -: 2663:  Notes:
        -: 2664:  This has the semantics of 'MPI_Bsend', except that it returns the internal
        -: 2665:  error code 'MPID_WOULD_BLOCK' if the message can''t be sent immediately
        -: 2666:  (t is for "try").  
        -: 2667: 
        -: 2668:  The reason that this interface is chosen over a query to check whether
        -: 2669:  a message `can` be sent is that the query approach is not
        -: 2670:  thread-safe.  Since the decision on whether a message can be sent
        -: 2671:  without blocking depends (among other things) on the state of flow
        -: 2672:  control managed by the device, this approach also gives the device
        -: 2673:  the greatest freedom in implementing flow control.  In particular,
        -: 2674:  if another MPI process can change the flow control parameters, then
        -: 2675:  even in a single-threaded implementation, it would not be safe to
        -: 2676:  return, for example, a message size that could be sent with 'MPI_Bsend'.
        -: 2677:
        -: 2678:  This routine allows an MPI implementation to optimize 'MPI_Bsend'
        -: 2679:  for the case when the message can be delivered without blocking the
        -: 2680:  calling process.  An ADI implementation is free to have this routine
        -: 2681:  always return 'MPID_WOULD_BLOCK', but is encouraged not to.
        -: 2682:
        -: 2683:  To allow the MPI implementation to avoid trying this routine when it
        -: 2684:  is not implemented by the ADI, the C preprocessor constant 'MPID_HAS_TBSEND'
        -: 2685:  should be defined if this routine has a nontrivial implementation.
        -: 2686:
        -: 2687:  This is an optional routine.  The MPI code for 'MPI_Bsend' will attempt
        -: 2688:  to call this routine only if the device defines 'MPID_HAS_TBSEND'.
        -: 2689:
        -: 2690:  Module:
        -: 2691:  Communication
        -: 2692:  @*/
        -: 2693:int MPID_tBsend( const void *buf, int count, MPI_Datatype datatype,
        -: 2694:		 int dest, int tag, MPID_Comm *comm, int context_offset );
        -: 2695:
        -: 2696:/*@
        -: 2697:  MPID_Isend - MPID entry point for MPI_Isend
        -: 2698:
        -: 2699:  Notes:
        -: 2700:  The only difference between this and 'MPI_Isend' is that the basic
        -: 2701:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2702:  have been made, the MPI opaque objects have been replaced by
        -: 2703:  MPID objects, and a context id offset is provided in addition to the 
        -: 2704:  communicator.  This offset is added to the context of the communicator
        -: 2705:  to get the context it used by the message.
        -: 2706:
        -: 2707:  Module:
        -: 2708:  Communication
        -: 2709:
        -: 2710:  @*/
        -: 2711:int MPID_Isend( const void *buf, int count, MPI_Datatype datatype,
        -: 2712:		int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2713:		MPID_Request **request );
        -: 2714:
        -: 2715:/*@
        -: 2716:  MPID_Irsend - MPID entry point for MPI_Irsend
        -: 2717:
        -: 2718:  Notes:
        -: 2719:  The only difference between this and 'MPI_Irsend' is that the basic
        -: 2720:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2721:  have been made, the MPI opaque objects have been replaced by
        -: 2722:  MPID objects, and a context id offset is provided in addition to the 
        -: 2723:  communicator.  This offset is added to the context of the communicator
        -: 2724:  to get the context it used by the message.
        -: 2725:
        -: 2726:  Module:
        -: 2727:  Communication
        -: 2728:
        -: 2729:  @*/
        -: 2730:int MPID_Irsend( const void *buf, int count, MPI_Datatype datatype,
        -: 2731:		 int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2732:		 MPID_Request **request );
        -: 2733:
        -: 2734:/*@
        -: 2735:  MPID_Issend - MPID entry point for MPI_Issend
        -: 2736:
        -: 2737:  Notes:
        -: 2738:  The only difference between this and 'MPI_Issend' is that the basic
        -: 2739:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2740:  have been made, the MPI opaque objects have been replaced by
        -: 2741:  MPID objects, and a context id offset is provided in addition to the 
        -: 2742:  communicator.  This offset is added to the context of the communicator
        -: 2743:  to get the context it used by the message.
        -: 2744:
        -: 2745:  Module:
        -: 2746:  Communication
        -: 2747:
        -: 2748:  @*/
        -: 2749:int MPID_Issend( const void *buf, int count, MPI_Datatype datatype,
        -: 2750:		 int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2751:		 MPID_Request **request );
        -: 2752:
        -: 2753:/*@
        -: 2754:  MPID_Recv - MPID entry point for MPI_Recv
        -: 2755:
        -: 2756:  Notes:
        -: 2757:  The only difference between this and 'MPI_Recv' is that the basic
        -: 2758:  error checks (e.g., valid communicator, datatype, source, and tag)
        -: 2759:  have been made, the MPI opaque objects have been replaced by
        -: 2760:  MPID objects, a context id offset is provided in addition to the 
        -: 2761:  communicator, and a request may be returned.  The context offset is added 
        -: 2762:  to the context of the communicator to get the context it used by the message.
        -: 2763:  As in 'MPID_Send', the request is returned only if the operation did not
        -: 2764:  complete.  Conversely, the status object is populated with valid information
        -: 2765:  only if the operation completed.
        -: 2766:
        -: 2767:  Module:
        -: 2768:  Communication
        -: 2769:
        -: 2770:  @*/
        -: 2771:int MPID_Recv( void *buf, int count, MPI_Datatype datatype,
        -: 2772:	       int source, int tag, MPID_Comm *comm, int context_offset,
        -: 2773:	       MPI_Status *status, MPID_Request **request );
        -: 2774:
        -: 2775:
        -: 2776:/*@
        -: 2777:  MPID_Irecv - MPID entry point for MPI_Irecv
        -: 2778:
        -: 2779:  Notes:
        -: 2780:  The only difference between this and 'MPI_Irecv' is that the basic
        -: 2781:  error checks (e.g., valid communicator, datatype, source, and tag)
        -: 2782:  have been made, the MPI opaque objects have been replaced by
        -: 2783:  MPID objects, and a context id offset is provided in addition to the 
        -: 2784:  communicator.  This offset is added to the context of the communicator
        -: 2785:  to get the context it used by the message.
        -: 2786:
        -: 2787:  Module:
        -: 2788:  Communication
        -: 2789:
        -: 2790:  @*/
        -: 2791:int MPID_Irecv( void *buf, int count, MPI_Datatype datatype,
        -: 2792:		int source, int tag, MPID_Comm *comm, int context_offset,
        -: 2793:		MPID_Request **request );
        -: 2794:
        -: 2795:/*@
        -: 2796:  MPID_Send_init - MPID entry point for MPI_Send_init
        -: 2797:
        -: 2798:  Notes:
        -: 2799:  The only difference between this and 'MPI_Send_init' is that the basic
        -: 2800:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2801:  have been made, the MPI opaque objects have been replaced by
        -: 2802:  MPID objects, and a context id offset is provided in addition to the 
        -: 2803:  communicator.  This offset is added to the context of the communicator
        -: 2804:  to get the context it used by the message.
        -: 2805:
        -: 2806:  Module:
        -: 2807:  Communication
        -: 2808:
        -: 2809:  @*/
        -: 2810:int MPID_Send_init( const void *buf, int count, MPI_Datatype datatype,
        -: 2811:		    int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2812:		    MPID_Request **request );
        -: 2813:
        -: 2814:int MPID_Bsend_init(const void *, int, MPI_Datatype, int, int, MPID_Comm *,
        -: 2815:		   int, MPID_Request **);
        -: 2816:/*@
        -: 2817:  MPID_Rsend_init - MPID entry point for MPI_Rsend_init
        -: 2818:
        -: 2819:  Notes:
        -: 2820:  The only difference between this and 'MPI_Rsend_init' is that the basic
        -: 2821:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2822:  have been made, the MPI opaque objects have been replaced by
        -: 2823:  MPID objects, and a context id offset is provided in addition to the 
        -: 2824:  communicator.  This offset is added to the context of the communicator
        -: 2825:  to get the context it used by the message.
        -: 2826:
        -: 2827:  Module:
        -: 2828:  Communication
        -: 2829:
        -: 2830:  @*/
        -: 2831:int MPID_Rsend_init( const void *buf, int count, MPI_Datatype datatype,
        -: 2832:		     int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2833:		     MPID_Request **request );
        -: 2834:/*@
        -: 2835:  MPID_Ssend_init - MPID entry point for MPI_Ssend_init
        -: 2836:
        -: 2837:  Notes:
        -: 2838:  The only difference between this and 'MPI_Ssend_init' is that the basic
        -: 2839:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2840:  have been made, the MPI opaque objects have been replaced by
        -: 2841:  MPID objects, and a context id offset is provided in addition to the 
        -: 2842:  communicator.  This offset is added to the context of the communicator
        -: 2843:  to get the context it used by the message.
        -: 2844:
        -: 2845:  Module:
        -: 2846:  Communication
        -: 2847:
        -: 2848:  @*/
        -: 2849:int MPID_Ssend_init( const void *buf, int count, MPI_Datatype datatype,
        -: 2850:		     int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2851:		     MPID_Request **request );
        -: 2852:
        -: 2853:/*@
        -: 2854:  MPID_Recv_init - MPID entry point for MPI_Recv_init
        -: 2855:
        -: 2856:  Notes:
        -: 2857:  The only difference between this and 'MPI_Recv_init' is that the basic
        -: 2858:  error checks (e.g., valid communicator, datatype, source, and tag)
        -: 2859:  have been made, the MPI opaque objects have been replaced by
        -: 2860:  MPID objects, and a context id offset is provided in addition to the 
        -: 2861:  communicator.  This offset is added to the context of the communicator
        -: 2862:  to get the context it used by the message.
        -: 2863:
        -: 2864:  Module:
        -: 2865:  Communication
        -: 2866:
        -: 2867:  @*/
        -: 2868:int MPID_Recv_init( void *buf, int count, MPI_Datatype datatype,
        -: 2869:		    int source, int tag, MPID_Comm *comm, int context_offset,
        -: 2870:		    MPID_Request **request );
        -: 2871:
        -: 2872:/*@
        -: 2873:  MPID_Startall - MPID entry point for MPI_Startall
        -: 2874:
        -: 2875:  Notes:
        -: 2876:  The only difference between this and 'MPI_Startall' is that the basic
        -: 2877:  error checks (e.g., count) have been made, and the MPI opaque objects
        -: 2878:  have been replaced by pointers to MPID objects.  
        -: 2879:
        -: 2880:  Rationale:
        -: 2881:  This allows the device to schedule communication involving multiple requests,
        -: 2882:  whereas an implementation built on just 'MPID_Start' would force the
        -: 2883:  ADI to initiate the communication in the order encountered.
        -: 2884:  @*/
        -: 2885:int MPID_Startall(int count, MPID_Request *requests[]);
        -: 2886:
        -: 2887:/*@
        -: 2888:   MPID_Probe - Block until a matching request is found and return information 
        -: 2889:   about it
        -: 2890:
        -: 2891:  Input Parameters:
        -: 2892:+ source - rank to match (or 'MPI_ANY_SOURCE')
        -: 2893:. tag - Tag to match (or 'MPI_ANY_TAG')
        -: 2894:. comm - communicator to match.
        -: 2895:- context_offset - context id offset of communicator to match
        -: 2896:
        -: 2897:  Output Parameter:
        -: 2898:. status - 'MPI_Status' set as defined by 'MPI_Probe'
        -: 2899:
        -: 2900:
        -: 2901:  Return Value:
        -: 2902:  Error code.
        -: 2903:  
        -: 2904:  Notes:
        -: 2905:  Note that the values returned in 'status' will be valid for a subsequent
        -: 2906:  MPI receive operation only if no other thread attempts to receive the same
        -: 2907:  message.  
        -: 2908:  (See the 
        -: 2909:  discussion of probe in Section 8.7.2 Clarifications of the MPI-2 standard.)
        -: 2910:
        -: 2911:  Providing the 'context_offset' is necessary at this level to support the 
        -: 2912:  way in which the MPICH implementation uses context ids in the implementation
        -: 2913:  of other operations.  The communicator is present to allow the device 
        -: 2914:  to use message-queues attached to particular communicators or connections
        -: 2915:  between processes.
        -: 2916:
        -: 2917:  Module:
        -: 2918:  Request
        -: 2919:
        -: 2920:  @*/
        -: 2921:int MPID_Probe(int, int, MPID_Comm *, int, MPI_Status *);
        -: 2922:/*@
        -: 2923:   MPID_Iprobe - Look for a matching request in the receive queue 
        -: 2924:   but do not remove or return it
        -: 2925:
        -: 2926:  Input Parameters:
        -: 2927:+ source - rank to match (or 'MPI_ANY_SOURCE')
        -: 2928:. tag - Tag to match (or 'MPI_ANY_TAG')
        -: 2929:. comm - communicator to match.
        -: 2930:- context_offset - context id offset of communicator to match
        -: 2931:
        -: 2932:  Output Parameter:
        -: 2933:+ flag - true if a matching request was found, false otherwise.
        -: 2934:- status - 'MPI_Status' set as defined by 'MPI_Iprobe' (only valid when return 
        -: 2935:  flag is true).
        -: 2936:
        -: 2937:  Return Value:
        -: 2938:  Error Code.
        -: 2939:
        -: 2940:  Notes:
        -: 2941:  Note that the values returned in 'status' will be valid for a subsequent
        -: 2942:  MPI receive operation only if no other thread attempts to receive the same
        -: 2943:  message.  
        -: 2944:  (See the 
        -: 2945:  discussion of probe in Section 8.7.2 (Clarifications) of the MPI-2 standard.)
        -: 2946:
        -: 2947:  Providing the 'context_offset' is necessary at this level to support the 
        -: 2948:  way in which the MPICH implementation uses context ids in the implementation
        -: 2949:  of other operations.  The communicator is present to allow the device 
        -: 2950:  to use message-queues attached to particular communicators or connections
        -: 2951:  between processes.
        -: 2952:
        -: 2953:  Devices that rely solely on polling to make progress should call
        -: 2954:  MPID_Progress_poke() (or some equivalent function) if a matching request
        -: 2955:  could not be found.  This insures that progress continues to be made even if
        -: 2956:  the application is calling MPI_Iprobe() from within a loop not containing
        -: 2957:  calls to any other MPI functions.
        -: 2958:  
        -: 2959:  Module:
        -: 2960:  Request
        -: 2961:
        -: 2962:  @*/
        -: 2963:int MPID_Iprobe(int, int, MPID_Comm *, int, int *, MPI_Status *);
        -: 2964:
        -: 2965:/*@
        -: 2966:  MPID_Cancel_send - Cancel the indicated send request
        -: 2967:
        -: 2968:  Input Parameter:
        -: 2969:. request - Send request to cancel
        -: 2970:
        -: 2971:  Return Value:
        -: 2972:  MPI error code.
        -: 2973:  
        -: 2974:  Notes:
        -: 2975:  Cancel is a tricky operation, particularly for sends.  Read the
        -: 2976:  discussion in the MPI-1 and MPI-2 documents carefully.  This call
        -: 2977:  only requests that the request be cancelled; a subsequent wait 
        -: 2978:  or test must first succeed (i.e., the request completion counter must be
        -: 2979:  zeroed).
        -: 2980:
        -: 2981:  Module:
        -: 2982:  Request
        -: 2983:
        -: 2984:  @*/
        -: 2985:int MPID_Cancel_send(MPID_Request *);
        -: 2986:/*@
        -: 2987:  MPID_Cancel_recv - Cancel the indicated recv request
        -: 2988:
        -: 2989:  Input Parameter:
        -: 2990:. request - Receive request to cancel
        -: 2991:
        -: 2992:  Return Value:
        -: 2993:  MPI error code.
        -: 2994:  
        -: 2995:  Notes:
        -: 2996:  This cancels a pending receive request.  In many cases, this is implemented
        -: 2997:  by simply removing the request from a pending receive request queue.  
        -: 2998:  However, some ADI implementations may maintain these queues in special 
        -: 2999:  places, such as within a NIC (Network Interface Card).
        -: 3000:  This call only requests that the request be cancelled; a subsequent wait 
        -: 3001:  or test must first succeed (i.e., the request completion counter must be
        -: 3002:  zeroed).
        -: 3003:
        -: 3004:  Module:
        -: 3005:  Request
        -: 3006:
        -: 3007:  @*/
        -: 3008:int MPID_Cancel_recv(MPID_Request *);
        -: 3009:
        -: 3010:int MPID_Win_create(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *,
        -: 3011:                    MPID_Win **);
        -: 3012:int MPID_Win_fence(int, MPID_Win *);
        -: 3013:int MPID_Put(void *, int, MPI_Datatype, int, MPI_Aint, int,
        -: 3014:            MPI_Datatype, MPID_Win *); 
        -: 3015:int MPID_Get(void *, int, MPI_Datatype, int, MPI_Aint, int,
        -: 3016:            MPI_Datatype, MPID_Win *);
        -: 3017:int MPID_Accumulate(void *, int, MPI_Datatype, int, MPI_Aint, int, 
        -: 3018:		   MPI_Datatype, MPI_Op, MPID_Win *);
        -: 3019:int MPID_Win_free(MPID_Win **); 
        -: 3020:int MPID_Win_test(MPID_Win *win_ptr, int *flag);
        -: 3021:int MPID_Win_wait(MPID_Win *win_ptr);
        -: 3022:int MPID_Win_complete(MPID_Win *win_ptr);
        -: 3023:int MPID_Win_post(MPID_Group *group_ptr, int assert, MPID_Win *win_ptr);
        -: 3024:int MPID_Win_start(MPID_Group *group_ptr, int assert, MPID_Win *win_ptr);
        -: 3025:int MPID_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr);
        -: 3026:int MPID_Win_unlock(int dest, MPID_Win *win_ptr);
        -: 3027:
        -: 3028:/*@
        -: 3029:  MPID_Progress_start - Begin a block of operations that check the completion
        -: 3030:  counters in requests.
        -: 3031:
        -: 3032:  Input parameters:
        -: 3033:. state - pointer to a progress state variable
        -: 3034:    
        -: 3035:  Notes:
        -: 3036:  This routine is informs the progress engine that a block of code follows that
        -: 3037:  will examine the completion counter of some 'MPID_Request' objects and then
        -: 3038:  call 'MPID_Progress_wait' zero or more times followed by a call to
        -: 3039:  'MPID_Progress_end'.
        -: 3040:  
        -: 3041:  The progress state variable must be specific to the thread calling it.  If at
        -: 3042:  all possible, the state should be declared as an auto variable and thus
        -: 3043:  allocated on the stack of the current thread.  Thread specific storage could
        -: 3044:  be used instead, but doing such would incur additional (and typically
        -: 3045:  unnecessary) overhead.
        -: 3046:  
        -: 3047:  This routine is needed to properly implement blocking tests when
        -: 3048:  multithreaded progress engines are used.  In a single-threaded implementation
        -: 3049:  of the ADI, this may be defined as an empty macro.
        -: 3050:
        -: 3051:  Module:
        -: 3052:  Communication
        -: 3053:  @*/
        -: 3054:void MPID_Progress_start(MPID_Progress_state * state);
        -: 3055:/*@
        -: 3056:  MPID_Progress_wait - Wait for some communication since 'MPID_Progress_start' 
        -: 3057:
        -: 3058:    Input parameters:
        -: 3059:.   state - pointer to the progress state initialized by MPID_Progress_start
        -: 3060:    
        -: 3061:    Return value:
        -: 3062:    An mpi error code.
        -: 3063:
        -: 3064:    Notes:
        -: 3065:    This instructs the progress engine to wait until some communication event
        -: 3066:    happens since 'MPID_Progress_start' was called.  This call blocks the 
        -: 3067:    calling thread (only, not the process).
        -: 3068:
        -: 3069:  Module:
        -: 3070:  Communication
        -: 3071: @*/
        -: 3072:int MPID_Progress_wait(MPID_Progress_state * state);
        -: 3073:/*@
        -: 3074:  MPID_Progress_end - End a block of operations begun with 'MPID_Progress_start'
        -: 3075:
        -: 3076:  Input parameters:
        -: 3077:  . state - pointer to the progress state variable passed to
        -: 3078:    'MPID_Progress_start'
        -: 3079:
        -: 3080:   Notes:
        -: 3081:   This routine instructs the progress engine to end the block begun with
        -: 3082:   'MPID_Progress_start'.  The progress engine is not required to check for any
        -: 3083:   pending communication.
        -: 3084:
        -: 3085:   The purpose of this call is to release any locks initiated by
        -: 3086:   'MPID_Progess_start' or 'MPID_Progess_wait'.  In a single threaded ADI
        -: 3087:   implementation, this may be defined as an empty macro.
        -: 3088:
        -: 3089:  Module:
        -: 3090:  Communication
        -: 3091:   @*/
        -: 3092:void MPID_Progress_end(MPID_Progress_state * stae);
        -: 3093:/*@
        -: 3094:  MPID_Progress_test - Check for communication
        -: 3095:
        -: 3096:  Return value:
        -: 3097:  An mpi error code.
        -: 3098:  
        -: 3099:  Notes:
        -: 3100:  Unlike 'MPID_Progress_wait', this routine is nonblocking.  Therefore, it
        -: 3101:  does not require the use of 'MPID_Progress_start' and 'MPID_Progress_end'.
        -: 3102:  
        -: 3103:  Module:
        -: 3104:  Communication
        -: 3105:  @*/
        -: 3106:int MPID_Progress_test(void);
        -: 3107:/*@
        -: 3108:  MPID_Progress_poke - Allow a progress engine to check for pending 
        -: 3109:  communication
        -: 3110:
        -: 3111:  Return value:
        -: 3112:  An mpi error code.
        -: 3113:  
        -: 3114:  Notes:
        -: 3115:  This routine provides a way to invoke the progress engine in a polling 
        -: 3116:  implementation of the ADI.  This routine must be nonblocking.
        -: 3117:
        -: 3118:  A multithreaded implementation is free to define this as an empty macro.
        -: 3119:
        -: 3120:  Module:
        -: 3121:  Communication
        -: 3122:  @*/
        -: 3123:int MPID_Progress_poke(void);
        -: 3124:
        -: 3125:/*@
        -: 3126:  MPID_Request_create - Create and return a bare request
        -: 3127:
        -: 3128:  Return value:
        -: 3129:  A pointer to a new request object.
        -: 3130:
        -: 3131:  Notes:
        -: 3132:  This routine is intended for use by 'MPI_Grequest_start' only.  Note that 
        -: 3133:  once a request is created with this routine, any progress engine must assume 
        -: 3134:  that an outside function can complete a request with 
        -: 3135:  'MPID_Request_set_completed'.
        -: 3136:
        -: 3137:  The request object returned by this routine should be initialized such that
        -: 3138:  ref_count is one and handle contains a valid handle referring to the object.
        -: 3139:  @*/
        -: 3140:MPID_Request * MPID_Request_create(void);
        -: 3141:void MPID_Request_set_completed(MPID_Request *);
        -: 3142:/*@
        -: 3143:  MPID_Request_release - Release a request 
        -: 3144:
        -: 3145:  Input Parameter:
        -: 3146:. request - request to release
        -: 3147:
        -: 3148:  Notes:
        -: 3149:  This routine is called to release a reference to request object.  If
        -: 3150:  the reference count of the request object has reached zero, the object will
        -: 3151:  be deallocated.
        -: 3152:
        -: 3153:  Module:
        -: 3154:  Request
        -: 3155:@*/
        -: 3156:void MPID_Request_release(MPID_Request *);
        -: 3157:
        -: 3158:typedef struct MPID_Grequest_class {
        -: 3159:     MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 3160:     MPI_Grequest_query_function *query_fn;
        -: 3161:     MPI_Grequest_free_function *free_fn;
        -: 3162:     MPI_Grequest_cancel_function *cancel_fn;
        -: 3163:     MPIX_Grequest_poll_function *poll_fn;
        -: 3164:     MPIX_Grequest_wait_function *wait_fn;
        -: 3165:     struct MPID_Grequest_class *next;
        -: 3166:} MPID_Grequest_class;
        -: 3167:
        -: 3168:
        -: 3169:/*TTopoOverview.tex
        -: 3170: *
        -: 3171: * The MPI collective and topology routines can benefit from information 
        -: 3172: * about the topology of the underlying interconnect.  Unfortunately, there
        -: 3173: * is no best form for the representation (the MPI-1 Forum tried to define
        -: 3174: * such a representation, but was unable to).  One useful decomposition
        -: 3175: * that has been used in cluster enviroments is a hierarchical decomposition.
        -: 3176: *
        -: 3177: * The other obviously useful topology information would match the needs of 
        -: 3178: * 'MPI_Cart_create'.  However, it may be simpler to for the device to 
        -: 3179: * implement this routine directly.
        -: 3180: *
        -: 3181: * Other useful information could be the topology information that matches
        -: 3182: * the needs of the collective operation, such as spanning trees and rings.
        -: 3183: * These may be added to ADI3 later.
        -: 3184: *
        -: 3185: * Question: Should we define a cart create function?  Dims create?
        -: 3186: *
        -: 3187: * Usage:
        -: 3188: * This routine has nothing to do with the choice of communication method
        -: 3189: * that a implementation of the ADI may make.  It is intended only to
        -: 3190: * communicate information on the heirarchy of processes, if any, to 
        -: 3191: * the implementation of the collective communication routines.  This routine
        -: 3192: * may also be useful for the MPI Graph topology functions.
        -: 3193: *
        -: 3194: T*/
        -: 3195:
        -: 3196:/*@
        -: 3197:  MPID_Topo_cluster_info - Return information on the hierarchy of 
        -: 3198:  interconnections
        -: 3199:
        -: 3200:  Input Parameter:
        -: 3201:. comm - Communicator to study.  May be 'NULL', in which case 'MPI_COMM_WORLD'
        -: 3202:  is the effective communicator.
        -: 3203:
        -: 3204:  Output Parameters:
        -: 3205:+ levels - The number of levels in the hierarchy.  
        -: 3206:  To simplify the use of this routine, the maximum value is 
        -: 3207:  'MPID_TOPO_CLUSTER_MAX_LEVELS' (typically 8 or less).
        -: 3208:. my_cluster - For each level, the id of the cluster that the calling process
        -: 3209:  belongs to.
        -: 3210:- my_rank - For each level, the rank of the calling process in its cluster
        -: 3211:
        -: 3212:  Notes:
        -: 3213:  This routine returns a description of the system in terms of nested 
        -: 3214:  clusters of processes.  Levels are numbered from zero.  At each level,
        -: 3215:  each process may belong to no more than cluster; if a process is in any
        -: 3216:  cluster at level i, it must be in some cluster at level i-1.
        -: 3217:
        -: 3218:  The communicator argument allows this routine to be used in the dynamic
        -: 3219:  process case (i.e., with communicators that are created after 'MPI_Init' 
        -: 3220:  and that involve processes that are not part of 'MPI_COMM_WORLD').
        -: 3221:
        -: 3222:  For non-hierarchical systems, this routine simply returns a single 
        -: 3223:  level containing all processes.
        -: 3224:
        -: 3225:  Sample Outputs:
        -: 3226:  For a single, switch-connected cluster or a uniform-memory-access (UMA)
        -: 3227:  symmetric multiprocessor (SMP), the return values could be
        -: 3228:.vb
        -: 3229:    level       my_cluster         my_rank
        -: 3230:    0           0                  rank in comm_world
        -: 3231:.ve
        -: 3232:  This is also a valid response for `any` device.
        -: 3233:
        -: 3234:  For a switch-connected cluster of 2 processor SMPs
        -: 3235:.vb
        -: 3236:    level       my_cluster         my_rank
        -: 3237:    0           0                  rank in comm_world
        -: 3238:    1           0 to p/2           0 or 1
        -: 3239:.ve
        -: 3240: where the value each process on the same SMP has the same value for
        -: 3241: 'my_cluster[1]' and a different value for 'my_rank[1]'.
        -: 3242:
        -: 3243:  For two SMPs connected by a network,
        -: 3244:.vb
        -: 3245:    level       my_cluster         my_rank
        -: 3246:    0           0                  rank in comm_world
        -: 3247:    1           0 or 1             0 to # on SMP
        -: 3248:.ve
        -: 3249:
        -: 3250:  An example with more than 2 levels is a collection of clusters, each with
        -: 3251:  SMP nodes.  
        -: 3252:
        -: 3253:  Limitations:
        -: 3254:  This approach does not provide a representations for topologies that
        -: 3255:  are not hierarchical.  For example, a mesh interconnect is a single-level
        -: 3256:  cluster in this view.
        -: 3257:
        -: 3258:  Module: 
        -: 3259:  Topology
        -: 3260:  @*/
        -: 3261:int MPID_Topo_cluster_info( MPID_Comm *comm, 
        -: 3262:			    int *levels, int my_cluster[], int my_rank[] );
        -: 3263:
        -: 3264:/*@
        -: 3265:  MPID_Get_processor_name - Return the name of the current processor
        -: 3266:
        -: 3267:  Input Parameter:
        -: 3268:. namelen - Length of name
        -: 3269:  
        -: 3270:  Output Parameters:
        -: 3271:+ name - A unique specifier for the actual (as opposed to virtual) node. This
        -: 3272:  must be an array of size at least 'MPI_MAX_PROCESSOR_NAME'.
        -: 3273:- resultlen - Length (in characters) of the name.  If this pointer is null,
        -: 3274:   this value is not set.
        -: 3275:
        -: 3276:  Notes:
        -: 3277:  The name returned should identify a particular piece of hardware; 
        -: 3278:  the exact format is implementation defined.  This name may or may not
        -: 3279:  be the same as might be returned by 'gethostname', 'uname', or 'sysinfo'.
        -: 3280:
        -: 3281:  This routine is essentially an MPID version of 'MPI_Get_processor_name' .  
        -: 3282:  It must be part of the device because not all environments support calls
        -: 3283:  to return the processor name.  The additional argument (input name 
        -: 3284:  length) is used to provide better error checking and to ensure that 
        -: 3285:  the input buffer is large enough (rather than assuming that it is
        -: 3286:  'MPI_MAX_PROCESSOR_NAME' long).
        -: 3287:  @*/
        -: 3288:int MPID_Get_processor_name( char *name, int namelen, int *resultlen);
        -: 3289:
        -: 3290:void MPID_Errhandler_free(MPID_Errhandler *errhan_ptr);
        -: 3291:
        -: 3292:/*@
        -: 3293:  MPID_Get_universe_size - Return the number of processes that the current
        -: 3294:  process management environment can handle
        -: 3295:
        -: 3296:  Output Parameters:
        -: 3297:. universe_size - the universe size; MPIR_UNIVERSE_SIZE_NOT_AVAILABLE if the
        -: 3298:  size cannot be determined
        -: 3299:  
        -: 3300:  Return value:
        -: 3301:  A MPI error code.
        -: 3302:@*/
        -: 3303:int MPID_Get_universe_size(int  * universe_size);
        -: 3304:
        -: 3305:#define MPIR_UNIVERSE_SIZE_NOT_SET -1
        -: 3306:#define MPIR_UNIVERSE_SIZE_NOT_AVAILABLE -2
        -: 3307:
        -: 3308:/*
        -: 3309: * FIXME: VCs should not be exposed to the top layer, which implies that these routines should not be exposed either.  Instead,
        -: 3310: * the creation, duplication and destruction of communicator objects should be communicated to the device, allowing the device to
        -: 3311: * manage the underlying connections in a way that is appropriate (and efficient).
        -: 3312: */
        -: 3313:
        -: 3314:/*@
        -: 3315:  MPID_VCRT_Create - Create a virtual connection reference table
        -: 3316:  @*/
        -: 3317:int MPID_VCRT_Create(int size, MPID_VCRT *vcrt_ptr);
        -: 3318:
        -: 3319:/*@
        -: 3320:  MPID_VCRT_Add_ref - Add a reference to a VCRT
        -: 3321:  @*/
        -: 3322:int MPID_VCRT_Add_ref(MPID_VCRT vcrt);
        -: 3323:
        -: 3324:/*@
        -: 3325:  MPID_VCRT_Release - Release a reference to a VCRT
        -: 3326:  
        -: 3327:  Notes:
        -: 3328:  The 'isDisconnect' argument allows this routine to handle the special
        -: 3329:  case of 'MPI_Comm_disconnect', which needs to take special action
        -: 3330:  if all references to a VC are removed.
        -: 3331:  @*/
        -: 3332:int MPID_VCRT_Release(MPID_VCRT vcrt, int isDisconnect);
        -: 3333:
        -: 3334:/*@
        -: 3335:  MPID_VCRT_Get_ptr - 
        -: 3336:  @*/
        -: 3337:int MPID_VCRT_Get_ptr(MPID_VCRT vcrt, MPID_VCR **vc_pptr);
        -: 3338:
        -: 3339:/*@
        -: 3340:  MPID_VCR_Dup - Create a duplicate reference to a virtual connection
        -: 3341:  @*/
        -: 3342:int MPID_VCR_Dup(MPID_VCR orig_vcr, MPID_VCR * new_vcr);
        -: 3343:
        -: 3344:/*@
        -: 3345:   MPID_VCR_Get_lpid - Get the local process id that corresponds to a 
        -: 3346:   virtual connection reference.
        -: 3347:
        -: 3348:   Notes:
        -: 3349:   The local process ids are described elsewhere.  Basically, they are
        -: 3350:   a nonnegative number by which this process can refer to other processes 
        -: 3351:   to which it is connected.  These are local process ids because different
        -: 3352:   processes may use different ids to identify the same target process
        -: 3353:  @*/
        -: 3354:int MPID_VCR_Get_lpid(MPID_VCR vcr, int * lpid_ptr);
        -: 3355:
        -: 3356:/* ------------------------------------------------------------------------- */
        -: 3357:/* Define a macro to allow us to select between statically selected functions
        -: 3358: * and dynamically loaded ones.  If USE_DYNAMIC_LIBRARIES is defined,
        -: 3359: * the macro MPIU_CALL(context,funccall) expands into
        -: 3360: *    MPIU_CALL##context.funccall.
        -: 3361: * For example,
        -: 3362: *    err = MPIU_CALL(MPIDI_CH3,iSend(...))
        -: 3363: * will expand into
        -: 3364: *    err = MPIU_CALL_MPIDI_CH3.iSend(...)
        -: 3365: * If USE_DYNAMIC_LIBS is not selected, then it expands into
        -: 3366: *    err = MPIDI_CH3_iSend(...)
        -: 3367: * 
        -: 3368: * In the case where dynamic libraries are used, a variable named 
        -: 3369: * MPIU_CALL_context must be defined that contains the function pointers;
        -: 3370: * initializing the function pointers must be done before the first use.
        -: 3371: * Typically, this variable will be the single instance of a structure that
        -: 3372: * contains the function pointers.
        -: 3373: */
        -: 3374:/* ------------------------------------------------------------------------- */
        -: 3375:#ifdef USE_DYNAMIC_LIBRARIES
        -: 3376:#define MPIU_CALL(context,funccall) MPIU_CALL_##context.funccall
        -: 3377:#else
        -: 3378:#define MPIU_CALL(context,funccall) context##_##funccall
        -: 3379:#endif
        -: 3380:
        -: 3381:/* Include definitions from the device which require items defined by this 
        -: 3382:   file (mpiimpl.h). */
        -: 3383:#include "mpidpost.h"
        -: 3384:
        -: 3385:/* ------------------------------------------------------------------------- */
        -: 3386:/* FIXME: Also for mpicoll.h, in src/mpi/coll?  */
        -: 3387:/* ------------------------------------------------------------------------- */
        -: 3388:/* thresholds to switch between long and short vector algorithms for
        -: 3389:   collective operations */ 
        -: 3390:/* FIXME: Should there be a way to (a) update/compute these at configure time
        -: 3391:   and (b) provide runtime control?  Should these be MPIR_xxx_DEFAULT 
        -: 3392:   instead? */
        -: 3393:#define MPIR_BCAST_SHORT_MSG          12288
        -: 3394:#define MPIR_BCAST_LONG_MSG           524288
        -: 3395:#define MPIR_BCAST_MIN_PROCS          8
        -: 3396:#define MPIR_ALLTOALL_SHORT_MSG       256
        -: 3397:#define MPIR_ALLTOALL_MEDIUM_MSG      32768
        -: 3398:#define MPIR_ALLTOALL_THROTTLE        4  /* max no. of irecvs/isends posted at a 
        -: 3399:time in some alltoall algorithms. Setting it to 0 causes all irecvs/isends to be 
        -: 3400:posted at once. */
        -: 3401:#define MPIR_REDSCAT_COMMUTATIVE_LONG_MSG 524288
        -: 3402:#define MPIR_REDSCAT_NONCOMMUTATIVE_SHORT_MSG 512
        -: 3403:#define MPIR_ALLGATHER_SHORT_MSG      81920
        -: 3404:#define MPIR_ALLGATHER_LONG_MSG       524288
        -: 3405:#define MPIR_REDUCE_SHORT_MSG         2048
        -: 3406:#define MPIR_ALLREDUCE_SHORT_MSG      2048
        -: 3407:#define MPIR_GATHER_VSMALL_MSG        1024
        -: 3408:#define MPIR_SCATTER_SHORT_MSG        2048  /* for intercommunicator scatter */
        -: 3409:#define MPIR_GATHER_SHORT_MSG         2048  /* for intercommunicator scatter */
        -: 3410:#define MPIR_GATHERV_MIN_PROCS        32
        -: 3411:
        -: 3412:/* For pipelined collectives */
        -: 3413:#define MPIR_ALLGATHERV_PIPELINE_MSGSIZE   32768
        -: 3414:
        -: 3415:/* Tags for point to point operations which implement collective operations */
        -: 3416:#define MPIR_BARRIER_TAG               1
        -: 3417:#define MPIR_BCAST_TAG                 2
        -: 3418:#define MPIR_GATHER_TAG                3
        -: 3419:#define MPIR_GATHERV_TAG               4
        -: 3420:#define MPIR_SCATTER_TAG               5
        -: 3421:#define MPIR_SCATTERV_TAG              6
        -: 3422:#define MPIR_ALLGATHER_TAG             7
        -: 3423:#define MPIR_ALLGATHERV_TAG            8
        -: 3424:#define MPIR_ALLTOALL_TAG              9
        -: 3425:#define MPIR_ALLTOALLV_TAG            10
        -: 3426:#define MPIR_REDUCE_TAG               11
        -: 3427:#define MPIR_USER_REDUCE_TAG          12
        -: 3428:#define MPIR_USER_REDUCEA_TAG         13
        -: 3429:#define MPIR_ALLREDUCE_TAG            14
        -: 3430:#define MPIR_USER_ALLREDUCE_TAG       15
        -: 3431:#define MPIR_USER_ALLREDUCEA_TAG      16
        -: 3432:#define MPIR_REDUCE_SCATTER_TAG       17
        -: 3433:#define MPIR_USER_REDUCE_SCATTER_TAG  18
        -: 3434:#define MPIR_USER_REDUCE_SCATTERA_TAG 19
        -: 3435:#define MPIR_SCAN_TAG                 20
        -: 3436:#define MPIR_USER_SCAN_TAG            21
        -: 3437:#define MPIR_USER_SCANA_TAG           22
        -: 3438:#define MPIR_LOCALCOPY_TAG            23
        -: 3439:#define MPIR_EXSCAN_TAG               24
        -: 3440:#define MPIR_ALLTOALLW_TAG            25
        -: 3441:#define MPIR_TOPO_A_TAG               26
        -: 3442:#define MPIR_TOPO_B_TAG               27
        -: 3443:
        -: 3444:/* These functions are used in the implementation of collective
        -: 3445:   operations. They are wrappers around MPID send/recv functions. They do
        -: 3446:   sends/receives by setting the context offset to
        -: 3447:   MPID_CONTEXT_INTRA_COLL. */
        -: 3448:int MPIC_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
        -: 3449:              MPI_Comm comm);
        -: 3450:int MPIC_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
        -: 3451:              MPI_Comm comm, MPI_Status *status);
        -: 3452:int MPIC_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
        -: 3453:               MPI_Comm comm);
        -: 3454:int MPIC_Sendrecv(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3455:                  int dest, int sendtag, void *recvbuf, int recvcount,
        -: 3456:                  MPI_Datatype recvtype, int source, int recvtag,
        -: 3457:                  MPI_Comm comm, MPI_Status *status);
        -: 3458:int MPIC_Sendrecv_replace(void *buf, int count, MPI_Datatype type,
        -: 3459:                          int dest, int sendtag,
        -: 3460:                          int source, int recvtag,
        -: 3461:                          MPI_Comm comm, MPI_Status *status);
        -: 3462:int MPIR_Localcopy(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3463:                   void *recvbuf, int recvcount, MPI_Datatype recvtype);
        -: 3464:int MPIC_Irecv(void *buf, int count, MPI_Datatype datatype, int
        -: 3465:               source, int tag, MPI_Comm comm, MPI_Request *request);
        -: 3466:int MPIC_Isend(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
        -: 3467:               MPI_Comm comm, MPI_Request *request);
        -: 3468:int MPIC_Wait(MPID_Request * request_ptr);
        -: 3469:
        -: 3470:
        -: 3471:void MPIR_MAXF  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3472:void MPIR_MINF  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3473:void MPIR_SUM  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3474:void MPIR_PROD  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3475:void MPIR_LAND  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3476:void MPIR_BAND  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3477:void MPIR_LOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3478:void MPIR_BOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3479:void MPIR_LXOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3480:void MPIR_BXOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3481:void MPIR_MAXLOC  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3482:void MPIR_MINLOC  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3483:
        -: 3484:int MPIR_MAXF_check_dtype  ( MPI_Datatype ) ;
        -: 3485:int MPIR_MINF_check_dtype ( MPI_Datatype ) ;
        -: 3486:int MPIR_SUM_check_dtype  ( MPI_Datatype ) ;
        -: 3487:int MPIR_PROD_check_dtype  ( MPI_Datatype ) ;
        -: 3488:int MPIR_LAND_check_dtype  ( MPI_Datatype ) ;
        -: 3489:int MPIR_BAND_check_dtype  ( MPI_Datatype ) ;
        -: 3490:int MPIR_LOR_check_dtype  ( MPI_Datatype ) ;
        -: 3491:int MPIR_BOR_check_dtype  ( MPI_Datatype ) ;
        -: 3492:int MPIR_LXOR_check_dtype ( MPI_Datatype ) ;
        -: 3493:int MPIR_BXOR_check_dtype  ( MPI_Datatype ) ;
        -: 3494:int MPIR_MAXLOC_check_dtype  ( MPI_Datatype ) ;
        -: 3495:int MPIR_MINLOC_check_dtype  ( MPI_Datatype ) ;
        -: 3496:
        -: 3497:#define MPIR_PREDEF_OP_COUNT 12
        -: 3498:extern MPI_User_function *MPIR_Op_table[];
        -: 3499:
        -: 3500:typedef int (MPIR_Op_check_dtype_fn) ( MPI_Datatype ); 
        -: 3501:extern MPIR_Op_check_dtype_fn *MPIR_Op_check_dtype_table[];
        -: 3502:
        -: 3503:#if !defined MPIR_MIN
        -: 3504:#define MPIR_MIN(a,b) (((a)>(b))?(b):(a))
        -: 3505:#endif /* MPIR_MIN */
        -: 3506:
        -: 3507:#if !defined MPIR_MAX
        -: 3508:#define MPIR_MAX(a,b) (((b)>(a))?(b):(a))
        -: 3509:#endif /* MPIR_MAX */
        -: 3510:
        -: 3511:int MPIR_Allgather(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3512:                   void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3513:                   MPID_Comm *comm_ptr );
        -: 3514:int MPIR_Allgather_inter(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3515:                         void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3516:                         MPID_Comm *comm_ptr );
        -: 3517:int MPIR_Allgatherv(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3518:                    void *recvbuf, int *recvcounts, int *displs,   
        -: 3519:                    MPI_Datatype recvtype, MPID_Comm *comm_ptr );
        -: 3520:int MPIR_Allgatherv_inter(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3521:                          void *recvbuf, int *recvcounts, int *displs,   
        -: 3522:                          MPI_Datatype recvtype, MPID_Comm *comm_ptr );
        -: 3523:int MPIR_Allreduce(void *sendbuf, void *recvbuf, int count, 
        -: 3524:                   MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr);
        -: 3525:int MPIR_Allreduce_inter(void *sendbuf, void *recvbuf, int count, 
        -: 3526:                        MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr);
        -: 3527:int MPIR_Alltoall(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3528:                  void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3529:                  MPID_Comm *comm_ptr);
        -: 3530:int MPIR_Alltoall_inter(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3531:                        void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3532:                        MPID_Comm *comm_ptr);
        -: 3533:int MPIR_Alltoallv(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3534:                   MPI_Datatype sendtype, void *recvbuf, int *recvcnts, 
        -: 3535:                   int *rdispls, MPI_Datatype recvtype, MPID_Comm *comm_ptr);
        -: 3536:int MPIR_Alltoallv_inter(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3537:                         MPI_Datatype sendtype, void *recvbuf, int *recvcnts, 
        -: 3538:                         int *rdispls, MPI_Datatype recvtype, 
        -: 3539:                         MPID_Comm *comm_ptr);
        -: 3540:int MPIR_Alltoallw(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3541:                   MPI_Datatype *sendtypes, void *recvbuf, int *recvcnts, 
        -: 3542:                   int *rdispls, MPI_Datatype *recvtypes, MPID_Comm *comm_ptr);
        -: 3543:int MPIR_Alltoallw_inter(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3544:                         MPI_Datatype *sendtypes, void *recvbuf, 
        -: 3545:                         int *recvcnts, int *rdispls, MPI_Datatype *recvtypes, 
        -: 3546:                         MPID_Comm *comm_ptr);
        -: 3547:int MPIR_Barrier_inter( MPID_Comm *comm_ptr);
        -: 3548:int MPIR_Bcast_inter(void *buffer, int count, MPI_Datatype datatype, 
        -: 3549:		     int root, MPID_Comm *comm_ptr);
        -: 3550:int MPIR_Bcast (void *buffer, int count, MPI_Datatype datatype, int
        -: 3551:                root, MPID_Comm *comm_ptr);
        -: 3552:int MPIR_Bcast_or_coll_fn (void *buffer, int count, MPI_Datatype datatype, int
        -: 3553:                root, MPID_Comm *comm_ptr);
        -: 3554:int MPIR_Exscan(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
        -: 3555:                MPI_Op op, MPID_Comm *comm_ptr );
        -: 3556:int MPIR_Gather (void *sendbuf, int sendcnt, MPI_Datatype sendtype,
        -: 3557:                 void *recvbuf, int recvcnt, MPI_Datatype recvtype,
        -: 3558:                 int root, MPID_Comm *comm_ptr);
        -: 3559:int MPIR_Gather_inter (void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3560:                       void *recvbuf, int recvcnt, MPI_Datatype recvtype, 
        -: 3561:                       int root, MPID_Comm *comm_ptr );
        -: 3562:int MPIR_Gatherv (void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3563:                  void *recvbuf, int *recvcnts, int *displs,
        -: 3564:                  MPI_Datatype recvtype, int root, MPID_Comm *comm_ptr); 
        -: 3565:int MPIR_Reduce_scatter(void *sendbuf, void *recvbuf, int *recvcnts, 
        -: 3566:                        MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr);
        -: 3567:int MPIR_Reduce_scatter_inter(void *sendbuf, void *recvbuf, int *recvcnts, 
        -: 3568:                              MPI_Datatype datatype, MPI_Op op, 
        -: 3569:                              MPID_Comm *comm_ptr);
        -: 3570:int MPIR_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
        -: 3571:                MPI_Op op, int root, MPID_Comm *comm_ptr );
        -: 3572:int MPIR_Reduce_or_coll_fn(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
        -: 3573:                MPI_Op op, int root, MPID_Comm *comm_ptr );
        -: 3574:int MPIR_Reduce_inter (void *sendbuf, void *recvbuf, int count, MPI_Datatype
        -: 3575:                 datatype, MPI_Op op, int root, MPID_Comm *comm_ptr); 
        -: 3576:int MPIR_Scan(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, 
        -: 3577:              MPI_Op op, MPID_Comm *comm_ptr);
        -: 3578:int MPIR_Scatter(void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3579:                 void *recvbuf, int recvcnt, MPI_Datatype recvtype, 
        -: 3580:                 int root, MPID_Comm *comm_ptr );
        -: 3581:int MPIR_Scatter_inter(void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3582:                       void *recvbuf, int recvcnt, MPI_Datatype recvtype, 
        -: 3583:                       int root, MPID_Comm *comm_ptr );
        -: 3584:int MPIR_Scatterv (void *sendbuf, int *sendcnts, int *displs,
        -: 3585:                   MPI_Datatype sendtype, void *recvbuf, int recvcnt,
        -: 3586:                   MPI_Datatype recvtype, int root, MPID_Comm
        -: 3587:                   *comm_ptr);
        -: 3588:int MPIR_Barrier( MPID_Comm *comm_ptr );
        -: 3589:
        -: 3590:int MPIR_Setup_intercomm_localcomm( MPID_Comm * );
        -: 3591:
        -: 3592:int MPIR_Comm_create( MPID_Comm ** );
        -: 3593:
        -: 3594:int MPIR_Comm_commit( MPID_Comm * );
        -: 3595:
        -: 3596:int MPIR_Comm_is_node_aware( MPID_Comm * );
        -: 3597:
        -: 3598:int MPIR_Comm_is_node_consecutive( MPID_Comm *);
        -: 3599:
        -: 3600:void MPIR_Free_err_dyncodes( void );
        -: 3601:
        -: 3602:
        -: 3603:/* Collective functions cannot be called from multiple threads. These
        -: 3604:   are stubs used in the collective communication calls to check for
        -: 3605:   user error. Currently they are just being macroed out. */
        -: 3606:#define MPIDU_ERR_CHECK_MULTIPLE_THREADS_ENTER(comm_ptr)
        -: 3607:#define MPIDU_ERR_CHECK_MULTIPLE_THREADS_EXIT(comm_ptr)
        -: 3608:
        -: 3609:/* Miscellaneous */
        -: 3610:void MPIU_SetTimeout( int );
        -: 3611:
        -: 3612:#if defined(HAVE_VSNPRINTF) && defined(NEEDS_VSNPRINTF_DECL) && !defined(vsnprintf)
        -: 3613:int vsnprintf(char *str, size_t size, const char *format, va_list ap);
        -: 3614:# endif
        -: 3615:
        -: 3616:/* Routines for determining local and remote processes */
        -: 3617:
        -: 3618:int MPIU_Find_local_and_external(struct MPID_Comm *comm, int *local_size_p, int *local_rank_p, int **local_ranks_p,
        -: 3619:                                 int *external_size_p, int *external_rank_p, int **external_ranks_p,
        -: 3620:                                 int **intranode_table, int **internode_table_p);
        -: 3621:int MPIU_Get_internode_rank(MPID_Comm *comm_ptr, int r);
        -: 3622:int MPIU_Get_intranode_rank(MPID_Comm *comm_ptr, int r);
        -: 3623:
        -: 3624:#endif /* MPIIMPL_INCLUDED */