-:    0:Source:/home/MPI/testing/mpich2/mpich2/src/include/mpiimpl.h
        -:    0:Graph:mpid_datatype_contents.gcno
        -:    0:Data:-
        -:    0:Runs:0
        -:    0:Programs:0
        -:    1:/* -*- Mode: C; c-basic-offset:4 ; -*- */
        -:    2:/*  
        -:    3: *  (C) 2001 by Argonne National Laboratory.
        -:    4: *      See COPYRIGHT in top-level directory.
        -:    5: */
        -:    6:#ifndef MPIIMPL_H_INCLUDED
        -:    7:#define MPIIMPL_H_INCLUDED
        -:    8:
        -:    9:/*
        -:   10: * This file is the temporary home of most of the definitions used to 
        -:   11: * implement MPICH.  We will eventually divide this file into logical
        -:   12: * pieces once we are certain of the relationships between the components.
        -:   13: */
        -:   14:
        -:   15:/* style: define:vsnprintf:1 sig:0 */
        -:   16:/* style: allow:printf:3 sig:0 */
        -:   17:
        -:   18:/* Include the mpi definitions */
        -:   19:#include "mpi.h"
        -:   20:
        -:   21:/* Include nested mpi (NMPI) definitions */
        -:   22:#include "nmpi.h"
        -:   23:
        -:   24:/* There are a few definitions that must be made *before* the mpichconf.h
        -:   25:   file is included.  These include the definitions of the error levels and some
        -:   26:   thread granularity constants */
        -:   27:#include "mpichconfconst.h"
        -:   28:
        -:   29:/* Data computed by configure.  This is included *after* mpi.h because we
        -:   30:   do not want mpi.h to depend on any other files or configure flags */
        -:   31:#include "mpichconf.h"
        -:   32:
        -:   33:#include <stdio.h>
        -:   34:#ifdef STDC_HEADERS
        -:   35:#include <stdlib.h>
        -:   36:#include <stdarg.h>
        -:   37:#include <string.h>
        -:   38:#else
        -:   39:#ifdef HAVE_STDLIB_H
        -:   40:#include <stdlib.h>
        -:   41:#endif
        -:   42:#ifdef HAVE_STDARG_H
        -:   43:#include <stdarg.h>
        -:   44:#endif
        -:   45:#ifdef HAVE_STRING_H
        -:   46:#include <string.h>
        -:   47:#endif
        -:   48:#endif
        -:   49:
        -:   50:#ifdef HAVE_SYS_TYPES_H
        -:   51:#include <sys/types.h>
        -:   52:#endif
        -:   53:
        -:   54:/* for MAXHOSTNAMELEN under Linux and OSX */
        -:   55:#ifdef HAVE_SYS_PARAM_H
        -:   56:#include <sys/param.h>
        -:   57:#endif
        -:   58:
        -:   59:#if (!defined MAXHOSTNAMELEN) && (!defined MAX_HOSTNAME_LEN)
        -:   60:#define MAX_HOSTNAME_LEN 256
        -:   61:#elif !defined MAX_HOSTNAME_LEN
        -:   62:#define MAX_HOSTNAME_LEN MAXHOSTNAMELEN
        -:   63:#endif
        -:   64:
        -:   65:/* Default PMI version to use */
        -:   66:#define MPIU_DEFAULT_PMI_VERSION 1
        -:   67:#define MPIU_DEFAULT_PMI_SUBVERSION 1
        -:   68:
        -:   69:/* This allows us to keep names local to a single file when we can use
        -:   70:   weak symbols */
        -:   71:#ifdef  USE_WEAK_SYMBOLS
        -:   72:#define PMPI_LOCAL static
        -:   73:#else
        -:   74:#define PMPI_LOCAL 
        -:   75:#endif
        -:   76:
        -:   77:/* Fix for universal endianess added in autoconf 2.62 */
        -:   78:#ifdef WORDS_UNIVERSAL_ENDIAN
        -:   79:#if defined(__BIG_ENDIAN__)
        -:   80:#elif defined(__LITTLE_ENDIAN__)
        -:   81:#define WORDS_LITTLEENDIAN
        -:   82:#else
        -:   83:#error 'Universal endianess defined without __BIG_ENDIAN__ or __LITTLE_ENDIAN__'
        -:   84:#endif
        -:   85:#endif
        -:   86:
        -:   87:/* Include some basic (and easily shared) definitions */
        -:   88:#include "mpibase.h"
        -:   89:
        -:   90:/* FIXME: The code base should not define two of these */
        -:   91:/* This is used to quote a name in a definition (see FUNCNAME/FCNAME below) */
        -:   92:#ifndef MPIDI_QUOTE
        -:   93:#define MPIDI_QUOTE(A) MPIDI_QUOTE2(A)
        -:   94:#define MPIDI_QUOTE2(A) #A
        -:   95:#endif
        -:   96:
        -:   97:/* 
        -:   98:   Include the implementation definitions (e.g., error reporting, thread
        -:   99:   portability)
        -:  100:   More detailed documentation is contained in the MPICH2 and ADI3 manuals.
        -:  101: */
        -:  102:/* FIXME: ... to do ... */
        -:  103:#include "mpitypedefs.h"
        -:  104:
        -:  105:/* This is the default implementation of MPIU_Memcpy.  We define this
        -:  106:   before including mpidpre.h so that it can be used when a device or
        -:  107:   channel can use it if it's overriding MPIU_Memcpy.  */
        -:  108:MPIU_DBG_ATTRIBUTE_NOINLINE
        -:  109:ATTRIBUTE((unused))
        -:  110:static MPIU_DBG_INLINE_KEYWORD void MPIUI_Memcpy(void * dst, const void * src, size_t len)
    #####:  111:{
    #####:  112:    memcpy(dst, src, len);
    #####:  113:}
        -:  114:
        -:  115:/* Include definitions from the device which must exist before items in this
        -:  116:   file (mpiimpl.h) can be defined. mpidpre.h must be included before any
        -:  117:   files that allow the device to override or extend any terms; this includes
        -:  118:   mpiimplthread.h and mpiutil.h */
        -:  119:/* ------------------------------------------------------------------------- */
        -:  120:#include "mpidpre.h"
        -:  121:/* ------------------------------------------------------------------------- */
        -:  122:
        -:  123:/* Overriding memcpy:
        -:  124:   Devices and channels can override the default implementation of
        -:  125:   MPIU_Memcpy by defining the MPIU_Memcpy macro.  The implementation
        -:  126:   can call MPIUI_Memcpy for the default memcpy implementation.   
        -:  127:   Note that MPIU_Memcpy and MPIUI_Memcpy return void rather than a
        -:  128:   pointer to the destination buffer.  This is different from C89
        -:  129:   memcpy.
        -:  130:*/
        -:  131:#ifndef MPIU_Memcpy
        -:  132:#define MPIU_Memcpy(dst, src, len)                \
        -:  133:    do {                                          \
        -:  134:        MPIU_MEM_CHECK_MEMCPY((dst),(src),(len)); \
        -:  135:        MPIUI_Memcpy((dst), (src), (len));        \
        -:  136:    } while (0)
        -:  137:#endif
        -:  138:
        -:  139:#include "mpiimplthread.h"
        -:  140:/* #include "mpiu_monitors.h" */
        -:  141:
        -:  142:#include "mpiutil.h"
        -:  143:
        -:  144:/* ------------------------------------------------------------------------- */
        -:  145:/* mpidebug.h */
        -:  146:/* ------------------------------------------------------------------------- */
        -:  147:/* Debugging and printf control */
        -:  148:/* Use these *only* for debugging output intended for the implementors
        -:  149:   and maintainers of MPICH.  Do *not* use these for any output that
        -:  150:   general users may normally see.  Use either the error code creation
        -:  151:   routines for error messages or MPIU_msg_printf etc. for general messages 
        -:  152:   (MPIU_msg_printf will go through gettext).  
        -:  153:
        -:  154:   FIXME: Document all of these macros
        -:  155:
        -:  156:   NOTE: These macros and values are deprecated.  See 
        -:  157:   www.mcs.anl.gov/mpi/mpich2/developer/design/debugmsg.htm for 
        -:  158:   the new design (only partially implemented at this time).
        -:  159:   
        -:  160:   The implementation is in mpidbg.h
        -:  161:*/
        -:  162:#include "mpidbg.h"
        -:  163:
        -:  164:#if defined(MPICH_DBG_OUTPUT)
        -:  165:#define MPIU_DBG_PRINTF(e)			\
        -:  166:{						\
        -:  167:    if (MPIU_dbg_state != MPIU_DBG_STATE_NONE)	\
        -:  168:    {						\
        -:  169:	MPIU_dbg_printf e;			\
        -:  170:    }						\
        -:  171:}
        -:  172:/* The first argument is a place holder to allow the selection of a subset
        -:  173:   of debugging events.  The second is a placeholder to allow a numeric
        -:  174:   level of debugging within that class.  The third is the debugging text */
        -:  175:#define MPIU_DBG_PRINTF_CLASS(_c,_l,_e) MPIU_DBG_PRINTF(_e)
        -:  176:#else
        -:  177:#define MPIU_DBG_PRINTF(e)
        -:  178:#define MPIU_DBG_PRINTF_CLASS(_c,_l,_e)
        -:  179:#endif
        -:  180:/* FIXME: These should use the MPIU_DBG_STMT macros and not be defined
        -:  181:   as macros themselves (to make it clear that they are macros, and not
        -:  182:   always called) */
        -:  183:#ifdef USE_MPIU_DBG_PRINT_VC
        -:  184:void MPIU_DBG_PrintVC(MPIDI_VC_t *vc);
        -:  185:void MPIU_DBG_PrintVCState2(MPIDI_VC_t *vc, MPIDI_VC_State_t new_state);
        -:  186:void MPIU_DBG_PrintVCState(MPIDI_VC_t *vc);
        -:  187:#else
        -:  188:#define MPIU_DBG_PrintVC(vc)
        -:  189:#define MPIU_DBG_PrintVCState2(vc, new_state)
        -:  190:#define MPIU_DBG_PrintVCState(vc)
        -:  191:#endif
        -:  192:
        -:  193:
        -:  194:/* The follow is temporarily provided for backward compatibility.  Any code
        -:  195:   using dbg_printf should be updated to use MPIU_DBG_PRINTF. */
        -:  196:#define dbg_printf MPIU_dbg_printf
        -:  197:
        -:  198:/* MPIR_IDebug withdrawn because the MPIU_DBG_MSG interface provides 
        -:  199:   a more flexible, integrated, and documented mechanism */
        -:  200:
        -:  201:/* ------------------------------------------------------------------------- */
        -:  202:/* end of mpidebug.h */
        -:  203:/* ------------------------------------------------------------------------- */
        -:  204:
        -:  205:/* Routines for memory management */
        -:  206:#include "mpimem.h"
        -:  207:
        -:  208:/*
        -:  209: * Use MPIU_SYSCALL to wrap system calls; this provides a convenient point
        -:  210: * for timing the calls and keeping track of the use of system calls.
        -:  211: * This macro simply invokes the system call and does not even handle
        -:  212: * EINTR.
        -:  213: * To use, 
        -:  214: *    MPIU_SYSCALL( return-value, name-of-call, args-in-parenthesis )
        -:  215: * e.g., change "n = read(fd,buf,maxn);" into
        -:  216: *    MPIU_SYSCALL( n,read,(fd,buf,maxn) );
        -:  217: * An example that prints each syscall to stdout is shown below. 
        -:  218: */
        -:  219:#ifdef USE_LOG_SYSCALLS
        -:  220:#define MPIU_SYSCALL(a_,b_,c_) { \
        -:  221:    printf( "[%d]about to call %s\n", MPIR_Process.comm_world->rank,#b_);\
        -:  222:          fflush(stdout); errno = 0;\
        -:  223:    a_ = b_ c_; \
        -:  224:    if ((a_)>=0 || errno==0) {\
        -:  225:    printf( "[%d]%s returned %d\n", \
        -:  226:          MPIR_Process.comm_world->rank, #b_, a_ );\
        -:  227:    } \
        -:  228: else { \
        -:  229:    printf( "[%d]%s returned %d (errno = %d,%s)\n", \
        -:  230:          MPIR_Process.comm_world->rank, \
        -:  231:          #b_, a_, errno, strerror(errno));\
        -:  232:    };           fflush(stdout);}
        -:  233:#else
        -:  234:#define MPIU_SYSCALL(a_,b_,c_) a_ = b_ c_
        -:  235:#endif
        -:  236:
        -:  237:/*TDSOverview.tex
        -:  238:  
        -:  239:  MPI has a number of data structures, most of which are represented by 
        -:  240:  an opaque handle in an MPI program.  In the MPICH implementation of MPI, 
        -:  241:  these handles are represented
        -:  242:  as integers; this makes implementation of the C/Fortran handle transfer 
        -:  243:  calls (part of MPI-2) easy.  
        -:  244: 
        -:  245:  MPID objects (again with the possible exception of 'MPI_Request's) 
        -:  246:  are allocated by a common set of object allocation functions.
        -:  247:  These are 
        -:  248:.vb
        -:  249:    void *MPIU_Handle_obj_create( MPIU_Object_alloc_t *objmem )
        -:  250:    void MPIU_Handle_obj_destroy( MPIU_Object_alloc_t *objmem, void *object )
        -:  251:.ve
        -:  252:  where 'objmem' is a pointer to a memory allocation object that knows 
        -:  253:  enough to allocate objects, including the
        -:  254:  size of the object and the location of preallocated memory, as well 
        -:  255:  as the type of memory allocator.  By providing the routines to allocate and
        -:  256:  free the memory, we make it easy to use the same interface to allocate both
        -:  257:  local and shared memory for objects (always using the same kind for each 
        -:  258:  type of object).
        -:  259:
        -:  260:  The names create/destroy were chosen because they are different from 
        -:  261:  new/delete (C++ operations) and malloc/free.  
        -:  262:  Any name choice will have some conflicts with other uses, of course.
        -:  263:
        -:  264:  Reference Counts:
        -:  265:  Many MPI objects have reference count semantics.  
        -:  266:  The semantics of MPI require that many objects that have been freed by the 
        -:  267:  user 
        -:  268:  (e.g., with 'MPI_Type_free' or 'MPI_Comm_free') remain valid until all 
        -:  269:  pending
        -:  270:  references to that object (e.g., by an 'MPI_Irecv') are complete.  There
        -:  271:  are several ways to implement this; MPICH uses `reference counts` in the
        -:  272:  objects.  To support the 'MPI_THREAD_MULTIPLE' level of thread-safety, these
        -:  273:  reference counts must be accessed and updated atomically.  
        -:  274:  A reference count for
        -:  275:  `any` object can be incremented (atomically) 
        -:  276:  with 'MPIU_Object_add_ref(objptr)'
        -:  277:  and decremented with 'MPIU_Object_release_ref(objptr,newval_ptr)'.  
        -:  278:  These have been designed so that then can be implemented as inlined 
        -:  279:  macros rather than function calls, even in the multithreaded case, and
        -:  280:  can use special processor instructions that guarantee atomicity to 
        -:  281:  avoid thread locks.
        -:  282:  The decrement routine sets the value pointed at by 'inuse_ptr' to 0 if 
        -:  283:  the postdecrement value of the reference counter is zero, and to a non-zero
        -:  284:  value otherwise.  If this value is zero, then the routine that decremented 
        -:  285:  the
        -:  286:  reference count should free the object.  This may be as simple as 
        -:  287:  calling 'MPIU_Handle_obj_destroy' (for simple objects with no other allocated
        -:  288:  storage) or may require calling a separate routine to destroy the object.
        -:  289:  Because MPI uses 'MPI_xxx_free' to both decrement the reference count and 
        -:  290:  free the object if the reference count is zero, we avoid the use of 'free'
        -:  291:  in the MPID routines.
        -:  292:
        -:  293:  The 'inuse_ptr' approach is used rather than requiring the post-decrement
        -:  294:  value because, for reference-count semantics, all that is necessary is
        -:  295:  to know when the reference count reaches zero, and this can sometimes
        -:  296:  be implemented more cheaply that requiring the post-decrement value (e.g.,
        -:  297:  on IA32, there is an instruction for this operation).
        -:  298:
        -:  299:  Question:
        -:  300:  Should we state that this is a macro so that we can use a register for
        -:  301:  the output value?  That avoids a store.  Alternately, have the macro 
        -:  302:  return the value as if it was a function?
        -:  303:
        -:  304:  Structure Definitions:
        -:  305:  The structure definitions in this document define `only` that part of
        -:  306:  a structure that may be used by code that is making use of the ADI.
        -:  307:  Thus, some structures, such as 'MPID_Comm', have many defined fields;
        -:  308:  these are used to support MPI routines such as 'MPI_Comm_size' and
        -:  309:  'MPI_Comm_remote_group'.  Other structures may have few or no defined
        -:  310:  members; these structures have no fields used outside of the ADI.  
        -:  311:  In C++ terms,  all members of these structures are 'private'.  
        -:  312:
        -:  313:  For the initial implementation, we expect that the structure definitions 
        -:  314:  will be designed for the multimethod device.  However, all items that are
        -:  315:  specific to a particular device (including the multi-method device) 
        -:  316:  will be placed at the end of the structure;
        -:  317:  the document will clearly identify the members that all implementations
        -:  318:  will provide.  This simplifies much of the code in both the ADI and the 
        -:  319:  implementation of the MPI routines because structure member can be directly
        -:  320:  accessed rather than using some macro or C++ style method interface.
        -:  321:  
        -:  322: T*/
        -:  323:
        -:  324:/* mpi_lang.h - Prototypes for language specific routines. Currently used to
        -:  325: * set keyval attribute callbacks
        -:  326: */
        -:  327:#include "mpi_lang.h"
        -:  328:/* Known language bindings */
        -:  329:/*E
        -:  330:  MPID_Lang_t - Known language bindings for MPI
        -:  331:
        -:  332:  A few operations in MPI need to know what language they were called from
        -:  333:  or created by.  This type enumerates the possible languages so that
        -:  334:  the MPI implementation can choose the correct behavior.  An example of this
        -:  335:  are the keyval attribute copy and delete functions.
        -:  336:
        -:  337:  Module:
        -:  338:  Attribute-DS
        -:  339:  E*/
        -:  340:typedef enum MPID_Lang_t { MPID_LANG_C 
        -:  341:#ifdef HAVE_FORTRAN_BINDING
        -:  342:			   , MPID_LANG_FORTRAN
        -:  343:			   , MPID_LANG_FORTRAN90
        -:  344:#endif
        -:  345:#ifdef HAVE_CXX_BINDING
        -:  346:			   , MPID_LANG_CXX
        -:  347:#endif
        -:  348:} MPID_Lang_t;
        -:  349:
        -:  350:/* Macros for the MPI handles (e.g., the object that encodes an
        -:  351:   MPI_Datatype) */
        -:  352:#include "mpihandlemem.h"
        -:  353:
        -:  354:/* This routine is used to install an attribute free routine for datatypes
        -:  355:   at finalize-time */
        -:  356:void MPIR_DatatypeAttrFinalize( void );
        -:  357:
        -:  358:/* ------------------------------------------------------------------------- */
        -:  359:/* Should the following be moved into mpihandlemem.h ?*/
        -:  360:/* ------------------------------------------------------------------------- */
        -:  361:
        -:  362:/* Routines to initialize handle allocations */
        -:  363:/* These are now internal to the handlemem package
        -:  364:void *MPIU_Handle_direct_init( void *, int, int, int );
        -:  365:void *MPIU_Handle_indirect_init( void *(**)[], int *, int, int, int, int );
        -:  366:int MPIU_Handle_free( void *((*)[]), int );
        -:  367:*/
        -:  368:/* Convert Handles to objects for MPI types that have predefined objects */
        -:  369:#define MPID_Getb_ptr(kind,a,bmsk,ptr)                                  \
        -:  370:{                                                                       \
        -:  371:   switch (HANDLE_GET_KIND(a)) {                                        \
        -:  372:      case HANDLE_KIND_BUILTIN:                                         \
        -:  373:          ptr=MPID_##kind##_builtin+((a)&(bmsk));                       \
        -:  374:          break;                                                        \
        -:  375:      case HANDLE_KIND_DIRECT:                                          \
        -:  376:          ptr=MPID_##kind##_direct+HANDLE_INDEX(a);                     \
        -:  377:          break;                                                        \
        -:  378:      case HANDLE_KIND_INDIRECT:                                        \
        -:  379:          ptr=((MPID_##kind*)                                           \
        -:  380:               MPIU_Handle_get_ptr_indirect(a,&MPID_##kind##_mem));     \
        -:  381:          break;                                                        \
        -:  382:      case HANDLE_KIND_INVALID:                                         \
        -:  383:      default:								\
        -:  384:          ptr=0;							\
        -:  385:          break;							\
        -:  386:    }                                                                   \
        -:  387:}
        -:  388:
        -:  389:/* Convert handles to objects for MPI types that do _not_ have any predefined
        -:  390:   objects */
        -:  391:#define MPID_Get_ptr(kind,a,ptr)					\
        -:  392:{									\
        -:  393:   switch (HANDLE_GET_KIND(a)) {					\
        -:  394:      case HANDLE_KIND_DIRECT:						\
        -:  395:          ptr=MPID_##kind##_direct+HANDLE_INDEX(a);			\
        -:  396:          break;							\
        -:  397:      case HANDLE_KIND_INDIRECT:					\
        -:  398:          ptr=((MPID_##kind*)						\
        -:  399:               MPIU_Handle_get_ptr_indirect(a,&MPID_##kind##_mem));	\
        -:  400:          break;							\
        -:  401:      case HANDLE_KIND_INVALID:						\
        -:  402:      case HANDLE_KIND_BUILTIN:						\
        -:  403:      default:								\
        -:  404:          ptr=0;							\
        -:  405:          break;							\
        -:  406:     }									\
        -:  407:}
        -:  408:
        -:  409:/* FIXME: the masks should be defined with the handle definitions instead
        -:  410:   of inserted here as literals */
        -:  411:#define MPID_Comm_get_ptr(a,ptr)       MPID_Getb_ptr(Comm,a,0x03ffffff,ptr)
        -:  412:#define MPID_Group_get_ptr(a,ptr)      MPID_Getb_ptr(Group,a,0x03ffffff,ptr)
        -:  413:#define MPID_File_get_ptr(a,ptr)       MPID_Get_ptr(File,a,ptr)
        -:  414:#define MPID_Errhandler_get_ptr(a,ptr) MPID_Getb_ptr(Errhandler,a,0x3,ptr)
        -:  415:#define MPID_Op_get_ptr(a,ptr)         MPID_Getb_ptr(Op,a,0x000000ff,ptr)
        -:  416:#define MPID_Info_get_ptr(a,ptr)       MPID_Get_ptr(Info,a,ptr)
        -:  417:#define MPID_Win_get_ptr(a,ptr)        MPID_Get_ptr(Win,a,ptr)
        -:  418:#define MPID_Request_get_ptr(a,ptr)    MPID_Get_ptr(Request,a,ptr)
        -:  419:#define MPID_Grequest_class_get_ptr(a,ptr) MPID_Get_ptr(Grequest_class,a,ptr)
        -:  420:/* Keyvals have a special format. This is roughly MPID_Get_ptrb, but
        -:  421:   the handle index is in a smaller bit field.  In addition, 
        -:  422:   there is no storage for the builtin keyvals.  
        -:  423:   For the indirect case, we mask off the part of the keyval that is
        -:  424:   in the bits normally used for the indirect block index.
        -:  425:*/
        -:  426:#define MPID_Keyval_get_ptr(a,ptr)     \
        -:  427:{                                                                       \
        -:  428:   switch (HANDLE_GET_KIND(a)) {                                        \
        -:  429:      case HANDLE_KIND_BUILTIN:                                         \
        -:  430:          ptr=0;                                                        \
        -:  431:          break;                                                        \
        -:  432:      case HANDLE_KIND_DIRECT:                                          \
        -:  433:          ptr=MPID_Keyval_direct+((a)&0x3fffff);                        \
        -:  434:          break;                                                        \
        -:  435:      case HANDLE_KIND_INDIRECT:                                        \
        -:  436:          ptr=((MPID_Keyval*)                                           \
        -:  437:             MPIU_Handle_get_ptr_indirect((a)&0xfc3fffff,&MPID_Keyval_mem)); \
        -:  438:          break;                                                        \
        -:  439:      case HANDLE_KIND_INVALID:                                         \
        -:  440:      default:								\
        -:  441:          ptr=0;							\
        -:  442:          break;							\
        -:  443:    }                                                                   \
        -:  444:}
        -:  445:
        -:  446:/* Valid pointer checks */
        -:  447:/* This test is lame.  Should eventually include cookie test 
        -:  448:   and in-range addresses */
        -:  449:#define MPID_Valid_ptr(kind,ptr,err) \
        -:  450:  {if (!(ptr)) { err = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, \
        -:  451:                                             "**nullptrtype", "**nullptrtype %s", #kind ); } }
        -:  452:#define MPID_Valid_ptr_class(kind,ptr,errclass,err) \
        -:  453:  {if (!(ptr)) { err = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, errclass, \
        -:  454:                                             "**nullptrtype", "**nullptrtype %s", #kind ); } }
        -:  455:
        -:  456:#define MPID_Info_valid_ptr(ptr,err) MPID_Valid_ptr_class(Info,ptr,MPI_ERR_INFO,err)
        -:  457:/* Check not only for a null pointer but for an invalid communicator,
        -:  458:   such as one that has been freed.  Let's try the ref_count as the test
        -:  459:   for now */
        -:  460:#define MPID_Comm_valid_ptr(ptr,err) {                \
        -:  461:     MPID_Valid_ptr_class(Comm,ptr,MPI_ERR_COMM,err); \
        -:  462:     if ((ptr) && MPIU_Object_get_ref(ptr) == 0) {    \
        -:  463:         MPIU_ERR_SET(err,MPI_ERR_COMM,"**comm");     \
        -:  464:         ptr = 0;                                     \
        -:  465:     }                                                \
        -:  466:}
        -:  467:#define MPID_Group_valid_ptr(ptr,err) MPID_Valid_ptr_class(Group,ptr,MPI_ERR_GROUP,err)
        -:  468:#define MPID_Win_valid_ptr(ptr,err) MPID_Valid_ptr_class(Win,ptr,MPI_ERR_WIN,err)
        -:  469:#define MPID_Op_valid_ptr(ptr,err) MPID_Valid_ptr_class(Op,ptr,MPI_ERR_OP,err)
        -:  470:#define MPID_Errhandler_valid_ptr(ptr,err) MPID_Valid_ptr_class(Errhandler,ptr,MPI_ERR_ARG,err)
        -:  471:#define MPID_File_valid_ptr(ptr,err) MPID_Valid_ptr_class(File,ptr,MPI_ERR_FILE,err)
        -:  472:#define MPID_Request_valid_ptr(ptr,err) MPID_Valid_ptr_class(Request,ptr,MPI_ERR_REQUEST,err)
        -:  473:#define MPID_Keyval_valid_ptr(ptr,err) MPID_Valid_ptr_class(Keyval,ptr,MPI_ERR_KEYVAL,err)
        -:  474:
        -:  475:/* FIXME: 
        -:  476:   Generic pointer test.  This is applied to any address, not just one from
        -:  477:   an MPI object.
        -:  478:   Currently unimplemented (returns success except for null pointers.
        -:  479:   With a little work, could check that the pointer is properly aligned,
        -:  480:   using something like 
        -:  481:   ((p) == 0 || ((char *)(p) & MPID_Alignbits[alignment] != 0)
        -:  482:   where MPID_Alignbits is set with a mask whose bits must be zero in a 
        -:  483:   properly aligned quantity.  For systems with no alignment rules, 
        -:  484:   all of these masks are zero, and this part of test can be eliminated.
        -:  485: */
        -:  486:#define MPID_Pointer_is_invalid(p,alignment) ((p) == 0)
        -:  487:/* Fixme: The following MPID_ALIGNED_xxx values are temporary.  They 
        -:  488:   need to be computed by configure and included in the mpichconf.h file.
        -:  489:   Note that they cannot be set conservatively (i.e., as sizeof(object)),
        -:  490:   since the runtime system may generate objects with lesser alignment
        -:  491:   rules if the processor allows them.
        -:  492: */
        -:  493:#define MPID_ALIGNED_PTR_INT   1
        -:  494:#define MPID_ALIGNED_PTR_LONG  1
        -:  495:#define MPID_ALIGNED_PTR_VOIDP 1
        -:  496:/* ------------------------------------------------------------------------- */
        -:  497:/* end of code that should the following be moved into mpihandlemem.h ?*/
        -:  498:/* ------------------------------------------------------------------------- */
        -:  499:
        -:  500:#include "mpiparam.h"
        -:  501:
        -:  502:/* ------------------------------------------------------------------------- */
        -:  503:/* Info */
        -:  504:/*TInfoOverview.tex
        -:  505:
        -:  506:  'MPI_Info' provides a way to create a list of '(key,value)' pairs
        -:  507:  where the 'key' and 'value' are both strings.  Because many routines, both
        -:  508:  in the MPI implementation and in related APIs such as the PMI process
        -:  509:  management interface, require 'MPI_Info' arguments, we define a simple 
        -:  510:  structure for each 'MPI_Info' element.  Elements are allocated by the 
        -:  511:  generic object allocator; the head element is always empty (no 'key'
        -:  512:  or 'value' is defined on the head element).  
        -:  513:  
        -:  514:  For simplicity, we have not abstracted the info data structures;
        -:  515:  routines that want to work with the linked list may do so directly.
        -:  516:  Because the 'MPI_Info' type is a handle and not a pointer, an MPIU
        -:  517:  (utility) routine is provided to handle the 
        -:  518:  deallocation of 'MPID_Info' elements.  See the implementation of
        -:  519:  'MPI_Info_create' for how an Info type is allocated.
        -:  520:
        -:  521:  Thread Safety:
        -:  522:
        -:  523:  The info interface itself is not thread-robust.  In particular, the routines
        -:  524:  'MPI_INFO_GET_NKEYS' and 'MPI_INFO_GET_NTHKEY' assume that no other 
        -:  525:  thread modifies the info key.  (If the info routines had the concept
        -:  526:  of a next value, they would not be thread safe.  As it stands, a user
        -:  527:  must be careful if several threads have access to the same info object.) 
        -:  528:  Further, 'MPI_INFO_DUP', while not 
        -:  529:  explicitly advising implementers to be careful of one thread modifying the
        -:  530:  'MPI_Info' structure while 'MPI_INFO_DUP' is copying it, requires that the
        -:  531:  operation take place in a thread-safe manner.
        -:  532:  There isn'' much that we can do about these cases.  There are other cases
        -:  533:  that must be handled.  In particular, multiple threads are allowed to 
        -:  534:  update the same info value.  Thus, all of the update routines must be thread
        -:  535:  safe; the simple implementation used in the MPICH implementation uses locks.
        -:  536:  Note that the 'MPI_Info_delete' call does not need a lock; the defintion of
        -:  537:  thread-safety means that any order of the calls functions correctly; since
        -:  538:  it invalid either to delete the same 'MPI_Info' twice or to modify an
        -:  539:  'MPI_Info' that has been deleted, only one thread at a time can call 
        -:  540:  'MPI_Info_free' on any particular 'MPI_Info' value.  
        -:  541:
        -:  542:  T*/
        -:  543:/*S
        -:  544:  MPID_Info - Structure of an MPID info
        -:  545:
        -:  546:  Notes:
        -:  547:  There is no reference count because 'MPI_Info' values, unlike other MPI 
        -:  548:  objects, may be changed after they are passed to a routine without 
        -:  549:  changing the routine''s behavior.  In other words, any routine that uses
        -:  550:  an 'MPI_Info' object must make a copy or otherwise act on any info value
        -:  551:  that it needs.
        -:  552:
        -:  553:  A linked list is used because the typical 'MPI_Info' list will be short
        -:  554:  and a simple linked list is easy to implement and to maintain.  Similarly,
        -:  555:  a single structure rather than separate header and element structures are
        -:  556:  defined for simplicity.  No separate thread lock is provided because
        -:  557:  info routines are not performance critical; they may use the single
        -:  558:  critical section lock in the 'MPIR_Process' structure when they need a
        -:  559:  thread lock.
        -:  560:  
        -:  561:  This particular form of linked list (in particular, with this particular
        -:  562:  choice of the first two members) is used because it allows us to use 
        -:  563:  the same routines to manage this list as are used to manage the 
        -:  564:  list of free objects (in the file 'src/util/mem/handlemem.c').  In 
        -:  565:  particular, if lock-free routines for updating a linked list are 
        -:  566:  provided, they can be used for managing the 'MPID_Info' structure as well.
        -:  567:
        -:  568:  The MPI standard requires that keys can be no less that 32 characters and
        -:  569:  no more than 255 characters.  There is no mandated limit on the size 
        -:  570:  of values.
        -:  571:
        -:  572:  Module:
        -:  573:  Info-DS
        -:  574:  S*/
        -:  575:typedef struct MPID_Info {
        -:  576:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  577:    struct MPID_Info   *next;
        -:  578:    char               *key;
        -:  579:    char               *value;
        -:  580:} MPID_Info;
        -:  581:extern MPIU_Object_alloc_t MPID_Info_mem;
        -:  582:/* Preallocated info objects */
        -:  583:extern MPID_Info MPID_Info_direct[];
        -:  584:/* ------------------------------------------------------------------------- */
        -:  585:
        -:  586:/* ------------------------------------------------------------------------- */
        -:  587:/* Error Handlers */
        -:  588:/*E
        -:  589:  MPID_Errhandler_fn - MPID Structure to hold an error handler function
        -:  590:
        -:  591:  Notes:
        -:  592:  The MPI-1 Standard declared only the C version of this, implicitly 
        -:  593:  assuming that 'int' and 'MPI_Fint' were the same. 
        -:  594:
        -:  595:  Since Fortran does not have a C-style variable number of arguments 
        -:  596:  interface, the Fortran interface simply accepts two arguments.  Some
        -:  597:  calling conventions for Fortran (particularly under Windows) require
        -:  598:  this.
        -:  599:
        -:  600:  Module:
        -:  601:  ErrHand-DS
        -:  602:  
        -:  603:  Questions:
        -:  604:  What do we want to do about C++?  Do we want a hook for a routine that can
        -:  605:  be called to throw an exception in C++, particularly if we give C++ access
        -:  606:  to this structure?  Does the C++ handler need to be different (not part
        -:  607:  of the union)?
        -:  608:
        -:  609:  E*/
        -:  610:typedef union MPID_Errhandler_fn {
        -:  611:   void (*C_Comm_Handler_function) ( MPI_Comm *, int *, ... );
        -:  612:   void (*F77_Handler_function) ( MPI_Fint *, MPI_Fint * );
        -:  613:   void (*C_Win_Handler_function) ( MPI_Win *, int *, ... );
        -:  614:   void (*C_File_Handler_function) ( MPI_File *, int *, ... );
        -:  615:} MPID_Errhandler_fn;
        -:  616:
        -:  617:/*S
        -:  618:  MPID_Errhandler - Description of the error handler structure
        -:  619:
        -:  620:  Notes:
        -:  621:  Device-specific information may indicate whether the error handler is active;
        -:  622:  this can help prevent infinite recursion in error handlers caused by 
        -:  623:  user-error without requiring the user to be as careful.  We might want to 
        -:  624:  make this part of the interface so that the 'MPI_xxx_call_errhandler' 
        -:  625:  routines would check.
        -:  626:
        -:  627:  It is useful to have a way to indicate that the errhandler is no longer
        -:  628:  valid, to help catch the case where the user has freed the errhandler but
        -:  629:  is still using a copy of the 'MPI_Errhandler' value.  We may want to 
        -:  630:  define the 'id' value for deleted errhandlers.
        -:  631:
        -:  632:  Module:
        -:  633:  ErrHand-DS
        -:  634:  S*/
        -:  635:typedef struct MPID_Errhandler {
        -:  636:  MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  637:  MPID_Lang_t        language;
        -:  638:  MPID_Object_kind   kind;
        -:  639:  MPID_Errhandler_fn errfn;
        -:  640:  /* Other, device-specific information */
        -:  641:#ifdef MPID_DEV_ERRHANDLER_DECL
        -:  642:    MPID_DEV_ERRHANDLER_DECL
        -:  643:#endif
        -:  644:} MPID_Errhandler;
        -:  645:extern MPIU_Object_alloc_t MPID_Errhandler_mem;
        -:  646:/* Preallocated errhandler objects */
        -:  647:extern MPID_Errhandler MPID_Errhandler_builtin[];
        -:  648:extern MPID_Errhandler MPID_Errhandler_direct[];
        -:  649:
        -:  650:/* We never reference count the builtin error handler objects, regardless of how
        -:  651: * we decide to reference count the other predefined objects.  If we get to the
        -:  652: * point where we never reference count *any* of the builtin objects then we
        -:  653: * should probably remove these checks and let them fall through to the checks
        -:  654: * for BUILTIN down in the MPIU_Object_* routines. */
        -:  655:#define MPIR_Errhandler_add_ref( _errhand )                               \
        -:  656:    do {                                                                  \
        -:  657:        if (HANDLE_GET_KIND((_errhand)->handle) != HANDLE_KIND_BUILTIN) { \
        -:  658:            MPIU_Object_add_ref( _errhand );                              \
        -:  659:        }                                                                 \
        -:  660:    } while (0)
        -:  661:#define MPIR_Errhandler_release_ref( _errhand, _inuse )                   \
        -:  662:    do {                                                                  \
        -:  663:        if (HANDLE_GET_KIND((_errhand)->handle) != HANDLE_KIND_BUILTIN) { \
        -:  664:            MPIU_Object_release_ref( (_errhand), (_inuse) );              \
        -:  665:        }                                                                 \
        -:  666:        else {                                                            \
        -:  667:            *(_inuse) = 1;                                                \
        -:  668:        }                                                                 \
        -:  669:    } while (0)
        -:  670:/* ------------------------------------------------------------------------- */
        -:  671:
        -:  672:/* ------------------------------------------------------------------------- */
        -:  673:/* Keyvals and attributes */
        -:  674:/*TKyOverview.tex
        -:  675:
        -:  676:  Keyvals are MPI objects that, unlike most MPI objects, are defined to be
        -:  677:  integers rather than a handle (e.g., 'MPI_Comm').  However, they really
        -:  678:  `are` MPI opaque objects and are handled by the MPICH implementation in
        -:  679:  the same way as all other MPI opaque objects.  The only difference is that
        -:  680:  there is no 'typedef int MPI_Keyval;' in 'mpi.h'.  In particular, keyvals
        -:  681:  are encoded (for direct and indirect references) in the same way that 
        -:  682:  other MPI opaque objects are
        -:  683:
        -:  684:  Each keyval has a copy and a delete function associated with it.
        -:  685:  Unfortunately, these have a slightly different calling sequence for
        -:  686:  each language, particularly when the size of a pointer is 
        -:  687:  different from the size of a Fortran integer.  The unions 
        -:  688:  'MPID_Copy_function' and 'MPID_Delete_function' capture the differences
        -:  689:  in a single union type.
        -:  690:
        -:  691:  Notes:
        -:  692:  One potential user error is to access an attribute in one language (say
        -:  693:  Fortran) that was created in another (say C).  We could add a check and
        -:  694:  generate an error message in this case; note that this would have to 
        -:  695:  be an option, because (particularly when accessing the attribute from C), 
        -:  696:  it may be what the user intended, and in any case, it is a valid operation.
        -:  697:
        -:  698:  T*/
        -:  699:/*TAttrOverview.tex
        -:  700: *
        -:  701: * The MPI standard allows `attributes`, essentially an '(integer,pointer)'
        -:  702: * pair, to be attached to communicators, windows, and datatypes.  
        -:  703: * The integer is a `keyval`, which is allocated by a call (at the MPI level)
        -:  704: * to 'MPI_Comm/Type/Win_create_keyval'.  The pointer is the value of 
        -:  705: * the attribute.
        -:  706: * Attributes are primarily intended for use by the user, for example, to save
        -:  707: * information on a communicator, but can also be used to pass data to the
        -:  708: * MPI implementation.  For example, an attribute may be used to pass 
        -:  709: * Quality of Service information to an implementation to be used with 
        -:  710: * communication on a particular communicator.  
        -:  711: * To provide the most general access by the ADI to all attributes, the
        -:  712: * ADI defines a collection of routines that are used by the implementation
        -:  713: * of the MPI attribute routines (such as 'MPI_Comm_get_attr').
        -:  714: * In addition, the MPI routines involving attributes will invoke the 
        -:  715: * corresponding 'hook' functions (e.g., 'MPID_Dev_comm_attr_set_hook') 
        -:  716: * should the device define them.
        -:  717: *
        -:  718: * Attributes on windows and datatypes are defined by MPI but not of 
        -:  719: * interest (as yet) to the device.
        -:  720: *
        -:  721: * In addition, there are seven predefined attributes that the device must
        -:  722: * supply to the implementation.  This is accomplished through 
        -:  723: * data values that are part of the 'MPIR_Process' data block.
        -:  724: *  The predefined keyvals on 'MPI_COMM_WORLD' are\:
        -:  725: *.vb
        -:  726: * Keyval                     Related Module
        -:  727: * MPI_APPNUM                 Dynamic
        -:  728: * MPI_HOST                   Core
        -:  729: * MPI_IO                     Core
        -:  730: * MPI_LASTUSEDCODE           Error
        -:  731: * MPI_TAG_UB                 Communication
        -:  732: * MPI_UNIVERSE_SIZE          Dynamic
        -:  733: * MPI_WTIME_IS_GLOBAL        Timer
        -:  734: *.ve
        -:  735: * The values stored in the 'MPIR_Process' block are the actual values.  For 
        -:  736: * example, the value of 'MPI_TAG_UB' is the integer value of the largest tag.
        -:  737: * The
        -:  738: * value of 'MPI_WTIME_IS_GLOBAL' is a '1' for true and '0' for false.  Likely
        -:  739: * values for 'MPI_IO' and 'MPI_HOST' are 'MPI_ANY_SOURCE' and 'MPI_PROC_NULL'
        -:  740: * respectively.
        -:  741: *
        -:  742: T*/
        -:  743:
        -:  744:/* Include the attribute access routines that permit access to the 
        -:  745:   attribute or its pointer, needed for cross-language access to attributes */
        -:  746:#include "mpi_attr.h"
        -:  747:
        -:  748:/* Because Comm, Datatype, and File handles are all ints, and because
        -:  749:   attributes are otherwise identical between the three types, we
        -:  750:   only store generic copy and delete functions.  This allows us to use
        -:  751:   common code for the attribute set, delete, and dup functions */
        -:  752:/*E
        -:  753:  MPID_Copy_function - MPID Structure to hold an attribute copy function
        -:  754:
        -:  755:  Notes:
        -:  756:  The appropriate element of this union is selected by using the language
        -:  757:  field of the 'keyval'.
        -:  758:
        -:  759:  Because 'MPI_Comm', 'MPI_Win', and 'MPI_Datatype' are all 'int's in 
        -:  760:  MPICH2, we use a single C copy function rather than have separate
        -:  761:  ones for the Communicator, Window, and Datatype attributes.
        -:  762:
        -:  763:  There are no corresponding typedefs for the Fortran functions.  The 
        -:  764:  F77 function corresponds to the Fortran 77 binding used in MPI-1 and the
        -:  765:  F90 function corresponds to the Fortran 90 binding used in MPI-2.
        -:  766:
        -:  767:  Module:
        -:  768:  Attribute-DS
        -:  769:
        -:  770:  E*/
        -:  771:int
        -:  772:MPIR_Attr_copy_c_proxy(
        -:  773:    MPI_Comm_copy_attr_function* user_function,
        -:  774:    int handle,
        -:  775:    int keyval,
        -:  776:    void* extra_state,
        -:  777:    MPIR_AttrType attrib_type,
        -:  778:    void* attrib,
        -:  779:    void** attrib_copy,
        -:  780:    int* flag
        -:  781:    );
        -:  782:
        -:  783:typedef struct MPID_Copy_function {
        -:  784:  int  (*C_CopyFunction)( int, int, void *, void *, void *, int * );
        -:  785:  void (*F77_CopyFunction)  ( MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, 
        -:  786:                              MPI_Fint *, MPI_Fint *, MPI_Fint * );
        -:  787:  void (*F90_CopyFunction)  ( MPI_Fint *, MPI_Fint *, MPI_Aint *, MPI_Aint *,
        -:  788:                              MPI_Aint *, MPI_Fint *, MPI_Fint * );
        -:  789:  /* The generic lang-independent user_function and proxy will
        -:  790:   * replace the lang dependent copy funcs above
        -:  791:   * Currently the lang-indpendent funcs are used only for keyvals
        -:  792:   */
        -:  793:  MPI_Comm_copy_attr_function *user_function;
        -:  794:  MPID_Attr_copy_proxy *proxy;
        -:  795:  /* The C++ function is the same as the C function */
        -:  796:} MPID_Copy_function;
        -:  797:
        -:  798:/*E
        -:  799:  MPID_Delete_function - MPID Structure to hold an attribute delete function
        -:  800:
        -:  801:  Notes:
        -:  802:  The appropriate element of this union is selected by using the language
        -:  803:  field of the 'keyval'.
        -:  804:
        -:  805:  Because 'MPI_Comm', 'MPI_Win', and 'MPI_Datatype' are all 'int's in 
        -:  806:  MPICH2, we use a single C delete function rather than have separate
        -:  807:  ones for the Communicator, Window, and Datatype attributes.
        -:  808:
        -:  809:  There are no corresponding typedefs for the Fortran functions.  The 
        -:  810:  F77 function corresponds to the Fortran 77 binding used in MPI-1 and the
        -:  811:  F90 function corresponds to the Fortran 90 binding used in MPI-2.
        -:  812:
        -:  813:  Module:
        -:  814:  Attribute-DS
        -:  815:
        -:  816:  E*/
        -:  817:int
        -:  818:MPIR_Attr_delete_c_proxy(
        -:  819:    MPI_Comm_delete_attr_function* user_function,
        -:  820:    int handle,
        -:  821:    int keyval,
        -:  822:    MPIR_AttrType attrib_type,
        -:  823:    void* attrib,
        -:  824:    void* extra_state
        -:  825:    );
        -:  826:
        -:  827:typedef struct MPID_Delete_function {
        -:  828:  int  (*C_DeleteFunction)  ( int, int, void *, void * );
        -:  829:  void (*F77_DeleteFunction)( MPI_Fint *, MPI_Fint *, MPI_Fint *, MPI_Fint *, 
        -:  830:                              MPI_Fint * );
        -:  831:  void (*F90_DeleteFunction)( MPI_Fint *, MPI_Fint *, MPI_Aint *, MPI_Aint *, 
        -:  832:                              MPI_Fint * );
        -:  833:  /* The generic lang-independent user_function and proxy will
        -:  834:   * replace the lang dependent copy funcs above
        -:  835:   * Currently the lang-indpendent funcs are used only for keyvals
        -:  836:   */
        -:  837:  MPI_Comm_delete_attr_function *user_function;
        -:  838:  MPID_Attr_delete_proxy *proxy;
        -:  839:} MPID_Delete_function;
        -:  840:
        -:  841:/*S
        -:  842:  MPID_Keyval - Structure of an MPID keyval
        -:  843:
        -:  844:  Module:
        -:  845:  Attribute-DS
        -:  846:
        -:  847:  S*/
        -:  848:typedef struct MPID_Keyval {
        -:  849:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  850:    MPID_Object_kind     kind;
        -:  851:    int                  was_freed;
        -:  852:    void                 *extra_state;
        -:  853:    MPID_Copy_function   copyfn;
        -:  854:    MPID_Delete_function delfn;
        -:  855:  /* other, device-specific information */
        -:  856:#ifdef MPID_DEV_KEYVAL_DECL
        -:  857:    MPID_DEV_KEYVAL_DECL
        -:  858:#endif
        -:  859:} MPID_Keyval;
        -:  860:
        -:  861:#define MPIR_Keyval_add_ref( _keyval )                                  \
        -:  862:    do {                                                                \
        -:  863:        MPIU_Object_add_ref( _keyval );                                 \
        -:  864:    } while(0)
        -:  865:
        -:  866:#define MPIR_Keyval_release_ref( _keyval, _inuse )                      \
        -:  867:    do {                                                                \
        -:  868:        MPIU_Object_release_ref( _keyval, _inuse );                     \
        -:  869:    } while(0)
        -:  870:
        -:  871:
        -:  872:/* Attribute values in C/C++ are void * and in Fortran are ADDRESS_SIZED
        -:  873:   integers.  Normally, these are the same size, but in at least one 
        -:  874:   case, the address-sized integers was selected as longer than void *
        -:  875:   to work with the datatype code used in the I/O library.  While this
        -:  876:   is really a limitation in the current Datatype implementation. */
        -:  877:#ifdef USE_AINT_FOR_ATTRVAL
        -:  878:typedef MPI_Aint MPID_AttrVal_t;
        -:  879:#else
        -:  880:typedef void * MPID_AttrVal_t;
        -:  881:#endif
        -:  882:
        -:  883:/* Attributes need no ref count or handle, but since we want to use the
        -:  884:   common block allocator for them, we must provide those elements 
        -:  885:*/
        -:  886:/*S
        -:  887:  MPID_Attribute - Structure of an MPID attribute
        -:  888:
        -:  889:  Notes:
        -:  890:  Attributes don''t have 'ref_count's because they don''t have reference
        -:  891:  count semantics.  That is, there are no shallow copies or duplicates
        -:  892:  of an attibute.  An attribute is copied when the communicator that
        -:  893:  it is attached to is duplicated.  Subsequent operations, such as
        -:  894:  'MPI_Comm_attr_free', can change the attribute list for one of the
        -:  895:  communicators but not the other, making it impractical to keep the
        -:  896:  same list.  (We could defer making the copy until the list is changed,
        -:  897:  but even then, there would be no reference count on the individual
        -:  898:  attributes.)
        -:  899: 
        -:  900:  A pointer to the keyval, rather than the (integer) keyval itself is
        -:  901:  used since there is no need within the attribute structure to make
        -:  902:  it any harder to find the keyval structure.
        -:  903:
        -:  904:  The attribute value is a 'void *'.  If 'sizeof(MPI_Fint)' > 'sizeof(void*)',
        -:  905:  then this must be changed (no such system has been encountered yet).
        -:  906:  For the Fortran 77 routines in the case where 'sizeof(MPI_Fint)' < 
        -:  907:  'sizeof(void*)', the high end of the 'void *' value is used.  That is,
        -:  908:  we cast it to 'MPI_Fint *' and use that value.
        -:  909:
        -:  910:  MPI defines three kinds of attributes (see MPI 2.1, Section 16.3, pages 
        -:  911:  487-488 (the standard says two, but there are really three, as discussed
        -:  912:  below).  These are pointer-valued attributes and two types of integer-valued
        -:  913:  attributes.  
        -:  914:  Pointer-valued attributes are used in C.
        -:  915:  Integer-valued attributes are used in Fortran.  These are of type either
        -:  916:  INTEGER or INTEGER(KIND=MPI_ADDRESS_KIND).
        -:  917:
        -:  918:  The predefined attributes are a combination of INTEGER and pointers.
        -:  919: 
        -:  920:  Module:
        -:  921:  Attribute-DS
        -:  922:
        -:  923: S*/
        -:  924:typedef struct MPID_Attribute {
        -:  925:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  926:    MPID_Keyval  *keyval;           /* Keyval structure for this attribute */
        -:  927:
        -:  928:    struct MPID_Attribute *next;    /* Pointer to next in the list */
        -:  929:    MPIR_AttrType attrType;         /* Type of the attribute */
        -:  930:    long        pre_sentinal;       /* Used to detect user errors in accessing
        -:  931:				       the value */
        -:  932:    MPID_AttrVal_t value;           /* Stored value. An Aint must be at least
        -:  933:				       as large as an address - some builds
        -:  934:				       may make an Aint larger than a void * */
        -:  935:    long        post_sentinal;      /* Like pre_sentinal */
        -:  936:    /* other, device-specific information */
        -:  937:#ifdef MPID_DEV_ATTR_DECL
        -:  938:    MPID_DEV_ATTR_DECL
        -:  939:#endif
        -:  940:} MPID_Attribute;
        -:  941:/* ------------------------------------------------------------------------- */
        -:  942:
        -:  943:/*---------------------------------------------------------------------------
        -:  944: * Groups are *not* a major data structure in MPICH-2.  They are provided
        -:  945: * only because they are required for the group operations (e.g., 
        -:  946: * MPI_Group_intersection) and for the scalable RMA synchronization
        -:  947: *---------------------------------------------------------------------------*/
        -:  948:/* This structure is used to implement the group operations such as 
        -:  949:   MPI_Group_translate_ranks */
        -:  950:typedef struct MPID_Group_pmap_t {
        -:  951:    int          lrank;     /* Local rank in group (between 0 and size-1) */
        -:  952:    int          lpid;      /* local process id, from VCONN */
        -:  953:    int          next_lpid; /* Index of next lpid (in lpid order) */
        -:  954:    int          flag;      /* marker, used to implement group operations */
        -:  955:} MPID_Group_pmap_t;
        -:  956:
        -:  957:/* Any changes in the MPID_Group structure must be made to the
        -:  958:   predefined value in MPID_Group_builtin for MPI_GROUP_EMPTY in 
        -:  959:   src/mpi/group/grouputil.c */
        -:  960:/*S
        -:  961: MPID_Group - Description of the Group data structure
        -:  962:
        -:  963: The processes in the group of 'MPI_COMM_WORLD' have lpid values 0 to 'size'-1,
        -:  964: where 'size' is the size of 'MPI_COMM_WORLD'.  Processes created by 
        -:  965: 'MPI_Comm_spawn' or 'MPI_Comm_spawn_multiple' or added by 'MPI_Comm_attach' 
        -:  966: or  
        -:  967: 'MPI_Comm_connect'
        -:  968: are numbered greater than 'size - 1' (on the calling process). See the 
        -:  969: discussion of LocalPID values.
        -:  970:
        -:  971: Note that when dynamic process creation is used, the pids are `not` unique
        -:  972: across the universe of connected MPI processes.  This is ok, as long as
        -:  973: pids are interpreted `only` on the process that owns them.
        -:  974:
        -:  975: Only for MPI-1 are the lpid''s equal to the `global` pids.  The local pids
        -:  976: can be thought of as a reference not to the remote process itself, but
        -:  977: how the remote process can be reached from this process.  We may want to 
        -:  978: have a structure 'MPID_Lpid_t' that contains information on the remote
        -:  979: process, such as (for TCP) the hostname, ip address (it may be different if
        -:  980: multiple interfaces are supported; we may even want plural ip addresses for
        -:  981: stripping communication), and port (or ports).  For shared memory connected
        -:  982: processes, it might have the address of a remote queue.  The lpid number 
        -:  983: is an index into a table of 'MPID_Lpid_t'''s that contain this (device- and
        -:  984: method-specific) information.
        -:  985:
        -:  986: Module:
        -:  987: Group-DS
        -:  988:
        -:  989: S*/
        -:  990:typedef struct MPID_Group {
        -:  991:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -:  992:    int          size;           /* Size of a group */
        -:  993:    int          rank;           /* rank of this process relative to this 
        -:  994:				    group */
        -:  995:    int          idx_of_first_lpid;
        -:  996:    MPID_Group_pmap_t *lrank_to_lpid; /* Array mapping a local rank to local 
        -:  997:					 process number */
        -:  998:    /* We may want some additional data for the RMA syncrhonization calls */
        -:  999:  /* Other, device-specific information */
        -: 1000:#ifdef MPID_DEV_GROUP_DECL
        -: 1001:    MPID_DEV_GROUP_DECL
        -: 1002:#endif
        -: 1003:} MPID_Group;
        -: 1004:
        -: 1005:extern MPIU_Object_alloc_t MPID_Group_mem;
        -: 1006:/* Preallocated group objects */
        -: 1007:#define MPID_GROUP_N_BUILTIN 1
        -: 1008:extern MPID_Group MPID_Group_builtin[MPID_GROUP_N_BUILTIN];
        -: 1009:extern MPID_Group MPID_Group_direct[];
        -: 1010:
        -: 1011:#define MPIR_Group_add_ref( _group ) \
        -: 1012:    do { MPIU_Object_add_ref( _group ); } while (0)
        -: 1013:
        -: 1014:#define MPIR_Group_release_ref( _group, _inuse ) \
        -: 1015:     do { MPIU_Object_release_ref( _group, _inuse ); } while (0)
        -: 1016:
        -: 1017:void MPIR_Group_setup_lpid_list( MPID_Group * );
        -: 1018:int MPIR_GroupCheckVCRSubset( MPID_Group *group_ptr, int vsize, MPID_VCR *vcr, int *idx );
        -: 1019:
        -: 1020:/* ------------------------------------------------------------------------- */
        -: 1021:
        -: 1022:/*E
        -: 1023:  MPID_Comm_kind_t - Name the two types of communicators
        -: 1024:  E*/
        -: 1025:typedef enum MPID_Comm_kind_t { 
        -: 1026:    MPID_INTRACOMM = 0, 
        -: 1027:    MPID_INTERCOMM = 1 } MPID_Comm_kind_t;
        -: 1028:/* Communicators */
        -: 1029:
        -: 1030:/*S
        -: 1031:  MPID_Comm - Description of the Communicator data structure
        -: 1032:
        -: 1033:  Notes:
        -: 1034:  Note that the size and rank duplicate data in the groups that
        -: 1035:  make up this communicator.  These are used often enough that this
        -: 1036:  optimization is valuable.  
        -: 1037:
        -: 1038:  This definition provides only a 16-bit integer for context id''s .
        -: 1039:  This should be sufficient for most applications.  However, extending
        -: 1040:  this to a 32-bit (or longer) integer should be easy.
        -: 1041:
        -: 1042:  There are two context ids.  One is used for sending and one for 
        -: 1043:  receiving.  In the case of an Intracommunicator, they are the same
        -: 1044:  context id.  They differ in the case of intercommunicators, where 
        -: 1045:  they may come from processes in different comm worlds (in the
        -: 1046:  case of MPI-2 dynamic process intercomms).  
        -: 1047:
        -: 1048:  The virtual connection table is an explicit member of this structure.
        -: 1049:  This contains the information used to contact a particular process,
        -: 1050:  indexed by the rank relative to this communicator.
        -: 1051:
        -: 1052:  Groups are allocated lazily.  That is, the group pointers may be
        -: 1053:  null, created only when needed by a routine such as 'MPI_Comm_group'.
        -: 1054:  The local process ids needed to form the group are available within
        -: 1055:  the virtual connection table.
        -: 1056:  For intercommunicators, we may want to always have the groups.  If not, 
        -: 1057:  we either need the 'local_group' or we need a virtual connection table
        -: 1058:  corresponding to the 'local_group' (we may want this anyway to simplify
        -: 1059:  the implementation of the intercommunicator collective routines).
        -: 1060:
        -: 1061:  The pointer to the structure 'MPID_Collops' containing pointers to the 
        -: 1062:  collective  
        -: 1063:  routines allows an implementation to replace each routine on a 
        -: 1064:  routine-by-routine basis.  By default, this pointer is null, as are the 
        -: 1065:  pointers within the structure.  If either pointer is null, the implementation
        -: 1066:  uses the generic provided implementation.  This choice, rather than
        -: 1067:  initializing the table with pointers to all of the collective routines,
        -: 1068:  is made to reduce the space used in the communicators and to eliminate the
        -: 1069:  need to include the implementation of all collective routines in all MPI 
        -: 1070:  executables, even if the routines are not used.
        -: 1071:
        -: 1072:  The macro 'MPID_HAS_HETERO' may be defined by a device to indicate that
        -: 1073:  the device supports MPI programs that must communicate between processes with
        -: 1074:  different data representations (e.g., different sized integers or different
        -: 1075:  byte orderings).  If the device does need to define this value, it should
        -: 1076:  be defined in the file 'mpidpre.h'. 
        -: 1077:
        -: 1078:  Please note that the local_size and remote_size fields can be confusing.  For
        -: 1079:  intracommunicators both fields are always equal to the size of the
        -: 1080:  communicator.  For intercommunicators local_size is equal to the size of
        -: 1081:  local_group while remote_size is equal to the size of remote_group.
        -: 1082:
        -: 1083:  Module:
        -: 1084:  Communicator-DS
        -: 1085:
        -: 1086:  Question:
        -: 1087:  For fault tolerance, do we want to have a standard field for communicator 
        -: 1088:  health?  For example, ok, failure detected, all (live) members of failed 
        -: 1089:  communicator have acked.
        -: 1090:  S*/
        -: 1091:typedef struct MPID_Comm {
        -: 1092:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1093:    MPIR_Context_id_t context_id; /* Send context id.  See notes */
        -: 1094:    MPIR_Context_id_t recvcontext_id; /* Send context id.  See notes */
        -: 1095:    int           remote_size;   /* Value of MPI_Comm_(remote)_size */
        -: 1096:    int           rank;          /* Value of MPI_Comm_rank */
        -: 1097:    MPID_VCRT     vcrt;          /* virtual connecton reference table */
        -: 1098:    MPID_VCR *    vcr;           /* alias to the array of virtual connections
        -: 1099:				    in vcrt */
        -: 1100:    MPID_VCRT     local_vcrt;    /* local virtual connecton reference table */
        -: 1101:    MPID_VCR *    local_vcr;     /* alias to the array of local virtual
        -: 1102:				    connections in local vcrt */
        -: 1103:    MPID_Attribute *attributes;  /* List of attributes */
        -: 1104:    int           local_size;    /* Value of MPI_Comm_size for local group */
        -: 1105:    MPID_Group   *local_group,   /* Groups in communicator. */
        -: 1106:                 *remote_group;  /* The local and remote groups are the
        -: 1107:                                    same for intra communicators */
        -: 1108:    MPID_Comm_kind_t comm_kind;  /* MPID_INTRACOMM or MPID_INTERCOMM */
        -: 1109:    char          name[MPI_MAX_OBJECT_NAME];  /* Required for MPI-2 */
        -: 1110:    MPID_Errhandler *errhandler; /* Pointer to the error handler structure */
        -: 1111:    struct MPID_Comm    *local_comm; /* Defined only for intercomms, holds
        -: 1112:				        an intracomm for the local group */
        -: 1113:    int is_node_aware;           /* true if node topology info is available,
        -: 1114:                                    such as node_comm and node_roots_comm */
        -: 1115:    struct MPID_Comm *node_comm; /* Comm of processes in this comm that are on
        -: 1116:                                    the same node as this process. */
        -: 1117:    struct MPID_Comm *node_roots_comm; /* Comm of root processes for other nodes. */
        -: 1118:    int *intranode_table;        /* intranode_table[i] gives the rank in
        -: 1119:                                    node_comm of rank i in this comm or -1 if i
        -: 1120:                                    is not in this process' node_comm.
        -: 1121:                                    It is of size 'local_size'. */
        -: 1122:    int *internode_table;        /* internode_table[i] gives the rank in
        -: 1123:                                    node_roots_comm of rank i in this comm.
        -: 1124:                                    It is of size 'local_size'. */
        -: 1125:    int           is_low_group;  /* For intercomms only, this boolean is
        -: 1126:				    set for all members of one of the 
        -: 1127:				    two groups of processes and clear for 
        -: 1128:				    the other.  It enables certain
        -: 1129:				    intercommunicator collective operations
        -: 1130:				    that wish to use half-duplex operations
        -: 1131:				    to implement a full-duplex operation */
        -: 1132:    struct MPID_Comm     *comm_next;/* Provides a chain through all active 
        -: 1133:				       communicators */
        -: 1134:    struct MPID_Collops  *coll_fns; /* Pointer to a table of functions 
        -: 1135:                                              implementing the collective 
        -: 1136:                                              routines */
        -: 1137:    struct MPID_TopoOps  *topo_fns; /* Pointer to a table of functions
        -: 1138:				       implementting the topology routines
        -: 1139:				    */
        -: 1140:#ifdef MPID_HAS_HETERO
        -: 1141:    int is_hetero;
        -: 1142:#endif
        -: 1143:  /* Other, device-specific information */
        -: 1144:#ifdef MPID_DEV_COMM_DECL
        -: 1145:    MPID_DEV_COMM_DECL
        -: 1146:#endif
        -: 1147:} MPID_Comm;
        -: 1148:extern MPIU_Object_alloc_t MPID_Comm_mem;
        -: 1149:
        -: 1150:/* MPIR_Comm_release is a helper routine that releases references to a comm.
        -: 1151:   The second arg is false unless this is called as part of 
        -: 1152:   MPI_Comm_disconnect .
        -: 1153:*/
        -: 1154:int MPIR_Comm_release(MPID_Comm *, int );
        -: 1155:int MPIR_Comm_release_always(MPID_Comm *comm_ptr, int isDisconnect);
        -: 1156:
        -: 1157:#define MPIR_Comm_add_ref(_comm) \
        -: 1158:    do { MPIU_Object_add_ref((_comm)); } while (0)
        -: 1159:
        -: 1160:#define MPIR_Comm_release_ref( _comm, _inuse ) \
        -: 1161:    do { MPIU_Object_release_ref( _comm, _inuse ); } while (0)
        -: 1162:
        -: 1163:/* Preallocated comm objects.  There are 3: comm_world, comm_self, and 
        -: 1164:   a private (non-user accessible) dup of comm world that is provided 
        -: 1165:   if needed in MPI_Finalize.  Having a separate version of comm_world
        -: 1166:   avoids possible interference with User code */
        -: 1167:#define MPID_COMM_N_BUILTIN 3
        -: 1168:extern MPID_Comm MPID_Comm_builtin[MPID_COMM_N_BUILTIN];
        -: 1169:extern MPID_Comm MPID_Comm_direct[];
        -: 1170:/* This is the handle for the internal MPI_COMM_WORLD .  The "2" at the end
        -: 1171:   of the handle is 3-1 (e.g., the index in the builtin array) */
        -: 1172:#define MPIR_ICOMM_WORLD  ((MPI_Comm)0x44000002)
        -: 1173:
        -: 1174:/* The following preprocessor macros provide bitfield access information for
        -: 1175: * context ID values.  They follow a uniform naming pattern:
        -: 1176: *
        -: 1177: * MPID_CONTEXT_foo_WIDTH - the width in bits of the field
        -: 1178: * MPID_CONTEXT_foo_MASK  - A valid bit mask for bit-wise AND and OR operations
        -: 1179: *                          with exactly all of the bits in the field set.
        -: 1180: * MPID_CONTEXT_foo_SHIFT - The number of bits that the field should be shifted
        -: 1181: *                          rightwards to place it in the least significant bits
        -: 1182: *                          of the ID.  There may still be higher order bits
        -: 1183: *                          from other fields, so the _MASK should be used first
        -: 1184: *                          if you want to reliably retrieve the exact value of
        -: 1185: *                          the field.
        -: 1186: */
        -: 1187:
        -: 1188:/* yields an rvalue that is the value of the field_name_ in the least significant bits */
        -: 1189:#define MPID_CONTEXT_READ_FIELD(field_name_,id_) \
        -: 1190:    (((id_) & MPID_CONTEXT_##field_name_##_MASK) >> MPID_CONTEXT_##field_name_##_SHIFT)
        -: 1191:/* yields an rvalue that is the old_id_ with field_name_ set to field_val_ */
        -: 1192:#define MPID_CONTEXT_SET_FIELD(field_name_,old_id_,field_val_) \
        -: 1193:    ((old_id_ & ~MPID_CONTEXT_##field_name_##_MASK) | ((field_val_) << MPID_CONTEXT_##field_name_##_SHIFT))
        -: 1194:
        -: 1195:/* Context suffixes for separating pt2pt and collective communication */
        -: 1196:#define MPID_CONTEXT_SUFFIX_WIDTH (1)
        -: 1197:#define MPID_CONTEXT_SUFFIX_SHIFT (0)
        -: 1198:#define MPID_CONTEXT_SUFFIX_MASK ((1 << MPID_CONTEXT_SUFFIX_WIDTH) - 1)
        -: 1199:#define MPID_CONTEXT_INTRA_PT2PT (0)
        -: 1200:#define MPID_CONTEXT_INTRA_COLL  (1)
        -: 1201:#define MPID_CONTEXT_INTER_PT2PT (0)
        -: 1202:#define MPID_CONTEXT_INTER_COLL  (1)
        -: 1203:
        -: 1204:/* Used to derive context IDs for sub-communicators from a parent communicator's
        -: 1205:   context ID value.  This field comes after the one bit suffix.
        -: 1206:   values are shifted left by 1. */
        -: 1207:#define MPID_CONTEXT_SUBCOMM_WIDTH (2)
        -: 1208:#define MPID_CONTEXT_SUBCOMM_SHIFT (MPID_CONTEXT_SUFFIX_WIDTH + MPID_CONTEXT_SUFFIX_SHIFT)
        -: 1209:#define MPID_CONTEXT_SUBCOMM_MASK      (((1 << MPID_CONTEXT_SUBCOMM_WIDTH) - 1) << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1210:
        -: 1211:/* these values may be added/subtracted directly to/from an existing context ID
        -: 1212: * in order to determine the context ID of the child/parent */
        -: 1213:#define MPID_CONTEXT_PARENT_OFFSET    (0 << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1214:#define MPID_CONTEXT_INTRANODE_OFFSET (1 << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1215:#define MPID_CONTEXT_INTERNODE_OFFSET (2 << MPID_CONTEXT_SUBCOMM_SHIFT)
        -: 1216:
        -: 1217:/* this field (IS_LOCALCOM) is used to derive a context ID for local
        -: 1218: * communicators of intercommunicators without communication */
        -: 1219:#define MPID_CONTEXT_IS_LOCALCOMM_WIDTH (1)
        -: 1220:#define MPID_CONTEXT_IS_LOCALCOMM_SHIFT (MPID_CONTEXT_SUBCOMM_SHIFT + MPID_CONTEXT_SUBCOMM_WIDTH)
        -: 1221:#define MPID_CONTEXT_IS_LOCALCOMM_MASK (((1 << MPID_CONTEXT_IS_LOCALCOMM_WIDTH) - 1) << MPID_CONTEXT_IS_LOCALCOMM_SHIFT)
        -: 1222:
        -: 1223:/* MPIR_MAX_CONTEXT_MASK is the number of ints that make up the bit vector that
        -: 1224: * describes the context ID prefix space.
        -: 1225: *
        -: 1226: * The following must hold:
        -: 1227: * (num_bits_in_vector) <= (maximum_context_id_prefix)
        -: 1228: *   which is the following in concrete terms:
        -: 1229: * MPIR_MAX_CONTEXT_MASK*MPIR_CONTEXT_INT_BITS <= 2**(MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_WIDTH))
        -: 1230: *
        -: 1231: * We currently always assume MPIR_CONTEXT_INT_BITS is 32, regardless of the
        -: 1232: * value of sizeof(int)*CHAR_BITS.  We also make the assumption that CHAR_BITS==8.
        -: 1233: *
        -: 1234: * For a 16-bit context id field and CHAR_BITS==8, this implies MPIR_MAX_CONTEXT_MASK <= 256
        -: 1235: */
        -: 1236:
        -: 1237:/* number of bits to shift right by in order to obtain the context ID prefix */
        -: 1238:#define MPID_CONTEXT_PREFIX_SHIFT (MPID_CONTEXT_IS_LOCALCOMM_SHIFT + MPID_CONTEXT_IS_LOCALCOMM_WIDTH)
        -: 1239:#define MPID_CONTEXT_PREFIX_WIDTH (MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_WIDTH))
        -: 1240:#define MPID_CONTEXT_PREFIX_MASK (((1 << MPID_CONTEXT_PREFIX_WIDTH) - 1) << MPID_CONTEXT_PREFIX_SHIFT)
        -: 1241:
        -: 1242:#define MPID_CONTEXT_DYNAMIC_PROC_WIDTH (1) /* the upper half is reserved for dynamic procs */
        -: 1243:#define MPID_CONTEXT_DYNAMIC_PROC_SHIFT (MPIR_CONTEXT_ID_BITS - MPID_CONTEXT_DYNAMIC_PROC_WIDTH) /* the upper half is reserved for dynamic procs */
        -: 1244:#define MPID_CONTEXT_DYNAMIC_PROC_MASK (((1 << MPID_CONTEXT_DYNAMIC_PROC_WIDTH) - 1) << MPID_CONTEXT_DYNAMIC_PROC_SHIFT)
        -: 1245:
        -: 1246:/* should probably be (sizeof(int)*CHAR_BITS) once we make the code CHAR_BITS-clean */
        -: 1247:#define MPIR_CONTEXT_INT_BITS (32)
        -: 1248:#define MPIR_CONTEXT_ID_BITS (sizeof(MPIR_Context_id_t)*8) /* 8 --> CHAR_BITS eventually */
        -: 1249:#define MPIR_MAX_CONTEXT_MASK \
        -: 1250:    ((1 << (MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_WIDTH))) / MPIR_CONTEXT_INT_BITS)
        -: 1251:
        -: 1252:/* Utility routines.  Where possible, these are kept in the source directory
        -: 1253:   with the other comm routines (src/mpi/comm, in mpicomm.h).  However,
        -: 1254:   to create a new communicator after a spawn or connect-accept operation, 
        -: 1255:   the device may need to create a new contextid */
        -: 1256:int MPIR_Get_contextid( MPID_Comm *, MPIR_Context_id_t *context_id );
        -: 1257:
        -: 1258:/* ------------------------------------------------------------------------- */
        -: 1259:
        -: 1260:/* Requests */
        -: 1261:/* This currently defines a single structure type for all requests.  
        -: 1262:   Eventually, we may want a union type, as used in MPICH-1 */
        -: 1263:/*E
        -: 1264:  MPID_Request_kind - Kinds of MPI Requests
        -: 1265:
        -: 1266:  Module:
        -: 1267:  Request-DS
        -: 1268:
        -: 1269:  E*/
        -: 1270:typedef enum MPID_Request_kind_t {
        -: 1271:    MPID_REQUEST_UNDEFINED,
        -: 1272:    MPID_REQUEST_SEND,
        -: 1273:    MPID_REQUEST_RECV,
        -: 1274:    MPID_PREQUEST_SEND,
        -: 1275:    MPID_PREQUEST_RECV,
        -: 1276:    MPID_UREQUEST,
        -: 1277:    MPID_LAST_REQUEST_KIND
        -: 1278:#ifdef MPID_DEV_REQUEST_KIND_DECL
        -: 1279:    , MPID_DEV_REQUEST_KIND_DECL
        -: 1280:#endif
        -: 1281:} MPID_Request_kind_t;
        -: 1282:
        -: 1283:/* Typedefs for Fortran generalized requests */
        -: 1284:typedef void (MPIR_Grequest_f77_cancel_function)(void *, int*, int *); 
        -: 1285:typedef void (MPIR_Grequest_f77_free_function)(void *, int *); 
        -: 1286:typedef void (MPIR_Grequest_f77_query_function)(void *, MPI_Status *, int *); 
        -: 1287:
        -: 1288:/*S
        -: 1289:  MPID_Request - Description of the Request data structure
        -: 1290:
        -: 1291:  Module:
        -: 1292:  Request-DS
        -: 1293:
        -: 1294:  Notes:
        -: 1295:  If it is necessary to remember the MPI datatype, this information is 
        -: 1296:  saved within the device-specific fields provided by 'MPID_DEV_REQUEST_DECL'.
        -: 1297:
        -: 1298:  Requests come in many flavors, as stored in the 'kind' field.  It is 
        -: 1299:  expected that each kind of request will have its own structure type 
        -: 1300:  (e.g., 'MPID_Request_send_t') that extends the 'MPID_Request'.
        -: 1301:  
        -: 1302:  S*/
        -: 1303:typedef struct MPID_Request {
        -: 1304:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1305:    MPID_Request_kind_t kind;
        -: 1306:    /* completion counter */
        -: 1307:    volatile int cc;
        -: 1308:    /* pointer to the completion counter */
        -: 1309:    /* This is necessary for the case when an operation is described by a 
        -: 1310:       list of requests */
        -: 1311:    int volatile *cc_ptr;
        -: 1312:    /* A comm is needed to find the proper error handler */
        -: 1313:    MPID_Comm *comm;
        -: 1314:    /* Status is needed for wait/test/recv */
        -: 1315:    MPI_Status status;
        -: 1316:    /* Persistent requests have their own "real" requests.  Receive requests
        -: 1317:       have partnering send requests when src=dest. etc. */
        -: 1318:    struct MPID_Request *partner_request;
        -: 1319:    /* User-defined request support */
        -: 1320:    MPI_Grequest_cancel_function *cancel_fn;
        -: 1321:    MPI_Grequest_free_function   *free_fn;
        -: 1322:    MPI_Grequest_query_function  *query_fn;
        -: 1323:    MPIX_Grequest_poll_function   *poll_fn;
        -: 1324:    MPIX_Grequest_wait_function   *wait_fn;
        -: 1325:    void             *grequest_extra_state;
        -: 1326:    MPIX_Grequest_class	        greq_class;
        -: 1327:    MPID_Lang_t                  greq_lang;         /* language that defined
        -: 1328:						       the generalize req */
        -: 1329:    
        -: 1330:    /* Other, device-specific information */
        -: 1331:#ifdef MPID_DEV_REQUEST_DECL
        -: 1332:    MPID_DEV_REQUEST_DECL
        -: 1333:#endif
        -: 1334:} MPID_Request;
        -: 1335:
        -: 1336:extern MPIU_Object_alloc_t MPID_Request_mem;
        -: 1337:/* Preallocated request objects */
        -: 1338:extern MPID_Request MPID_Request_direct[];
        -: 1339:
        -: 1340:#define MPIR_Request_add_ref( _req ) \
        -: 1341:    do { MPIU_Object_add_ref( _req ); } while (0)
        -: 1342:
        -: 1343:#define MPIR_Request_release_ref( _req, _inuse ) \
        -: 1344:    do { MPIU_Object_release_ref( _req, _inuse ); } while (0)
        -: 1345:
        -: 1346:/* These macros allow us to implement a sendq when debugger support is
        -: 1347:   selected.  As there is extra overhead for this, we only do this
        -: 1348:   when specifically requested 
        -: 1349:*/
        -: 1350:#ifdef HAVE_DEBUGGER_SUPPORT
        -: 1351:void MPIR_WaitForDebugger( void );
        -: 1352:void MPIR_DebuggerSetAborting( const char * );
        -: 1353:void MPIR_Sendq_remember(MPID_Request *, int, int, int );
        -: 1354:void MPIR_Sendq_forget(MPID_Request *);
        -: 1355:void MPIR_CommL_remember( MPID_Comm * );
        -: 1356:void MPIR_CommL_forget( MPID_Comm * );
        -: 1357:
        -: 1358:#define MPIR_SENDQ_REMEMBER(_a,_b,_c,_d) MPIR_Sendq_remember(_a,_b,_c,_d)
        -: 1359:#define MPIR_SENDQ_FORGET(_a) MPIR_Sendq_forget(_a)
        -: 1360:#define MPIR_COMML_REMEMBER(_a) MPIR_CommL_remember( _a )
        -: 1361:#define MPIR_COMML_FORGET(_a) MPIR_CommL_forget( _a )
        -: 1362:#else
        -: 1363:#define MPIR_SENDQ_REMEMBER(a,b,c,d)
        -: 1364:#define MPIR_SENDQ_FORGET(a)
        -: 1365:#define MPIR_COMML_REMEMBER(_a) 
        -: 1366:#define MPIR_COMML_FORGET(_a) 
        -: 1367:#endif
        -: 1368:
        -: 1369:
        -: 1370:/* ------------------------------------------------------------------------- */
        -: 1371:/* Prototypes and definitions for the node ID code.  This is used to support
        -: 1372:   hierarchical collectives in a (mostly) device-independent way. */
        -: 1373:#if defined(MPID_USE_NODE_IDS)
        -: 1374:/* MPID_Node_id_t is a signed integer type defined by the device in mpidpre.h. */
        -: 1375:int MPID_Get_node_id(MPID_Comm *comm, int rank, MPID_Node_id_t *id_p);
        -: 1376:int MPID_Get_max_node_id(MPID_Comm *comm, MPID_Node_id_t *max_id_p);
        -: 1377:#endif
        -: 1378:
        -: 1379:/* ------------------------------------------------------------------------- */
        -: 1380:/*S
        -: 1381:  MPID_Progress_state - object to hold progress state when using the blocking
        -: 1382:  progress routines.
        -: 1383:
        -: 1384:  Module:
        -: 1385:  Misc
        -: 1386:
        -: 1387:  Notes:
        -: 1388:  The device must define MPID_PROGRESS_STATE_DECL.  It should  include any state
        -: 1389:  that needs to be maintained between calls to MPID_Progress_{start,wait,end}.
        -: 1390:  S*/
        -: 1391:typedef struct MPID_Progress_state
        -: 1392:{
        -: 1393:    MPID_PROGRESS_STATE_DECL
        -: 1394:}
        -: 1395:MPID_Progress_state;
        -: 1396:/* ------------------------------------------------------------------------- */
        -: 1397:
        -: 1398:/* ------------------------------------------------------------------------- */
        -: 1399:/* end of mpirma.h (in src/mpi/rma?) */
        -: 1400:/* ------------------------------------------------------------------------- */
        -: 1401:
        -: 1402:/* Windows */
        -: 1403:#ifdef USE_MPID_RMA_TABLE
        -: 1404:struct MPID_Win;
        -: 1405:typedef struct MPIRI_RMA_Ops {
        -: 1406:    int (*Win_free)(struct MPID_Win **);
        -: 1407:    int (*Put)(void *, int, MPI_Datatype, int, MPI_Aint, int, MPI_Datatype, 
        -: 1408:		struct MPID_Win *);
        -: 1409:    int (*Get)(void *, int, MPI_Datatype, int, MPI_Aint, int, MPI_Datatype, 
        -: 1410:		struct MPID_Win *);
        -: 1411:    int (*Accumulate)(void *, int, MPI_Datatype, int, MPI_Aint, int, 
        -: 1412:		       MPI_Datatype, MPI_Op, struct MPID_Win *);
        -: 1413:    int (*Win_fence)(int, struct MPID_Win *);
        -: 1414:    int (*Win_post)(MPID_Group *, int, struct MPID_Win *);
        -: 1415:    int (*Win_start)(MPID_Group *, int, struct MPID_Win *);
        -: 1416:    int (*Win_complete)(struct MPID_Win *);
        -: 1417:    int (*Win_wait)(struct MPID_Win *);
        -: 1418:    int (*Win_test)(struct MPID_Win *, int *);
        -: 1419:    int (*Win_lock)(int, int, int, struct MPID_Win *);
        -: 1420:    int (*Win_unlock)(int, struct MPID_Win *);
        -: 1421:} MPIRI_RMAFns;
        -: 1422:#define MPIRI_RMAFNS_VERSION 2
        -: 1423:/* Note that the memory allocation/free routines do not take a window, 
        -: 1424:   so they must be initialized separately, and are a per-run, not per-window
        -: 1425:   object.  If the device can manage different kinds of memory allocations,
        -: 1426:   these routines must internally provide that flexibility. */
        -: 1427:/* 
        -: 1428:    void *(*Alloc_mem)(size_t, MPID_Info *);
        -: 1429:    int (*Free_mem)(void *);
        -: 1430:*/
        -: 1431:#endif
        -: 1432:
        -: 1433:/*S
        -: 1434:  MPID_Win - Description of the Window Object data structure.
        -: 1435:
        -: 1436:  Module:
        -: 1437:  Win-DS
        -: 1438:
        -: 1439:  Notes:
        -: 1440:  The following 3 keyvals are defined for attributes on all MPI 
        -: 1441:  Window objects\:
        -: 1442:.vb
        -: 1443: MPI_WIN_SIZE
        -: 1444: MPI_WIN_BASE
        -: 1445: MPI_WIN_DISP_UNIT
        -: 1446:.ve
        -: 1447:  These correspond to the values in 'length', 'start_address', and 
        -: 1448:  'disp_unit'.
        -: 1449:
        -: 1450:  The communicator in the window is the same communicator that the user
        -: 1451:  provided to 'MPI_Win_create' (not a dup).  However, each intracommunicator
        -: 1452:  has a special context id that may be used if MPI communication is used 
        -: 1453:  by the implementation to implement the RMA operations.
        -: 1454:
        -: 1455:  There is no separate window group; the group of the communicator should be
        -: 1456:  used.
        -: 1457:
        -: 1458:  Question:
        -: 1459:  Should a 'MPID_Win' be defined after 'MPID_Segment' in case the device 
        -: 1460:  wants to 
        -: 1461:  store a queue of pending put/get operations, described with 'MPID_Segment'
        -: 1462:  (or 'MPID_Request')s?
        -: 1463:
        -: 1464:  S*/
        -: 1465:typedef struct MPID_Win {
        -: 1466:    MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1467:    int fence_cnt;     /* 0 = no fence has been called; 
        -: 1468:                          1 = fence has been called */ 
        -: 1469:    MPID_Errhandler *errhandler;  /* Pointer to the error handler structure */
        -: 1470:    void *base;
        -: 1471:    MPI_Aint    size;        
        -: 1472:    int          disp_unit;      /* Displacement unit of *local* window */
        -: 1473:    MPID_Attribute *attributes;
        -: 1474:    MPID_Group *start_group_ptr; /* group passed in MPI_Win_start */
        -: 1475:    int start_assert;            /* assert passed to MPI_Win_start */
        -: 1476:    MPI_Comm    comm;         /* communicator of window (dup) */
        -: 1477:#ifdef USE_THREADED_WINDOW_CODE
        -: 1478:    /* These were causing compilation errors.  We need to figure out how to
        -: 1479:       integrate threads into MPICH2 before including these fields. */
        -: 1480:    /* FIXME: The test here should be within a test for threaded support */
        -: 1481:#ifdef HAVE_PTHREAD_H
        -: 1482:    pthread_t wait_thread_id; /* id of thread handling MPI_Win_wait */
        -: 1483:    pthread_t passive_target_thread_id; /* thread for passive target RMA */
        -: 1484:#elif defined(HAVE_WINTHREADS)
        -: 1485:    HANDLE wait_thread_id;
        -: 1486:    HANDLE passive_target_thread_id;
        -: 1487:#endif
        -: 1488:#endif
        -: 1489:    /* */
        -: 1490:#ifdef USE_MPID_RMA_TABLE
        -: 1491:    MPIRI_RMAFns RMAFns;
        -: 1492:#endif    
        -: 1493:    /* These are COPIES of the values so that addresses to them
        -: 1494:       can be returned as attributes.  They are initialized by the
        -: 1495:       MPI_Win_get_attr function */
        -: 1496:    int  copyDispUnit;
        -: 1497:    MPI_Aint copySize;
        -: 1498:    
        -: 1499:    char          name[MPI_MAX_OBJECT_NAME];  
        -: 1500:  /* Other, device-specific information */
        -: 1501:#ifdef MPID_DEV_WIN_DECL
        -: 1502:    MPID_DEV_WIN_DECL
        -: 1503:#endif
        -: 1504:} MPID_Win;
        -: 1505:extern MPIU_Object_alloc_t MPID_Win_mem;
        -: 1506:/* Preallocated win objects */
        -: 1507:extern MPID_Win MPID_Win_direct[];
        -: 1508:
        -: 1509:
        -: 1510:/* ------------------------------------------------------------------------- */
        -: 1511:/* also in mpirma.h ?*/
        -: 1512:/* ------------------------------------------------------------------------- */
        -: 1513:
        -: 1514:/*
        -: 1515: * Good Memory (may be required for passive target operations on MPI_Win)
        -: 1516: */
        -: 1517:
        -: 1518:/*@
        -: 1519:  MPID_Alloc_mem - Allocate memory suitable for passive target RMA operations
        -: 1520:
        -: 1521:  Input Parameter:
        -: 1522:+ size - Number of types to allocate.
        -: 1523:- info - Info object
        -: 1524:
        -: 1525:  Return value:
        -: 1526:  Pointer to the allocated memory.  If the memory is not available, 
        -: 1527:  returns null.
        -: 1528:
        -: 1529:  Notes:
        -: 1530:  This routine is used to implement 'MPI_Alloc_mem'.  It is for that reason
        -: 1531:  that there is no communicator argument.  
        -: 1532:
        -: 1533:  This memory may `only` be freed with 'MPID_Free_mem'.
        -: 1534:
        -: 1535:  This is a `local`, not a collective operation.  It functions more like a
        -: 1536:  good form of 'malloc' than collective shared-memory allocators such as
        -: 1537:  the 'shmalloc' found on SGI systems.
        -: 1538:
        -: 1539:  Implementations of this routine may wish to use 'MPID_Memory_register'.  
        -: 1540:  However, this routine has slighly different requirements, so a separate
        -: 1541:  entry point is provided.
        -: 1542:
        -: 1543:  Question:
        -: 1544:  Since this takes an info object, should there be an error routine in the 
        -: 1545:  case that the info object contains an error?
        -: 1546:
        -: 1547:  Module:
        -: 1548:  Win
        -: 1549:  @*/
        -: 1550:void *MPID_Alloc_mem( size_t size, MPID_Info *info );
        -: 1551:
        -: 1552:/*@
        -: 1553:  MPID_Free_mem - Frees memory allocated with 'MPID_Alloc_mem'
        -: 1554:
        -: 1555:  Input Parameter:
        -: 1556:. ptr - Pointer to memory allocated by 'MPID_Alloc_mem'.
        -: 1557:
        -: 1558:  Return value:
        -: 1559:  'MPI_SUCCESS' if memory was successfully freed; an MPI error code otherwise.
        -: 1560:
        -: 1561:  Notes:
        -: 1562:  The return value is provided because it may not be easy to validate the
        -: 1563:  value of 'ptr' without attempting to free the memory.
        -: 1564:
        -: 1565:  Module:
        -: 1566:  Win
        -: 1567:  @*/
        -: 1568:int MPID_Free_mem( void *ptr );
        -: 1569:
        -: 1570:/*@
        -: 1571:  MPID_Mem_was_alloced - Return true if this memory was allocated with 
        -: 1572:  'MPID_Alloc_mem'
        -: 1573:
        -: 1574:  Input Parameters:
        -: 1575:+ ptr  - Address of memory
        -: 1576:- size - Size of reqion in bytes.
        -: 1577:
        -: 1578:  Return value:
        -: 1579:  True if the memory was allocated with 'MPID_Alloc_mem', false otherwise.
        -: 1580:
        -: 1581:  Notes:
        -: 1582:  This routine may be needed by 'MPI_Win_create' to ensure that the memory 
        -: 1583:  for passive target RMA operations was allocated with 'MPI_Mem_alloc'.
        -: 1584:  This may be used, for example, for ensuring that memory used with
        -: 1585:  passive target operations was allocated with 'MPID_Alloc_mem'.
        -: 1586:
        -: 1587:  Module:
        -: 1588:  Win
        -: 1589:  @*/
        -: 1590:int MPID_Mem_was_alloced( void *ptr );  /* brad : this isn't used or implemented anywhere */
        -: 1591:
        -: 1592:/* ------------------------------------------------------------------------- */
        -: 1593:/* end of also in mpirma.h ? */
        -: 1594:/* ------------------------------------------------------------------------- */
        -: 1595:
        -: 1596:/* ------------------------------------------------------------------------- */
        -: 1597:/* Reduction and accumulate operations */
        -: 1598:/*E
        -: 1599:  MPID_Op_kind - Enumerates types of MPI_Op types
        -: 1600:
        -: 1601:  Notes:
        -: 1602:  These are needed for implementing 'MPI_Accumulate', since only predefined
        -: 1603:  operations are allowed for that operation.  
        -: 1604:
        -: 1605:  A gap in the enum values was made allow additional predefined operations
        -: 1606:  to be inserted.  This might include future additions to MPI or experimental
        -: 1607:  extensions (such as a Read-Modify-Write operation).
        -: 1608:
        -: 1609:  Module:
        -: 1610:  Collective-DS
        -: 1611:  E*/
        -: 1612:typedef enum MPID_Op_kind { MPID_OP_MAX=1, MPID_OP_MIN=2, 
        -: 1613:			    MPID_OP_SUM=3, MPID_OP_PROD=4, 
        -: 1614:	       MPID_OP_LAND=5, MPID_OP_BAND=6, MPID_OP_LOR=7, MPID_OP_BOR=8,
        -: 1615:	       MPID_OP_LXOR=9, MPID_OP_BXOR=10, MPID_OP_MAXLOC=11, 
        -: 1616:               MPID_OP_MINLOC=12, MPID_OP_REPLACE=13, 
        -: 1617:               MPID_OP_USER_NONCOMMUTE=32, MPID_OP_USER=33 }
        -: 1618:  MPID_Op_kind;
        -: 1619:
        -: 1620:/*S
        -: 1621:  MPID_User_function - Definition of a user function for MPI_Op types.
        -: 1622:
        -: 1623:  Notes:
        -: 1624:  This includes a 'const' to make clear which is the 'in' argument and 
        -: 1625:  which the 'inout' argument, and to indicate that the 'count' and 'datatype'
        -: 1626:  arguments are unchanged (they are addresses in an attempt to allow 
        -: 1627:  interoperation with Fortran).  It includes 'restrict' to emphasize that 
        -: 1628:  no overlapping operations are allowed.
        -: 1629:
        -: 1630:  We need to include a Fortran version, since those arguments will
        -: 1631:  have type 'MPI_Fint *' instead.  We also need to add a test to the
        -: 1632:  test suite for this case; in fact, we need tests for each of the handle
        -: 1633:  types to ensure that the transfered handle works correctly.
        -: 1634:
        -: 1635:  This is part of the collective module because user-defined operations
        -: 1636:  are valid only for the collective computation routines and not for 
        -: 1637:  RMA accumulate.
        -: 1638:
        -: 1639:  Yes, the 'restrict' is in the correct location.  C compilers that 
        -: 1640:  support 'restrict' should be able to generate code that is as good as a
        -: 1641:  Fortran compiler would for these functions.
        -: 1642:
        -: 1643:  We should note on the manual pages for user-defined operations that
        -: 1644:  'restrict' should be used when available, and that a cast may be 
        -: 1645:  required when passing such a function to 'MPI_Op_create'.
        -: 1646:
        -: 1647:  Question:
        -: 1648:  Should each of these function types have an associated typedef?
        -: 1649:
        -: 1650:  Should there be a C++ function here?
        -: 1651:
        -: 1652:  Module:
        -: 1653:  Collective-DS
        -: 1654:  S*/
        -: 1655:typedef union MPID_User_function {
        -: 1656:    void (*c_function) ( const void *, void *, 
        -: 1657:			 const int *, const MPI_Datatype * ); 
        -: 1658:    void (*f77_function) ( const void *, void *,
        -: 1659:			  const MPI_Fint *, const MPI_Fint * );
        -: 1660:} MPID_User_function;
        -: 1661:/* FIXME: Should there be "restrict" in the definitions above, e.g., 
        -: 1662:   (*c_function)( const void restrict * , void restrict *, ... )? */
        -: 1663:
        -: 1664:/*S
        -: 1665:  MPID_Op - MPI_Op structure
        -: 1666:
        -: 1667:  Notes:
        -: 1668:  All of the predefined functions are commutative.  Only user functions may 
        -: 1669:  be noncummutative, so there are two separate op types for commutative and
        -: 1670:  non-commutative user-defined operations.
        -: 1671:
        -: 1672:  Operations do not require reference counts because there are no nonblocking
        -: 1673:  operations that accept user-defined operations.  Thus, there is no way that
        -: 1674:  a valid program can free an 'MPI_Op' while it is in use.
        -: 1675:
        -: 1676:  Module:
        -: 1677:  Collective-DS
        -: 1678:  S*/
        -: 1679:typedef struct MPID_Op {
        -: 1680:     MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 1681:     MPID_Op_kind       kind;
        -: 1682:     MPID_Lang_t        language;
        -: 1683:     MPID_User_function function;
        -: 1684:  } MPID_Op;
        -: 1685:#define MPID_OP_N_BUILTIN 14
        -: 1686:extern MPID_Op MPID_Op_builtin[MPID_OP_N_BUILTIN];
        -: 1687:extern MPID_Op MPID_Op_direct[];
        -: 1688:extern MPIU_Object_alloc_t MPID_Op_mem;
        -: 1689:
        -: 1690:#define MPIR_Op_release_ref( _op, _inuse ) \
        -: 1691:    do { MPIU_Object_release_ref( _op, _inuse ); } while (0)
        -: 1692:
        -: 1693:/* ------------------------------------------------------------------------- */
        -: 1694:
        -: 1695:/* ------------------------------------------------------------------------- */
        -: 1696:/* mpicoll.h (in src/mpi/coll?) */
        -: 1697:/* ------------------------------------------------------------------------- */
        -: 1698:
        -: 1699:/* Collective operations */
        -: 1700:typedef struct MPID_Collops {
        -: 1701:    int ref_count;   /* Supports lazy copies */
        -: 1702:    /* Contains pointers to the functions for the MPI collectives */
        -: 1703:    int (*Barrier) (MPID_Comm *);
        -: 1704:    int (*Bcast) (void*, int, MPI_Datatype, int, MPID_Comm * );
        -: 1705:    int (*Gather) (void*, int, MPI_Datatype, void*, int, MPI_Datatype, 
        -: 1706:                   int, MPID_Comm *); 
        -: 1707:    int (*Gatherv) (void*, int, MPI_Datatype, void*, int *, int *, 
        -: 1708:                    MPI_Datatype, int, MPID_Comm *); 
        -: 1709:    int (*Scatter) (void*, int, MPI_Datatype, void*, int, MPI_Datatype, 
        -: 1710:                    int, MPID_Comm *);
        -: 1711:    int (*Scatterv) (void*, int *, int *, MPI_Datatype, void*, int, 
        -: 1712:                    MPI_Datatype, int, MPID_Comm *);
        -: 1713:    int (*Allgather) (void*, int, MPI_Datatype, void*, int, 
        -: 1714:                      MPI_Datatype, MPID_Comm *);
        -: 1715:    int (*Allgatherv) (void*, int, MPI_Datatype, void*, int *, int *, 
        -: 1716:                       MPI_Datatype, MPID_Comm *);
        -: 1717:    int (*Alltoall) (void*, int, MPI_Datatype, void*, int, MPI_Datatype, 
        -: 1718:                               MPID_Comm *);
        -: 1719:    int (*Alltoallv) (void*, int *, int *, MPI_Datatype, void*, int *, 
        -: 1720:                     int *, MPI_Datatype, MPID_Comm *);
        -: 1721:    int (*Alltoallw) (void*, int *, int *, MPI_Datatype *, void*, int *, 
        -: 1722:                     int *, MPI_Datatype *, MPID_Comm *);
        -: 1723:    int (*Reduce) (void*, void*, int, MPI_Datatype, MPI_Op, int, 
        -: 1724:                   MPID_Comm *);
        -: 1725:    int (*Allreduce) (void*, void*, int, MPI_Datatype, MPI_Op, 
        -: 1726:                      MPID_Comm *);
        -: 1727:    int (*Reduce_scatter) (void*, void*, int *, MPI_Datatype, MPI_Op, 
        -: 1728:                           MPID_Comm *);
        -: 1729:    int (*Scan) (void*, void*, int, MPI_Datatype, MPI_Op, MPID_Comm * );
        -: 1730:    int (*Exscan) (void*, void*, int, MPI_Datatype, MPI_Op, MPID_Comm * );
        -: 1731:    
        -: 1732:} MPID_Collops;
        -: 1733:
        -: 1734:#define MPIR_BARRIER_TAG 1
        -: 1735:/* ------------------------------------------------------------------------- */
        -: 1736:/* end of mpicoll.h (in src/mpi/coll? */
        -: 1737:/* ------------------------------------------------------------------------- */
        -: 1738:
        -: 1739:/* ------------------------------------------------------------------------- */
        -: 1740:/* mpitopo.h (in src/mpi/topo? */
        -: 1741:/*
        -: 1742: * The following struture allows the device detailed control over the 
        -: 1743: * functions that are used to implement the topology routines.  If either
        -: 1744: * the pointer to this structure is null or any individual entry is null,
        -: 1745: * the default function is used (this follows exactly the same rules as the
        -: 1746: * collective operations, provided in the MPID_Collops structure).
        -: 1747: */
        -: 1748:/* ------------------------------------------------------------------------- */
        -: 1749:
        -: 1750:typedef struct MPID_TopoOps {
        -: 1751:    int (*cartCreate)( const MPID_Comm *, int, const int[], const int [],
        -: 1752:		       int, MPI_Comm * );
        -: 1753:    int (*cartMap)   ( const MPID_Comm *, int, const int[], const int [], 
        -: 1754:		       int * );
        -: 1755:    int (*graphCreate)( const MPID_Comm *, int, const int[], const int [],
        -: 1756:			int, MPI_Comm * );
        -: 1757:    int (*graphMap)   ( const MPID_Comm *, int, const int[], const int[], 
        -: 1758:			int * );
        -: 1759:} MPID_TopoOps;
        -: 1760:/* ------------------------------------------------------------------------- */
        -: 1761:/* end of mpitopo.h (in src/mpi/topo? */
        -: 1762:/* ------------------------------------------------------------------------- */
        -: 1763:
        -: 1764:/* Time stamps */
        -: 1765:/* Get the timer definitions.  The source file for this include is
        -: 1766:   src/mpi/timer/mpichtimer.h.in */
        -: 1767:#include "mpichtimer.h"
        -: 1768:
        -: 1769:typedef struct MPID_Stateinfo_t {
        -: 1770:    MPID_Time_t stamp;
        -: 1771:    int count;
        -: 1772:} MPID_Stateinfo_t;
        -: 1773:#define MPICH_MAX_STATES 512
        -: 1774:/* Timer state routines (src/util/instrm/states.c) */
        -: 1775:void MPID_TimerStateBegin( int, MPID_Time_t * );
        -: 1776:void MPID_TimerStateEnd( int, MPID_Time_t * );
        -: 1777:
        -: 1778:/* ------------------------------------------------------------------------- */
        -: 1779:/* Thread types */
        -: 1780:/* Temporary; this will include "mpichthread.h" eventually */
        -: 1781:
        -: 1782:#ifdef MPICH_DEBUG_NESTING
        -: 1783:#define MPICH_MAX_NESTFILENAME 256
        -: 1784:typedef struct MPICH_Nestinfo { 
        -: 1785:    char file[MPICH_MAX_NESTFILENAME];
        -: 1786:    int  line;
        -: 1787:} MPICH_Nestinfo_t;
        -: 1788:#define MPICH_MAX_NESTINFO 16
        -: 1789:#endif /* MPICH_DEBUG_NESTING */
        -: 1790:
        -: 1791:typedef struct MPICH_PerThread_t {
        -: 1792:    int              nest_count;   /* For layered MPI implementation */
        -: 1793:    int              op_errno;     /* For errors in predefined MPI_Ops */
        -: 1794:#ifdef MPICH_DEBUG_NESTING
        -: 1795:    MPICH_Nestinfo_t nestinfo[MPICH_MAX_NESTINFO];
        -: 1796:#endif
        -: 1797:    /* FIXME: Is this used anywhere? */
        -: 1798:#ifdef HAVE_TIMING
        -: 1799:    MPID_Stateinfo_t timestamps[MPICH_MAX_STATES];  /* per thread state info */
        -: 1800:#endif
        -: 1801:#if defined(MPID_DEV_PERTHREAD_DECL)
        -: 1802:    MPID_DEV_PERTHREAD_DECL
        -: 1803:#endif    
        -: 1804:} MPICH_PerThread_t;
        -: 1805:
        -: 1806:#if !defined(MPICH_IS_THREADED)
        -: 1807:/* If single threaded, make this point at a pre-allocated segment.
        -: 1808:   This structure is allocated in src/mpi/init/initthread.c */
        -: 1809:extern MPICH_PerThread_t MPIR_Thread;
        -: 1810:
        -: 1811:/* The following three macros define a way to portably access thread-private
        -: 1812:   storage in MPICH2, and avoid extra overhead when MPICH2 is single 
        -: 1813:   threaded
        -: 1814:   INITKEY - Create the key.  Must happen *before* the other threads 
        -: 1815:             are created
        -: 1816:   INIT    - Create the thread-private storage.  Must happen once per thread
        -: 1817:   DECL    - Declare local variables
        -: 1818:   GET     - Access the thread-private storage
        -: 1819:   FIELD   - Access the thread-private field (by name)
        -: 1820:
        -: 1821:   The "DECL" is the extern so that there is always a statement for
        -: 1822:   the declaration.
        -: 1823:*/
        -: 1824:#define MPIU_THREADPRIV_INITKEY
        -: 1825:#define MPIU_THREADPRIV_INIT 
        -: 1826:/* Empty declarations are not allowed in C. However multiple decls are allowed */
        -: 1827:#define MPIU_THREADPRIV_DECL extern MPICH_PerThread_t MPIR_Thread
        -: 1828:#define MPIU_THREADPRIV_GET
        -: 1829:#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread._a)
        -: 1830:
        -: 1831:#elif  defined(HAVE_RUNTIME_THREADCHECK)
        -: 1832:/* In the case where the thread level is set in MPI_Init_thread, we
        -: 1833:   need a blended version of the non-threaded and the thread-multiple
        -: 1834:   definitions.
        -: 1835:   
        -: 1836:   The approach is to have TWO MPICH_PerThread_t pointers.  One is local
        -: 1837:   (The MPIU_THREADPRIV_DECL is used in the routines local definitions), 
        -: 1838:   as in the threaded version of these macros.  This is set by using a routine
        -: 1839:   to get thread-private storage.  The second is a preallocated, extern 
        -: 1840:   MPICH_PerThread_t struct, as in the single threaded case.  Based on
        -: 1841:   MPIR_Process.isThreaded, one or the other is used.
        -: 1842:   
        -: 1843: */
        -: 1844:/* For the single threaded case, we use a preallocated structure 
        -: 1845:   This structure is allocated in src/mpi/init/initthread.c */
        -: 1846:extern MPICH_PerThread_t MPIR_ThreadSingle;
        -: 1847:
        -: 1848:/* We need to provide a function that will cleanup the storage attached
        -: 1849:   to the key.  */
        -: 1850:#define MPIU_THREADPRIV_INITKEY  \
        -: 1851:    {if (MPIR_Process.isThreaded) {\
        -: 1852:	    MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_Process.thread_storage,NULL);}}
        -: 1853:#define MPIU_THREADPRIV_INIT {if (MPIR_Process.isThreaded) {\
        -: 1854:	MPICH_PerThread_t *(pt_) = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t));	\
        -: 1855:	MPID_Thread_tls_set(&MPIR_Process.thread_storage, (void *) (pt_)); \
        -: 1856:        }}
        -: 1857:#define MPIU_THREADPRIV_DECL \
        -: 1858:    MPICH_PerThread_t *MPIR_Thread=0
        -: 1859:#define MPIU_THREADPRIV_GET  \
        -: 1860:    {if (!MPIR_Thread){MPIR_GetPerThread( &MPIR_Thread );}}
        -: 1861:#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread->_a)
        -: 1862:
        -: 1863:#else /* Thread multiple */
        -: 1864:/* The following three macros define a way to portably access thread-private
        -: 1865:   storage in MPICH2, and avoid extra overhead when MPICH2 is single 
        -: 1866:   threaded.  We initialize the MPIR_Thread pointer to null so that
        -: 1867:   we need call the routine to get the thread-private storage only once
        -: 1868:   in an invocation of a routine.  */
        -: 1869:
        -: 1870:#define MPIU_THREADPRIV_INITKEY  \
        -: 1871:    MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_Process.thread_storage,NULL)
        -: 1872:#define MPIU_THREADPRIV_INIT {\
        -: 1873:	MPICH_PerThread_t *(pt_) = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t));	\
        -: 1874:	MPID_Thread_tls_set(&MPIR_Process.thread_storage, (void *) (pt_)); \
        -: 1875:        }
        -: 1876:#define MPIU_THREADPRIV_DECL MPICH_PerThread_t *MPIR_Thread=0
        -: 1877:#define MPIU_THREADPRIV_GET {if (!MPIR_Thread)MPIR_GetPerThread( &MPIR_Thread );}
        -: 1878:#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread->_a)
        -: 1879:#endif
        -: 1880:
        -: 1881:/* Per process data */
        -: 1882:typedef enum MPIR_MPI_State_t { MPICH_PRE_INIT=0, MPICH_WITHIN_MPI=1,
        -: 1883:               MPICH_POST_FINALIZED=2 } MPIR_MPI_State_t;
        -: 1884:
        -: 1885:typedef struct PreDefined_attrs {
        -: 1886:    int appnum;          /* Application number provided by mpiexec (MPI-2) */
        -: 1887:    int host;            /* host */
        -: 1888:    int io;              /* standard io allowed */
        -: 1889:    int lastusedcode;    /* last used error code (MPI-2) */
        -: 1890:    int tag_ub;          /* Maximum message tag */
        -: 1891:    int universe;        /* Universe size from mpiexec (MPI-2) */
        -: 1892:    int wtime_is_global; /* Wtime is global over processes in COMM_WORLD */
        -: 1893:} PreDefined_attrs;
        -: 1894:
        -: 1895:struct MPID_Datatype;
        -: 1896:
        -: 1897:typedef struct MPICH_PerProcess_t {
        -: 1898:    MPIR_MPI_State_t  initialized;      /* Is MPI initalized? */
        -: 1899:    int               do_error_checks;  /* runtime error check control */
        -: 1900:    struct MPID_Comm  *comm_world;      /* Easy access to comm_world for
        -: 1901:                                           error handler */
        -: 1902:    struct MPID_Comm  *comm_self;       /* Easy access to comm_self */
        -: 1903:    struct MPID_Comm  *comm_parent;     /* Easy access to comm_parent */
        -: 1904:    struct MPID_Comm  *icomm_world;     /* An internal version of comm_world
        -: 1905:					   that is separate from user's 
        -: 1906:					   versions */
        -: 1907:    PreDefined_attrs  attrs;            /* Predefined attribute values */
        -: 1908:
        -: 1909:    /* The topology routines dimsCreate is independent of any communicator.
        -: 1910:       If this pointer is null, the default routine is used */
        -: 1911:    int (*dimsCreate)( int, int, int *);
        -: 1912:
        -: 1913:    /* Attribute dup functions.  Here for lazy initialization */
        -: 1914:    int (*attr_dup)( int, MPID_Attribute *, MPID_Attribute ** );
        -: 1915:    int (*attr_free)( int, MPID_Attribute ** );
        -: 1916:    /* There is no win_attr_dup function because there can be no MPI_Win_dup
        -: 1917:       function */
        -: 1918:    /* Routine to get the messages corresponding to dynamically created
        -: 1919:       error messages */
        -: 1920:    const char *(*errcode_to_string)( int );
        -: 1921:#ifdef HAVE_CXX_BINDING
        -: 1922:    /* Routines to call C++ functions from the C implementation of the
        -: 1923:       MPI reduction and attribute routines */
        -: 1924:    void (*cxx_call_op_fn)( void *, void *, int, MPI_Datatype, 
        -: 1925:			    MPI_User_function * );
        -: 1926:    /* Error handling functions.  As for the attribute functions,
        -: 1927:       we pass the integer file/comm/win, the address of the error code, 
        -: 1928:       and the C function to call (itself a function defined by the
        -: 1929:       C++ interface and exported to C).  The first argument is used
        -: 1930:       to specify the kind (comm,file,win) */
        -: 1931:    void  (*cxx_call_errfn) ( int, int *, int *, void (*)(void) );
        -: 1932:#endif /* HAVE_CXX_BINDING */
        -: 1933:} MPICH_PerProcess_t;
        -: 1934:extern MPICH_PerProcess_t MPIR_Process;
        -: 1935:
        -: 1936:/*D
        -: 1937:  MPICH_THREAD_LEVEL - Indicates the maximum level of thread
        -: 1938:  support provided at compile time.
        -: 1939: 
        -: 1940:  Values:
        -: 1941:  Any of the 'MPI_THREAD_xxx' values (these are preprocessor-time constants)
        -: 1942:
        -: 1943:  Notes:
        -: 1944:  The macro 'MPICH_THREAD_LEVEL' defines the maximum level of
        -: 1945:  thread support provided, and may be used at compile time to remove
        -: 1946:  thread locks and other code needed only in a multithreaded environment.
        -: 1947:
        -: 1948:  A typical use is 
        -: 1949:.vb
        -: 1950:  #ifdef MPICH_IS_THREADED
        -: 1951:     lock((r)->lock_ptr);
        -: 1952:     (r)->ref_count++;
        -: 1953:     unlock((r)->lock_ptr);
        -: 1954:  #else
        -: 1955:     (r)->ref_count ++;
        -: 1956:  #fi
        -: 1957:.ve
        -: 1958:
        -: 1959:  Note that 'MPICH_IS_THREADED' is defined as 1 if 
        -: 1960:.vb
        -: 1961:  MPICH_THREAD_LEVEL >= MPI_THREAD_MULTIPLE
        -: 1962:.ve
        -: 1963:  is true.  The test should be used only for special cases (such as 
        -: 1964:  handling 'SERIALIZED').
        -: 1965:
        -: 1966:  Module:
        -: 1967:  Environment-DS
        -: 1968:  D*/
        -: 1969:
        -: 1970:/* ------------------------------------------------------------------------- */
        -: 1971:/* In MPICH2, each function has an "enter" and "exit" macro.  These can be 
        -: 1972: * used to add various features to each function at compile time, or they
        -: 1973: * can be set to empty to provide the fastest possible production version.
        -: 1974: *
        -: 1975: * There are at this time three choices of features (beyond the empty choice)
        -: 1976: * 1. timing (controlled by macros in mpitimerimpl.h)
        -: 1977: *    These collect data on when each function began and finished; the
        -: 1978: *    resulting data can be displayed using special programs
        -: 1979: * 2. nesting (selected with --enable-g=nesting)
        -: 1980: *    Checks that the "NMPI" functions and the Nest_incr/decr calls
        -: 1981: *    are properly nested at runtime.  
        -: 1982: * 3. Debug logging (selected with --enable-g=log)
        -: 1983: *    Invokes MPIU_DBG_MSG at the entry and exit for each routine            
        -: 1984: * 4. Additional memory validation of the memory arena (--enable-g=memarena)
        -: 1985: */
        -: 1986:/* ------------------------------------------------------------------------- */
        -: 1987:/* if fine-grain nest testing is enabled then define the function enter/exit
        -: 1988:   macros to track the nesting level; otherwise, allow the timing module the
        -: 1989:   opportunity to define the macros */
        -: 1990:#if defined(MPICH_DEBUG_FINE_GRAIN_NESTING)
        -: 1991:#   include "mpiu_func_nesting.h"
        -: 1992:#elif defined(MPICH_DEBUG_MEMARENA)
        -: 1993:#   include "mpifuncmem.h"
        -: 1994:#elif defined(USE_DBG_LOGGING)
        -: 1995:#   include "mpifunclog.h"
        -: 1996:#elif !defined(NEEDS_FUNC_ENTER_EXIT_DEFS)
        -: 1997:    /* If no timing choice is selected, this sets the entry/exit macros 
        -: 1998:       to empty */
        -: 1999:#   include "mpitimerimpl.h"
        -: 2000:#endif
        -: 2001:
        -: 2002:#ifdef NEEDS_FUNC_ENTER_EXIT_DEFS
        -: 2003:/* mpich layer definitions */
        -: 2004:#define MPID_MPI_FUNC_ENTER(a)			MPIR_FUNC_ENTER(a)
        -: 2005:#define MPID_MPI_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2006:#define MPID_MPI_PT2PT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2007:#define MPID_MPI_PT2PT_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2008:#define MPID_MPI_PT2PT_FUNC_ENTER_FRONT(a)	MPIR_FUNC_ENTER(a)
        -: 2009:#define MPID_MPI_PT2PT_FUNC_EXIT_FRONT(a)	MPIR_FUNC_EXIT(a)
        -: 2010:#define MPID_MPI_PT2PT_FUNC_ENTER_BACK(a)	MPIR_FUNC_ENTER(a)
        -: 2011:#define MPID_MPI_PT2PT_FUNC_ENTER_BOTH(a)	MPIR_FUNC_ENTER(a)
        -: 2012:#define MPID_MPI_PT2PT_FUNC_EXIT_BACK(a)	MPIR_FUNC_EXIT(a)
        -: 2013:#define MPID_MPI_PT2PT_FUNC_EXIT_BOTH(a)	MPIR_FUNC_EXIT(a)
        -: 2014:#define MPID_MPI_COLL_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2015:#define MPID_MPI_COLL_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2016:#define MPID_MPI_RMA_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2017:#define MPID_MPI_RMA_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2018:#define MPID_MPI_INIT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2019:#define MPID_MPI_INIT_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2020:#define MPID_MPI_FINALIZE_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2021:#define MPID_MPI_FINALIZE_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2022:
        -: 2023:/* device layer definitions */
        -: 2024:#define MPIDI_FUNC_ENTER(a)			MPIR_FUNC_ENTER(a)
        -: 2025:#define MPIDI_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2026:#define MPIDI_PT2PT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2027:#define MPIDI_PT2PT_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2028:#define MPIDI_PT2PT_FUNC_ENTER_FRONT(a)		MPIR_FUNC_ENTER(a)
        -: 2029:#define MPIDI_PT2PT_FUNC_EXIT_FRONT(a)		MPIR_FUNC_EXIT(a)
        -: 2030:#define MPIDI_PT2PT_FUNC_ENTER_BACK(a)		MPIR_FUNC_ENTER(a)
        -: 2031:#define MPIDI_PT2PT_FUNC_ENTER_BOTH(a)		MPIR_FUNC_ENTER(a)
        -: 2032:#define MPIDI_PT2PT_FUNC_EXIT_BACK(a)		MPIR_FUNC_EXIT(a)
        -: 2033:#define MPIDI_PT2PT_FUNC_EXIT_BOTH(a)		MPIR_FUNC_EXIT(a)
        -: 2034:#define MPIDI_COLL_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2035:#define MPIDI_COLL_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2036:#define MPIDI_RMA_FUNC_ENTER(a)			MPIR_FUNC_ENTER(a)
        -: 2037:#define MPIDI_RMA_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2038:#define MPIDI_INIT_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2039:#define MPIDI_INIT_FUNC_EXIT(a)			MPIR_FUNC_EXIT(a)
        -: 2040:#define MPIDI_FINALIZE_FUNC_ENTER(a)		MPIR_FUNC_ENTER(a)
        -: 2041:#define MPIDI_FINALIZE_FUNC_EXIT(a)		MPIR_FUNC_EXIT(a)
        -: 2042:
        -: 2043:/* evaporate the timing macros since timing is not selected */
        -: 2044:#define MPIU_Timer_init(rank, size)
        -: 2045:#define MPIU_Timer_finalize()
        -: 2046:#endif /* NEEDS_FUNC_ENTER_EXIT_DEFS */
        -: 2047:
        -: 2048:/* Definitions for error handling and reporting */
        -: 2049:#include "mpierror.h"
        -: 2050:#include "mpierrs.h"
        -: 2051:
        -: 2052:/* FIXME: This routine is only used within mpi/src/err/errutil.c and 
        -: 2053:   smpd.  We may not want to export it.  */
        -: 2054:void MPIR_Err_print_stack(FILE *, int);
        -: 2055:
        -: 2056:
        -: 2057:/* ------------------------------------------------------------------------- */
        -: 2058:/* XXX DJG FIXME delete this? */
        -: 2059:/* FIXME: Merge these with the object refcount update routines (perhaps as
        -: 2060:   part of a general "atomic update" file */
        -: 2061:/*
        -: 2062: * Standardized general-purpose atomic update routines.  Some comments:
        -: 2063: * Setmax atomically implements *a_ptr = max(b,*a_ptr) .  This can
        -: 2064: * be implemented using compare-and-swap (form max, if new max is 
        -: 2065: * larger, compare-and-swap against old max.  if failure, restart).
        -: 2066: * Fetch_and_increment can be implemented in a similar way; for
        -: 2067: * example, in IA32, 
        -: 2068: * loop:
        -: 2069: *   mov eax, valptr
        -: 2070: *   mov oldvalptr, eax
        -: 2071: *   mov ebx, eax
        -: 2072: *   inc ebx
        -: 2073: *   lock: cmpxchg valptr, ebx
        -: 2074: *   jnz loop
        -: 2075: *
        -: 2076: * Implementations using LoadLink/StoreConditional are similar.
        -: 2077: *
        -: 2078: * Question: can we use the simple code for MPI_THREAD_SERIALIZED?
        -: 2079: * If not, do we want a separate set of definitions that can be used
        -: 2080: * in the code where serialized is ok.
        -: 2081: *
        -: 2082: * Currently, these are used only in the routines to create new error classes
        -: 2083: * and codes (src/mpi/errhan/dynerrutil.c and add_error_class.c).  
        -: 2084: * Note that MPI object reference counts are handled with their own routines.
        -: 2085: *
        -: 2086: * Because of the current use of these routines is within the SINGLE_CS
        -: 2087: * thread lock (for the THREAD_MULTIPLE case), we currently
        -: 2088: * do *not* include a separate Critical section for these operations.
        -: 2089: *
        -: 2090: */
        -: 2091:#define MPIR_Setmax(a_ptr,b) if (b>*(a_ptr)) { *(a_ptr) = b; }
        -: 2092:#define MPIR_Fetch_and_increment(count_ptr,value_ptr) \
        -: 2093:    { *value_ptr = *count_ptr; *count_ptr += 1; }
        -: 2094:
        -: 2095:/* ------------------------------------------------------------------------- */
        -: 2096:
        -: 2097:/* FIXME: Move these to the communicator block; make sure that all 
        -: 2098:   objects have such hooks */
        -: 2099:#ifndef HAVE_DEV_COMM_HOOK
        -: 2100:#define MPID_Dev_comm_create_hook( a )
        -: 2101:#define MPID_Dev_comm_destroy_hook( a )
        -: 2102:#endif
        -: 2103:
        -: 2104:/* ------------------------------------------------------------------------- */
        -: 2105:/* FIXME: What is the scope of these functions?  Can they be moved into
        -: 2106:   src/mpi/pt2pt? */
        -: 2107:/* ------------------------------------------------------------------------- */
        -: 2108:
        -: 2109:/* Do not set MPI_ERROR (only set if ERR_IN_STATUS is returned */
        -: 2110:#define MPIR_Status_set_empty(status_)			\
        -: 2111:{							\
        -: 2112:    if ((status_) != MPI_STATUS_IGNORE)			\
        -: 2113:    {							\
        -: 2114:	(status_)->MPI_SOURCE = MPI_ANY_SOURCE;		\
        -: 2115:	(status_)->MPI_TAG = MPI_ANY_TAG;		\
        -: 2116:	(status_)->count = 0;				\
        -: 2117:	(status_)->cancelled = FALSE;			\
        -: 2118:    }							\
        -: 2119:}
        -: 2120:/* See MPI 1.1, section 3.11, Null Processes */
        -: 2121:/* Do not set MPI_ERROR (only set if ERR_IN_STATUS is returned */
        -: 2122:#define MPIR_Status_set_procnull(status_)		\
        -: 2123:{							\
        -: 2124:    if ((status_) != MPI_STATUS_IGNORE)			\
        -: 2125:    {							\
        -: 2126:	(status_)->MPI_SOURCE = MPI_PROC_NULL;		\
        -: 2127:	(status_)->MPI_TAG = MPI_ANY_TAG;		\
        -: 2128:	(status_)->count = 0;				\
        -: 2129:	(status_)->cancelled = FALSE;			\
        -: 2130:    }							\
        -: 2131:}
        -: 2132:
        -: 2133:#define MPIR_Request_extract_status(request_ptr_, status_)								\
        -: 2134:{															\
        -: 2135:    if ((status_) != MPI_STATUS_IGNORE)											\
        -: 2136:    {															\
        -: 2137:	int error__;													\
        -: 2138:															\
        -: 2139:	/* According to the MPI 1.1 standard page 22 lines 9-12, the MPI_ERROR field may not be modified except by the	\
        -: 2140:	   functions in section 3.7.5 which return MPI_ERR_IN_STATUSES (MPI_Wait{all,some} and MPI_Test{all,some}). */	\
        -: 2141:	error__ = (status_)->MPI_ERROR;											\
        -: 2142:	*(status_) = (request_ptr_)->status;										\
        -: 2143:	(status_)->MPI_ERROR = error__;											\
        -: 2144:    }															\
        -: 2145:}
        -: 2146:/* ------------------------------------------------------------------------- */
        -: 2147:
        -: 2148:/* FIXME: The bindings should be divided into three groups:
        -: 2149:   1. ADI3 routines.  These should have structure comment documentation, e.g., 
        -: 2150:   the text from doc/adi3/adi3.c
        -: 2151:   2. General utility routines.  These should have a short description
        -: 2152:   3. Local utility routines, e.g., routines used within a single subdirectory.
        -: 2153:   These should be moved into an include file in that subdirectory 
        -: 2154:*/
        -: 2155:/* Bindings for internal routines */
        -: 2156:/*@ MPIR_Add_finalize - Add a routine to be called when MPI_Finalize is invoked
        -: 2157:
        -: 2158:+ routine - Routine to call
        -: 2159:. extra   - Void pointer to data to pass to the routine
        -: 2160:- priority - Indicates the priority of this callback and controls the order
        -: 2161:  in which callbacks are executed.  Use a priority of zero for most handlers;
        -: 2162:  higher priorities will be executed first.
        -: 2163:
        -: 2164:Notes:
        -: 2165:  The routine 'MPID_Finalize' is executed with priority 
        -: 2166:  'MPIR_FINALIZE_CALLBACK_PRIO' (currently defined as 5).  Handlers with
        -: 2167:  a higher priority execute before 'MPID_Finalize' is called; those with
        -: 2168:  a lower priority after 'MPID_Finalize' is called.  
        -: 2169:@*/
        -: 2170:void MPIR_Add_finalize( int (*routine)( void * ), void *extra, int priority );
        -: 2171:
        -: 2172:#define MPIR_FINALIZE_CALLBACK_PRIO 5
        -: 2173:#define MPIR_FINALIZE_CALLBACK_HANDLE_CHECK_PRIO 1
        -: 2174:#define MPIR_FINALIZE_CALLBACK_DEFAULT_PRIO 0
        -: 2175:#define MPIR_FINALIZE_CALLBACK_MAX_PRIO 10
        -: 2176:
        -: 2177:/* For no error checking, we could define MPIR_Nest_incr/decr as empty */
        -: 2178:
        -: 2179:/* These routines export the nesting controls for use in ROMIO */
        -: 2180:void MPIR_Nest_incr_export(void);
        -: 2181:void MPIR_Nest_decr_export(void);
        -: 2182:
        -: 2183:#ifdef MPICH_DEBUG_NESTING
        -: 2184:/* These two routines export the versions of the nest macros that
        -: 2185:   provide the file/line where the nest value changes, also for use in ROMIO */
        -: 2186:void MPIR_Nest_incr_export_dbg(const char *, int);
        -: 2187:void MPIR_Nest_decr_export_dbg(const char *, int);
        -: 2188:
        -: 2189:/* FIXME: We should move the initialization and error reporting into
        -: 2190:   routines that can be called when necessary */
        -: 2191:#define MPIR_Nest_init() {\
        -: 2192:   int _i;\
        -: 2193:   for (_i=0;_i<MPICH_MAX_NESTINFO;_i++) {\
        -: 2194:      MPIU_THREADPRIV_FIELD(nestinfo)[_i].file[0] = 0;\
        -: 2195:      MPIU_THREADPRIV_FIELD(nestinfo)[_i].line = 0;}}
        -: 2196:#define MPIR_Nest_incr() {\
        -: 2197:     if (MPIU_THREADPRIV_FIELD(nest_count) >= MPICH_MAX_NESTINFO) {\
        -: 2198:     MPIU_Internal_error_printf("nest stack exceeded at %s:%d\n",\
        -: 2199:          __FILE__,__LINE__);\
        -: 2200:     }else{\
        -: 2201:     MPIU_Strncpy(MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,__FILE__,\
        -: 2202:                  MPICH_MAX_NESTFILENAME);\
        -: 2203:     MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line=__LINE__;}\
        -: 2204:     MPIU_THREADPRIV_FIELD(nest_count)++; }
        -: 2205:/* Set the line for the current entry to - the old line - this can help
        -: 2206:   identify increments that did not set the fields */
        -: 2207:#define MPIR_Nest_decr() {\
        -: 2208:    if (MPIU_THREADPRIV_FIELD(nest_count) >= 0) {\
        -: 2209:	MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line=-__LINE__;} \
        -: 2210:     MPIU_THREADPRIV_FIELD(nest_count)--; \
        -: 2211:     if (MPIU_THREADPRIV_FIELD(nest_count) < MPICH_MAX_NESTINFO && \
        -: 2212:    strcmp(MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,__FILE__) != 0) {\
        -: 2213:         MPIU_Msg_printf( "Decremented nest count in file %s:%d but incremented in different file (%s:%d)\n",\
        -: 2214:                          __FILE__,__LINE__,\
        -: 2215:                          MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,\
        -: 2216:                          MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line);\
        -: 2217:     }else if (MPIU_THREADPRIV_FIELD(nest_count) < 0){\
        -: 2218:	 MPIU_Msg_printf("Decremented nest count in file %s:%d is negative\n",\
        -: 2219:			 __FILE__,__LINE__);}\
        -: 2220:}
        -: 2221:#else
        -: 2222:#define MPIR_Nest_init()
        -: 2223:#define MPIR_Nest_incr() {MPIU_THREADPRIV_FIELD(nest_count)++;}
        -: 2224:#define MPIR_Nest_decr() {MPIU_THREADPRIV_FIELD(nest_count)--;}
        -: 2225:#endif /* MPICH_DEBUG_NESTING */
        -: 2226:
        -: 2227:#define MPIR_Nest_value() (MPIU_THREADPRIV_FIELD(nest_count))
        -: 2228:
        -: 2229:
        -: 2230:/*int MPIR_Comm_attr_dup(MPID_Comm *, MPID_Attribute **);
        -: 2231:  int MPIR_Comm_attr_delete(MPID_Comm *, MPID_Attribute *);*/
        -: 2232:int MPIR_Comm_copy( MPID_Comm *, int, MPID_Comm ** );
        -: 2233:/* Fortran keyvals are set with functions in mpi_f77interface.h */
        -: 2234:#ifdef HAVE_CXX_BINDING
        -: 2235:extern void MPIR_Keyval_set_cxx( int, void (*)(void), void (*)(void) );
        -: 2236:extern void MPIR_Op_set_cxx( MPI_Op, void (*)(void) );
        -: 2237:extern void MPIR_Errhandler_set_cxx( MPI_Errhandler, void (*)(void) );
        -: 2238:#endif
        -: 2239:
        -: 2240:int MPIR_Group_create( int, MPID_Group ** );
        -: 2241:int MPIR_Group_release(MPID_Group *group_ptr);
        -: 2242:
        -: 2243:int MPIR_dup_fn ( MPI_Comm, int, void *, void *, void *, int * );
        -: 2244:int MPIR_Request_complete(MPI_Request *, MPID_Request *, MPI_Status *, int *);
        -: 2245:int MPIR_Request_get_error(MPID_Request *);
        -: 2246:
        -: 2247:/* The following routines perform the callouts to the user routines registered
        -: 2248:   as part of a generalized request.  They handle any language binding issues
        -: 2249:   that are necessary. They are used when completing, freeing, cancelling or
        -: 2250:   extracting the status from a generalized request. */
        -: 2251:int MPIR_Grequest_cancel(MPID_Request * request_ptr, int complete);
        -: 2252:int MPIR_Grequest_query(MPID_Request * request_ptr);
        -: 2253:int MPIR_Grequest_free(MPID_Request * request_ptr);
        -: 2254:
        -: 2255:/* this routine was added to support our extension relaxing the progress rules
        -: 2256: * for generalized requests */
        -: 2257:int MPIR_Grequest_progress_poke(int count, MPID_Request **request_ptrs, 
        -: 2258:		MPI_Status array_of_statuses[] );
        -: 2259:int MPIR_Grequest_waitall(int count, MPID_Request * const *  request_ptrs);
        -: 2260:
        -: 2261:/* ------------------------------------------------------------------------- */
        -: 2262:/* Prototypes for language-specific routines, such as routines to set
        -: 2263:   Fortran keyval attributes */
        -: 2264:#ifdef HAVE_FORTRAN_BINDING
        -: 2265:#include "mpi_f77interface.h"
        -: 2266:#endif
        -: 2267:
        -: 2268:/* ADI Bindings */
        -: 2269:/*@
        -: 2270:  MPID_Init - Initialize the device
        -: 2271:
        -: 2272:  Input Parameters:
        -: 2273:+ argc_p - Pointer to the argument count
        -: 2274:. argv_p - Pointer to the argument list
        -: 2275:- requested - Requested level of thread support.  Values are the same as
        -: 2276:  for the 'required' argument to 'MPI_Init_thread', except that we define
        -: 2277:  an enum for these values.
        -: 2278:
        -: 2279:  Output Parameters:
        -: 2280:+ provided - Provided level of thread support.  May be less than the 
        -: 2281:  requested level of support.
        -: 2282:. has_args - Set to true if 'argc_p' and 'argv_p' contain the command
        -: 2283:  line arguments.  See below.
        -: 2284:- has_env  - Set to true if the environment of the process has been 
        -: 2285:  set as the user expects.  See below.
        -: 2286:
        -: 2287:  Return value:
        -: 2288:  Returns 'MPI_SUCCESS' on success and an MPI error code on failure.  Failure
        -: 2289:  can happen when, for example, the device is unable  to start or contact the
        -: 2290:  number of processes specified by the 'mpiexec' command.
        -: 2291:
        -: 2292:  Notes:
        -: 2293:  Null arguments for 'argc_p' and 'argv_p' `must` be valid (see MPI-2, section
        -: 2294:  4.2)
        -: 2295:
        -: 2296:  Multi-method devices should initialize each method within this call.
        -: 2297:  They can use environment variables and/or command-line arguments
        -: 2298:  to decide which methods to initialize (but note that they must not
        -: 2299:  `depend` on using command-line arguments).
        -: 2300:
        -: 2301:  This call also initializes all MPID data needed by the device.  This
        -: 2302:  includes the 'MPID_Request's and any other data structures used by 
        -: 2303:  the device.
        -: 2304:
        -: 2305:  The arguments 'has_args' and 'has_env' indicate whether the process was
        -: 2306:  started with command-line arguments or environment variables.  In some
        -: 2307:  cases, only the root process is started with these values; in others, 
        -: 2308:  the startup environment ensures that each process receives the 
        -: 2309:  command-line arguments and environment variables that the user expects. 
        -: 2310:  While the MPI standard makes no requirements that command line arguments or 
        -: 2311:  environment variables are provided to all processes, most users expect a
        -: 2312:  common environment.  These variables allow an MPI implementation (that is
        -: 2313:  based on ADI-3) to provide both of these by making use of MPI communication
        -: 2314:  after 'MPID_Init' is called but before 'MPI_Init' returns to the user, if
        -: 2315:  the process management environment does not provide this service.
        -: 2316:
        -: 2317:
        -: 2318:  This routine is used to implement both 'MPI_Init' and 'MPI_Init_thread'.
        -: 2319:
        -: 2320:  Setting the environment requires a 'setenv' function.  Some
        -: 2321:  systems may not have this.  In that case, the documentation must make 
        -: 2322:  clear that the environment may not be propagated to the generated processes.
        -: 2323:
        -: 2324:  Module:
        -: 2325:  MPID_CORE
        -: 2326:
        -: 2327:  Questions:
        -: 2328:
        -: 2329:  The values for 'has_args' and 'has_env' are boolean.  
        -: 2330:  They could be more specific.  For 
        -: 2331:  example, the value could indicate the rank in 'MPI_COMM_WORLD' of a 
        -: 2332:  process that has the values; the value 'MPI_ANY_SOURCE' (or a '-1') could
        -: 2333:  indicate that the value is available on all processes (including this one).
        -: 2334:  We may want this since otherwise the processes may need to determine whether
        -: 2335:  any process needs the command line.  Another option would be to use positive 
        -: 2336:  values in the same way that the 'color' argument is used in 'MPI_Comm_split';
        -: 2337:  a negative value indicates the member of the processes with that color that 
        -: 2338:  has the values of the command line arguments (or environment).  This allows
        -: 2339:  for non-SPMD programs.
        -: 2340:
        -: 2341:  Do we require that the startup environment (e.g., whatever 'mpiexec' is 
        -: 2342:  using to start processes) is responsible for delivering
        -: 2343:  the command line arguments and environment variables that the user expects?
        -: 2344:  That is, if the user is running an SPMD program, and expects each process
        -: 2345:  to get the same command line argument, who is responsible for this?  
        -: 2346:  The 'has_args' and 'has_env' values are intended to allow the ADI to 
        -: 2347:  handle this while taking advantage of any support that the process 
        -: 2348:  manager framework may provide.
        -: 2349:
        -: 2350:  Alternately, how do we find out from the process management environment
        -: 2351:  whether it took care of the environment or the command line arguments?  
        -: 2352:  Do we need a 'PMI_Env_query' function that can answer these questions
        -: 2353:  dynamically (in case a different process manager is used through the same
        -: 2354:  interface)?
        -: 2355:
        -: 2356:  Can we fix the Fortran command-line arguments?  That is, can we arrange for
        -: 2357:  'iargc' and 'getarg' (and the POSIX equivalents) to return the correct 
        -: 2358:  values?  See, for example, the Absoft implementations of 'getarg'.  
        -: 2359:  We could also contact PGI about the Portland Group compilers, and of 
        -: 2360:  course the 'g77' source code is available.
        -: 2361:  Does each process have the same values for the environment variables 
        -: 2362:  when this routine returns?
        -: 2363:
        -: 2364:  If we don''t require that all processes get the same argument list, 
        -: 2365:  we need to find out if they did anyway so that 'MPI_Init_thread' can
        -: 2366:  fixup the list for the user.  This argues for another return value that
        -: 2367:  flags how much of the environment the 'MPID_Init' routine set up
        -: 2368:  so that the 'MPI_Init_thread' call can provide the rest.  The reason
        -: 2369:  for this is that, even though the MPI standard does not require it, 
        -: 2370:  a user-friendly implementation should, in the SPMD mode, give each
        -: 2371:  process the same environment and argument lists unless the user 
        -: 2372:  explicitly directed otherwise.
        -: 2373:
        -: 2374:  How does this interface to PMI?  Do we need to know anything?  Should
        -: 2375:  this call have an info argument to support PMI?
        -: 2376:
        -: 2377:  The following questions involve how environment variables and command
        -: 2378:  line arguments are used to control the behavior of the implementation. 
        -: 2379:  Many of these values must be determined at the time that 'MPID_Init' 
        -: 2380:  is called.  These all should be considered in the context of the 
        -: 2381:  parameter routines described in the MPICH2 Design Document.
        -: 2382:
        -: 2383:  Are there recommended environment variable names?  For example, in ADI-2,
        -: 2384:  there are many debugging options that are part of the common device.
        -: 2385:  In MPI-2, we can''t require command line arguments, so any such options
        -: 2386:  must also have environment variables.  E.g., 'MPICH_ADI_DEBUG' or
        -: 2387:  'MPICH_ADI_DB'.
        -: 2388:
        -: 2389:  Names that are explicitly prohibited?  For example, do we want to 
        -: 2390:  reserve any names that 'MPI_Init_thread' (as opposed to 'MPID_Init')
        -: 2391:  might use?  
        -: 2392:
        -: 2393:  How does information on command-line arguments and environment variables
        -: 2394:  recognized by the device get added to the documentation?
        -: 2395:
        -: 2396:  What about control for other impact on the environment?  For example,
        -: 2397:  what signals should the device catch (e.g., 'SIGFPE'? 'SIGTRAP'?)?  
        -: 2398:  Which of these should be optional (e.g., ignore or leave signal alone) 
        -: 2399:  or selectable (e.g., port to listen on)?  For example, catching 'SIGTRAP'
        -: 2400:  causes problems for 'gdb', so we''d like to be able to leave 'SIGTRAP' 
        -: 2401:  unchanged in some cases.
        -: 2402:
        -: 2403:  Another environment variable should control whether fault-tolerance is 
        -: 2404:  desired.  If fault-tolerance is selected, then some collective operations 
        -: 2405:  will need to use different algorithms and most fatal errors detected by the 
        -: 2406:  MPI implementation should abort only the affected process, not all processes.
        -: 2407:  @*/
        -: 2408:int MPID_Init( int *argc_p, char ***argv_p, int requested, 
        -: 2409:	       int *provided, int *has_args, int *has_env );
        -: 2410:
        -: 2411:/* was: 
        -: 2412: int MPID_Init( int *argc_p, char ***argv_p, 
        -: 2413:	       int requested, int *provided,
        -: 2414:	       MPID_Comm **parent_comm, int *has_args, int *has_env ); */
        -: 2415:
        -: 2416:/*@ 
        -: 2417:  MPID_InitCompleted - Notify the device that the MPI_Init or MPI_Initthread
        -: 2418:  call has completed setting up MPI
        -: 2419:
        -: 2420: Notes:
        -: 2421: This call allows the device to complete any setup that it wishes to perform
        -: 2422: and for which it needs to access any of the structures (such as 'MPIR_Process')
        -: 2423: that are initialized after 'MPID_Init' is called.  If the device does not need
        -: 2424: any extra operations, then it may provide either an empty function or even
        -: 2425: define this as a macro with the value 'MPI_SUCCESS'.
        -: 2426:  @*/
        -: 2427:int MPID_InitCompleted( void );
        -: 2428:
        -: 2429:/*@
        -: 2430:  MPID_Finalize - Perform the device-specific termination of an MPI job
        -: 2431:
        -: 2432:  Return Value:
        -: 2433:  'MPI_SUCCESS' or a valid MPI error code.  Normally, this routine will
        -: 2434:  return 'MPI_SUCCESS'.  Only in extrordinary circumstances can this
        -: 2435:  routine fail; for example, if some process stops responding during the
        -: 2436:  finalize step.  In this case, 'MPID_Finalize' should return an MPI 
        -: 2437:  error code indicating the reason that it failed.
        -: 2438:
        -: 2439:  Notes:
        -: 2440:
        -: 2441:  Module:
        -: 2442:  MPID_CORE
        -: 2443:
        -: 2444:  Questions:
        -: 2445:  Need to check the MPI-2 requirements on 'MPI_Finalize' with respect to
        -: 2446:  things like which process must remain after 'MPID_Finalize' is called.
        -: 2447:  @*/
        -: 2448:int MPID_Finalize(void);
        -: 2449:/*@
        -: 2450:  MPID_Abort - Abort at least the processes in the specified communicator.
        -: 2451:
        -: 2452:  Input Parameters:
        -: 2453:+ comm        - Communicator of processes to abort
        -: 2454:. mpi_errno   - MPI error code containing the reason for the abort
        -: 2455:. exit_code   - Exit code to return to the calling environment.  See notes.
        -: 2456:- error_msg   - Optional error message
        -: 2457:
        -: 2458:  Return value:
        -: 2459:  'MPI_SUCCESS' or an MPI error code.  Normally, this routine should not 
        -: 2460:  return, since the calling process must be a member of the communicator.  
        -: 2461:  However, under some circumstances, the 'MPID_Abort' might fail; in this 
        -: 2462:  case, returning an error indication is appropriate.
        -: 2463:
        -: 2464:  Notes:
        -: 2465:
        -: 2466:  In a fault-tolerant MPI implementation, this operation should abort `only` 
        -: 2467:  the processes in the specified communicator.  Any communicator that shares
        -: 2468:  processes with the aborted communicator becomes invalid.  For more 
        -: 2469:  details, see (paper not yet written on fault-tolerant MPI).
        -: 2470:
        -: 2471:  In particular, if the communicator is 'MPI_COMM_SELF', only the calling 
        -: 2472:  process should be aborted.
        -: 2473:
        -: 2474:  The 'exit_code' is the exit code that this particular process will 
        -: 2475:  attempt to provide to the 'mpiexec' or other program invocation 
        -: 2476:  environment.  See 'mpiexec' for a discussion of how exit codes from 
        -: 2477:  many processes may be combined.
        -: 2478:
        -: 2479:  If the error_msg field is non-NULL this string will be used as the message
        -: 2480:  with the abort output.  Otherwise, the output message will be base on the
        -: 2481:  error message associated with the mpi_errno.
        -: 2482:
        -: 2483:  An external agent that is aborting processes can invoke this with either
        -: 2484:  'MPI_COMM_WORLD' or 'MPI_COMM_SELF'.  For example, if the process manager
        -: 2485:  wishes to abort a group of processes, it should cause 'MPID_Abort' to 
        -: 2486:  be invoked with 'MPI_COMM_SELF' on each process in the group.
        -: 2487:
        -: 2488:  Question:
        -: 2489:  An alternative design is to provide an 'MPID_Group' instead of a
        -: 2490:  communicator.  This would allow a process manager to ask the ADI 
        -: 2491:  to kill an entire group of processes without needing a communicator.
        -: 2492:  However, the implementation of 'MPID_Abort' will either do this by
        -: 2493:  communicating with other processes or by requesting the process manager
        -: 2494:  to kill the processes.  That brings up this question: should 
        -: 2495:  'MPID_Abort' use 'PMI' to kill processes?  Should it be required to
        -: 2496:  notify the process manager?  What about persistent resources (such 
        -: 2497:  as SYSV segments or forked processes)?  
        -: 2498:
        -: 2499:  This suggests that for any persistent resource, an exit handler be
        -: 2500:  defined.  These would be executed by 'MPID_Abort' or 'MPID_Finalize'.  
        -: 2501:  See the implementation of 'MPI_Finalize' for an example of exit callbacks.
        -: 2502:  In addition, code that registered persistent resources could use persistent
        -: 2503:  storage (i.e., a file) to record that information, allowing cleanup 
        -: 2504:  utilities (such as 'mpiexec') to remove any resources left after the 
        -: 2505:  process exits.
        -: 2506:
        -: 2507:  'MPI_Finalize' requires that attributes on 'MPI_COMM_SELF' be deleted 
        -: 2508:  before anything else happens; this allows libraries to attach end-of-job
        -: 2509:  actions to 'MPI_Finalize'.  It is valuable to have a similar 
        -: 2510:  capability on 'MPI_Abort', with the caveat that 'MPI_Abort' may not 
        -: 2511:  guarantee that the run-on-abort routines were called.  This provides a
        -: 2512:  consistent way for the MPICH implementation to handle freeing any 
        -: 2513:  persistent resources.  However, such callbacks must be limited since
        -: 2514:  communication may not be possible once 'MPI_Abort' is called.  Further,
        -: 2515:  any callbacks must guarantee that they have finite termination.  
        -: 2516:  
        -: 2517:  One possible extension would be to allow `users` to add actions to be 
        -: 2518:  run when 'MPI_Abort' is called, perhaps through a special attribute value
        -: 2519:  applied to 'MPI_COMM_SELF'.  Note that is is incorrect to call the delete 
        -: 2520:  functions for the normal attributes on 'MPI_COMM_SELF' because MPI
        -: 2521:  only specifies that those are run on 'MPI_Finalize' (i.e., normal 
        -: 2522:  termination). 
        -: 2523:
        -: 2524:  Module:
        -: 2525:  MPID_CORE
        -: 2526:  @*/
        -: 2527:
        -: 2528:/* FIXME: the 4th argument isn't part of the original design and isn't documented */
        -: 2529:
        -: 2530:# if 0
        -: 2531:int MPID_Abort( MPID_Comm *comm, int mpi_errno, int exit_code, const char *error_msg );
        -: 2532:#endif
        -: 2533:/* FIXME: Should we turn off this flag and only declare MPID_Abort in mpiutil.h? */
        -: 2534:/* We want to also declare MPID_Abort in mpiutil.h if mpiimpl.h is not used */
        -: 2535:#define HAS_MPID_ABORT_DECL
        -: 2536:
        -: 2537:int MPID_Open_port(MPID_Info *, char *);
        -: 2538:int MPID_Close_port(const char *);
        -: 2539:
        -: 2540:/*@
        -: 2541:   MPID_Comm_accept - MPID entry point for MPI_Comm_accept
        -: 2542:
        -: 2543:   Input Parameters:
        -: 2544:+  port_name - port name
        -: 2545:.  info - info
        -: 2546:.  root - root
        -: 2547:-  comm - communicator
        -: 2548:
        -: 2549:   Output Parameters:
        -: 2550:.  MPI_Comm *newcomm - new communicator
        -: 2551:
        -: 2552:  Return Value:
        -: 2553:  'MPI_SUCCESS' or a valid MPI error code.
        -: 2554:@*/
        -: 2555:int MPID_Comm_accept(char *, MPID_Info *, int, MPID_Comm *, MPID_Comm **);
        -: 2556:
        -: 2557:/*@
        -: 2558:   MPID_Comm_connect - MPID entry point for MPI_Comm_connect
        -: 2559:
        -: 2560:   Input Parameters:
        -: 2561:+  port_name - port name
        -: 2562:.  info - info
        -: 2563:.  root - root
        -: 2564:-  comm - communicator
        -: 2565:
        -: 2566:   Output Parameters:
        -: 2567:.  newcomm_ptr - new intercommunicator
        -: 2568:
        -: 2569:  Return Value:
        -: 2570:  'MPI_SUCCESS' or a valid MPI error code.
        -: 2571:@*/
        -: 2572:int MPID_Comm_connect(const char *, MPID_Info *, int, MPID_Comm *, MPID_Comm **);
        -: 2573:
        -: 2574:int MPID_Comm_disconnect(MPID_Comm *);
        -: 2575:
        -: 2576:int MPID_Comm_spawn_multiple(int, char *[], char* *[], int [], MPID_Info* [],
        -: 2577:                             int, MPID_Comm *, MPID_Comm **, int []);
        -: 2578:
        -: 2579:/*@
        -: 2580:  MPID_Send - MPID entry point for MPI_Send
        -: 2581:
        -: 2582:  Notes:
        -: 2583:  The only difference between this and 'MPI_Send' is that the basic
        -: 2584:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2585:  have been made, the MPI opaque objects have been replaced by
        -: 2586:  MPID objects, a context id offset is provided in addition to the 
        -: 2587:  communicator, and a request may be returned.  The context offset is 
        -: 2588:  added to the context of the communicator
        -: 2589:  to get the context it used by the message.
        -: 2590:  A request is returned only if the ADI implementation was unable to 
        -: 2591:  complete the send of the message.  In that case, the usual 'MPI_Wait'
        -: 2592:  logic should be used to complete the request.  This approach is used to 
        -: 2593:  allow a simple implementation of the ADI.  The ADI is free to always 
        -: 2594:  complete the message and never return a request.
        -: 2595:
        -: 2596:  Module:
        -: 2597:  Communication
        -: 2598:
        -: 2599:  @*/
        -: 2600:int MPID_Send( const void *buf, int count, MPI_Datatype datatype,
        -: 2601:	       int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2602:	       MPID_Request **request );
        -: 2603:
        -: 2604:/*@
        -: 2605:  MPID_Rsend - MPID entry point for MPI_Rsend
        -: 2606:
        -: 2607:  Notes:
        -: 2608:  The only difference between this and 'MPI_Rsend' is that the basic
        -: 2609:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2610:  have been made, the MPI opaque objects have been replaced by
        -: 2611:  MPID objects, a context id offset is provided in addition to the 
        -: 2612:  communicator, and a request may be returned.  The context offset is 
        -: 2613:  added to the context of the communicator
        -: 2614:  to get the context it used by the message.
        -: 2615:  A request is returned only if the ADI implementation was unable to 
        -: 2616:  complete the send of the message.  In that case, the usual 'MPI_Wait'
        -: 2617:  logic should be used to complete the request.  This approach is used to 
        -: 2618:  allow a simple implementation of the ADI.  The ADI is free to always 
        -: 2619:  complete the message and never return a request.
        -: 2620:
        -: 2621:  Module:
        -: 2622:  Communication
        -: 2623:
        -: 2624:  @*/
        -: 2625:int MPID_Rsend( const void *buf, int count, MPI_Datatype datatype,
        -: 2626:		int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2627:		MPID_Request **request );
        -: 2628:
        -: 2629:/*@
        -: 2630:  MPID_Ssend - MPID entry point for MPI_Ssend
        -: 2631:
        -: 2632:  Notes:
        -: 2633:  The only difference between this and 'MPI_Ssend' is that the basic
        -: 2634:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2635:  have been made, the MPI opaque objects have been replaced by
        -: 2636:  MPID objects, a context id offset is provided in addition to the 
        -: 2637:  communicator, and a request may be returned.  The context offset is 
        -: 2638:  added to the context of the communicator
        -: 2639:  to get the context it used by the message.
        -: 2640:  A request is returned only if the ADI implementation was unable to 
        -: 2641:  complete the send of the message.  In that case, the usual 'MPI_Wait'
        -: 2642:  logic should be used to complete the request.  This approach is used to 
        -: 2643:  allow a simple implementation of the ADI.  The ADI is free to always 
        -: 2644:  complete the message and never return a request.
        -: 2645:
        -: 2646:  Module:
        -: 2647:  Communication
        -: 2648:
        -: 2649:  @*/
        -: 2650:int MPID_Ssend( const void *buf, int count, MPI_Datatype datatype,
        -: 2651:		int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2652:		MPID_Request **request );
        -: 2653:
        -: 2654:/*@
        -: 2655:  MPID_tBsend - Attempt a send and return if it would block
        -: 2656:
        -: 2657:  Notes:
        -: 2658:  This has the semantics of 'MPI_Bsend', except that it returns the internal
        -: 2659:  error code 'MPID_WOULD_BLOCK' if the message can''t be sent immediately
        -: 2660:  (t is for "try").  
        -: 2661: 
        -: 2662:  The reason that this interface is chosen over a query to check whether
        -: 2663:  a message `can` be sent is that the query approach is not
        -: 2664:  thread-safe.  Since the decision on whether a message can be sent
        -: 2665:  without blocking depends (among other things) on the state of flow
        -: 2666:  control managed by the device, this approach also gives the device
        -: 2667:  the greatest freedom in implementing flow control.  In particular,
        -: 2668:  if another MPI process can change the flow control parameters, then
        -: 2669:  even in a single-threaded implementation, it would not be safe to
        -: 2670:  return, for example, a message size that could be sent with 'MPI_Bsend'.
        -: 2671:
        -: 2672:  This routine allows an MPI implementation to optimize 'MPI_Bsend'
        -: 2673:  for the case when the message can be delivered without blocking the
        -: 2674:  calling process.  An ADI implementation is free to have this routine
        -: 2675:  always return 'MPID_WOULD_BLOCK', but is encouraged not to.
        -: 2676:
        -: 2677:  To allow the MPI implementation to avoid trying this routine when it
        -: 2678:  is not implemented by the ADI, the C preprocessor constant 'MPID_HAS_TBSEND'
        -: 2679:  should be defined if this routine has a nontrivial implementation.
        -: 2680:
        -: 2681:  This is an optional routine.  The MPI code for 'MPI_Bsend' will attempt
        -: 2682:  to call this routine only if the device defines 'MPID_HAS_TBSEND'.
        -: 2683:
        -: 2684:  Module:
        -: 2685:  Communication
        -: 2686:  @*/
        -: 2687:int MPID_tBsend( const void *buf, int count, MPI_Datatype datatype,
        -: 2688:		 int dest, int tag, MPID_Comm *comm, int context_offset );
        -: 2689:
        -: 2690:/*@
        -: 2691:  MPID_Isend - MPID entry point for MPI_Isend
        -: 2692:
        -: 2693:  Notes:
        -: 2694:  The only difference between this and 'MPI_Isend' is that the basic
        -: 2695:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2696:  have been made, the MPI opaque objects have been replaced by
        -: 2697:  MPID objects, and a context id offset is provided in addition to the 
        -: 2698:  communicator.  This offset is added to the context of the communicator
        -: 2699:  to get the context it used by the message.
        -: 2700:
        -: 2701:  Module:
        -: 2702:  Communication
        -: 2703:
        -: 2704:  @*/
        -: 2705:int MPID_Isend( const void *buf, int count, MPI_Datatype datatype,
        -: 2706:		int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2707:		MPID_Request **request );
        -: 2708:
        -: 2709:/*@
        -: 2710:  MPID_Irsend - MPID entry point for MPI_Irsend
        -: 2711:
        -: 2712:  Notes:
        -: 2713:  The only difference between this and 'MPI_Irsend' is that the basic
        -: 2714:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2715:  have been made, the MPI opaque objects have been replaced by
        -: 2716:  MPID objects, and a context id offset is provided in addition to the 
        -: 2717:  communicator.  This offset is added to the context of the communicator
        -: 2718:  to get the context it used by the message.
        -: 2719:
        -: 2720:  Module:
        -: 2721:  Communication
        -: 2722:
        -: 2723:  @*/
        -: 2724:int MPID_Irsend( const void *buf, int count, MPI_Datatype datatype,
        -: 2725:		 int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2726:		 MPID_Request **request );
        -: 2727:
        -: 2728:/*@
        -: 2729:  MPID_Issend - MPID entry point for MPI_Issend
        -: 2730:
        -: 2731:  Notes:
        -: 2732:  The only difference between this and 'MPI_Issend' is that the basic
        -: 2733:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2734:  have been made, the MPI opaque objects have been replaced by
        -: 2735:  MPID objects, and a context id offset is provided in addition to the 
        -: 2736:  communicator.  This offset is added to the context of the communicator
        -: 2737:  to get the context it used by the message.
        -: 2738:
        -: 2739:  Module:
        -: 2740:  Communication
        -: 2741:
        -: 2742:  @*/
        -: 2743:int MPID_Issend( const void *buf, int count, MPI_Datatype datatype,
        -: 2744:		 int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2745:		 MPID_Request **request );
        -: 2746:
        -: 2747:/*@
        -: 2748:  MPID_Recv - MPID entry point for MPI_Recv
        -: 2749:
        -: 2750:  Notes:
        -: 2751:  The only difference between this and 'MPI_Recv' is that the basic
        -: 2752:  error checks (e.g., valid communicator, datatype, source, and tag)
        -: 2753:  have been made, the MPI opaque objects have been replaced by
        -: 2754:  MPID objects, a context id offset is provided in addition to the 
        -: 2755:  communicator, and a request may be returned.  The context offset is added 
        -: 2756:  to the context of the communicator to get the context it used by the message.
        -: 2757:  As in 'MPID_Send', the request is returned only if the operation did not
        -: 2758:  complete.  Conversely, the status object is populated with valid information
        -: 2759:  only if the operation completed.
        -: 2760:
        -: 2761:  Module:
        -: 2762:  Communication
        -: 2763:
        -: 2764:  @*/
        -: 2765:int MPID_Recv( void *buf, int count, MPI_Datatype datatype,
        -: 2766:	       int source, int tag, MPID_Comm *comm, int context_offset,
        -: 2767:	       MPI_Status *status, MPID_Request **request );
        -: 2768:
        -: 2769:
        -: 2770:/*@
        -: 2771:  MPID_Irecv - MPID entry point for MPI_Irecv
        -: 2772:
        -: 2773:  Notes:
        -: 2774:  The only difference between this and 'MPI_Irecv' is that the basic
        -: 2775:  error checks (e.g., valid communicator, datatype, source, and tag)
        -: 2776:  have been made, the MPI opaque objects have been replaced by
        -: 2777:  MPID objects, and a context id offset is provided in addition to the 
        -: 2778:  communicator.  This offset is added to the context of the communicator
        -: 2779:  to get the context it used by the message.
        -: 2780:
        -: 2781:  Module:
        -: 2782:  Communication
        -: 2783:
        -: 2784:  @*/
        -: 2785:int MPID_Irecv( void *buf, int count, MPI_Datatype datatype,
        -: 2786:		int source, int tag, MPID_Comm *comm, int context_offset,
        -: 2787:		MPID_Request **request );
        -: 2788:
        -: 2789:/*@
        -: 2790:  MPID_Send_init - MPID entry point for MPI_Send_init
        -: 2791:
        -: 2792:  Notes:
        -: 2793:  The only difference between this and 'MPI_Send_init' is that the basic
        -: 2794:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2795:  have been made, the MPI opaque objects have been replaced by
        -: 2796:  MPID objects, and a context id offset is provided in addition to the 
        -: 2797:  communicator.  This offset is added to the context of the communicator
        -: 2798:  to get the context it used by the message.
        -: 2799:
        -: 2800:  Module:
        -: 2801:  Communication
        -: 2802:
        -: 2803:  @*/
        -: 2804:int MPID_Send_init( const void *buf, int count, MPI_Datatype datatype,
        -: 2805:		    int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2806:		    MPID_Request **request );
        -: 2807:
        -: 2808:int MPID_Bsend_init(const void *, int, MPI_Datatype, int, int, MPID_Comm *,
        -: 2809:		   int, MPID_Request **);
        -: 2810:/*@
        -: 2811:  MPID_Rsend_init - MPID entry point for MPI_Rsend_init
        -: 2812:
        -: 2813:  Notes:
        -: 2814:  The only difference between this and 'MPI_Rsend_init' is that the basic
        -: 2815:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2816:  have been made, the MPI opaque objects have been replaced by
        -: 2817:  MPID objects, and a context id offset is provided in addition to the 
        -: 2818:  communicator.  This offset is added to the context of the communicator
        -: 2819:  to get the context it used by the message.
        -: 2820:
        -: 2821:  Module:
        -: 2822:  Communication
        -: 2823:
        -: 2824:  @*/
        -: 2825:int MPID_Rsend_init( const void *buf, int count, MPI_Datatype datatype,
        -: 2826:		     int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2827:		     MPID_Request **request );
        -: 2828:/*@
        -: 2829:  MPID_Ssend_init - MPID entry point for MPI_Ssend_init
        -: 2830:
        -: 2831:  Notes:
        -: 2832:  The only difference between this and 'MPI_Ssend_init' is that the basic
        -: 2833:  error checks (e.g., valid communicator, datatype, dest, and tag)
        -: 2834:  have been made, the MPI opaque objects have been replaced by
        -: 2835:  MPID objects, and a context id offset is provided in addition to the 
        -: 2836:  communicator.  This offset is added to the context of the communicator
        -: 2837:  to get the context it used by the message.
        -: 2838:
        -: 2839:  Module:
        -: 2840:  Communication
        -: 2841:
        -: 2842:  @*/
        -: 2843:int MPID_Ssend_init( const void *buf, int count, MPI_Datatype datatype,
        -: 2844:		     int dest, int tag, MPID_Comm *comm, int context_offset,
        -: 2845:		     MPID_Request **request );
        -: 2846:
        -: 2847:/*@
        -: 2848:  MPID_Recv_init - MPID entry point for MPI_Recv_init
        -: 2849:
        -: 2850:  Notes:
        -: 2851:  The only difference between this and 'MPI_Recv_init' is that the basic
        -: 2852:  error checks (e.g., valid communicator, datatype, source, and tag)
        -: 2853:  have been made, the MPI opaque objects have been replaced by
        -: 2854:  MPID objects, and a context id offset is provided in addition to the 
        -: 2855:  communicator.  This offset is added to the context of the communicator
        -: 2856:  to get the context it used by the message.
        -: 2857:
        -: 2858:  Module:
        -: 2859:  Communication
        -: 2860:
        -: 2861:  @*/
        -: 2862:int MPID_Recv_init( void *buf, int count, MPI_Datatype datatype,
        -: 2863:		    int source, int tag, MPID_Comm *comm, int context_offset,
        -: 2864:		    MPID_Request **request );
        -: 2865:
        -: 2866:/*@
        -: 2867:  MPID_Startall - MPID entry point for MPI_Startall
        -: 2868:
        -: 2869:  Notes:
        -: 2870:  The only difference between this and 'MPI_Startall' is that the basic
        -: 2871:  error checks (e.g., count) have been made, and the MPI opaque objects
        -: 2872:  have been replaced by pointers to MPID objects.  
        -: 2873:
        -: 2874:  Rationale:
        -: 2875:  This allows the device to schedule communication involving multiple requests,
        -: 2876:  whereas an implementation built on just 'MPID_Start' would force the
        -: 2877:  ADI to initiate the communication in the order encountered.
        -: 2878:  @*/
        -: 2879:int MPID_Startall(int count, MPID_Request *requests[]);
        -: 2880:
        -: 2881:/*@
        -: 2882:   MPID_Probe - Block until a matching request is found and return information 
        -: 2883:   about it
        -: 2884:
        -: 2885:  Input Parameters:
        -: 2886:+ source - rank to match (or 'MPI_ANY_SOURCE')
        -: 2887:. tag - Tag to match (or 'MPI_ANY_TAG')
        -: 2888:. comm - communicator to match.
        -: 2889:- context_offset - context id offset of communicator to match
        -: 2890:
        -: 2891:  Output Parameter:
        -: 2892:. status - 'MPI_Status' set as defined by 'MPI_Probe'
        -: 2893:
        -: 2894:
        -: 2895:  Return Value:
        -: 2896:  Error code.
        -: 2897:  
        -: 2898:  Notes:
        -: 2899:  Note that the values returned in 'status' will be valid for a subsequent
        -: 2900:  MPI receive operation only if no other thread attempts to receive the same
        -: 2901:  message.  
        -: 2902:  (See the 
        -: 2903:  discussion of probe in Section 8.7.2 Clarifications of the MPI-2 standard.)
        -: 2904:
        -: 2905:  Providing the 'context_offset' is necessary at this level to support the 
        -: 2906:  way in which the MPICH implementation uses context ids in the implementation
        -: 2907:  of other operations.  The communicator is present to allow the device 
        -: 2908:  to use message-queues attached to particular communicators or connections
        -: 2909:  between processes.
        -: 2910:
        -: 2911:  Module:
        -: 2912:  Request
        -: 2913:
        -: 2914:  @*/
        -: 2915:int MPID_Probe(int, int, MPID_Comm *, int, MPI_Status *);
        -: 2916:/*@
        -: 2917:   MPID_Iprobe - Look for a matching request in the receive queue 
        -: 2918:   but do not remove or return it
        -: 2919:
        -: 2920:  Input Parameters:
        -: 2921:+ source - rank to match (or 'MPI_ANY_SOURCE')
        -: 2922:. tag - Tag to match (or 'MPI_ANY_TAG')
        -: 2923:. comm - communicator to match.
        -: 2924:- context_offset - context id offset of communicator to match
        -: 2925:
        -: 2926:  Output Parameter:
        -: 2927:+ flag - true if a matching request was found, false otherwise.
        -: 2928:- status - 'MPI_Status' set as defined by 'MPI_Iprobe' (only valid when return 
        -: 2929:  flag is true).
        -: 2930:
        -: 2931:  Return Value:
        -: 2932:  Error Code.
        -: 2933:
        -: 2934:  Notes:
        -: 2935:  Note that the values returned in 'status' will be valid for a subsequent
        -: 2936:  MPI receive operation only if no other thread attempts to receive the same
        -: 2937:  message.  
        -: 2938:  (See the 
        -: 2939:  discussion of probe in Section 8.7.2 (Clarifications) of the MPI-2 standard.)
        -: 2940:
        -: 2941:  Providing the 'context_offset' is necessary at this level to support the 
        -: 2942:  way in which the MPICH implementation uses context ids in the implementation
        -: 2943:  of other operations.  The communicator is present to allow the device 
        -: 2944:  to use message-queues attached to particular communicators or connections
        -: 2945:  between processes.
        -: 2946:
        -: 2947:  Devices that rely solely on polling to make progress should call
        -: 2948:  MPID_Progress_poke() (or some equivalent function) if a matching request
        -: 2949:  could not be found.  This insures that progress continues to be made even if
        -: 2950:  the application is calling MPI_Iprobe() from within a loop not containing
        -: 2951:  calls to any other MPI functions.
        -: 2952:  
        -: 2953:  Module:
        -: 2954:  Request
        -: 2955:
        -: 2956:  @*/
        -: 2957:int MPID_Iprobe(int, int, MPID_Comm *, int, int *, MPI_Status *);
        -: 2958:
        -: 2959:/*@
        -: 2960:  MPID_Cancel_send - Cancel the indicated send request
        -: 2961:
        -: 2962:  Input Parameter:
        -: 2963:. request - Send request to cancel
        -: 2964:
        -: 2965:  Return Value:
        -: 2966:  MPI error code.
        -: 2967:  
        -: 2968:  Notes:
        -: 2969:  Cancel is a tricky operation, particularly for sends.  Read the
        -: 2970:  discussion in the MPI-1 and MPI-2 documents carefully.  This call
        -: 2971:  only requests that the request be cancelled; a subsequent wait 
        -: 2972:  or test must first succeed (i.e., the request completion counter must be
        -: 2973:  zeroed).
        -: 2974:
        -: 2975:  Module:
        -: 2976:  Request
        -: 2977:
        -: 2978:  @*/
        -: 2979:int MPID_Cancel_send(MPID_Request *);
        -: 2980:/*@
        -: 2981:  MPID_Cancel_recv - Cancel the indicated recv request
        -: 2982:
        -: 2983:  Input Parameter:
        -: 2984:. request - Receive request to cancel
        -: 2985:
        -: 2986:  Return Value:
        -: 2987:  MPI error code.
        -: 2988:  
        -: 2989:  Notes:
        -: 2990:  This cancels a pending receive request.  In many cases, this is implemented
        -: 2991:  by simply removing the request from a pending receive request queue.  
        -: 2992:  However, some ADI implementations may maintain these queues in special 
        -: 2993:  places, such as within a NIC (Network Interface Card).
        -: 2994:  This call only requests that the request be cancelled; a subsequent wait 
        -: 2995:  or test must first succeed (i.e., the request completion counter must be
        -: 2996:  zeroed).
        -: 2997:
        -: 2998:  Module:
        -: 2999:  Request
        -: 3000:
        -: 3001:  @*/
        -: 3002:int MPID_Cancel_recv(MPID_Request *);
        -: 3003:
        -: 3004:int MPID_Win_create(void *, MPI_Aint, int, MPID_Info *, MPID_Comm *,
        -: 3005:                    MPID_Win **);
        -: 3006:int MPID_Win_fence(int, MPID_Win *);
        -: 3007:int MPID_Put(void *, int, MPI_Datatype, int, MPI_Aint, int,
        -: 3008:            MPI_Datatype, MPID_Win *); 
        -: 3009:int MPID_Get(void *, int, MPI_Datatype, int, MPI_Aint, int,
        -: 3010:            MPI_Datatype, MPID_Win *);
        -: 3011:int MPID_Accumulate(void *, int, MPI_Datatype, int, MPI_Aint, int, 
        -: 3012:		   MPI_Datatype, MPI_Op, MPID_Win *);
        -: 3013:int MPID_Win_free(MPID_Win **); 
        -: 3014:int MPID_Win_test(MPID_Win *win_ptr, int *flag);
        -: 3015:int MPID_Win_wait(MPID_Win *win_ptr);
        -: 3016:int MPID_Win_complete(MPID_Win *win_ptr);
        -: 3017:int MPID_Win_post(MPID_Group *group_ptr, int assert, MPID_Win *win_ptr);
        -: 3018:int MPID_Win_start(MPID_Group *group_ptr, int assert, MPID_Win *win_ptr);
        -: 3019:int MPID_Win_lock(int lock_type, int dest, int assert, MPID_Win *win_ptr);
        -: 3020:int MPID_Win_unlock(int dest, MPID_Win *win_ptr);
        -: 3021:
        -: 3022:/*@
        -: 3023:  MPID_Progress_start - Begin a block of operations that check the completion
        -: 3024:  counters in requests.
        -: 3025:
        -: 3026:  Input parameters:
        -: 3027:. state - pointer to a progress state variable
        -: 3028:    
        -: 3029:  Notes:
        -: 3030:  This routine is informs the progress engine that a block of code follows that
        -: 3031:  will examine the completion counter of some 'MPID_Request' objects and then
        -: 3032:  call 'MPID_Progress_wait' zero or more times followed by a call to
        -: 3033:  'MPID_Progress_end'.
        -: 3034:  
        -: 3035:  The progress state variable must be specific to the thread calling it.  If at
        -: 3036:  all possible, the state should be declared as an auto variable and thus
        -: 3037:  allocated on the stack of the current thread.  Thread specific storage could
        -: 3038:  be used instead, but doing such would incur additional (and typically
        -: 3039:  unnecessary) overhead.
        -: 3040:  
        -: 3041:  This routine is needed to properly implement blocking tests when
        -: 3042:  multithreaded progress engines are used.  In a single-threaded implementation
        -: 3043:  of the ADI, this may be defined as an empty macro.
        -: 3044:
        -: 3045:  Module:
        -: 3046:  Communication
        -: 3047:  @*/
        -: 3048:void MPID_Progress_start(MPID_Progress_state * state);
        -: 3049:/*@
        -: 3050:  MPID_Progress_wait - Wait for some communication since 'MPID_Progress_start' 
        -: 3051:
        -: 3052:    Input parameters:
        -: 3053:.   state - pointer to the progress state initialized by MPID_Progress_start
        -: 3054:    
        -: 3055:    Return value:
        -: 3056:    An mpi error code.
        -: 3057:
        -: 3058:    Notes:
        -: 3059:    This instructs the progress engine to wait until some communication event
        -: 3060:    happens since 'MPID_Progress_start' was called.  This call blocks the 
        -: 3061:    calling thread (only, not the process).
        -: 3062:
        -: 3063:  Module:
        -: 3064:  Communication
        -: 3065: @*/
        -: 3066:int MPID_Progress_wait(MPID_Progress_state * state);
        -: 3067:/*@
        -: 3068:  MPID_Progress_end - End a block of operations begun with 'MPID_Progress_start'
        -: 3069:
        -: 3070:  Input parameters:
        -: 3071:  . state - pointer to the progress state variable passed to
        -: 3072:    'MPID_Progress_start'
        -: 3073:
        -: 3074:   Notes:
        -: 3075:   This routine instructs the progress engine to end the block begun with
        -: 3076:   'MPID_Progress_start'.  The progress engine is not required to check for any
        -: 3077:   pending communication.
        -: 3078:
        -: 3079:   The purpose of this call is to release any locks initiated by
        -: 3080:   'MPID_Progess_start' or 'MPID_Progess_wait'.  In a single threaded ADI
        -: 3081:   implementation, this may be defined as an empty macro.
        -: 3082:
        -: 3083:  Module:
        -: 3084:  Communication
        -: 3085:   @*/
        -: 3086:void MPID_Progress_end(MPID_Progress_state * stae);
        -: 3087:/*@
        -: 3088:  MPID_Progress_test - Check for communication
        -: 3089:
        -: 3090:  Return value:
        -: 3091:  An mpi error code.
        -: 3092:  
        -: 3093:  Notes:
        -: 3094:  Unlike 'MPID_Progress_wait', this routine is nonblocking.  Therefore, it
        -: 3095:  does not require the use of 'MPID_Progress_start' and 'MPID_Progress_end'.
        -: 3096:  
        -: 3097:  Module:
        -: 3098:  Communication
        -: 3099:  @*/
        -: 3100:int MPID_Progress_test(void);
        -: 3101:/*@
        -: 3102:  MPID_Progress_poke - Allow a progress engine to check for pending 
        -: 3103:  communication
        -: 3104:
        -: 3105:  Return value:
        -: 3106:  An mpi error code.
        -: 3107:  
        -: 3108:  Notes:
        -: 3109:  This routine provides a way to invoke the progress engine in a polling 
        -: 3110:  implementation of the ADI.  This routine must be nonblocking.
        -: 3111:
        -: 3112:  A multithreaded implementation is free to define this as an empty macro.
        -: 3113:
        -: 3114:  Module:
        -: 3115:  Communication
        -: 3116:  @*/
        -: 3117:int MPID_Progress_poke(void);
        -: 3118:
        -: 3119:/*@
        -: 3120:  MPID_Request_create - Create and return a bare request
        -: 3121:
        -: 3122:  Return value:
        -: 3123:  A pointer to a new request object.
        -: 3124:
        -: 3125:  Notes:
        -: 3126:  This routine is intended for use by 'MPI_Grequest_start' only.  Note that 
        -: 3127:  once a request is created with this routine, any progress engine must assume 
        -: 3128:  that an outside function can complete a request with 
        -: 3129:  'MPID_Request_set_completed'.
        -: 3130:
        -: 3131:  The request object returned by this routine should be initialized such that
        -: 3132:  ref_count is one and handle contains a valid handle referring to the object.
        -: 3133:  @*/
        -: 3134:MPID_Request * MPID_Request_create(void);
        -: 3135:void MPID_Request_set_completed(MPID_Request *);
        -: 3136:/*@
        -: 3137:  MPID_Request_release - Release a request 
        -: 3138:
        -: 3139:  Input Parameter:
        -: 3140:. request - request to release
        -: 3141:
        -: 3142:  Notes:
        -: 3143:  This routine is called to release a reference to request object.  If
        -: 3144:  the reference count of the request object has reached zero, the object will
        -: 3145:  be deallocated.
        -: 3146:
        -: 3147:  Module:
        -: 3148:  Request
        -: 3149:@*/
        -: 3150:void MPID_Request_release(MPID_Request *);
        -: 3151:
        -: 3152:typedef struct MPID_Grequest_class {
        -: 3153:     MPIU_OBJECT_HEADER; /* adds handle and ref_count fields */
        -: 3154:     MPI_Grequest_query_function *query_fn;
        -: 3155:     MPI_Grequest_free_function *free_fn;
        -: 3156:     MPI_Grequest_cancel_function *cancel_fn;
        -: 3157:     MPIX_Grequest_poll_function *poll_fn;
        -: 3158:     MPIX_Grequest_wait_function *wait_fn;
        -: 3159:     struct MPID_Grequest_class *next;
        -: 3160:} MPID_Grequest_class;
        -: 3161:
        -: 3162:
        -: 3163:/*TTopoOverview.tex
        -: 3164: *
        -: 3165: * The MPI collective and topology routines can benefit from information 
        -: 3166: * about the topology of the underlying interconnect.  Unfortunately, there
        -: 3167: * is no best form for the representation (the MPI-1 Forum tried to define
        -: 3168: * such a representation, but was unable to).  One useful decomposition
        -: 3169: * that has been used in cluster enviroments is a hierarchical decomposition.
        -: 3170: *
        -: 3171: * The other obviously useful topology information would match the needs of 
        -: 3172: * 'MPI_Cart_create'.  However, it may be simpler to for the device to 
        -: 3173: * implement this routine directly.
        -: 3174: *
        -: 3175: * Other useful information could be the topology information that matches
        -: 3176: * the needs of the collective operation, such as spanning trees and rings.
        -: 3177: * These may be added to ADI3 later.
        -: 3178: *
        -: 3179: * Question: Should we define a cart create function?  Dims create?
        -: 3180: *
        -: 3181: * Usage:
        -: 3182: * This routine has nothing to do with the choice of communication method
        -: 3183: * that a implementation of the ADI may make.  It is intended only to
        -: 3184: * communicate information on the heirarchy of processes, if any, to 
        -: 3185: * the implementation of the collective communication routines.  This routine
        -: 3186: * may also be useful for the MPI Graph topology functions.
        -: 3187: *
        -: 3188: T*/
        -: 3189:
        -: 3190:/*@
        -: 3191:  MPID_Topo_cluster_info - Return information on the hierarchy of 
        -: 3192:  interconnections
        -: 3193:
        -: 3194:  Input Parameter:
        -: 3195:. comm - Communicator to study.  May be 'NULL', in which case 'MPI_COMM_WORLD'
        -: 3196:  is the effective communicator.
        -: 3197:
        -: 3198:  Output Parameters:
        -: 3199:+ levels - The number of levels in the hierarchy.  
        -: 3200:  To simplify the use of this routine, the maximum value is 
        -: 3201:  'MPID_TOPO_CLUSTER_MAX_LEVELS' (typically 8 or less).
        -: 3202:. my_cluster - For each level, the id of the cluster that the calling process
        -: 3203:  belongs to.
        -: 3204:- my_rank - For each level, the rank of the calling process in its cluster
        -: 3205:
        -: 3206:  Notes:
        -: 3207:  This routine returns a description of the system in terms of nested 
        -: 3208:  clusters of processes.  Levels are numbered from zero.  At each level,
        -: 3209:  each process may belong to no more than cluster; if a process is in any
        -: 3210:  cluster at level i, it must be in some cluster at level i-1.
        -: 3211:
        -: 3212:  The communicator argument allows this routine to be used in the dynamic
        -: 3213:  process case (i.e., with communicators that are created after 'MPI_Init' 
        -: 3214:  and that involve processes that are not part of 'MPI_COMM_WORLD').
        -: 3215:
        -: 3216:  For non-hierarchical systems, this routine simply returns a single 
        -: 3217:  level containing all processes.
        -: 3218:
        -: 3219:  Sample Outputs:
        -: 3220:  For a single, switch-connected cluster or a uniform-memory-access (UMA)
        -: 3221:  symmetric multiprocessor (SMP), the return values could be
        -: 3222:.vb
        -: 3223:    level       my_cluster         my_rank
        -: 3224:    0           0                  rank in comm_world
        -: 3225:.ve
        -: 3226:  This is also a valid response for `any` device.
        -: 3227:
        -: 3228:  For a switch-connected cluster of 2 processor SMPs
        -: 3229:.vb
        -: 3230:    level       my_cluster         my_rank
        -: 3231:    0           0                  rank in comm_world
        -: 3232:    1           0 to p/2           0 or 1
        -: 3233:.ve
        -: 3234: where the value each process on the same SMP has the same value for
        -: 3235: 'my_cluster[1]' and a different value for 'my_rank[1]'.
        -: 3236:
        -: 3237:  For two SMPs connected by a network,
        -: 3238:.vb
        -: 3239:    level       my_cluster         my_rank
        -: 3240:    0           0                  rank in comm_world
        -: 3241:    1           0 or 1             0 to # on SMP
        -: 3242:.ve
        -: 3243:
        -: 3244:  An example with more than 2 levels is a collection of clusters, each with
        -: 3245:  SMP nodes.  
        -: 3246:
        -: 3247:  Limitations:
        -: 3248:  This approach does not provide a representations for topologies that
        -: 3249:  are not hierarchical.  For example, a mesh interconnect is a single-level
        -: 3250:  cluster in this view.
        -: 3251:
        -: 3252:  Module: 
        -: 3253:  Topology
        -: 3254:  @*/
        -: 3255:int MPID_Topo_cluster_info( MPID_Comm *comm, 
        -: 3256:			    int *levels, int my_cluster[], int my_rank[] );
        -: 3257:
        -: 3258:/*@
        -: 3259:  MPID_Get_processor_name - Return the name of the current processor
        -: 3260:
        -: 3261:  Input Parameter:
        -: 3262:. namelen - Length of name
        -: 3263:  
        -: 3264:  Output Parameters:
        -: 3265:+ name - A unique specifier for the actual (as opposed to virtual) node. This
        -: 3266:  must be an array of size at least 'MPI_MAX_PROCESSOR_NAME'.
        -: 3267:- resultlen - Length (in characters) of the name.  If this pointer is null,
        -: 3268:   this value is not set.
        -: 3269:
        -: 3270:  Notes:
        -: 3271:  The name returned should identify a particular piece of hardware; 
        -: 3272:  the exact format is implementation defined.  This name may or may not
        -: 3273:  be the same as might be returned by 'gethostname', 'uname', or 'sysinfo'.
        -: 3274:
        -: 3275:  This routine is essentially an MPID version of 'MPI_Get_processor_name' .  
        -: 3276:  It must be part of the device because not all environments support calls
        -: 3277:  to return the processor name.  The additional argument (input name 
        -: 3278:  length) is used to provide better error checking and to ensure that 
        -: 3279:  the input buffer is large enough (rather than assuming that it is
        -: 3280:  'MPI_MAX_PROCESSOR_NAME' long).
        -: 3281:  @*/
        -: 3282:int MPID_Get_processor_name( char *name, int namelen, int *resultlen);
        -: 3283:
        -: 3284:void MPID_Errhandler_free(MPID_Errhandler *errhan_ptr);
        -: 3285:
        -: 3286:/*@
        -: 3287:  MPID_Get_universe_size - Return the number of processes that the current
        -: 3288:  process management environment can handle
        -: 3289:
        -: 3290:  Output Parameters:
        -: 3291:. universe_size - the universe size; MPIR_UNIVERSE_SIZE_NOT_AVAILABLE if the
        -: 3292:  size cannot be determined
        -: 3293:  
        -: 3294:  Return value:
        -: 3295:  A MPI error code.
        -: 3296:@*/
        -: 3297:int MPID_Get_universe_size(int  * universe_size);
        -: 3298:
        -: 3299:#define MPIR_UNIVERSE_SIZE_NOT_SET -1
        -: 3300:#define MPIR_UNIVERSE_SIZE_NOT_AVAILABLE -2
        -: 3301:
        -: 3302:/*
        -: 3303: * FIXME: VCs should not be exposed to the top layer, which implies that these routines should not be exposed either.  Instead,
        -: 3304: * the creation, duplication and destruction of communicator objects should be communicated to the device, allowing the device to
        -: 3305: * manage the underlying connections in a way that is appropriate (and efficient).
        -: 3306: */
        -: 3307:
        -: 3308:/*@
        -: 3309:  MPID_VCRT_Create - Create a virtual connection reference table
        -: 3310:  @*/
        -: 3311:int MPID_VCRT_Create(int size, MPID_VCRT *vcrt_ptr);
        -: 3312:
        -: 3313:/*@
        -: 3314:  MPID_VCRT_Add_ref - Add a reference to a VCRT
        -: 3315:  @*/
        -: 3316:int MPID_VCRT_Add_ref(MPID_VCRT vcrt);
        -: 3317:
        -: 3318:/*@
        -: 3319:  MPID_VCRT_Release - Release a reference to a VCRT
        -: 3320:  
        -: 3321:  Notes:
        -: 3322:  The 'isDisconnect' argument allows this routine to handle the special
        -: 3323:  case of 'MPI_Comm_disconnect', which needs to take special action
        -: 3324:  if all references to a VC are removed.
        -: 3325:  @*/
        -: 3326:int MPID_VCRT_Release(MPID_VCRT vcrt, int isDisconnect);
        -: 3327:
        -: 3328:/*@
        -: 3329:  MPID_VCRT_Get_ptr - 
        -: 3330:  @*/
        -: 3331:int MPID_VCRT_Get_ptr(MPID_VCRT vcrt, MPID_VCR **vc_pptr);
        -: 3332:
        -: 3333:/*@
        -: 3334:  MPID_VCR_Dup - Create a duplicate reference to a virtual connection
        -: 3335:  @*/
        -: 3336:int MPID_VCR_Dup(MPID_VCR orig_vcr, MPID_VCR * new_vcr);
        -: 3337:
        -: 3338:/*@
        -: 3339:   MPID_VCR_Get_lpid - Get the local process id that corresponds to a 
        -: 3340:   virtual connection reference.
        -: 3341:
        -: 3342:   Notes:
        -: 3343:   The local process ids are described elsewhere.  Basically, they are
        -: 3344:   a nonnegative number by which this process can refer to other processes 
        -: 3345:   to which it is connected.  These are local process ids because different
        -: 3346:   processes may use different ids to identify the same target process
        -: 3347:  @*/
        -: 3348:int MPID_VCR_Get_lpid(MPID_VCR vcr, int * lpid_ptr);
        -: 3349:
        -: 3350:/* ------------------------------------------------------------------------- */
        -: 3351:/* Define a macro to allow us to select between statically selected functions
        -: 3352: * and dynamically loaded ones.  If USE_DYNAMIC_LIBRARIES is defined,
        -: 3353: * the macro MPIU_CALL(context,funccall) expands into
        -: 3354: *    MPIU_CALL##context.funccall.
        -: 3355: * For example,
        -: 3356: *    err = MPIU_CALL(MPIDI_CH3,iSend(...))
        -: 3357: * will expand into
        -: 3358: *    err = MPIU_CALL_MPIDI_CH3.iSend(...)
        -: 3359: * If USE_DYNAMIC_LIBS is not selected, then it expands into
        -: 3360: *    err = MPIDI_CH3_iSend(...)
        -: 3361: * 
        -: 3362: * In the case where dynamic libraries are used, a variable named 
        -: 3363: * MPIU_CALL_context must be defined that contains the function pointers;
        -: 3364: * initializing the function pointers must be done before the first use.
        -: 3365: * Typically, this variable will be the single instance of a structure that
        -: 3366: * contains the function pointers.
        -: 3367: */
        -: 3368:/* ------------------------------------------------------------------------- */
        -: 3369:#ifdef USE_DYNAMIC_LIBRARIES
        -: 3370:#define MPIU_CALL(context,funccall) MPIU_CALL_##context.funccall
        -: 3371:#else
        -: 3372:#define MPIU_CALL(context,funccall) context##_##funccall
        -: 3373:#endif
        -: 3374:
        -: 3375:/* Include definitions from the device which require items defined by this 
        -: 3376:   file (mpiimpl.h). */
        -: 3377:#include "mpidpost.h"
        -: 3378:
        -: 3379:/* ------------------------------------------------------------------------- */
        -: 3380:/* FIXME: Also for mpicoll.h, in src/mpi/coll?  */
        -: 3381:/* ------------------------------------------------------------------------- */
        -: 3382:/* thresholds to switch between long and short vector algorithms for
        -: 3383:   collective operations */ 
        -: 3384:/* FIXME: Should there be a way to (a) update/compute these at configure time
        -: 3385:   and (b) provide runtime control?  Should these be MPIR_xxx_DEFAULT 
        -: 3386:   instead? */
        -: 3387:#define MPIR_BCAST_SHORT_MSG          12288
        -: 3388:#define MPIR_BCAST_LONG_MSG           524288
        -: 3389:#define MPIR_BCAST_MIN_PROCS          8
        -: 3390:#define MPIR_ALLTOALL_SHORT_MSG       256
        -: 3391:#define MPIR_ALLTOALL_MEDIUM_MSG      32768
        -: 3392:#define MPIR_ALLTOALL_THROTTLE        4  /* max no. of irecvs/isends posted at a 
        -: 3393:time in some alltoall algorithms. Setting it to 0 causes all irecvs/isends to be 
        -: 3394:posted at once. */
        -: 3395:#define MPIR_REDSCAT_COMMUTATIVE_LONG_MSG 524288
        -: 3396:#define MPIR_REDSCAT_NONCOMMUTATIVE_SHORT_MSG 512
        -: 3397:#define MPIR_ALLGATHER_SHORT_MSG      81920
        -: 3398:#define MPIR_ALLGATHER_LONG_MSG       524288
        -: 3399:#define MPIR_REDUCE_SHORT_MSG         2048
        -: 3400:#define MPIR_ALLREDUCE_SHORT_MSG      2048
        -: 3401:#define MPIR_GATHER_VSMALL_MSG        1024
        -: 3402:#define MPIR_SCATTER_SHORT_MSG        2048  /* for intercommunicator scatter */
        -: 3403:#define MPIR_GATHER_SHORT_MSG         2048  /* for intercommunicator scatter */
        -: 3404:#define MPIR_GATHERV_MIN_PROCS        32
        -: 3405:
        -: 3406:/* For pipelined collectives */
        -: 3407:#define MPIR_ALLGATHERV_PIPELINE_MSGSIZE   32768
        -: 3408:
        -: 3409:/* Tags for point to point operations which implement collective operations */
        -: 3410:#define MPIR_BARRIER_TAG               1
        -: 3411:#define MPIR_BCAST_TAG                 2
        -: 3412:#define MPIR_GATHER_TAG                3
        -: 3413:#define MPIR_GATHERV_TAG               4
        -: 3414:#define MPIR_SCATTER_TAG               5
        -: 3415:#define MPIR_SCATTERV_TAG              6
        -: 3416:#define MPIR_ALLGATHER_TAG             7
        -: 3417:#define MPIR_ALLGATHERV_TAG            8
        -: 3418:#define MPIR_ALLTOALL_TAG              9
        -: 3419:#define MPIR_ALLTOALLV_TAG            10
        -: 3420:#define MPIR_REDUCE_TAG               11
        -: 3421:#define MPIR_USER_REDUCE_TAG          12
        -: 3422:#define MPIR_USER_REDUCEA_TAG         13
        -: 3423:#define MPIR_ALLREDUCE_TAG            14
        -: 3424:#define MPIR_USER_ALLREDUCE_TAG       15
        -: 3425:#define MPIR_USER_ALLREDUCEA_TAG      16
        -: 3426:#define MPIR_REDUCE_SCATTER_TAG       17
        -: 3427:#define MPIR_USER_REDUCE_SCATTER_TAG  18
        -: 3428:#define MPIR_USER_REDUCE_SCATTERA_TAG 19
        -: 3429:#define MPIR_SCAN_TAG                 20
        -: 3430:#define MPIR_USER_SCAN_TAG            21
        -: 3431:#define MPIR_USER_SCANA_TAG           22
        -: 3432:#define MPIR_LOCALCOPY_TAG            23
        -: 3433:#define MPIR_EXSCAN_TAG               24
        -: 3434:#define MPIR_ALLTOALLW_TAG            25
        -: 3435:#define MPIR_TOPO_A_TAG               26
        -: 3436:#define MPIR_TOPO_B_TAG               27
        -: 3437:
        -: 3438:/* These functions are used in the implementation of collective
        -: 3439:   operations. They are wrappers around MPID send/recv functions. They do
        -: 3440:   sends/receives by setting the context offset to
        -: 3441:   MPID_CONTEXT_INTRA_COLL. */
        -: 3442:int MPIC_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
        -: 3443:              MPI_Comm comm);
        -: 3444:int MPIC_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
        -: 3445:              MPI_Comm comm, MPI_Status *status);
        -: 3446:int MPIC_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
        -: 3447:               MPI_Comm comm);
        -: 3448:int MPIC_Sendrecv(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3449:                  int dest, int sendtag, void *recvbuf, int recvcount,
        -: 3450:                  MPI_Datatype recvtype, int source, int recvtag,
        -: 3451:                  MPI_Comm comm, MPI_Status *status);
        -: 3452:int MPIC_Sendrecv_replace(void *buf, int count, MPI_Datatype type,
        -: 3453:                          int dest, int sendtag,
        -: 3454:                          int source, int recvtag,
        -: 3455:                          MPI_Comm comm, MPI_Status *status);
        -: 3456:int MPIR_Localcopy(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3457:                   void *recvbuf, int recvcount, MPI_Datatype recvtype);
        -: 3458:int MPIC_Irecv(void *buf, int count, MPI_Datatype datatype, int
        -: 3459:               source, int tag, MPI_Comm comm, MPI_Request *request);
        -: 3460:int MPIC_Isend(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
        -: 3461:               MPI_Comm comm, MPI_Request *request);
        -: 3462:int MPIC_Wait(MPID_Request * request_ptr);
        -: 3463:
        -: 3464:
        -: 3465:void MPIR_MAXF  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3466:void MPIR_MINF  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3467:void MPIR_SUM  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3468:void MPIR_PROD  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3469:void MPIR_LAND  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3470:void MPIR_BAND  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3471:void MPIR_LOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3472:void MPIR_BOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3473:void MPIR_LXOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3474:void MPIR_BXOR  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3475:void MPIR_MAXLOC  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3476:void MPIR_MINLOC  ( void *, void *, int *, MPI_Datatype * ) ;
        -: 3477:
        -: 3478:int MPIR_MAXF_check_dtype  ( MPI_Datatype ) ;
        -: 3479:int MPIR_MINF_check_dtype ( MPI_Datatype ) ;
        -: 3480:int MPIR_SUM_check_dtype  ( MPI_Datatype ) ;
        -: 3481:int MPIR_PROD_check_dtype  ( MPI_Datatype ) ;
        -: 3482:int MPIR_LAND_check_dtype  ( MPI_Datatype ) ;
        -: 3483:int MPIR_BAND_check_dtype  ( MPI_Datatype ) ;
        -: 3484:int MPIR_LOR_check_dtype  ( MPI_Datatype ) ;
        -: 3485:int MPIR_BOR_check_dtype  ( MPI_Datatype ) ;
        -: 3486:int MPIR_LXOR_check_dtype ( MPI_Datatype ) ;
        -: 3487:int MPIR_BXOR_check_dtype  ( MPI_Datatype ) ;
        -: 3488:int MPIR_MAXLOC_check_dtype  ( MPI_Datatype ) ;
        -: 3489:int MPIR_MINLOC_check_dtype  ( MPI_Datatype ) ;
        -: 3490:
        -: 3491:#define MPIR_PREDEF_OP_COUNT 12
        -: 3492:extern MPI_User_function *MPIR_Op_table[];
        -: 3493:
        -: 3494:typedef int (MPIR_Op_check_dtype_fn) ( MPI_Datatype ); 
        -: 3495:extern MPIR_Op_check_dtype_fn *MPIR_Op_check_dtype_table[];
        -: 3496:
        -: 3497:#if !defined MPIR_MIN
        -: 3498:#define MPIR_MIN(a,b) (((a)>(b))?(b):(a))
        -: 3499:#endif /* MPIR_MIN */
        -: 3500:
        -: 3501:#if !defined MPIR_MAX
        -: 3502:#define MPIR_MAX(a,b) (((b)>(a))?(b):(a))
        -: 3503:#endif /* MPIR_MAX */
        -: 3504:
        -: 3505:int MPIR_Allgather(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3506:                   void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3507:                   MPID_Comm *comm_ptr );
        -: 3508:int MPIR_Allgather_inter(void *sendbuf, int sendcount, MPI_Datatype sendtype,
        -: 3509:                         void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3510:                         MPID_Comm *comm_ptr );
        -: 3511:int MPIR_Allgatherv(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3512:                    void *recvbuf, int *recvcounts, int *displs,   
        -: 3513:                    MPI_Datatype recvtype, MPID_Comm *comm_ptr );
        -: 3514:int MPIR_Allgatherv_inter(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3515:                          void *recvbuf, int *recvcounts, int *displs,   
        -: 3516:                          MPI_Datatype recvtype, MPID_Comm *comm_ptr );
        -: 3517:int MPIR_Allreduce(void *sendbuf, void *recvbuf, int count, 
        -: 3518:                   MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr);
        -: 3519:int MPIR_Allreduce_inter(void *sendbuf, void *recvbuf, int count, 
        -: 3520:                        MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr);
        -: 3521:int MPIR_Alltoall(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3522:                  void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3523:                  MPID_Comm *comm_ptr);
        -: 3524:int MPIR_Alltoall_inter(void *sendbuf, int sendcount, MPI_Datatype sendtype, 
        -: 3525:                        void *recvbuf, int recvcount, MPI_Datatype recvtype, 
        -: 3526:                        MPID_Comm *comm_ptr);
        -: 3527:int MPIR_Alltoallv(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3528:                   MPI_Datatype sendtype, void *recvbuf, int *recvcnts, 
        -: 3529:                   int *rdispls, MPI_Datatype recvtype, MPID_Comm *comm_ptr);
        -: 3530:int MPIR_Alltoallv_inter(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3531:                         MPI_Datatype sendtype, void *recvbuf, int *recvcnts, 
        -: 3532:                         int *rdispls, MPI_Datatype recvtype, 
        -: 3533:                         MPID_Comm *comm_ptr);
        -: 3534:int MPIR_Alltoallw(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3535:                   MPI_Datatype *sendtypes, void *recvbuf, int *recvcnts, 
        -: 3536:                   int *rdispls, MPI_Datatype *recvtypes, MPID_Comm *comm_ptr);
        -: 3537:int MPIR_Alltoallw_inter(void *sendbuf, int *sendcnts, int *sdispls, 
        -: 3538:                         MPI_Datatype *sendtypes, void *recvbuf, 
        -: 3539:                         int *recvcnts, int *rdispls, MPI_Datatype *recvtypes, 
        -: 3540:                         MPID_Comm *comm_ptr);
        -: 3541:int MPIR_Barrier_inter( MPID_Comm *comm_ptr);
        -: 3542:int MPIR_Bcast_inter(void *buffer, int count, MPI_Datatype datatype, 
        -: 3543:		     int root, MPID_Comm *comm_ptr);
        -: 3544:int MPIR_Bcast (void *buffer, int count, MPI_Datatype datatype, int
        -: 3545:                root, MPID_Comm *comm_ptr);
        -: 3546:int MPIR_Bcast_or_coll_fn (void *buffer, int count, MPI_Datatype datatype, int
        -: 3547:                root, MPID_Comm *comm_ptr);
        -: 3548:int MPIR_Exscan(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
        -: 3549:                MPI_Op op, MPID_Comm *comm_ptr );
        -: 3550:int MPIR_Gather (void *sendbuf, int sendcnt, MPI_Datatype sendtype,
        -: 3551:                 void *recvbuf, int recvcnt, MPI_Datatype recvtype,
        -: 3552:                 int root, MPID_Comm *comm_ptr);
        -: 3553:int MPIR_Gather_inter (void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3554:                       void *recvbuf, int recvcnt, MPI_Datatype recvtype, 
        -: 3555:                       int root, MPID_Comm *comm_ptr );
        -: 3556:int MPIR_Gatherv (void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3557:                  void *recvbuf, int *recvcnts, int *displs,
        -: 3558:                  MPI_Datatype recvtype, int root, MPID_Comm *comm_ptr); 
        -: 3559:int MPIR_Reduce_scatter(void *sendbuf, void *recvbuf, int *recvcnts, 
        -: 3560:                        MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr);
        -: 3561:int MPIR_Reduce_scatter_inter(void *sendbuf, void *recvbuf, int *recvcnts, 
        -: 3562:                              MPI_Datatype datatype, MPI_Op op, 
        -: 3563:                              MPID_Comm *comm_ptr);
        -: 3564:int MPIR_Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
        -: 3565:                MPI_Op op, int root, MPID_Comm *comm_ptr );
        -: 3566:int MPIR_Reduce_or_coll_fn(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype,
        -: 3567:                MPI_Op op, int root, MPID_Comm *comm_ptr );
        -: 3568:int MPIR_Reduce_inter (void *sendbuf, void *recvbuf, int count, MPI_Datatype
        -: 3569:                 datatype, MPI_Op op, int root, MPID_Comm *comm_ptr); 
        -: 3570:int MPIR_Scan(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, 
        -: 3571:              MPI_Op op, MPID_Comm *comm_ptr);
        -: 3572:int MPIR_Scatter(void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3573:                 void *recvbuf, int recvcnt, MPI_Datatype recvtype, 
        -: 3574:                 int root, MPID_Comm *comm_ptr );
        -: 3575:int MPIR_Scatter_inter(void *sendbuf, int sendcnt, MPI_Datatype sendtype, 
        -: 3576:                       void *recvbuf, int recvcnt, MPI_Datatype recvtype, 
        -: 3577:                       int root, MPID_Comm *comm_ptr );
        -: 3578:int MPIR_Scatterv (void *sendbuf, int *sendcnts, int *displs,
        -: 3579:                   MPI_Datatype sendtype, void *recvbuf, int recvcnt,
        -: 3580:                   MPI_Datatype recvtype, int root, MPID_Comm
        -: 3581:                   *comm_ptr);
        -: 3582:int MPIR_Barrier( MPID_Comm *comm_ptr );
        -: 3583:
        -: 3584:int MPIR_Setup_intercomm_localcomm( MPID_Comm * );
        -: 3585:
        -: 3586:int MPIR_Comm_create( MPID_Comm ** );
        -: 3587:
        -: 3588:int MPIR_Comm_commit( MPID_Comm * );
        -: 3589:
        -: 3590:int MPIR_Comm_is_node_aware( MPID_Comm * );
        -: 3591:
        -: 3592:int MPIR_Comm_is_node_consecutive( MPID_Comm *);
        -: 3593:
        -: 3594:void MPIR_Free_err_dyncodes( void );
        -: 3595:
        -: 3596:
        -: 3597:/* Collective functions cannot be called from multiple threads. These
        -: 3598:   are stubs used in the collective communication calls to check for
        -: 3599:   user error. Currently they are just being macroed out. */
        -: 3600:#define MPIDU_ERR_CHECK_MULTIPLE_THREADS_ENTER(comm_ptr)
        -: 3601:#define MPIDU_ERR_CHECK_MULTIPLE_THREADS_EXIT(comm_ptr)
        -: 3602:
        -: 3603:/* Miscellaneous */
        -: 3604:void MPIU_SetTimeout( int );
        -: 3605:
        -: 3606:#if defined(HAVE_VSNPRINTF) && defined(NEEDS_VSNPRINTF_DECL) && !defined(vsnprintf)
        -: 3607:int vsnprintf(char *str, size_t size, const char *format, va_list ap);
        -: 3608:# endif
        -: 3609:
        -: 3610:/* Routines for determining local and remote processes */
        -: 3611:
        -: 3612:int MPIU_Find_local_and_external(struct MPID_Comm *comm, int *local_size_p, int *local_rank_p, int **local_ranks_p,
        -: 3613:                                 int *external_size_p, int *external_rank_p, int **external_ranks_p,
        -: 3614:                                 int **intranode_table, int **internode_table_p);
        -: 3615:int MPIU_Get_internode_rank(MPID_Comm *comm_ptr, int r);
        -: 3616:int MPIU_Get_intranode_rank(MPID_Comm *comm_ptr, int r);
        -: 3617:
        -: 3618:#endif /* MPIIMPL_INCLUDED */