-:    0:Source:/home/MPI/testing/mpich2/mpich2/src/mpid/ch3/src/ch3u_recvq.c
        -:    0:Graph:ch3u_recvq.gcno
        -:    0:Data:ch3u_recvq.gcda
        -:    0:Runs:3459
        -:    0:Programs:899
        -:    1:/* -*- Mode: C; c-basic-offset:4 ; -*- */
        -:    2:/*
        -:    3: *  (C) 2001 by Argonne National Laboratory.
        -:    4: *      See COPYRIGHT in top-level directory.
        -:    5: */
        -:    6:
        -:    7:#include "mpidimpl.h"
        -:    8:
        -:    9:/* MPIDI_POSTED_RECV_ENQUEUE_HOOK(req): Notifies channel that req has
        -:   10:   been enqueued on the posted recv queue.  Returns void. */
        -:   11:#ifndef MPIDI_POSTED_RECV_ENQUEUE_HOOK
        -:   12:#define MPIDI_POSTED_RECV_ENQUEUE_HOOK(req) do{}while(0)
        -:   13:#endif
        -:   14:/* MPIDI_POSTED_RECV_DEQUEUE_HOOK(req): Notifies channel that req has
        -:   15:   been dequeued from the posted recv queue.  Returns non-zero if the
        -:   16:   channel has already matched the request; 0 otherwise.  This happens
        -:   17:   when the channel supports shared-memory and network communication
        -:   18:   with a network capable of matching, and the same request is matched
        -:   19:   by the network and, e.g., shared-memory.  When that happens the
        -:   20:   dequeue functions below should, either search for the next matching
        -:   21:   request, or report that no request was found. */
        -:   22:#ifndef MPIDI_POSTED_RECV_DEQUEUE_HOOK
        -:   23:#define MPIDI_POSTED_RECV_DEQUEUE_HOOK(req) 0
        -:   24:#endif
        -:   25:
        -:   26:/* FIXME: 
        -:   27: * Recvq_lock/unlock removed because it is not needed for the SINGLE_CS
        -:   28: * approach and we might want a different, non-lock-based approach in 
        -:   29: * a finer-grained thread-sync version.  For example, 
        -:   30: * some routines can be implemented in a lock-free
        -:   31: * fashion (because the user is required to guarantee non-conflicting 
        -:   32: * accesses, such as doing a probe and a receive that matches in different 
        -:   33: * threads).  
        -:   34: *
        -:   35: * There are a lot of routines here.  Do we really need them all?
        -:   36: * 
        -:   37: * The search criteria can be implemented in a single 64 bit compare on
        -:   38: * systems with efficient 64-bit operations.  The rank and contextid can also
        -:   39: * in many cases be combined into a single 32-bit word for the comparison
        -:   40: * (in which case the message info should be stored in the queue in a 
        -:   41: * naturally aligned, 64-bit word.
        -:   42: * 
        -:   43: */
        -:   44:
        -:   45:static MPID_Request * recvq_posted_head = 0;
        -:   46:static MPID_Request * recvq_posted_tail = 0;
        -:   47:static MPID_Request * recvq_unexpected_head = 0;
        -:   48:static MPID_Request * recvq_unexpected_tail = 0;
        -:   49:
        -:   50:/* Export the location of the queue heads if debugger support is enabled.
        -:   51: * This allows the queue code to rely on the local variables for the
        -:   52: * queue heads while also exporting those variables to the debugger.
        -:   53: * See src/mpi/debugger/dll_mpich2.c for how this is used to 
        -:   54: * access the message queues.
        -:   55: */
        -:   56:#ifdef HAVE_DEBUGGER_SUPPORT
        -:   57:MPID_Request ** const MPID_Recvq_posted_head_ptr     = &recvq_posted_head;
        -:   58:MPID_Request ** const MPID_Recvq_unexpected_head_ptr = &recvq_unexpected_head;
        -:   59:#endif
        -:   60:
        -:   61:/* If the MPIDI_Message_match structure fits into a pointer size, we
        -:   62: * can directly work on it */
        -:   63:/* MATCH_WITH_NO_MASK compares the match values without masking
        -:   64: * them. This is useful for the case where there are no ANY_TAG or
        -:   65: * ANY_SOURCE wild cards. */
        -:   66:#define MATCH_WITH_NO_MASK(match1, match2)                              \
        -:   67:    ((sizeof(MPIDI_Message_match) == SIZEOF_VOID_P) ? ((match1).whole == (match2).whole) : \
        -:   68:     (((match1).parts.rank == (match2).parts.rank) &&                   \
        -:   69:      ((match1).parts.tag == (match2).parts.tag) &&                     \
        -:   70:      ((match1).parts.context_id == (match2).parts.context_id)))
        -:   71:
        -:   72:/* MATCH_WITH_LEFT_MASK compares the match values after masking only
        -:   73: * the left field. This is useful for the case where the right match
        -:   74: * is a part of the unexpected queue and has no ANY_TAG or ANY_SOURCE
        -:   75: * wild cards, but the left match might have them. */
        -:   76:#define MATCH_WITH_LEFT_MASK(match1, match2, mask)                      \
        -:   77:    ((sizeof(MPIDI_Message_match) == SIZEOF_VOID_P) ?                   \
        -:   78:     (((match1).whole & (mask).whole) == (match2).whole) :              \
        -:   79:     ((((match1).parts.rank & (mask).parts.rank) == (match2).parts.rank) && \
        -:   80:      (((match1).parts.tag & (mask).parts.tag) == (match2).parts.tag) && \
        -:   81:      ((match1).parts.context_id == (match2).parts.context_id)))
        -:   82:
        -:   83:/* This is the most general case where both matches have to be
        -:   84: * masked. Both matches are masked with the same value. There doesn't
        -:   85: * seem to be a need for two different masks at this time. */
        -:   86:#define MATCH_WITH_LEFT_RIGHT_MASK(match1, match2, mask)                \
        -:   87:    ((sizeof(MPIDI_Message_match) == SIZEOF_VOID_P) ?                   \
        -:   88:     (((match1).whole & (mask).whole) == ((match2).whole & (mask).whole)) : \
        -:   89:     ((((match1).parts.rank & (mask).parts.rank) == ((match2).parts.rank & (mask).parts.rank)) && \
        -:   90:      (((match1).parts.tag & (mask).parts.tag) == ((match2).parts.tag & (mask).parts.tag)) && \
        -:   91:      ((match1).parts.context_id == (match2).parts.context_id)))
        -:   92:
        -:   93:
        -:   94:/* FIXME: If this routine is only used by probe/iprobe, then we don't need
        -:   95:   to set the cancelled field in status (only set for nonblocking requests) */
        -:   96:/*
        -:   97: * MPIDI_CH3U_Recvq_FU()
        -:   98: *
        -:   99: * Search for a matching request in the unexpected receive queue.  Return 
        -:  100: * true if one is found, false otherwise.  If the status arguement is
        -:  101: * not MPI_STATUS_IGNORE, return information about the request in that
        -:  102: * parameter.  This routine is used by mpid_probe and mpid_iprobe.
        -:  103: *
        -:  104: * Multithread - As this is a read-only routine, it need not
        -:  105: * require an external critical section (careful organization of the
        -:  106: * queue updates would not even require a critical section within this
        -:  107: * routine).  However, this routine is used both from within the progress
        -:  108: * engine and from without it.  To make that work with the current
        -:  109: * design for MSGQUEUE and the brief-global mode, the critical section 
        -:  110: * is *outside* of this routine.
        -:  111: *
        -:  112: * This routine is used only in mpid_iprobe and mpid_probe
        -:  113: *
        -:  114: */
        -:  115:#undef FUNCNAME
        -:  116:#define FUNCNAME MPIDI_CH3U_Recvq_FU
        -:  117:#undef FCNAME
        -:  118:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  119:int MPIDI_CH3U_Recvq_FU(int source, int tag, int context_id, MPI_Status *s)
 15755693:  120:{
        -:  121:    MPID_Request * rreq;
 15755693:  122:    int found = 0;
        -:  123:    MPIDI_Message_match match, mask;
        -:  124:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
        -:  125:
        -:  126:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
        -:  127:
 15755693:  128:    rreq = recvq_unexpected_head;
        -:  129:
 15755693:  130:    match.parts.context_id = context_id;
 15755693:  131:    match.parts.tag = tag;
 15755693:  132:    match.parts.rank = source;
        -:  133:
 15755693:  134:    if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
 30843628:  135:	while (rreq != NULL) {
 15421717:  136:	    if (MATCH_WITH_NO_MASK(rreq->dev.match, match))
       16:  137:		break;
 15421701:  138:	    rreq = rreq->dev.next;
        -:  139:	}
        -:  140:    }
        -:  141:    else {
   333766:  142:	mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
   333766:  143:	if (tag == MPI_ANY_TAG)
   130069:  144:	    match.parts.tag = mask.parts.tag = 0;
   333766:  145:	if (source == MPI_ANY_SOURCE)
   333754:  146:	    match.parts.rank = mask.parts.rank = 0;
        -:  147:
   492902:  148:	while (rreq != NULL) {
   160612:  149:	    if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask))
     1476:  150:		break;
   159136:  151:	    rreq = rreq->dev.next;
        -:  152:	}
        -:  153:    }
        -:  154:
        -:  155:    /* Save the information about the request before releasing the 
        -:  156:       queue */
 15755693:  157:    if (rreq) {
     1492:  158:	if (s != MPI_STATUS_IGNORE) {
        -:  159:	    /* Avoid setting "extra" fields like MPI_ERROR */
     1478:  160:	    s->MPI_SOURCE = rreq->status.MPI_SOURCE;
     1478:  161:	    s->MPI_TAG    = rreq->status.MPI_TAG;
     1478:  162:	    s->count      = rreq->status.count;
     1478:  163:	    s->cancelled  = rreq->status.cancelled;
        -:  164:	}
     1492:  165:	found = 1;
        -:  166:    }
        -:  167:
        -:  168:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
 15755693:  169:    return found;
        -:  170:}
        -:  171:
        -:  172:/*
        -:  173: * MPIDI_CH3U_Recvq_FDU()
        -:  174: *
        -:  175: * Find a request in the unexpected queue and dequeue it; otherwise return NULL.
        -:  176: *
        -:  177: * Multithread - This routine must be atomic (since it dequeues a
        -:  178: * request).  However, once the request is dequeued, no other thread can
        -:  179: * see it, so this routine provides its own atomicity.
        -:  180: *
        -:  181: * This routine is used only in the case of send_cancel.  However, it is used both
        -:  182: * within mpid_send_cancel and within a packet handler.
        -:  183: */
        -:  184:#undef FUNCNAME
        -:  185:#define FUNCNAME MPIDI_CH3U_Recvq_FDU
        -:  186:#undef FCNAME
        -:  187:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  188:MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id, 
        -:  189:				    MPIDI_Message_match * match)
       43:  190:{
        -:  191:    MPID_Request * rreq;
        -:  192:    MPID_Request * prev_rreq;
        -:  193:    MPID_Request * cur_rreq;
        -:  194:    MPID_Request * matching_prev_rreq;
        -:  195:    MPID_Request * matching_cur_rreq;
        -:  196:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
        -:  197:
        -:  198:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
        -:  199:
       43:  200:    matching_prev_rreq = NULL;
       43:  201:    matching_cur_rreq = NULL;
       43:  202:    prev_rreq = NULL;
        -:  203:
        -:  204:    /* Note that since this routine is used only in the case of send_cancel,
        -:  205:       there can be only one match if at all. */
        -:  206:    /* FIXME: Why doesn't this exit after it finds the first match? */
       43:  207:    cur_rreq = recvq_unexpected_head;
      121:  208:    while (cur_rreq != NULL) {
       35:  209:	if (cur_rreq->dev.sender_req_id == sreq_id &&
        -:  210:	    (MATCH_WITH_NO_MASK(cur_rreq->dev.match, *match))) {
       35:  211:	    matching_prev_rreq = prev_rreq;
       35:  212:	    matching_cur_rreq = cur_rreq;
        -:  213:	}
       35:  214:	prev_rreq = cur_rreq;
       35:  215:	cur_rreq = cur_rreq->dev.next;
        -:  216:    }
        -:  217:
       43:  218:    if (matching_cur_rreq != NULL) {
       35:  219:	if (matching_prev_rreq != NULL) {
    #####:  220:	    matching_prev_rreq->dev.next = matching_cur_rreq->dev.next;
        -:  221:	}
        -:  222:	else {
       35:  223:	    recvq_unexpected_head = matching_cur_rreq->dev.next;
        -:  224:	}
        -:  225:	
       35:  226:	if (matching_cur_rreq->dev.next == NULL) {
       35:  227:	    recvq_unexpected_tail = matching_prev_rreq;
        -:  228:	}
        -:  229:
       35:  230:	rreq = matching_cur_rreq;
        -:  231:    }
        -:  232:    else {
        8:  233:	rreq = NULL;
        -:  234:    }
        -:  235:
        -:  236:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
       43:  237:    return rreq;
        -:  238:}
        -:  239:
        -:  240:
        -:  241:/*
        -:  242: * MPIDI_CH3U_Recvq_FDU_or_AEP()
        -:  243: *
        -:  244: * Atomically find a request in the unexpected queue and dequeue it, or 
        -:  245: * allocate a new request and enqueue it in the posted queue
        -:  246: *
        -:  247: * Multithread - This routine must be called from within a MSGQUEUE 
        -:  248: * critical section.  If a request is allocated, it must not release
        -:  249: * the MSGQUEUE until the request is completely valid, as another thread
        -:  250: * may then find it and dequeue it.
        -:  251: *
        -:  252: * This routine is used in mpid_irecv and mpid_recv.
        -:  253: *
        -:  254: */
        -:  255:#undef FUNCNAME
        -:  256:#define FUNCNAME MPIDI_CH3U_Recvq_FDU_or_AEP
        -:  257:#undef FCNAME
        -:  258:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  259:MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag, 
        -:  260:                                           int context_id, MPID_Comm *comm, void *user_buf,
        -:  261:                                           int user_count, MPI_Datatype datatype, int * foundp)
 12853863:  262:{
        -:  263:    int found;
        -:  264:    MPID_Request *rreq, *prev_rreq;
        -:  265:    MPIDI_Message_match match;
        -:  266:    MPIDI_Message_match mask;
        -:  267:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
        -:  268:
        -:  269:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
        -:  270:
        -:  271:    /* Optimize this loop for an empty unexpected receive queue */
 12853863:  272:    rreq = recvq_unexpected_head;
 12853863:  273:    if (rreq) {
  1417938:  274:	prev_rreq = NULL;
        -:  275:
  1417938:  276:	match.parts.context_id = context_id;
  1417938:  277:	match.parts.tag = tag;
  1417938:  278:	match.parts.rank = source;
        -:  279:
  1417938:  280:	if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
        -:  281:	    do {
  1589382:  282:		if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
  1155667:  283:		    if (prev_rreq != NULL) {
    11518:  284:			prev_rreq->dev.next = rreq->dev.next;
        -:  285:		    }
        -:  286:		    else {
  1144149:  287:			recvq_unexpected_head = rreq->dev.next;
        -:  288:		    }
        -:  289:
  1155667:  290:		    if (rreq->dev.next == NULL) {
  1028419:  291:			recvq_unexpected_tail = prev_rreq;
        -:  292:		    }
        -:  293:
  1155667:  294:		    rreq->comm = comm;
  1155667:  295:		    MPIR_Comm_add_ref(comm);
  1155667:  296:		    rreq->dev.user_buf = user_buf;
  1155667:  297:		    rreq->dev.user_count = user_count;
  1155667:  298:		    rreq->dev.datatype = datatype;
  1155667:  299:		    found = TRUE;
  1155667:  300:		    goto lock_exit;
        -:  301:		}
   433715:  302:		prev_rreq = rreq;
   433715:  303:		rreq      = rreq->dev.next;
   433715:  304:	    } while (rreq);
        -:  305:	}
        -:  306:	else {
    12125:  307:	    mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
    12125:  308:	    if (tag == MPI_ANY_TAG)
     6314:  309:		match.parts.tag = mask.parts.tag = 0;
    12125:  310:	    if (source == MPI_ANY_SOURCE)
     5816:  311:		match.parts.rank = mask.parts.rank = 0;
        -:  312:
        -:  313:	    do {
    14249:  314:		if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
    11539:  315:		    if (prev_rreq != NULL) {
     1611:  316:			prev_rreq->dev.next = rreq->dev.next;
        -:  317:		    }
        -:  318:		    else {
     9928:  319:			recvq_unexpected_head = rreq->dev.next;
        -:  320:		    }
    11539:  321:		    if (rreq->dev.next == NULL) {
     9607:  322:			recvq_unexpected_tail = prev_rreq;
        -:  323:		    }
    11539:  324:		    rreq->comm                 = comm;
    11539:  325:		    MPIR_Comm_add_ref(comm);
    11539:  326:		    rreq->dev.user_buf         = user_buf;
    11539:  327:		    rreq->dev.user_count       = user_count;
    11539:  328:		    rreq->dev.datatype         = datatype;
    11539:  329:		    found = TRUE;
    11539:  330:		    goto lock_exit;
        -:  331:		}
     2710:  332:		prev_rreq = rreq;
     2710:  333:		rreq = rreq->dev.next;
     2710:  334:	    } while (rreq);
        -:  335:	}
        -:  336:    }
        -:  337:    
        -:  338:    /* A matching request was not found in the unexpected queue, so we 
        -:  339:       need to allocate a new request and add it to the posted queue */
        -:  340:    {
 11686657:  341:	int mpi_errno=0;
 11686657:  342:	MPIDI_Request_create_rreq( rreq, mpi_errno, 
        -:  343:				   found = FALSE;goto lock_exit );
 11686657:  344:	rreq->dev.match.parts.tag	   = tag;
 11686657:  345:	rreq->dev.match.parts.rank	   = source;
 11686657:  346:	rreq->dev.match.parts.context_id   = context_id;
        -:  347:
        -:  348:	/* Added a mask for faster search on 64-bit capable
        -:  349:	 * platforms */
 11686657:  350:	rreq->dev.mask.parts.context_id = ~0;
 11686657:  351:	if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
     8341:  352:	    rreq->dev.mask.parts.rank = 0;
        -:  353:	else
 11678316:  354:	    rreq->dev.mask.parts.rank = ~0;
 11686657:  355:	if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
    17143:  356:	    rreq->dev.mask.parts.tag = 0;
        -:  357:	else
 11669514:  358:	    rreq->dev.mask.parts.tag = ~0;
        -:  359:
 11686657:  360:        rreq->comm                 = comm;
 11686657:  361:        MPIR_Comm_add_ref(comm);
 11686657:  362:        rreq->dev.user_buf         = user_buf;
 11686657:  363:        rreq->dev.user_count       = user_count;
 11686657:  364:        rreq->dev.datatype         = datatype;
 11686657:  365:	rreq->dev.next		   = NULL;
 11686657:  366:	if (recvq_posted_tail != NULL) {
  1308275:  367:	    recvq_posted_tail->dev.next = rreq;
        -:  368:	}
        -:  369:	else {
 10378382:  370:	    recvq_posted_head = rreq;
        -:  371:	}
 11686657:  372:	recvq_posted_tail = rreq;
        -:  373:	MPIDI_POSTED_RECV_ENQUEUE_HOOK(rreq);
        -:  374:    }
        -:  375:    
 11686657:  376:    found = FALSE;
        -:  377:
 12853863:  378:  lock_exit:
        -:  379:
 12853863:  380:    *foundp = found;
        -:  381:    
        -:  382:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
 12853863:  383:    return rreq;
        -:  384:}
        -:  385:
        -:  386:
        -:  387:/*
        -:  388: * MPIDI_CH3U_Recvq_DP()
        -:  389: *
        -:  390: * Given an existing request, dequeue that request from the posted queue, or 
        -:  391: * return NULL if the request was not in the posted queued
        -:  392: *
        -:  393: * Multithread - This routine is atomic
        -:  394: */
        -:  395:#undef FUNCNAME
        -:  396:#define FUNCNAME MPIDI_CH3U_Recvq_DP
        -:  397:#undef FCNAME
        -:  398:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  399:int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
     2079:  400:{
        -:  401:    int found;
        -:  402:    MPID_Request * cur_rreq;
        -:  403:    MPID_Request * prev_rreq;
        -:  404:    int dequeue_failed;
        -:  405:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
        -:  406:
        -:  407:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
        -:  408:
     2079:  409:    found = FALSE;
     2079:  410:    prev_rreq = NULL;
        -:  411:    
        -:  412:    MPIU_THREAD_CS_ENTER(MSGQUEUE,);
     2079:  413:    cur_rreq = recvq_posted_head;
     4163:  414:    while (cur_rreq != NULL) {
     2065:  415:	if (cur_rreq == rreq) {
     2060:  416:	    if (prev_rreq != NULL) {
        3:  417:		prev_rreq->dev.next = cur_rreq->dev.next;
        -:  418:	    }
        -:  419:	    else {
     2057:  420:		recvq_posted_head = cur_rreq->dev.next;
        -:  421:	    }
     2060:  422:	    if (cur_rreq->dev.next == NULL) {
       10:  423:		recvq_posted_tail = prev_rreq;
        -:  424:	    }
        -:  425:            /* Notify channel that rreq has been dequeued and check if
        -:  426:               it has already matched rreq, fail if so */
     2060:  427:	    dequeue_failed = MPIDI_POSTED_RECV_DEQUEUE_HOOK(rreq);
     2060:  428:            if (!dequeue_failed)
     2060:  429:                found = TRUE;
        -:  430:	    break;
        -:  431:	}
        -:  432:	
        5:  433:	prev_rreq = cur_rreq;
        5:  434:	cur_rreq = cur_rreq->dev.next;
        -:  435:    }
        -:  436:
        -:  437:    MPIU_THREAD_CS_EXIT(MSGQUEUE,);
        -:  438:
        -:  439:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
     2079:  440:    return found;
        -:  441:}
        -:  442:
        -:  443:/*
        -:  444: * MPIDI_CH3U_Recvq_FDP_or_AEU()
        -:  445: *
        -:  446: * Locate a request in the posted queue and dequeue it, or allocate a new 
        -:  447: * request and enqueue it in the unexpected queue
        -:  448: *
        -:  449: * Multithread - This routine must be called from within a MSGQUEUE 
        -:  450: * critical section.  If a request is allocated, it must not release
        -:  451: * the MSGQUEUE until the request is completely valid, as another thread
        -:  452: * may then find it and dequeue it.
        -:  453: *
        -:  454: * This routine is used in ch3u_eager, ch3u_eagersync, ch3u_handle_recv_pkt,
        -:  455: * ch3u_rndv, and mpidi_isend_self.  Routines within the progress engine
        -:  456: * will need to be careful to avoid nested critical sections.  
        -:  457: *
        -:  458: * FIXME: Currently, the routines called from within the progress engine
        -:  459: * do not use the MSGQUEUE CS, because in the brief-global mode, that
        -:  460: * simply uses the global_mutex .  
        -:  461: */
        -:  462:#undef FUNCNAME
        -:  463:#define FUNCNAME MPIDI_CH3U_Recvq_FDP_or_AEU
        -:  464:#undef FCNAME
        -:  465:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  466:MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match, 
        -:  467:					   int * foundp)
 12851838:  468:{
        -:  469:    int found;
        -:  470:    MPID_Request * rreq;
        -:  471:    MPID_Request * prev_rreq;
        -:  472:    int channel_matched;
        -:  473:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
        -:  474:
        -:  475:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
        -:  476:
 12851838:  477: top_loop:
 12851838:  478:    prev_rreq = NULL;
        -:  479:
 12851838:  480:    rreq = recvq_posted_head;
        -:  481:
 26999516:  482:    while (rreq != NULL) {
 12980437:  483:	if (MATCH_WITH_LEFT_RIGHT_MASK(rreq->dev.match, *match, rreq->dev.mask)) {
 11684597:  484:	    if (prev_rreq != NULL) {
   108452:  485:		prev_rreq->dev.next = rreq->dev.next;
        -:  486:	    }
        -:  487:	    else {
 11576145:  488:		recvq_posted_head = rreq->dev.next;
        -:  489:	    }
 11684597:  490:	    if (rreq->dev.next == NULL) {
 10436500:  491:		recvq_posted_tail = prev_rreq;
        -:  492:	    }
        -:  493:
        -:  494:            /* give channel a chance to match the request, try again if so */
 11684597:  495:	    channel_matched = MPIDI_POSTED_RECV_DEQUEUE_HOOK(rreq);
 11684597:  496:            if (channel_matched)
    #####:  497:                goto top_loop;
        -:  498:            
 11684597:  499:	    found = TRUE;                
 11684597:  500:	    goto lock_exit;
        -:  501:	}
  1295840:  502:	prev_rreq = rreq;
  1295840:  503:	rreq = rreq->dev.next;
        -:  504:    }
        -:  505:
        -:  506:    /* A matching request was not found in the posted queue, so we 
        -:  507:       need to allocate a new request and add it to the unexpected queue */
        -:  508:    {
  1167241:  509:	int mpi_errno=0;
  1167241:  510:	MPIDI_Request_create_rreq( rreq, mpi_errno, 
        -:  511:				   found=FALSE;goto lock_exit );
  1167241:  512:        rreq->dev.recv_pending_count = 1;
  1167241:  513:	rreq->dev.match	= *match;
  1167241:  514:	rreq->dev.next	= NULL;
  1167241:  515:	if (recvq_unexpected_tail != NULL) {
   138495:  516:	    recvq_unexpected_tail->dev.next = rreq;
        -:  517:	}
        -:  518:	else {
  1028746:  519:	    recvq_unexpected_head = rreq;
        -:  520:	}
  1167241:  521:	recvq_unexpected_tail = rreq;
        -:  522:    }
        -:  523:    
  1167241:  524:    found = FALSE;
        -:  525:
 12851838:  526:  lock_exit:
        -:  527:
 12851838:  528:    *foundp = found;
        -:  529:
        -:  530:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
 12851838:  531:    return rreq;
        -:  532:}
        -:  533:
        -:  534:/* --BEGIN ERROR HANDLING-- */
        -:  535:/* pretty prints tag, returns out for calling convenience */
        -:  536:static char *tag_val_to_str(int tag, char *out, int max)
    #####:  537:{
    #####:  538:    if (tag == MPI_ANY_TAG) {
    #####:  539:        MPIU_Strncpy(out, "MPI_ANY_TAG", max);
        -:  540:    }
        -:  541:    else {
    #####:  542:        MPIU_Snprintf(out, max, "%d", tag);
        -:  543:    }
    #####:  544:    return out;
        -:  545:}
        -:  546:
        -:  547:/* pretty prints rank, returns out for calling convenience */
        -:  548:static char *rank_val_to_str(int rank, char *out, int max)
    #####:  549:{
    #####:  550:    if (rank == MPI_ANY_SOURCE) {
    #####:  551:        MPIU_Strncpy(out, "MPI_ANY_SOURCE", max);
        -:  552:    }
        -:  553:    else {
    #####:  554:        MPIU_Snprintf(out, max, "%d", rank);
        -:  555:    }
    #####:  556:    return out;
        -:  557:}
        -:  558:
        -:  559:/* satisfy the compiler */
        -:  560:void MPIDI_CH3U_Dbg_print_recvq(FILE *stream);
        -:  561:
        -:  562:/* This function can be called by a debugger to dump the recvq state to the
        -:  563: * given stream. */
        -:  564:#undef FUNCNAME
        -:  565:#define FUNCNAME MPIDI_CH3U_Dbg_print_recvq
        -:  566:#undef FCNAME
        -:  567:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  568:void MPIDI_CH3U_Dbg_print_recvq(FILE *stream)
    #####:  569:{
        -:  570:    MPID_Request * rreq;
        -:  571:    int i;
        -:  572:    char tag_buf[128];
        -:  573:    char rank_buf[128];
        -:  574:
    #####:  575:    fprintf(stream, "========================================\n");
    #####:  576:    fprintf(stream, "MPI_COMM_WORLD  ctx=%#x rank=%d\n", MPIR_Process.comm_world->context_id, MPIR_Process.comm_world->rank);
    #####:  577:    fprintf(stream, "MPI_COMM_SELF   ctx=%#x\n", MPIR_Process.comm_self->context_id);
    #####:  578:    if (MPIR_Process.comm_parent) {
    #####:  579:        fprintf(stream, "MPI_COMM_PARENT ctx=%#x recvctx=%#x\n",
        -:  580:                MPIR_Process.comm_self->context_id,
        -:  581:                MPIR_Process.comm_parent->recvcontext_id);
        -:  582:    }
        -:  583:    else {
    #####:  584:        fprintf(stream, "MPI_COMM_PARENT (NULL)\n");
        -:  585:    }
        -:  586:
    #####:  587:    fprintf(stream, "CH3 Posted RecvQ:\n");
    #####:  588:    rreq = recvq_posted_head;
    #####:  589:    i = 0;
    #####:  590:    while (rreq != NULL) {
    #####:  591:        fprintf(stream, "..[%d] rreq=%p ctx=%#x rank=%s tag=%s\n", i, rreq,
        -:  592:                        rreq->dev.match.parts.context_id,
        -:  593:                        rank_val_to_str(rreq->dev.match.parts.rank, rank_buf, sizeof(rank_buf)),
        -:  594:                        tag_val_to_str(rreq->dev.match.parts.tag, tag_buf, sizeof(tag_buf)));
    #####:  595:        ++i;
    #####:  596:        rreq = rreq->dev.next;
        -:  597:    }
        -:  598:
    #####:  599:    fprintf(stream, "CH3 Unexpected RecvQ:\n");
    #####:  600:    rreq = recvq_unexpected_head;
    #####:  601:    i = 0;
    #####:  602:    while (rreq != NULL) {
    #####:  603:        fprintf(stream, "..[%d] rreq=%p ctx=%#x rank=%s tag=%s\n", i, rreq,
        -:  604:                        rreq->dev.match.parts.context_id,
        -:  605:                        rank_val_to_str(rreq->dev.match.parts.rank, rank_buf, sizeof(rank_buf)),
        -:  606:                        tag_val_to_str(rreq->dev.match.parts.tag, tag_buf, sizeof(tag_buf)));
    #####:  607:        fprintf(stream, "..    status.src=%s status.tag=%s\n",
        -:  608:                        rank_val_to_str(rreq->status.MPI_SOURCE, rank_buf, sizeof(rank_buf)),
        -:  609:                        tag_val_to_str(rreq->status.MPI_TAG, tag_buf, sizeof(tag_buf)));
    #####:  610:        ++i;
    #####:  611:        rreq = rreq->dev.next;
        -:  612:    }
    #####:  613:    fprintf(stream, "========================================\n");
    #####:  614:}
        -:  615:/* --END ERROR HANDLING-- */
        -:  616:
        -:  617:/* returns the number of elements in the unexpected queue */
        -:  618:#undef FUNCNAME
        -:  619:#define FUNCNAME MPIDI_CH3U_Recvq_count_unexp
        -:  620:#undef FCNAME
        -:  621:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  622:int MPIDI_CH3U_Recvq_count_unexp(void)
    #####:  623:{
    #####:  624:    int count = 0;
    #####:  625:    MPID_Request *req = recvq_unexpected_head;
        -:  626:
    #####:  627:    while (req)
        -:  628:    {
    #####:  629:        ++count;
    #####:  630:        req = req->dev.next;
        -:  631:    }
        -:  632:
    #####:  633:    return count;
        -:  634:}