-:    0:Source:/home/MPI/testing/mpich2/mpich2/src/mpid/ch3/src/ch3u_handle_recv_req.c
        -:    0:Graph:ch3u_handle_recv_req.gcno
        -:    0:Data:ch3u_handle_recv_req.gcda
        -:    0:Runs:3459
        -:    0:Programs:899
        -:    1:/* -*- Mode: C; c-basic-offset:4 ; -*- */
        -:    2:/*
        -:    3: *  (C) 2001 by Argonne National Laboratory.
        -:    4: *      See COPYRIGHT in top-level directory.
        -:    5: */
        -:    6:
        -:    7:#include "mpidimpl.h"
        -:    8:#include "mpidrma.h"
        -:    9:
        -:   10:static int create_derived_datatype(MPID_Request * rreq, MPID_Datatype ** dtp);
        -:   11:static int do_accumulate_op(MPID_Request * rreq);
        -:   12:static int do_simple_accumulate(MPIDI_PT_single_op *single_op);
        -:   13:static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue);
        -:   14:
        -:   15:#undef FUNCNAME
        -:   16:#define FUNCNAME MPIDI_CH3U_Handle_recv_req
        -:   17:#undef FCNAME
        -:   18:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:   19:int MPIDI_CH3U_Handle_recv_req(MPIDI_VC_t * vc, MPID_Request * rreq, 
        -:   20:			       int * complete)
    #####:   21:{
        -:   22:    static int in_routine = FALSE;
    #####:   23:    int mpi_errno = MPI_SUCCESS;
        -:   24:    int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
        -:   25:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
        -:   26:
        -:   27:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
        -:   28:
    #####:   29:    MPIU_Assert(in_routine == FALSE);
    #####:   30:    in_routine = TRUE;
        -:   31:
    #####:   32:    reqFn = rreq->dev.OnDataAvail;
    #####:   33:    if (!reqFn) {
    #####:   34:	MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_RECV);
    #####:   35:	MPIDI_CH3U_Request_complete(rreq);
    #####:   36:	*complete = TRUE;
        -:   37:    }
        -:   38:    else {
    #####:   39:        mpi_errno = reqFn( vc, rreq, complete );
        -:   40:    }
        -:   41:
    #####:   42:    in_routine = FALSE;
        -:   43:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
    #####:   44:    return mpi_errno;
        -:   45:}
        -:   46:
        -:   47:/* ----------------------------------------------------------------------- */
        -:   48:/* Here are the functions that implement the actions that are taken when 
        -:   49: * data is available for a receive request (or other completion operations)
        -:   50: * These include "receive" requests that are part of the RMA implementation.
        -:   51: *
        -:   52: * The convention for the names of routines that are called when data is
        -:   53: * available is
        -:   54: *    MPIDI_CH3_ReqHandler_<type>( MPIDI_VC_t *, MPID_Request *, int * )
        -:   55: * as in 
        -:   56: *    MPIDI_CH3_ReqHandler_...
        -:   57: *
        -:   58: * ToDo: 
        -:   59: *    We need a way for each of these functions to describe what they are,
        -:   60: *    so that given a pointer to one of these functions, we can retrieve
        -:   61: *    a description of the routine.  We may want to use a static string 
        -:   62: *    and require the user to maintain thread-safety, at least while
        -:   63: *    accessing the string.
        -:   64: */
        -:   65:/* ----------------------------------------------------------------------- */
        -:   66:int MPIDI_CH3_ReqHandler_RecvComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), 
        -:   67:				       MPID_Request *rreq, 
        -:   68:				       int *complete )
    #####:   69:{
        -:   70:    /* mark data transfer as complete and decrement CC */
    #####:   71:    MPIDI_CH3U_Request_complete(rreq);
    #####:   72:    *complete = TRUE;
    #####:   73:    return MPI_SUCCESS;
        -:   74:}
        -:   75:
        -:   76:#undef FUNCNAME
        -:   77:#define FUNCNAME MPIDI_CH3_ReqHandler_PutAccumRespComplete
        -:   78:#undef FCNAME
        -:   79:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:   80:int MPIDI_CH3_ReqHandler_PutAccumRespComplete( MPIDI_VC_t *vc, 
        -:   81:					       MPID_Request *rreq, 
        -:   82:					       int *complete )
     7661:   83:{
     7661:   84:    int mpi_errno = MPI_SUCCESS;
        -:   85:    MPID_Win *win_ptr;
        -:   86:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_PUTACCUMRESPCOMPLETE);
        -:   87:    
        -:   88:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_PUTACCUMRESPCOMPLETE);
        -:   89:
     7661:   90:    if (MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_ACCUM_RESP) {
        -:   91:	/* accumulate data from tmp_buf into user_buf */
     1573:   92:	mpi_errno = do_accumulate_op(rreq);
     1573:   93:	if (mpi_errno) {
    #####:   94:	    MPIU_ERR_POP(mpi_errno);
        -:   95:	}
        -:   96:    }
        -:   97:    
     7661:   98:    MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
        -:   99:    
        -:  100:    /* if passive target RMA, increment counter */
     7661:  101:    if (win_ptr->current_lock_type != MPID_LOCK_NONE)
      797:  102:	win_ptr->my_pt_rma_puts_accs++;
        -:  103:    
     7661:  104:    if (rreq->dev.source_win_handle != MPI_WIN_NULL) {
        -:  105:	/* Last RMA operation from source. If active
        -:  106:	   target RMA, decrement window counter. If
        -:  107:	   passive target RMA, release lock on window and
        -:  108:	   grant next lock in the lock queue if there is
        -:  109:	   any. If it's a shared lock or a lock-put-unlock
        -:  110:	   type of optimization, we also need to send an
        -:  111:	   ack to the source. */ 
        -:  112:	
     6703:  113:	if (win_ptr->current_lock_type == MPID_LOCK_NONE) {
        -:  114:	    /* FIXME: MT: this has to be done atomically */
     6398:  115:	    win_ptr->my_counter -= 1;
        -:  116:	}
        -:  117:	else {
      305:  118:	    if ((win_ptr->current_lock_type == MPI_LOCK_SHARED) ||
        -:  119:		(rreq->dev.single_op_opt == 1)) {
      301:  120:		mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, 
        -:  121:				    rreq->dev.source_win_handle);
      301:  122:		if (mpi_errno) {
    #####:  123:		    MPIU_ERR_POP(mpi_errno);
        -:  124:		}
        -:  125:	    }
      305:  126:	    mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
        -:  127:	}
        -:  128:    }
        -:  129:    
        -:  130:    /* mark data transfer as complete and decrement CC */
     7661:  131:    MPIDI_CH3U_Request_complete(rreq);
     7661:  132:    *complete = TRUE;
     7661:  133: fn_fail:
        -:  134:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_PUTACCUMRESPCOMPLETE);
     7661:  135:    return MPI_SUCCESS;
        -:  136:}
        -:  137:
        -:  138:#undef FUNCNAME
        -:  139:#define FUNCNAME MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete
        -:  140:#undef FCNAME
        -:  141:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  142:int MPIDI_CH3_ReqHandler_PutRespDerivedDTComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), 
        -:  143:						   MPID_Request *rreq, 
        -:  144:						   int *complete )
     1205:  145:{
     1205:  146:    int mpi_errno = MPI_SUCCESS;
     1205:  147:    MPID_Datatype *new_dtp = NULL;
        -:  148:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_PUTRESPDERIVEDDTCOMPLETE);
        -:  149:    
        -:  150:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_PUTRESPDERIVEDDTCOMPLETE);
        -:  151:                
        -:  152:    /* create derived datatype */
     1205:  153:    create_derived_datatype(rreq, &new_dtp);
        -:  154:    
        -:  155:    /* update request to get the data */
     1205:  156:    MPIDI_Request_set_type(rreq, MPIDI_REQUEST_TYPE_PUT_RESP);
     1205:  157:    rreq->dev.datatype = new_dtp->handle;
     1205:  158:    rreq->dev.recv_data_sz = new_dtp->size * rreq->dev.user_count; 
        -:  159:    
     1205:  160:    rreq->dev.datatype_ptr = new_dtp;
        -:  161:    /* this will cause the datatype to be freed when the
        -:  162:       request is freed. free dtype_info here. */
     1205:  163:    MPIU_Free(rreq->dev.dtype_info);
        -:  164:    
     1205:  165:    rreq->dev.segment_ptr = MPID_Segment_alloc( );
     1205:  166:    MPIU_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
        -:  167:
     1205:  168:    MPID_Segment_init(rreq->dev.user_buf,
        -:  169:		      rreq->dev.user_count,
        -:  170:		      rreq->dev.datatype,
        -:  171:		      rreq->dev.segment_ptr, 0);
     1205:  172:    rreq->dev.segment_first = 0;
     1205:  173:    rreq->dev.segment_size = rreq->dev.recv_data_sz;
        -:  174:    
     1205:  175:    mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
     1205:  176:    if (mpi_errno != MPI_SUCCESS) {
    #####:  177:	MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
        -:  178:			    "**ch3|loadrecviov");
        -:  179:    }
     1205:  180:    if (!rreq->dev.OnDataAvail) 
      805:  181:	rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
        -:  182:    
     1205:  183:    *complete = FALSE;
     1205:  184: fn_fail:
        -:  185:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_PUTRESPDERIVEDDTCOMPLETE);
     1205:  186:    return mpi_errno;
        -:  187:}
        -:  188:
        -:  189:#undef FUNCNAME
        -:  190:#define FUNCNAME MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete
        -:  191:#undef FCNAME
        -:  192:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  193:int MPIDI_CH3_ReqHandler_AccumRespDerivedDTComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), 
        -:  194:						     MPID_Request *rreq, 
        -:  195:						     int *complete )
      434:  196:{
      434:  197:    int mpi_errno = MPI_SUCCESS;
      434:  198:    MPID_Datatype *new_dtp = NULL;
        -:  199:    MPI_Aint true_lb, true_extent, extent;
        -:  200:    void *tmp_buf;
      434:  201:    MPIU_THREADPRIV_DECL;
        -:  202:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_ACCUMRESPDERIVEDDTCOMPLETE);
        -:  203:    
        -:  204:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_ACCUMRESPDERIVEDDTCOMPLETE);
        -:  205:    
      434:  206:    MPIU_THREADPRIV_GET;
        -:  207:    
        -:  208:    /* create derived datatype */
      434:  209:    create_derived_datatype(rreq, &new_dtp);
        -:  210:    
        -:  211:    /* update new request to get the data */
      434:  212:    MPIDI_Request_set_type(rreq, MPIDI_REQUEST_TYPE_ACCUM_RESP);
        -:  213:    
        -:  214:    /* first need to allocate tmp_buf to recv the data into */
        -:  215:    
      434:  216:    MPIR_Nest_incr();
      434:  217:    mpi_errno = NMPI_Type_get_true_extent(new_dtp->handle, 
        -:  218:					  &true_lb, &true_extent);
      434:  219:    MPIR_Nest_decr();
      434:  220:    if (mpi_errno) {
    #####:  221:	MPIU_ERR_POP(mpi_errno);
        -:  222:    }
        -:  223:    
      434:  224:    MPID_Datatype_get_extent_macro(new_dtp->handle, extent); 
        -:  225:    
      434:  226:    tmp_buf = MPIU_Malloc(rreq->dev.user_count * 
        -:  227:			  (MPIR_MAX(extent,true_extent)));  
      434:  228:    if (!tmp_buf) {
    #####:  229:	MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
        -:  230:    }
        -:  231:    
        -:  232:    /* adjust for potential negative lower bound in datatype */
      434:  233:    tmp_buf = (void *)((char*)tmp_buf - true_lb);
        -:  234:    
      434:  235:    rreq->dev.user_buf = tmp_buf;
      434:  236:    rreq->dev.datatype = new_dtp->handle;
      434:  237:    rreq->dev.recv_data_sz = new_dtp->size *
        -:  238:	rreq->dev.user_count; 
      434:  239:    rreq->dev.datatype_ptr = new_dtp;
        -:  240:    /* this will cause the datatype to be freed when the
        -:  241:       request is freed. free dtype_info here. */
      434:  242:    MPIU_Free(rreq->dev.dtype_info);
        -:  243:    
      434:  244:    rreq->dev.segment_ptr = MPID_Segment_alloc( );
      434:  245:    MPIU_ERR_CHKANDJUMP1((rreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
        -:  246:
      434:  247:    MPID_Segment_init(rreq->dev.user_buf,
        -:  248:		      rreq->dev.user_count,
        -:  249:		      rreq->dev.datatype,
        -:  250:		      rreq->dev.segment_ptr, 0);
      434:  251:    rreq->dev.segment_first = 0;
      434:  252:    rreq->dev.segment_size = rreq->dev.recv_data_sz;
        -:  253:    
      434:  254:    mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
      434:  255:    if (mpi_errno != MPI_SUCCESS) {
    #####:  256:	MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
        -:  257:			    "**ch3|loadrecviov");
        -:  258:    }
      434:  259:    if (!rreq->dev.OnDataAvail)
      367:  260:	rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_PutAccumRespComplete;
        -:  261:    
      434:  262:    *complete = FALSE;
      434:  263: fn_fail:
        -:  264:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_ACCUMRESPDERIVEDDTCOMPLETE);
      434:  265:    return mpi_errno;
        -:  266:}
        -:  267:
        -:  268:#undef FUNCNAME
        -:  269:#define FUNCNAME MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete
        -:  270:#undef FCNAME
        -:  271:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  272:int MPIDI_CH3_ReqHandler_GetRespDerivedDTComplete( MPIDI_VC_t *vc, 
        -:  273:						   MPID_Request *rreq, 
        -:  274:						   int *complete )
      914:  275:{
      914:  276:    int mpi_errno = MPI_SUCCESS;
      914:  277:    MPID_Datatype *new_dtp = NULL;
        -:  278:    MPIDI_CH3_Pkt_t upkt;
      914:  279:    MPIDI_CH3_Pkt_get_resp_t * get_resp_pkt = &upkt.get_resp;
        -:  280:    MPID_Request * sreq;
        -:  281:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_GETRESPDERIVEDDTCOMPLETE);
        -:  282:    
        -:  283:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_GETRESPDERIVEDDTCOMPLETE);
        -:  284:                
        -:  285:    /* create derived datatype */
      914:  286:    create_derived_datatype(rreq, &new_dtp);
      914:  287:    MPIU_Free(rreq->dev.dtype_info);
        -:  288:    
        -:  289:    /* create request for sending data */
      914:  290:    sreq = MPID_Request_create();
      914:  291:    MPIU_ERR_CHKANDJUMP(sreq == NULL, mpi_errno,MPI_ERR_OTHER,"**nomem");
        -:  292:    
      914:  293:    sreq->kind = MPID_REQUEST_SEND;
      914:  294:    MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_GET_RESP);
      914:  295:    sreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete;
      914:  296:    sreq->dev.OnFinal     = MPIDI_CH3_ReqHandler_GetSendRespComplete;
      914:  297:    sreq->dev.user_buf = rreq->dev.user_buf;
      914:  298:    sreq->dev.user_count = rreq->dev.user_count;
      914:  299:    sreq->dev.datatype = new_dtp->handle;
      914:  300:    sreq->dev.datatype_ptr = new_dtp;
      914:  301:    sreq->dev.target_win_handle = rreq->dev.target_win_handle;
      914:  302:    sreq->dev.source_win_handle = rreq->dev.source_win_handle;
        -:  303:    
      914:  304:    MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP);
      914:  305:    get_resp_pkt->request_handle = rreq->dev.request_handle;    
        -:  306:    
      914:  307:    sreq->dev.segment_ptr = MPID_Segment_alloc( );
      914:  308:    MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
        -:  309:
      914:  310:    MPID_Segment_init(sreq->dev.user_buf,
        -:  311:		      sreq->dev.user_count,
        -:  312:		      sreq->dev.datatype,
        -:  313:		      sreq->dev.segment_ptr, 0);
      914:  314:    sreq->dev.segment_first = 0;
      914:  315:    sreq->dev.segment_size = new_dtp->size * sreq->dev.user_count;
        -:  316:
        -:  317:    /* Because this is in a packet handler, it is already within a critical section */	
        -:  318:    /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
      914:  319:    mpi_errno = vc->sendNoncontig_fn(vc, sreq, get_resp_pkt, sizeof(*get_resp_pkt));
        -:  320:    /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
        -:  321:    /* --BEGIN ERROR HANDLING-- */
      914:  322:    if (mpi_errno != MPI_SUCCESS)
        -:  323:    {
    #####:  324:        MPIU_Object_set_ref(sreq, 0);
    #####:  325:        MPIDI_CH3_Request_destroy(sreq);
    #####:  326:        sreq = NULL;
    #####:  327:        MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
        -:  328:    }
        -:  329:    /* --END ERROR HANDLING-- */
        -:  330:    
        -:  331:    /* mark receive data transfer as complete and decrement CC in receive 
        -:  332:       request */
      914:  333:    MPIDI_CH3U_Request_complete(rreq);
      914:  334:    *complete = TRUE;
        -:  335:    
      914:  336: fn_fail:
        -:  337:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_GETRESPDERIVEDDTCOMPLETE);
      914:  338:    return mpi_errno;
        -:  339:}
        -:  340:
        -:  341:#undef FUNCNAME
        -:  342:#define FUNCNAME MPIDI_CH3_ReqHandler_SinglePutAccumComplete
        -:  343:#undef FCNAME
        -:  344:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  345:int MPIDI_CH3_ReqHandler_SinglePutAccumComplete( MPIDI_VC_t *vc, 
        -:  346:						 MPID_Request *rreq, 
        -:  347:						 int *complete )
    #####:  348:{
    #####:  349:    int mpi_errno = MPI_SUCCESS;
        -:  350:    MPID_Win *win_ptr;
        -:  351:    MPIDI_Win_lock_queue *lock_queue_entry, *curr_ptr, **curr_ptr_ptr;
        -:  352:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_SINGLEPUTACCUMCOMPLETE);
        -:  353:    
        -:  354:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_SINGLEPUTACCUMCOMPLETE);
        -:  355:
        -:  356:    /* received all the data for single lock-put(accum)-unlock 
        -:  357:       optimization where the lock was not acquired in 
        -:  358:       ch3u_handle_recv_pkt. Try to acquire the lock and do the 
        -:  359:       operation. */
        -:  360:    
    #####:  361:    MPID_Win_get_ptr(rreq->dev.target_win_handle, win_ptr);
        -:  362:    
    #####:  363:    lock_queue_entry = rreq->dev.lock_queue_entry;
        -:  364:    
    #####:  365:    if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, 
        -:  366:					lock_queue_entry->lock_type) == 1)
        -:  367:    {
        -:  368:	
    #####:  369:	if (MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_PT_SINGLE_PUT) {
        -:  370:	    /* copy the data over */
    #####:  371:	    mpi_errno = MPIR_Localcopy(rreq->dev.user_buf,
        -:  372:				       rreq->dev.user_count,
        -:  373:				       rreq->dev.datatype,
        -:  374:				       lock_queue_entry->pt_single_op->addr,
        -:  375:				       lock_queue_entry->pt_single_op->count,
        -:  376:				       lock_queue_entry->pt_single_op->datatype);
        -:  377:	}
        -:  378:	else {
    #####:  379:	    mpi_errno = do_simple_accumulate(lock_queue_entry->pt_single_op);
        -:  380:	}
        -:  381:	
    #####:  382:	if (mpi_errno) {
    #####:  383:	    MPIU_ERR_POP(mpi_errno);
        -:  384:	}
        -:  385:	
        -:  386:	/* increment counter */
    #####:  387:	win_ptr->my_pt_rma_puts_accs++;
        -:  388:	
        -:  389:	/* send done packet */
    #####:  390:	mpi_errno = MPIDI_CH3I_Send_pt_rma_done_pkt(vc, 
        -:  391:				    lock_queue_entry->source_win_handle);
    #####:  392:	if (mpi_errno) {
    #####:  393:	    MPIU_ERR_POP(mpi_errno);
        -:  394:	}
        -:  395:	
        -:  396:	/* free lock_queue_entry including data buffer and remove 
        -:  397:	   it from the queue. */
    #####:  398:	curr_ptr = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
    #####:  399:	curr_ptr_ptr = (MPIDI_Win_lock_queue **) &(win_ptr->lock_queue);
    #####:  400:	while (curr_ptr != lock_queue_entry) {
    #####:  401:	    curr_ptr_ptr = &(curr_ptr->next);
    #####:  402:	    curr_ptr = curr_ptr->next;
        -:  403:	}                    
    #####:  404:	*curr_ptr_ptr = curr_ptr->next;
        -:  405:	
    #####:  406:	MPIU_Free(lock_queue_entry->pt_single_op->data);
    #####:  407:	MPIU_Free(lock_queue_entry->pt_single_op);
    #####:  408:	MPIU_Free(lock_queue_entry);
        -:  409:	
        -:  410:	/* Release lock and grant next lock if there is one. */
    #####:  411:	mpi_errno = MPIDI_CH3I_Release_lock(win_ptr);
        -:  412:    }
        -:  413:    else {
        -:  414:	/* could not acquire lock. mark data recd as 1 */
    #####:  415:	lock_queue_entry->pt_single_op->data_recd = 1;
        -:  416:    }
        -:  417:    
        -:  418:    /* mark data transfer as complete and decrement CC */
    #####:  419:    MPIDI_CH3U_Request_complete(rreq);
    #####:  420:    *complete = TRUE;
    #####:  421: fn_fail:
        -:  422:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_SINGLEPUTACCUMCOMPLETE);
    #####:  423:    return mpi_errno;
        -:  424:}
        -:  425:
        -:  426:#undef FUNCNAME
        -:  427:#define FUNCNAME MPIDI_CH3_ReqHandler_UnpackUEBufComplete
        -:  428:#undef FCNAME
        -:  429:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  430:int MPIDI_CH3_ReqHandler_UnpackUEBufComplete( MPIDI_VC_t *vc ATTRIBUTE((unused)), 
        -:  431:					      MPID_Request *rreq, 
        -:  432:					      int *complete )
   983889:  433:{
        -:  434:    int recv_pending;
        -:  435:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKUEBUFCOMPLETE);
        -:  436:    
        -:  437:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKUEBUFCOMPLETE);
        -:  438:    
   983889:  439:    MPIDI_Request_decr_pending(rreq);
   983889:  440:    MPIDI_Request_check_pending(rreq, &recv_pending);
   983889:  441:    if (!recv_pending)
        -:  442:    { 
       97:  443:	if (rreq->dev.recv_data_sz > 0)
        -:  444:	{
       97:  445:	    MPIDI_CH3U_Request_unpack_uebuf(rreq);
       97:  446:	    MPIU_Free(rreq->dev.tmpbuf);
        -:  447:	}
        -:  448:    }
        -:  449:    else
        -:  450:    {
        -:  451:	/* The receive has not been posted yet.  MPID_{Recv/Irecv}() 
        -:  452:	   is responsible for unpacking the buffer. */
        -:  453:    }
        -:  454:    
        -:  455:    /* mark data transfer as complete and decrement CC */
   983889:  456:    MPIDI_CH3U_Request_complete(rreq);
   983889:  457:    *complete = TRUE;
        -:  458:    
        -:  459:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKUEBUFCOMPLETE);
   983889:  460:    return MPI_SUCCESS;
        -:  461:}
        -:  462:
        -:  463:#undef FUNCNAME
        -:  464:#define FUNCNAME MPIDI_CH3_ReqHandler_UnpackSRBufComplete
        -:  465:#undef FCNAME
        -:  466:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  467:int MPIDI_CH3_ReqHandler_UnpackSRBufComplete( MPIDI_VC_t *vc, 
        -:  468:					      MPID_Request *rreq, 
        -:  469:					      int *complete )
    77366:  470:{
    77366:  471:    int mpi_errno = MPI_SUCCESS;
        -:  472:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKSRBUFCOMPLETE);
        -:  473:    
        -:  474:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKSRBUFCOMPLETE);
        -:  475:
    77366:  476:    MPIDI_CH3U_Request_unpack_srbuf(rreq);
        -:  477:
    77366:  478:    if ((MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_PUT_RESP) ||
        -:  479:	(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_ACCUM_RESP))
        -:  480:    {
      467:  481:	mpi_errno = MPIDI_CH3_ReqHandler_PutAccumRespComplete( 
        -:  482:	    vc, rreq, complete );
        -:  483:    }
        -:  484:    else {
        -:  485:	/* mark data transfer as complete and decrement CC */
    76899:  486:	MPIDI_CH3U_Request_complete(rreq);
    76899:  487:	*complete = TRUE;
        -:  488:    }
        -:  489:
        -:  490:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKSRBUFCOMPLETE);
    77366:  491:    return mpi_errno;
        -:  492:}
        -:  493:
        -:  494:#undef FUNCNAME
        -:  495:#define FUNCNAME MPIDI_CH3_ReqHandler_UnpackSRBufReloadIOV
        -:  496:#undef FCNAME
        -:  497:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  498:int MPIDI_CH3_ReqHandler_UnpackSRBufReloadIOV( MPIDI_VC_t *vc ATTRIBUTE((unused)), 
        -:  499:					      MPID_Request *rreq, 
        -:  500:					      int *complete )
      213:  501:{
      213:  502:    int mpi_errno = MPI_SUCCESS;
        -:  503:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKSRBUFRELOADIOV);
        -:  504:    
        -:  505:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKSRBUFRELOADIOV);
        -:  506:
      213:  507:    MPIDI_CH3U_Request_unpack_srbuf(rreq);
      213:  508:    mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
      213:  509:    if (mpi_errno != MPI_SUCCESS) {
    #####:  510:	MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|loadrecviov" );
        -:  511:    }
      213:  512:    *complete = FALSE;
      213:  513: fn_fail:
        -:  514:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_UNPACKSRBUFRELOADIOV);
      213:  515:    return mpi_errno;
        -:  516:}
        -:  517:
        -:  518:#undef FUNCNAME
        -:  519:#define FUNCNAME MPIDI_CH3_ReqHandler_ReloadIOV
        -:  520:#undef FCNAME
        -:  521:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  522:int MPIDI_CH3_ReqHandler_ReloadIOV( MPIDI_VC_t *vc ATTRIBUTE((unused)), 
        -:  523:				    MPID_Request *rreq, int *complete )
       13:  524:{
       13:  525:    int mpi_errno = MPI_SUCCESS;
        -:  526:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_REQHANDLER_RELOADIOV);
        -:  527:    
        -:  528:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_REQHANDLER_RELOADIOV);
        -:  529:
       13:  530:    mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
       13:  531:    if (mpi_errno != MPI_SUCCESS) {
    #####:  532:	MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|loadrecviov");
        -:  533:    }
       13:  534:    *complete = FALSE;
       13:  535: fn_fail:
        -:  536:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_REQHANDLER_RELOADIOV);
       13:  537:    return mpi_errno;
        -:  538:}
        -:  539:
        -:  540:/* ----------------------------------------------------------------------- */
        -:  541:/* ----------------------------------------------------------------------- */
        -:  542:
        -:  543:#undef FUNCNAME
        -:  544:#define FUNCNAME create_derived_datatype
        -:  545:#undef FCNAME
        -:  546:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  547:static int create_derived_datatype(MPID_Request *req, MPID_Datatype **dtp)
     2553:  548:{
        -:  549:    MPIDI_RMA_dtype_info *dtype_info;
        -:  550:    void *dataloop;
        -:  551:    MPID_Datatype *new_dtp;
     2553:  552:    int mpi_errno=MPI_SUCCESS;
        -:  553:    MPI_Aint ptrdiff;
        -:  554:    MPIDI_STATE_DECL(MPID_STATE_CREATE_DERIVED_DATATYPE);
        -:  555:    
        -:  556:    MPIDI_FUNC_ENTER(MPID_STATE_CREATE_DERIVED_DATATYPE);
        -:  557:
     2553:  558:    dtype_info = req->dev.dtype_info;
        -:  559:    /* FIXME: What is this variable for (it is never referenced)? */
     2553:  560:    dataloop   = req->dev.dataloop;
        -:  561:
        -:  562:    /* allocate new datatype object and handle */
     2553:  563:    new_dtp = (MPID_Datatype *) MPIU_Handle_obj_alloc(&MPID_Datatype_mem);
     2553:  564:    if (!new_dtp) {
    #####:  565:	MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
        -:  566:    }
        -:  567:
     2553:  568:    *dtp = new_dtp;
        -:  569:            
        -:  570:    /* Note: handle is filled in by MPIU_Handle_obj_alloc() */
     2553:  571:    MPIU_Object_set_ref(new_dtp, 1);
     2553:  572:    new_dtp->is_permanent = 0;
     2553:  573:    new_dtp->is_committed = 1;
     2553:  574:    new_dtp->attributes   = 0;
     2553:  575:    new_dtp->cache_id     = 0;
     2553:  576:    new_dtp->name[0]      = 0;
     2553:  577:    new_dtp->is_contig = dtype_info->is_contig;
     2553:  578:    new_dtp->max_contig_blocks = dtype_info->max_contig_blocks; 
     2553:  579:    new_dtp->size = dtype_info->size;
     2553:  580:    new_dtp->extent = dtype_info->extent;
     2553:  581:    new_dtp->dataloop_size = dtype_info->dataloop_size;
     2553:  582:    new_dtp->dataloop_depth = dtype_info->dataloop_depth; 
     2553:  583:    new_dtp->eltype = dtype_info->eltype;
        -:  584:    /* set dataloop pointer */
     2553:  585:    new_dtp->dataloop = req->dev.dataloop;
        -:  586:    
     2553:  587:    new_dtp->ub = dtype_info->ub;
     2553:  588:    new_dtp->lb = dtype_info->lb;
     2553:  589:    new_dtp->true_ub = dtype_info->true_ub;
     2553:  590:    new_dtp->true_lb = dtype_info->true_lb;
     2553:  591:    new_dtp->has_sticky_ub = dtype_info->has_sticky_ub;
     2553:  592:    new_dtp->has_sticky_lb = dtype_info->has_sticky_lb;
        -:  593:    /* update pointers in dataloop */
     2553:  594:    ptrdiff = (MPI_Aint)((char *) (new_dtp->dataloop) - (char *)
        -:  595:                         (dtype_info->dataloop));
        -:  596:    
        -:  597:    /* FIXME: Temp to avoid SEGV when memory tracing */
     2553:  598:    new_dtp->hetero_dloop = 0;
        -:  599:
     2553:  600:    MPID_Dataloop_update(new_dtp->dataloop, ptrdiff);
        -:  601:
     2553:  602:    new_dtp->contents = NULL;
        -:  603:
     2553:  604: fn_fail:
        -:  605:    MPIDI_FUNC_EXIT(MPID_STATE_CREATE_DERIVED_DATATYPE);
        -:  606:
     2553:  607:    return mpi_errno;
        -:  608:}
        -:  609:
        -:  610:
        -:  611:#undef FUNCNAME
        -:  612:#define FUNCNAME do_accumulate_op
        -:  613:#undef FCNAME
        -:  614:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  615:static int do_accumulate_op(MPID_Request *rreq)
     1573:  616:{
     1573:  617:    int mpi_errno = MPI_SUCCESS, predefined;
        -:  618:    MPI_Aint true_lb, true_extent;
        -:  619:    MPI_User_function *uop;
     1573:  620:    MPIU_THREADPRIV_DECL;
        -:  621:    MPIDI_STATE_DECL(MPID_STATE_DO_ACCUMULATE_OP);
        -:  622:    
        -:  623:    MPIDI_FUNC_ENTER(MPID_STATE_DO_ACCUMULATE_OP);
        -:  624:
     1573:  625:    MPIU_THREADPRIV_GET;
        -:  626:
     1573:  627:    if (rreq->dev.op == MPI_REPLACE)
        -:  628:    {
        -:  629:        /* simply copy the data */
      872:  630:        mpi_errno = MPIR_Localcopy(rreq->dev.user_buf, rreq->dev.user_count,
        -:  631:                                   rreq->dev.datatype,
        -:  632:                                   rreq->dev.real_user_buf,
        -:  633:                                   rreq->dev.user_count,
        -:  634:                                   rreq->dev.datatype);
      872:  635:        if (mpi_errno) {
    #####:  636:	    MPIU_ERR_POP(mpi_errno);
        -:  637:	}
        -:  638:        goto fn_exit;
        -:  639:    }
        -:  640:
      701:  641:    if (HANDLE_GET_KIND(rreq->dev.op) == HANDLE_KIND_BUILTIN)
        -:  642:    {
        -:  643:        /* get the function by indexing into the op table */
      701:  644:        uop = MPIR_Op_table[(rreq->dev.op)%16 - 1];
        -:  645:    }
        -:  646:    else
        -:  647:    {
        -:  648:	/* --BEGIN ERROR HANDLING-- */
    #####:  649:        mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", rreq->dev.op );
    #####:  650:        return mpi_errno;
        -:  651:	/* --END ERROR HANDLING-- */
        -:  652:    }
        -:  653:    
      943:  654:    MPIDI_CH3I_DATATYPE_IS_PREDEFINED(rreq->dev.datatype, predefined);
      701:  655:    if (predefined)
        -:  656:    {
      459:  657:        (*uop)(rreq->dev.user_buf, rreq->dev.real_user_buf,
        -:  658:               &(rreq->dev.user_count), &(rreq->dev.datatype));
        -:  659:    }
        -:  660:    else
        -:  661:    {
        -:  662:	/* derived datatype */
        -:  663:        MPID_Segment *segp;
        -:  664:        DLOOP_VECTOR *dloop_vec;
        -:  665:        MPI_Aint first, last;
        -:  666:        int vec_len, i, type_size, count;
        -:  667:        MPI_Datatype type;
        -:  668:        MPID_Datatype *dtp;
        -:  669:        
      242:  670:        segp = MPID_Segment_alloc();
        -:  671:	/* --BEGIN ERROR HANDLING-- */
      242:  672:        if (!segp)
        -:  673:	{
    #####:  674:            mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 ); 
        -:  675:	    MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
    #####:  676:            return mpi_errno;
        -:  677:        }
        -:  678:	/* --END ERROR HANDLING-- */
      242:  679:        MPID_Segment_init(NULL, rreq->dev.user_count,
        -:  680:			  rreq->dev.datatype, segp, 0);
      242:  681:        first = 0;
      242:  682:        last  = SEGMENT_IGNORE_LAST;
        -:  683:        
      242:  684:        MPID_Datatype_get_ptr(rreq->dev.datatype, dtp);
      242:  685:        vec_len = dtp->max_contig_blocks * rreq->dev.user_count + 1; 
        -:  686:        /* +1 needed because Rob says so */
      242:  687:        dloop_vec = (DLOOP_VECTOR *)
        -:  688:            MPIU_Malloc(vec_len * sizeof(DLOOP_VECTOR));
        -:  689:	/* --BEGIN ERROR HANDLING-- */
      242:  690:        if (!dloop_vec)
        -:  691:	{
    #####:  692:            mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0 ); 
        -:  693:	    MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
    #####:  694:            return mpi_errno;
        -:  695:        }
        -:  696:	/* --END ERROR HANDLING-- */
        -:  697:
      242:  698:        MPID_Segment_pack_vector(segp, first, &last, dloop_vec, &vec_len);
        -:  699:        
      242:  700:        type = dtp->eltype;
      242:  701:        MPID_Datatype_get_size_macro(type, type_size);
  1000723:  702:        for (i=0; i<vec_len; i++)
        -:  703:	{
  1000481:  704:            count = (dloop_vec[i].DLOOP_VECTOR_LEN)/type_size;
  1000481:  705:            (*uop)((char *)rreq->dev.user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
        -:  706:                   (char *)rreq->dev.real_user_buf + MPIU_PtrToAint(dloop_vec[i].DLOOP_VECTOR_BUF),
        -:  707:                   &count, &type);
        -:  708:        }
        -:  709:        
      242:  710:        MPID_Segment_free(segp);
      242:  711:        MPIU_Free(dloop_vec);
        -:  712:    }
        -:  713:
     1573:  714: fn_exit:
        -:  715:    /* free the temporary buffer */
     1573:  716:    MPIR_Nest_incr();
     1573:  717:    mpi_errno = NMPI_Type_get_true_extent(rreq->dev.datatype, &true_lb, &true_extent);
     1573:  718:    MPIR_Nest_decr();
     1573:  719:    if (mpi_errno) {
    #####:  720:	MPIU_ERR_POP(mpi_errno);
        -:  721:    }
        -:  722:    
     1573:  723:    MPIU_Free((char *) rreq->dev.user_buf + true_lb);
        -:  724:
        -:  725:    MPIDI_FUNC_EXIT(MPID_STATE_DO_ACCUMULATE_OP);
        -:  726:
     1573:  727:    return mpi_errno;
        -:  728: fn_fail:
        -:  729:    goto fn_exit;
        -:  730:}
        -:  731:
        -:  732:static int entered_flag = 0;
        -:  733:static int entered_count = 0;
        -:  734:
        -:  735:/* Release the current lock on the window and grant the next lock in the
        -:  736:   queue if any */
        -:  737:#undef FUNCNAME
        -:  738:#define FUNCNAME MPIDI_CH3I_Release_lock
        -:  739:#undef FCNAME
        -:  740:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  741:int MPIDI_CH3I_Release_lock(MPID_Win *win_ptr)
     1095:  742:{
        -:  743:    MPIDI_Win_lock_queue *lock_queue, **lock_queue_ptr;
     1095:  744:    int requested_lock, mpi_errno = MPI_SUCCESS, temp_entered_count;
        -:  745:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_RELEASE_LOCK);
        -:  746:    
        -:  747:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_RELEASE_LOCK);
        -:  748:
     1095:  749:    if (win_ptr->current_lock_type == MPI_LOCK_SHARED) {
        -:  750:        /* decr ref cnt */
        -:  751:        /* FIXME: MT: Must be done atomically */
      405:  752:        win_ptr->shared_lock_ref_cnt--;
        -:  753:    }
        -:  754:
        -:  755:    /* If shared lock ref count is 0 (which is also true if the lock is an
        -:  756:       exclusive lock), release the lock. */
     1095:  757:    if (win_ptr->shared_lock_ref_cnt == 0) {
        -:  758:
        -:  759:	/* This function needs to be reentrant even in the single-threaded case
        -:  760:           because when going through the lock queue, the do_simple_get 
        -:  761:	   called in the 
        -:  762:	   lock-get-unlock case may itself cause a request to complete, and 
        -:  763:	   this function
        -:  764:           may again get called in the completion action in 
        -:  765:	   ch3u_handle_send_req.c. To
        -:  766:           handle this possibility, we use an entered_flag. If the flag is 
        -:  767:	   not 0, we simply
        -:  768:	   increment the entered_count and return. The loop through the lock 
        -:  769:	   queue is repeated 
        -:  770:	   if the entered_count has changed while we are in the loop.
        -:  771:	 */
     1095:  772:	if (entered_flag != 0) {
    #####:  773:	    entered_count++;
    #####:  774:	    goto fn_exit;
        -:  775:	}
        -:  776:	else {
     1095:  777:	    entered_flag = 1;
     1095:  778:	    temp_entered_count = entered_count;
        -:  779:	}
        -:  780:
        -:  781:	do { 
     1095:  782:	    if (temp_entered_count != entered_count) temp_entered_count++;
        -:  783:
        -:  784:	    /* FIXME: MT: The setting of the lock type must be done atomically */
     1095:  785:	    win_ptr->current_lock_type = MPID_LOCK_NONE;
        -:  786:	    
        -:  787:	    /* If there is a lock queue, try to satisfy as many lock requests as 
        -:  788:	       possible. If the first one is a shared lock, grant it and grant all 
        -:  789:	       other shared locks. If the first one is an exclusive lock, grant 
        -:  790:	       only that one. */
        -:  791:	    
        -:  792:	    /* FIXME: MT: All queue accesses need to be made atomic */
     1095:  793:	    lock_queue = (MPIDI_Win_lock_queue *) win_ptr->lock_queue;
     1095:  794:	    lock_queue_ptr = (MPIDI_Win_lock_queue **) &(win_ptr->lock_queue);
     2190:  795:	    while (lock_queue) {
        -:  796:		/* if it is not a lock-op-unlock type case or if it is a 
        -:  797:		   lock-op-unlock type case but all the data has been received, 
        -:  798:		   try to acquire the lock */
      474:  799:		if ((lock_queue->pt_single_op == NULL) || 
        -:  800:		    (lock_queue->pt_single_op->data_recd == 1)) {
        -:  801:		    
      474:  802:		    requested_lock = lock_queue->lock_type;
      474:  803:		    if (MPIDI_CH3I_Try_acquire_win_lock(win_ptr, requested_lock) 
        -:  804:			== 1) {
        -:  805:			
      474:  806:			if (lock_queue->pt_single_op != NULL) {
        -:  807:			    /* single op. do it here */
        -:  808:			    MPIDI_PT_single_op * single_op;
        -:  809:			    
    #####:  810:			    single_op = lock_queue->pt_single_op;
    #####:  811:			    if (single_op->type == MPIDI_RMA_PUT) {
    #####:  812:				mpi_errno = MPIR_Localcopy(single_op->data,
        -:  813:							   single_op->count,
        -:  814:							   single_op->datatype,
        -:  815:							   single_op->addr,
        -:  816:							   single_op->count,
        -:  817:							   single_op->datatype);
        -:  818:			    }   
    #####:  819:			    else if (single_op->type == MPIDI_RMA_ACCUMULATE) {
    #####:  820:				mpi_errno = do_simple_accumulate(single_op);
        -:  821:			    }
    #####:  822:			    else if (single_op->type == MPIDI_RMA_GET) {
    #####:  823:				mpi_errno = do_simple_get(win_ptr, lock_queue);
        -:  824:			    }
        -:  825:			    
    #####:  826:			    if (mpi_errno != MPI_SUCCESS) goto fn_exit;
        -:  827:			    
        -:  828:			    /* if put or accumulate, send rma done packet and release lock. */
    #####:  829:			    if (single_op->type != MPIDI_RMA_GET) {
        -:  830:				/* increment counter */
    #####:  831:				win_ptr->my_pt_rma_puts_accs++;
        -:  832:				
    #####:  833:				mpi_errno = 
        -:  834:				    MPIDI_CH3I_Send_pt_rma_done_pkt(lock_queue->vc, 
        -:  835:								    lock_queue->source_win_handle);
    #####:  836:				if (mpi_errno != MPI_SUCCESS) goto fn_exit;
        -:  837:				
        -:  838:				/* release the lock */
    #####:  839:				if (win_ptr->current_lock_type == MPI_LOCK_SHARED) {
        -:  840:				    /* decr ref cnt */
        -:  841:				    /* FIXME: MT: Must be done atomically */
    #####:  842:				    win_ptr->shared_lock_ref_cnt--;
        -:  843:				}
        -:  844:				
        -:  845:				/* If shared lock ref count is 0 
        -:  846:				   (which is also true if the lock is an
        -:  847:				   exclusive lock), release the lock. */
    #####:  848:				if (win_ptr->shared_lock_ref_cnt == 0) {
        -:  849:				    /* FIXME: MT: The setting of the lock type 
        -:  850:				       must be done atomically */
    #####:  851:				    win_ptr->current_lock_type = MPID_LOCK_NONE;
        -:  852:				}
        -:  853:				
        -:  854:				/* dequeue entry from lock queue */
    #####:  855:				MPIU_Free(single_op->data);
    #####:  856:				MPIU_Free(single_op);
    #####:  857:				*lock_queue_ptr = lock_queue->next;
    #####:  858:				MPIU_Free(lock_queue);
    #####:  859:				lock_queue = *lock_queue_ptr;
        -:  860:			    }
        -:  861:			    
        -:  862:			    else {
        -:  863:				/* it's a get. The operation is not complete. It 
        -:  864:				   will be completed in ch3u_handle_send_req.c. 
        -:  865:				   Free the single_op structure. If it's an 
        -:  866:				   exclusive lock, break. Otherwise continue to the
        -:  867:				   next operation. */
        -:  868:				
    #####:  869:				MPIU_Free(single_op);
    #####:  870:				*lock_queue_ptr = lock_queue->next;
    #####:  871:				MPIU_Free(lock_queue);
    #####:  872:				lock_queue = *lock_queue_ptr;
        -:  873:				
    #####:  874:				if (requested_lock == MPI_LOCK_EXCLUSIVE)
    #####:  875:				    break;
        -:  876:			    }
        -:  877:			}
        -:  878:			
        -:  879:			else {
        -:  880:			    /* send lock granted packet. */
      474:  881:			    mpi_errno = 
        -:  882:				MPIDI_CH3I_Send_lock_granted_pkt(lock_queue->vc,
        -:  883:								 lock_queue->source_win_handle);
        -:  884:			    
        -:  885:			    /* dequeue entry from lock queue */
      474:  886:			    *lock_queue_ptr = lock_queue->next;
      474:  887:			    MPIU_Free(lock_queue);
      474:  888:			    lock_queue = *lock_queue_ptr;
        -:  889:			    
        -:  890:			    /* if the granted lock is exclusive, 
        -:  891:			       no need to continue */
      474:  892:			    if (requested_lock == MPI_LOCK_EXCLUSIVE)
      474:  893:				break;
        -:  894:			}
        -:  895:		    }
        -:  896:		}
        -:  897:		else {
    #####:  898:		    lock_queue_ptr = &(lock_queue->next);
    #####:  899:		    lock_queue = lock_queue->next;
        -:  900:		}
        -:  901:	    }
     1095:  902:	} while (temp_entered_count != entered_count);
     1095:  903:	entered_count = entered_flag = 0;
        -:  904:    }
        -:  905:
     1095:  906: fn_exit:
        -:  907:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_RELEASE_LOCK);
     1095:  908:    return mpi_errno;
        -:  909:}
        -:  910:
        -:  911:
        -:  912:#undef FUNCNAME
        -:  913:#define FUNCNAME MPIDI_CH3I_Send_pt_rma_done_pkt
        -:  914:#undef FCNAME 
        -:  915:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  916:int MPIDI_CH3I_Send_pt_rma_done_pkt(MPIDI_VC_t *vc, MPI_Win source_win_handle)
      301:  917:{
        -:  918:    MPIDI_CH3_Pkt_t upkt;
      301:  919:    MPIDI_CH3_Pkt_pt_rma_done_t *pt_rma_done_pkt = &upkt.pt_rma_done;
        -:  920:    MPID_Request *req;
      301:  921:    int mpi_errno=MPI_SUCCESS;
        -:  922:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
        -:  923:    
        -:  924:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
        -:  925:
      301:  926:    MPIDI_Pkt_init(pt_rma_done_pkt, MPIDI_CH3_PKT_PT_RMA_DONE);
      301:  927:    pt_rma_done_pkt->source_win_handle = source_win_handle;
        -:  928:
        -:  929:    /* Because this is in a packet handler, it is already within a critical section */	
        -:  930:    /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
      301:  931:    mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsg(vc, pt_rma_done_pkt,
        -:  932:					      sizeof(*pt_rma_done_pkt), &req));
        -:  933:    /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
      301:  934:    if (mpi_errno != MPI_SUCCESS) {
    #####:  935:	MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
        -:  936:    }
        -:  937:
      301:  938:    if (req != NULL)
        -:  939:    {
    #####:  940:        MPID_Request_release(req);
        -:  941:    }
        -:  942:
      301:  943: fn_fail:
        -:  944:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SEND_PT_RMA_DONE_PKT);
      301:  945:    return mpi_errno;
        -:  946:}
        -:  947:
        -:  948:
        -:  949:#undef FUNCNAME
        -:  950:#define FUNCNAME do_simple_accumulate
        -:  951:#undef FCNAME 
        -:  952:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  953:static int do_simple_accumulate(MPIDI_PT_single_op *single_op)
    #####:  954:{
    #####:  955:    int mpi_errno = MPI_SUCCESS;
        -:  956:    MPI_User_function *uop;
        -:  957:    MPIDI_STATE_DECL(MPID_STATE_DO_SIMPLE_ACCUMULATE);
        -:  958:    
        -:  959:    MPIDI_FUNC_ENTER(MPID_STATE_DO_SIMPLE_ACCUMULATE);
        -:  960:
    #####:  961:    if (single_op->op == MPI_REPLACE)
        -:  962:    {
        -:  963:        /* simply copy the data */
    #####:  964:        mpi_errno = MPIR_Localcopy(single_op->data, single_op->count,
        -:  965:                                   single_op->datatype, single_op->addr,
        -:  966:                                   single_op->count, single_op->datatype);
    #####:  967:        if (mpi_errno) {
    #####:  968:	    MPIU_ERR_POP(mpi_errno);
        -:  969:	}
        -:  970:        goto fn_exit;
        -:  971:    }
        -:  972:
    #####:  973:    if (HANDLE_GET_KIND(single_op->op) == HANDLE_KIND_BUILTIN)
        -:  974:    {
        -:  975:        /* get the function by indexing into the op table */
    #####:  976:        uop = MPIR_Op_table[(single_op->op)%16 - 1];
        -:  977:    }
        -:  978:    else
        -:  979:    {
        -:  980:	/* --BEGIN ERROR HANDLING-- */
    #####:  981:        mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OP, "**opnotpredefined", "**opnotpredefined %d", single_op->op );
    #####:  982:        goto fn_fail;
        -:  983:	/* --END ERROR HANDLING-- */
        -:  984:    }
        -:  985:    
        -:  986:    /* only basic datatypes supported for this optimization. */
    #####:  987:    (*uop)(single_op->data, single_op->addr,
        -:  988:           &(single_op->count), &(single_op->datatype));
        -:  989:
    #####:  990: fn_fail:
    #####:  991: fn_exit:
        -:  992:    MPIDI_FUNC_EXIT(MPID_STATE_DO_SIMPLE_ACCUMULATE);
    #####:  993:    return mpi_errno;
        -:  994:}
        -:  995:
        -:  996:
        -:  997:
        -:  998:#undef FUNCNAME
        -:  999:#define FUNCNAME do_simple_get
        -: 1000:#undef FCNAME 
        -: 1001:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -: 1002:static int do_simple_get(MPID_Win *win_ptr, MPIDI_Win_lock_queue *lock_queue)
    #####: 1003:{
        -: 1004:    MPIDI_CH3_Pkt_t upkt;
    #####: 1005:    MPIDI_CH3_Pkt_get_resp_t * get_resp_pkt = &upkt.get_resp;
        -: 1006:    MPID_Request *req;
        -: 1007:    MPID_IOV iov[MPID_IOV_LIMIT];
    #####: 1008:    int type_size, mpi_errno=MPI_SUCCESS;
        -: 1009:    MPIDI_STATE_DECL(MPID_STATE_DO_SIMPLE_GET);
        -: 1010:    
        -: 1011:    MPIDI_FUNC_ENTER(MPID_STATE_DO_SIMPLE_GET);
        -: 1012:
    #####: 1013:    req = MPID_Request_create();
    #####: 1014:    if (req == NULL) {
    #####: 1015:        MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
        -: 1016:    }
    #####: 1017:    req->dev.target_win_handle = win_ptr->handle;
    #####: 1018:    req->dev.source_win_handle = lock_queue->source_win_handle;
    #####: 1019:    req->dev.single_op_opt = 1;
        -: 1020:    
    #####: 1021:    MPIDI_Request_set_type(req, MPIDI_REQUEST_TYPE_GET_RESP); 
    #####: 1022:    req->kind = MPID_REQUEST_SEND;
    #####: 1023:    req->dev.OnDataAvail = MPIDI_CH3_ReqHandler_GetSendRespComplete;
    #####: 1024:    req->dev.OnFinal     = MPIDI_CH3_ReqHandler_GetSendRespComplete;
        -: 1025:    
    #####: 1026:    MPIDI_Pkt_init(get_resp_pkt, MPIDI_CH3_PKT_GET_RESP);
    #####: 1027:    get_resp_pkt->request_handle = lock_queue->pt_single_op->request_handle;
        -: 1028:    
    #####: 1029:    iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) get_resp_pkt;
    #####: 1030:    iov[0].MPID_IOV_LEN = sizeof(*get_resp_pkt);
        -: 1031:    
    #####: 1032:    iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST)lock_queue->pt_single_op->addr;
    #####: 1033:    MPID_Datatype_get_size_macro(lock_queue->pt_single_op->datatype, type_size);
    #####: 1034:    iov[1].MPID_IOV_LEN = lock_queue->pt_single_op->count * type_size;
        -: 1035:    
        -: 1036:    /* Because this is in a packet handler, it is already within a critical section */	
        -: 1037:    /* MPIU_THREAD_CS_ENTER(CH3COMM,vc); */
    #####: 1038:    mpi_errno = MPIU_CALL(MPIDI_CH3,iSendv(lock_queue->vc, req, iov, 2));
        -: 1039:    /* MPIU_THREAD_CS_EXIT(CH3COMM,vc); */
        -: 1040:    /* --BEGIN ERROR HANDLING-- */
    #####: 1041:    if (mpi_errno != MPI_SUCCESS)
        -: 1042:    {
    #####: 1043:        MPIU_Object_set_ref(req, 0);
    #####: 1044:        MPIDI_CH3_Request_destroy(req);
    #####: 1045:	MPIU_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|rmamsg");
        -: 1046:    }
        -: 1047:    /* --END ERROR HANDLING-- */
        -: 1048:
    #####: 1049: fn_fail:
        -: 1050:    MPIDI_FUNC_EXIT(MPID_STATE_DO_SIMPLE_GET);
        -: 1051:
    #####: 1052:    return mpi_errno;
        -: 1053:}