Actual source code: sf.c

petsc-3.4.4 2014-03-13
  1: #include <petsc-private/sfimpl.h> /*I "petscsf.h" I*/
  2: #include <petscctable.h>

  4: #if defined(PETSC_USE_DEBUG)
  5: #  define PetscSFCheckGraphSet(sf,arg) do {                          \
  6:     if (PetscUnlikely(!(sf)->graphset))                              \
  7:       SETERRQ3(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE,"Must call PetscSFSetGraph() on argument %D \"%s\" before %s()",(arg),#sf,PETSC_FUNCTION_NAME); \
  8:   } while (0)
  9: #else
 10: #  define PetscSFCheckGraphSet(sf,arg) do {} while (0)
 11: #endif

 13: const char *const PetscSFDuplicateOptions[] = {"CONFONLY","RANKS","GRAPH","PetscSFDuplicateOption","PETSCSF_DUPLICATE_",0};

 17: /*@C
 18:    PetscSFCreate - create a star forest communication context

 20:    Not Collective

 22:    Input Arguments:
 23: .  comm - communicator on which the star forest will operate

 25:    Output Arguments:
 26: .  sf - new star forest context

 28:    Level: intermediate

 30: .seealso: PetscSFSetGraph(), PetscSFDestroy()
 31: @*/
 32: PetscErrorCode PetscSFCreate(MPI_Comm comm,PetscSF *sf)
 33: {
 35:   PetscSF        b;

 39: #if !defined(PETSC_USE_DYNAMIC_LIBRARIES)
 40:   PetscSFInitializePackage();
 41: #endif

 43:   PetscHeaderCreate(b,_p_PetscSF,struct _PetscSFOps,PETSCSF_CLASSID,"PetscSF","Star Forest","PetscSF",comm,PetscSFDestroy,PetscSFView);

 45:   b->nroots    = -1;
 46:   b->nleaves   = -1;
 47:   b->nranks    = -1;
 48:   b->rankorder = PETSC_TRUE;
 49:   b->ingroup   = MPI_GROUP_NULL;
 50:   b->outgroup  = MPI_GROUP_NULL;
 51:   b->graphset  = PETSC_FALSE;

 53:   *sf = b;
 54:   return(0);
 55: }

 59: /*@C
 60:    PetscSFReset - Reset a star forest so that different sizes or neighbors can be used

 62:    Collective

 64:    Input Arguments:
 65: .  sf - star forest

 67:    Level: advanced

 69: .seealso: PetscSFCreate(), PetscSFSetGraph(), PetscSFDestroy()
 70: @*/
 71: PetscErrorCode PetscSFReset(PetscSF sf)
 72: {

 77:   sf->mine   = NULL;
 78:   PetscFree(sf->mine_alloc);
 79:   sf->remote = NULL;
 80:   PetscFree(sf->remote_alloc);
 81:   PetscFree4(sf->ranks,sf->roffset,sf->rmine,sf->rremote);
 82:   PetscFree(sf->degree);
 83:   if (sf->ingroup  != MPI_GROUP_NULL) {MPI_Group_free(&sf->ingroup);}
 84:   if (sf->outgroup != MPI_GROUP_NULL) {MPI_Group_free(&sf->outgroup);}
 85:   PetscSFDestroy(&sf->multi);
 86:   sf->graphset = PETSC_FALSE;
 87:   if (sf->ops->Reset) {(*sf->ops->Reset)(sf);}
 88:   sf->setupcalled = PETSC_FALSE;
 89:   return(0);
 90: }

 94: /*@C
 95:    PetscSFSetType - set the PetscSF communication implementation

 97:    Collective on PetscSF

 99:    Input Parameters:
100: +  sf - the PetscSF context
101: -  type - a known method

103:    Options Database Key:
104: .  -sf_type <type> - Sets the method; use -help for a list
105:    of available methods (for instance, window, pt2pt, neighbor)

107:    Notes:
108:    See "include/petscsf.h" for available methods (for instance)
109: +    PETSCSFWINDOW - MPI-2/3 one-sided
110: -    PETSCSFBASIC - basic implementation using MPI-1 two-sided

112:   Level: intermediate

114: .keywords: PetscSF, set, type

116: .seealso: PetscSFType, PetscSFCreate()
117: @*/
118: PetscErrorCode PetscSFSetType(PetscSF sf,PetscSFType type)
119: {
120:   PetscErrorCode ierr,(*r)(PetscSF);
121:   PetscBool      match;


127:   PetscObjectTypeCompare((PetscObject)sf,type,&match);
128:   if (match) return(0);

130:   PetscFunctionListFind(PetscSFList,type,&r);
131:   if (!r) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_UNKNOWN_TYPE,"Unable to find requested PetscSF type %s",type);
132:   /* Destroy the previous private PetscSF context */
133:   if (sf->ops->Destroy) {
134:     (*(sf)->ops->Destroy)(sf);
135:   }
136:   PetscMemzero(sf->ops,sizeof(*sf->ops));
137:   PetscObjectChangeTypeName((PetscObject)sf,type);
138:   (*r)(sf);
139:   return(0);
140: }

144: /*@C
145:    PetscSFDestroy - destroy star forest

147:    Collective

149:    Input Arguments:
150: .  sf - address of star forest

152:    Level: intermediate

154: .seealso: PetscSFCreate(), PetscSFReset()
155: @*/
156: PetscErrorCode PetscSFDestroy(PetscSF *sf)
157: {

161:   if (!*sf) return(0);
163:   if (--((PetscObject)(*sf))->refct > 0) {*sf = 0; return(0);}
164:   PetscSFReset(*sf);
165:   if ((*sf)->ops->Destroy) {(*(*sf)->ops->Destroy)(*sf);}
166:   PetscHeaderDestroy(sf);
167:   return(0);
168: }

172: /*@
173:    PetscSFSetUp - set up communication structures

175:    Collective

177:    Input Arguments:
178: .  sf - star forest communication object

180:    Level: beginner

182: .seealso: PetscSFSetFromOptions(), PetscSFSetType()
183: @*/
184: PetscErrorCode PetscSFSetUp(PetscSF sf)
185: {

189:   if (sf->setupcalled) return(0);
190:   if (!((PetscObject)sf)->type_name) {PetscSFSetType(sf,PETSCSFBASIC);}
191:   if (sf->ops->SetUp) {(*sf->ops->SetUp)(sf);}
192:   sf->setupcalled = PETSC_TRUE;
193:   return(0);
194: }

198: /*@C
199:    PetscSFSetFromOptions - set PetscSF options using the options database

201:    Logically Collective

203:    Input Arguments:
204: .  sf - star forest

206:    Options Database Keys:
207: .  -sf_synchronization - synchronization type used by PetscSF

209:    Level: intermediate

211: .keywords: KSP, set, from, options, database

213: .seealso: PetscSFWindowSetSyncType()
214: @*/
215: PetscErrorCode PetscSFSetFromOptions(PetscSF sf)
216: {
217:   PetscSFType    deft;
218:   char           type[256];
220:   PetscBool      flg;

224:   PetscObjectOptionsBegin((PetscObject)sf);
225:   deft = ((PetscObject)sf)->type_name ? ((PetscObject)sf)->type_name : PETSCSFBASIC;
226:   PetscOptionsList("-sf_type","PetscSF implementation type","PetscSFSetType",PetscSFList,deft,type,256,&flg);
227:   PetscSFSetType(sf,flg ? type : deft);
228:   PetscOptionsBool("-sf_rank_order","sort composite points for gathers and scatters in rank order, gathers are non-deterministic otherwise","PetscSFSetRankOrder",sf->rankorder,&sf->rankorder,NULL);
229:   if (sf->ops->SetFromOptions) {(*sf->ops->SetFromOptions)(sf);}
230:   PetscOptionsEnd();
231:   return(0);
232: }

236: /*@C
237:    PetscSFSetRankOrder - sort multi-points for gathers and scatters by rank order

239:    Logically Collective

241:    Input Arguments:
242: +  sf - star forest
243: -  flg - PETSC_TRUE to sort, PETSC_FALSE to skip sorting (lower setup cost, but non-deterministic)

245:    Level: advanced

247: .seealso: PetscSFGatherBegin(), PetscSFScatterBegin()
248: @*/
249: PetscErrorCode PetscSFSetRankOrder(PetscSF sf,PetscBool flg)
250: {

255:   if (sf->multi) SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_WRONGSTATE,"Rank ordering must be set before first call to PetscSFGatherBegin() or PetscSFScatterBegin()");
256:   sf->rankorder = flg;
257:   return(0);
258: }

262: /*@C
263:    PetscSFSetGraph - Set a parallel star forest

265:    Collective

267:    Input Arguments:
268: +  sf - star forest
269: .  nroots - number of root vertices on the current process (these are possible targets for other process to attach leaves)
270: .  nleaves - number of leaf vertices on the current process, each of these references a root on any process
271: .  ilocal - locations of leaves in leafdata buffers, pass NULL for contiguous storage
272: .  localmode - copy mode for ilocal
273: .  iremote - remote locations of root vertices for each leaf on the current process
274: -  remotemode - copy mode for iremote

276:    Level: intermediate

278: .seealso: PetscSFCreate(), PetscSFView(), PetscSFGetGraph()
279: @*/
280: PetscErrorCode PetscSFSetGraph(PetscSF sf,PetscInt nroots,PetscInt nleaves,const PetscInt *ilocal,PetscCopyMode localmode,const PetscSFNode *iremote,PetscCopyMode remotemode)
281: {
282:   PetscErrorCode     ierr;
283:   PetscTable         table;
284:   PetscTablePosition pos;
285:   PetscMPIInt        size;
286:   PetscInt           i,*rcount,*ranks;

292:   if (nroots < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"roots %D, cannot be negative",nroots);
293:   if (nleaves < 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"nleaves %D, cannot be negative",nleaves);
294:   PetscSFReset(sf);
295:   sf->nroots  = nroots;
296:   sf->nleaves = nleaves;
297:   if (ilocal) {
298:     switch (localmode) {
299:     case PETSC_COPY_VALUES:
300:       PetscMalloc(nleaves*sizeof(*sf->mine),&sf->mine_alloc);
301:       sf->mine    = sf->mine_alloc;
302:       PetscMemcpy(sf->mine,ilocal,nleaves*sizeof(*sf->mine));
303:       sf->minleaf = PETSC_MAX_INT;
304:       sf->maxleaf = PETSC_MIN_INT;
305:       for (i=0; i<nleaves; i++) {
306:         sf->minleaf = PetscMin(sf->minleaf,ilocal[i]);
307:         sf->maxleaf = PetscMax(sf->maxleaf,ilocal[i]);
308:       }
309:       break;
310:     case PETSC_OWN_POINTER:
311:       sf->mine_alloc = (PetscInt*)ilocal;
312:       sf->mine       = sf->mine_alloc;
313:       break;
314:     case PETSC_USE_POINTER:
315:       sf->mine = (PetscInt*)ilocal;
316:       break;
317:     default: SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_OUTOFRANGE,"Unknown localmode");
318:     }
319:   }
320:   if (!ilocal || nleaves > 0) {
321:     sf->minleaf = 0;
322:     sf->maxleaf = nleaves - 1;
323:   }
324:   switch (remotemode) {
325:   case PETSC_COPY_VALUES:
326:     PetscMalloc(nleaves*sizeof(*sf->remote),&sf->remote_alloc);
327:     sf->remote = sf->remote_alloc;
328:     PetscMemcpy(sf->remote,iremote,nleaves*sizeof(*sf->remote));
329:     break;
330:   case PETSC_OWN_POINTER:
331:     sf->remote_alloc = (PetscSFNode*)iremote;
332:     sf->remote       = sf->remote_alloc;
333:     break;
334:   case PETSC_USE_POINTER:
335:     sf->remote = (PetscSFNode*)iremote;
336:     break;
337:   default: SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_OUTOFRANGE,"Unknown remotemode");
338:   }

340:   MPI_Comm_size(PetscObjectComm((PetscObject)sf),&size);
341:   PetscTableCreate(10,size,&table);
342:   for (i=0; i<nleaves; i++) {
343:     /* Log 1-based rank */
344:     PetscTableAdd(table,iremote[i].rank+1,1,ADD_VALUES);
345:   }
346:   PetscTableGetCount(table,&sf->nranks);
347:   PetscMalloc4(sf->nranks,PetscInt,&sf->ranks,sf->nranks+1,PetscInt,&sf->roffset,nleaves,PetscInt,&sf->rmine,nleaves,PetscInt,&sf->rremote);
348:   PetscMalloc2(sf->nranks,PetscInt,&rcount,sf->nranks,PetscInt,&ranks);
349:   PetscTableGetHeadPosition(table,&pos);
350:   for (i=0; i<sf->nranks; i++) {
351:     PetscTableGetNext(table,&pos,&ranks[i],&rcount[i]);
352:     ranks[i]--;             /* Convert back to 0-based */
353:   }
354:   PetscTableDestroy(&table);
355:   PetscSortIntWithArray(sf->nranks,ranks,rcount);
356:   sf->roffset[0] = 0;
357:   for (i=0; i<sf->nranks; i++) {
358:     PetscMPIIntCast(ranks[i],sf->ranks+i);
359:     sf->roffset[i+1] = sf->roffset[i] + rcount[i];
360:     rcount[i]        = 0;
361:   }
362:   for (i=0; i<nleaves; i++) {
363:     PetscInt lo,hi,irank;
364:     /* Search for index of iremote[i].rank in sf->ranks */
365:     lo = 0; hi = sf->nranks;
366:     while (hi - lo > 1) {
367:       PetscInt mid = lo + (hi - lo)/2;
368:       if (iremote[i].rank < sf->ranks[mid]) hi = mid;
369:       else                                  lo = mid;
370:     }
371:     if (hi - lo == 1 && iremote[i].rank == sf->ranks[lo]) irank = lo;
372:     else SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Could not find rank %D in array",iremote[i].rank);
373:     sf->rmine[sf->roffset[irank] + rcount[irank]]   = ilocal ? ilocal[i] : i;
374:     sf->rremote[sf->roffset[irank] + rcount[irank]] = iremote[i].index;
375:     rcount[irank]++;
376:   }
377:   PetscFree2(rcount,ranks);
378: #if !defined(PETSC_USE_64BIT_INDICES)
379:   if (nroots == PETSC_DETERMINE) {
380:     /* Jed, if you have a better way to do this, put it in */
381:     PetscInt *numRankLeaves, *leafOff, *leafIndices, *numRankRoots, *rootOff, *rootIndices, maxRoots = 0;

383:     /* All to all to determine number of leaf indices from each (you can do this using Scan and asynch messages) */
384:     PetscMalloc4(size,PetscInt,&numRankLeaves,size+1,PetscInt,&leafOff,size,PetscInt,&numRankRoots,size+1,PetscInt,&rootOff);
385:     PetscMemzero(numRankLeaves, size * sizeof(PetscInt));
386:     for (i = 0; i < nleaves; ++i) ++numRankLeaves[iremote[i].rank];
387:     MPI_Alltoall(numRankLeaves, 1, MPIU_INT, numRankRoots, 1, MPIU_INT, PetscObjectComm((PetscObject)sf));
388:     /* Could set nroots to this maximum */
389:     for (i = 0; i < size; ++i) maxRoots += numRankRoots[i];

391:     /* Gather all indices */
392:     PetscMalloc2(nleaves,PetscInt,&leafIndices,maxRoots,PetscInt,&rootIndices);
393:     leafOff[0] = 0;
394:     for (i = 0; i < size; ++i) leafOff[i+1] = leafOff[i] + numRankLeaves[i];
395:     for (i = 0; i < nleaves; ++i) leafIndices[leafOff[iremote[i].rank]++] = iremote[i].index;
396:     leafOff[0] = 0;
397:     for (i = 0; i < size; ++i) leafOff[i+1] = leafOff[i] + numRankLeaves[i];
398:     rootOff[0] = 0;
399:     for (i = 0; i < size; ++i) rootOff[i+1] = rootOff[i] + numRankRoots[i];
400:     MPI_Alltoallv(leafIndices, numRankLeaves, leafOff, MPIU_INT, rootIndices, numRankRoots, rootOff, MPIU_INT, PetscObjectComm((PetscObject)sf));
401:     /* Sort and reduce */
402:     PetscSortRemoveDupsInt(&maxRoots, rootIndices);
403:     PetscFree2(leafIndices,rootIndices);
404:     PetscFree4(numRankLeaves,leafOff,numRankRoots,rootOff);
405:     sf->nroots = maxRoots;
406:   }
407: #endif

409:   sf->graphset = PETSC_TRUE;
410:   return(0);
411: }

415: /*@C
416:    PetscSFCreateInverseSF - given a PetscSF in which all vertices have degree 1, creates the inverse map

418:    Collective

420:    Input Arguments:
421: .  sf - star forest to invert

423:    Output Arguments:
424: .  isf - inverse of sf

426:    Level: advanced

428:    Notes:
429:    All roots must have degree 1.

431:    The local space may be a permutation, but cannot be sparse.

433: .seealso: PetscSFSetGraph()
434: @*/
435: PetscErrorCode PetscSFCreateInverseSF(PetscSF sf,PetscSF *isf)
436: {
438:   PetscMPIInt    rank;
439:   PetscInt       i,nroots,nleaves,maxlocal,count,*newilocal;
440:   const PetscInt *ilocal;
441:   PetscSFNode    *roots,*leaves;

444:   MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
445:   PetscSFGetGraph(sf,&nroots,&nleaves,&ilocal,NULL);
446:   for (i=0,maxlocal=0; i<nleaves; i++) maxlocal = PetscMax(maxlocal,(ilocal ? ilocal[i] : i)+1);
447:   PetscMalloc2(nroots,PetscSFNode,&roots,nleaves,PetscSFNode,&leaves);
448:   for (i=0; i<nleaves; i++) {
449:     leaves[i].rank  = rank;
450:     leaves[i].index = i;
451:   }
452:   for (i=0; i <nroots; i++) {
453:     roots[i].rank  = -1;
454:     roots[i].index = -1;
455:   }
456:   PetscSFReduceBegin(sf,MPIU_2INT,leaves,roots,MPIU_REPLACE);
457:   PetscSFReduceEnd(sf,MPIU_2INT,leaves,roots,MPIU_REPLACE);

459:   /* Check whether our leaves are sparse */
460:   for (i=0,count=0; i<nroots; i++) if (roots[i].rank >= 0) count++;
461:   if (count == nroots) newilocal = NULL;
462:   else {                        /* Index for sparse leaves and compact "roots" array (which is to become our leaves). */
463:     PetscMalloc(count*sizeof(PetscInt),&newilocal);
464:     for (i=0,count=0; i<nroots; i++) {
465:       if (roots[i].rank >= 0) {
466:         newilocal[count]   = i;
467:         roots[count].rank  = roots[i].rank;
468:         roots[count].index = roots[i].index;
469:         count++;
470:       }
471:     }
472:   }

474:   PetscSFDuplicate(sf,PETSCSF_DUPLICATE_CONFONLY,isf);
475:   PetscSFSetGraph(*isf,maxlocal,count,newilocal,PETSC_OWN_POINTER,roots,PETSC_COPY_VALUES);
476:   PetscFree2(roots,leaves);
477:   return(0);
478: }

482: /*@
483:    PetscSFDuplicate - duplicate a PetscSF, optionally preserving rank connectivity and graph

485:    Collective

487:    Input Arguments:
488: +  sf - communication object to duplicate
489: -  opt - PETSCSF_DUPLICATE_CONFONLY, PETSCSF_DUPLICATE_RANKS, or PETSCSF_DUPLICATE_GRAPH (see PetscSFDuplicateOption)

491:    Output Arguments:
492: .  newsf - new communication object

494:    Level: beginner

496: .seealso: PetscSFCreate(), PetscSFSetType(), PetscSFSetGraph()
497: @*/
498: PetscErrorCode PetscSFDuplicate(PetscSF sf,PetscSFDuplicateOption opt,PetscSF *newsf)
499: {

503:   PetscSFCreate(PetscObjectComm((PetscObject)sf),newsf);
504:   PetscSFSetType(*newsf,((PetscObject)sf)->type_name);
505:   if (sf->ops->Duplicate) {(*sf->ops->Duplicate)(sf,opt,*newsf);}
506:   if (opt == PETSCSF_DUPLICATE_GRAPH) {
507:     PetscInt          nroots,nleaves;
508:     const PetscInt    *ilocal;
509:     const PetscSFNode *iremote;
510:     PetscSFGetGraph(sf,&nroots,&nleaves,&ilocal,&iremote);
511:     PetscSFSetGraph(*newsf,nroots,nleaves,ilocal,PETSC_COPY_VALUES,iremote,PETSC_COPY_VALUES);
512:   }
513:   return(0);
514: }

518: /*@C
519:    PetscSFGetGraph - Get the graph specifying a parallel star forest

521:    Not Collective

523:    Input Arguments:
524: .  sf - star forest

526:    Output Arguments:
527: +  nroots - number of root vertices on the current process (these are possible targets for other process to attach leaves)
528: .  nleaves - number of leaf vertices on the current process, each of these references a root on any process
529: .  ilocal - locations of leaves in leafdata buffers
530: -  iremote - remote locations of root vertices for each leaf on the current process

532:    Level: intermediate

534: .seealso: PetscSFCreate(), PetscSFView(), PetscSFSetGraph()
535: @*/
536: PetscErrorCode PetscSFGetGraph(PetscSF sf,PetscInt *nroots,PetscInt *nleaves,const PetscInt **ilocal,const PetscSFNode **iremote)
537: {

541:   /* We are not currently requiring that the graph is set, thus returning nroots=-1 if it has not been set */
542:   /* if (!sf->graphset) SETERRQ(PetscObjectComm((PetscObject)sf),PETSC_ERR_ARG_WRONGSTATE,"Graph has not been set, must call PetscSFSetGraph()"); */
543:   if (nroots) *nroots = sf->nroots;
544:   if (nleaves) *nleaves = sf->nleaves;
545:   if (ilocal) *ilocal = sf->mine;
546:   if (iremote) *iremote = sf->remote;
547:   return(0);
548: }

552: /*@C
553:    PetscSFGetLeafRange - Get the active leaf ranges

555:    Not Collective

557:    Input Arguments:
558: .  sf - star forest

560:    Output Arguments:
561: +  minleaf - minimum active leaf on this process
562: -  maxleaf - maximum active leaf on this process

564:    Level: developer

566: .seealso: PetscSFCreate(), PetscSFView(), PetscSFSetGraph(), PetscSFGetGraph()
567: @*/
568: PetscErrorCode PetscSFGetLeafRange(PetscSF sf,PetscInt *minleaf,PetscInt *maxleaf)
569: {

573:   if (minleaf) *minleaf = sf->minleaf;
574:   if (maxleaf) *maxleaf = sf->maxleaf;
575:   return(0);
576: }

580: /*@C
581:    PetscSFView - view a star forest

583:    Collective

585:    Input Arguments:
586: +  sf - star forest
587: -  viewer - viewer to display graph, for example PETSC_VIEWER_STDOUT_WORLD

589:    Level: beginner

591: .seealso: PetscSFCreate(), PetscSFSetGraph()
592: @*/
593: PetscErrorCode PetscSFView(PetscSF sf,PetscViewer viewer)
594: {
595:   PetscErrorCode    ierr;
596:   PetscBool         iascii;
597:   PetscViewerFormat format;

601:   if (!viewer) {PetscViewerASCIIGetStdout(PetscObjectComm((PetscObject)sf),&viewer);}
604:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
605:   if (iascii) {
606:     PetscMPIInt rank;
607:     PetscInt    i,j;

609:     PetscObjectPrintClassNamePrefixType((PetscObject)sf,viewer,"Star Forest Object");
610:     PetscViewerASCIIPushTab(viewer);
611:     if (sf->ops->View) {(*sf->ops->View)(sf,viewer);}
612:     MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
613:     PetscViewerASCIISynchronizedAllow(viewer,PETSC_TRUE);
614:     PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of roots=%D, leaves=%D, remote ranks=%D\n",rank,sf->nroots,sf->nleaves,sf->nranks);
615:     for (i=0; i<sf->nleaves; i++) {
616:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %D <- (%D,%D)\n",rank,sf->mine ? sf->mine[i] : i,sf->remote[i].rank,sf->remote[i].index);
617:     }
618:     PetscViewerFlush(viewer);
619:     PetscViewerGetFormat(viewer,&format);
620:     if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
621:       PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Roots referenced by my leaves, by rank\n",rank);
622:       for (i=0; i<sf->nranks; i++) {
623:         PetscViewerASCIISynchronizedPrintf(viewer,"[%d] %D: %D edges\n",rank,sf->ranks[i],sf->roffset[i+1]-sf->roffset[i]);
624:         for (j=sf->roffset[i]; j<sf->roffset[i+1]; j++) {
625:           PetscViewerASCIISynchronizedPrintf(viewer,"[%d]    %D <- %D\n",rank,sf->rmine[j],sf->rremote[j]);
626:         }
627:       }
628:     }
629:     PetscViewerFlush(viewer);
630:     PetscViewerASCIISynchronizedAllow(viewer,PETSC_FALSE);
631:     PetscViewerASCIIPopTab(viewer);
632:   }
633:   return(0);
634: }

638: /*@C
639:    PetscSFGetRanks - Get ranks and number of vertices referenced by leaves on this process

641:    Not Collective

643:    Input Arguments:
644: .  sf - star forest

646:    Output Arguments:
647: +  nranks - number of ranks referenced by local part
648: .  ranks - array of ranks
649: .  roffset - offset in rmine/rremote for each rank (length nranks+1)
650: .  rmine - concatenated array holding local indices referencing each remote rank
651: -  rremote - concatenated array holding remote indices referenced for each remote rank

653:    Level: developer

655: .seealso: PetscSFSetGraph()
656: @*/
657: PetscErrorCode PetscSFGetRanks(PetscSF sf,PetscInt *nranks,const PetscMPIInt **ranks,const PetscInt **roffset,const PetscInt **rmine,const PetscInt **rremote)
658: {

662:   if (nranks)  *nranks  = sf->nranks;
663:   if (ranks)   *ranks   = sf->ranks;
664:   if (roffset) *roffset = sf->roffset;
665:   if (rmine)   *rmine   = sf->rmine;
666:   if (rremote) *rremote = sf->rremote;
667:   return(0);
668: }

672: /*@C
673:    PetscSFGetGroups - gets incoming and outgoing process groups

675:    Collective

677:    Input Argument:
678: .  sf - star forest

680:    Output Arguments:
681: +  incoming - group of origin processes for incoming edges (leaves that reference my roots)
682: -  outgoing - group of destination processes for outgoing edges (roots that I reference)

684:    Level: developer

686: .seealso: PetscSFGetWindow(), PetscSFRestoreWindow()
687: @*/
688: PetscErrorCode PetscSFGetGroups(PetscSF sf,MPI_Group *incoming,MPI_Group *outgoing)
689: {
691:   MPI_Group      group;

694:   if (sf->ingroup == MPI_GROUP_NULL) {
695:     PetscInt       i;
696:     const PetscInt *indegree;
697:     PetscMPIInt    rank,*outranks,*inranks;
698:     PetscSFNode    *remote;
699:     PetscSF        bgcount;

701:     /* Compute the number of incoming ranks */
702:     PetscMalloc(sf->nranks*sizeof(PetscSFNode),&remote);
703:     for (i=0; i<sf->nranks; i++) {
704:       remote[i].rank  = sf->ranks[i];
705:       remote[i].index = 0;
706:     }
707:     PetscSFDuplicate(sf,PETSCSF_DUPLICATE_CONFONLY,&bgcount);
708:     PetscSFSetGraph(bgcount,1,sf->nranks,NULL,PETSC_COPY_VALUES,remote,PETSC_OWN_POINTER);
709:     PetscSFComputeDegreeBegin(bgcount,&indegree);
710:     PetscSFComputeDegreeEnd(bgcount,&indegree);

712:     /* Enumerate the incoming ranks */
713:     PetscMalloc2(indegree[0],PetscMPIInt,&inranks,sf->nranks,PetscMPIInt,&outranks);
714:     MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
715:     for (i=0; i<sf->nranks; i++) outranks[i] = rank;
716:     PetscSFGatherBegin(bgcount,MPI_INT,outranks,inranks);
717:     PetscSFGatherEnd(bgcount,MPI_INT,outranks,inranks);
718:     MPI_Comm_group(PetscObjectComm((PetscObject)sf),&group);
719:     MPI_Group_incl(group,indegree[0],inranks,&sf->ingroup);
720:     MPI_Group_free(&group);
721:     PetscFree2(inranks,outranks);
722:     PetscSFDestroy(&bgcount);
723:   }
724:   *incoming = sf->ingroup;

726:   if (sf->outgroup == MPI_GROUP_NULL) {
727:     MPI_Comm_group(PetscObjectComm((PetscObject)sf),&group);
728:     MPI_Group_incl(group,sf->nranks,sf->ranks,&sf->outgroup);
729:     MPI_Group_free(&group);
730:   }
731:   *outgoing = sf->outgroup;
732:   return(0);
733: }

737: /*@C
738:    PetscSFGetMultiSF - gets the inner SF implemeting gathers and scatters

740:    Collective

742:    Input Argument:
743: .  sf - star forest that may contain roots with 0 or with more than 1 vertex

745:    Output Arguments:
746: .  multi - star forest with split roots, such that each root has degree exactly 1

748:    Level: developer

750:    Notes:

752:    In most cases, users should use PetscSFGatherBegin() and PetscSFScatterBegin() instead of manipulating multi
753:    directly. Since multi satisfies the stronger condition that each entry in the global space has exactly one incoming
754:    edge, it is a candidate for future optimization that might involve its removal.

756: .seealso: PetscSFSetGraph(), PetscSFGatherBegin(), PetscSFScatterBegin()
757: @*/
758: PetscErrorCode PetscSFGetMultiSF(PetscSF sf,PetscSF *multi)
759: {

765:   if (sf->nroots < 0) {         /* Graph has not been set yet; why do we need this? */
766:     PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);
767:     *multi = sf->multi;
768:     return(0);
769:   }
770:   if (!sf->multi) {
771:     const PetscInt *indegree;
772:     PetscInt       i,*inoffset,*outones,*outoffset;
773:     PetscSFNode    *remote;
774:     PetscSFComputeDegreeBegin(sf,&indegree);
775:     PetscSFComputeDegreeEnd(sf,&indegree);
776:     PetscMalloc3(sf->nroots+1,PetscInt,&inoffset,sf->nleaves,PetscInt,&outones,sf->nleaves,PetscInt,&outoffset);
777:     inoffset[0] = 0;
778: #if 1
779:     for (i=0; i<sf->nroots; i++) inoffset[i+1] = PetscMax(i+1, inoffset[i] + indegree[i]);
780: #else
781:     for (i=0; i<sf->nroots; i++) inoffset[i+1] = inoffset[i] + indegree[i];
782: #endif
783:     for (i=0; i<sf->nleaves; i++) outones[i] = 1;
784:     PetscSFFetchAndOpBegin(sf,MPIU_INT,inoffset,outones,outoffset,MPIU_SUM);
785:     PetscSFFetchAndOpEnd(sf,MPIU_INT,inoffset,outones,outoffset,MPIU_SUM);
786:     for (i=0; i<sf->nroots; i++) inoffset[i] -= indegree[i]; /* Undo the increment */
787: #if 0
788: #if defined(PETSC_USE_DEBUG)                                 /* Check that the expected number of increments occurred */
789:     for (i=0; i<sf->nroots; i++) {
790:       if (inoffset[i] + indegree[i] != inoffset[i+1]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Incorrect result after PetscSFFetchAndOp");
791:     }
792: #endif
793: #endif
794:     PetscMalloc(sf->nleaves*sizeof(*remote),&remote);
795:     for (i=0; i<sf->nleaves; i++) {
796:       remote[i].rank  = sf->remote[i].rank;
797:       remote[i].index = outoffset[i];
798:     }
799:     PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,&sf->multi);
800:     PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,NULL,PETSC_COPY_VALUES,remote,PETSC_OWN_POINTER);
801:     if (sf->rankorder) {        /* Sort the ranks */
802:       PetscMPIInt rank;
803:       PetscInt    *inranks,*newoffset,*outranks,*newoutoffset,*tmpoffset,maxdegree;
804:       PetscSFNode *newremote;
805:       MPI_Comm_rank(PetscObjectComm((PetscObject)sf),&rank);
806:       for (i=0,maxdegree=0; i<sf->nroots; i++) maxdegree = PetscMax(maxdegree,indegree[i]);
807:       PetscMalloc5(sf->multi->nroots,PetscInt,&inranks,sf->multi->nroots,PetscInt,&newoffset,sf->nleaves,PetscInt,&outranks,sf->nleaves,PetscInt,&newoutoffset,maxdegree,PetscInt,&tmpoffset);
808:       for (i=0; i<sf->nleaves; i++) outranks[i] = rank;
809:       PetscSFReduceBegin(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);
810:       PetscSFReduceEnd(sf->multi,MPIU_INT,outranks,inranks,MPIU_REPLACE);
811:       /* Sort the incoming ranks at each vertex, build the inverse map */
812:       for (i=0; i<sf->nroots; i++) {
813:         PetscInt j;
814:         for (j=0; j<indegree[i]; j++) tmpoffset[j] = j;
815:         PetscSortIntWithArray(indegree[i],inranks+inoffset[i],tmpoffset);
816:         for (j=0; j<indegree[i]; j++) newoffset[inoffset[i] + tmpoffset[j]] = inoffset[i] + j;
817:       }
818:       PetscSFBcastBegin(sf->multi,MPIU_INT,newoffset,newoutoffset);
819:       PetscSFBcastEnd(sf->multi,MPIU_INT,newoffset,newoutoffset);
820:       PetscMalloc(sf->nleaves*sizeof(*newremote),&newremote);
821:       for (i=0; i<sf->nleaves; i++) {
822:         newremote[i].rank  = sf->remote[i].rank;
823:         newremote[i].index = newoutoffset[i];
824:       }
825:       PetscSFSetGraph(sf->multi,inoffset[sf->nroots],sf->nleaves,NULL,PETSC_COPY_VALUES,newremote,PETSC_OWN_POINTER);
826:       PetscFree5(inranks,newoffset,outranks,newoutoffset,tmpoffset);
827:     }
828:     PetscFree3(inoffset,outones,outoffset);
829:   }
830:   *multi = sf->multi;
831:   return(0);
832: }

836: /*@C
837:    PetscSFCreateEmbeddedSF - removes edges from all but the selected roots, does not remap indices

839:    Collective

841:    Input Arguments:
842: +  sf - original star forest
843: .  nroots - number of roots to select on this process
844: -  selected - selected roots on this process

846:    Output Arguments:
847: .  newsf - new star forest

849:    Level: advanced

851:    Note:
852:    To use the new PetscSF, it may be necessary to know the indices of the leaves that are still participating. This can
853:    be done by calling PetscSFGetGraph().

855: .seealso: PetscSFSetGraph(), PetscSFGetGraph()
856: @*/
857: PetscErrorCode PetscSFCreateEmbeddedSF(PetscSF sf,PetscInt nroots,const PetscInt *selected,PetscSF *newsf)
858: {
860:   PetscInt       i,nleaves,*ilocal,*rootdata,*leafdata;
861:   PetscSFNode    *iremote;

867:   PetscMalloc2(sf->nroots,PetscInt,&rootdata,sf->nleaves,PetscInt,&leafdata);
868:   PetscMemzero(rootdata,sf->nroots*sizeof(PetscInt));
869:   PetscMemzero(leafdata,sf->nleaves*sizeof(PetscInt));
870:   for (i=0; i<nroots; i++) rootdata[selected[i]] = 1;
871:   PetscSFBcastBegin(sf,MPIU_INT,rootdata,leafdata);
872:   PetscSFBcastEnd(sf,MPIU_INT,rootdata,leafdata);

874:   for (i=0,nleaves=0; i<sf->nleaves; i++) nleaves += leafdata[i];
875:   PetscMalloc(nleaves*sizeof(PetscInt),&ilocal);
876:   PetscMalloc(nleaves*sizeof(PetscSFNode),&iremote);
877:   for (i=0,nleaves=0; i<sf->nleaves; i++) {
878:     if (leafdata[i]) {
879:       ilocal[nleaves]        = sf->mine ? sf->mine[i] : i;
880:       iremote[nleaves].rank  = sf->remote[i].rank;
881:       iremote[nleaves].index = sf->remote[i].index;
882:       nleaves++;
883:     }
884:   }
885:   PetscSFDuplicate(sf,PETSCSF_DUPLICATE_RANKS,newsf);
886:   PetscSFSetGraph(*newsf,sf->nroots,nleaves,ilocal,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);
887:   PetscFree2(rootdata,leafdata);
888:   return(0);
889: }

893: /*@C
894:    PetscSFBcastBegin - begin pointwise broadcast to be concluded with call to PetscSFBcastEnd()

896:    Collective on PetscSF

898:    Input Arguments:
899: +  sf - star forest on which to communicate
900: .  unit - data type associated with each node
901: -  rootdata - buffer to broadcast

903:    Output Arguments:
904: .  leafdata - buffer to update with values from each leaf's respective root

906:    Level: intermediate

908: .seealso: PetscSFCreate(), PetscSFSetGraph(), PetscSFView(), PetscSFBcastEnd(), PetscSFReduceBegin()
909: @*/
910: PetscErrorCode PetscSFBcastBegin(PetscSF sf,MPI_Datatype unit,const void *rootdata,void *leafdata)
911: {

916:   PetscSFCheckGraphSet(sf,1);
917:   PetscSFSetUp(sf);
918:   (*sf->ops->BcastBegin)(sf,unit,rootdata,leafdata);
919:   return(0);
920: }

924: /*@C
925:    PetscSFBcastEnd - end a broadcast operation started with PetscSFBcastBegin()

927:    Collective

929:    Input Arguments:
930: +  sf - star forest
931: .  unit - data type
932: -  rootdata - buffer to broadcast

934:    Output Arguments:
935: .  leafdata - buffer to update with values from each leaf's respective root

937:    Level: intermediate

939: .seealso: PetscSFSetGraph(), PetscSFReduceEnd()
940: @*/
941: PetscErrorCode PetscSFBcastEnd(PetscSF sf,MPI_Datatype unit,const void *rootdata,void *leafdata)
942: {

947:   PetscSFCheckGraphSet(sf,1);
948:   PetscSFSetUp(sf);
949:   (*sf->ops->BcastEnd)(sf,unit,rootdata,leafdata);
950:   return(0);
951: }

955: /*@C
956:    PetscSFReduceBegin - begin reduction of leafdata into rootdata, to be completed with call to PetscSFReduceEnd()

958:    Collective

960:    Input Arguments:
961: +  sf - star forest
962: .  unit - data type
963: .  leafdata - values to reduce
964: -  op - reduction operation

966:    Output Arguments:
967: .  rootdata - result of reduction of values from all leaves of each root

969:    Level: intermediate

971: .seealso: PetscSFBcastBegin()
972: @*/
973: PetscErrorCode PetscSFReduceBegin(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *rootdata,MPI_Op op)
974: {

979:   PetscSFCheckGraphSet(sf,1);
980:   PetscSFSetUp(sf);
981:   (sf->ops->ReduceBegin)(sf,unit,leafdata,rootdata,op);
982:   return(0);
983: }

987: /*@C
988:    PetscSFReduceEnd - end a reduction operation started with PetscSFReduceBegin()

990:    Collective

992:    Input Arguments:
993: +  sf - star forest
994: .  unit - data type
995: .  leafdata - values to reduce
996: -  op - reduction operation

998:    Output Arguments:
999: .  rootdata - result of reduction of values from all leaves of each root

1001:    Level: intermediate

1003: .seealso: PetscSFSetGraph(), PetscSFBcastEnd()
1004: @*/
1005: PetscErrorCode PetscSFReduceEnd(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *rootdata,MPI_Op op)
1006: {

1011:   PetscSFCheckGraphSet(sf,1);
1012:   PetscSFSetUp(sf);
1013:   (*sf->ops->ReduceEnd)(sf,unit,leafdata,rootdata,op);
1014:   return(0);
1015: }

1019: /*@C
1020:    PetscSFComputeDegreeBegin - begin computation of degree for each root vertex, to be completed with PetscSFComputeDegreeEnd()

1022:    Collective

1024:    Input Arguments:
1025: .  sf - star forest

1027:    Output Arguments:
1028: .  degree - degree of each root vertex

1030:    Level: advanced

1032: .seealso: PetscSFGatherBegin()
1033: @*/
1034: PetscErrorCode PetscSFComputeDegreeBegin(PetscSF sf,const PetscInt **degree)
1035: {

1040:   PetscSFCheckGraphSet(sf,1);
1042:   if (!sf->degree) {
1043:     PetscInt i;
1044:     PetscMalloc(sf->nroots*sizeof(PetscInt),&sf->degree);
1045:     PetscMalloc(sf->nleaves*sizeof(PetscInt),&sf->degreetmp);
1046:     for (i=0; i<sf->nroots; i++) sf->degree[i] = 0;
1047:     for (i=0; i<sf->nleaves; i++) sf->degreetmp[i] = 1;
1048:     PetscSFReduceBegin(sf,MPIU_INT,sf->degreetmp,sf->degree,MPIU_SUM);
1049:   }
1050:   *degree = NULL;
1051:   return(0);
1052: }

1056: /*@C
1057:    PetscSFComputeDegreeEnd - complete computation of degree for each root vertex, started with PetscSFComputeDegreeBegin()

1059:    Collective

1061:    Input Arguments:
1062: .  sf - star forest

1064:    Output Arguments:
1065: .  degree - degree of each root vertex

1067:    Level: developer

1069: .seealso:
1070: @*/
1071: PetscErrorCode PetscSFComputeDegreeEnd(PetscSF sf,const PetscInt **degree)
1072: {

1077:   PetscSFCheckGraphSet(sf,1);
1078:   if (!sf->degreeknown) {
1079:     PetscSFReduceEnd(sf,MPIU_INT,sf->degreetmp,sf->degree,MPIU_SUM);
1080:     PetscFree(sf->degreetmp);

1082:     sf->degreeknown = PETSC_TRUE;
1083:   }
1084:   *degree = sf->degree;
1085:   return(0);
1086: }

1090: /*@C
1091:    PetscSFFetchAndOpBegin - begin operation that fetches values from root and updates atomically by applying operation using my leaf value, to be completed with PetscSFFetchAndOpEnd()

1093:    Collective

1095:    Input Arguments:
1096: +  sf - star forest
1097: .  unit - data type
1098: .  leafdata - leaf values to use in reduction
1099: -  op - operation to use for reduction

1101:    Output Arguments:
1102: +  rootdata - root values to be updated, input state is seen by first process to perform an update
1103: -  leafupdate - state at each leaf's respective root immediately prior to my atomic update

1105:    Level: advanced

1107:    Note:
1108:    The update is only atomic at the granularity provided by the hardware. Different roots referenced by the same process
1109:    might be updated in a different order. Furthermore, if a composite type is used for the unit datatype, atomicity is
1110:    not guaranteed across the whole vertex. Therefore, this function is mostly only used with primitive types such as
1111:    integers.

1113: .seealso: PetscSFComputeDegreeBegin(), PetscSFReduceBegin(), PetscSFSetGraph()
1114: @*/
1115: PetscErrorCode PetscSFFetchAndOpBegin(PetscSF sf,MPI_Datatype unit,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op op)
1116: {

1121:   PetscSFCheckGraphSet(sf,1);
1122:   PetscSFSetUp(sf);
1123:   (*sf->ops->FetchAndOpBegin)(sf,unit,rootdata,leafdata,leafupdate,op);
1124:   return(0);
1125: }

1129: /*@C
1130:    PetscSFFetchAndOpEnd - end operation started in matching call to PetscSFFetchAndOpBegin() to fetch values from roots and update atomically by applying operation using my leaf value

1132:    Collective

1134:    Input Arguments:
1135: +  sf - star forest
1136: .  unit - data type
1137: .  leafdata - leaf values to use in reduction
1138: -  op - operation to use for reduction

1140:    Output Arguments:
1141: +  rootdata - root values to be updated, input state is seen by first process to perform an update
1142: -  leafupdate - state at each leaf's respective root immediately prior to my atomic update

1144:    Level: advanced

1146: .seealso: PetscSFComputeDegreeEnd(), PetscSFReduceEnd(), PetscSFSetGraph()
1147: @*/
1148: PetscErrorCode PetscSFFetchAndOpEnd(PetscSF sf,MPI_Datatype unit,void *rootdata,const void *leafdata,void *leafupdate,MPI_Op op)
1149: {

1154:   PetscSFCheckGraphSet(sf,1);
1155:   PetscSFSetUp(sf);
1156:   (*sf->ops->FetchAndOpEnd)(sf,unit,rootdata,leafdata,leafupdate,op);
1157:   return(0);
1158: }

1162: /*@C
1163:    PetscSFGatherBegin - begin pointwise gather of all leaves into multi-roots, to be completed with PetscSFGatherEnd()

1165:    Collective

1167:    Input Arguments:
1168: +  sf - star forest
1169: .  unit - data type
1170: -  leafdata - leaf data to gather to roots

1172:    Output Argument:
1173: .  multirootdata - root buffer to gather into, amount of space per root is equal to its degree

1175:    Level: intermediate

1177: .seealso: PetscSFComputeDegreeBegin(), PetscSFScatterBegin()
1178: @*/
1179: PetscErrorCode PetscSFGatherBegin(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *multirootdata)
1180: {
1182:   PetscSF        multi;

1186:   PetscSFGetMultiSF(sf,&multi);
1187:   PetscSFReduceBegin(multi,unit,leafdata,multirootdata,MPIU_REPLACE);
1188:   return(0);
1189: }

1193: /*@C
1194:    PetscSFGatherEnd - ends pointwise gather operation that was started with PetscSFGatherBegin()

1196:    Collective

1198:    Input Arguments:
1199: +  sf - star forest
1200: .  unit - data type
1201: -  leafdata - leaf data to gather to roots

1203:    Output Argument:
1204: .  multirootdata - root buffer to gather into, amount of space per root is equal to its degree

1206:    Level: intermediate

1208: .seealso: PetscSFComputeDegreeEnd(), PetscSFScatterEnd()
1209: @*/
1210: PetscErrorCode PetscSFGatherEnd(PetscSF sf,MPI_Datatype unit,const void *leafdata,void *multirootdata)
1211: {
1213:   PetscSF        multi;

1217:   PetscSFCheckGraphSet(sf,1);
1218:   PetscSFSetUp(sf);
1219:   PetscSFGetMultiSF(sf,&multi);
1220:   PetscSFReduceEnd(multi,unit,leafdata,multirootdata,MPIU_REPLACE);
1221:   return(0);
1222: }

1226: /*@C
1227:    PetscSFScatterBegin - begin pointwise scatter operation from multi-roots to leaves, to be completed with PetscSFScatterEnd()

1229:    Collective

1231:    Input Arguments:
1232: +  sf - star forest
1233: .  unit - data type
1234: -  multirootdata - root buffer to send to each leaf, one unit of data per leaf

1236:    Output Argument:
1237: .  leafdata - leaf data to be update with personal data from each respective root

1239:    Level: intermediate

1241: .seealso: PetscSFComputeDegreeBegin(), PetscSFScatterBegin()
1242: @*/
1243: PetscErrorCode PetscSFScatterBegin(PetscSF sf,MPI_Datatype unit,const void *multirootdata,void *leafdata)
1244: {
1246:   PetscSF        multi;

1250:   PetscSFCheckGraphSet(sf,1);
1251:   PetscSFSetUp(sf);
1252:   PetscSFGetMultiSF(sf,&multi);
1253:   PetscSFBcastBegin(multi,unit,multirootdata,leafdata);
1254:   return(0);
1255: }

1259: /*@C
1260:    PetscSFScatterEnd - ends pointwise scatter operation that was started with PetscSFScatterBegin()

1262:    Collective

1264:    Input Arguments:
1265: +  sf - star forest
1266: .  unit - data type
1267: -  multirootdata - root buffer to send to each leaf, one unit of data per leaf

1269:    Output Argument:
1270: .  leafdata - leaf data to be update with personal data from each respective root

1272:    Level: intermediate

1274: .seealso: PetscSFComputeDegreeEnd(), PetscSFScatterEnd()
1275: @*/
1276: PetscErrorCode PetscSFScatterEnd(PetscSF sf,MPI_Datatype unit,const void *multirootdata,void *leafdata)
1277: {
1279:   PetscSF        multi;

1283:   PetscSFCheckGraphSet(sf,1);
1284:   PetscSFSetUp(sf);
1285:   PetscSFGetMultiSF(sf,&multi);
1286:   PetscSFBcastEnd(multi,unit,multirootdata,leafdata);
1287:   return(0);
1288: }