Actual source code: subcomm.c

petsc-master 2016-05-26
Report Typos and Errors
  2: /*
  3:      Provides utility routines for split MPI communicator.
  4: */
  5: #include <petscsys.h>    /*I   "petscsys.h"    I*/
  6: #include <petscviewer.h>

  8: const char *const PetscSubcommTypes[] = {"GENERAL","CONTIGUOUS","INTERLACED","PetscSubcommType","PETSC_SUBCOMM_",0};

 10: extern PetscErrorCode PetscSubcommCreate_contiguous(PetscSubcomm);
 11: extern PetscErrorCode PetscSubcommCreate_interlaced(PetscSubcomm);

 15: /*@C
 16:    PetscSubcommSetFromOptions - Allows setting options from a PetscSubcomm

 18:    Collective on PetscSubcomm

 20:    Input Parameter:
 21: .  psubcomm - PetscSubcomm context

 23:    Level: beginner

 25: @*/
 26: PetscErrorCode PetscSubcommSetFromOptions(PetscSubcomm psubcomm)
 27: {
 28:   PetscErrorCode   ierr;
 29:   PetscSubcommType type;
 30:   PetscBool        flg;

 33:   if (!psubcomm) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NULL,"Must call PetscSubcommCreate firt");

 35:   PetscOptionsBegin(psubcomm->parent,psubcomm->subcommprefix,"Options for PetscSubcomm",NULL);
 36:   PetscOptionsEnum("-psubcomm_type",NULL,NULL,PetscSubcommTypes,(PetscEnum)psubcomm->type,(PetscEnum*)&type,&flg);
 37:   if (flg && psubcomm->type != type) {
 38:     /* free old structures */
 39:     PetscCommDestroy(&(psubcomm)->dupparent);
 40:     PetscCommDestroy(&(psubcomm)->child);
 41:     PetscFree((psubcomm)->subsize);
 42:     switch (type) {
 43:     case PETSC_SUBCOMM_GENERAL:
 44:       SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Runtime option PETSC_SUBCOMM_GENERAL is not supported, use PetscSubcommSetTypeGeneral()");
 45:     case PETSC_SUBCOMM_CONTIGUOUS:
 46:       PetscSubcommCreate_contiguous(psubcomm);
 47:       break;
 48:     case PETSC_SUBCOMM_INTERLACED:
 49:       PetscSubcommCreate_interlaced(psubcomm);
 50:       break;
 51:     default:
 52:       SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SUP,"PetscSubcommType %s is not supported yet",PetscSubcommTypes[type]);
 53:     }
 54:   }

 56:   PetscOptionsName("-psubcomm_view","Triggers display of PetscSubcomm context","PetscSubcommView",&flg);
 57:   if (flg) {
 58:     PetscSubcommView(psubcomm,PETSC_VIEWER_STDOUT_WORLD);
 59:   }
 60:   PetscOptionsEnd();
 61:   return(0);
 62: }

 66: /*@C
 67:   PetscSubcommSetOptionsPrefix - Sets the prefix used for searching for all
 68:   PetscSubcomm items in the options database.

 70:   Logically collective on PetscSubcomm.

 72:   Level: Intermediate

 74:   Input Parameters:
 75: +   psubcomm - PetscSubcomm context
 76: -   prefix - the prefix to prepend all PetscSubcomm item names with.

 78: @*/
 79: PetscErrorCode PetscSubcommSetOptionsPrefix(PetscSubcomm psubcomm,const char pre[])
 80: {
 81:   PetscErrorCode   ierr;

 84:    if (!pre) {
 85:     PetscFree(psubcomm->subcommprefix);
 86:   } else {
 87:     if (pre[0] == '-') SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Options prefix should not begin with a hypen");
 88:     PetscFree(psubcomm->subcommprefix);
 89:     PetscStrallocpy(pre,&(psubcomm->subcommprefix));
 90:   }
 91:   return(0);
 92: }

 96: /*@C
 97:    PetsSubcommcView - Views a PetscSubcomm of values as either ASCII text or a binary file

 99:    Collective on PetscSubcomm

101:    Input Parameter:
102: +  psubcomm - PetscSubcomm context
103: -  viewer - location to view the values

105:    Level: beginner
106: @*/
107: PetscErrorCode PetscSubcommView(PetscSubcomm psubcomm,PetscViewer viewer)
108: {
109:   PetscErrorCode    ierr;
110:   PetscBool         iascii;
111:   PetscViewerFormat format;

114:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
115:   if (iascii) {
116:     PetscViewerGetFormat(viewer,&format);
117:     if (format == PETSC_VIEWER_DEFAULT) {
118:       MPI_Comm    comm=psubcomm->parent;
119:       PetscMPIInt rank,size,subsize,subrank,duprank;

121:       MPI_Comm_size(comm,&size);
122:       PetscViewerASCIIPrintf(viewer,"PetscSubcomm type %s with total %d MPI processes:\n",PetscSubcommTypes[psubcomm->type],size);
123:       MPI_Comm_rank(comm,&rank);
124:       MPI_Comm_size(psubcomm->child,&subsize);
125:       MPI_Comm_rank(psubcomm->child,&subrank);
126:       MPI_Comm_rank(psubcomm->dupparent,&duprank);
127:       PetscSynchronizedPrintf(comm,"  [%d], color %d, sub-size %d, sub-rank %d, duprank %d\n",rank,psubcomm->color,subsize,subrank,duprank);
128:       PetscSynchronizedFlush(comm,PETSC_STDOUT);
129:     }
130:   } else SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Not supported yet");
131:   return(0);
132: }

136: /*@C
137:   PetscSubcommSetNumber - Set total number of subcommunicators.

139:    Collective on MPI_Comm

141:    Input Parameter:
142: +  psubcomm - PetscSubcomm context
143: -  nsubcomm - the total number of subcommunicators in psubcomm

145:    Level: advanced

147: .keywords: communicator

149: .seealso: PetscSubcommCreate(),PetscSubcommDestroy(),PetscSubcommSetType(),PetscSubcommSetTypeGeneral()
150: @*/
151: PetscErrorCode  PetscSubcommSetNumber(PetscSubcomm psubcomm,PetscInt nsubcomm)
152: {
154:   MPI_Comm       comm=psubcomm->parent;
155:   PetscMPIInt    msub,size;

158:   if (!psubcomm) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NULL,"PetscSubcomm is not created. Call PetscSubcommCreate() first");
159:   MPI_Comm_size(comm,&size);
160:   PetscMPIIntCast(nsubcomm,&msub);
161:   if (msub < 1 || msub > size) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE, "Num of subcommunicators %d cannot be < 1 or > input comm size %d",msub,size);

163:   psubcomm->n = msub;
164:   return(0);
165: }

169: /*@C
170:   PetscSubcommSetType - Set type of subcommunicators.

172:    Collective on MPI_Comm

174:    Input Parameter:
175: +  psubcomm - PetscSubcomm context
176: -  subcommtype - subcommunicator type, PETSC_SUBCOMM_CONTIGUOUS,PETSC_SUBCOMM_INTERLACED

178:    Level: advanced

180: .keywords: communicator

182: .seealso: PetscSubcommCreate(),PetscSubcommDestroy(),PetscSubcommSetNumber(),PetscSubcommSetTypeGeneral()
183: @*/
184: PetscErrorCode  PetscSubcommSetType(PetscSubcomm psubcomm,PetscSubcommType subcommtype)
185: {

189:   if (!psubcomm) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NULL,"PetscSubcomm is not created. Call PetscSubcommCreate()");
190:   if (psubcomm->n < 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"number of subcommunicators %d is incorrect. Call PetscSubcommSetNumber()",psubcomm->n);

192:   if (subcommtype == PETSC_SUBCOMM_CONTIGUOUS) {
193:     PetscSubcommCreate_contiguous(psubcomm);
194:   } else if (subcommtype == PETSC_SUBCOMM_INTERLACED) {
195:     PetscSubcommCreate_interlaced(psubcomm);
196:   } else SETERRQ1(psubcomm->parent,PETSC_ERR_SUP,"PetscSubcommType %s is not supported yet",PetscSubcommTypes[subcommtype]);
197:   return(0);
198: }

202: /*@C
203:   PetscSubcommSetTypeGeneral - Set a PetscSubcomm from user's specifications

205:    Collective on MPI_Comm

207:    Input Parameter:
208: +  psubcomm - PetscSubcomm context
209: .  color   - control of subset assignment (nonnegative integer). Processes with the same color are in the same subcommunicator.
210: -  subrank - rank in the subcommunicator

212:    Level: advanced

214: .keywords: communicator, create

216: .seealso: PetscSubcommCreate(),PetscSubcommDestroy(),PetscSubcommSetNumber(),PetscSubcommSetType()
217: @*/
218: PetscErrorCode PetscSubcommSetTypeGeneral(PetscSubcomm psubcomm,PetscMPIInt color,PetscMPIInt subrank)
219: {
221:   MPI_Comm       subcomm=0,dupcomm=0,comm=psubcomm->parent;
222:   PetscMPIInt    size,icolor,duprank,*recvbuf,sendbuf[3],mysubsize,rank,*subsize;
223:   PetscMPIInt    i,nsubcomm=psubcomm->n;

226:   if (!psubcomm) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NULL,"PetscSubcomm is not created. Call PetscSubcommCreate()");
227:   if (nsubcomm < 1) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"number of subcommunicators %d is incorrect. Call PetscSubcommSetNumber()",nsubcomm);

229:   MPI_Comm_split(comm,color,subrank,&subcomm);

231:   /* create dupcomm with same size as comm, but its rank, duprank, maps subcomm's contiguously into dupcomm */
232:   /* TODO: this can be done in an ostensibly scalale way (i.e., without allocating an array of size 'size') as is done in PetscObjectsCreateGlobalOrdering(). */
233:   MPI_Comm_size(comm,&size);
234:   PetscMalloc1(2*size,&recvbuf);

236:   MPI_Comm_rank(comm,&rank);
237:   MPI_Comm_size(subcomm,&mysubsize);

239:   sendbuf[0] = color;
240:   sendbuf[1] = mysubsize;
241:   MPI_Allgather(sendbuf,2,MPI_INT,recvbuf,2,MPI_INT,comm);

243:   PetscCalloc1(nsubcomm,&subsize);
244:   for (i=0; i<2*size; i+=2) {
245:     subsize[recvbuf[i]] = recvbuf[i+1];
246:   }
247:   PetscFree(recvbuf);

249:   duprank = 0;
250:   for (icolor=0; icolor<nsubcomm; icolor++) {
251:     if (icolor != color) { /* not color of this process */
252:       duprank += subsize[icolor];
253:     } else {
254:       duprank += subrank;
255:       break;
256:     }
257:   }
258:   MPI_Comm_split(comm,0,duprank,&dupcomm);

260:   PetscCommDuplicate(dupcomm,&psubcomm->dupparent,NULL);
261:   PetscCommDuplicate(subcomm,&psubcomm->child,NULL);
262:   MPI_Comm_free(&dupcomm);
263:   MPI_Comm_free(&subcomm);

265:   psubcomm->color   = color;
266:   psubcomm->subsize = subsize;
267:   psubcomm->type    = PETSC_SUBCOMM_GENERAL;
268:   return(0);
269: }

273: /*@C
274:   PetscSubcommDestroy - Destroys a PetscSubcomm object

276:    Collective on PetscSubcomm

278:    Input Parameter:
279:    .  psubcomm - the PetscSubcomm context

281:    Level: advanced

283: .seealso: PetscSubcommCreate(),PetscSubcommSetType()
284: @*/
285: PetscErrorCode  PetscSubcommDestroy(PetscSubcomm *psubcomm)
286: {

290:   if (!*psubcomm) return(0);
291:   PetscCommDestroy(&(*psubcomm)->dupparent);
292:   PetscCommDestroy(&(*psubcomm)->child);
293:   PetscFree((*psubcomm)->subsize);
294:   if ((*psubcomm)->subcommprefix) { PetscFree((*psubcomm)->subcommprefix); }
295:   PetscFree((*psubcomm));
296:   return(0);
297: }

301: /*@C
302:   PetscSubcommCreate - Create a PetscSubcomm context.

304:    Collective on MPI_Comm

306:    Input Parameter:
307: .  comm - MPI communicator

309:    Output Parameter:
310: .  psubcomm - location to store the PetscSubcomm context

312:    Level: advanced

314: .keywords: communicator, create

316: .seealso: PetscSubcommDestroy()
317: @*/
318: PetscErrorCode  PetscSubcommCreate(MPI_Comm comm,PetscSubcomm *psubcomm)
319: {
321:   PetscMPIInt    rank,size;

324:   PetscNew(psubcomm);

326:   /* set defaults */
327:   MPI_Comm_rank(comm,&rank);
328:   MPI_Comm_size(comm,&size);

330:   (*psubcomm)->parent    = comm;
331:   (*psubcomm)->dupparent = comm;
332:   (*psubcomm)->child     = PETSC_COMM_SELF;
333:   (*psubcomm)->n         = size;
334:   (*psubcomm)->color     = rank;
335:   (*psubcomm)->subsize   = NULL;
336:   (*psubcomm)->type      = PETSC_SUBCOMM_INTERLACED;
337:   return(0);
338: }

342: PetscErrorCode PetscSubcommCreate_contiguous(PetscSubcomm psubcomm)
343: {
345:   PetscMPIInt    rank,size,*subsize,duprank=-1,subrank=-1;
346:   PetscMPIInt    np_subcomm,nleftover,i,color=-1,rankstart,nsubcomm=psubcomm->n;
347:   MPI_Comm       subcomm=0,dupcomm=0,comm=psubcomm->parent;

350:   MPI_Comm_rank(comm,&rank);
351:   MPI_Comm_size(comm,&size);

353:   /* get size of each subcommunicator */
354:   PetscMalloc1(1+nsubcomm,&subsize);

356:   np_subcomm = size/nsubcomm;
357:   nleftover  = size - nsubcomm*np_subcomm;
358:   for (i=0; i<nsubcomm; i++) {
359:     subsize[i] = np_subcomm;
360:     if (i<nleftover) subsize[i]++;
361:   }

363:   /* get color and subrank of this proc */
364:   rankstart = 0;
365:   for (i=0; i<nsubcomm; i++) {
366:     if (rank >= rankstart && rank < rankstart+subsize[i]) {
367:       color   = i;
368:       subrank = rank - rankstart;
369:       duprank = rank;
370:       break;
371:     } else rankstart += subsize[i];
372:   }

374:   MPI_Comm_split(comm,color,subrank,&subcomm);

376:   /* create dupcomm with same size as comm, but its rank, duprank, maps subcomm's contiguously into dupcomm */
377:   MPI_Comm_split(comm,0,duprank,&dupcomm);
378:   PetscCommDuplicate(dupcomm,&psubcomm->dupparent,NULL);
379:   PetscCommDuplicate(subcomm,&psubcomm->child,NULL);
380:   MPI_Comm_free(&dupcomm);
381:   MPI_Comm_free(&subcomm);

383:   psubcomm->color   = color;
384:   psubcomm->subsize = subsize;
385:   psubcomm->type    = PETSC_SUBCOMM_CONTIGUOUS;
386:   return(0);
387: }

391: /*
392:    Note:
393:    In PCREDUNDANT, to avoid data scattering from subcomm back to original comm, we create subcommunicators
394:    by iteratively taking a process into a subcommunicator.
395:    Example: size=4, nsubcomm=(*psubcomm)->n=3
396:      comm=(*psubcomm)->parent:
397:       rank:     [0]  [1]  [2]  [3]
398:       color:     0    1    2    0

400:      subcomm=(*psubcomm)->comm:
401:       subrank:  [0]  [0]  [0]  [1]

403:      dupcomm=(*psubcomm)->dupparent:
404:       duprank:  [0]  [2]  [3]  [1]

406:      Here, subcomm[color = 0] has subsize=2, owns process [0] and [3]
407:            subcomm[color = 1] has subsize=1, owns process [1]
408:            subcomm[color = 2] has subsize=1, owns process [2]
409:            dupcomm has same number of processes as comm, and its duprank maps
410:            processes in subcomm contiguously into a 1d array:
411:             duprank: [0] [1]      [2]         [3]
412:             rank:    [0] [3]      [1]         [2]
413:                     subcomm[0] subcomm[1]  subcomm[2]
414: */

416: PetscErrorCode PetscSubcommCreate_interlaced(PetscSubcomm psubcomm)
417: {
419:   PetscMPIInt    rank,size,*subsize,duprank,subrank;
420:   PetscMPIInt    np_subcomm,nleftover,i,j,color,nsubcomm=psubcomm->n;
421:   MPI_Comm       subcomm=0,dupcomm=0,comm=psubcomm->parent;

424:   MPI_Comm_rank(comm,&rank);
425:   MPI_Comm_size(comm,&size);

427:   /* get size of each subcommunicator */
428:   PetscMalloc1(1+nsubcomm,&subsize);

430:   np_subcomm = size/nsubcomm;
431:   nleftover  = size - nsubcomm*np_subcomm;
432:   for (i=0; i<nsubcomm; i++) {
433:     subsize[i] = np_subcomm;
434:     if (i<nleftover) subsize[i]++;
435:   }

437:   /* find color for this proc */
438:   color   = rank%nsubcomm;
439:   subrank = rank/nsubcomm;

441:   MPI_Comm_split(comm,color,subrank,&subcomm);

443:   j = 0; duprank = 0;
444:   for (i=0; i<nsubcomm; i++) {
445:     if (j == color) {
446:       duprank += subrank;
447:       break;
448:     }
449:     duprank += subsize[i]; j++;
450:   }

452:   /* create dupcomm with same size as comm, but its rank, duprank, maps subcomm's contiguously into dupcomm */
453:   MPI_Comm_split(comm,0,duprank,&dupcomm);
454:   PetscCommDuplicate(dupcomm,&psubcomm->dupparent,NULL);
455:   PetscCommDuplicate(subcomm,&psubcomm->child,NULL);
456:   MPI_Comm_free(&dupcomm);
457:   MPI_Comm_free(&subcomm);

459:   psubcomm->color   = color;
460:   psubcomm->subsize = subsize;
461:   psubcomm->type    = PETSC_SUBCOMM_INTERLACED;
462:   return(0);
463: }