Actual source code: init.c

petsc-master 2020-08-05
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot be known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9:  #include <petscsys.h>
 10:  #include <petsc/private/petscimpl.h>
 11:  #include <petscvalgrind.h>
 12:  #include <petscviewer.h>
 13: #if defined(PETSC_USE_LOG)
 14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
 15: #endif

 17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 18: #include <sys/sysinfo.h>
 19: #endif
 20: #if defined(PETSC_HAVE_UNISTD_H)
 21: #include <unistd.h>
 22: #endif
 23: #if defined(PETSC_HAVE_CUDA)
 24: #include <cuda_runtime.h>
 25:  #include <petsccublas.h>
 26: #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
 27: #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
 28: #endif
 29: #endif

 31: #if defined(PETSC_HAVE_VIENNACL)
 32: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 33: #endif

 35: /* ------------------------Nasty global variables -------------------------------*/
 36: /*
 37:      Indicates if PETSc started up MPI, or it was
 38:    already started before PETSc was initialized.
 39: */
 40: PetscBool   PetscBeganMPI         = PETSC_FALSE;
 41: PetscBool   PetscErrorHandlingInitialized = PETSC_FALSE;
 42: PetscBool   PetscInitializeCalled = PETSC_FALSE;
 43: PetscBool   PetscFinalizeCalled   = PETSC_FALSE;
 44: PetscBool   PetscCUDAInitialized  = PETSC_FALSE;

 46: PetscMPIInt PetscGlobalRank       = -1;
 47: PetscMPIInt PetscGlobalSize       = -1;

 49: PetscBool   use_gpu_aware_mpi     = PETSC_TRUE;

 51: #if defined(PETSC_HAVE_COMPLEX)
 52: #if defined(PETSC_COMPLEX_INSTANTIATE)
 53: template <> class std::complex<double>; /* instantiate complex template class */
 54: #endif
 55: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 56: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 57: MPI_Datatype MPIU_C_COMPLEX;
 58: #endif

 60: /*MC
 61:    PETSC_i - the imaginary number i

 63:    Synopsis:
 64:  #include <petscsys.h>
 65:    PetscComplex PETSC_i;

 67:    Level: beginner

 69:    Note:
 70:    Complex numbers are automatically available if PETSc located a working complex implementation

 72: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 73: M*/
 74: PetscComplex PETSC_i;
 75: #endif
 76: #if defined(PETSC_USE_REAL___FLOAT128)
 77: MPI_Datatype MPIU___FLOAT128 = 0;
 78: #if defined(PETSC_HAVE_COMPLEX)
 79: MPI_Datatype MPIU___COMPLEX128 = 0;
 80: #endif
 81: #elif defined(PETSC_USE_REAL___FP16)
 82: MPI_Datatype MPIU___FP16 = 0;
 83: #endif
 84: MPI_Datatype MPIU_2SCALAR = 0;
 85: #if defined(PETSC_USE_64BIT_INDICES)
 86: MPI_Datatype MPIU_2INT = 0;
 87: #endif
 88: MPI_Datatype MPIU_BOOL;
 89: MPI_Datatype MPIU_ENUM;
 90: MPI_Datatype MPIU_FORTRANADDR;
 91: MPI_Datatype MPIU_SIZE_T;

 93: /*
 94:        Function that is called to display all error messages
 95: */
 96: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
 97: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
 98: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
 99: /*
100:   This is needed to turn on/off GPU synchronization
101: */
102: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
103: PetscBool PetscCUDASynchronize = PETSC_FALSE;

105: /* ------------------------------------------------------------------------------*/
106: /*
107:    Optional file where all PETSc output from various prints is saved
108: */
109: PETSC_INTERN FILE *petsc_history;
110: FILE *petsc_history = NULL;

112: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
113: {
115:   PetscMPIInt    rank,size;
116:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
117:   char           version[256];

120:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
121:   if (!rank) {
122:     char        arch[10];
123:     int         err;

125:     PetscGetArchType(arch,10);
126:     PetscGetDate(date,64);
127:     PetscGetVersion(version,256);
128:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
129:     if (filename) {
130:       PetscFixFilename(filename,fname);
131:     } else {
132:       PetscGetHomeDirectory(pfile,sizeof(pfile));
133:       PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));
134:       PetscFixFilename(pfile,fname);
135:     }

137:     *fd = fopen(fname,"a");
138:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

140:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
141:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
142:     PetscGetProgramName(pname,sizeof(pname));
143:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
144:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");

146:     err = fflush(*fd);
147:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
148:   }
149:   return(0);
150: }

152: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
153: {
155:   PetscMPIInt    rank;
156:   char           date[64];
157:   int            err;

160:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
161:   if (!rank) {
162:     PetscGetDate(date,64);
163:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
164:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
165:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
166:     err  = fflush(*fd);
167:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
168:     err = fclose(*fd);
169:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
170:   }
171:   return(0);
172: }

174: /* ------------------------------------------------------------------------------*/

176: /*
177:    This is ugly and probably belongs somewhere else, but I want to
178:   be able to put a true MPI abort error handler with command line args.

180:     This is so MPI errors in the debugger will leave all the stack
181:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
182:   in the debugger hence we call abort() instead of MPI_Abort().
183: */

185: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
186: {
188:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
189:   abort();
190: }

192: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
193: {

197:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
198:   PetscAttachDebugger();
199:   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
200: }

202: #if defined(PETSC_HAVE_CUDA)
203: /* CUDA validation after it is lazily initialized */
204: static PetscErrorCode PetscCUDAValidate(void)
205: {
206:   PetscBool             mpi_gpu_awareness;

209:   if (use_gpu_aware_mpi) {
210: #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
211:     /* Trust OpenMPI's compile time cuda query interface */
212:     mpi_gpu_awareness = PETSC_TRUE;
213: #else
214:     /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults.
215:       Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT.
216:     */
218: #endif
219:     if (!mpi_gpu_awareness) {
220:       (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n");
221:       (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n");
222:       (*PetscErrorPrintf)("If you do care, for IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n");
223:       (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n");
224:       (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n");
225:       (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n");
226:       PETSCABORT(PETSC_COMM_SELF,PETSC_ERR_LIB);
227:     }
228:   }
229:   return(0);
230: }

232: /* Initialize the CUDA device lazily just before creating the first CUDA object. */
233: static PetscBool PetscNotUseGpu = PETSC_FALSE; /* Assert the code will not use GPUs */
234: PetscErrorCode PetscCUDAInitializeLazily(void)
235: {
236:   PetscErrorCode        ierr;
237:   cudaError_t           cerr;
238:   int                   devId,devCount;
239:   PetscMPIInt           rank;
240:   static PetscBool      cudaValdidateChecked = PETSC_FALSE;

243:   if (PetscNotUseGpu) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"You asserted the code won't use GPUs with -cuda_set_device none, but it is trying to create a cuda object. Remove this option or see manpage of PetscCUDAInitialize().");
244:   if (!PetscCUDAInitialized) {
245:     cerr = cudaGetDeviceCount(&devCount);CHKERRCUDA(cerr);
246:     if (devCount > 1) {
247:       cerr = cudaSetDeviceFlags(cudaDeviceMapHost);
248:       cudaGetLastError(); /* Reset the last error */
249:       if (cerr == cudaSuccess) { /* It implies cuda runtime has not been initialized? */
250:         MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
251:         devId = rank % devCount;
252:         cerr  = cudaSetDevice(devId);CHKERRCUDA(cerr);
253:       } else if (cerr == cudaErrorSetOnActiveProcess) {
254:         /* It implies user has initialized cuda runtime outside of petsc. We do nothing to respect the device choice. */
255:       }
256:     }
257:     PetscCUDAInitialized = PETSC_TRUE;
258:   }
259:   if (!cudaValdidateChecked) {
260:     PetscCUDAValidate();
261:     cudaValdidateChecked = PETSC_TRUE;
262:   }
263:   return(0);
264: }

266: /*@C
267:      PetscCUDAInitialize - Initializes the CUDA device (eagerly in PetscInitialize()) and cuBLAS/cuSPARSE on the device

269:      Logically collective

271:   Input Parameter:
272:   comm   - the MPI communicator that will utilize the CUDA devices
273:   device - the device assigned to current MPI process. Special values like PETSC_DECIDE/DEFAULT have special meanings (see details below)

275:   Options Database:
276: +  -cuda_set_device <device> - the device assigned to current MPI rank. <device> is case-insensitive and can be:
277:        NONE (or none, or -3) : the code will not use GPUs, otherwise it will error out;
278:        PETSC_DEFAULT(or DEFAULT, or -2) : do not explicitly set device, i.e., use whatever device already set by user (probably before PetscInitialize()). Init cuda runtime etc;
279:        PETSC_DECIDE (or DECIDE, or -1) : assign MPI ranks in comm to available devices in round-robin, and init cuda runtime etc on the selected device;
280:        >= 0 integer  : assign the device with this id to current MPI process. Error out if <device> is invalid. Init cuda runtime etc on this device;
281:      With PETSC_{DECIDE, DEFAULT}, if there are actually no GPUs, the code can still run, but it will error out when trying to create cuda objects.
282: .  -cuda_view                - view information about the CUDA devices.
283: .  -cuda_synchronize         - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view.
284: .  -log_view                 - logging, however if alone or combined with `-cuda_set_device DEFAULT | DECIDE | >=0 int`, will int cuda; if combined with `-cuda_set_device none`, won't init cuda.
285: -  -use_gpu_aware_mpi        - assume the MPI is GPU-aware when communicating data on GPUs.

287:   Level: beginner

289:   Notes:
290:    Initializing cuBLAS takes about 1/2 second therefore it is done by default in PetscCUDAInitialize() before logging begins.

292: @*/
293: PetscErrorCode PetscCUDAInitialize(MPI_Comm comm,PetscInt device)
294: {
295:   PetscErrorCode        ierr;
296:   cudaError_t           cerr;
297:   int                   devId,devCount=0;
298:   const PetscInt        PETSC_NONE=-3; /* Unlike PETSC_DECIDE, we don't have a macro PETSC_NONE in petsc headers */
299:   PetscMPIInt           rank;

302:   if (!PetscCUDAInitialized) {
303:     cerr = cudaGetDeviceCount(&devCount);
304:     cudaGetLastError(); /* Reset the last error */
305:     if (cerr != cudaSuccess) devCount = 0;
306:     if (device >= 0) { /* User wants to use this specific device */
307:       cerr = cudaSetDeviceFlags(cudaDeviceMapHost); /* Allow it to fail since user might have already initialized the device. */
308:       cudaGetLastError(); /* Reset the last error */
309:       cerr = cudaSetDevice((int)device);CHKERRCUDA(cerr);
310:     } else if (device == PETSC_DECIDE) { /* Assign MPI ranks to available devices in round-robin */
311:       if (devCount > 0) { /* Allow no-GPU as long as user does not use GPUs */
312:         /* Set the device flags so that it can map host memory */
313:         cerr  = cudaSetDeviceFlags(cudaDeviceMapHost);CHKERRCUDA(cerr);
314:         MPI_Comm_rank(comm,&rank);
315:         devId = rank % devCount;
316:         cerr  = cudaSetDevice(devId);CHKERRCUDA(cerr);
317:       }
318:     } else if (device == PETSC_DEFAULT) {
319:       /* Do nothing, i.e., use whatever device set by user before PetscInitialize() */
320:     } else if (device == PETSC_NONE) {
321:       PetscNotUseGpu = PETSC_TRUE; /* Assert the code won't use GPUs even there are */
322:     } else SETERRQ1(comm,PETSC_ERR_ARG_OUTOFRANGE,"Wrong device (%D) passed to -cuda_set_device <dev>. Must be NONE(-3),PETSC_DEFAULT(-2),PETSC_DECIDE(-1) or a non-negative integer.",device);

324:     if (devCount > 0 && device != PETSC_NONE) {
325:       /* Do costly CUDA handles initialization here to not to distort petsc logging later */
326:       PetscCUBLASInitializeHandle();
327:       PetscCUSOLVERDnInitializeHandle();
328:       PetscCUDAInitialized = PETSC_TRUE;
329:     }
330:   }
331:   return(0);
332: }
333: #endif

335: /*@C
336:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
337:      wishes a clean exit somewhere deep in the program.

339:    Collective on PETSC_COMM_WORLD

341:    Options Database Keys are the same as for PetscFinalize()

343:    Level: advanced

345:    Note:
346:    See PetscInitialize() for more general runtime options.

348: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
349: @*/
350: PetscErrorCode  PetscEnd(void)
351: {
353:   PetscFinalize();
354:   exit(0);
355:   return 0;
356: }

358: PetscBool PetscOptionsPublish = PETSC_FALSE;
359: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
360: PETSC_INTERN PetscBool      petscsetmallocvisited;
361: static       char           emacsmachinename[256];

363: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
364: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;

366: /*@C
367:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
368:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
369:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

371:    Input Parameter:
372: +  help - the help function (may be NULL)
373: -  version - the version function (may be NULL)

375:    Level: developer

377: @*/
378: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
379: {
381:   PetscExternalHelpFunction    = help;
382:   PetscExternalVersionFunction = version;
383:   return(0);
384: }

386: #if defined(PETSC_USE_LOG)
387: PETSC_INTERN PetscBool   PetscObjectsLog;
388: #endif

390: void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
391: {
392:   if (PetscUnlikely(*err)) {
393:     PetscMPIInt len;
394:     char        errstring[MPI_MAX_ERROR_STRING];

396:     MPI_Error_string(*err,errstring,&len);
397:     PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
398:   }
399:   return;
400: }

402: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(const char help[])
403: {
404:   char              string[64];
405:   MPI_Comm          comm = PETSC_COMM_WORLD;
406:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp;
407:   PetscErrorCode    ierr;
408:   PetscReal         si;
409:   PetscInt          intensity;
410:   int               i;
411:   PetscMPIInt       rank;
412:   char              version[256];
413: #if defined(PETSC_USE_LOG)
414:   char              mname[PETSC_MAX_PATH_LEN];
415:   PetscViewerFormat format;
416:   PetscBool         flg4 = PETSC_FALSE;
417: #endif

420:   MPI_Comm_rank(comm,&rank);

422: #if !defined(PETSC_HAVE_THREADSAFETY)
423:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
424:     /*
425:       Setup the memory management; support for tracing malloc() usage
426:     */
427:     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

429:     if (PetscDefined(USE_DEBUG)) {
430:       mdebug        = PETSC_TRUE;
431:       initializenan = PETSC_TRUE;
432:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
433:     } else {
434:       /* don't warn about unused option */
435:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
436:       flg1 = PETSC_FALSE;
437:     }
438:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
439:     if (flg1 || flg2) {
440:       mdebug        = PETSC_TRUE;
441:       eachcall      = PETSC_TRUE;
442:       initializenan = PETSC_TRUE;
443:     } else if (flg3 && !flg2) {
444:       mdebug        = PETSC_FALSE;
445:       eachcall      = PETSC_FALSE;
446:       initializenan = PETSC_FALSE;
447:     }

449:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
450:     if (mlog) {
451:       mdebug = PETSC_TRUE;
452:     }
453:     /* the next line is deprecated */
454:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
455:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
456:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
457:     if (mdebug) {
458:       PetscMallocSetDebug(eachcall,initializenan);
459:     }
460:     if (mlog) {
461:       PetscReal logthreshold = 0;
462:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
463:       PetscMallocViewSet(logthreshold);
464:     }
465: #if defined(PETSC_USE_LOG)
466:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
467: #endif
468:   }

470:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
471:   if (flg2) {PetscMallocSetCoalesce(flg1);}
472:   flg1 = PETSC_FALSE;
473:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
474:   /* ignore this option if malloc is already set */
475:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

477:   flg1 = PETSC_FALSE;
478:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
479:   if (!flg1) {
480:     flg1 = PETSC_FALSE;
481:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
482:   }
483:   if (flg1) {
484:     PetscMemorySetGetMaximumUsage();
485:   }
486: #endif

488: #if defined(PETSC_USE_LOG)
489:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
490: #endif

492:   /*
493:       Set the display variable for graphics
494:   */
495:   PetscSetDisplay();

497:   /*
498:      Print main Section 1.5 Writing Application Codes with PETSc help message
499:   */
500:   PetscOptionsHasHelp(NULL,&hasHelp);
501:   if (help && hasHelp) {
502:     PetscPrintf(comm,help);
503:     PetscPrintf(comm,"----------------------------------------\n");
504:   }

506:   /*
507:       Print the PETSc version information
508:   */
509:   PetscOptionsHasName(NULL,NULL,"-version",&flg1);
510:   if (flg1 || hasHelp) {
511:     /*
512:        Print "higher-level" package version message
513:     */
514:     if (PetscExternalVersionFunction) {
515:       (*PetscExternalVersionFunction)(comm);
516:     }

518:     PetscGetVersion(version,256);
519:     (*PetscHelpPrintf)(comm,"%s\n",version);
520:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
521:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
522:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
523:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
524:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
525:     (*PetscHelpPrintf)(comm,"----------------------------------------\n");
526:   }

528:   /*
529:        Print "higher-level" package help message
530:   */
531:   if (hasHelp) {
532:     PetscBool hasHelpIntro;

534:     if (PetscExternalHelpFunction) {
535:       (*PetscExternalHelpFunction)(comm);
536:     }
537:     PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);
538:     if (hasHelpIntro) {
539:       PetscOptionsDestroyDefault();
540:       PetscFreeMPIResources();
541:       MPI_Finalize();
542:       exit(0);
543:     }
544:   }

546:   /*
547:       Setup the error handling
548:   */
549:   flg1 = PETSC_FALSE;
550:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
551:   if (flg1) {
552:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
553:     PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
554:   }
555:   flg1 = PETSC_FALSE;
556:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
557:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
558:   flg1 = PETSC_FALSE;
559:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
560:   if (flg1) {
561:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
562:   }
563:   /* experimental */
564:   flg1 = PETSC_FALSE;
565:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);
566:   if (flg1) {
567:     MPI_Errhandler eh;

569:     MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);
570:     MPI_Comm_set_errhandler(comm,eh);
571:     MPI_Errhandler_free(&eh);
572:   }
573:   flg1 = PETSC_FALSE;
574:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
575:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}

577:   /*
578:       Setup debugger information
579:   */
580:   PetscSetDefaultDebugger();
581:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);
582:   if (flg1) {
583:     MPI_Errhandler err_handler;

585:     PetscSetDebuggerFromString(string);
586:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
587:     MPI_Comm_set_errhandler(comm,err_handler);
588:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
589:   }
590:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);
591:   if (flg1) { PetscSetDebugTerminal(string); }
592:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);
593:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);
594:   if (flg1 || flg2) {
595:     PetscMPIInt    size;
596:     PetscInt       lsize,*nodes;
597:     MPI_Errhandler err_handler;
598:     /*
599:        we have to make sure that all processors have opened
600:        connections to all other processors, otherwise once the
601:        debugger has stated it is likely to receive a SIGUSR1
602:        and kill the program.
603:     */
604:     MPI_Comm_size(comm,&size);
605:     if (size > 2) {
606:       PetscMPIInt dummy = 0;
607:       MPI_Status  status;
608:       for (i=0; i<size; i++) {
609:         if (rank != i) {
610:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
611:         }
612:       }
613:       for (i=0; i<size; i++) {
614:         if (rank != i) {
615:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
616:         }
617:       }
618:     }
619:     /* check if this processor node should be in debugger */
620:     PetscMalloc1(size,&nodes);
621:     lsize = size;
622:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
623:     if (flag) {
624:       for (i=0; i<lsize; i++) {
625:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
626:       }
627:     }
628:     if (!flag) {
629:       PetscSetDebuggerFromString(string);
630:       PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
631:       if (flg1) {
632:         PetscAttachDebugger();
633:       } else {
634:         PetscStopForDebugger();
635:       }
636:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
637:       MPI_Comm_set_errhandler(comm,err_handler);
638:     }
639:     PetscFree(nodes);
640:   }

642:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);
643:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

645:   /*
646:         Setup profiling and logging
647:   */
648: #if defined(PETSC_USE_INFO)
649:   {
650:     PetscInfoSetFromOptions(NULL);
651:   }
652: #endif
653:   PetscDetermineInitialFPTrap();
654:   flg1 = PETSC_FALSE;
655:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
656:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
657:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
659: #if defined(PETSC_USE_LOG)
660:   mname[0] = 0;
661:   PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);
662:   if (flg1) {
663:     if (mname[0]) {
664:       PetscOpenHistoryFile(mname,&petsc_history);
665:     } else {
666:       PetscOpenHistoryFile(NULL,&petsc_history);
667:     }
668:   }

670:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

672: #if defined(PETSC_HAVE_MPE)
673:   flg1 = PETSC_FALSE;
674:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
675:   if (flg1) {PetscLogMPEBegin();}
676: #endif
677:   flg1 = PETSC_FALSE;
678:   flg3 = PETSC_FALSE;
679:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
680:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
681:   if (flg1)                      { PetscLogAllBegin(); }
682:   else if (flg3)                 { PetscLogDefaultBegin();}

684:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);
685:   if (flg1) {
686:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
687:     FILE *file;
688:     if (mname[0]) {
689:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
690:       PetscFixFilename(name,fname);
691:       file = fopen(fname,"w");
692:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
693:     } else file = PETSC_STDOUT;
694:     PetscLogTraceBegin(file);
695:   }

697:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
698:   if (flg4) {
699:     if (format == PETSC_VIEWER_ASCII_XML) {
700:       PetscLogNestedBegin();
701:     } else {
702:       PetscLogDefaultBegin();
703:     }
704:   }
705:   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
706:     PetscReal threshold = PetscRealConstant(0.01);
707:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
708:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
709:   }
710: #endif

712:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);

714: #if defined(PETSC_HAVE_CUDA)
715:   {
716:     cudaError_t           cerr;
717:     PetscBool             initCUDA = PETSC_FALSE,cudaView = PETSC_FALSE,logView = PETSC_FALSE,devNone = PETSC_FALSE;
718:     struct cudaDeviceProp prop;
719:     PetscInt              device;
720:     char                  devStr[16]={0};
721:     int                   devId,devCount;
722:     /*
723:       If collecting logging information, by default, wait for GPU to complete its operations
724:       before returning to the CPU in order to get accurate timings of each event
725:     */
726:     PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);
727:     if (!logView) {
728:       PetscOptionsHasName(NULL,NULL,"-log_view",&logView);
729:     }
730:     PetscCUDASynchronize = logView;

732:     PetscOptionsBegin(PETSC_COMM_WORLD,NULL,"CUDA options","Sys");
733:     PetscOptionsString("-cuda_set_device","Set which MPI ranks to use which CUDA devices","PetscCUDAInitialize",devStr,devStr,sizeof(devStr),&initCUDA);
734:     PetscStrcasecmp("none",devStr,&devNone);
735:     if (devNone) device = -3; /* -3 is the locally used PETSC_NONE in PetscCUDAInitialize() */
736:     else {PetscOptionsInt("-cuda_set_device","Set which MPI ranks to use which CUDA devices","PetscCUDAInitialize",device,&device,&initCUDA);}
737:     PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU (on by default with -log_summary or -log_view)",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);
738:     PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);
739:     PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cudaView);
740:     /* Get use_gpu_aware_mpi ASAP since it might be accessed even before lazy CUDA initialization */
741:     PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);
742:     PetscOptionsEnd();

744:     if (initCUDA) {PetscCUDAInitialize(PETSC_COMM_WORLD,device);}
745:     else if (logView) { /* With -log_view, we want to do costly cuda runtime initialization early so that not to distort the timing later. */
746:       devCount = 0;
747:       cerr = cudaGetDeviceCount(&devCount);
748:       cudaGetLastError(); /* Reset the last error */
749:       if (cerr == cudaSuccess && devCount >= 1) { /* There are GPU(s) */
750:         devId = 0;
751:         if (devCount > 1) { /* Decide which GPU to init when there are multiple GPUs */
752:           cerr = cudaSetDeviceFlags(cudaDeviceMapHost);
753:           cudaGetLastError(); /* Reset the last error */
754:           if (cerr == cudaSuccess) { /* It implies cuda runtime has not been initialized */
755:             MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
756:             devId = rank % devCount;
757:             cerr  = cudaSetDevice(devId);CHKERRCUDA(cerr);
758:           } else if (cerr == cudaErrorSetOnActiveProcess) {
759:             /* It means user initialized cuda runtime outside of petsc. We respect the device choice. */
760:             cerr = cudaGetDevice(&devId);CHKERRCUDA(cerr);
761:           }
762:         }
763:         PetscCUDAInitialize(PETSC_COMM_WORLD,(PetscInt)devId);
764:       }
765:     }

767:     if (cudaView) {
768:       MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
769:       cerr = cudaGetDeviceCount(&devCount);CHKERRCUDA(cerr);
770:       for (devId = 0; devId < devCount; ++devId) {
771:         cerr = cudaGetDeviceProperties(&prop,devId);CHKERRCUDA(cerr);
772:         PetscPrintf(PETSC_COMM_WORLD, "CUDA device %d: %s\n", devId, prop.name);
773:       }
774:       cerr = cudaGetDevice(&devId);CHKERRCUDA(cerr);
775:       PetscSynchronizedPrintf(PETSC_COMM_WORLD,"[%d] Using CUDA device %d.\n",rank,devId);
776:       PetscSynchronizedFlush(PETSC_COMM_WORLD,PETSC_STDOUT);
777:     }
778:   }
779: #endif

781:   /*
782:        Print basic help message
783:   */
784:   if (hasHelp) {
785:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
786:     (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");
787:     (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");
788:     (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");
789:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
790:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
791:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
792:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
793:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
794:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
795:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
796:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
797:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
798:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
799:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
800:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
801:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
802:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
803:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
804:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
805:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
806:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
807:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
808:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
809:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
810:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
811:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
812:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
813:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
814:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
815:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
816:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
817:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
818:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
819:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
820: #if defined(PETSC_USE_LOG)
821:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
822:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
823:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
824:     (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
825: #if defined(PETSC_HAVE_MPE)
826:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
827: #endif
828: #endif
829: #if defined(PETSC_USE_INFO)
830:     (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
831: #endif
832:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
833:     (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");
834:     (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");
835:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
836:   }

838: #if defined(PETSC_HAVE_POPEN)
839:   {
840:   char machine[128];
841:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);
842:   if (flg1) {
843:     PetscPOpenSetMachine(machine);
844:   }
845:   }
846: #endif

848:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
849:   if (flg1) {
850:     PetscSleep(si);
851:   }

853: #if defined(PETSC_HAVE_VIENNACL)
854:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
855:   if (!flg3) {
856:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
857:   }
858:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
859:   PetscViennaCLSynchronize = flg3;
860:   PetscViennaCLInit();
861: #endif

863:   /*
864:      Creates the logging data structures; this is enabled even if logging is not turned on
865:      This is the last thing we do before returning to the user code to prevent having the
866:      logging numbers contaminated by any startup time associated with MPI and the GPUs
867:   */
868: #if defined(PETSC_USE_LOG)
869:   PetscLogInitialize();
870: #endif

872:   return(0);
873: }