Actual source code: init.c

petsc-master 2020-06-05
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9:  #include <petscsys.h>
 10:  #include <petsc/private/petscimpl.h>
 11:  #include <petscvalgrind.h>
 12:  #include <petscviewer.h>
 13: #if defined(PETSC_USE_LOG)
 14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
 15: #endif

 17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 18: #include <sys/sysinfo.h>
 19: #endif
 20: #if defined(PETSC_HAVE_UNISTD_H)
 21: #include <unistd.h>
 22: #endif
 23: #if defined(PETSC_HAVE_CUDA)
 24: #include <cuda_runtime.h>
 25:  #include <petsccublas.h>
 26: #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
 27: #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
 28: #endif
 29: #endif

 31: #if defined(PETSC_HAVE_VIENNACL)
 32: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 33: #endif

 35: /* ------------------------Nasty global variables -------------------------------*/
 36: /*
 37:      Indicates if PETSc started up MPI, or it was
 38:    already started before PETSc was initialized.
 39: */
 40: PetscBool   PetscBeganMPI         = PETSC_FALSE;
 41: PetscBool   PetscInitializeCalled = PETSC_FALSE;
 42: PetscBool   PetscFinalizeCalled   = PETSC_FALSE;
 43: PetscBool   PetscCUDAInitialized  = PETSC_FALSE;

 45: PetscMPIInt PetscGlobalRank       = -1;
 46: PetscMPIInt PetscGlobalSize       = -1;

 48: PetscBool   use_gpu_aware_mpi     = PETSC_TRUE;

 50: #if defined(PETSC_HAVE_COMPLEX)
 51: #if defined(PETSC_COMPLEX_INSTANTIATE)
 52: template <> class std::complex<double>; /* instantiate complex template class */
 53: #endif
 54: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 55: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 56: MPI_Datatype MPIU_C_COMPLEX;
 57: #endif

 59: /*MC
 60:    PETSC_i - the imaginary number i

 62:    Synopsis:
 63:  #include <petscsys.h>
 64:    PetscComplex PETSC_i;

 66:    Level: beginner

 68:    Note:
 69:    Complex numbers are automatically available if PETSc located a working complex implementation

 71: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 72: M*/
 73: PetscComplex PETSC_i;
 74: #endif
 75: #if defined(PETSC_USE_REAL___FLOAT128)
 76: MPI_Datatype MPIU___FLOAT128 = 0;
 77: #if defined(PETSC_HAVE_COMPLEX)
 78: MPI_Datatype MPIU___COMPLEX128 = 0;
 79: #endif
 80: #elif defined(PETSC_USE_REAL___FP16)
 81: MPI_Datatype MPIU___FP16 = 0;
 82: #endif
 83: MPI_Datatype MPIU_2SCALAR = 0;
 84: #if defined(PETSC_USE_64BIT_INDICES)
 85: MPI_Datatype MPIU_2INT = 0;
 86: #endif
 87: MPI_Datatype MPIU_BOOL;
 88: MPI_Datatype MPIU_ENUM;
 89: MPI_Datatype MPIU_FORTRANADDR;
 90: MPI_Datatype MPIU_SIZE_T;

 92: /*
 93:        Function that is called to display all error messages
 94: */
 95: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
 96: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
 97: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
 98: /*
 99:   This is needed to turn on/off GPU synchronization
100: */
101: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
102: PetscBool PetscCUDASynchronize = PETSC_FALSE;

104: /* ------------------------------------------------------------------------------*/
105: /*
106:    Optional file where all PETSc output from various prints is saved
107: */
108: PETSC_INTERN FILE *petsc_history;
109: FILE *petsc_history = NULL;

111: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
112: {
114:   PetscMPIInt    rank,size;
115:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
116:   char           version[256];

119:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
120:   if (!rank) {
121:     char        arch[10];
122:     int         err;

124:     PetscGetArchType(arch,10);
125:     PetscGetDate(date,64);
126:     PetscGetVersion(version,256);
127:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
128:     if (filename) {
129:       PetscFixFilename(filename,fname);
130:     } else {
131:       PetscGetHomeDirectory(pfile,240);
132:       PetscStrcat(pfile,"/.petschistory");
133:       PetscFixFilename(pfile,fname);
134:     }

136:     *fd = fopen(fname,"a");
137:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

139:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
140:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
141:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
142:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
143:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");

145:     err = fflush(*fd);
146:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
147:   }
148:   return(0);
149: }

151: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
152: {
154:   PetscMPIInt    rank;
155:   char           date[64];
156:   int            err;

159:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
160:   if (!rank) {
161:     PetscGetDate(date,64);
162:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
163:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
164:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
165:     err  = fflush(*fd);
166:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
167:     err = fclose(*fd);
168:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
169:   }
170:   return(0);
171: }

173: /* ------------------------------------------------------------------------------*/

175: /*
176:    This is ugly and probably belongs somewhere else, but I want to
177:   be able to put a true MPI abort error handler with command line args.

179:     This is so MPI errors in the debugger will leave all the stack
180:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
181:   in the debugger hence we call abort() instead of MPI_Abort().
182: */

184: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
185: {
187:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
188:   abort();
189: }

191: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
192: {

196:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
197:   PetscAttachDebugger();
198:   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
199: }

201: #if defined(PETSC_HAVE_CUDA)
202: /*@C
203:      PetscCUDAInitialize - Initializes the CUDA device and cuBLAS on the device

205:      Logically collective

207:   Input Parameter:
208:   comm - the MPI communicator that will utilize the CUDA devices

210:   Options Database:
211: +  -cuda_initialize <yes,no> - Default no. Do the initialization in PetscInitialize(). If -cuda_initialize no is used then the default initialization is done automatically
212:                                when the first CUDA call is made unless you call PetscCUDAInitialize() before any CUDA operations are performed
213: .  -cuda_view - view information about the CUDA devices
214: .  -cuda_synchronize - wait at the end of asynchronize CUDA calls so that their time gets credited to the current event; default with -log_view
215: .  -cuda_set_device <gpu> - integer number of the device
216: -  -use_gpu_aware_mpi     - Assume the MPI is GPU-aware when communicating data on GPU

218:   Level: beginner

220:   Notes:
221:    Initializing cuBLAS takes about 1/2 second there it is done by default in PetscInitialize() before logging begins

223: @*/
224: PetscErrorCode PetscCUDAInitialize(MPI_Comm comm)
225: {
226:   PetscErrorCode        ierr;
227:   PetscInt              deviceOpt = 0;
228:   PetscBool             cuda_view_flag = PETSC_FALSE,flg;
229:   struct cudaDeviceProp prop;
230:   int                   devCount,device,devicecnt;
231:   cudaError_t           err = cudaSuccess;
232:   PetscMPIInt           rank,size;

235:   /*
236:      If collecting logging information, by default, wait for GPU to complete its operations
237:      before returning to the CPU in order to get accurate timings of each event
238:   */
239:   PetscOptionsHasName(NULL,NULL,"-log_summary",&PetscCUDASynchronize);
240:   if (!PetscCUDASynchronize) {
241:     PetscOptionsHasName(NULL,NULL,"-log_view",&PetscCUDASynchronize);
242:   }

244:   PetscOptionsBegin(comm,NULL,"CUDA options","Sys");
245:   PetscOptionsInt("-cuda_set_device","Set all MPI ranks to use the specified CUDA device",NULL,deviceOpt,&deviceOpt,&flg);
246:   device = (int)deviceOpt;
247:   PetscOptionsBool("-cuda_synchronize","Wait for the GPU to complete operations before returning to the CPU",NULL,PetscCUDASynchronize,&PetscCUDASynchronize,NULL);
248:   PetscOptionsDeprecated("-cuda_show_devices","-cuda_view","3.12",NULL);
249:   PetscOptionsName("-cuda_view","Display CUDA device information and assignments",NULL,&cuda_view_flag);
250:   PetscOptionsEnd();
251:   if (!PetscCUDAInitialized) {
252:     MPI_Comm_size(comm,&size);

254:     if (size>1 && !flg) {
255:       /* check to see if we force multiple ranks to hit the same GPU */
256:       /* we're not using the same GPU on multiple MPI threads. So try to allocated different   GPUs to different processes */

258:       /* First get the device count */
259:       err   = cudaGetDeviceCount(&devCount);
260:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));

262:       /* next determine the rank and then set the device via a mod */
263:       MPI_Comm_rank(comm,&rank);
264:       device = rank % devCount;
265:     }
266:     err = cudaSetDevice(device);
267:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));

269:     /* set the device flags so that it can map host memory */
270:     err = cudaSetDeviceFlags(cudaDeviceMapHost);
271:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));

273:     PetscCUBLASInitializeHandle();
274:     PetscCUSOLVERDnInitializeHandle();
275:     PetscCUDAInitialized = PETSC_TRUE;
276:   }
277:   if (cuda_view_flag) {
278:     MPI_Comm_rank(comm,&rank);
279:     err  = cudaGetDeviceCount(&devCount);
280:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
281:     for (devicecnt = 0; devicecnt < devCount; ++devicecnt) {
282:       err = cudaGetDeviceProperties(&prop,devicecnt);
283:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
284:       PetscPrintf(comm, "CUDA device %d: %s\n", devicecnt, prop.name);
285:     }
286:     PetscSynchronizedPrintf(comm,"[%d] Using CUDA device %d.\n",rank,device);
287:     PetscSynchronizedFlush(comm,PETSC_STDOUT);
288:   }
289:   return(0);
290: }
291: #endif

293: /*@C
294:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
295:      wishes a clean exit somewhere deep in the program.

297:    Collective on PETSC_COMM_WORLD

299:    Options Database Keys are the same as for PetscFinalize()

301:    Level: advanced

303:    Note:
304:    See PetscInitialize() for more general runtime options.

306: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
307: @*/
308: PetscErrorCode  PetscEnd(void)
309: {
311:   PetscFinalize();
312:   exit(0);
313:   return 0;
314: }

316: PetscBool PetscOptionsPublish = PETSC_FALSE;
317: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
318: PETSC_INTERN PetscBool      petscsetmallocvisited;
319: static       char           emacsmachinename[256];

321: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
322: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;

324: /*@C
325:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
326:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
327:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

329:    Input Parameter:
330: +  help - the help function (may be NULL)
331: -  version - the version function (may be NULL)

333:    Level: developer

335: @*/
336: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
337: {
339:   PetscExternalHelpFunction    = help;
340:   PetscExternalVersionFunction = version;
341:   return(0);
342: }

344: #if defined(PETSC_USE_LOG)
345: PETSC_INTERN PetscBool   PetscObjectsLog;
346: #endif

348: void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
349: {
350:   if (PetscUnlikely(*err)) {
351:     PetscMPIInt len;
352:     char        errstring[MPI_MAX_ERROR_STRING];

354:     MPI_Error_string(*err,errstring,&len);
355:     PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
356:   }
357:   return;
358: }

360: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(void)
361: {
362:   char              string[64];
363:   MPI_Comm          comm = PETSC_COMM_WORLD;
364:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
365:   PetscErrorCode    ierr;
366:   PetscReal         si;
367:   PetscInt          intensity;
368:   int               i;
369:   PetscMPIInt       rank;
370:   char              version[256],helpoptions[256];
371: #if defined(PETSC_USE_LOG)
372:   char              mname[PETSC_MAX_PATH_LEN];
373:   PetscViewerFormat format;
374:   PetscBool         flg4 = PETSC_FALSE;
375: #endif
376: #if defined(PETSC_HAVE_CUDA)
377:   PetscBool         initCUDA = PETSC_FALSE,mpi_gpu_awareness;
378:   cudaError_t       cerr;
379:   int               devCount = 0;
380: #endif

383:   MPI_Comm_rank(comm,&rank);

385: #if !defined(PETSC_HAVE_THREADSAFETY)
386:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
387:     /*
388:       Setup the memory management; support for tracing malloc() usage
389:     */
390:     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

392:     if (PetscDefined(USE_DEBUG)) {
393:       mdebug        = PETSC_TRUE;
394:       initializenan = PETSC_TRUE;
395:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
396:     } else {
397:       /* don't warn about unused option */
398:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
399:       flg1 = PETSC_FALSE;
400:     }
401:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
402:     if (flg1 || flg2) {
403:       mdebug        = PETSC_TRUE;
404:       eachcall      = PETSC_TRUE;
405:       initializenan = PETSC_TRUE;
406:     } else if (flg3 && !flg2) {
407:       mdebug        = PETSC_FALSE;
408:       eachcall      = PETSC_FALSE;
409:       initializenan = PETSC_FALSE;
410:     }

412:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
413:     if (mlog) {
414:       mdebug = PETSC_TRUE;
415:     }
416:     /* the next line is deprecated */
417:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
418:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
419:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
420:     if (mdebug) {
421:       PetscMallocSetDebug(eachcall,initializenan);
422:     }
423:     if (mlog) {
424:       PetscReal logthreshold = 0;
425:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
426:       PetscMallocViewSet(logthreshold);
427:     }
428: #if defined(PETSC_USE_LOG)
429:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
430: #endif
431:   }

433:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
434:   if (flg2) {PetscMallocSetCoalesce(flg1);}
435:   flg1 = PETSC_FALSE;
436:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
437:   /* ignore this option if malloc is already set */
438:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

440:   flg1 = PETSC_FALSE;
441:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
442:   if (!flg1) {
443:     flg1 = PETSC_FALSE;
444:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
445:   }
446:   if (flg1) {
447:     PetscMemorySetGetMaximumUsage();
448:   }
449: #endif

451: #if defined(PETSC_USE_LOG)
452:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
453: #endif

455:   /*
456:       Set the display variable for graphics
457:   */
458:   PetscSetDisplay();

460:   /*
461:       Print the PETSc version information
462:   */
463:   PetscOptionsHasName(NULL,NULL,"-v",&flg1);
464:   PetscOptionsHasName(NULL,NULL,"-version",&flg2);
465:   PetscOptionsHasHelp(NULL,&flg3);
466:   if (flg1 || flg2 || flg3) {

468:     /*
469:        Print "higher-level" package version message
470:     */
471:     if (PetscExternalVersionFunction) {
472:       (*PetscExternalVersionFunction)(comm);
473:     }

475:     PetscGetVersion(version,256);
476:     (*PetscHelpPrintf)(comm,"%s\n",version);
477:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
478:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
479:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
480:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
481:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
482:     (*PetscHelpPrintf)(comm,"----------------------------------------\n");
483:   }

485:   /*
486:        Print "higher-level" package help message
487:   */
488:   if (flg3) {
489:     if (PetscExternalHelpFunction) {
490:       (*PetscExternalHelpFunction)(comm);
491:     }
492:   }

494:   PetscOptionsGetString(NULL,NULL,"-help",helpoptions,sizeof(helpoptions),&flg1);
495:   if (flg1) {
496:     PetscStrcmp(helpoptions,"intro",&flg2);
497:     if (flg2) {
498:       PetscOptionsDestroyDefault();
499:       PetscFreeMPIResources();
500:       MPI_Finalize();
501:       exit(0);
502:     }
503:   }

505:   /*
506:       Setup the error handling
507:   */
508:   flg1 = PETSC_FALSE;
509:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
510:   if (flg1) {
511:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
512:     PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
513:   }
514:   flg1 = PETSC_FALSE;
515:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
516:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
517:   flg1 = PETSC_FALSE;
518:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
519:   if (flg1) {
520:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
521:   }
522:   /* experimental */
523:   flg1 = PETSC_FALSE;
524:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);
525:   if (flg1) {
526:     MPI_Errhandler eh;

528:     MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);
529:     MPI_Comm_set_errhandler(comm,eh);
530:     MPI_Errhandler_free(&eh);
531:   }
532:   flg1 = PETSC_FALSE;
533:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
534:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
535:   flg1 = PETSC_FALSE;
536:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
537:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
538:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);

541:   /*
542:       Setup debugger information
543:   */
544:   PetscSetDefaultDebugger();
545:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
546:   if (flg1) {
547:     MPI_Errhandler err_handler;

549:     PetscSetDebuggerFromString(string);
550:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
551:     MPI_Comm_set_errhandler(comm,err_handler);
552:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
553:   }
554:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
555:   if (flg1) { PetscSetDebugTerminal(string); }
556:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
557:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
558:   if (flg1 || flg2) {
559:     PetscMPIInt    size;
560:     PetscInt       lsize,*nodes;
561:     MPI_Errhandler err_handler;
562:     /*
563:        we have to make sure that all processors have opened
564:        connections to all other processors, otherwise once the
565:        debugger has stated it is likely to receive a SIGUSR1
566:        and kill the program.
567:     */
568:     MPI_Comm_size(comm,&size);
569:     if (size > 2) {
570:       PetscMPIInt dummy = 0;
571:       MPI_Status  status;
572:       for (i=0; i<size; i++) {
573:         if (rank != i) {
574:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
575:         }
576:       }
577:       for (i=0; i<size; i++) {
578:         if (rank != i) {
579:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
580:         }
581:       }
582:     }
583:     /* check if this processor node should be in debugger */
584:     PetscMalloc1(size,&nodes);
585:     lsize = size;
586:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
587:     if (flag) {
588:       for (i=0; i<lsize; i++) {
589:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
590:       }
591:     }
592:     if (!flag) {
593:       PetscSetDebuggerFromString(string);
594:       PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
595:       if (flg1) {
596:         PetscAttachDebugger();
597:       } else {
598:         PetscStopForDebugger();
599:       }
600:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
601:       MPI_Comm_set_errhandler(comm,err_handler);
602:     }
603:     PetscFree(nodes);
604:   }

606:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
607:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

609:   /*
610:         Setup profiling and logging
611:   */
612: #if defined(PETSC_USE_INFO)
613:   {
614:     PetscInfoSetFromOptions(NULL);
615:   }
616: #endif
617: #if defined(PETSC_USE_LOG)
618:   mname[0] = 0;
619:   PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
620:   if (flg1) {
621:     if (mname[0]) {
622:       PetscOpenHistoryFile(mname,&petsc_history);
623:     } else {
624:       PetscOpenHistoryFile(NULL,&petsc_history);
625:     }
626:   }

628:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

630: #if defined(PETSC_HAVE_MPE)
631:   flg1 = PETSC_FALSE;
632:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
633:   if (flg1) {PetscLogMPEBegin();}
634: #endif
635:   flg1 = PETSC_FALSE;
636:   flg3 = PETSC_FALSE;
637:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
638:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
639:   if (flg1)                      { PetscLogAllBegin(); }
640:   else if (flg3)                 { PetscLogDefaultBegin();}

642:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
643:   if (flg1) {
644:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
645:     FILE *file;
646:     if (mname[0]) {
647:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
648:       PetscFixFilename(name,fname);
649:       file = fopen(fname,"w");
650:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
651:     } else file = PETSC_STDOUT;
652:     PetscLogTraceBegin(file);
653:   }

655:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
656:   if (flg4) {
657:     if (format == PETSC_VIEWER_ASCII_XML) {
658:       PetscLogNestedBegin();
659:     } else {
660:       PetscLogDefaultBegin();
661:     }
662:   }
663:   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
664:     PetscReal threshold = PetscRealConstant(0.01);
665:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
666:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
667:   }
668: #endif

670:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);

672: #if defined(PETSC_HAVE_CUDA)
673:   PetscOptionsBegin(comm,NULL,"CUDA initialize","Sys");
674:   PetscOptionsBool("-cuda_initialize","Initialize the CUDA devices and cuBLAS during PetscInitialize()",NULL,initCUDA,&initCUDA,NULL);
675:   PetscOptionsBool("-use_gpu_aware_mpi","Use GPU-aware MPI",NULL,use_gpu_aware_mpi,&use_gpu_aware_mpi,NULL);
676:   PetscOptionsEnd();
677:   if (initCUDA) {PetscCUDAInitialize(PETSC_COMM_WORLD);}
678:   if (use_gpu_aware_mpi) {
679:     cerr = cudaGetDeviceCount(&devCount);{if (cerr != cudaErrorNoDevice) CHKERRCUDA(cerr);} /* Catch other errors */
680:     if (cerr == cudaErrorNoDevice) devCount = 0; /* CUDA does not say what devCount is under this error */
681:   }
682:   if (devCount > 0 && use_gpu_aware_mpi) { /* Only do the MPI GPU awareness check when there are GPU(s) */
683: #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION) && defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
684:     /* Trust OpenMPI's compile time cuda query interface */
685:     mpi_gpu_awareness = PETSC_TRUE;
686: #else
687:     /* For other MPI implementations without cuda query API, we do a GPU MPI call to see if it segfaults.
688:        Note that Spectrum MPI sets OMPI_MAJOR_VERSION and is CUDA-aware, but does not have MPIX_CUDA_AWARE_SUPPORT.
689:     */
691: #endif
692:     if (!mpi_gpu_awareness) {
693:       (*PetscErrorPrintf)("PETSc is configured with GPU support, but your MPI is not GPU-aware. For better performance, please use a GPU-aware MPI.\n");
694:       (*PetscErrorPrintf)("For IBM Spectrum MPI on OLCF Summit, you may need jsrun --smpiargs=-gpu.\n");
695:       (*PetscErrorPrintf)("For OpenMPI, you need to configure it --with-cuda (https://www.open-mpi.org/faq/?category=buildcuda)\n");
696:       (*PetscErrorPrintf)("For MVAPICH2-GDR, you need to set MV2_USE_CUDA=1 (http://mvapich.cse.ohio-state.edu/userguide/gdr/)\n");
697:       (*PetscErrorPrintf)("For Cray-MPICH, you need to set MPICH_RDMA_ENABLED_CUDA=1 (https://www.olcf.ornl.gov/tutorials/gpudirect-mpich-enabled-cuda/)\n");
698:       (*PetscErrorPrintf)("If you do not care, use option -use_gpu_aware_mpi 0, then PETSc will copy data from GPU to CPU for communication.\n");
699:       PETSCABORT(PETSC_COMM_WORLD,PETSC_ERR_LIB);
700:     }
701:   }
702: #endif

704:   /*
705:        Print basic help message
706:   */
707:   PetscOptionsHasHelp(NULL,&flg1);
708:   if (flg1) {
709:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
710:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
711:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
712:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
713:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
714:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
715:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
716:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
717:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
718:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
719:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
720:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
721:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
722:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
723:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
724:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
725:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
726:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
727:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
728:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
729:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
730:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
731:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
732:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
733:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
734:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
735:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
736:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
737:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
738:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
739:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
740:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
741:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
742: #if defined(PETSC_USE_LOG)
743:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
744:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
745:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
746:     (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
747: #if defined(PETSC_HAVE_MPE)
748:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
749: #endif
750: #endif
751: #if defined(PETSC_USE_INFO)
752:     (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
753: #endif
754:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
755:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
756:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
757:   }

759: #if defined(PETSC_HAVE_POPEN)
760:   {
761:   char machine[128];
762:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
763:   if (flg1) {
764:     PetscPOpenSetMachine(machine);
765:   }
766:   }
767: #endif

769:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
770:   if (flg1) {
771:     PetscSleep(si);
772:   }

774: #if defined(PETSC_HAVE_VIENNACL)
775:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
776:   if (!flg3) {
777:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
778:   }
779:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
780:   PetscViennaCLSynchronize = flg3;
781:   PetscViennaCLInit();
782: #endif

784:   /*
785:      Creates the logging data structures; this is enabled even if logging is not turned on
786:      This is the last thing we do before returning to the user code to prevent having the
787:      logging numbers contaminated by any startup time associated with MPI and the GPUs
788:   */
789: #if defined(PETSC_USE_LOG)
790:   PetscLogInitialize();
791: #endif

793:   return(0);
794: }