Actual source code: init.c

petsc-master 2017-05-24
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9:  #include <petscsys.h>
 10:  #include <petsc/private/petscimpl.h>
 11:  #include <petscvalgrind.h>
 12:  #include <petscviewer.h>

 14: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 15: #include <sys/sysinfo.h>
 16: #endif
 17: #if defined(PETSC_HAVE_UNISTD_H)
 18: #include <unistd.h>
 19: #endif
 20: #if defined(PETSC_HAVE_CUDA)
 21: #include <cuda_runtime.h>
 22: #endif

 24: #if defined(PETSC_HAVE_VIENNACL)
 25: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 26: #endif

 28: /* ------------------------Nasty global variables -------------------------------*/
 29: /*
 30:      Indicates if PETSc started up MPI, or it was
 31:    already started before PETSc was initialized.
 32: */
 33: PetscBool   PetscBeganMPI         = PETSC_FALSE;
 34: PetscBool   PetscInitializeCalled = PETSC_FALSE;
 35: PetscBool   PetscFinalizeCalled   = PETSC_FALSE;
 36: PetscBool   PetscCUDAInitialized  = PETSC_FALSE;

 38: PetscMPIInt PetscGlobalRank       = -1;
 39: PetscMPIInt PetscGlobalSize       = -1;

 41: #if defined(PETSC_HAVE_COMPLEX)
 42: #if defined(PETSC_COMPLEX_INSTANTIATE)
 43: template <> class std::complex<double>; /* instantiate complex template class */
 44: #endif
 45: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 46: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 47: MPI_Datatype MPIU_C_COMPLEX;
 48: #endif

 50: /*MC
 51:    PETSC_i - the imaginary number i

 53:    Synopsis:
 54:    #include <petscsys.h>
 55:    PetscComplex PETSC_i;

 57:    Level: beginner

 59:    Note:
 60:    Complex numbers are automatically available if PETSc located a working complex implementation

 62: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 63: M*/
 64: PetscComplex PETSC_i;
 65: #endif
 66: #if defined(PETSC_USE_REAL___FLOAT128)
 67: MPI_Datatype MPIU___FLOAT128 = 0;
 68: #if defined(PETSC_HAVE_COMPLEX)
 69: MPI_Datatype MPIU___COMPLEX128 = 0;
 70: #endif
 71: #elif defined(PETSC_USE_REAL___FP16)
 72: MPI_Datatype MPIU___FP16 = 0;
 73: #endif
 74: MPI_Datatype MPIU_2SCALAR = 0;
 75: #if defined(PETSC_USE_64BIT_INDICES) || !defined(MPI_2INT)
 76: MPI_Datatype MPIU_2INT = 0;
 77: #endif
 78: MPI_Datatype MPIU_BOOL;
 79: MPI_Datatype MPIU_ENUM;

 81: /*
 82:        Function that is called to display all error messages
 83: */
 84: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
 85: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
 86: #if defined(PETSC_HAVE_MATLAB_ENGINE)
 87: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintf_Matlab;
 88: #else
 89: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
 90: #endif
 91: /*
 92:   This is needed to turn on/off GPU synchronization
 93: */
 94: PetscBool PetscCUSPSynchronize = PETSC_FALSE;
 95: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
 96: PetscBool PetscCUDASynchronize = PETSC_FALSE;

 98: /* ------------------------------------------------------------------------------*/
 99: /*
100:    Optional file where all PETSc output from various prints is saved
101: */
102: FILE *petsc_history = NULL;

104: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
105: {
107:   PetscMPIInt    rank,size;
108:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
109:   char           version[256];

112:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
113:   if (!rank) {
114:     char        arch[10];
115:     int         err;

117:     PetscGetArchType(arch,10);
118:     PetscGetDate(date,64);
119:     PetscGetVersion(version,256);
120:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
121:     if (filename) {
122:       PetscFixFilename(filename,fname);
123:     } else {
124:       PetscGetHomeDirectory(pfile,240);
125:       PetscStrcat(pfile,"/.petschistory");
126:       PetscFixFilename(pfile,fname);
127:     }

129:     *fd = fopen(fname,"a");
130:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

132:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
133:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
134:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
135:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
136:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");

138:     err = fflush(*fd);
139:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
140:   }
141:   return(0);
142: }

144: PetscErrorCode  PetscCloseHistoryFile(FILE **fd)
145: {
147:   PetscMPIInt    rank;
148:   char           date[64];
149:   int            err;

152:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
153:   if (!rank) {
154:     PetscGetDate(date,64);
155:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
156:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
157:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
158:     err  = fflush(*fd);
159:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
160:     err = fclose(*fd);
161:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
162:   }
163:   return(0);
164: }

166: /* ------------------------------------------------------------------------------*/

168: /*
169:    This is ugly and probably belongs somewhere else, but I want to
170:   be able to put a true MPI abort error handler with command line args.

172:     This is so MPI errors in the debugger will leave all the stack
173:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
174:   in the debugger hence we call abort() instead of MPI_Abort().
175: */

177: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
178: {
180:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
181:   abort();
182: }

184: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
185: {

189:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
190:   PetscAttachDebugger();
191:   if (ierr) MPI_Abort(*comm,*flag); /* hopeless so get out */
192: }

194: /*@C
195:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
196:      wishes a clean exit somewhere deep in the program.

198:    Collective on PETSC_COMM_WORLD

200:    Options Database Keys are the same as for PetscFinalize()

202:    Level: advanced

204:    Note:
205:    See PetscInitialize() for more general runtime options.

207: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
208: @*/
209: PetscErrorCode  PetscEnd(void)
210: {
212:   PetscFinalize();
213:   exit(0);
214:   return 0;
215: }

217: PetscBool PetscOptionsPublish = PETSC_FALSE;
218: extern PetscErrorCode PetscSetUseTrMalloc_Private(void);
219: extern PetscErrorCode PetscSetUseHBWMalloc_Private(void);
220: extern PetscBool      petscsetmallocvisited;
221: static char           emacsmachinename[256];

223: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
224: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = 0;

226: /*@C
227:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
228:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
229:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

231:    Input Parameter:
232: +  help - the help function (may be NULL)
233: -  version - the version function (may be NULL)

235:    Level: developer

237:    Concepts: package help message

239: @*/
240: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
241: {
243:   PetscExternalHelpFunction    = help;
244:   PetscExternalVersionFunction = version;
245:   return(0);
246: }

248: #if defined(PETSC_USE_LOG)
249: extern PetscBool   PetscObjectsLog;
250: #endif

252: PetscErrorCode  PetscOptionsCheckInitial_Private(void)
253: {
254:   char              string[64],mname[PETSC_MAX_PATH_LEN],*f;
255:   MPI_Comm          comm = PETSC_COMM_WORLD;
256:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag;
257:   PetscErrorCode    ierr;
258:   PetscReal         si;
259:   PetscInt          intensity;
260:   int               i;
261:   PetscMPIInt       rank;
262:   char              version[256];
263: #if !defined(PETSC_HAVE_THREADSAFETY)
264:   PetscReal         logthreshold;
265: #endif
266: #if defined(PETSC_USE_LOG)
267:   PetscViewerFormat format;
268:   PetscBool         flg4 = PETSC_FALSE;
269: #endif

272:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);

274: #if !defined(PETSC_HAVE_THREADSAFETY)
275:   /*
276:       Setup the memory management; support for tracing malloc() usage
277:   */
278:   PetscOptionsHasName(NULL,NULL,"-malloc_log",&flg3);
279:   logthreshold = 0.0;
280:   PetscOptionsGetReal(NULL,NULL,"-malloc_log_threshold",&logthreshold,&flg1);
281:   if (flg1) flg3 = PETSC_TRUE;
282: #if defined(PETSC_USE_DEBUG)
283:   PetscOptionsGetBool(NULL,NULL,"-malloc",&flg1,&flg2);
284:   if ((!flg2 || flg1) && !petscsetmallocvisited) {
285:     if (flg2 || !(PETSC_RUNNING_ON_VALGRIND)) {
286:       /* turn off default -malloc if valgrind is being used */
287:       PetscSetUseTrMalloc_Private();
288:     }
289:   }
290: #else
291:   PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&flg1,NULL);
292:   PetscOptionsGetBool(NULL,NULL,"-malloc",&flg2,NULL);
293:   if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
294: #endif
295:   if (flg3) {
296:     PetscMallocSetDumpLogThreshold((PetscLogDouble)logthreshold);
297:   }
298:   flg1 = PETSC_FALSE;
299:   PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg1,NULL);
300:   if (flg1) {
301:     PetscSetUseTrMalloc_Private();
302:     PetscMallocDebug(PETSC_TRUE);
303:   }
304:   flg1 = PETSC_FALSE;
305:   PetscOptionsGetBool(NULL,NULL,"-malloc_test",&flg1,NULL);
306: #if defined(PETSC_USE_DEBUG)
307:   if (flg1 && !PETSC_RUNNING_ON_VALGRIND) {
308:     PetscSetUseTrMalloc_Private();
309:     PetscMallocSetDumpLog();
310:     PetscMallocDebug(PETSC_TRUE);
311:   }
312: #endif
313:   flg1 = PETSC_FALSE;
314:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
315:   if (flg1) {PetscSetUseHBWMalloc_Private();}

317:   flg1 = PETSC_FALSE;
318:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
319:   if (!flg1) {
320:     flg1 = PETSC_FALSE;
321:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
322:   }
323:   if (flg1) {
324:     PetscMemorySetGetMaximumUsage();
325:   }
326: #endif

328: #if defined(PETSC_USE_LOG)
329:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
330: #endif

332:   /*
333:       Set the display variable for graphics
334:   */
335:   PetscSetDisplay();

337:   /*
338:       Print the PETSc version information
339:   */
340:   PetscOptionsHasName(NULL,NULL,"-v",&flg1);
341:   PetscOptionsHasName(NULL,NULL,"-version",&flg2);
342:   PetscOptionsHasName(NULL,NULL,"-help",&flg3);
343:   if (flg1 || flg2 || flg3) {

345:     /*
346:        Print "higher-level" package version message
347:     */
348:     if (PetscExternalVersionFunction) {
349:       (*PetscExternalVersionFunction)(comm);
350:     }

352:     PetscGetVersion(version,256);
353:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
354: ------------------------------\n");
355:     (*PetscHelpPrintf)(comm,"%s\n",version);
356:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
357:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
358:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
359:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
360:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
361:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
362: ------------------------------\n");
363:   }

365:   /*
366:        Print "higher-level" package help message
367:   */
368:   if (flg3) {
369:     if (PetscExternalHelpFunction) {
370:       (*PetscExternalHelpFunction)(comm);
371:     }
372:   }

374:   /*
375:       Setup the error handling
376:   */
377:   flg1 = PETSC_FALSE;
378:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
379:   if (flg1) {
380:     MPI_Comm_set_errhandler(PETSC_COMM_WORLD,MPI_ERRORS_ARE_FATAL);
381:     PetscPushErrorHandler(PetscAbortErrorHandler,0);
382:   }
383:   flg1 = PETSC_FALSE;
384:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
385:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
386:   flg1 = PETSC_FALSE;
387:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
388:   if (flg1) {
389:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
390:   }
391:   flg1 = PETSC_FALSE;
392:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
393:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}
394:   flg1 = PETSC_FALSE;
395:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,NULL);
396:   if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}
397:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);

400:   /*
401:       Setup debugger information
402:   */
403:   PetscSetDefaultDebugger();
404:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,64,&flg1);
405:   if (flg1) {
406:     MPI_Errhandler err_handler;

408:     PetscSetDebuggerFromString(string);
409:     MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
410:     MPI_Comm_set_errhandler(comm,err_handler);
411:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
412:   }
413:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,64,&flg1);
414:   if (flg1) { PetscSetDebugTerminal(string); }
415:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,64,&flg1);
416:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,64,&flg2);
417:   if (flg1 || flg2) {
418:     PetscMPIInt    size;
419:     PetscInt       lsize,*nodes;
420:     MPI_Errhandler err_handler;
421:     /*
422:        we have to make sure that all processors have opened
423:        connections to all other processors, otherwise once the
424:        debugger has stated it is likely to receive a SIGUSR1
425:        and kill the program.
426:     */
427:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
428:     if (size > 2) {
429:       PetscMPIInt dummy = 0;
430:       MPI_Status  status;
431:       for (i=0; i<size; i++) {
432:         if (rank != i) {
433:           MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);
434:         }
435:       }
436:       for (i=0; i<size; i++) {
437:         if (rank != i) {
438:           MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);
439:         }
440:       }
441:     }
442:     /* check if this processor node should be in debugger */
443:     PetscMalloc1(size,&nodes);
444:     lsize = size;
445:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
446:     if (flag) {
447:       for (i=0; i<lsize; i++) {
448:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
449:       }
450:     }
451:     if (!flag) {
452:       PetscSetDebuggerFromString(string);
453:       PetscPushErrorHandler(PetscAbortErrorHandler,0);
454:       if (flg1) {
455:         PetscAttachDebugger();
456:       } else {
457:         PetscStopForDebugger();
458:       }
459:       MPI_Comm_create_errhandler((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
460:       MPI_Comm_set_errhandler(comm,err_handler);
461:     }
462:     PetscFree(nodes);
463:   }

465:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
466:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

468:   /*
469:         Setup profiling and logging
470:   */
471: #if defined(PETSC_USE_INFO)
472:   {
473:     char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
474:     PetscOptionsGetString(NULL,NULL,"-info",logname,250,&flg1);
475:     if (flg1 && logname[0]) {
476:       PetscInfoAllow(PETSC_TRUE,logname);
477:     } else if (flg1) {
478:       PetscInfoAllow(PETSC_TRUE,NULL);
479:     }
480:   }
481: #endif
482: #if defined(PETSC_USE_LOG)
483:   mname[0] = 0;
484:   PetscOptionsGetString(NULL,NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
485:   if (flg1) {
486:     if (mname[0]) {
487:       PetscOpenHistoryFile(mname,&petsc_history);
488:     } else {
489:       PetscOpenHistoryFile(NULL,&petsc_history);
490:     }
491:   }
492: #if defined(PETSC_HAVE_MPE)
493:   flg1 = PETSC_FALSE;
494:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
495:   if (flg1) {PetscLogMPEBegin();}
496: #endif
497:   flg1 = PETSC_FALSE;
498:   flg3 = PETSC_FALSE;
499:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
500:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
501:   if (flg1)                      { PetscLogAllBegin(); }
502:   else if (flg3)                 { PetscLogDefaultBegin();}

504:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,250,&flg1);
505:   if (flg1) {
506:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
507:     FILE *file;
508:     if (mname[0]) {
509:       sprintf(name,"%s.%d",mname,rank);
510:       PetscFixFilename(name,fname);
511:       file = fopen(fname,"w");
512:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
513:     } else file = PETSC_STDOUT;
514:     PetscLogTraceBegin(file);
515:   }

517:   PetscOptionsGetViewer(PETSC_COMM_WORLD,NULL,"-log_view",NULL,&format,&flg4);
518:   if (flg4) {
519:     if (format == PETSC_VIEWER_ASCII_XML){
520:       PetscLogNestedBegin();
521:     } else {
522:       PetscLogDefaultBegin();
523:     }
524:   }
525: #endif

527:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);

529: #if defined(PETSC_HAVE_CUDA)
530:   PetscOptionsHasName(NULL,NULL,"-cuda_show_devices",&flg1);
531:   if (flg1) {
532:     struct cudaDeviceProp prop;
533:     int                   devCount;
534:     PetscInt              device;
535:     cudaError_t           err = cudaSuccess;

537:     err = cudaGetDeviceCount(&devCount);
538:     if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));
539:     for (device = 0; device < devCount; ++device) {
540:       err = cudaGetDeviceProperties(&prop, (int)device);
541:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceProperties %s",cudaGetErrorString(err));
542:       PetscPrintf(PETSC_COMM_WORLD, "CUDA device %D: %s\n", device, prop.name);
543:     }
544:   }
545:   if (!PetscCUDAInitialized) {
546:     PetscMPIInt size;
547:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
548:     if (size>1) {
549:       int         devCount;
550:       PetscInt    device;
551:       PetscMPIInt rank;
552:       cudaError_t err = cudaSuccess;

554:       /* check to see if we force multiple ranks to hit the same GPU */
555:       PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
556:       if (flg1) {
557:         err = cudaSetDevice((int)device);
558:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
559:       } else {
560:         /* we're not using the same GPU on multiple MPI threads. So try to allocated different   GPUs to different processes */

562:         /* First get the device count */
563:         err   = cudaGetDeviceCount(&devCount);
564:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaGetDeviceCount %s",cudaGetErrorString(err));

566:         /* next determine the rank and then set the device via a mod */
567:         MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
568:         device = rank % devCount;
569:         err    = cudaSetDevice((int)device);
570:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
571:       }

573:       /* set the device flags so that it can map host memory ... do NOT throw exception on err!=cudaSuccess
574:        multiple devices may try to set the flags on the same device. So long as one of them succeeds, things
575:        are ok. */
576:       err = cudaSetDeviceFlags(cudaDeviceMapHost);
577:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
578:     } else {
579:       PetscInt    device;
580:       cudaError_t err = cudaSuccess;

582:       /* the code below works for serial GPU simulations */
583:       PetscOptionsGetInt(NULL,NULL,"-cuda_set_device", &device, &flg1);
584:       if (flg1) {
585:         err = cudaSetDevice((int)device);
586:         if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDevice %s",cudaGetErrorString(err));
587:       }

589:       /* set the device flags so that it can map host memory ... here, we error check. */
590:       err = cudaSetDeviceFlags(cudaDeviceMapHost);
591:       if (err != cudaSuccess) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_SYS,"error in cudaSetDeviceFlags %s",cudaGetErrorString(err));
592:     }

594:     PetscCUDAInitialized = PETSC_TRUE;
595:   }
596: #endif


599:   /*
600:        Print basic help message
601:   */
602:   PetscOptionsHasName(NULL,NULL,"-help",&flg1);
603:   if (flg1) {
604:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
605:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
606:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
607:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
608:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
609:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
610:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
611:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
612:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
613:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
614:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
615:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
616:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
617:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
618:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
619:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
620:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
621:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
622:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
623:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
624:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
625:     (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
626:     (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
627:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
628:     (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
629:     (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
630:     (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");
631:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
632:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
633:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
634:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
635:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
636:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
637: #if defined(PETSC_USE_LOG)
638:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
639:     (*PetscHelpPrintf)(comm," -log[_summary _summary_python]: logging objects and events\n");
640:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
641: #if defined(PETSC_HAVE_MPE)
642:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
643: #endif
644:     (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
645: #endif
646:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
647:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
648:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
649:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
650:   }

652: #if defined(PETSC_HAVE_POPEN)
653:   {
654:   char machine[128];
655:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,128,&flg1);
656:   if (flg1) {
657:     PetscPOpenSetMachine(machine);
658:   }
659:   }
660: #endif

662:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
663:   if (flg1) {
664:     PetscSleep(si);
665:   }

667:   PetscOptionsGetString(NULL,NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
668:   if (flg1) {
669:     PetscStrstr(mname,"null",&f);
670:     if (f) {
671:       PetscInfoDeactivateClass(0);
672:     }
673:   }

675: #if defined(PETSC_HAVE_CUSP) || defined(PETSC_HAVE_VIENNACL) || defined(PETSC_HAVE_VECCUDA)
676:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
677:   if (!flg3) {
678:   PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
679:   }
680: #endif
681: #if defined(PETSC_HAVE_CUSP)
682:   PetscOptionsGetBool(NULL,NULL,"-cusp_synchronize",&flg3,NULL);
683:   PetscCUSPSynchronize = flg3;
684: #elif defined(PETSC_HAVE_VIENNACL)
685:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
686:   PetscViennaCLSynchronize = flg3;
687: #elif defined(PETSC_HAVE_VECCUDA)
688:   PetscOptionsGetBool(NULL,NULL,"-cuda_synchronize",&flg3,NULL);
689:   PetscCUDASynchronize = flg3;
690: #endif

692: #if defined(PETSC_HAVE_VIENNACL)
693:   PetscViennaCLInit();
694: #endif

696:   return(0);
697: }