Actual source code: init.c

  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot know 
  6:   what malloc is being used until it has already processed the input.
  7: */

  9: #include <petscsys.h>        /*I  "petscsys.h"   I*/

 11: #ifndef _GNU_SOURCE
 12: #define _GNU_SOURCE
 13: #endif
 14: #if defined(PETSC_HAVE_SCHED_H)
 15: #include <sched.h>
 16: #endif
 17: #if defined(PETSC_HAVE_PTHREAD_H)
 18: #include <pthread.h>
 19: #endif

 21: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 22: #include <sys/sysinfo.h>
 23: #endif
 24: #if defined(PETSC_HAVE_UNISTD_H)
 25: #include <unistd.h>
 26: #endif
 27: #if defined(PETSC_HAVE_STDLIB_H)
 28: #include <stdlib.h>
 29: #endif
 30: #if defined(PETSC_HAVE_MALLOC_H)
 31: #include <malloc.h>
 32: #endif
 33: #if defined(PETSC_HAVE_VALGRIND)
 34: #include <valgrind/valgrind.h>
 35: #endif

 37: /* ------------------------Nasty global variables -------------------------------*/
 38: /*
 39:      Indicates if PETSc started up MPI, or it was 
 40:    already started before PETSc was initialized.
 41: */
 42: PetscBool    PetscBeganMPI         = PETSC_FALSE;
 43: PetscBool    PetscInitializeCalled = PETSC_FALSE;
 44: PetscBool    PetscFinalizeCalled   = PETSC_FALSE;
 45: PetscBool    PetscUseThreadPool    = PETSC_FALSE;
 46: PetscBool    PetscThreadGo         = PETSC_TRUE;
 47: PetscMPIInt  PetscGlobalRank = -1;
 48: PetscMPIInt  PetscGlobalSize = -1;

 50: #if defined(PETSC_HAVE_PTHREADCLASSES)
 51: PetscMPIInt  PetscMaxThreads = 2;
 52: pthread_t*   PetscThreadPoint;
 53: #define PETSC_HAVE_PTHREAD_BARRIER
 54: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
 55: pthread_barrier_t* BarrPoint;   /* used by 'true' thread pool */
 56: #endif
 57: PetscErrorCode ithreaderr = 0;
 58: int*         pVal;

 60: #define CACHE_LINE_SIZE 64  /* used by 'chain', 'main','tree' thread pools */
 61: int* ThreadCoreAffinity;

 63: typedef enum {JobInitiated,ThreadsWorking,JobCompleted} estat;  /* used by 'chain','tree' thread pool */

 65: typedef struct {
 66:   pthread_mutex_t** mutexarray;
 67:   pthread_cond_t**  cond1array;
 68:   pthread_cond_t** cond2array;
 69:   void* (*pfunc)(void*);
 70:   void** pdata;
 71:   PetscBool startJob;
 72:   estat eJobStat;
 73:   PetscBool** arrThreadStarted;
 74:   PetscBool** arrThreadReady;
 75: } sjob_tree;
 76: sjob_tree job_tree;
 77: typedef struct {
 78:   pthread_mutex_t** mutexarray;
 79:   pthread_cond_t**  cond1array;
 80:   pthread_cond_t** cond2array;
 81:   void* (*pfunc)(void*);
 82:   void** pdata;
 83:   PetscBool** arrThreadReady;
 84: } sjob_main;
 85: sjob_main job_main;
 86: typedef struct {
 87:   pthread_mutex_t** mutexarray;
 88:   pthread_cond_t**  cond1array;
 89:   pthread_cond_t** cond2array;
 90:   void* (*pfunc)(void*);
 91:   void** pdata;
 92:   PetscBool startJob;
 93:   estat eJobStat;
 94:   PetscBool** arrThreadStarted;
 95:   PetscBool** arrThreadReady;
 96: } sjob_chain;
 97: sjob_chain job_chain;
 98: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
 99: typedef struct {
100:   pthread_mutex_t mutex;
101:   pthread_cond_t cond;
102:   void* (*pfunc)(void*);
103:   void** pdata;
104:   pthread_barrier_t* pbarr;
105:   int iNumJobThreads;
106:   int iNumReadyThreads;
107:   PetscBool startJob;
108: } sjob_true;
109: sjob_true job_true = {PTHREAD_MUTEX_INITIALIZER,PTHREAD_COND_INITIALIZER,NULL,NULL,NULL,0,0,PETSC_FALSE};
110: #endif

112: pthread_cond_t  main_cond  = PTHREAD_COND_INITIALIZER;  /* used by 'true', 'chain','tree' thread pools */
113: char* arrmutex; /* used by 'chain','main','tree' thread pools */
114: char* arrcond1; /* used by 'chain','main','tree' thread pools */
115: char* arrcond2; /* used by 'chain','main','tree' thread pools */
116: char* arrstart; /* used by 'chain','main','tree' thread pools */
117: char* arrready; /* used by 'chain','main','tree' thread pools */

119: /* Function Pointers */
120: void*          (*PetscThreadFunc)(void*) = NULL;
121: void*          (*PetscThreadInitialize)(PetscInt) = NULL;
122: PetscErrorCode (*PetscThreadFinalize)(void) = NULL;
123: void           (*MainWait)(void) = NULL;
124: PetscErrorCode (*MainJob)(void* (*pFunc)(void*),void**,PetscInt) = NULL;
125: /**** Tree Thread Pool Functions ****/
126: void*          PetscThreadFunc_Tree(void*);
127: void*          PetscThreadInitialize_Tree(PetscInt);
128: PetscErrorCode PetscThreadFinalize_Tree(void);
129: void           MainWait_Tree(void);
130: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void**,PetscInt);
131: /**** Main Thread Pool Functions ****/
132: void*          PetscThreadFunc_Main(void*);
133: void*          PetscThreadInitialize_Main(PetscInt);
134: PetscErrorCode PetscThreadFinalize_Main(void);
135: void           MainWait_Main(void);
136: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void**,PetscInt);
137: /**** Chain Thread Pool Functions ****/
138: void*          PetscThreadFunc_Chain(void*);
139: void*          PetscThreadInitialize_Chain(PetscInt);
140: PetscErrorCode PetscThreadFinalize_Chain(void);
141: void           MainWait_Chain(void);
142: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void**,PetscInt);
143: /**** True Thread Pool Functions ****/
144: void*          PetscThreadFunc_True(void*);
145: void*          PetscThreadInitialize_True(PetscInt);
146: PetscErrorCode PetscThreadFinalize_True(void);
147: void           MainWait_True(void);
148: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void**,PetscInt);
149: /**** NO Thread Pool Function  ****/
150: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void**,PetscInt);
151: /****  ****/
152: void* FuncFinish(void*);
153: void* PetscThreadRun(MPI_Comm Comm,void* (*pFunc)(void*),int,pthread_t*,void**);
154: void* PetscThreadStop(MPI_Comm Comm,int,pthread_t*);
155: #endif

157: #if defined(PETSC_USE_COMPLEX)
158: #if defined(PETSC_COMPLEX_INSTANTIATE)
159: template <> class std::complex<double>; /* instantiate complex template class */
160: #endif
161: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
162: MPI_Datatype   MPIU_C_DOUBLE_COMPLEX;
163: MPI_Datatype   MPIU_C_COMPLEX;
164: #endif
165: PetscScalar    PETSC_i;
166: #else
167: PetscScalar    PETSC_i = 0.0;
168: #endif
169: #if defined(PETSC_USE_REAL___FLOAT128)
170: MPI_Datatype   MPIU___FLOAT128 = 0;
171: #endif
172: MPI_Datatype   MPIU_2SCALAR = 0;
173: MPI_Datatype   MPIU_2INT = 0;

175: /*
176:      These are needed by petscbt.h
177: */
178: #include <petscbt.h>
179: char      _BT_mask = ' ';
180: char      _BT_c = ' ';
181: PetscInt  _BT_idx  = 0;

183: /*
184:        Function that is called to display all error messages
185: */
186: PetscErrorCode  (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
187: PetscErrorCode  (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
188: #if defined(PETSC_HAVE_MATLAB_ENGINE)
189: PetscErrorCode  (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintf_Matlab;
190: #else
191: PetscErrorCode  (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
192: #endif
193: /*
194:   This is needed to turn on/off cusp synchronization */
195: PetscBool   synchronizeCUSP = PETSC_FALSE;

197: /* ------------------------------------------------------------------------------*/
198: /* 
199:    Optional file where all PETSc output from various prints is saved
200: */
201: FILE *petsc_history = PETSC_NULL;

205: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
206: {
208:   PetscMPIInt    rank,size;
209:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
210:   char           version[256];

213:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
214:   if (!rank) {
215:     char        arch[10];
216:     int         err;
217:     PetscViewer viewer;

219:     PetscGetArchType(arch,10);
220:     PetscGetDate(date,64);
221:     PetscGetVersion(version,256);
222:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
223:     if (filename) {
224:       PetscFixFilename(filename,fname);
225:     } else {
226:       PetscGetHomeDirectory(pfile,240);
227:       PetscStrcat(pfile,"/.petschistory");
228:       PetscFixFilename(pfile,fname);
229:     }

231:     *fd = fopen(fname,"a"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
232:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
233:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
234:     PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
235:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
236:     PetscViewerASCIIOpenWithFILE(PETSC_COMM_WORLD,*fd,&viewer);
237:     PetscOptionsView(viewer);
238:     PetscViewerDestroy(&viewer);
239:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
240:     err = fflush(*fd);
241:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
242:   }
243:   return(0);
244: }

248: PetscErrorCode  PetscCloseHistoryFile(FILE **fd)
249: {
251:   PetscMPIInt    rank;
252:   char           date[64];
253:   int            err;

256:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
257:   if (!rank) {
258:     PetscGetDate(date,64);
259:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
260:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
261:     PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
262:     err = fflush(*fd);
263:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
264:     err = fclose(*fd);
265:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
266:   }
267:   return(0);
268: }

270: /* ------------------------------------------------------------------------------*/

272: /* 
273:    This is ugly and probably belongs somewhere else, but I want to 
274:   be able to put a true MPI abort error handler with command line args.

276:     This is so MPI errors in the debugger will leave all the stack 
277:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
278:   in the debugger hence we call abort() instead of MPI_Abort().
279: */

283: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
284: {
286:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
287:   abort();
288: }

292: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
293: {

297:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
298:   PetscAttachDebugger();
299:   if (ierr) { /* hopeless so get out */
300:     MPI_Abort(*comm,*flag);
301:   }
302: }

306: /*@C 
307:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one 
308:      wishes a clean exit somewhere deep in the program.

310:    Collective on PETSC_COMM_WORLD

312:    Options Database Keys are the same as for PetscFinalize()

314:    Level: advanced

316:    Note:
317:    See PetscInitialize() for more general runtime options.

319: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
320: @*/
321: PetscErrorCode  PetscEnd(void)
322: {
324:   PetscFinalize();
325:   exit(0);
326:   return 0;
327: }

329: PetscBool    PetscOptionsPublish = PETSC_FALSE;
332: static char       emacsmachinename[256];

334: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
335: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = 0;

339: /*@C 
340:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
341:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
342:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

344:    Input Parameter:
345: +  help - the help function (may be PETSC_NULL)
346: -  version - the version function (may be PETSC_NULL)

348:    Level: developer

350:    Concepts: package help message

352: @*/
353: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
354: {
356:   PetscExternalHelpFunction    = help;
357:   PetscExternalVersionFunction = version;
358:   return(0);
359: }

363: PetscErrorCode  PetscOptionsCheckInitial_Private(void)
364: {
365:   char           string[64],mname[PETSC_MAX_PATH_LEN],*f;
366:   MPI_Comm       comm = PETSC_COMM_WORLD;
367:   PetscBool      flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flg4 = PETSC_FALSE,flag,flgz,flgzout;
369:   PetscReal      si;
370:   int            i;
371:   PetscMPIInt    rank;
372:   char           version[256];

375:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);

377:   /*
378:       Setup the memory management; support for tracing malloc() usage 
379:   */
380:   PetscOptionsHasName(PETSC_NULL,"-malloc_log",&flg3);
381: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
382:   PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg1,&flg2);
383:   if ((!flg2 || flg1) && !petscsetmallocvisited) {
384: #if defined(PETSC_HAVE_VALGRIND)
385:     if (flg2 || !(RUNNING_ON_VALGRIND)) {
386:       /* turn off default -malloc if valgrind is being used */
387: #endif
388:       PetscSetUseTrMalloc_Private();
389: #if defined(PETSC_HAVE_VALGRIND)
390:     }
391: #endif
392:   }
393: #else
394:   PetscOptionsGetBool(PETSC_NULL,"-malloc_dump",&flg1,PETSC_NULL);
395:   PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg2,PETSC_NULL);
396:   if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
397: #endif
398:   if (flg3) {
399:     PetscMallocSetDumpLog();
400:   }
401:   flg1 = PETSC_FALSE;
402:   PetscOptionsGetBool(PETSC_NULL,"-malloc_debug",&flg1,PETSC_NULL);
403:   if (flg1) {
404:     PetscSetUseTrMalloc_Private();
405:     PetscMallocDebug(PETSC_TRUE);
406:   }

408:   flg1 = PETSC_FALSE;
409:   PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg1,PETSC_NULL);
410:   if (!flg1) {
411:     flg1 = PETSC_FALSE;
412:     PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg1,PETSC_NULL);
413:   }
414:   if (flg1) {
415:     PetscMemorySetGetMaximumUsage();
416:   }

418:   /*
419:       Set the display variable for graphics
420:   */
421:   PetscSetDisplay();


424:   /*
425:       Print the PETSc version information
426:   */
427:   PetscOptionsHasName(PETSC_NULL,"-v",&flg1);
428:   PetscOptionsHasName(PETSC_NULL,"-version",&flg2);
429:   PetscOptionsHasName(PETSC_NULL,"-help",&flg3);
430:   if (flg1 || flg2 || flg3){

432:     /*
433:        Print "higher-level" package version message 
434:     */
435:     if (PetscExternalVersionFunction) {
436:       (*PetscExternalVersionFunction)(comm);
437:     }

439:     PetscGetVersion(version,256);
440:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
441: ------------------------------\n");
442:     (*PetscHelpPrintf)(comm,"%s\n",version);
443:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
444:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
445:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
446:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
447:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
448:     (*PetscHelpPrintf)(comm,"--------------------------------------------\
449: ------------------------------\n");
450:   }

452:   /*
453:        Print "higher-level" package help message 
454:   */
455:   if (flg3){
456:     if (PetscExternalHelpFunction) {
457:       (*PetscExternalHelpFunction)(comm);
458:     }
459:   }

461:   /*
462:       Setup the error handling
463:   */
464:   flg1 = PETSC_FALSE;
465:   PetscOptionsGetBool(PETSC_NULL,"-on_error_abort",&flg1,PETSC_NULL);
466:   if (flg1) { PetscPushErrorHandler(PetscAbortErrorHandler,0);}
467:   flg1 = PETSC_FALSE;
468:   PetscOptionsGetBool(PETSC_NULL,"-on_error_mpiabort",&flg1,PETSC_NULL);
469:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
470:   flg1 = PETSC_FALSE;
471:   PetscOptionsGetBool(PETSC_NULL,"-mpi_return_on_error",&flg1,PETSC_NULL);
472:   if (flg1) {
473:     MPI_Errhandler_set(comm,MPI_ERRORS_RETURN);
474:   }
475:   flg1 = PETSC_FALSE;
476:   PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);
477:   if (!flg1) {PetscPushSignalHandler(PetscDefaultSignalHandler,(void*)0);}
478:   flg1 = PETSC_FALSE;
479:   PetscOptionsGetBool(PETSC_NULL,"-fp_trap",&flg1,PETSC_NULL);
480:   if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}

482:   /*
483:       Setup debugger information
484:   */
485:   PetscSetDefaultDebugger();
486:   PetscOptionsGetString(PETSC_NULL,"-on_error_attach_debugger",string,64,&flg1);
487:   if (flg1) {
488:     MPI_Errhandler err_handler;

490:     PetscSetDebuggerFromString(string);
491:     MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
492:     MPI_Errhandler_set(comm,err_handler);
493:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
494:   }
495:   PetscOptionsGetString(PETSC_NULL,"-debug_terminal",string,64,&flg1);
496:   if (flg1) { PetscSetDebugTerminal(string); }
497:   PetscOptionsGetString(PETSC_NULL,"-start_in_debugger",string,64,&flg1);
498:   PetscOptionsGetString(PETSC_NULL,"-stop_for_debugger",string,64,&flg2);
499:   if (flg1 || flg2) {
500:     PetscMPIInt    size;
501:     PetscInt       lsize,*nodes;
502:     MPI_Errhandler err_handler;
503:     /*
504:        we have to make sure that all processors have opened 
505:        connections to all other processors, otherwise once the 
506:        debugger has stated it is likely to receive a SIGUSR1
507:        and kill the program. 
508:     */
509:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
510:     if (size > 2) {
511:       PetscMPIInt dummy = 0;
512:       MPI_Status  status;
513:       for (i=0; i<size; i++) {
514:         if (rank != i) {
515:           MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);
516:         }
517:       }
518:       for (i=0; i<size; i++) {
519:         if (rank != i) {
520:           MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);
521:         }
522:       }
523:     }
524:     /* check if this processor node should be in debugger */
525:     PetscMalloc(size*sizeof(PetscInt),&nodes);
526:     lsize = size;
527:     PetscOptionsGetIntArray(PETSC_NULL,"-debugger_nodes",nodes,&lsize,&flag);
528:     if (flag) {
529:       for (i=0; i<lsize; i++) {
530:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
531:       }
532:     }
533:     if (!flag) {
534:       PetscSetDebuggerFromString(string);
535:       PetscPushErrorHandler(PetscAbortErrorHandler,0);
536:       if (flg1) {
537:         PetscAttachDebugger();
538:       } else {
539:         PetscStopForDebugger();
540:       }
541:       MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
542:       MPI_Errhandler_set(comm,err_handler);
543:     }
544:     PetscFree(nodes);
545:   }

547:   PetscOptionsGetString(PETSC_NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
548:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

550: #if defined(PETSC_USE_SOCKET_VIEWER)
551:   /*
552:     Activates new sockets for zope if needed
553:   */
554:   PetscOptionsHasName(PETSC_NULL,"-zope", &flgz);
555:   PetscOptionsHasName(PETSC_NULL,"-nostdout", &flgzout);
556:   if (flgz){
557:     int  sockfd;
558:     char hostname[256];
559:     char username[256];
560:     int  remoteport = 9999;

562:     PetscOptionsGetString(PETSC_NULL, "-zope", hostname, 256, &flgz);
563:     if (!hostname[0]){
564:       PetscGetHostName(hostname,256);
565:     }
566:     PetscOpenSocket(hostname, remoteport, &sockfd);
567:     PetscGetUserName(username, 256);
568:     PETSC_ZOPEFD = fdopen(sockfd, "w");
569:     if (flgzout){
570:       PETSC_STDOUT = PETSC_ZOPEFD;
571:       fprintf(PETSC_STDOUT, "<<<user>>> %s\n",username);
572:       fprintf(PETSC_STDOUT, "<<<start>>>");
573:     } else {
574:       fprintf(PETSC_ZOPEFD, "<<<user>>> %s\n",username);
575:       fprintf(PETSC_ZOPEFD, "<<<start>>>");
576:     }
577:   }
578: #endif
579: #if defined(PETSC_USE_SERVER)
580:   PetscOptionsHasName(PETSC_NULL,"-server", &flgz);
581:   if (flgz){
582:     PetscInt port = PETSC_DECIDE;
583:     PetscOptionsGetInt(PETSC_NULL,"-server",&port,PETSC_NULL);
584:     PetscWebServe(PETSC_COMM_WORLD,(int)port);
585:   }
586: #endif

588:   /*
589:         Setup profiling and logging
590:   */
591: #if defined (PETSC_USE_INFO)
592:   {
593:     char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
594:     PetscOptionsGetString(PETSC_NULL,"-info",logname,250,&flg1);
595:     if (flg1 && logname[0]) {
596:       PetscInfoAllow(PETSC_TRUE,logname);
597:     } else if (flg1) {
598:       PetscInfoAllow(PETSC_TRUE,PETSC_NULL);
599:     }
600:   }
601: #endif
602: #if defined(PETSC_USE_LOG)
603:   mname[0] = 0;
604:   PetscOptionsGetString(PETSC_NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
605:   if (flg1) {
606:     if (mname[0]) {
607:       PetscOpenHistoryFile(mname,&petsc_history);
608:     } else {
609:       PetscOpenHistoryFile(0,&petsc_history);
610:     }
611:   }
612: #if defined(PETSC_HAVE_MPE)
613:   flg1 = PETSC_FALSE;
614:   PetscOptionsHasName(PETSC_NULL,"-log_mpe",&flg1);
615:   if (flg1) PetscLogMPEBegin();
616: #endif
617:   flg1 = PETSC_FALSE;
618:   flg2 = PETSC_FALSE;
619:   flg3 = PETSC_FALSE;
620:   PetscOptionsGetBool(PETSC_NULL,"-log_all",&flg1,PETSC_NULL);
621:   PetscOptionsGetBool(PETSC_NULL,"-log",&flg2,PETSC_NULL);
622:   PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
623:   PetscOptionsHasName(PETSC_NULL,"-log_summary_python",&flg4);
624:   if (flg1)                      {  PetscLogAllBegin(); }
625:   else if (flg2 || flg3 || flg4) {  PetscLogBegin();}
626: 
627:   PetscOptionsGetString(PETSC_NULL,"-log_trace",mname,250,&flg1);
628:   if (flg1) {
629:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
630:     FILE *file;
631:     if (mname[0]) {
632:       sprintf(name,"%s.%d",mname,rank);
633:       PetscFixFilename(name,fname);
634:       file = fopen(fname,"w");
635:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
636:     } else {
637:       file = PETSC_STDOUT;
638:     }
639:     PetscLogTraceBegin(file);
640:   }
641: #endif

643:   /*
644:       Setup building of stack frames for all function calls
645:   */
646: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
647:   PetscStackCreate();
648: #endif

650:   PetscOptionsGetBool(PETSC_NULL,"-options_gui",&PetscOptionsPublish,PETSC_NULL);

652: #if defined(PETSC_HAVE_PTHREADCLASSES)
653:   /*
654:       Determine whether user specified maximum number of threads
655:    */
656:   PetscOptionsGetInt(PETSC_NULL,"-thread_max",&PetscMaxThreads,PETSC_NULL);

658:   PetscOptionsHasName(PETSC_NULL,"-main",&flg1);
659:   if(flg1) {
660:     cpu_set_t mset;
661:     int icorr,ncorr = get_nprocs();
662:     PetscOptionsGetInt(PETSC_NULL,"-main",&icorr,PETSC_NULL);
663:     CPU_ZERO(&mset);
664:     CPU_SET(icorr%ncorr,&mset);
665:     sched_setaffinity(0,sizeof(cpu_set_t),&mset);
666:   }

668:   PetscInt N_CORES = get_nprocs();
669:   ThreadCoreAffinity = (int*)malloc(N_CORES*sizeof(int));
670:   char tstr[9];
671:   char tbuf[2];
672:   strcpy(tstr,"-thread");
673:   for(i=0;i<PetscMaxThreads;i++) {
674:     ThreadCoreAffinity[i] = i;
675:     sprintf(tbuf,"%d",i);
676:     strcat(tstr,tbuf);
677:     PetscOptionsHasName(PETSC_NULL,tstr,&flg1);
678:     if(flg1) {
679:       PetscOptionsGetInt(PETSC_NULL,tstr,&ThreadCoreAffinity[i],PETSC_NULL);
680:       ThreadCoreAffinity[i] = ThreadCoreAffinity[i]%N_CORES; /* check on the user */
681:     }
682:     tstr[7] = '\0';
683:   }

685:   /*
686:       Determine whether to use thread pool
687:    */
688:   PetscOptionsHasName(PETSC_NULL,"-use_thread_pool",&flg1);
689:   if (flg1) {
690:     PetscUseThreadPool = PETSC_TRUE;
691:     /* get the thread pool type */
692:     PetscInt ipool = 0;
693:     const char *choices[4] = {"true","tree","main","chain"};

695:     PetscOptionsGetEList(PETSC_NULL,"-use_thread_pool",choices,4,&ipool,PETSC_NULL);
696:     switch(ipool) {
697:     case 1:
698:       PetscThreadFunc       = &PetscThreadFunc_Tree;
699:       PetscThreadInitialize = &PetscThreadInitialize_Tree;
700:       PetscThreadFinalize   = &PetscThreadFinalize_Tree;
701:       MainWait              = &MainWait_Tree;
702:       MainJob               = &MainJob_Tree;
703:       PetscInfo(PETSC_NULL,"Using tree thread pool\n");
704:       break;
705:     case 2:
706:       PetscThreadFunc       = &PetscThreadFunc_Main;
707:       PetscThreadInitialize = &PetscThreadInitialize_Main;
708:       PetscThreadFinalize   = &PetscThreadFinalize_Main;
709:       MainWait              = &MainWait_Main;
710:       MainJob               = &MainJob_Main;
711:       PetscInfo(PETSC_NULL,"Using main thread pool\n");
712:       break;
713: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
714:     case 3:
715: #else
716:     default:
717: #endif
718:       PetscThreadFunc       = &PetscThreadFunc_Chain;
719:       PetscThreadInitialize = &PetscThreadInitialize_Chain;
720:       PetscThreadFinalize   = &PetscThreadFinalize_Chain;
721:       MainWait              = &MainWait_Chain;
722:       MainJob               = &MainJob_Chain;
723:       PetscInfo(PETSC_NULL,"Using chain thread pool\n");
724:       break;
725: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
726:     default:
727:       PetscThreadFunc       = &PetscThreadFunc_True;
728:       PetscThreadInitialize = &PetscThreadInitialize_True;
729:       PetscThreadFinalize   = &PetscThreadFinalize_True;
730:       MainWait              = &MainWait_True;
731:       MainJob               = &MainJob_True;
732:       PetscInfo(PETSC_NULL,"Using true thread pool\n");
733:       break;
734: #endif
735:     }
736:     PetscThreadInitialize(PetscMaxThreads);
737:   } else {
738:     //need to define these in the case on 'no threads' or 'thread create/destroy'
739:     //could take any of the above versions
740:     MainJob               = &MainJob_Spawn;
741:   }
742: #endif
743:   /*
744:        Print basic help message
745:   */
746:   PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
747:   if (flg1) {
748:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
749:     (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
750:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
751:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
752:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
753:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
754:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
755:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
756:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
757:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
758:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
759:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
760:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
761:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
762:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
763:     (*PetscHelpPrintf)(comm," -display display: Location where graphics and debuggers are displayed\n");
764:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
765:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
766:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
767:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
768:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
769:     (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
770:     (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
771:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
772:     (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
773:     (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
774:     (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");
775:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
776:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
777:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
778:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
779:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
780:     (*PetscHelpPrintf)(comm," -memory_info: print memory usage at end of run\n");
781:     (*PetscHelpPrintf)(comm," -server <port>: Run PETSc webserver (default port is 8080) see PetscWebServe()\n");
782: #if defined(PETSC_USE_LOG)
783:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
784:     (*PetscHelpPrintf)(comm," -log[_all _summary _summary_python]: logging objects and events\n");
785:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
786: #if defined(PETSC_HAVE_MPE)
787:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through upshot\n");
788: #endif
789:     (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
790: #endif
791:     (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
792:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
793:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
794:     (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
795:   }

797:   PetscOptionsGetReal(PETSC_NULL,"-petsc_sleep",&si,&flg1);
798:   if (flg1) {
799:     PetscSleep(si);
800:   }

802:   PetscOptionsGetString(PETSC_NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
803:   PetscStrstr(mname,"null",&f);
804:   if (f) {
805:     PetscInfoDeactivateClass(PETSC_NULL);
806:   }

808: #if defined(PETSC_HAVE_CUSP)
809:   PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
810:   if (flg3) flg1 = PETSC_TRUE;
811:   else flg1 = PETSC_FALSE;
812:   PetscOptionsGetBool(PETSC_NULL,"-cusp_synchronize",&flg1,PETSC_NULL);
813:   if (flg1) synchronizeCUSP = PETSC_TRUE;
814: #endif

816:   return(0);
817: }

819: #if defined(PETSC_HAVE_PTHREADCLASSES)

821: /**** 'Tree' Thread Pool Functions ****/
822: void* PetscThreadFunc_Tree(void* arg) {
823:   PetscErrorCode iterr;
824:   int icorr,ierr;
825:   int* pId = (int*)arg;
826:   int ThreadId = *pId,Mary = 2,i,SubWorker;
827:   PetscBool PeeOn;
828:   cpu_set_t mset;
829:   //printf("Thread %d In Tree Thread Function\n",ThreadId);
830:   icorr = ThreadCoreAffinity[ThreadId];
831:   CPU_ZERO(&mset);
832:   CPU_SET(icorr,&mset);
833:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

835:   if((Mary*ThreadId+1)>(PetscMaxThreads-1)) {
836:     PeeOn = PETSC_TRUE;
837:   }
838:   else {
839:     PeeOn = PETSC_FALSE;
840:   }
841:   if(PeeOn==PETSC_FALSE) {
842:     /* check your subordinates, wait for them to be ready */
843:     for(i=1;i<=Mary;i++) {
844:       SubWorker = Mary*ThreadId+i;
845:       if(SubWorker<PetscMaxThreads) {
846:         pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
847:         while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE) {
848:           /* upon entry, automically releases the lock and blocks
849:            upon return, has the lock */
850:           pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
851:         }
852:         pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
853:       }
854:     }
855:     /* your subordinates are now ready */
856:   }
857:   pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
858:   /* update your ready status */
859:   *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
860:   if(ThreadId==0) {
861:     job_tree.eJobStat = JobCompleted;
862:     /* ignal main */
863:     pthread_cond_signal(&main_cond);
864:   }
865:   else {
866:     /* tell your boss that you're ready to work */
867:     pthread_cond_signal(job_tree.cond1array[ThreadId]);
868:   }
869:   /* the while loop needs to have an exit
870:   the 'main' thread can terminate all the threads by performing a broadcast
871:    and calling FuncFinish */
872:   while(PetscThreadGo) {
873:     /*need to check the condition to ensure we don't have to wait
874:       waiting when you don't have to causes problems
875:      also need to check the condition to ensure proper handling of spurious wakeups */
876:     while(*(job_tree.arrThreadReady[ThreadId])==PETSC_TRUE) {
877:       /* upon entry, automically releases the lock and blocks
878:        upon return, has the lock */
879:         pthread_cond_wait(job_tree.cond2array[ThreadId],job_tree.mutexarray[ThreadId]);
880:         *(job_tree.arrThreadStarted[ThreadId]) = PETSC_TRUE;
881:         *(job_tree.arrThreadReady[ThreadId])   = PETSC_FALSE;
882:     }
883:     if(ThreadId==0) {
884:       job_tree.startJob = PETSC_FALSE;
885:       job_tree.eJobStat = ThreadsWorking;
886:     }
887:     pthread_mutex_unlock(job_tree.mutexarray[ThreadId]);
888:     if(PeeOn==PETSC_FALSE) {
889:       /* tell your subordinates it's time to get to work */
890:       for(i=1; i<=Mary; i++) {
891:         SubWorker = Mary*ThreadId+i;
892:         if(SubWorker<PetscMaxThreads) {
893:           pthread_cond_signal(job_tree.cond2array[SubWorker]);
894:         }
895:       }
896:     }
897:     /* do your job */
898:     if(job_tree.pdata==NULL) {
899:       iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata);
900:     }
901:     else {
902:       iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata[ThreadId]);
903:     }
904:     if(iterr!=0) {
905:       ithreaderr = 1;
906:     }
907:     if(PetscThreadGo) {
908:       /* reset job, get ready for more */
909:       if(PeeOn==PETSC_FALSE) {
910:         /* check your subordinates, waiting for them to be ready
911:          how do you know for a fact that a given subordinate has actually started? */
912:         for(i=1;i<=Mary;i++) {
913:           SubWorker = Mary*ThreadId+i;
914:           if(SubWorker<PetscMaxThreads) {
915:             pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
916:             while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_tree.arrThreadStarted[SubWorker])==PETSC_FALSE) {
917:               /* upon entry, automically releases the lock and blocks
918:                upon return, has the lock */
919:               pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
920:             }
921:             pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
922:           }
923:         }
924:         /* your subordinates are now ready */
925:       }
926:       pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
927:       *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
928:       if(ThreadId==0) {
929:         job_tree.eJobStat = JobCompleted; /* oot thread: last thread to complete, guaranteed! */
930:         /* root thread signals 'main' */
931:         pthread_cond_signal(&main_cond);
932:       }
933:       else {
934:         /* signal your boss before you go to sleep */
935:         pthread_cond_signal(job_tree.cond1array[ThreadId]);
936:       }
937:     }
938:   }
939:   return NULL;
940: }

944: void* PetscThreadInitialize_Tree(PetscInt N) {
945:   PetscInt i,ierr;
946:   int status;

948:   if(PetscUseThreadPool) {
949:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
950:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
951:     arrmutex = (char*)memalign(Val1,Val2);
952:     arrcond1 = (char*)memalign(Val1,Val2);
953:     arrcond2 = (char*)memalign(Val1,Val2);
954:     arrstart = (char*)memalign(Val1,Val2);
955:     arrready = (char*)memalign(Val1,Val2);
956:     job_tree.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
957:     job_tree.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
958:     job_tree.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
959:     job_tree.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
960:     job_tree.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
961:     /* initialize job structure */
962:     for(i=0; i<PetscMaxThreads; i++) {
963:       job_tree.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
964:       job_tree.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
965:       job_tree.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
966:       job_tree.arrThreadStarted[i]  = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
967:       job_tree.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
968:     }
969:     for(i=0; i<PetscMaxThreads; i++) {
970:       pthread_mutex_init(job_tree.mutexarray[i],NULL);
971:       pthread_cond_init(job_tree.cond1array[i],NULL);
972:       pthread_cond_init(job_tree.cond2array[i],NULL);
973:       *(job_tree.arrThreadStarted[i])  = PETSC_FALSE;
974:       *(job_tree.arrThreadReady[i])    = PETSC_FALSE;
975:     }
976:     job_tree.pfunc = NULL;
977:     job_tree.pdata = (void**)malloc(N*sizeof(void*));
978:     job_tree.startJob = PETSC_FALSE;
979:     job_tree.eJobStat = JobInitiated;
980:     pVal = (int*)malloc(N*sizeof(int));
981:     /* allocate memory in the heap for the thread structure */
982:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
983:     /* create threads */
984:     for(i=0; i<N; i++) {
985:       pVal[i] = i;
986:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
987:       /* should check status */
988:     }
989:   }
990:   return NULL;
991: }

995: PetscErrorCode PetscThreadFinalize_Tree() {
996:   int i,ierr;
997:   void* jstatus;


1001:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1002:   /* join the threads */
1003:   for(i=0; i<PetscMaxThreads; i++) {
1004:     pthread_join(PetscThreadPoint[i],&jstatus);
1005:     /* do error checking*/
1006:   }
1007:   free(PetscThreadPoint);
1008:   free(arrmutex);
1009:   free(arrcond1);
1010:   free(arrcond2);
1011:   free(arrstart);
1012:   free(arrready);
1013:   free(job_tree.pdata);
1014:   free(pVal);

1016:   return(0);
1017: }

1021: void MainWait_Tree() {
1022:   int ierr;
1023:   pthread_mutex_lock(job_tree.mutexarray[0]);
1024:   while(job_tree.eJobStat<JobCompleted||job_tree.startJob==PETSC_TRUE) {
1025:     pthread_cond_wait(&main_cond,job_tree.mutexarray[0]);
1026:   }
1027:   pthread_mutex_unlock(job_tree.mutexarray[0]);
1028: }

1032: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void** data,PetscInt n) {
1033:   int i,ierr;
1034:   PetscErrorCode ijoberr = 0;

1036:   MainWait();
1037:   job_tree.pfunc = pFunc;
1038:   job_tree.pdata = data;
1039:   job_tree.startJob = PETSC_TRUE;
1040:   for(i=0; i<PetscMaxThreads; i++) {
1041:     *(job_tree.arrThreadStarted[i]) = PETSC_FALSE;
1042:   }
1043:   job_tree.eJobStat = JobInitiated;
1044:   pthread_cond_signal(job_tree.cond2array[0]);
1045:   if(pFunc!=FuncFinish) {
1046:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1047:   }

1049:   if(ithreaderr) {
1050:     ijoberr = ithreaderr;
1051:   }
1052:   return ijoberr;
1053: }
1054: /****  ****/

1056: /**** 'Main' Thread Pool Functions ****/
1057: void* PetscThreadFunc_Main(void* arg) {
1058:   PetscErrorCode iterr;
1059:   int icorr,ierr;
1060:   int* pId = (int*)arg;
1061:   int ThreadId = *pId;
1062:   cpu_set_t mset;
1063:   //printf("Thread %d In Main Thread Function\n",ThreadId);
1064:   icorr = ThreadCoreAffinity[ThreadId];
1065:   CPU_ZERO(&mset);
1066:   CPU_SET(icorr,&mset);
1067:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1069:   pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1070:   /* update your ready status */
1071:   *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1072:   /* tell the BOSS that you're ready to work before you go to sleep */
1073:   pthread_cond_signal(job_main.cond1array[ThreadId]);

1075:   /* the while loop needs to have an exit
1076:      the 'main' thread can terminate all the threads by performing a broadcast
1077:      and calling FuncFinish */
1078:   while(PetscThreadGo) {
1079:     /* need to check the condition to ensure we don't have to wait
1080:        waiting when you don't have to causes problems
1081:      also need to check the condition to ensure proper handling of spurious wakeups */
1082:     while(*(job_main.arrThreadReady[ThreadId])==PETSC_TRUE) {
1083:       /* upon entry, atomically releases the lock and blocks
1084:        upon return, has the lock */
1085:         pthread_cond_wait(job_main.cond2array[ThreadId],job_main.mutexarray[ThreadId]);
1086:         /* (job_main.arrThreadReady[ThreadId])   = PETSC_FALSE; */
1087:     }
1088:     pthread_mutex_unlock(job_main.mutexarray[ThreadId]);
1089:     if(job_main.pdata==NULL) {
1090:       iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata);
1091:     }
1092:     else {
1093:       iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata[ThreadId]);
1094:     }
1095:     if(iterr!=0) {
1096:       ithreaderr = 1;
1097:     }
1098:     if(PetscThreadGo) {
1099:       /* reset job, get ready for more */
1100:       pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1101:       *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1102:       /* tell the BOSS that you're ready to work before you go to sleep */
1103:       pthread_cond_signal(job_main.cond1array[ThreadId]);
1104:     }
1105:   }
1106:   return NULL;
1107: }

1111: void* PetscThreadInitialize_Main(PetscInt N) {
1112:   PetscInt i,ierr;
1113:   int status;

1115:   if(PetscUseThreadPool) {
1116:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
1117:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1118:     arrmutex = (char*)memalign(Val1,Val2);
1119:     arrcond1 = (char*)memalign(Val1,Val2);
1120:     arrcond2 = (char*)memalign(Val1,Val2);
1121:     arrstart = (char*)memalign(Val1,Val2);
1122:     arrready = (char*)memalign(Val1,Val2);
1123:     job_main.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1124:     job_main.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1125:     job_main.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1126:     job_main.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1127:     /* initialize job structure */
1128:     for(i=0; i<PetscMaxThreads; i++) {
1129:       job_main.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1130:       job_main.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1131:       job_main.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1132:       job_main.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1133:     }
1134:     for(i=0; i<PetscMaxThreads; i++) {
1135:       pthread_mutex_init(job_main.mutexarray[i],NULL);
1136:       pthread_cond_init(job_main.cond1array[i],NULL);
1137:       pthread_cond_init(job_main.cond2array[i],NULL);
1138:       *(job_main.arrThreadReady[i])    = PETSC_FALSE;
1139:     }
1140:     job_main.pfunc = NULL;
1141:     job_main.pdata = (void**)malloc(N*sizeof(void*));
1142:     pVal = (int*)malloc(N*sizeof(int));
1143:     /* allocate memory in the heap for the thread structure */
1144:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1145:     /* create threads */
1146:     for(i=0; i<N; i++) {
1147:       pVal[i] = i;
1148:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1149:       /* error check */
1150:     }
1151:   }
1152:   else {
1153:   }
1154:   return NULL;
1155: }

1159: PetscErrorCode PetscThreadFinalize_Main() {
1160:   int i,ierr;
1161:   void* jstatus;


1165:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1166:   /* join the threads */
1167:   for(i=0; i<PetscMaxThreads; i++) {
1168:     pthread_join(PetscThreadPoint[i],&jstatus);
1169:   }
1170:   free(PetscThreadPoint);
1171:   free(arrmutex);
1172:   free(arrcond1);
1173:   free(arrcond2);
1174:   free(arrstart);
1175:   free(arrready);
1176:   free(job_main.pdata);
1177:   free(pVal);

1179:   return(0);
1180: }

1184: void MainWait_Main() {
1185:   int i,ierr;
1186:   for(i=0; i<PetscMaxThreads; i++) {
1187:     pthread_mutex_lock(job_main.mutexarray[i]);
1188:     while(*(job_main.arrThreadReady[i])==PETSC_FALSE) {
1189:       pthread_cond_wait(job_main.cond1array[i],job_main.mutexarray[i]);
1190:     }
1191:     pthread_mutex_unlock(job_main.mutexarray[i]);
1192:   }
1193: }

1197: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void** data,PetscInt n) {
1198:   int i,ierr;
1199:   PetscErrorCode ijoberr = 0;

1201:   MainWait(); /* you know everyone is waiting to be signalled! */
1202:   job_main.pfunc = pFunc;
1203:   job_main.pdata = data;
1204:   for(i=0; i<PetscMaxThreads; i++) {
1205:     *(job_main.arrThreadReady[i]) = PETSC_FALSE; /* why do this?  suppose you get into MainWait first */
1206:   }
1207:   /* tell the threads to go to work */
1208:   for(i=0; i<PetscMaxThreads; i++) {
1209:     pthread_cond_signal(job_main.cond2array[i]);
1210:   }
1211:   if(pFunc!=FuncFinish) {
1212:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1213:   }

1215:   if(ithreaderr) {
1216:     ijoberr = ithreaderr;
1217:   }
1218:   return ijoberr;
1219: }
1220: /****  ****/

1222: /**** Chain Thread Functions ****/
1223: void* PetscThreadFunc_Chain(void* arg) {
1224:   PetscErrorCode iterr;
1225:   int icorr,ierr;
1226:   int* pId = (int*)arg;
1227:   int ThreadId = *pId;
1228:   int SubWorker = ThreadId + 1;
1229:   PetscBool PeeOn;
1230:   cpu_set_t mset;
1231:   //printf("Thread %d In Chain Thread Function\n",ThreadId);
1232:   icorr = ThreadCoreAffinity[ThreadId];
1233:   CPU_ZERO(&mset);
1234:   CPU_SET(icorr,&mset);
1235:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1237:   if(ThreadId==(PetscMaxThreads-1)) {
1238:     PeeOn = PETSC_TRUE;
1239:   }
1240:   else {
1241:     PeeOn = PETSC_FALSE;
1242:   }
1243:   if(PeeOn==PETSC_FALSE) {
1244:     /* check your subordinate, wait for him to be ready */
1245:     pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1246:     while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE) {
1247:       /* upon entry, automically releases the lock and blocks
1248:        upon return, has the lock */
1249:       pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1250:     }
1251:     pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1252:     /* your subordinate is now ready*/
1253:   }
1254:   pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1255:   /* update your ready status */
1256:   *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1257:   if(ThreadId==0) {
1258:     job_chain.eJobStat = JobCompleted;
1259:     /* signal main */
1260:     pthread_cond_signal(&main_cond);
1261:   }
1262:   else {
1263:     /* tell your boss that you're ready to work */
1264:     pthread_cond_signal(job_chain.cond1array[ThreadId]);
1265:   }
1266:   /*  the while loop needs to have an exit
1267:      the 'main' thread can terminate all the threads by performing a broadcast
1268:    and calling FuncFinish */
1269:   while(PetscThreadGo) {
1270:     /* need to check the condition to ensure we don't have to wait
1271:        waiting when you don't have to causes problems
1272:      also need to check the condition to ensure proper handling of spurious wakeups */
1273:     while(*(job_chain.arrThreadReady[ThreadId])==PETSC_TRUE) {
1274:       /*upon entry, automically releases the lock and blocks
1275:        upon return, has the lock */
1276:         pthread_cond_wait(job_chain.cond2array[ThreadId],job_chain.mutexarray[ThreadId]);
1277:         *(job_chain.arrThreadStarted[ThreadId]) = PETSC_TRUE;
1278:         *(job_chain.arrThreadReady[ThreadId])   = PETSC_FALSE;
1279:     }
1280:     if(ThreadId==0) {
1281:       job_chain.startJob = PETSC_FALSE;
1282:       job_chain.eJobStat = ThreadsWorking;
1283:     }
1284:     pthread_mutex_unlock(job_chain.mutexarray[ThreadId]);
1285:     if(PeeOn==PETSC_FALSE) {
1286:       /* tell your subworker it's time to get to work */
1287:       pthread_cond_signal(job_chain.cond2array[SubWorker]);
1288:     }
1289:     /* do your job */
1290:     if(job_chain.pdata==NULL) {
1291:       iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata);
1292:     }
1293:     else {
1294:       iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata[ThreadId]);
1295:     }
1296:     if(iterr!=0) {
1297:       ithreaderr = 1;
1298:     }
1299:     if(PetscThreadGo) {
1300:       /* reset job, get ready for more */
1301:       if(PeeOn==PETSC_FALSE) {
1302:         /* check your subordinate, wait for him to be ready
1303:          how do you know for a fact that your subordinate has actually started? */
1304:         pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1305:         while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_chain.arrThreadStarted[SubWorker])==PETSC_FALSE) {
1306:           /* upon entry, automically releases the lock and blocks
1307:            upon return, has the lock */
1308:           pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1309:         }
1310:         pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1311:         /* your subordinate is now ready */
1312:       }
1313:       pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1314:       *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1315:       if(ThreadId==0) {
1316:         job_chain.eJobStat = JobCompleted; /* foreman: last thread to complete, guaranteed! */
1317:         /* root thread (foreman) signals 'main' */
1318:         pthread_cond_signal(&main_cond);
1319:       }
1320:       else {
1321:         /* signal your boss before you go to sleep */
1322:         pthread_cond_signal(job_chain.cond1array[ThreadId]);
1323:       }
1324:     }
1325:   }
1326:   return NULL;
1327: }

1331: void* PetscThreadInitialize_Chain(PetscInt N) {
1332:   PetscInt i,ierr;
1333:   int status;

1335:   if(PetscUseThreadPool) {
1336:     size_t Val1 = (size_t)CACHE_LINE_SIZE;
1337:     size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1338:     arrmutex = (char*)memalign(Val1,Val2);
1339:     arrcond1 = (char*)memalign(Val1,Val2);
1340:     arrcond2 = (char*)memalign(Val1,Val2);
1341:     arrstart = (char*)memalign(Val1,Val2);
1342:     arrready = (char*)memalign(Val1,Val2);
1343:     job_chain.mutexarray       = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1344:     job_chain.cond1array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1345:     job_chain.cond2array       = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1346:     job_chain.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1347:     job_chain.arrThreadReady   = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1348:     /* initialize job structure */
1349:     for(i=0; i<PetscMaxThreads; i++) {
1350:       job_chain.mutexarray[i]        = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1351:       job_chain.cond1array[i]        = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1352:       job_chain.cond2array[i]        = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1353:       job_chain.arrThreadStarted[i]  = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
1354:       job_chain.arrThreadReady[i]    = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1355:     }
1356:     for(i=0; i<PetscMaxThreads; i++) {
1357:       pthread_mutex_init(job_chain.mutexarray[i],NULL);
1358:       pthread_cond_init(job_chain.cond1array[i],NULL);
1359:       pthread_cond_init(job_chain.cond2array[i],NULL);
1360:       *(job_chain.arrThreadStarted[i])  = PETSC_FALSE;
1361:       *(job_chain.arrThreadReady[i])    = PETSC_FALSE;
1362:     }
1363:     job_chain.pfunc = NULL;
1364:     job_chain.pdata = (void**)malloc(N*sizeof(void*));
1365:     job_chain.startJob = PETSC_FALSE;
1366:     job_chain.eJobStat = JobInitiated;
1367:     pVal = (int*)malloc(N*sizeof(int));
1368:     /* allocate memory in the heap for the thread structure */
1369:     PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1370:     /* create threads */
1371:     for(i=0; i<N; i++) {
1372:       pVal[i] = i;
1373:       status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1374:       /* should check error */
1375:     }
1376:   }
1377:   else {
1378:   }
1379:   return NULL;
1380: }


1385: PetscErrorCode PetscThreadFinalize_Chain() {
1386:   int i,ierr;
1387:   void* jstatus;


1391:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1392:   /* join the threads */
1393:   for(i=0; i<PetscMaxThreads; i++) {
1394:     pthread_join(PetscThreadPoint[i],&jstatus);
1395:     /* should check error */
1396:   }
1397:   free(PetscThreadPoint);
1398:   free(arrmutex);
1399:   free(arrcond1);
1400:   free(arrcond2);
1401:   free(arrstart);
1402:   free(arrready);
1403:   free(job_chain.pdata);
1404:   free(pVal);

1406:   return(0);
1407: }

1411: void MainWait_Chain() {
1412:   int ierr;
1413:   pthread_mutex_lock(job_chain.mutexarray[0]);
1414:   while(job_chain.eJobStat<JobCompleted||job_chain.startJob==PETSC_TRUE) {
1415:     pthread_cond_wait(&main_cond,job_chain.mutexarray[0]);
1416:   }
1417:   pthread_mutex_unlock(job_chain.mutexarray[0]);
1418: }

1422: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void** data,PetscInt n) {
1423:   int i,ierr;
1424:   PetscErrorCode ijoberr = 0;

1426:   MainWait();
1427:   job_chain.pfunc = pFunc;
1428:   job_chain.pdata = data;
1429:   job_chain.startJob = PETSC_TRUE;
1430:   for(i=0; i<PetscMaxThreads; i++) {
1431:     *(job_chain.arrThreadStarted[i]) = PETSC_FALSE;
1432:   }
1433:   job_chain.eJobStat = JobInitiated;
1434:   pthread_cond_signal(job_chain.cond2array[0]);
1435:   if(pFunc!=FuncFinish) {
1436:     MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1437:   }

1439:   if(ithreaderr) {
1440:     ijoberr = ithreaderr;
1441:   }
1442:   return ijoberr;
1443: }
1444: /****  ****/

1446: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
1447: /**** True Thread Functions ****/
1448: void* PetscThreadFunc_True(void* arg) {
1449:   int icorr,ierr,iVal;
1450:   int* pId = (int*)arg;
1451:   int ThreadId = *pId;
1452:   PetscErrorCode iterr;
1453:   cpu_set_t mset;
1454:   //printf("Thread %d In True Pool Thread Function\n",ThreadId);
1455:   icorr = ThreadCoreAffinity[ThreadId];
1456:   CPU_ZERO(&mset);
1457:   CPU_SET(icorr,&mset);
1458:   sched_setaffinity(0,sizeof(cpu_set_t),&mset);

1460:   pthread_mutex_lock(&job_true.mutex);
1461:   job_true.iNumReadyThreads++;
1462:   if(job_true.iNumReadyThreads==PetscMaxThreads) {
1463:     pthread_cond_signal(&main_cond);
1464:   }
1465:   /*the while loop needs to have an exit
1466:     the 'main' thread can terminate all the threads by performing a broadcast
1467:    and calling FuncFinish */
1468:   while(PetscThreadGo) {
1469:     /*need to check the condition to ensure we don't have to wait
1470:       waiting when you don't have to causes problems
1471:      also need to wait if another thread sneaks in and messes with the predicate */
1472:     while(job_true.startJob==PETSC_FALSE&&job_true.iNumJobThreads==0) {
1473:       /* upon entry, automically releases the lock and blocks
1474:        upon return, has the lock */
1475:       //printf("Thread %d Going to Sleep!\n",ThreadId);
1476:       pthread_cond_wait(&job_true.cond,&job_true.mutex);
1477:     }
1478:     job_true.startJob = PETSC_FALSE;
1479:     job_true.iNumJobThreads--;
1480:     job_true.iNumReadyThreads--;
1481:     iVal = PetscMaxThreads-job_true.iNumReadyThreads-1;
1482:     pthread_mutex_unlock(&job_true.mutex);
1483:     if(job_true.pdata==NULL) {
1484:       iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata);
1485:     }
1486:     else {
1487:       iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata[iVal]);
1488:     }
1489:     if(iterr!=0) {
1490:       ithreaderr = 1;
1491:     }
1492:     //printf("Thread %d Finished Job\n",ThreadId);
1493:     /* the barrier is necessary BECAUSE: look at job_true.iNumReadyThreads
1494:       what happens if a thread finishes before they all start? BAD!
1495:      what happens if a thread finishes before any else start? BAD! */
1496:     pthread_barrier_wait(job_true.pbarr); /* ensures all threads are finished */
1497:     /* reset job */
1498:     if(PetscThreadGo) {
1499:       pthread_mutex_lock(&job_true.mutex);
1500:       job_true.iNumReadyThreads++;
1501:       if(job_true.iNumReadyThreads==PetscMaxThreads) {
1502:         /* signal the 'main' thread that the job is done! (only done once) */
1503:         pthread_cond_signal(&main_cond);
1504:       }
1505:     }
1506:   }
1507:   return NULL;
1508: }

1512: void* PetscThreadInitialize_True(PetscInt N) {
1513:   PetscInt i;
1514:   int status;

1516:   pVal = (int*)malloc(N*sizeof(int));
1517:   /* allocate memory in the heap for the thread structure */
1518:   PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1519:   BarrPoint = (pthread_barrier_t*)malloc((N+1)*sizeof(pthread_barrier_t)); /* BarrPoint[0] makes no sense, don't use it! */
1520:   job_true.pdata = (void**)malloc(N*sizeof(void*));
1521:   for(i=0; i<N; i++) {
1522:     pVal[i] = i;
1523:     status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1524:     /* error check to ensure proper thread creation */
1525:     status = pthread_barrier_init(&BarrPoint[i+1],NULL,i+1);
1526:     /* should check error */
1527:   }
1528:   //printf("Finished True Thread Pool Initialization\n");
1529:   return NULL;
1530: }


1535: PetscErrorCode PetscThreadFinalize_True() {
1536:   int i,ierr;
1537:   void* jstatus;


1541:   MainJob(FuncFinish,NULL,PetscMaxThreads);  /* set up job and broadcast work */
1542:   /* join the threads */
1543:   for(i=0; i<PetscMaxThreads; i++) {
1544:     pthread_join(PetscThreadPoint[i],&jstatus);
1545:   }
1546:   free(BarrPoint);
1547:   free(PetscThreadPoint);

1549:   return(0);
1550: }

1554: void MainWait_True() {
1555:   int ierr;
1556:   pthread_mutex_lock(&job_true.mutex);
1557:   while(job_true.iNumReadyThreads<PetscMaxThreads||job_true.startJob==PETSC_TRUE) {
1558:     pthread_cond_wait(&main_cond,&job_true.mutex);
1559:   }
1560:   pthread_mutex_unlock(&job_true.mutex);
1561: }

1565: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void** data,PetscInt n) {
1566:   int ierr;
1567:   PetscErrorCode ijoberr = 0;

1569:   MainWait();
1570:   job_true.pfunc = pFunc;
1571:   job_true.pdata = data;
1572:   job_true.pbarr = &BarrPoint[n];
1573:   job_true.iNumJobThreads = n;
1574:   job_true.startJob = PETSC_TRUE;
1575:   pthread_cond_broadcast(&job_true.cond);
1576:   if(pFunc!=FuncFinish) {
1577:     MainWait(); /* why wait after? guarantees that job gets done */
1578:   }

1580:   if(ithreaderr) {
1581:     ijoberr = ithreaderr;
1582:   }
1583:   return ijoberr;
1584: }
1585: /**** NO THREAD POOL FUNCTION ****/
1588: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void** data,PetscInt n) {
1589:   PetscErrorCode ijoberr = 0;

1591:   pthread_t* apThread = (pthread_t*)malloc(n*sizeof(pthread_t));
1592:   PetscThreadPoint = apThread; /* point to same place */
1593:   PetscThreadRun(MPI_COMM_WORLD,pFunc,n,apThread,data);
1594:   PetscThreadStop(MPI_COMM_WORLD,n,apThread); /* ensures that all threads are finished with the job */
1595:   free(apThread);

1597:   return ijoberr;
1598: }
1599: /****  ****/
1600: #endif

1602: void* FuncFinish(void* arg) {
1603:   PetscThreadGo = PETSC_FALSE;
1604:   return(0);
1605: }

1607: #endif