Actual source code: init.c
1: /*
3: This file defines part of the initialization of PETSc
5: This file uses regular malloc and free because it cannot know
6: what malloc is being used until it has already processed the input.
7: */
9: #include <petscsys.h> /*I "petscsys.h" I*/
11: #ifndef _GNU_SOURCE
12: #define _GNU_SOURCE
13: #endif
14: #if defined(PETSC_HAVE_SCHED_H)
15: #ifndef __USE_GNU
17: #endif
18: #include <sched.h>
19: #endif
20: #if defined(PETSC_HAVE_PTHREAD_H)
21: #include <pthread.h>
22: #endif
24: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
25: #include <sys/sysinfo.h>
26: #endif
27: #if defined(PETSC_HAVE_UNISTD_H)
28: #include <unistd.h>
29: #endif
30: #if defined(PETSC_HAVE_STDLIB_H)
31: #include <stdlib.h>
32: #endif
33: #if defined(PETSC_HAVE_MALLOC_H)
34: #include <malloc.h>
35: #endif
36: #if defined(PETSC_HAVE_VALGRIND)
37: #include <valgrind/valgrind.h>
38: #endif
40: /* ------------------------Nasty global variables -------------------------------*/
41: /*
42: Indicates if PETSc started up MPI, or it was
43: already started before PETSc was initialized.
44: */
45: PetscBool PetscBeganMPI = PETSC_FALSE;
46: PetscBool PetscInitializeCalled = PETSC_FALSE;
47: PetscBool PetscFinalizeCalled = PETSC_FALSE;
48: PetscBool PetscUseThreadPool = PETSC_FALSE;
49: PetscBool PetscThreadGo = PETSC_TRUE;
50: PetscMPIInt PetscGlobalRank = -1;
51: PetscMPIInt PetscGlobalSize = -1;
53: #if defined(PETSC_HAVE_PTHREADCLASSES)
54: PetscMPIInt PetscMaxThreads = 2;
55: pthread_t* PetscThreadPoint;
56: #define PETSC_HAVE_PTHREAD_BARRIER
57: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
58: pthread_barrier_t* BarrPoint; /* used by 'true' thread pool */
59: #endif
60: PetscErrorCode ithreaderr = 0;
61: int* pVal;
63: #define CACHE_LINE_SIZE 64 /* used by 'chain', 'main','tree' thread pools */
64: int* ThreadCoreAffinity;
66: typedef enum {JobInitiated,ThreadsWorking,JobCompleted} estat; /* used by 'chain','tree' thread pool */
68: typedef struct {
69: pthread_mutex_t** mutexarray;
70: pthread_cond_t** cond1array;
71: pthread_cond_t** cond2array;
72: void* (*pfunc)(void*);
73: void** pdata;
74: PetscBool startJob;
75: estat eJobStat;
76: PetscBool** arrThreadStarted;
77: PetscBool** arrThreadReady;
78: } sjob_tree;
79: sjob_tree job_tree;
80: typedef struct {
81: pthread_mutex_t** mutexarray;
82: pthread_cond_t** cond1array;
83: pthread_cond_t** cond2array;
84: void* (*pfunc)(void*);
85: void** pdata;
86: PetscBool** arrThreadReady;
87: } sjob_main;
88: sjob_main job_main;
89: typedef struct {
90: pthread_mutex_t** mutexarray;
91: pthread_cond_t** cond1array;
92: pthread_cond_t** cond2array;
93: void* (*pfunc)(void*);
94: void** pdata;
95: PetscBool startJob;
96: estat eJobStat;
97: PetscBool** arrThreadStarted;
98: PetscBool** arrThreadReady;
99: } sjob_chain;
100: sjob_chain job_chain;
101: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
102: typedef struct {
103: pthread_mutex_t mutex;
104: pthread_cond_t cond;
105: void* (*pfunc)(void*);
106: void** pdata;
107: pthread_barrier_t* pbarr;
108: int iNumJobThreads;
109: int iNumReadyThreads;
110: PetscBool startJob;
111: } sjob_true;
112: sjob_true job_true = {PTHREAD_MUTEX_INITIALIZER,PTHREAD_COND_INITIALIZER,NULL,NULL,NULL,0,0,PETSC_FALSE};
113: #endif
115: pthread_cond_t main_cond = PTHREAD_COND_INITIALIZER; /* used by 'true', 'chain','tree' thread pools */
116: char* arrmutex; /* used by 'chain','main','tree' thread pools */
117: char* arrcond1; /* used by 'chain','main','tree' thread pools */
118: char* arrcond2; /* used by 'chain','main','tree' thread pools */
119: char* arrstart; /* used by 'chain','main','tree' thread pools */
120: char* arrready; /* used by 'chain','main','tree' thread pools */
122: /* Function Pointers */
123: void* (*PetscThreadFunc)(void*) = NULL;
124: void* (*PetscThreadInitialize)(PetscInt) = NULL;
125: PetscErrorCode (*PetscThreadFinalize)(void) = NULL;
126: void (*MainWait)(void) = NULL;
127: PetscErrorCode (*MainJob)(void* (*pFunc)(void*),void**,PetscInt) = NULL;
128: /**** Tree Thread Pool Functions ****/
129: void* PetscThreadFunc_Tree(void*);
130: void* PetscThreadInitialize_Tree(PetscInt);
131: PetscErrorCode PetscThreadFinalize_Tree(void);
132: void MainWait_Tree(void);
133: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void**,PetscInt);
134: /**** Main Thread Pool Functions ****/
135: void* PetscThreadFunc_Main(void*);
136: void* PetscThreadInitialize_Main(PetscInt);
137: PetscErrorCode PetscThreadFinalize_Main(void);
138: void MainWait_Main(void);
139: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void**,PetscInt);
140: /**** Chain Thread Pool Functions ****/
141: void* PetscThreadFunc_Chain(void*);
142: void* PetscThreadInitialize_Chain(PetscInt);
143: PetscErrorCode PetscThreadFinalize_Chain(void);
144: void MainWait_Chain(void);
145: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void**,PetscInt);
146: /**** True Thread Pool Functions ****/
147: void* PetscThreadFunc_True(void*);
148: void* PetscThreadInitialize_True(PetscInt);
149: PetscErrorCode PetscThreadFinalize_True(void);
150: void MainWait_True(void);
151: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void**,PetscInt);
152: /**** NO Thread Pool Function ****/
153: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void**,PetscInt);
154: /**** ****/
155: void* FuncFinish(void*);
156: void* PetscThreadRun(MPI_Comm Comm,void* (*pFunc)(void*),int,pthread_t*,void**);
157: void* PetscThreadStop(MPI_Comm Comm,int,pthread_t*);
158: #endif
160: #if defined(PETSC_USE_COMPLEX)
161: #if defined(PETSC_COMPLEX_INSTANTIATE)
162: template <> class std::complex<double>; /* instantiate complex template class */
163: #endif
164: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
165: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
166: MPI_Datatype MPIU_C_COMPLEX;
167: #endif
168: PetscScalar PETSC_i;
169: #else
170: PetscScalar PETSC_i = 0.0;
171: #endif
172: #if defined(PETSC_USE_REAL___FLOAT128)
173: MPI_Datatype MPIU___FLOAT128 = 0;
174: #endif
175: MPI_Datatype MPIU_2SCALAR = 0;
176: MPI_Datatype MPIU_2INT = 0;
178: /*
179: These are needed by petscbt.h
180: */
181: #include <petscbt.h>
182: char _BT_mask = ' ';
183: char _BT_c = ' ';
184: PetscInt _BT_idx = 0;
186: /*
187: Function that is called to display all error messages
188: */
189: PetscErrorCode (*PetscErrorPrintf)(const char [],...) = PetscErrorPrintfDefault;
190: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...) = PetscHelpPrintfDefault;
191: #if defined(PETSC_HAVE_MATLAB_ENGINE)
192: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintf_Matlab;
193: #else
194: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list) = PetscVFPrintfDefault;
195: #endif
196: /*
197: This is needed to turn on/off cusp synchronization */
198: PetscBool synchronizeCUSP = PETSC_FALSE;
200: /* ------------------------------------------------------------------------------*/
201: /*
202: Optional file where all PETSc output from various prints is saved
203: */
204: FILE *petsc_history = PETSC_NULL;
208: PetscErrorCode PetscOpenHistoryFile(const char filename[],FILE **fd)
209: {
211: PetscMPIInt rank,size;
212: char pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
213: char version[256];
216: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
217: if (!rank) {
218: char arch[10];
219: int err;
220: PetscViewer viewer;
222: PetscGetArchType(arch,10);
223: PetscGetDate(date,64);
224: PetscGetVersion(version,256);
225: MPI_Comm_size(PETSC_COMM_WORLD,&size);
226: if (filename) {
227: PetscFixFilename(filename,fname);
228: } else {
229: PetscGetHomeDirectory(pfile,240);
230: PetscStrcat(pfile,"/.petschistory");
231: PetscFixFilename(pfile,fname);
232: }
234: *fd = fopen(fname,"a"); if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);
235: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
236: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
237: PetscGetProgramName(pname,PETSC_MAX_PATH_LEN);
238: PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
239: PetscViewerASCIIOpenWithFILE(PETSC_COMM_WORLD,*fd,&viewer);
240: PetscOptionsView(viewer);
241: PetscViewerDestroy(&viewer);
242: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
243: err = fflush(*fd);
244: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
245: }
246: return(0);
247: }
251: PetscErrorCode PetscCloseHistoryFile(FILE **fd)
252: {
254: PetscMPIInt rank;
255: char date[64];
256: int err;
259: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
260: if (!rank) {
261: PetscGetDate(date,64);
262: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
263: PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
264: PetscFPrintf(PETSC_COMM_SELF,*fd,"---------------------------------------------------------\n");
265: err = fflush(*fd);
266: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
267: err = fclose(*fd);
268: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
269: }
270: return(0);
271: }
273: /* ------------------------------------------------------------------------------*/
275: /*
276: This is ugly and probably belongs somewhere else, but I want to
277: be able to put a true MPI abort error handler with command line args.
279: This is so MPI errors in the debugger will leave all the stack
280: frames. The default MP_Abort() cleans up and exits thus providing no useful information
281: in the debugger hence we call abort() instead of MPI_Abort().
282: */
286: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag)
287: {
289: (*PetscErrorPrintf)("MPI error %d\n",*flag);
290: abort();
291: }
295: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag)
296: {
300: (*PetscErrorPrintf)("MPI error %d\n",*flag);
301: PetscAttachDebugger();
302: if (ierr) { /* hopeless so get out */
303: MPI_Abort(*comm,*flag);
304: }
305: }
309: /*@C
310: PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
311: wishes a clean exit somewhere deep in the program.
313: Collective on PETSC_COMM_WORLD
315: Options Database Keys are the same as for PetscFinalize()
317: Level: advanced
319: Note:
320: See PetscInitialize() for more general runtime options.
322: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
323: @*/
324: PetscErrorCode PetscEnd(void)
325: {
327: PetscFinalize();
328: exit(0);
329: return 0;
330: }
332: PetscBool PetscOptionsPublish = PETSC_FALSE;
335: static char emacsmachinename[256];
337: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = 0;
338: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm) = 0;
342: /*@C
343: PetscSetHelpVersionFunctions - Sets functions that print help and version information
344: before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
345: This routine enables a "higher-level" package that uses PETSc to print its messages first.
347: Input Parameter:
348: + help - the help function (may be PETSC_NULL)
349: - version - the version function (may be PETSC_NULL)
351: Level: developer
353: Concepts: package help message
355: @*/
356: PetscErrorCode PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
357: {
359: PetscExternalHelpFunction = help;
360: PetscExternalVersionFunction = version;
361: return(0);
362: }
366: PetscErrorCode PetscOptionsCheckInitial_Private(void)
367: {
368: char string[64],mname[PETSC_MAX_PATH_LEN],*f;
369: MPI_Comm comm = PETSC_COMM_WORLD;
370: PetscBool flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flg4 = PETSC_FALSE,flag,flgz,flgzout;
372: PetscReal si;
373: int i;
374: PetscMPIInt rank;
375: char version[256];
378: MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
380: /*
381: Setup the memory management; support for tracing malloc() usage
382: */
383: PetscOptionsHasName(PETSC_NULL,"-malloc_log",&flg3);
384: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
385: PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg1,&flg2);
386: if ((!flg2 || flg1) && !petscsetmallocvisited) {
387: #if defined(PETSC_HAVE_VALGRIND)
388: if (flg2 || !(RUNNING_ON_VALGRIND)) {
389: /* turn off default -malloc if valgrind is being used */
390: #endif
391: PetscSetUseTrMalloc_Private();
392: #if defined(PETSC_HAVE_VALGRIND)
393: }
394: #endif
395: }
396: #else
397: PetscOptionsGetBool(PETSC_NULL,"-malloc_dump",&flg1,PETSC_NULL);
398: PetscOptionsGetBool(PETSC_NULL,"-malloc",&flg2,PETSC_NULL);
399: if (flg1 || flg2 || flg3) {PetscSetUseTrMalloc_Private();}
400: #endif
401: if (flg3) {
402: PetscMallocSetDumpLog();
403: }
404: flg1 = PETSC_FALSE;
405: PetscOptionsGetBool(PETSC_NULL,"-malloc_debug",&flg1,PETSC_NULL);
406: if (flg1) {
407: PetscSetUseTrMalloc_Private();
408: PetscMallocDebug(PETSC_TRUE);
409: }
411: flg1 = PETSC_FALSE;
412: PetscOptionsGetBool(PETSC_NULL,"-malloc_info",&flg1,PETSC_NULL);
413: if (!flg1) {
414: flg1 = PETSC_FALSE;
415: PetscOptionsGetBool(PETSC_NULL,"-memory_info",&flg1,PETSC_NULL);
416: }
417: if (flg1) {
418: PetscMemorySetGetMaximumUsage();
419: }
421: /*
422: Set the display variable for graphics
423: */
424: PetscSetDisplay();
427: /*
428: Print the PETSc version information
429: */
430: PetscOptionsHasName(PETSC_NULL,"-v",&flg1);
431: PetscOptionsHasName(PETSC_NULL,"-version",&flg2);
432: PetscOptionsHasName(PETSC_NULL,"-help",&flg3);
433: if (flg1 || flg2 || flg3){
435: /*
436: Print "higher-level" package version message
437: */
438: if (PetscExternalVersionFunction) {
439: (*PetscExternalVersionFunction)(comm);
440: }
442: PetscGetVersion(version,256);
443: (*PetscHelpPrintf)(comm,"--------------------------------------------\
444: ------------------------------\n");
445: (*PetscHelpPrintf)(comm,"%s\n",version);
446: (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
447: (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
448: (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
449: (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
450: (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
451: (*PetscHelpPrintf)(comm,"--------------------------------------------\
452: ------------------------------\n");
453: }
455: /*
456: Print "higher-level" package help message
457: */
458: if (flg3){
459: if (PetscExternalHelpFunction) {
460: (*PetscExternalHelpFunction)(comm);
461: }
462: }
464: /*
465: Setup the error handling
466: */
467: flg1 = PETSC_FALSE;
468: PetscOptionsGetBool(PETSC_NULL,"-on_error_abort",&flg1,PETSC_NULL);
469: if (flg1) { PetscPushErrorHandler(PetscAbortErrorHandler,0);}
470: flg1 = PETSC_FALSE;
471: PetscOptionsGetBool(PETSC_NULL,"-on_error_mpiabort",&flg1,PETSC_NULL);
472: if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,0);}
473: flg1 = PETSC_FALSE;
474: PetscOptionsGetBool(PETSC_NULL,"-mpi_return_on_error",&flg1,PETSC_NULL);
475: if (flg1) {
476: MPI_Errhandler_set(comm,MPI_ERRORS_RETURN);
477: }
478: flg1 = PETSC_FALSE;
479: PetscOptionsGetBool(PETSC_NULL,"-no_signal_handler",&flg1,PETSC_NULL);
480: if (!flg1) {PetscPushSignalHandler(PetscDefaultSignalHandler,(void*)0);}
481: flg1 = PETSC_FALSE;
482: PetscOptionsGetBool(PETSC_NULL,"-fp_trap",&flg1,PETSC_NULL);
483: if (flg1) {PetscSetFPTrap(PETSC_FP_TRAP_ON);}
485: /*
486: Setup debugger information
487: */
488: PetscSetDefaultDebugger();
489: PetscOptionsGetString(PETSC_NULL,"-on_error_attach_debugger",string,64,&flg1);
490: if (flg1) {
491: MPI_Errhandler err_handler;
493: PetscSetDebuggerFromString(string);
494: MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_DebuggerOnError,&err_handler);
495: MPI_Errhandler_set(comm,err_handler);
496: PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,0);
497: }
498: PetscOptionsGetString(PETSC_NULL,"-debug_terminal",string,64,&flg1);
499: if (flg1) { PetscSetDebugTerminal(string); }
500: PetscOptionsGetString(PETSC_NULL,"-start_in_debugger",string,64,&flg1);
501: PetscOptionsGetString(PETSC_NULL,"-stop_for_debugger",string,64,&flg2);
502: if (flg1 || flg2) {
503: PetscMPIInt size;
504: PetscInt lsize,*nodes;
505: MPI_Errhandler err_handler;
506: /*
507: we have to make sure that all processors have opened
508: connections to all other processors, otherwise once the
509: debugger has stated it is likely to receive a SIGUSR1
510: and kill the program.
511: */
512: MPI_Comm_size(PETSC_COMM_WORLD,&size);
513: if (size > 2) {
514: PetscMPIInt dummy = 0;
515: MPI_Status status;
516: for (i=0; i<size; i++) {
517: if (rank != i) {
518: MPI_Send(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD);
519: }
520: }
521: for (i=0; i<size; i++) {
522: if (rank != i) {
523: MPI_Recv(&dummy,1,MPI_INT,i,109,PETSC_COMM_WORLD,&status);
524: }
525: }
526: }
527: /* check if this processor node should be in debugger */
528: PetscMalloc(size*sizeof(PetscInt),&nodes);
529: lsize = size;
530: PetscOptionsGetIntArray(PETSC_NULL,"-debugger_nodes",nodes,&lsize,&flag);
531: if (flag) {
532: for (i=0; i<lsize; i++) {
533: if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
534: }
535: }
536: if (!flag) {
537: PetscSetDebuggerFromString(string);
538: PetscPushErrorHandler(PetscAbortErrorHandler,0);
539: if (flg1) {
540: PetscAttachDebugger();
541: } else {
542: PetscStopForDebugger();
543: }
544: MPI_Errhandler_create((MPI_Handler_function*)Petsc_MPI_AbortOnError,&err_handler);
545: MPI_Errhandler_set(comm,err_handler);
546: }
547: PetscFree(nodes);
548: }
550: PetscOptionsGetString(PETSC_NULL,"-on_error_emacs",emacsmachinename,128,&flg1);
551: if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}
553: #if defined(PETSC_USE_SOCKET_VIEWER)
554: /*
555: Activates new sockets for zope if needed
556: */
557: PetscOptionsHasName(PETSC_NULL,"-zope", &flgz);
558: PetscOptionsHasName(PETSC_NULL,"-nostdout", &flgzout);
559: if (flgz){
560: int sockfd;
561: char hostname[256];
562: char username[256];
563: int remoteport = 9999;
565: PetscOptionsGetString(PETSC_NULL, "-zope", hostname, 256, &flgz);
566: if (!hostname[0]){
567: PetscGetHostName(hostname,256);
568: }
569: PetscOpenSocket(hostname, remoteport, &sockfd);
570: PetscGetUserName(username, 256);
571: PETSC_ZOPEFD = fdopen(sockfd, "w");
572: if (flgzout){
573: PETSC_STDOUT = PETSC_ZOPEFD;
574: fprintf(PETSC_STDOUT, "<<<user>>> %s\n",username);
575: fprintf(PETSC_STDOUT, "<<<start>>>");
576: } else {
577: fprintf(PETSC_ZOPEFD, "<<<user>>> %s\n",username);
578: fprintf(PETSC_ZOPEFD, "<<<start>>>");
579: }
580: }
581: #endif
582: #if defined(PETSC_USE_SERVER)
583: PetscOptionsHasName(PETSC_NULL,"-server", &flgz);
584: if (flgz){
585: PetscInt port = PETSC_DECIDE;
586: PetscOptionsGetInt(PETSC_NULL,"-server",&port,PETSC_NULL);
587: PetscWebServe(PETSC_COMM_WORLD,(int)port);
588: }
589: #endif
591: /*
592: Setup profiling and logging
593: */
594: #if defined (PETSC_USE_INFO)
595: {
596: char logname[PETSC_MAX_PATH_LEN]; logname[0] = 0;
597: PetscOptionsGetString(PETSC_NULL,"-info",logname,250,&flg1);
598: if (flg1 && logname[0]) {
599: PetscInfoAllow(PETSC_TRUE,logname);
600: } else if (flg1) {
601: PetscInfoAllow(PETSC_TRUE,PETSC_NULL);
602: }
603: }
604: #endif
605: #if defined(PETSC_USE_LOG)
606: mname[0] = 0;
607: PetscOptionsGetString(PETSC_NULL,"-history",mname,PETSC_MAX_PATH_LEN,&flg1);
608: if (flg1) {
609: if (mname[0]) {
610: PetscOpenHistoryFile(mname,&petsc_history);
611: } else {
612: PetscOpenHistoryFile(0,&petsc_history);
613: }
614: }
615: #if defined(PETSC_HAVE_MPE)
616: flg1 = PETSC_FALSE;
617: PetscOptionsHasName(PETSC_NULL,"-log_mpe",&flg1);
618: if (flg1) PetscLogMPEBegin();
619: #endif
620: flg1 = PETSC_FALSE;
621: flg2 = PETSC_FALSE;
622: flg3 = PETSC_FALSE;
623: PetscOptionsGetBool(PETSC_NULL,"-log_all",&flg1,PETSC_NULL);
624: PetscOptionsGetBool(PETSC_NULL,"-log",&flg2,PETSC_NULL);
625: PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
626: PetscOptionsHasName(PETSC_NULL,"-log_summary_python",&flg4);
627: if (flg1) { PetscLogAllBegin(); }
628: else if (flg2 || flg3 || flg4) { PetscLogBegin();}
629:
630: PetscOptionsGetString(PETSC_NULL,"-log_trace",mname,250,&flg1);
631: if (flg1) {
632: char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
633: FILE *file;
634: if (mname[0]) {
635: sprintf(name,"%s.%d",mname,rank);
636: PetscFixFilename(name,fname);
637: file = fopen(fname,"w");
638: if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
639: } else {
640: file = PETSC_STDOUT;
641: }
642: PetscLogTraceBegin(file);
643: }
644: #endif
646: /*
647: Setup building of stack frames for all function calls
648: */
649: #if defined(PETSC_USE_DEBUG) && !defined(PETSC_USE_PTHREAD)
650: PetscStackCreate();
651: #endif
653: PetscOptionsGetBool(PETSC_NULL,"-options_gui",&PetscOptionsPublish,PETSC_NULL);
655: #if defined(PETSC_HAVE_PTHREADCLASSES)
656: /*
657: Determine whether user specified maximum number of threads
658: */
659: PetscOptionsGetInt(PETSC_NULL,"-thread_max",&PetscMaxThreads,PETSC_NULL);
661: PetscOptionsHasName(PETSC_NULL,"-main",&flg1);
662: if(flg1) {
663: cpu_set_t mset;
664: int icorr,ncorr = get_nprocs();
665: PetscOptionsGetInt(PETSC_NULL,"-main",&icorr,PETSC_NULL);
666: CPU_ZERO(&mset);
667: CPU_SET(icorr%ncorr,&mset);
668: sched_setaffinity(0,sizeof(cpu_set_t),&mset);
669: }
671: PetscInt N_CORES = get_nprocs();
672: ThreadCoreAffinity = (int*)malloc(N_CORES*sizeof(int));
673: char tstr[9];
674: char tbuf[2];
675: strcpy(tstr,"-thread");
676: for(i=0;i<PetscMaxThreads;i++) {
677: ThreadCoreAffinity[i] = i;
678: sprintf(tbuf,"%d",i);
679: strcat(tstr,tbuf);
680: PetscOptionsHasName(PETSC_NULL,tstr,&flg1);
681: if(flg1) {
682: PetscOptionsGetInt(PETSC_NULL,tstr,&ThreadCoreAffinity[i],PETSC_NULL);
683: ThreadCoreAffinity[i] = ThreadCoreAffinity[i]%N_CORES; /* check on the user */
684: }
685: tstr[7] = '\0';
686: }
688: /*
689: Determine whether to use thread pool
690: */
691: PetscOptionsHasName(PETSC_NULL,"-use_thread_pool",&flg1);
692: if (flg1) {
693: PetscUseThreadPool = PETSC_TRUE;
694: /* get the thread pool type */
695: PetscInt ipool = 0;
696: const char *choices[4] = {"true","tree","main","chain"};
698: PetscOptionsGetEList(PETSC_NULL,"-use_thread_pool",choices,4,&ipool,PETSC_NULL);
699: switch(ipool) {
700: case 1:
701: PetscThreadFunc = &PetscThreadFunc_Tree;
702: PetscThreadInitialize = &PetscThreadInitialize_Tree;
703: PetscThreadFinalize = &PetscThreadFinalize_Tree;
704: MainWait = &MainWait_Tree;
705: MainJob = &MainJob_Tree;
706: PetscInfo(PETSC_NULL,"Using tree thread pool\n");
707: break;
708: case 2:
709: PetscThreadFunc = &PetscThreadFunc_Main;
710: PetscThreadInitialize = &PetscThreadInitialize_Main;
711: PetscThreadFinalize = &PetscThreadFinalize_Main;
712: MainWait = &MainWait_Main;
713: MainJob = &MainJob_Main;
714: PetscInfo(PETSC_NULL,"Using main thread pool\n");
715: break;
716: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
717: case 3:
718: #else
719: default:
720: #endif
721: PetscThreadFunc = &PetscThreadFunc_Chain;
722: PetscThreadInitialize = &PetscThreadInitialize_Chain;
723: PetscThreadFinalize = &PetscThreadFinalize_Chain;
724: MainWait = &MainWait_Chain;
725: MainJob = &MainJob_Chain;
726: PetscInfo(PETSC_NULL,"Using chain thread pool\n");
727: break;
728: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
729: default:
730: PetscThreadFunc = &PetscThreadFunc_True;
731: PetscThreadInitialize = &PetscThreadInitialize_True;
732: PetscThreadFinalize = &PetscThreadFinalize_True;
733: MainWait = &MainWait_True;
734: MainJob = &MainJob_True;
735: PetscInfo(PETSC_NULL,"Using true thread pool\n");
736: break;
737: #endif
738: }
739: PetscThreadInitialize(PetscMaxThreads);
740: } else {
741: //need to define these in the case on 'no threads' or 'thread create/destroy'
742: //could take any of the above versions
743: MainJob = &MainJob_Spawn;
744: }
745: #endif
746: /*
747: Print basic help message
748: */
749: PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
750: if (flg1) {
751: (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
752: (*PetscHelpPrintf)(comm," -help: prints help method for each option\n");
753: (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
754: (*PetscHelpPrintf)(comm," only when run in the debugger\n");
755: (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
756: (*PetscHelpPrintf)(comm," start the debugger in new xterm\n");
757: (*PetscHelpPrintf)(comm," unless noxterm is given\n");
758: (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
759: (*PetscHelpPrintf)(comm," start all processes in the debugger\n");
760: (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
761: (*PetscHelpPrintf)(comm," emacs jumps to error file\n");
762: (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
763: (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
764: (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
765: (*PetscHelpPrintf)(comm," waits the delay for you to attach\n");
766: (*PetscHelpPrintf)(comm," -display display: Location where graphics and debuggers are displayed\n");
767: (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
768: (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
769: (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
770: (*PetscHelpPrintf)(comm," note on IBM RS6000 this slows run greatly\n");
771: (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
772: (*PetscHelpPrintf)(comm," -malloc: use our error checking malloc\n");
773: (*PetscHelpPrintf)(comm," -malloc no: don't use error checking malloc\n");
774: (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
775: (*PetscHelpPrintf)(comm," -malloc_log: keeps log of all memory allocations\n");
776: (*PetscHelpPrintf)(comm," -malloc_debug: enables extended checking for memory corruption\n");
777: (*PetscHelpPrintf)(comm," -options_table: dump list of options inputted\n");
778: (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
779: (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
780: (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
781: (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
782: (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
783: (*PetscHelpPrintf)(comm," -memory_info: print memory usage at end of run\n");
784: (*PetscHelpPrintf)(comm," -server <port>: Run PETSc webserver (default port is 8080) see PetscWebServe()\n");
785: #if defined(PETSC_USE_LOG)
786: (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
787: (*PetscHelpPrintf)(comm," -log[_all _summary _summary_python]: logging objects and events\n");
788: (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
789: #if defined(PETSC_HAVE_MPE)
790: (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through upshot\n");
791: #endif
792: (*PetscHelpPrintf)(comm," -info <optional filename>: print informative messages about the calculations\n");
793: #endif
794: (*PetscHelpPrintf)(comm," -v: prints PETSc version number and release date\n");
795: (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
796: (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
797: (*PetscHelpPrintf)(comm,"-----------------------------------------------\n");
798: }
800: PetscOptionsGetReal(PETSC_NULL,"-petsc_sleep",&si,&flg1);
801: if (flg1) {
802: PetscSleep(si);
803: }
805: PetscOptionsGetString(PETSC_NULL,"-info_exclude",mname,PETSC_MAX_PATH_LEN,&flg1);
806: PetscStrstr(mname,"null",&f);
807: if (f) {
808: PetscInfoDeactivateClass(PETSC_NULL);
809: }
811: #if defined(PETSC_HAVE_CUSP)
812: PetscOptionsHasName(PETSC_NULL,"-log_summary",&flg3);
813: if (flg3) flg1 = PETSC_TRUE;
814: else flg1 = PETSC_FALSE;
815: PetscOptionsGetBool(PETSC_NULL,"-cusp_synchronize",&flg1,PETSC_NULL);
816: if (flg1) synchronizeCUSP = PETSC_TRUE;
817: #endif
819: return(0);
820: }
822: #if defined(PETSC_HAVE_PTHREADCLASSES)
824: /**** 'Tree' Thread Pool Functions ****/
825: void* PetscThreadFunc_Tree(void* arg) {
826: PetscErrorCode iterr;
827: int icorr,ierr;
828: int* pId = (int*)arg;
829: int ThreadId = *pId,Mary = 2,i,SubWorker;
830: PetscBool PeeOn;
831: cpu_set_t mset;
832: //printf("Thread %d In Tree Thread Function\n",ThreadId);
833: icorr = ThreadCoreAffinity[ThreadId];
834: CPU_ZERO(&mset);
835: CPU_SET(icorr,&mset);
836: sched_setaffinity(0,sizeof(cpu_set_t),&mset);
838: if((Mary*ThreadId+1)>(PetscMaxThreads-1)) {
839: PeeOn = PETSC_TRUE;
840: }
841: else {
842: PeeOn = PETSC_FALSE;
843: }
844: if(PeeOn==PETSC_FALSE) {
845: /* check your subordinates, wait for them to be ready */
846: for(i=1;i<=Mary;i++) {
847: SubWorker = Mary*ThreadId+i;
848: if(SubWorker<PetscMaxThreads) {
849: pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
850: while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE) {
851: /* upon entry, automically releases the lock and blocks
852: upon return, has the lock */
853: pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
854: }
855: pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
856: }
857: }
858: /* your subordinates are now ready */
859: }
860: pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
861: /* update your ready status */
862: *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
863: if(ThreadId==0) {
864: job_tree.eJobStat = JobCompleted;
865: /* ignal main */
866: pthread_cond_signal(&main_cond);
867: }
868: else {
869: /* tell your boss that you're ready to work */
870: pthread_cond_signal(job_tree.cond1array[ThreadId]);
871: }
872: /* the while loop needs to have an exit
873: the 'main' thread can terminate all the threads by performing a broadcast
874: and calling FuncFinish */
875: while(PetscThreadGo) {
876: /*need to check the condition to ensure we don't have to wait
877: waiting when you don't have to causes problems
878: also need to check the condition to ensure proper handling of spurious wakeups */
879: while(*(job_tree.arrThreadReady[ThreadId])==PETSC_TRUE) {
880: /* upon entry, automically releases the lock and blocks
881: upon return, has the lock */
882: pthread_cond_wait(job_tree.cond2array[ThreadId],job_tree.mutexarray[ThreadId]);
883: *(job_tree.arrThreadStarted[ThreadId]) = PETSC_TRUE;
884: *(job_tree.arrThreadReady[ThreadId]) = PETSC_FALSE;
885: }
886: if(ThreadId==0) {
887: job_tree.startJob = PETSC_FALSE;
888: job_tree.eJobStat = ThreadsWorking;
889: }
890: pthread_mutex_unlock(job_tree.mutexarray[ThreadId]);
891: if(PeeOn==PETSC_FALSE) {
892: /* tell your subordinates it's time to get to work */
893: for(i=1; i<=Mary; i++) {
894: SubWorker = Mary*ThreadId+i;
895: if(SubWorker<PetscMaxThreads) {
896: pthread_cond_signal(job_tree.cond2array[SubWorker]);
897: }
898: }
899: }
900: /* do your job */
901: if(job_tree.pdata==NULL) {
902: iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata);
903: }
904: else {
905: iterr = (PetscErrorCode)(long int)job_tree.pfunc(job_tree.pdata[ThreadId]);
906: }
907: if(iterr!=0) {
908: ithreaderr = 1;
909: }
910: if(PetscThreadGo) {
911: /* reset job, get ready for more */
912: if(PeeOn==PETSC_FALSE) {
913: /* check your subordinates, waiting for them to be ready
914: how do you know for a fact that a given subordinate has actually started? */
915: for(i=1;i<=Mary;i++) {
916: SubWorker = Mary*ThreadId+i;
917: if(SubWorker<PetscMaxThreads) {
918: pthread_mutex_lock(job_tree.mutexarray[SubWorker]);
919: while(*(job_tree.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_tree.arrThreadStarted[SubWorker])==PETSC_FALSE) {
920: /* upon entry, automically releases the lock and blocks
921: upon return, has the lock */
922: pthread_cond_wait(job_tree.cond1array[SubWorker],job_tree.mutexarray[SubWorker]);
923: }
924: pthread_mutex_unlock(job_tree.mutexarray[SubWorker]);
925: }
926: }
927: /* your subordinates are now ready */
928: }
929: pthread_mutex_lock(job_tree.mutexarray[ThreadId]);
930: *(job_tree.arrThreadReady[ThreadId]) = PETSC_TRUE;
931: if(ThreadId==0) {
932: job_tree.eJobStat = JobCompleted; /* oot thread: last thread to complete, guaranteed! */
933: /* root thread signals 'main' */
934: pthread_cond_signal(&main_cond);
935: }
936: else {
937: /* signal your boss before you go to sleep */
938: pthread_cond_signal(job_tree.cond1array[ThreadId]);
939: }
940: }
941: }
942: return NULL;
943: }
947: void* PetscThreadInitialize_Tree(PetscInt N) {
948: PetscInt i,ierr;
949: int status;
951: if(PetscUseThreadPool) {
952: size_t Val1 = (size_t)CACHE_LINE_SIZE;
953: size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
954: arrmutex = (char*)memalign(Val1,Val2);
955: arrcond1 = (char*)memalign(Val1,Val2);
956: arrcond2 = (char*)memalign(Val1,Val2);
957: arrstart = (char*)memalign(Val1,Val2);
958: arrready = (char*)memalign(Val1,Val2);
959: job_tree.mutexarray = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
960: job_tree.cond1array = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
961: job_tree.cond2array = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
962: job_tree.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
963: job_tree.arrThreadReady = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
964: /* initialize job structure */
965: for(i=0; i<PetscMaxThreads; i++) {
966: job_tree.mutexarray[i] = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
967: job_tree.cond1array[i] = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
968: job_tree.cond2array[i] = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
969: job_tree.arrThreadStarted[i] = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
970: job_tree.arrThreadReady[i] = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
971: }
972: for(i=0; i<PetscMaxThreads; i++) {
973: pthread_mutex_init(job_tree.mutexarray[i],NULL);
974: pthread_cond_init(job_tree.cond1array[i],NULL);
975: pthread_cond_init(job_tree.cond2array[i],NULL);
976: *(job_tree.arrThreadStarted[i]) = PETSC_FALSE;
977: *(job_tree.arrThreadReady[i]) = PETSC_FALSE;
978: }
979: job_tree.pfunc = NULL;
980: job_tree.pdata = (void**)malloc(N*sizeof(void*));
981: job_tree.startJob = PETSC_FALSE;
982: job_tree.eJobStat = JobInitiated;
983: pVal = (int*)malloc(N*sizeof(int));
984: /* allocate memory in the heap for the thread structure */
985: PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
986: /* create threads */
987: for(i=0; i<N; i++) {
988: pVal[i] = i;
989: status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
990: /* should check status */
991: }
992: }
993: return NULL;
994: }
998: PetscErrorCode PetscThreadFinalize_Tree() {
999: int i,ierr;
1000: void* jstatus;
1004: MainJob(FuncFinish,NULL,PetscMaxThreads); /* set up job and broadcast work */
1005: /* join the threads */
1006: for(i=0; i<PetscMaxThreads; i++) {
1007: pthread_join(PetscThreadPoint[i],&jstatus);
1008: /* do error checking*/
1009: }
1010: free(PetscThreadPoint);
1011: free(arrmutex);
1012: free(arrcond1);
1013: free(arrcond2);
1014: free(arrstart);
1015: free(arrready);
1016: free(job_tree.pdata);
1017: free(pVal);
1019: return(0);
1020: }
1024: void MainWait_Tree() {
1025: int ierr;
1026: pthread_mutex_lock(job_tree.mutexarray[0]);
1027: while(job_tree.eJobStat<JobCompleted||job_tree.startJob==PETSC_TRUE) {
1028: pthread_cond_wait(&main_cond,job_tree.mutexarray[0]);
1029: }
1030: pthread_mutex_unlock(job_tree.mutexarray[0]);
1031: }
1035: PetscErrorCode MainJob_Tree(void* (*pFunc)(void*),void** data,PetscInt n) {
1036: int i,ierr;
1037: PetscErrorCode ijoberr = 0;
1039: MainWait();
1040: job_tree.pfunc = pFunc;
1041: job_tree.pdata = data;
1042: job_tree.startJob = PETSC_TRUE;
1043: for(i=0; i<PetscMaxThreads; i++) {
1044: *(job_tree.arrThreadStarted[i]) = PETSC_FALSE;
1045: }
1046: job_tree.eJobStat = JobInitiated;
1047: pthread_cond_signal(job_tree.cond2array[0]);
1048: if(pFunc!=FuncFinish) {
1049: MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1050: }
1052: if(ithreaderr) {
1053: ijoberr = ithreaderr;
1054: }
1055: return ijoberr;
1056: }
1057: /**** ****/
1059: /**** 'Main' Thread Pool Functions ****/
1060: void* PetscThreadFunc_Main(void* arg) {
1061: PetscErrorCode iterr;
1062: int icorr,ierr;
1063: int* pId = (int*)arg;
1064: int ThreadId = *pId;
1065: cpu_set_t mset;
1066: //printf("Thread %d In Main Thread Function\n",ThreadId);
1067: icorr = ThreadCoreAffinity[ThreadId];
1068: CPU_ZERO(&mset);
1069: CPU_SET(icorr,&mset);
1070: sched_setaffinity(0,sizeof(cpu_set_t),&mset);
1072: pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1073: /* update your ready status */
1074: *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1075: /* tell the BOSS that you're ready to work before you go to sleep */
1076: pthread_cond_signal(job_main.cond1array[ThreadId]);
1078: /* the while loop needs to have an exit
1079: the 'main' thread can terminate all the threads by performing a broadcast
1080: and calling FuncFinish */
1081: while(PetscThreadGo) {
1082: /* need to check the condition to ensure we don't have to wait
1083: waiting when you don't have to causes problems
1084: also need to check the condition to ensure proper handling of spurious wakeups */
1085: while(*(job_main.arrThreadReady[ThreadId])==PETSC_TRUE) {
1086: /* upon entry, atomically releases the lock and blocks
1087: upon return, has the lock */
1088: pthread_cond_wait(job_main.cond2array[ThreadId],job_main.mutexarray[ThreadId]);
1089: /* (job_main.arrThreadReady[ThreadId]) = PETSC_FALSE; */
1090: }
1091: pthread_mutex_unlock(job_main.mutexarray[ThreadId]);
1092: if(job_main.pdata==NULL) {
1093: iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata);
1094: }
1095: else {
1096: iterr = (PetscErrorCode)(long int)job_main.pfunc(job_main.pdata[ThreadId]);
1097: }
1098: if(iterr!=0) {
1099: ithreaderr = 1;
1100: }
1101: if(PetscThreadGo) {
1102: /* reset job, get ready for more */
1103: pthread_mutex_lock(job_main.mutexarray[ThreadId]);
1104: *(job_main.arrThreadReady[ThreadId]) = PETSC_TRUE;
1105: /* tell the BOSS that you're ready to work before you go to sleep */
1106: pthread_cond_signal(job_main.cond1array[ThreadId]);
1107: }
1108: }
1109: return NULL;
1110: }
1114: void* PetscThreadInitialize_Main(PetscInt N) {
1115: PetscInt i,ierr;
1116: int status;
1118: if(PetscUseThreadPool) {
1119: size_t Val1 = (size_t)CACHE_LINE_SIZE;
1120: size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1121: arrmutex = (char*)memalign(Val1,Val2);
1122: arrcond1 = (char*)memalign(Val1,Val2);
1123: arrcond2 = (char*)memalign(Val1,Val2);
1124: arrstart = (char*)memalign(Val1,Val2);
1125: arrready = (char*)memalign(Val1,Val2);
1126: job_main.mutexarray = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1127: job_main.cond1array = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1128: job_main.cond2array = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1129: job_main.arrThreadReady = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1130: /* initialize job structure */
1131: for(i=0; i<PetscMaxThreads; i++) {
1132: job_main.mutexarray[i] = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1133: job_main.cond1array[i] = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1134: job_main.cond2array[i] = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1135: job_main.arrThreadReady[i] = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1136: }
1137: for(i=0; i<PetscMaxThreads; i++) {
1138: pthread_mutex_init(job_main.mutexarray[i],NULL);
1139: pthread_cond_init(job_main.cond1array[i],NULL);
1140: pthread_cond_init(job_main.cond2array[i],NULL);
1141: *(job_main.arrThreadReady[i]) = PETSC_FALSE;
1142: }
1143: job_main.pfunc = NULL;
1144: job_main.pdata = (void**)malloc(N*sizeof(void*));
1145: pVal = (int*)malloc(N*sizeof(int));
1146: /* allocate memory in the heap for the thread structure */
1147: PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1148: /* create threads */
1149: for(i=0; i<N; i++) {
1150: pVal[i] = i;
1151: status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1152: /* error check */
1153: }
1154: }
1155: else {
1156: }
1157: return NULL;
1158: }
1162: PetscErrorCode PetscThreadFinalize_Main() {
1163: int i,ierr;
1164: void* jstatus;
1168: MainJob(FuncFinish,NULL,PetscMaxThreads); /* set up job and broadcast work */
1169: /* join the threads */
1170: for(i=0; i<PetscMaxThreads; i++) {
1171: pthread_join(PetscThreadPoint[i],&jstatus);
1172: }
1173: free(PetscThreadPoint);
1174: free(arrmutex);
1175: free(arrcond1);
1176: free(arrcond2);
1177: free(arrstart);
1178: free(arrready);
1179: free(job_main.pdata);
1180: free(pVal);
1182: return(0);
1183: }
1187: void MainWait_Main() {
1188: int i,ierr;
1189: for(i=0; i<PetscMaxThreads; i++) {
1190: pthread_mutex_lock(job_main.mutexarray[i]);
1191: while(*(job_main.arrThreadReady[i])==PETSC_FALSE) {
1192: pthread_cond_wait(job_main.cond1array[i],job_main.mutexarray[i]);
1193: }
1194: pthread_mutex_unlock(job_main.mutexarray[i]);
1195: }
1196: }
1200: PetscErrorCode MainJob_Main(void* (*pFunc)(void*),void** data,PetscInt n) {
1201: int i,ierr;
1202: PetscErrorCode ijoberr = 0;
1204: MainWait(); /* you know everyone is waiting to be signalled! */
1205: job_main.pfunc = pFunc;
1206: job_main.pdata = data;
1207: for(i=0; i<PetscMaxThreads; i++) {
1208: *(job_main.arrThreadReady[i]) = PETSC_FALSE; /* why do this? suppose you get into MainWait first */
1209: }
1210: /* tell the threads to go to work */
1211: for(i=0; i<PetscMaxThreads; i++) {
1212: pthread_cond_signal(job_main.cond2array[i]);
1213: }
1214: if(pFunc!=FuncFinish) {
1215: MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1216: }
1218: if(ithreaderr) {
1219: ijoberr = ithreaderr;
1220: }
1221: return ijoberr;
1222: }
1223: /**** ****/
1225: /**** Chain Thread Functions ****/
1226: void* PetscThreadFunc_Chain(void* arg) {
1227: PetscErrorCode iterr;
1228: int icorr,ierr;
1229: int* pId = (int*)arg;
1230: int ThreadId = *pId;
1231: int SubWorker = ThreadId + 1;
1232: PetscBool PeeOn;
1233: cpu_set_t mset;
1234: //printf("Thread %d In Chain Thread Function\n",ThreadId);
1235: icorr = ThreadCoreAffinity[ThreadId];
1236: CPU_ZERO(&mset);
1237: CPU_SET(icorr,&mset);
1238: sched_setaffinity(0,sizeof(cpu_set_t),&mset);
1240: if(ThreadId==(PetscMaxThreads-1)) {
1241: PeeOn = PETSC_TRUE;
1242: }
1243: else {
1244: PeeOn = PETSC_FALSE;
1245: }
1246: if(PeeOn==PETSC_FALSE) {
1247: /* check your subordinate, wait for him to be ready */
1248: pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1249: while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE) {
1250: /* upon entry, automically releases the lock and blocks
1251: upon return, has the lock */
1252: pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1253: }
1254: pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1255: /* your subordinate is now ready*/
1256: }
1257: pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1258: /* update your ready status */
1259: *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1260: if(ThreadId==0) {
1261: job_chain.eJobStat = JobCompleted;
1262: /* signal main */
1263: pthread_cond_signal(&main_cond);
1264: }
1265: else {
1266: /* tell your boss that you're ready to work */
1267: pthread_cond_signal(job_chain.cond1array[ThreadId]);
1268: }
1269: /* the while loop needs to have an exit
1270: the 'main' thread can terminate all the threads by performing a broadcast
1271: and calling FuncFinish */
1272: while(PetscThreadGo) {
1273: /* need to check the condition to ensure we don't have to wait
1274: waiting when you don't have to causes problems
1275: also need to check the condition to ensure proper handling of spurious wakeups */
1276: while(*(job_chain.arrThreadReady[ThreadId])==PETSC_TRUE) {
1277: /*upon entry, automically releases the lock and blocks
1278: upon return, has the lock */
1279: pthread_cond_wait(job_chain.cond2array[ThreadId],job_chain.mutexarray[ThreadId]);
1280: *(job_chain.arrThreadStarted[ThreadId]) = PETSC_TRUE;
1281: *(job_chain.arrThreadReady[ThreadId]) = PETSC_FALSE;
1282: }
1283: if(ThreadId==0) {
1284: job_chain.startJob = PETSC_FALSE;
1285: job_chain.eJobStat = ThreadsWorking;
1286: }
1287: pthread_mutex_unlock(job_chain.mutexarray[ThreadId]);
1288: if(PeeOn==PETSC_FALSE) {
1289: /* tell your subworker it's time to get to work */
1290: pthread_cond_signal(job_chain.cond2array[SubWorker]);
1291: }
1292: /* do your job */
1293: if(job_chain.pdata==NULL) {
1294: iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata);
1295: }
1296: else {
1297: iterr = (PetscErrorCode)(long int)job_chain.pfunc(job_chain.pdata[ThreadId]);
1298: }
1299: if(iterr!=0) {
1300: ithreaderr = 1;
1301: }
1302: if(PetscThreadGo) {
1303: /* reset job, get ready for more */
1304: if(PeeOn==PETSC_FALSE) {
1305: /* check your subordinate, wait for him to be ready
1306: how do you know for a fact that your subordinate has actually started? */
1307: pthread_mutex_lock(job_chain.mutexarray[SubWorker]);
1308: while(*(job_chain.arrThreadReady[SubWorker])==PETSC_FALSE||*(job_chain.arrThreadStarted[SubWorker])==PETSC_FALSE) {
1309: /* upon entry, automically releases the lock and blocks
1310: upon return, has the lock */
1311: pthread_cond_wait(job_chain.cond1array[SubWorker],job_chain.mutexarray[SubWorker]);
1312: }
1313: pthread_mutex_unlock(job_chain.mutexarray[SubWorker]);
1314: /* your subordinate is now ready */
1315: }
1316: pthread_mutex_lock(job_chain.mutexarray[ThreadId]);
1317: *(job_chain.arrThreadReady[ThreadId]) = PETSC_TRUE;
1318: if(ThreadId==0) {
1319: job_chain.eJobStat = JobCompleted; /* foreman: last thread to complete, guaranteed! */
1320: /* root thread (foreman) signals 'main' */
1321: pthread_cond_signal(&main_cond);
1322: }
1323: else {
1324: /* signal your boss before you go to sleep */
1325: pthread_cond_signal(job_chain.cond1array[ThreadId]);
1326: }
1327: }
1328: }
1329: return NULL;
1330: }
1334: void* PetscThreadInitialize_Chain(PetscInt N) {
1335: PetscInt i,ierr;
1336: int status;
1338: if(PetscUseThreadPool) {
1339: size_t Val1 = (size_t)CACHE_LINE_SIZE;
1340: size_t Val2 = (size_t)PetscMaxThreads*CACHE_LINE_SIZE;
1341: arrmutex = (char*)memalign(Val1,Val2);
1342: arrcond1 = (char*)memalign(Val1,Val2);
1343: arrcond2 = (char*)memalign(Val1,Val2);
1344: arrstart = (char*)memalign(Val1,Val2);
1345: arrready = (char*)memalign(Val1,Val2);
1346: job_chain.mutexarray = (pthread_mutex_t**)malloc(PetscMaxThreads*sizeof(pthread_mutex_t*));
1347: job_chain.cond1array = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1348: job_chain.cond2array = (pthread_cond_t**)malloc(PetscMaxThreads*sizeof(pthread_cond_t*));
1349: job_chain.arrThreadStarted = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1350: job_chain.arrThreadReady = (PetscBool**)malloc(PetscMaxThreads*sizeof(PetscBool*));
1351: /* initialize job structure */
1352: for(i=0; i<PetscMaxThreads; i++) {
1353: job_chain.mutexarray[i] = (pthread_mutex_t*)(arrmutex+CACHE_LINE_SIZE*i);
1354: job_chain.cond1array[i] = (pthread_cond_t*)(arrcond1+CACHE_LINE_SIZE*i);
1355: job_chain.cond2array[i] = (pthread_cond_t*)(arrcond2+CACHE_LINE_SIZE*i);
1356: job_chain.arrThreadStarted[i] = (PetscBool*)(arrstart+CACHE_LINE_SIZE*i);
1357: job_chain.arrThreadReady[i] = (PetscBool*)(arrready+CACHE_LINE_SIZE*i);
1358: }
1359: for(i=0; i<PetscMaxThreads; i++) {
1360: pthread_mutex_init(job_chain.mutexarray[i],NULL);
1361: pthread_cond_init(job_chain.cond1array[i],NULL);
1362: pthread_cond_init(job_chain.cond2array[i],NULL);
1363: *(job_chain.arrThreadStarted[i]) = PETSC_FALSE;
1364: *(job_chain.arrThreadReady[i]) = PETSC_FALSE;
1365: }
1366: job_chain.pfunc = NULL;
1367: job_chain.pdata = (void**)malloc(N*sizeof(void*));
1368: job_chain.startJob = PETSC_FALSE;
1369: job_chain.eJobStat = JobInitiated;
1370: pVal = (int*)malloc(N*sizeof(int));
1371: /* allocate memory in the heap for the thread structure */
1372: PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1373: /* create threads */
1374: for(i=0; i<N; i++) {
1375: pVal[i] = i;
1376: status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1377: /* should check error */
1378: }
1379: }
1380: else {
1381: }
1382: return NULL;
1383: }
1388: PetscErrorCode PetscThreadFinalize_Chain() {
1389: int i,ierr;
1390: void* jstatus;
1394: MainJob(FuncFinish,NULL,PetscMaxThreads); /* set up job and broadcast work */
1395: /* join the threads */
1396: for(i=0; i<PetscMaxThreads; i++) {
1397: pthread_join(PetscThreadPoint[i],&jstatus);
1398: /* should check error */
1399: }
1400: free(PetscThreadPoint);
1401: free(arrmutex);
1402: free(arrcond1);
1403: free(arrcond2);
1404: free(arrstart);
1405: free(arrready);
1406: free(job_chain.pdata);
1407: free(pVal);
1409: return(0);
1410: }
1414: void MainWait_Chain() {
1415: int ierr;
1416: pthread_mutex_lock(job_chain.mutexarray[0]);
1417: while(job_chain.eJobStat<JobCompleted||job_chain.startJob==PETSC_TRUE) {
1418: pthread_cond_wait(&main_cond,job_chain.mutexarray[0]);
1419: }
1420: pthread_mutex_unlock(job_chain.mutexarray[0]);
1421: }
1425: PetscErrorCode MainJob_Chain(void* (*pFunc)(void*),void** data,PetscInt n) {
1426: int i,ierr;
1427: PetscErrorCode ijoberr = 0;
1429: MainWait();
1430: job_chain.pfunc = pFunc;
1431: job_chain.pdata = data;
1432: job_chain.startJob = PETSC_TRUE;
1433: for(i=0; i<PetscMaxThreads; i++) {
1434: *(job_chain.arrThreadStarted[i]) = PETSC_FALSE;
1435: }
1436: job_chain.eJobStat = JobInitiated;
1437: pthread_cond_signal(job_chain.cond2array[0]);
1438: if(pFunc!=FuncFinish) {
1439: MainWait(); /* why wait after? guarantees that job gets done before proceeding with result collection (if any) */
1440: }
1442: if(ithreaderr) {
1443: ijoberr = ithreaderr;
1444: }
1445: return ijoberr;
1446: }
1447: /**** ****/
1449: #if defined(PETSC_HAVE_PTHREAD_BARRIER)
1450: /**** True Thread Functions ****/
1451: void* PetscThreadFunc_True(void* arg) {
1452: int icorr,ierr,iVal;
1453: int* pId = (int*)arg;
1454: int ThreadId = *pId;
1455: PetscErrorCode iterr;
1456: cpu_set_t mset;
1457: //printf("Thread %d In True Pool Thread Function\n",ThreadId);
1458: icorr = ThreadCoreAffinity[ThreadId];
1459: CPU_ZERO(&mset);
1460: CPU_SET(icorr,&mset);
1461: sched_setaffinity(0,sizeof(cpu_set_t),&mset);
1463: pthread_mutex_lock(&job_true.mutex);
1464: job_true.iNumReadyThreads++;
1465: if(job_true.iNumReadyThreads==PetscMaxThreads) {
1466: pthread_cond_signal(&main_cond);
1467: }
1468: /*the while loop needs to have an exit
1469: the 'main' thread can terminate all the threads by performing a broadcast
1470: and calling FuncFinish */
1471: while(PetscThreadGo) {
1472: /*need to check the condition to ensure we don't have to wait
1473: waiting when you don't have to causes problems
1474: also need to wait if another thread sneaks in and messes with the predicate */
1475: while(job_true.startJob==PETSC_FALSE&&job_true.iNumJobThreads==0) {
1476: /* upon entry, automically releases the lock and blocks
1477: upon return, has the lock */
1478: //printf("Thread %d Going to Sleep!\n",ThreadId);
1479: pthread_cond_wait(&job_true.cond,&job_true.mutex);
1480: }
1481: job_true.startJob = PETSC_FALSE;
1482: job_true.iNumJobThreads--;
1483: job_true.iNumReadyThreads--;
1484: iVal = PetscMaxThreads-job_true.iNumReadyThreads-1;
1485: pthread_mutex_unlock(&job_true.mutex);
1486: if(job_true.pdata==NULL) {
1487: iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata);
1488: }
1489: else {
1490: iterr = (PetscErrorCode)(long int)job_true.pfunc(job_true.pdata[iVal]);
1491: }
1492: if(iterr!=0) {
1493: ithreaderr = 1;
1494: }
1495: //printf("Thread %d Finished Job\n",ThreadId);
1496: /* the barrier is necessary BECAUSE: look at job_true.iNumReadyThreads
1497: what happens if a thread finishes before they all start? BAD!
1498: what happens if a thread finishes before any else start? BAD! */
1499: pthread_barrier_wait(job_true.pbarr); /* ensures all threads are finished */
1500: /* reset job */
1501: if(PetscThreadGo) {
1502: pthread_mutex_lock(&job_true.mutex);
1503: job_true.iNumReadyThreads++;
1504: if(job_true.iNumReadyThreads==PetscMaxThreads) {
1505: /* signal the 'main' thread that the job is done! (only done once) */
1506: pthread_cond_signal(&main_cond);
1507: }
1508: }
1509: }
1510: return NULL;
1511: }
1515: void* PetscThreadInitialize_True(PetscInt N) {
1516: PetscInt i;
1517: int status;
1519: pVal = (int*)malloc(N*sizeof(int));
1520: /* allocate memory in the heap for the thread structure */
1521: PetscThreadPoint = (pthread_t*)malloc(N*sizeof(pthread_t));
1522: BarrPoint = (pthread_barrier_t*)malloc((N+1)*sizeof(pthread_barrier_t)); /* BarrPoint[0] makes no sense, don't use it! */
1523: job_true.pdata = (void**)malloc(N*sizeof(void*));
1524: for(i=0; i<N; i++) {
1525: pVal[i] = i;
1526: status = pthread_create(&PetscThreadPoint[i],NULL,PetscThreadFunc,&pVal[i]);
1527: /* error check to ensure proper thread creation */
1528: status = pthread_barrier_init(&BarrPoint[i+1],NULL,i+1);
1529: /* should check error */
1530: }
1531: //printf("Finished True Thread Pool Initialization\n");
1532: return NULL;
1533: }
1538: PetscErrorCode PetscThreadFinalize_True() {
1539: int i,ierr;
1540: void* jstatus;
1544: MainJob(FuncFinish,NULL,PetscMaxThreads); /* set up job and broadcast work */
1545: /* join the threads */
1546: for(i=0; i<PetscMaxThreads; i++) {
1547: pthread_join(PetscThreadPoint[i],&jstatus);
1548: }
1549: free(BarrPoint);
1550: free(PetscThreadPoint);
1552: return(0);
1553: }
1557: void MainWait_True() {
1558: int ierr;
1559: pthread_mutex_lock(&job_true.mutex);
1560: while(job_true.iNumReadyThreads<PetscMaxThreads||job_true.startJob==PETSC_TRUE) {
1561: pthread_cond_wait(&main_cond,&job_true.mutex);
1562: }
1563: pthread_mutex_unlock(&job_true.mutex);
1564: }
1568: PetscErrorCode MainJob_True(void* (*pFunc)(void*),void** data,PetscInt n) {
1569: int ierr;
1570: PetscErrorCode ijoberr = 0;
1572: MainWait();
1573: job_true.pfunc = pFunc;
1574: job_true.pdata = data;
1575: job_true.pbarr = &BarrPoint[n];
1576: job_true.iNumJobThreads = n;
1577: job_true.startJob = PETSC_TRUE;
1578: pthread_cond_broadcast(&job_true.cond);
1579: if(pFunc!=FuncFinish) {
1580: MainWait(); /* why wait after? guarantees that job gets done */
1581: }
1583: if(ithreaderr) {
1584: ijoberr = ithreaderr;
1585: }
1586: return ijoberr;
1587: }
1588: /**** NO THREAD POOL FUNCTION ****/
1591: PetscErrorCode MainJob_Spawn(void* (*pFunc)(void*),void** data,PetscInt n) {
1592: PetscErrorCode ijoberr = 0;
1594: pthread_t* apThread = (pthread_t*)malloc(n*sizeof(pthread_t));
1595: PetscThreadPoint = apThread; /* point to same place */
1596: PetscThreadRun(MPI_COMM_WORLD,pFunc,n,apThread,data);
1597: PetscThreadStop(MPI_COMM_WORLD,n,apThread); /* ensures that all threads are finished with the job */
1598: free(apThread);
1600: return ijoberr;
1601: }
1602: /**** ****/
1603: #endif
1605: void* FuncFinish(void* arg) {
1606: PetscThreadGo = PETSC_FALSE;
1607: return(0);
1608: }
1610: #endif