moab
|
00001 #include "IODebugTrack.hpp" 00002 #include "moab/Range.hpp" 00003 #include <iostream> 00004 #include <vector> 00005 #include <assert.h> 00006 00007 #ifdef USE_MPI 00008 # include "moab_mpi.h" 00009 #endif 00010 00011 const char PFX[] = ">>> "; 00012 00013 namespace moab { 00014 00015 IODebugTrack::IODebugTrack( bool enabled, 00016 const std::string name, 00017 std::ostream output_stream, 00018 unsigned long table_size ) 00019 : enableOutput(enabled), 00020 tableName(name), 00021 ostr(output_stream), 00022 maxSize(table_size) 00023 { 00024 #ifdef USE_MPI 00025 MPI_Comm_rank( MPI_COMM_WORLD, &mpiRank ); 00026 #else 00027 mpiRank = 0; 00028 #endif 00029 } 00030 00031 00032 IODebugTrack::IODebugTrack( bool enabled, 00033 const std::string name, 00034 unsigned long table_size ) 00035 : enableOutput(enabled), 00036 tableName(name), 00037 ostr(std::cerr), 00038 maxSize(table_size) 00039 { 00040 mpiRank = 0; 00041 haveMPI = false; 00042 #ifdef USE_MPI 00043 int have_init = 0; 00044 MPI_Initialized(&have_init); 00045 if (have_init) { 00046 haveMPI = true; 00047 MPI_Comm_rank( MPI_COMM_WORLD, &mpiRank ); 00048 } 00049 #endif 00050 } 00051 00052 IODebugTrack::~IODebugTrack() 00053 { 00054 if (!enableOutput || mpiRank) // only root prints gap summary 00055 return; 00056 00057 if (dataSet.empty()) { 00058 ostr << PFX << tableName << " : No Data Written!!!!" << std::endl; 00059 return; 00060 } 00061 00062 std::list<DRange>::const_iterator i; 00063 if (!maxSize) { 00064 for (i = dataSet.begin(); i != dataSet.end(); ++i) 00065 if (i->end >= maxSize) 00066 maxSize = i->end + 1; 00067 } 00068 Range processed; 00069 Range::iterator h = processed.begin(); 00070 bool wrote_zero = false; 00071 for (i = dataSet.begin(); i != dataSet.end(); ++i) { 00072 // ranges cannot contain zero 00073 assert(i->begin <= i->end); 00074 if (i->begin) 00075 h = processed.insert( h, i->begin, i->end ); 00076 else { 00077 wrote_zero = true; 00078 if (i->end) 00079 h = processed.insert( h, i->begin+1, i->end ); 00080 } 00081 } 00082 00083 // ranges cannot contain zero 00084 Range unprocessed; 00085 if (maxSize > 1) 00086 unprocessed.insert( 1, maxSize - 1 ); 00087 unprocessed = subtract( unprocessed, processed ); 00088 if (unprocessed.empty()) 00089 return; 00090 00091 Range::const_pair_iterator j; 00092 for (j = unprocessed.const_pair_begin(); j != unprocessed.const_pair_end(); ++j) { 00093 unsigned long b = j->first; 00094 unsigned long e = j->second; 00095 if (b == 1 && !wrote_zero) 00096 b = 0; 00097 00098 ostr << PFX << tableName << " : range not read/written: [" 00099 << b << "," << e << "]" << std::endl; 00100 ostr.flush(); 00101 } 00102 } 00103 00104 void IODebugTrack::record_io( unsigned long begin, unsigned long count ) 00105 { 00106 if (enableOutput && count) { 00107 DRange ins = { begin, begin+count-1, static_cast<long unsigned>(mpiRank) }; 00108 record_io( ins ); 00109 } 00110 } 00111 00112 void IODebugTrack::record_io( DRange ins ) 00113 { 00114 if (!enableOutput) 00115 return; 00116 00117 // only root should get non-local data 00118 assert(!mpiRank || ins.rank == (unsigned)mpiRank); 00119 assert( ins.begin <= ins.end ); 00120 00121 // test for out-of-bounds write 00122 if (maxSize && ins.end >= maxSize) 00123 ostr << ": Out of bounds write on rank " << mpiRank 00124 << ": [" << ins.begin << "," << ins.end << "] >= " << maxSize 00125 << std::endl; 00126 00127 // test for overlap with all existing ranges 00128 std::list<DRange>::iterator i; 00129 for (i = dataSet.begin(); i != dataSet.end(); ++i) { 00130 if (i->end >= ins.begin && i->begin <= ins.end) { // if overlap 00131 ostr << PFX << tableName; 00132 if (i->rank == ins.rank) { 00133 if (mpiRank == (int)ins.rank) 00134 ostr << ": Local overwrite on rank " << mpiRank; 00135 00136 // otherwise should have been logged on remote proc, do nothing here 00137 } 00138 else 00139 ostr << ": Conflicting write for ranks " << i->rank << " and " << ins.rank; 00140 00141 ostr << ": [" << i->begin << "," << i->end << "] and [" << ins.begin 00142 << "," << ins.end << "]" << std::endl; 00143 ostr.flush(); 00144 } 00145 } 00146 00147 dataSet.push_back( ins ); 00148 } 00149 00150 void IODebugTrack::all_reduce() 00151 { 00152 #ifdef USE_MPI 00153 if (!enableOutput || !haveMPI) 00154 return; 00155 00156 int commsize; 00157 MPI_Comm_size( MPI_COMM_WORLD, &commsize); 00158 int count = 3*dataSet.size(); 00159 std::vector<int> displs(commsize), counts(commsize); 00160 MPI_Gather( &count, 1, MPI_INT, 00161 &counts[0], 1, MPI_INT, 00162 0, MPI_COMM_WORLD ); 00163 displs[0] = 0; 00164 for (int i = 1; i < commsize; ++i) 00165 displs[i] = displs[i-1] + counts[i-1]; 00166 int total = (displs.back() + counts.back()) / 3; 00167 count /= 3; 00168 00169 std::vector<DRange> send(dataSet.size()), recv(total); 00170 std::copy( dataSet.begin(), dataSet.end(), send.begin() ); 00171 MPI_Gatherv( &send[0], 3*send.size(), MPI_UNSIGNED_LONG, 00172 &recv[0], &counts[0], &displs[0], MPI_UNSIGNED_LONG, 00173 0, MPI_COMM_WORLD ); 00174 00175 if (0 == mpiRank) { 00176 for (int i = count; i < total; ++i) 00177 record_io( recv[i] ); 00178 } 00179 else { 00180 dataSet.clear(); 00181 } 00182 #endif 00183 } 00184 00185 00186 00187 } // namespace moab