|
moab
|
00001 00017 #ifndef MOAB_PARALLEL_COMM_HPP 00018 #define MOAB_PARALLEL_COMM_HPP 00019 00020 #include "moab/Forward.hpp" 00021 #include "moab/Interface.hpp" 00022 #include "moab/Range.hpp" 00023 #include "moab/ProcConfig.hpp" 00024 #include <map> 00025 #include <set> 00026 #include <vector> 00027 #include <iostream> 00028 #include <fstream> 00029 #include <assert.h> 00030 #include <stdlib.h> 00031 #include "math.h" 00032 #include "moab/TupleList.hpp" 00033 00034 namespace moab { 00035 00036 class SequenceManager; 00037 class Error; 00038 template <typename KeyType, typename ValType, ValType NullVal> class RangeMap; 00039 typedef RangeMap<EntityHandle, EntityHandle, 0> HandleMap; 00040 class ParallelMergeMesh; 00041 class DebugOutput; 00042 class SharedSetData; 00043 00044 #define MAX_SHARING_PROCS 64 00045 00053 class ParallelComm 00054 { 00055 public: 00056 00057 friend class ParallelMergeMesh; 00058 00059 // ================================== 00060 // \section CONSTRUCTORS/DESTRUCTORS/PCOMM MANAGEMENT 00061 // ================================== 00062 00064 ParallelComm(Interface *impl, 00065 MPI_Comm comm, 00066 int* pcomm_id_out = 0); 00067 00069 ParallelComm(Interface *impl, 00070 std::vector<unsigned char> &tmp_buff, 00071 MPI_Comm comm, 00072 int* pcomm_id_out = 0); 00073 00075 int get_id() const { return pcommID; } 00076 00078 static ParallelComm *get_pcomm(Interface *impl, const int index); 00079 00083 static ParallelComm *get_pcomm( Interface* impl, 00084 EntityHandle partitioning, 00085 const MPI_Comm* comm = 0 ); 00086 00087 static ErrorCode get_all_pcomm( Interface* impl, 00088 std::vector<ParallelComm*>& list ); 00089 00091 ~ParallelComm(); 00092 00093 static unsigned char PROC_SHARED, PROC_OWNER; 00094 00095 // ================================== 00096 // \section GLOBAL IDS 00097 // ================================== 00098 00103 ErrorCode assign_global_ids(EntityHandle this_set, 00104 const int dimension, 00105 const int start_id = 1, 00106 const bool largest_dim_only = true, 00107 const bool parallel = true, 00108 const bool owned_only = false); 00109 00112 ErrorCode assign_global_ids( Range entities[], 00113 const int dimension, 00114 const int start_id, 00115 const bool parallel, 00116 const bool owned_only); 00117 00122 ErrorCode check_global_ids(EntityHandle this_set, 00123 const int dimension, 00124 const int start_id = 1, 00125 const bool largest_dim_only = true, 00126 const bool parallel = true, 00127 const bool owned_only = false); 00128 00129 // ================================== 00130 // \section HIGH-LEVEL COMMUNICATION (send/recv/bcast/scatter ents, exchange tags) 00131 // ================================== 00132 00148 ErrorCode send_entities(const int to_proc, 00149 Range &orig_ents, 00150 const bool adjs, 00151 const bool tags, 00152 const bool store_remote_handles, 00153 const bool is_iface, 00154 Range &final_ents, 00155 int &incoming1, 00156 int &incoming2, // newly added 00157 TupleList& entprocs, // newly added 00158 std::vector<MPI_Request> &recv_remoteh_reqs, // newly added 00159 bool wait_all = true); 00160 00161 ErrorCode send_entities(std::vector<unsigned int>& send_procs, 00162 std::vector<Range*>& send_ents, 00163 int& incoming1, int& incoming2, 00164 const bool store_remote_handles); 00165 00177 ErrorCode recv_entities(const int from_proc, 00178 const bool store_remote_handles, 00179 const bool is_iface, 00180 Range &final_ents, 00181 int& incomming1, 00182 int& incoming2, 00183 std::vector<std::vector<EntityHandle> > &L1hloc, 00184 std::vector<std::vector<EntityHandle> > &L1hrem, 00185 std::vector<std::vector<int> > &L1p, 00186 std::vector<EntityHandle> &L2hloc, 00187 std::vector<EntityHandle> &L2hrem, 00188 std::vector<unsigned int> &L2p, 00189 std::vector<MPI_Request> &recv_remoteh_reqs, 00190 bool wait_all = true); 00191 00192 ErrorCode recv_entities(std::set<unsigned int>& recv_procs, 00193 int incoming1, int incoming2, 00194 const bool store_remote_handles, 00195 const bool migrate = false); 00196 00205 ErrorCode recv_messages(const int from_proc, 00206 const bool store_remote_handles, 00207 const bool is_iface, 00208 Range &final_ents, 00209 int& incoming1, 00210 int& incoming2, 00211 std::vector<std::vector<EntityHandle> > &L1hloc, 00212 std::vector<std::vector<EntityHandle> > &L1hrem, 00213 std::vector<std::vector<int> > &L1p, 00214 std::vector<EntityHandle> &L2hloc, 00215 std::vector<EntityHandle> &L2hrem, 00216 std::vector<unsigned int> &L2p, 00217 std::vector<MPI_Request> &recv_remoteh_reqs); 00218 00219 ErrorCode recv_remote_handle_messages(const int from_proc, 00220 int& incoming2, 00221 std::vector<EntityHandle> &L2hloc, 00222 std::vector<EntityHandle> &L2hrem, 00223 std::vector<unsigned int> &L2p, 00224 std::vector<MPI_Request> &recv_remoteh_reqs); 00225 00245 ErrorCode exchange_ghost_cells(int ghost_dim, int bridge_dim, 00246 int num_layers, int addl_ents, 00247 bool store_remote_handles, 00248 bool wait_all = true, 00249 EntityHandle *file_set = NULL); 00250 00254 static ErrorCode exchange_ghost_cells(ParallelComm **pc, 00255 unsigned int num_procs, 00256 int ghost_dim, int bridge_dim, 00257 int num_layers, int addl_ents, 00258 bool store_remote_handles, 00259 EntityHandle *file_sets = NULL); 00260 00264 ErrorCode post_irecv(std::vector<unsigned int>& exchange_procs); 00265 00266 ErrorCode post_irecv(std::vector<unsigned int>& shared_procs, 00267 std::set<unsigned int>& recv_procs); 00268 00277 ErrorCode exchange_owned_meshs(std::vector<unsigned int>& exchange_procs, 00278 std::vector<Range*>& exchange_ents, 00279 std::vector<MPI_Request>& recv_ent_reqs, 00280 std::vector<MPI_Request>& recv_remoteh_reqs, 00281 bool store_remote_handles, 00282 bool wait_all = true, 00283 bool migrate = false, 00284 int dim = 0); 00285 00290 ErrorCode exchange_owned_mesh(std::vector<unsigned int>& exchange_procs, 00291 std::vector<Range*>& exchange_ents, 00292 std::vector<MPI_Request>& recv_ent_reqs, 00293 std::vector<MPI_Request>& recv_remoteh_reqs, 00294 const bool recv_posted, 00295 bool store_remote_handles, 00296 bool wait_all, 00297 bool migrate = false); 00298 00309 ErrorCode exchange_tags( const std::vector<Tag> &src_tags, 00310 const std::vector<Tag> &dst_tags, 00311 const Range &entities); 00312 00321 ErrorCode exchange_tags( const char *tag_name, 00322 const Range &entities); 00323 00332 ErrorCode exchange_tags( Tag tagh, 00333 const Range &entities); 00334 00346 ErrorCode reduce_tags( const std::vector<Tag> &src_tags, 00347 const std::vector<Tag> &dst_tags, 00348 const MPI_Op mpi_op, 00349 const Range &entities); 00350 00358 ErrorCode reduce_tags( const char *tag_name, 00359 const MPI_Op mpi_op, 00360 const Range &entities); 00361 00369 ErrorCode reduce_tags( Tag tag_handle, 00370 const MPI_Op mpi_op, 00371 const Range &entities); 00372 00381 ErrorCode broadcast_entities(const int from_proc, 00382 Range& entities, 00383 const bool adjacencies = false, 00384 const bool tags = true ); 00385 00394 ErrorCode scatter_entities(const int from_proc, 00395 std::vector<Range> &entities, 00396 const bool adjacencies = false, 00397 const bool tags = true); 00398 00399 // ================================== 00400 // \section INITIALIZATION OF PARALLEL DATA (resolve_shared_ents, etc.) 00401 // ================================== 00402 00419 ErrorCode resolve_shared_ents(EntityHandle this_set, 00420 Range &proc_ents, 00421 int resolve_dim = -1, 00422 int shared_dim = -1, 00423 Range *skin_ents = NULL, 00424 const Tag* id_tag = 0); 00425 00437 ErrorCode resolve_shared_ents(EntityHandle this_set, 00438 int resolve_dim = 3, 00439 int shared_dim = -1, 00440 const Tag* id_tag = 0); 00441 00442 static ErrorCode resolve_shared_ents(ParallelComm **pc, 00443 const unsigned int np, 00444 EntityHandle this_set, 00445 const int to_dim); 00446 00456 ErrorCode resolve_shared_sets( EntityHandle this_set, const Tag* id_tag = 0 ); 00457 00465 ErrorCode resolve_shared_sets( Range& candidate_sets, Tag id_tag ); 00466 00467 // ================================== 00468 // \section GET PARALLEL DATA (shared/owned/iface entities, etc.) 00469 // ================================== 00470 00477 ErrorCode get_pstatus(EntityHandle entity, 00478 unsigned char &pstatus_val); 00479 00487 ErrorCode get_pstatus_entities(int dim, 00488 unsigned char pstatus_val, 00489 Range &pstatus_ents); 00490 00493 ErrorCode get_owner(EntityHandle entity, int &owner); 00494 00497 ErrorCode get_owner_handle(EntityHandle entity, 00498 int &owner, 00499 EntityHandle &handle); 00500 00510 ErrorCode get_sharing_data(const EntityHandle entity, 00511 int *ps, 00512 EntityHandle *hs, 00513 unsigned char &pstat, 00514 unsigned int &num_ps); 00515 00523 ErrorCode get_sharing_data(const EntityHandle entity, 00524 int *ps, 00525 EntityHandle *hs, 00526 unsigned char &pstat, 00527 int &num_ps); 00528 00537 ErrorCode get_sharing_data(const EntityHandle *entities, 00538 int num_entities, 00539 std::set<int> &procs, 00540 int op = Interface::INTERSECT); 00541 00545 ErrorCode get_sharing_data(const Range &entities, 00546 std::set<int> &procs, 00547 int op = Interface::INTERSECT); 00548 00558 ErrorCode get_shared_entities(int other_proc, 00559 Range &shared_ents, 00560 int dim = -1, 00561 const bool iface = false, 00562 const bool owned_filter = false); 00563 /* 00566 ErrorCode get_partition_sets(EntityHandle this_set, 00567 Range &part_sets, 00568 const char *tag_name = NULL); 00569 */ 00571 ErrorCode get_interface_procs(std::set<unsigned int> &iface_procs, 00572 const bool get_buffs = false); 00573 00575 ErrorCode get_comm_procs(std::set<unsigned int> &procs); 00576 00577 // ================================== 00578 // \section SHARED SETS 00579 // ================================== 00580 00583 ErrorCode get_entityset_procs( EntityHandle entity_set, 00584 std::vector<unsigned>& ranks ) const; 00585 00589 ErrorCode get_entityset_owner( EntityHandle entity_set, 00590 unsigned& owner_rank, 00591 EntityHandle* remote_handle = 0 ) const; 00592 00594 ErrorCode get_entityset_local_handle( unsigned owning_rank, 00595 EntityHandle remote_handle, 00596 EntityHandle& local_handle ) const; 00597 00599 ErrorCode get_shared_sets( Range& result ) const; 00600 00604 ErrorCode get_entityset_owners( std::vector<unsigned>& ranks ) const; 00605 00607 ErrorCode get_owned_sets( unsigned owning_rank, Range& sets_out ) const; 00608 00609 // ================================== 00610 // \section LOW-LEVEL DATA (tags, sets on interface/partition, etc.) 00611 // ================================== 00612 00614 const ProcConfig &proc_config() const {return procConfig;} 00615 00617 ProcConfig &proc_config() {return procConfig;} 00618 00619 unsigned rank() const { return proc_config().proc_rank(); } 00620 unsigned size() const { return proc_config().proc_size(); } 00621 MPI_Comm comm() const { return proc_config().proc_comm(); } 00622 00624 ErrorCode get_shared_proc_tags(Tag &sharedp_tag, 00625 Tag &sharedps_tag, 00626 Tag &sharedh_tag, 00627 Tag &sharedhs_tag, 00628 Tag &pstatus_tag); 00629 00631 Range &partition_sets() {return partitionSets;} 00632 const Range &partition_sets() const {return partitionSets;} 00633 Range &interface_sets() {return interfaceSets;} 00634 const Range &interface_sets() const {return interfaceSets;} 00635 00637 Tag sharedp_tag(); 00638 00640 Tag sharedps_tag(); 00641 00643 Tag sharedh_tag(); 00644 00646 Tag sharedhs_tag(); 00647 00649 Tag pstatus_tag(); 00650 00653 static Tag pcomm_tag(Interface *impl, 00654 bool create_if_missing = true); 00655 00657 Tag partition_tag(); 00658 Tag part_tag() { return partition_tag(); } 00659 00660 // ================================== 00661 // \section DEBUGGING AIDS 00662 // ================================== 00663 00665 void print_pstatus(unsigned char pstat, std::string &ostr); 00666 00668 void print_pstatus(unsigned char pstat); 00669 00670 // ================================== 00671 // \section IMESHP-RELATED FUNCTIONS 00672 // ================================== 00673 00675 ErrorCode get_part_entities(Range &ents, int dim = -1); 00676 00677 EntityHandle get_partitioning() const { return partitioningSet; } 00678 ErrorCode set_partitioning( EntityHandle h ); 00679 ErrorCode get_global_part_count( int& count_out ) const; 00680 ErrorCode get_part_owner( int part_id, int& owner_out ) const; 00681 ErrorCode get_part_id( EntityHandle part, int& id_out ) const; 00682 ErrorCode get_part_handle( int id, EntityHandle& handle_out ) const; 00683 ErrorCode create_part( EntityHandle& part_out ); 00684 ErrorCode destroy_part( EntityHandle part ) ; 00685 ErrorCode collective_sync_partition(); 00686 ErrorCode get_part_neighbor_ids( EntityHandle part, 00687 int neighbors_out[MAX_SHARING_PROCS], 00688 int& num_neighbors_out ); 00689 ErrorCode get_interface_sets( EntityHandle part, 00690 Range& iface_sets_out, 00691 int* adj_part_id = 0 ); 00692 ErrorCode get_owning_part( EntityHandle entity, 00693 int& owning_part_id_out, 00694 EntityHandle* owning_handle = 0 ); 00695 ErrorCode get_sharing_parts( EntityHandle entity, 00696 int part_ids_out[MAX_SHARING_PROCS], 00697 int& num_part_ids_out, 00698 EntityHandle remote_handles[MAX_SHARING_PROCS] = 0); 00699 00720 ErrorCode filter_pstatus( Range &ents, 00721 const unsigned char pstatus_val, 00722 const unsigned char op, 00723 int to_proc = -1, 00724 Range *returned_ents = NULL); 00725 00732 ErrorCode get_iface_entities(int other_proc, 00733 int dim, 00734 Range &iface_ents); 00735 00736 Interface* get_moab() const { return mbImpl; } 00737 00738 00739 ErrorCode clean_shared_tags(std::vector<Range*>& exchange_ents); 00740 00741 class Buffer { 00742 public: 00743 unsigned char *mem_ptr; 00744 unsigned char *buff_ptr; 00745 unsigned int alloc_size; 00746 00747 Buffer(unsigned int sz = 0); 00748 Buffer(const Buffer &); 00749 ~Buffer(); 00750 void reset_buffer(size_t buff_pos = 0) {reset_ptr(buff_pos); reserve(INITIAL_BUFF_SIZE);} 00751 void reset_ptr(size_t buff_pos = 0) {assert((!mem_ptr && !buff_pos)|| (alloc_size >= buff_pos)); buff_ptr = mem_ptr + buff_pos;} 00752 inline void reserve(unsigned int new_size); 00753 void set_stored_size() {*((int*)mem_ptr) = (int)(buff_ptr - mem_ptr);} 00754 int get_stored_size() {return *((int*)mem_ptr);} 00755 00756 void check_space(unsigned int addl_space); 00757 }; 00758 00760 ErrorCode pack_buffer(Range &orig_ents, 00761 const bool adjacencies, 00762 const bool tags, 00763 const bool store_remote_handles, 00764 const int to_proc, 00765 Buffer *buff, 00766 TupleList *entprocs = NULL, 00767 Range *allsent = NULL); 00768 00769 ErrorCode unpack_buffer(unsigned char *buff_ptr, 00770 const bool store_remote_handles, 00771 const int from_proc, 00772 const int ind, 00773 std::vector<std::vector<EntityHandle> > &L1hloc, 00774 std::vector<std::vector<EntityHandle> > &L1hrem, 00775 std::vector<std::vector<int> > &L1p, 00776 std::vector<EntityHandle> &L2hloc, 00777 std::vector<EntityHandle> &L2hrem, 00778 std::vector<unsigned int> &L2p, 00779 std::vector<EntityHandle> &new_ents, 00780 const bool created_iface = false); 00781 00782 ErrorCode pack_entities(Range &entities, 00783 Buffer *buff, 00784 const bool store_remote_handles, 00785 const int to_proc, 00786 const bool is_iface, 00787 TupleList *entprocs = NULL, 00788 Range *allsent = NULL); 00789 00791 ErrorCode unpack_entities(unsigned char *&buff_ptr, 00792 const bool store_remote_handles, 00793 const int from_ind, 00794 const bool is_iface, 00795 std::vector<std::vector<EntityHandle> > &L1hloc, 00796 std::vector<std::vector<EntityHandle> > &L1hrem, 00797 std::vector<std::vector<int> > &L1p, 00798 std::vector<EntityHandle> &L2hloc, 00799 std::vector<EntityHandle> &L2hrem, 00800 std::vector<unsigned int> &L2p, 00801 std::vector<EntityHandle> &new_ents, 00802 const bool created_iface = false); 00803 00806 ErrorCode check_all_shared_handles(bool print_em = false); 00807 00808 static ErrorCode check_all_shared_handles(ParallelComm **pcs, 00809 int num_pcs); 00810 00811 struct SharedEntityData { 00812 EntityHandle local; 00813 EntityHandle remote; 00814 EntityID owner; 00815 }; 00816 00817 ErrorCode pack_shared_handles( 00818 std::vector<std::vector<SharedEntityData> > &send_data); 00819 00820 // check consistency of sharedEnts against their tags and their 00821 // vertices' tags 00822 ErrorCode check_local_shared(); 00823 00824 // check contents of communicated shared entity data against tags 00825 ErrorCode check_my_shared_handles( 00826 std::vector<std::vector<SharedEntityData> > &shents, 00827 const char *prefix = NULL); 00828 00830 void set_rank(unsigned int r); 00831 00833 void set_size(unsigned int r); 00834 00839 int get_buffers(int to_proc, bool *is_new = NULL); 00840 00841 00843 const std::vector<unsigned int> &buff_procs() const; 00844 00845 /* \brief Unpack message with remote handles 00846 * PUBLIC ONLY FOR TESTING! 00847 */ 00848 ErrorCode unpack_remote_handles(unsigned int from_proc, 00849 unsigned char *&buff_ptr, 00850 std::vector<EntityHandle> &L2hloc, 00851 std::vector<EntityHandle> &L2hrem, 00852 std::vector<unsigned int> &L2p); 00853 00854 /* \brief Pack message with remote handles 00855 * PUBLIC ONLY FOR TESTING! 00856 */ 00857 ErrorCode pack_remote_handles(std::vector<EntityHandle> &L1hloc, 00858 std::vector<EntityHandle> &L1hrem, 00859 std::vector<int> &procs, 00860 unsigned int to_proc, 00861 Buffer *buff); 00862 00863 // each iterate in proc_nvecs contains a set of procs and the entities *possibly* 00864 // on the interface between those procs; this function makes sets for each, 00865 // and tags the set with the procs sharing it; interface sets are optionally 00866 // returned; NOTE: a subsequent step is used to verify entities on the interface 00867 // and remove them if they're not shared 00868 ErrorCode create_interface_sets(std::map<std::vector<int>, std::vector<EntityHandle> > &proc_nvecs); 00869 00870 // do the same but working straight from sharedEnts 00871 ErrorCode create_interface_sets(EntityHandle this_set, int resolve_dim, int shared_dim); 00872 00873 ErrorCode tag_shared_verts(TupleList &shared_ents, 00874 std::map<std::vector<int>, std::vector<EntityHandle> > &proc_nvecs, 00875 Range &proc_verts, 00876 unsigned int i_extra = 1); 00877 00878 ErrorCode list_entities(const EntityHandle *ents, int num_ents); 00879 00880 ErrorCode list_entities(const Range &ents); 00881 00882 void set_send_request(int n_request); // set send request array 00883 00884 void set_recv_request(int n_request); // set recv request array 00885 00887 // changed to public function (HJK) 00888 void reset_all_buffers(); 00889 00890 static const unsigned int INITIAL_BUFF_SIZE; 00891 00893 void set_debug_verbosity(int verb); 00894 00896 int get_debug_verbosity(); 00897 00898 /* \brief Gather tag value from entities down to a specified root proc 00899 * This function gathers data from a domain-decomposed mesh onto a global mesh 00900 * represented on the root processor. On the root, this gather mesh is distinct from 00901 * the root's domain-decomposed subdomain. Entities are matched by global id, or by 00902 * another tag if its handle is input. The dimension of all entities in gather_ents should 00903 * be the same, since this is the dimension of entities in gather_set that are queried for 00904 * matching global id tags. 00905 * \param gather_ents (Local) entities from which to gather data 00906 * \param tag_handle Tag whose values are being gathered 00907 * \param id_tag Tag to use for matching entities (global id used by default) 00908 * \param gather_set On root, set containing global mesh onto which to put data 00909 * \param root_proc_rank Rank of the specified root processor (default rank is 0) 00910 */ 00911 ErrorCode gather_data(Range &gather_ents, Tag &tag_handle, 00912 Tag id_tag = 0, EntityHandle gather_set = 0, int root_proc_rank = 0); 00913 00914 /* \brief communicate extra points positions on boundary 00915 * This function is called after intersection of 2 meshes, to settle the 00916 * position of the intersection points on the boundary (interface) 00917 * The initial mesh distributed on each processor is decomposed after 00918 * intersection with another mesh, such as that new points are created on the 00919 * boundary. these points should better match at the interface ! 00920 * we perform an extra caution step, to ensure the robustness of the 00921 * intersection algorithm; only shared edges extra nodes 00922 * will be actually needed to be communicated, but we just pass by reference 00923 * the whole extraNodesVec structure, we do 00924 * not need to construct another data structure 00925 * The node positions on edges that are owned will be communicated to other 00926 * processors 00927 * 00928 * \param edges total range of entities 00929 * \param shared_edges_owned edges for which to communicate data 00930 * \param extraNodesVec handles of intersection vertices on all edges; 00931 */ 00932 ErrorCode settle_intersection_points(Range & edges, Range & shared_edges_owned, 00933 std::vector<std::vector<EntityHandle> *> & extraNodesVec, double tolerance); 00934 00935 private: 00936 00937 ErrorCode reduce_void(int tag_data_type, const MPI_Op mpi_op, int num_ents, void *old_vals, void *new_vals); 00938 00939 template <class T> ErrorCode reduce(const MPI_Op mpi_op, int num_ents, void *old_vals, void *new_vals); 00940 00941 void print_debug_isend(int from, int to, unsigned char *buff, 00942 int tag, int size); 00943 00944 void print_debug_irecv(int to, int from, unsigned char *buff, int size, 00945 int tag, int incoming); 00946 00947 void print_debug_recd(MPI_Status status); 00948 00949 void print_debug_waitany(std::vector<MPI_Request> &reqs, int tag, int proc); 00950 00951 // common initialization code, called from various constructors 00952 void initialize(); 00953 00954 ErrorCode set_sharing_data(EntityHandle ent, unsigned char pstatus, 00955 int old_nump, int new_nump, 00956 int *ps, EntityHandle *hs); 00957 00958 ErrorCode check_clean_iface(Range &allsent); 00959 00960 void define_mpe(); 00961 00962 ErrorCode get_sent_ents(const bool is_iface, 00963 const int bridge_dim, const int ghost_dim, 00964 const int num_layers, const int addl_ents, 00965 Range *sent_ents, Range &allsent, 00966 TupleList &entprocs); 00967 00978 ErrorCode set_pstatus_entities(Range &pstatus_ents, 00979 unsigned char pstatus_val, 00980 bool lower_dim_ents = false, 00981 bool verts_too = true, 00982 int operation = Interface::UNION); 00983 00994 ErrorCode set_pstatus_entities(EntityHandle *pstatus_ents, 00995 int num_ents, 00996 unsigned char pstatus_val, 00997 bool lower_dim_ents = false, 00998 bool verts_too = true, 00999 int operation = Interface::UNION); 01000 01001 int num_subranges(const Range &this_range); 01002 01004 int estimate_ents_buffer_size(Range &entities, 01005 const bool store_remote_handles); 01006 01008 int estimate_sets_buffer_size(Range &entities, 01009 const bool store_remote_handles); 01010 01012 ErrorCode send_buffer(const unsigned int to_proc, 01013 Buffer *send_buff, 01014 const int msg_tag, 01015 MPI_Request &send_req, 01016 MPI_Request &ack_recv_req, 01017 int *ack_buff, 01018 int &this_incoming, 01019 int next_mesg_tag = -1, 01020 Buffer *next_recv_buff = NULL, 01021 MPI_Request *next_recv_req = NULL, 01022 int *next_incoming = NULL); 01023 01027 ErrorCode recv_buffer(int mesg_tag_expected, 01028 const MPI_Status &mpi_status, 01029 Buffer *recv_buff, 01030 MPI_Request &recv_2nd_req, 01031 MPI_Request &ack_req, 01032 int &this_incoming, 01033 Buffer *send_buff, 01034 MPI_Request &send_req, 01035 MPI_Request &sent_ack_req, 01036 bool &done, 01037 Buffer *next_buff = NULL, 01038 int next_tag = -1, 01039 MPI_Request *next_req = NULL, 01040 int *next_incoming = NULL); 01041 01044 ErrorCode pack_entity_seq(const int nodes_per_entity, 01045 const bool store_remote_handles, 01046 const int to_proc, 01047 Range &these_ents, 01048 std::vector<EntityHandle> &entities, 01049 Buffer *buff); 01050 01051 ErrorCode print_buffer(unsigned char *buff_ptr, int mesg_type, int from_proc, 01052 bool sent); 01053 01056 ErrorCode unpack_iface_entities(unsigned char *&buff_ptr, 01057 const int from_proc, 01058 const int ind, 01059 std::vector<EntityHandle> &recd_ents); 01060 01061 ErrorCode pack_sets(Range &entities, 01062 Buffer *buff, 01063 const bool store_handles, 01064 const int to_proc); 01065 01066 ErrorCode unpack_sets(unsigned char *&buff_ptr, 01067 std::vector<EntityHandle> &entities, 01068 const bool store_handles, 01069 const int to_proc); 01070 01071 ErrorCode pack_adjacencies(Range &entities, 01072 Range::const_iterator &start_rit, 01073 Range &whole_range, 01074 unsigned char *&buff_ptr, 01075 int &count, 01076 const bool just_count, 01077 const bool store_handles, 01078 const int to_proc); 01079 01080 ErrorCode unpack_adjacencies(unsigned char *&buff_ptr, 01081 Range &entities, 01082 const bool store_handles, 01083 const int from_proc); 01084 01085 01086 /* \brief Unpack message with remote handles (const pointer to buffer) 01087 */ 01088 ErrorCode unpack_remote_handles(unsigned int from_proc, 01089 const unsigned char *buff_ptr, 01090 std::vector<EntityHandle> &L2hloc, 01091 std::vector<EntityHandle> &L2hrem, 01092 std::vector<unsigned int> &L2p); 01093 01095 ErrorCode find_existing_entity(const bool is_iface, 01096 const int owner_p, 01097 const EntityHandle owner_h, 01098 const int num_ents, 01099 const EntityHandle *connect, 01100 const int num_connect, 01101 const EntityType this_type, 01102 std::vector<EntityHandle> &L2hloc, 01103 std::vector<EntityHandle> &L2hrem, 01104 std::vector<unsigned int> &L2p, 01105 EntityHandle &new_h); 01106 01107 ErrorCode build_sharedhps_list(const EntityHandle entity, 01108 const unsigned char pstatus, 01109 const int sharedp, 01110 const std::set<unsigned int> &procs, 01111 unsigned int &num_ents, 01112 int *tmp_procs, 01113 EntityHandle *tmp_handles); 01114 01129 ErrorCode get_tag_send_list( const Range& all_entities, 01130 std::vector<Tag>& all_tags, 01131 std::vector<Range>& tag_ranges ); 01132 01177 ErrorCode pack_tags(Range &entities, 01178 const std::vector<Tag> &src_tags, 01179 const std::vector<Tag> &dst_tags, 01180 const std::vector<Range> &tag_ranges, 01181 Buffer *buff, 01182 const bool store_handles, 01183 const int to_proc); 01184 01190 ErrorCode packed_tag_size( Tag source_tag, 01191 const Range& entities, 01192 int& count_out ); 01193 01219 ErrorCode pack_tag( Tag source_tag, 01220 Tag destination_tag, 01221 const Range &entities, 01222 const std::vector<EntityHandle> &whole_range, 01223 Buffer *buff, 01224 const bool store_remote_handles, 01225 const int to_proc ); 01226 01227 ErrorCode unpack_tags(unsigned char *&buff_ptr, 01228 std::vector<EntityHandle> &entities, 01229 const bool store_handles, 01230 const int to_proc, 01231 const MPI_Op * const mpi_op = NULL); 01232 01233 ErrorCode tag_shared_verts(TupleList &shared_verts, 01234 Range *skin_ents, 01235 std::map<std::vector<int>, std::vector<EntityHandle> > &proc_nvecs, 01236 Range &proc_verts); 01237 01238 ErrorCode get_proc_nvecs(int resolve_dim, 01239 int shared_dim, 01240 Range *skin_ents, 01241 std::map<std::vector<int>, std::vector<EntityHandle> > &proc_nvecs); 01242 01243 // after verifying shared entities, now parent/child links between sets can be established 01244 ErrorCode create_iface_pc_links(); 01245 01248 ErrorCode pack_range_map(Range &this_range, EntityHandle actual_start, 01249 HandleMap &handle_map); 01250 01252 bool is_iface_proc(EntityHandle this_set, int to_proc); 01253 01256 ErrorCode update_iface_sets(Range &sent_ents, 01257 std::vector<EntityHandle> &remote_handles, 01258 int from_proc); 01259 01263 ErrorCode get_ghosted_entities(int bridge_dim, 01264 int ghost_dim, 01265 int to_proc, 01266 int num_layers, 01267 int addl_ents, 01268 Range &ghosted_ents); 01269 01271 ErrorCode add_verts(Range &sent_ents); 01272 01277 ErrorCode exchange_all_shared_handles( 01278 std::vector<std::vector<SharedEntityData> > &send_data, 01279 std::vector<std::vector<SharedEntityData> > &result); 01280 01286 ErrorCode get_remote_handles(const bool store_remote_handles, 01287 EntityHandle *from_vec, 01288 EntityHandle *to_vec_tmp, 01289 int num_ents, int to_proc, 01290 const std::vector<EntityHandle> &new_ents); 01291 01294 ErrorCode get_remote_handles(const bool store_remote_handles, 01295 const Range &from_range, 01296 Range &to_range, 01297 int to_proc, 01298 const std::vector<EntityHandle> &new_ents); 01299 01301 ErrorCode get_remote_handles(const bool store_remote_handles, 01302 const Range &from_range, 01303 EntityHandle *to_vec, 01304 int to_proc, 01305 const std::vector<EntityHandle> &new_ents); 01306 01309 ErrorCode get_local_handles(EntityHandle *from_vec, 01310 int num_ents, 01311 const Range &new_ents); 01312 01314 ErrorCode get_local_handles(const Range &remote_handles, 01315 Range &local_handles, 01316 const std::vector<EntityHandle> &new_ents); 01317 01319 ErrorCode get_local_handles(EntityHandle *from_vec, 01320 int num_ents, 01321 const std::vector<EntityHandle> &new_ents); 01322 01323 ErrorCode update_remote_data(Range &local_range, 01324 Range &remote_range, 01325 int other_proc, 01326 const unsigned char add_pstat); 01327 01328 ErrorCode update_remote_data(const EntityHandle new_h, 01329 const int *ps, 01330 const EntityHandle *hs, 01331 const int num_ps, 01332 const unsigned char add_pstat); 01333 01334 ErrorCode update_remote_data_old(const EntityHandle new_h, 01335 const int *ps, 01336 const EntityHandle *hs, 01337 const int num_ps, 01338 const unsigned char add_pstat); 01339 01342 ErrorCode tag_iface_entities(); 01343 01345 int add_pcomm(ParallelComm *pc); 01346 01348 void remove_pcomm(ParallelComm *pc); 01349 01352 ErrorCode check_sent_ents(Range &allsent); 01353 01355 ErrorCode assign_entities_part(std::vector<EntityHandle> &entities, const int proc); 01356 01358 ErrorCode remove_entities_part(Range &entities, const int proc); 01359 01361 Interface *mbImpl; 01362 01364 ProcConfig procConfig; 01365 01367 SequenceManager *sequenceManager; 01368 01370 Error *errorHandler; 01371 01373 std::vector<Buffer*> localOwnedBuffs, remoteOwnedBuffs; 01374 01376 //void reset_all_buffers(); 01377 01379 void delete_all_buffers(); 01380 01382 std::vector<MPI_Request> sendReqs; 01383 01385 std::vector<MPI_Request> recvReqs, recvRemotehReqs; 01386 01388 std::vector<unsigned int> buffProcs; 01389 01391 Range partitionSets, interfaceSets; 01392 01394 std::vector<EntityHandle> sharedEnts; 01395 01397 Tag sharedpTag, sharedpsTag, sharedhTag, sharedhsTag, pstatusTag, 01398 ifaceSetsTag, partitionTag; 01399 01400 int globalPartCount; 01401 01402 EntityHandle partitioningSet; 01403 01404 std::ofstream myFile; 01405 01406 int pcommID; 01407 01408 int ackbuff; 01409 01411 DebugOutput *myDebug; 01412 01414 SharedSetData* sharedSetData; 01415 01416 }; 01417 01418 inline ParallelComm::Buffer::Buffer(const Buffer &other_buff) 01419 { 01420 alloc_size = other_buff.alloc_size; 01421 mem_ptr = (unsigned char *)malloc(alloc_size); 01422 memcpy(mem_ptr, other_buff.mem_ptr, alloc_size); 01423 buff_ptr = mem_ptr + (other_buff.buff_ptr - other_buff.mem_ptr); 01424 } 01425 01426 inline ParallelComm::Buffer::Buffer(unsigned int new_size) 01427 : mem_ptr(NULL), buff_ptr(NULL), alloc_size(0) 01428 { 01429 if (new_size) this->reserve(new_size); 01430 } 01431 01432 inline ParallelComm::Buffer::~Buffer() 01433 { 01434 if (mem_ptr) { 01435 free(mem_ptr); 01436 mem_ptr = NULL; 01437 } 01438 } 01439 01440 #define DEBUG_BUFFER 0 01441 01442 inline void ParallelComm::Buffer::reserve(unsigned int new_size) { 01443 01444 #ifdef DEBUG_BUFFER 01445 int tmp_pos = 0; 01446 if (mem_ptr) { 01447 tmp_pos = buff_ptr - mem_ptr; 01448 } 01449 buff_ptr = (unsigned char *)malloc(new_size); 01450 assert(0 <= tmp_pos && tmp_pos <= (int)alloc_size); 01451 if (tmp_pos) memcpy(buff_ptr, mem_ptr, tmp_pos); 01452 if (mem_ptr) free(mem_ptr); 01453 mem_ptr = buff_ptr; 01454 alloc_size = new_size; 01455 buff_ptr = mem_ptr + tmp_pos; 01456 #else 01457 if (mem_ptr && alloc_size < new_size) { 01458 size_t tmp_pos = mem_ptr ? buff_ptr - mem_ptr : 0; 01459 mem_ptr = (unsigned char *)realloc(mem_ptr, new_size); 01460 alloc_size = new_size; 01461 buff_ptr = mem_ptr + tmp_pos; 01462 } 01463 else if (!mem_ptr) { 01464 mem_ptr = (unsigned char *)malloc(new_size); 01465 alloc_size = new_size; 01466 buff_ptr = mem_ptr; 01467 } 01468 #endif 01469 } 01470 01471 inline void ParallelComm::Buffer::check_space(unsigned int addl_space ) 01472 { 01473 assert(buff_ptr >= mem_ptr && buff_ptr <= mem_ptr+alloc_size); 01474 unsigned int new_size = buff_ptr - mem_ptr + addl_space; 01475 if (new_size > alloc_size) 01476 reserve(3*new_size/2); 01477 } 01478 01479 inline void ParallelComm::reset_all_buffers() 01480 { 01481 std::vector<Buffer*>::iterator vit; 01482 for (vit = localOwnedBuffs.begin(); vit != localOwnedBuffs.end(); vit++) 01483 (*vit)->reset_buffer(); 01484 for (vit = remoteOwnedBuffs.begin(); vit != remoteOwnedBuffs.end(); vit++) 01485 (*vit)->reset_buffer(); 01486 } 01487 01488 inline void ParallelComm::delete_all_buffers() 01489 { 01490 std::vector<Buffer*>::iterator vit; 01491 for (vit = localOwnedBuffs.begin(); vit != localOwnedBuffs.end(); vit++) 01492 delete (*vit); 01493 localOwnedBuffs.clear(); 01494 01495 for (vit = remoteOwnedBuffs.begin(); vit != remoteOwnedBuffs.end(); vit++) 01496 delete (*vit); 01497 remoteOwnedBuffs.clear(); 01498 } 01499 01500 inline const std::vector<unsigned int> &ParallelComm::buff_procs() const 01501 { 01502 return buffProcs; 01503 } 01504 01505 inline ErrorCode ParallelComm::get_shared_proc_tags(Tag &sharedp, 01506 Tag &sharedps, 01507 Tag &sharedh, 01508 Tag &sharedhs, 01509 Tag &pstatus) 01510 { 01511 sharedp = sharedp_tag(); 01512 sharedps = sharedps_tag(); 01513 sharedh = sharedh_tag(); 01514 sharedhs = sharedhs_tag(); 01515 pstatus = pstatus_tag(); 01516 01517 return MB_SUCCESS; 01518 } 01519 01520 inline ErrorCode ParallelComm::exchange_tags( const char *tag_name, 01521 const Range &entities) 01522 { 01523 // get the tag handle 01524 std::vector<Tag> tags(1); 01525 ErrorCode result = mbImpl->tag_get_handle(tag_name, 0, MB_TYPE_OPAQUE, tags[0], MB_TAG_ANY); 01526 if (MB_SUCCESS != result) return result; 01527 else if (!tags[0]) return MB_TAG_NOT_FOUND; 01528 01529 return exchange_tags(tags, tags, entities); 01530 } 01531 01532 inline ErrorCode ParallelComm::exchange_tags( Tag tagh, 01533 const Range &entities) 01534 { 01535 // get the tag handle 01536 std::vector<Tag> tags; 01537 tags.push_back(tagh); 01538 01539 return exchange_tags(tags, tags, entities); 01540 } 01541 01542 inline ErrorCode ParallelComm::reduce_tags( const char *tag_name, 01543 const MPI_Op mpi_op, 01544 const Range &entities) 01545 { 01546 // get the tag handle 01547 std::vector<Tag> tags(1); 01548 ErrorCode result = mbImpl->tag_get_handle(tag_name, 0, MB_TYPE_OPAQUE, tags[0], MB_TAG_ANY); 01549 if (MB_SUCCESS != result) return result; 01550 else if (!tags[0]) return MB_TAG_NOT_FOUND; 01551 01552 return reduce_tags(tags, tags, mpi_op, entities); 01553 } 01554 01555 inline ErrorCode ParallelComm::reduce_tags( Tag tagh, 01556 const MPI_Op mpi_op, 01557 const Range &entities) 01558 { 01559 // get the tag handle 01560 std::vector<Tag> tags; 01561 tags.push_back(tagh); 01562 01563 return reduce_tags(tags, tags, mpi_op, entities); 01564 } 01565 01566 inline ErrorCode ParallelComm::get_comm_procs(std::set<unsigned int> &procs) 01567 { 01568 ErrorCode result = get_interface_procs(procs); 01569 if (MB_SUCCESS != result) return result; 01570 01571 std::copy(buffProcs.begin(), buffProcs.end(), std::inserter(procs, procs.begin())); 01572 01573 return MB_SUCCESS; 01574 } 01575 01576 inline ErrorCode ParallelComm::get_owner(EntityHandle entity, 01577 int &owner) 01578 { 01579 EntityHandle tmp_handle; 01580 return get_owner_handle(entity, owner, tmp_handle); 01581 } 01582 01583 /* \brief Unpack message with remote handles (const pointer to buffer) 01584 */ 01585 inline ErrorCode ParallelComm::unpack_remote_handles(unsigned int from_proc, 01586 const unsigned char *buff_ptr, 01587 std::vector<EntityHandle> &L2hloc, 01588 std::vector<EntityHandle> &L2hrem, 01589 std::vector<unsigned int> &L2p) 01590 { 01591 // cast away const-ness, we won't be passing back a modified ptr 01592 unsigned char *tmp_buff = const_cast<unsigned char*>(buff_ptr); 01593 return unpack_remote_handles(from_proc, tmp_buff, L2hloc, L2hrem, L2p); 01594 } 01595 01596 inline void ParallelComm::set_rank(unsigned int r) 01597 { 01598 procConfig.proc_rank(r); 01599 if (procConfig.proc_size() < r) procConfig.proc_size(r+1); 01600 } 01601 01602 inline void ParallelComm::set_size(unsigned int s) 01603 { 01604 procConfig.proc_size(s); 01605 } 01606 01607 inline ErrorCode ParallelComm::get_sharing_data(const EntityHandle *entities, 01608 int num_entities, 01609 std::set<int> &procs, 01610 int op) 01611 { 01612 Range dum_range; 01613 // cast away constness 'cuz the range is passed as const 01614 EntityHandle *ents_cast = const_cast<EntityHandle*>(entities); 01615 std::copy(ents_cast, ents_cast+num_entities, range_inserter(dum_range)); 01616 return get_sharing_data(dum_range, procs, op); 01617 } 01618 01619 inline ErrorCode ParallelComm::get_sharing_data(const EntityHandle entity, 01620 int *ps, 01621 EntityHandle *hs, 01622 unsigned char &pstat, 01623 int &num_ps) 01624 { 01625 unsigned int dum_ps; 01626 ErrorCode result = get_sharing_data(entity, ps, hs, pstat, dum_ps); 01627 if (MB_SUCCESS == result) 01628 num_ps = dum_ps; 01629 return result; 01630 } 01631 01632 inline void ParallelComm::set_send_request(int n_request) 01633 { 01634 sendReqs.resize(n_request, MPI_REQUEST_NULL); 01635 } 01636 01637 inline void ParallelComm::set_recv_request(int n_request) 01638 { 01639 recvReqs.resize(n_request, MPI_REQUEST_NULL); 01640 } 01641 01642 } // namespace moab 01643 01644 #endif
1.7.6.1 
