2 #ifndef DUNE_INDICESSYNCER_HH
3 #define DUNE_INDICESSYNCER_HH
51 typedef typename ParallelIndexSet::LocalIndex::Attribute
Attribute;
92 void sync(T1& numberer);
106 char* receiveBuffer_;
109 std::size_t* sendBufferSizes_;
112 int receiveBufferSize_;
117 struct MessageInformation
134 class DefaultNumberer
144 return std::numeric_limits<size_t>::max();
149 MPI_Datatype datatype_;
170 typedef std::map<int, GlobalIndexList> GlobalIndicesMap;
180 GlobalIndicesMap globalMap_;
196 typedef std::map<int,BoolList> BoolMap;
205 std::map<int,MessageInformation> infoSend_;
223 typedef Dune::tuple<RemoteIndexModifier,GlobalIndexModifier,BoolListModifier,
224 const ConstRemoteIndexIterator> IteratorTuple;
257 Iterators& operator++();
265 const std::pair<GlobalIndex,Attribute>& global);
277 std::pair<GlobalIndex,Attribute>& globalIndexPair()
const;
305 bool isNotAtEnd()
const;
312 bool isAtEnd()
const;
328 typedef std::map<int,Iterators> IteratorsMap;
341 IteratorsMap iteratorsMap_;
344 void calculateMessageSizes();
353 void packAndSend(
int destination,
char* buffer, std::size_t bufferSize, MPI_Request& req);
359 template<
typename T1>
360 void recvAndUnpack(T1& numberer);
365 void registerMessageDatatype();
370 void insertIntoRemoteIndexList(
int process,
371 const std::pair<GlobalIndex,Attribute>& global,
377 void resetIteratorsMap();
397 template<
typename TG,
typename TA>
398 bool operator<(const IndexPair<TG,ParallelLocalIndex<TA> >& i1,
399 const std::pair<TG,TA>& i2)
401 return i1.global() < i2.first ||
402 (i1.global() == i2.first && i1.local().attribute()<i2.second);
405 template<
typename TG,
typename TA>
406 bool operator<(const std::pair<TG,TA>& i1,
409 return i1.first < i2.
global() ||
410 (i1.first == i2.global() && i1.second<i2.local().attribute());
413 template<
typename TG,
typename TA>
415 const std::pair<TG,TA>& i2)
417 return (i1.global() == i2.first && i1.local().attribute()==i2.second);
420 template<
typename TG,
typename TA>
422 const std::pair<TG,TA>& i2)
424 return (i1.global() != i2.first || i1.local().attribute()!=i2.second);
427 template<
typename TG,
typename TA>
431 return (i1.global() == i2.first && i1.local().attribute()==i2.second);
434 template<
typename TG,
typename TA>
438 return (i1.global() != i2.first || i1.local().attribute()!=i2.second);
457 template<
typename T,
typename A,
typename A1>
464 for(RemoteIterator remote = remoteIndices.
begin(), end =remoteIndices.
end(); remote != end; ++remote){
466 typedef typename RemoteIndexList::const_iterator RemoteIndexIterator;
468 GlobalIndexList& global = globalMap[remote->first];
469 RemoteIndexList& rList = *(remote->second.first);
471 for(RemoteIndexIterator index = rList.begin(), riEnd = rList.end();
472 index != riEnd; ++index){
473 global.
push_back(std::make_pair(index->localIndexPair().global(),
474 index->localIndexPair().local().attribute()));
487 template<
typename T,
typename A,
typename A1>
489 SLList<std::pair<
typename T::GlobalIndex,
490 typename T::LocalIndex::Attribute>,A> >& globalMap,
496 typedef typename T::GlobalIndex GlobalIndex;
497 typedef typename T::LocalIndex::Attribute Attribute;
498 typedef std::pair<GlobalIndex,Attribute> GlobalIndexPair;
500 typedef typename GlobalIndexPairList::iterator GlobalIndexIterator;
502 assert(globalMap.size()==
static_cast<std::size_t
>(remoteIndices.
neighbours()));
504 typename std::map<int,GlobalIndexPairList>::iterator global = globalMap.begin();
505 RemoteIterator end = remoteIndices.remoteIndices_.end();
507 for(RemoteIterator remote = remoteIndices.remoteIndices_.begin(); remote != end; ++remote, ++global){
508 typedef typename T::const_iterator IndexIterator;
510 assert(remote->first==global->first);
511 assert(remote->second.first->size() == global->second.size());
513 RemoteIndexIterator riEnd = remote->second.first->end();
514 RemoteIndexIterator rIndex = remote->second.first->begin();
515 GlobalIndexIterator gIndex = global->second.begin();
516 IndexIterator index = indexSet.begin();
518 assert(rIndex==riEnd || gIndex != global->second.end());
519 while(rIndex != riEnd){
521 assert(gIndex != global->second.end());
523 while(!(index->global() == gIndex->first
524 && index->local().attribute() == gIndex->second)){
529 if (index->global() > gIndex->first){
530 index=indexSet.begin();
534 assert(index != indexSet.end() && *index == *gIndex);
536 rIndex->localIndex_ = &(*index);
542 remoteIndices.sourceSeqNo_ = remoteIndices.source_->
seqNo();
543 remoteIndices.destSeqNo_ = remoteIndices.target_->
seqNo();
549 : indexSet_(indexSet), remoteIndices_(remoteIndices)
552 assert(remoteIndices.source_ == remoteIndices.target_);
553 assert(remoteIndices.source_ == &indexSet);
561 : iterators_(remoteIndices.beginModify(), globalIndices.beginModify(),
562 booleans.beginModify(), remoteIndices.end())
573 ++(get<0>(iterators_));
574 ++(get<1>(iterators_));
575 ++(get<2>(iterators_));
581 const std::pair<GlobalIndex,Attribute>& global)
583 get<0>(iterators_).insert(index);
584 get<1>(iterators_).insert(global);
585 get<2>(iterators_).insert(
false);
592 return *(get<0>(iterators_));
599 return *(get<1>(iterators_));
605 return *(get<2>(iterators_));
621 return get<0>(iterators_)!=get<3>(iterators_);
627 return get<0>(iterators_)==get<3>(iterators_);
633 MPI_Datatype type[2] = {MPI_INT, MPI_INT};
634 int blocklength[2] = {1,1};
635 MPI_Aint displacement[2];
639 MessageInformation message;
641 MPI_Address( &(message.publish), displacement);
642 MPI_Address( &(message.pairs), displacement+1);
645 MPI_Address(&message, &base);
646 displacement[0] -= base;
647 displacement[1] -= base;
649 MPI_Type_struct( 2, blocklength, displacement, type, &datatype_);
650 MPI_Type_commit(&datatype_);
654 void IndicesSyncer<T>::calculateMessageSizes()
657 typedef CollectiveIterator<T,typename RemoteIndices::Allocator> CollectiveIterator;
659 IndexIterator iEnd = indexSet_.end();
660 CollectiveIterator collIter = remoteIndices_.template
iterator<true>();
662 for(IndexIterator index = indexSet_.begin(); index != iEnd; ++index){
663 collIter.advance(index->global(), index->local().attribute());
668 typedef typename CollectiveIterator::iterator ValidIterator;
669 ValidIterator end = collIter.end();
672 for(ValidIterator valid = collIter.begin(); valid != end; ++valid){
677 Dune::dverb<<rank_<<
": publishing "<<knownRemote<<
" for index "<<index->global()<<
" for processes ";
680 for(ValidIterator valid = collIter.begin(); valid != end; ++valid){
681 ++(infoSend_[valid.process()].publish);
682 (infoSend_[valid.process()].pairs) += knownRemote;
684 Dune::dverb<<
"(publish="<<infoSend_[valid.process()].publish<<
", pairs="<<infoSend_[valid.process()].pairs
691 typedef typename std::map<int,MessageInformation>::const_iterator
694 const MessageIterator end = infoSend_.end();
699 MessageInformation dummy;
701 MessageIterator messageIter= infoSend_.begin();
703 typedef typename RemoteIndices::RemoteIndexMap::const_iterator RemoteIterator;
704 const RemoteIterator rend = remoteIndices_.
end();
707 for(RemoteIterator remote = remoteIndices_.
begin(); remote != rend; ++remote, ++neighbour){
708 MessageInformation* message;
709 MessageInformation recv;
711 if(messageIter != end && messageIter->first==remote->first){
713 message =
const_cast<MessageInformation*
>(&(messageIter->second));
719 sendBufferSizes_[neighbour]=0;
722 MPI_Pack_size(1, MPI_INT,remoteIndices_.
communicator(), &tsize);
723 sendBufferSizes_[neighbour] += tsize;
725 for(
int i=0; i < message->publish; ++i){
727 MPI_Pack_size(1, MPITraits<GlobalIndex>::getType(), remoteIndices_.
communicator(), &tsize);
728 sendBufferSizes_[neighbour] += tsize;
730 MPI_Pack_size(1, MPI_CHAR, remoteIndices_.
communicator(), &tsize);
731 sendBufferSizes_[neighbour] += tsize;
733 MPI_Pack_size(1, MPI_INT, remoteIndices_.
communicator(), &tsize);
734 sendBufferSizes_[neighbour] += tsize;
736 for(
int i=0; i < message->pairs; ++i){
738 MPI_Pack_size(1, MPI_INT, remoteIndices_.
communicator(), &tsize);
739 sendBufferSizes_[neighbour] += tsize;
741 MPI_Pack_size(1, MPI_CHAR, remoteIndices_.
communicator(), &tsize);
742 sendBufferSizes_[neighbour] += tsize;
745 Dune::dverb<<rank_<<
": Buffer (neighbour="<<remote->first<<
") size is "<< sendBufferSizes_[neighbour]<<
" for publish="<<message->publish<<
" pairs="<<message->pairs<<std::endl;
753 DefaultNumberer numberer;
758 template<
typename T1>
770 const RemoteIterator end = remoteIndices_.
end();
774 std::size_t noOldNeighbours = remoteIndices_.
neighbours();
775 int* oldNeighbours =
new int[noOldNeighbours];
776 sendBufferSizes_ =
new std::size_t[noOldNeighbours];
779 for(RemoteIterator remote = remoteIndices_.
begin(); remote != end; ++remote, ++i){
783 oldNeighbours[i]=remote->first;
786 assert(remote->second.first==remote->second.second);
792 BoolList& added = oldMap_[remote->first];
793 RemoteIndexIterator riEnd = rList.
end();
795 for(RemoteIndexIterator index = rList.
begin();
796 index != riEnd; ++index){
797 global.
push_back(std::make_pair(index->localIndexPair().global(),
798 index->localIndexPair().local().attribute()));
802 Iterators iterators(rList, global, added);
803 iteratorsMap_.insert(std::make_pair(remote->first, iterators));
804 assert(checkReset(iteratorsMap_[remote->first], rList,global,added));
808 calculateMessageSizes();
811 receiveBufferSize_=1;
812 sendBuffers_ =
new char*[noOldNeighbours];
814 for(std::size_t i=0; i<noOldNeighbours; ++i){
815 sendBuffers_[i] =
new char[sendBufferSizes_[i]];
816 receiveBufferSize_ = std::max(receiveBufferSize_, static_cast<int>(sendBufferSizes_[i]));
819 receiveBuffer_=
new char[receiveBufferSize_];
821 indexSet_.beginResize();
825 for(i = 0; i<noOldNeighbours; ++i)
830 MPI_Request* requests =
new MPI_Request[noOldNeighbours];
831 MPI_Status* statuses =
new MPI_Status[noOldNeighbours];
834 for(i = 0; i<noOldNeighbours; ++i)
835 packAndSend(oldNeighbours[i], sendBuffers_[i], sendBufferSizes_[i], requests[i]);
838 for(i = 0; i<noOldNeighbours; ++i)
839 recvAndUnpack(numberer);
846 delete[] receiveBuffer_;
850 if(MPI_SUCCESS!=MPI_Waitall(noOldNeighbours, requests, statuses)){
851 std::cerr<<
": MPI_Error occurred while sending message"<<std::endl;
852 for(i=0;i< noOldNeighbours;i++)
853 if(MPI_SUCCESS!=statuses[i].MPI_ERROR)
854 std::cerr<<
"Destination "<<statuses[i].MPI_SOURCE<<
" error code: "<<statuses[i].MPI_ERROR<<std::endl;
860 for(std::size_t i=0; i<noOldNeighbours; ++i)
861 delete[] sendBuffers_[i];
863 delete[] sendBuffers_;
864 delete[] sendBufferSizes_;
867 iteratorsMap_.clear();
869 indexSet_.endResize();
871 delete[] oldNeighbours;
879 remoteIndices_.sourceSeqNo_ = remoteIndices_.destSeqNo_ = indexSet_.seqNo();
887 IndexIterator iEnd = indexSet_.end();
892 assert(checkReset());
895 MPI_Pack(&(infoSend_[destination].publish), 1, MPI_INT, buffer, bufferSize, &bpos,
898 for(IndexIterator index = indexSet_.begin(); index != iEnd; ++index){
900 typedef typename IteratorsMap::iterator Iterator;
901 Iterator iteratorsEnd = iteratorsMap_.end();
904 for(Iterator iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators){
905 while(iterators->second.isNotAtEnd() &&
906 iterators->second.globalIndexPair().first < index->global())
907 ++(iterators->second);
908 assert(!iterators->second.isNotAtEnd() || iterators->second.globalIndexPair().first >= index->global());
915 bool knownRemote =
false;
917 for(Iterator iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators)
919 std::pair<GlobalIndex,Attribute> p;
920 if (iterators->second.isNotAtEnd())
922 p = iterators->second.globalIndexPair();
925 if(iterators->second.isNotAtEnd() && iterators->second.isOld()
926 && iterators->second.globalIndexPair().first == index->global()){
928 if(destination == iterators->first)
937 Dune::dverb<<rank_<<
": sending "<<indices<<
" for index "<<index->global()<<
" to "<<destination<<std::endl;
941 MPI_Pack(const_cast<GlobalIndex*>(&(index->global())), 1, MPITraits<GlobalIndex>::getType(), buffer, bufferSize, &bpos,
944 char attr = index->local().attribute();
945 MPI_Pack(&attr, 1, MPI_CHAR, buffer, bufferSize, &bpos,
949 MPI_Pack(&indices, 1, MPI_INT, buffer, bufferSize, &bpos,
953 for(Iterator iterators = iteratorsMap_.begin(); iteratorsEnd != iterators; ++iterators)
954 if(iterators->second.isNotAtEnd() && iterators->second.isOld()
955 && iterators->second.globalIndexPair().first == index->global()){
956 int process = iterators->first;
959 assert(pairs <= infoSend_[destination].pairs);
960 MPI_Pack(&process, 1, MPI_INT, buffer, bufferSize, &bpos,
962 char attr = iterators->second.remoteIndex().attribute();
964 MPI_Pack(&attr, 1, MPI_CHAR, buffer, bufferSize, &bpos,
970 Dune::dvverb<<
" (publish="<<published<<
", pairs="<<pairs<<
")"<<std::endl;
971 assert(published <= infoSend_[destination].publish);
975 assert(published == infoSend_[destination].publish);
976 assert(pairs == infoSend_[destination].pairs);
979 Dune::dverb << rank_<<
": Sending message of "<<bpos<<
" bytes to "<<destination<<std::endl;
981 MPI_Issend(buffer, bpos, MPI_PACKED, destination, 345, remoteIndices_.
communicator(),&request);
985 inline void IndicesSyncer<T>::insertIntoRemoteIndexList(
int process,
986 const std::pair<GlobalIndex,Attribute>& globalPair,
989 Dune::dverb<<
"Inserting from "<<process<<
" "<<globalPair.first<<
", "<<
990 globalPair.second<<
" "<<attribute<<std::endl;
995 typename IteratorsMap::iterator found = iteratorsMap_.find(process);
997 if( found == iteratorsMap_.end() ){
998 Dune::dverb<<
"Discovered new neighbour "<<process<<std::endl;
999 RemoteIndexList* rlist =
new RemoteIndexList();
1000 remoteIndices_.remoteIndices_.insert(std::make_pair(process,std::make_pair(rlist,rlist)));
1001 Iterators iterators = Iterators(*rlist, globalMap_[process], oldMap_[process]);
1002 found = iteratorsMap_.insert(std::make_pair(process, iterators)).first;
1005 Iterators& iterators = found->second;
1008 while(iterators.isNotAtEnd() && iterators.globalIndexPair() < globalPair){
1014 if(iterators.isAtEnd() || iterators.globalIndexPair() != globalPair){
1017 iterators.insert(RemoteIndex(
Attribute(attribute)),globalPair);
1022 bool indexIsThere=
false;
1023 for(Iterators tmpIterators = iterators;
1024 !tmpIterators.isAtEnd() && tmpIterators.globalIndexPair() == globalPair;
1027 if(tmpIterators.globalIndexPair().second == attribute){
1035 iterators.insert(RemoteIndex(
Attribute(attribute)),globalPair);
1038 template<
typename T>
1039 template<
typename T1>
1040 void IndicesSyncer<T>::recvAndUnpack(T1& numberer)
1044 IndexIterator iEnd = indexSet_.end();
1045 IndexIterator index = indexSet_.begin();
1049 assert(checkReset());
1054 MPI_Probe(MPI_ANY_SOURCE, 345, remoteIndices_.
communicator(), &status);
1056 int source=status.MPI_SOURCE;
1058 MPI_Get_count(&status, MPI_PACKED, &count);
1060 Dune::dvverb<<rank_<<
": Receiving message from "<< source<<
" with "<<count<<
" bytes"<<std::endl;
1062 if(count>receiveBufferSize_){
1063 receiveBufferSize_=count;
1064 delete[] receiveBuffer_;
1065 receiveBuffer_ =
new char[receiveBufferSize_];
1068 MPI_Recv(receiveBuffer_, count, MPI_PACKED, source, 345, remoteIndices_.
communicator(), &status);
1071 MPI_Unpack(receiveBuffer_, count, &bpos, &publish, 1, MPI_INT, remoteIndices_.
communicator());
1078 char sourceAttribute;
1081 MPI_Unpack(receiveBuffer_, count, &bpos, &global, 1, MPITraits<GlobalIndex>::getType(),
1083 MPI_Unpack(receiveBuffer_, count, &bpos, &sourceAttribute, 1, MPI_CHAR,
1085 MPI_Unpack(receiveBuffer_, count, &bpos, &pairs, 1, MPI_INT,
1090 SLList<std::pair<int,Attribute> > sourceAttributeList;
1091 sourceAttributeList.push_back(std::make_pair(source,
Attribute(sourceAttribute)));
1093 bool foundSelf =
false;
1098 for(;pairs>0;--pairs){
1102 MPI_Unpack(receiveBuffer_, count, &bpos, &process, 1, MPI_INT,
1105 MPI_Unpack(receiveBuffer_, count, &bpos, &attribute, 1, MPI_CHAR,
1116 IndexIterator pos = std::lower_bound(index, iEnd,
IndexPair(global));
1118 if(pos == iEnd || pos->global() != global){
1120 indexSet_.add(global,
1121 ParallelLocalIndex<Attribute>(numberer(global),
1122 myAttribute,
true));
1123 Dune::dvverb <<
"Adding "<<global<<
" "<<myAttribute<<std::endl;
1128 bool indexIsThere =
false;
1131 for(;pos->global()==global;++pos)
1132 if(pos->local().attribute() == myAttribute){
1133 Dune::dvverb<<
"found "<<global<<
" "<<myAttribute<<std::endl;
1134 indexIsThere =
true;
1139 indexSet_.add(global,
1140 ParallelLocalIndex<Attribute>(numberer(global),
1141 myAttribute,
true));
1142 Dune::dvverb <<
"Adding "<<global<<
" "<<myAttribute<<std::endl;
1146 sourceAttributeList.push_back(std::make_pair(process,
Attribute(attribute)));
1151 typedef typename SLList<std::pair<int,Attribute> >::const_iterator Iter;
1152 for(Iter i=sourceAttributeList.begin(), end=sourceAttributeList.end();
1154 insertIntoRemoteIndexList(i->first, std::make_pair(global, myAttribute),
1159 resetIteratorsMap();
1162 template<
typename T>
1163 void IndicesSyncer<T>::resetIteratorsMap(){
1166 typedef typename IteratorsMap::iterator Iterator;
1167 typedef typename RemoteIndices::RemoteIndexMap::iterator
1169 typedef typename GlobalIndicesMap::iterator GlobalIterator;
1170 typedef typename BoolMap::iterator BoolIterator;
1172 const RemoteIterator remoteEnd = remoteIndices_.remoteIndices_.end();
1173 Iterator iterators = iteratorsMap_.begin();
1174 GlobalIterator global = globalMap_.begin();
1175 BoolIterator added = oldMap_.begin();
1177 for(RemoteIterator remote = remoteIndices_.remoteIndices_.begin();
1178 remote != remoteEnd; ++remote, ++global, ++added, ++iterators){
1179 iterators->second.reset(*(remote->second.first), global->second, added->second);
1183 template<
typename T>
1184 bool IndicesSyncer<T>::checkReset(
const Iterators& iterators, RemoteIndexList& rList, GlobalIndexList& gList,
1187 if(get<0>(iterators.iterators_)!=rList.begin())
1189 if(get<1>(iterators.iterators_)!=gList.begin())
1191 if(get<2>(iterators.iterators_)!=bList.begin())
1197 template<
typename T>
1198 bool IndicesSyncer<T>::checkReset(){
1201 typedef typename IteratorsMap::iterator Iterator;
1202 typedef typename RemoteIndices::RemoteIndexMap::iterator
1204 typedef typename GlobalIndicesMap::iterator GlobalIterator;
1205 typedef typename BoolMap::iterator BoolIterator;
1207 const RemoteIterator remoteEnd = remoteIndices_.remoteIndices_.end();
1208 Iterator iterators = iteratorsMap_.begin();
1209 GlobalIterator global = globalMap_.begin();
1210 BoolIterator added = oldMap_.begin();
1213 for(RemoteIterator remote = remoteIndices_.remoteIndices_.begin();
1214 remote != remoteEnd; ++remote, ++global, ++added, ++iterators){
1215 if(!checkReset(iterators->second, *(remote->second.first), global->second,