00001
00002 #ifndef DUNE_REMOTEINDICES_HH
00003 #define DUNE_REMOTEINDICES_HH
00004
00005 #include"indexset.hh"
00006 #include"istlexception.hh"
00007 #include"plocalindex.hh"
00008 #include<dune/common/poolallocator.hh>
00009 #include<dune/common/sllist.hh>
00010 #include<dune/common/helpertemplates.hh>
00011 #include<map>
00012 #include<utility>
00013 #include<iostream>
00014
00015 #if HAVE_MPI
00016 #include"mpitraits.hh"
00017 #include"mpi.h"
00018
00019 namespace Dune{
00030
00031 template<typename TG, typename TA>
00032 class MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >
00033 {
00034 public:
00035 inline static MPI_Datatype getType();
00036 private:
00037 static MPI_Datatype type;
00038 };
00039
00040
00041 template<typename T>
00042 class RemoteIndices;
00043
00044 template<typename T1, typename T2>
00045 class RemoteIndex;
00046
00047 template<typename T>
00048 class IndicesSyncer;
00049
00050 template<typename T1, typename T2>
00051 std::ostream& operator<<(std::ostream& os, const RemoteIndex<T1,T2>& index);
00052
00053
00054 template<typename T, bool mode>
00055 class RemoteIndexListModifier;
00056
00060 template<typename T1, typename T2>
00061 class RemoteIndex
00062 {
00063 template<typename T>
00064 friend class IndicesSyncer;
00065
00066 template<typename T, typename A>
00067 friend void repairLocalIndexPointers(std::map<int,SLList<typename T::GlobalIndex,A> >&, RemoteIndices<T>&,
00068 const T&);
00069
00070 template<typename T, bool mode>
00071 friend class RemoteIndexListModifier;
00072
00073 public:
00078 typedef T1 GlobalIndex;
00087 typedef T2 Attribute;
00088
00092 typedef IndexPair<GlobalIndex,ParallelLocalIndex<Attribute> >
00093 PairType;
00094
00099 const Attribute attribute() const;
00100
00106 const PairType& localIndexPair() const;
00107
00111 RemoteIndex();
00112
00113
00119 RemoteIndex(const T2& attribute,
00120 const PairType* local);
00121
00122
00128 RemoteIndex(const T2& attribute);
00129
00130 private:
00132 const PairType* localIndex_;
00133
00135 char attribute_;
00136 };
00137
00138 template<class T>
00139 std::ostream& operator<<(std::ostream& os, const RemoteIndices<T>& indices);
00140
00141 template<typename T>
00142 class InterfaceBuilder;
00143
00144 template<class T>
00145 class CollectiveIterator;
00146
00147 template<class T>
00148 class IndicesSyncer;
00149
00153 template<class T>
00154 class RemoteIndices
00155 {
00156 friend class InterfaceBuilder<T>;
00157 friend class IndicesSyncer<T>;
00158 template<typename T1, typename A>
00159 friend void repairLocalIndexPointers(std::map<int,SLList<typename T1::GlobalIndex,A> >&,
00160 RemoteIndices<T1>&,
00161 const T1&);
00162 friend std::ostream& operator<<<>(std::ostream&, const RemoteIndices<T>&);
00163
00164 public:
00165
00169 typedef T ParallelIndexSet;
00170
00174 typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00175
00176
00180 typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00181
00185 typedef typename LocalIndex::Attribute Attribute;
00186
00190 typedef RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00191
00192
00196 typedef PoolAllocator<RemoteIndex,1> Allocator;
00197
00199 typedef SLList<RemoteIndex,Allocator>
00200 RemoteIndexList;
00201
00203 typedef std::map<int, std::pair<RemoteIndexList*,RemoteIndexList*> >
00204 RemoteIndexMap;
00205
00206 typedef typename RemoteIndexMap::const_iterator const_iterator;
00207
00218 inline RemoteIndices(const ParallelIndexSet& source, const ParallelIndexSet& destination,
00219 const MPI_Comm& comm);
00220
00221 RemoteIndices();
00222
00236 void setIndexSets(const ParallelIndexSet& source, const ParallelIndexSet& destination,
00237 const MPI_Comm& comm);
00241 ~RemoteIndices();
00242
00252 template<bool ignorePublic>
00253 void rebuild();
00254
00262 inline bool isSynced() const;
00263
00267 inline MPI_Comm communicator() const;
00268
00278 template<bool mode, bool send>
00279 inline RemoteIndexListModifier<T,mode> getModifier(int process);
00280
00285 inline const_iterator begin() const;
00286
00291 inline const_iterator end() const;
00292
00296 template<bool send>
00297 inline CollectiveIterator<T> iterator() const;
00298
00302 inline void free();
00303
00308 inline int neighbours() const;
00309
00310 private:
00312 const ParallelIndexSet* source_;
00313
00315 const ParallelIndexSet* target_;
00316
00318 MPI_Comm comm_;
00319
00321 const static int commTag_=333;
00322
00327 int sourceSeqNo_;
00328
00333 int destSeqNo_;
00334
00338 bool publicIgnored;
00339
00343 bool firstBuild;
00344
00346 typedef IndexPair<GlobalIndex, LocalIndex>
00347 PairType;
00348
00355 RemoteIndexMap remoteIndices_;
00356
00363 template<bool ignorePublic>
00364 inline void buildRemote();
00365
00371 inline int noPublic(const ParallelIndexSet& indexSet);
00372
00384 template<bool ignorePublic>
00385 inline void packEntries(PairType** myPairs, const ParallelIndexSet& indexSet,
00386 char* p_out, MPI_Datatype type, int bufferSize,
00387 int* position, int n);
00388
00402 inline void unpackIndices(RemoteIndexList& remote, int remoteEntries,
00403 PairType** local, int localEntries, char* p_in,
00404 MPI_Datatype type, int* positon, int bufferSize);
00405
00406 inline void unpackIndices(RemoteIndexList& send, RemoteIndexList& receive,
00407 int remoteEntries, PairType** localSource,
00408 int localSourceEntries, PairType** localDest,
00409 int localDestEntries, char* p_in,
00410 MPI_Datatype type, int* position, int bufferSize);
00411
00412 };
00413
00431 template<class T, bool mode>
00432 class RemoteIndexListModifier
00433 {
00434 public:
00435 class InvalidPosition : public ISTLError
00436 {};
00437
00438 enum {
00447 MODIFYINDEXSET=mode
00448 };
00449
00453 typedef T ParallelIndexSet;
00454
00458 typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00459
00463 typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00464
00468 typedef typename LocalIndex::Attribute Attribute;
00469
00473 typedef RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00474
00478 typedef PoolAllocator<RemoteIndex,1> Allocator;
00479
00481 typedef SLList<RemoteIndex,Allocator>
00482 RemoteIndexList;
00483
00487 typedef SLListModifyIterator<RemoteIndex,Allocator> ModifyIterator;
00488
00492 typedef typename RemoteIndexList::const_iterator ConstIterator;
00493
00507 void insert(const RemoteIndex& index) throw(InvalidPosition);
00508
00509
00524 void insert(const RemoteIndex& index, const GlobalIndex& global) throw(InvalidPosition);
00525
00533 bool remove(const GlobalIndex& global) throw(InvalidPosition);
00534
00540 RemoteIndexListModifier(const ParallelIndexSet& indexSet,
00541 RemoteIndexList& rList);
00542
00543
00544 RemoteIndexListModifier(const RemoteIndexListModifier<T,mode>&);
00545
00558 void repairLocalIndexPointers() throw(InvalidIndexSetState);
00559
00560 private:
00561
00562 typedef SLList<GlobalIndex,Allocator> GlobalList;
00563 typedef typename GlobalList::ModifyIterator GlobalModifyIterator;
00564 RemoteIndices<ParallelIndexSet>* remoteIndices_;
00565 RemoteIndexList* rList_;
00566 const ParallelIndexSet* indexSet_;
00567 GlobalList* glist_;
00568 ModifyIterator iter_;
00569 GlobalModifyIterator giter_;
00570 ConstIterator end_;
00571 bool first_;
00572 GlobalIndex last_;
00573 };
00574
00579 template<class T>
00580 class CollectiveIterator
00581 {
00582
00586 typedef T ParallelIndexSet;
00587
00591 typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00592
00596 typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00597
00601 typedef typename LocalIndex::Attribute Attribute;
00602
00604 typedef RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00605
00607 typedef PoolAllocator<RemoteIndex,1> Allocator;
00608
00610 typedef SLList<RemoteIndex,Allocator> RemoteIndexList;
00611
00613 typedef std::map<int,std::pair<typename RemoteIndexList::const_iterator,
00614 const typename RemoteIndexList::const_iterator> >
00615 Map;
00616
00617 public:
00618
00620 typedef std::map<int, std::pair<RemoteIndexList*,RemoteIndexList*> >
00621 RemoteIndexMap;
00622
00628 inline CollectiveIterator(const RemoteIndexMap& map_, bool send);
00629
00638 inline void advance(const GlobalIndex& global);
00639
00643 inline bool empty();
00644
00651 class iterator
00652 {
00653 public:
00654 typedef typename Map::iterator RealIterator;
00655 typedef typename Map::iterator ConstRealIterator;
00656
00657
00659 iterator(const RealIterator& iter, const ConstRealIterator& end, GlobalIndex& index)
00660 : iter_(iter), end_(end), index_(index)
00661 {
00662
00663 while(iter_!=end_ && iter_->second.first->localIndexPair().global()!=index_)
00664 ++iter_;
00665 }
00666
00668 iterator(const iterator& other)
00669 : iter_(other.iter_), end_(other.end_), index_(other.index_)
00670 { }
00671
00673 iterator& operator++()
00674 {
00675 ++iter_;
00676
00677 while(iter_!=end_ && iter_->second.first->localIndexPair().global()!=index_)
00678 ++iter_;
00679
00680 return *this;
00681 }
00682
00684 const RemoteIndex& operator*()const
00685 {
00686 return *(iter_->second.first);
00687 }
00688
00690 int process() const
00691 {
00692 return iter_->first;
00693 }
00694
00696 const RemoteIndex* operator->()const
00697 {
00698 return iter_->second.first.operator->();
00699 }
00700
00702 bool operator==(const iterator& other)
00703 {
00704 return other.iter_==iter_;
00705 }
00706
00708 bool operator!=(const iterator& other)
00709 {
00710 return other.iter_!=iter_;
00711 }
00712
00713 private:
00714 iterator();
00715
00716 RealIterator iter_;
00717 RealIterator end_;
00718 GlobalIndex index_;
00719 };
00720
00721 iterator begin();
00722
00723 iterator end();
00724
00725 private:
00726
00727 Map map_;
00728 GlobalIndex index_;
00729 };
00730
00731 template<typename TG, typename TA>
00732 MPI_Datatype MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >::getType()
00733 {
00734 if(type==MPI_DATATYPE_NULL){
00735 int length[4];
00736 MPI_Aint disp[4];
00737 MPI_Datatype types[4] = {MPI_LB, MPITraits<TG>::getType(),
00738 MPITraits<ParallelLocalIndex<TA> >::getType(), MPI_UB};
00739 IndexPair<TG,ParallelLocalIndex<TA> > rep[2];
00740 length[0]=length[1]=length[2]=length[3]=1;
00741 MPI_Address(rep, disp);
00742 MPI_Address(&(rep[0].global_), disp+1);
00743 MPI_Address(&(rep[0].local_), disp+2);
00744 MPI_Address(rep+1, disp+3);
00745 for(int i=3; i >= 0; --i)
00746 disp[i] -= disp[0];
00747 MPI_Type_struct(4, length, disp, types, &type);
00748 MPI_Type_commit(&type);
00749 }
00750 return type;
00751 }
00752
00753 template<typename TG, typename TA>
00754 MPI_Datatype MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >::type=MPI_DATATYPE_NULL;
00755
00756 template<typename T1, typename T2>
00757 RemoteIndex<T1,T2>::RemoteIndex(const T2& attribute, const PairType* local)
00758 : localIndex_(local), attribute_(attribute)
00759 {}
00760
00761 template<typename T1, typename T2>
00762 RemoteIndex<T1,T2>::RemoteIndex(const T2& attribute)
00763 : localIndex_(0), attribute_(attribute)
00764 {}
00765
00766 template<typename T1, typename T2>
00767 RemoteIndex<T1,T2>::RemoteIndex()
00768 : localIndex_(0), attribute_()
00769 {}
00770
00771 template<typename T1, typename T2>
00772 inline const T2 RemoteIndex<T1,T2>::attribute() const
00773 {
00774 return T2(attribute_);
00775 }
00776
00777 template<typename T1, typename T2>
00778 inline const IndexPair<T1,ParallelLocalIndex<T2> >& RemoteIndex<T1,T2>::localIndexPair() const
00779 {
00780 return *localIndex_;
00781 }
00782
00783 template<typename T>
00784 inline RemoteIndices<T>::RemoteIndices(const ParallelIndexSet& source,
00785 const ParallelIndexSet& destination,
00786 const MPI_Comm& comm)
00787 : source_(&source), target_(&destination), comm_(comm),
00788 sourceSeqNo_(-1), destSeqNo_(-1), publicIgnored(false), firstBuild(true)
00789 {}
00790
00791 template<typename T>
00792 RemoteIndices<T>::RemoteIndices()
00793 :source_(0), target_(0), sourceSeqNo_(-1),
00794 destSeqNo_(-1), publicIgnored(false), firstBuild(true)
00795 {}
00796
00797 template<class T>
00798 void RemoteIndices<T>::setIndexSets(const ParallelIndexSet& source,
00799 const ParallelIndexSet& destination,
00800 const MPI_Comm& comm)
00801 {
00802 free();
00803 source_ = &source;
00804 target_ = &destination;
00805 comm_ = comm;
00806 firstBuild = true;
00807 }
00808
00809 template<typename T>
00810 RemoteIndices<T>::~RemoteIndices()
00811 {
00812 free();
00813 }
00814
00815 template<typename T>
00816 template<bool ignorePublic>
00817 inline void RemoteIndices<T>::packEntries(IndexPair<GlobalIndex,LocalIndex>** pairs,
00818 const ParallelIndexSet& indexSet,
00819 char* p_out, MPI_Datatype type,
00820 int bufferSize,
00821 int *position, int n)
00822 {
00823
00824 typedef typename ParallelIndexSet::const_iterator const_iterator;
00825 typedef IndexPair<GlobalIndex,LocalIndex> PairType;
00826 const const_iterator end = indexSet.end();
00827
00828
00829 int i=0;
00830 for(const_iterator index = indexSet.begin(); index != end; ++index)
00831 if(ignorePublic || index->local().isPublic()){
00832
00833 MPI_Pack(const_cast<PairType*>(&(*index)), 1,
00834 type,
00835 p_out, bufferSize, position, comm_);
00836 pairs[i++] = const_cast<PairType*>(&(*index));
00837
00838 }
00839 assert(i==n);
00840 }
00841
00842 template<typename T>
00843 inline int RemoteIndices<T>::noPublic(const ParallelIndexSet& indexSet)
00844 {
00845 typedef typename ParallelIndexSet::const_iterator const_iterator;
00846
00847 int noPublic=0;
00848
00849 const const_iterator end=indexSet.end();
00850 for(const_iterator index=indexSet.begin(); index!=end; ++index)
00851 if(index->local().isPublic())
00852 noPublic++;
00853
00854 return noPublic;
00855
00856 }
00857
00858
00859 template<typename T>
00860 template<bool ignorePublic>
00861 inline void RemoteIndices<T>::buildRemote()
00862 {
00863
00864
00865 int rank, procs;
00866 MPI_Comm_rank(comm_, &rank);
00867 MPI_Comm_size(comm_, &procs);
00868
00869
00870
00871 int sourcePublish, destPublish;
00872
00873
00874 char sendTwo = (source_ != target_);
00875
00876 sourcePublish = (ignorePublic)? source_->size() : noPublic(*source_);
00877
00878 if(sendTwo)
00879 destPublish = (ignorePublic)? target_->size() : noPublic(*target_);
00880 else
00881
00882 destPublish = 0;
00883
00884 int maxPublish, publish=sourcePublish+destPublish;
00885
00886
00887 MPI_Allreduce(&publish, &maxPublish, 1, MPI_INT, MPI_MAX, comm_);
00888
00889
00890 typedef IndexPair<GlobalIndex,LocalIndex> PairType;
00891
00892 PairType** destPairs;
00893 PairType** sourcePairs = new PairType*[sourcePublish];
00894
00895 if(sendTwo)
00896 destPairs = new PairType*[destPublish];
00897 else
00898 destPairs=sourcePairs;
00899
00900 char** buffer = new char*[2];
00901 int bufferSize;
00902 int position=0;
00903 int intSize;
00904 int charSize;
00905
00906
00907 MPI_Datatype type = MPITraits<PairType>::getType();
00908
00909 MPI_Pack_size(maxPublish, type, comm_,
00910 &bufferSize);
00911 MPI_Pack_size(1, MPI_INT, comm_,
00912 &intSize);
00913 MPI_Pack_size(1, MPI_CHAR, comm_,
00914 &charSize);
00915
00916
00917
00918
00919 bufferSize += 2 * intSize + charSize;
00920
00921 buffer[0] = new char[bufferSize];
00922 buffer[1] = new char[bufferSize];
00923
00924
00925 for(int proc=0; proc<procs; proc++){
00926
00927 char* p_out = buffer[1-(proc%2)];
00928 char* p_in = buffer[proc%2];
00929
00930 if(proc==0 && (procs>0 || sendTwo)){
00931 if(!sendTwo){
00932
00933 p_out=buffer[0];
00934 assert(p_in==p_out);
00935 }
00936
00937
00938 MPI_Pack(&sendTwo, 1, MPI_CHAR, p_out, bufferSize, &position,
00939 comm_);
00940
00941
00942 MPI_Pack(&sourcePublish, 1, MPI_INT, p_out, bufferSize, &position,
00943 comm_);
00944 MPI_Pack(&destPublish, 1, MPI_INT, p_out, bufferSize, &position,
00945 comm_);
00946
00947
00948
00949
00950 packEntries<ignorePublic>(sourcePairs, *source_, p_out, type,
00951 bufferSize, &position, sourcePublish);
00952
00953 if(sendTwo)
00954 packEntries<ignorePublic>(destPairs, *target_, p_out, type,
00955 bufferSize, &position, destPublish);
00956
00957
00958
00959
00960 }
00961
00962 MPI_Status status;
00963 if(proc==0){
00964 if(sendTwo)
00965
00966 MPI_Sendrecv(p_out, position, MPI_PACKED, rank, commTag_, p_in, bufferSize, MPI_PACKED, rank, commTag_,
00967 comm_, &status);
00968 else
00969 continue;
00970 }else if(rank%2==0){
00971 MPI_Ssend(p_out, position, MPI_PACKED, (rank+1)%procs,
00972 commTag_, comm_);
00973 MPI_Recv(p_in, bufferSize, MPI_PACKED, (rank+procs-1)%procs,
00974 commTag_, comm_, &status);
00975 }else{
00976 MPI_Recv(p_in, bufferSize, MPI_PACKED, (rank+procs-1)%procs,
00977 commTag_, comm_, &status);
00978 MPI_Ssend(p_out, position, MPI_PACKED, (rank+1)%procs,
00979 commTag_, comm_);
00980 }
00981
00982
00983 int noRemoteSource=-1, noRemoteDest=-1;
00984 char twoIndexSets=0;
00985 position=0;
00986
00987 MPI_Unpack(p_in, bufferSize, &position, &twoIndexSets, 1, MPI_CHAR, comm_);
00988
00989 MPI_Unpack(p_in, bufferSize, &position, &noRemoteSource, 1, MPI_INT, comm_);
00990
00991 MPI_Unpack(p_in, bufferSize, &position, &noRemoteDest, 1, MPI_INT, comm_);
00992
00993
00994 int remoteProc = (rank+procs-proc)%procs;
00995
00996
00997
00998
00999
01000
01001
01002
01003
01004 RemoteIndexList* receive= new RemoteIndexList();
01005
01006 RemoteIndexList* send=0;
01007
01008 if(!twoIndexSets){
01009 if(sendTwo){
01010 send = new RemoteIndexList();
01011
01012 unpackIndices(*send, *receive, noRemoteSource, sourcePairs, sourcePublish,
01013 destPairs, destPublish, p_in, type, &position, bufferSize);
01014 }else{
01015
01016 unpackIndices(*receive, noRemoteSource, sourcePairs, sourcePublish,
01017 p_in, type, &position, bufferSize);
01018 send=receive;
01019 }
01020 }else{
01021
01022 if(sendTwo){
01023 unpackIndices(*receive, noRemoteSource, destPairs, destPublish,
01024 p_in, type, &position, bufferSize);
01025 }else{
01026 unpackIndices(*receive, noRemoteSource, sourcePairs, sourcePublish,
01027 p_in, type, &position, bufferSize);
01028 }
01029
01030 send = new RemoteIndexList();
01031 unpackIndices(*send, noRemoteDest, sourcePairs, sourcePublish,
01032 p_in, type, &position, bufferSize);
01033 }
01034
01035 if(receive->empty() && send->empty()){
01036 if(send==receive){
01037 delete send;
01038 }else{
01039 delete send;
01040 delete receive;
01041 }
01042 }else{
01043 remoteIndices_.insert(std::make_pair(remoteProc,
01044 std::make_pair(send,receive)));
01045 }
01046 }
01047 MPI_Barrier(comm_);
01048
01049
01050 if(destPairs!=sourcePairs)
01051 delete[] destPairs;
01052
01053 delete[] sourcePairs;
01054 delete[] buffer[0];
01055 delete[] buffer[1];
01056 delete[] buffer;
01057 }
01058
01059 template<typename T>
01060 inline void RemoteIndices<T>::unpackIndices(RemoteIndexList& remote,
01061 int remoteEntries,
01062 PairType** local,
01063 int localEntries,
01064 char* p_in,
01065 MPI_Datatype type,
01066 int* position,
01067 int bufferSize)
01068 {
01069 if(remoteEntries==0)
01070 return;
01071
01072 PairType index;
01073 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01074 type, comm_);
01075
01076 int n_in=0, localIndex=0;
01077
01078
01079 while(localIndex<localEntries){
01080 if(local[localIndex]->global()==index.global()){
01081 remote.push_back(RemoteIndex(index.local().attribute(),
01082 local[localIndex]));
01083 localIndex++;
01084
01085 if((++n_in) < remoteEntries){
01086 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01087 type, comm_);
01088 }else{
01089
01090 break;
01091 }
01092 }else if (local[localIndex]->global()<index.global()){
01093 ++localIndex;
01094 }else{
01095
01096 if((++n_in) < remoteEntries){
01097 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01098 type, comm_);
01099 }else
01100
01101 break;
01102 }
01103 }
01104
01105
01106 while(++n_in < remoteEntries)
01107 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01108 type, comm_);
01109 }
01110
01111
01112 template<typename T>
01113 inline void RemoteIndices<T>::unpackIndices(RemoteIndexList& send,
01114 RemoteIndexList& receive,
01115 int remoteEntries,
01116 PairType** localSource,
01117 int localSourceEntries,
01118 PairType** localDest,
01119 int localDestEntries,
01120 char* p_in,
01121 MPI_Datatype type,
01122 int* position,
01123 int bufferSize)
01124 {
01125 int n_in=0, sourceIndex=0, destIndex=0;
01126
01127
01128 while(n_in<remoteEntries && (sourceIndex<localSourceEntries || destIndex<localDestEntries)){
01129
01130 PairType index;
01131 MPI_Unpack(p_in, bufferSize, position, &index, 1,
01132 type, comm_);
01133 n_in++;
01134
01135
01136 while(sourceIndex<localSourceEntries && localSource[sourceIndex]->global()<index.global())
01137 sourceIndex++;
01138
01139 while(destIndex<localDestEntries && localDest[destIndex]->global()<index.global())
01140 destIndex++;
01141
01142
01143 if(sourceIndex<localSourceEntries && localSource[sourceIndex]->global()==index.global())
01144 send.push_back(RemoteIndex(index.local().attribute(),
01145 localSource[sourceIndex]));
01146
01147 if(destIndex < localDestEntries && localDest[destIndex]->global() == index.global())
01148 receive.push_back(RemoteIndex(index.local().attribute(),
01149 localDest[sourceIndex]));
01150 }
01151
01152 }
01153
01154 template<typename T>
01155 inline void RemoteIndices<T>::free()
01156 {
01157 typedef typename RemoteIndexMap::iterator Iterator;
01158 Iterator lend = remoteIndices_.end();
01159 for(Iterator lists=remoteIndices_.begin(); lists != lend; ++lists){
01160 if(lists->second.first==lists->second.second){
01161
01162 delete lists->second.first;
01163 }else{
01164 delete lists->second.first;
01165 delete lists->second.second;
01166 }
01167 }
01168 remoteIndices_.clear();
01169 }
01170
01171 template<typename T>
01172 inline int RemoteIndices<T>::neighbours() const
01173 {
01174 return remoteIndices_.size();
01175 }
01176
01177 template<typename T>
01178 template<bool ignorePublic>
01179 inline void RemoteIndices<T>::rebuild()
01180 {
01181
01182 if(firstBuild ||
01183 ignorePublic!=publicIgnored || !
01184 isSynced()){
01185 free();
01186
01187 buildRemote<ignorePublic>();
01188
01189 sourceSeqNo_ = source_->seqNo();
01190 destSeqNo_ = target_->seqNo();
01191 firstBuild=false;
01192 publicIgnored=ignorePublic;
01193 }
01194
01195
01196 }
01197
01198 template<typename T>
01199 inline bool RemoteIndices<T>::isSynced() const
01200 {
01201 return sourceSeqNo_==source_->seqNo() && destSeqNo_ ==target_->seqNo();
01202 }
01203
01204 template<typename T>
01205 template<bool mode, bool send>
01206 RemoteIndexListModifier<T,mode> RemoteIndices<T>::getModifier(int process)
01207 {
01208 typename RemoteIndexMap::iterator found = remoteIndices_.find(process);
01209
01210 if(found == remoteIndices_.end())
01211 if(source_ != target_)
01212 remoteIndices_.insert(std::make_pair(process,
01213 std::make_pair(new RemoteIndexList(),
01214 new RemoteIndexList())));
01215 else{
01216 RemoteIndexList* rlist = new RemoteIndexList();
01217 remoteIndices_.insert(std::make_pair(process,
01218 std::make_pair(rlist, rlist)));
01219
01220 found = remoteIndices_.find(process);
01221 }
01222
01223 firstBuild = false;
01224
01225 if(send)
01226 return RemoteIndexListModifier<T,mode>(*source_, *(found->second.first));
01227 else
01228 return RemoteIndexListModifier<T,mode>(*target_, *(found->second.second));
01229 }
01230
01231 template<typename T>
01232 inline typename std::map<int, std::pair<SLList<typename RemoteIndices<T>::RemoteIndex,
01233 PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*,
01234 SLList<typename RemoteIndices<T>::RemoteIndex,
01235 PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*> >::const_iterator
01236 RemoteIndices<T>::begin() const
01237 {
01238 return remoteIndices_.begin();
01239 }
01240
01241 template<typename T>
01242 inline typename std::map<int, std::pair<SLList<typename RemoteIndices<T>::RemoteIndex,
01243 PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*,
01244 SLList<typename RemoteIndices<T>::RemoteIndex,
01245 PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*> >::const_iterator
01246 RemoteIndices<T>::end() const
01247 {
01248 return remoteIndices_.end();
01249 }
01250
01251 template<class T, bool mode>
01252 RemoteIndexListModifier<T,mode>::RemoteIndexListModifier(const ParallelIndexSet& indexSet,
01253 RemoteIndexList& rList)
01254 : rList_(&rList), indexSet_(&indexSet), glist_(new GlobalList()), iter_(rList.beginModify()), end_(rList.end()), first_(true)
01255 {
01256 if(MODIFYINDEXSET){
01257 assert(indexSet_);
01258 for(ConstIterator iter=iter_; iter != end_; ++iter)
01259 glist_->push_back(iter->localIndexPair().global());
01260 giter_ = glist_->beginModify();
01261 }
01262 }
01263
01264 template<typename T, bool mode>
01265 RemoteIndexListModifier<T,mode>::RemoteIndexListModifier(const RemoteIndexListModifier<T,mode>& other)
01266 : rList_(other.rList_), indexSet_(other.indexSet_),
01267 glist_(other.glist_), iter_(other.iter_), giter_(other.giter_), end_(other.end_),
01268 first_(other.first_), last_(other.last_)
01269 {}
01270
01271 template<typename T, bool mode>
01272 inline void RemoteIndexListModifier<T,mode>::repairLocalIndexPointers() throw(InvalidIndexSetState)
01273 {
01274 IsTrue<mode>::yes();
01275
01276 if(MODIFYINDEXSET){
01277
01278 #ifdef DUNE_ISTL_WITH_CHECKING
01279 if(indexSet_->state()!=GROUND)
01280 DUNE_THROW(InvalidIndexSetState, "Index has to be in ground mode for repairing pointers to indices");
01281 #endif
01282 typedef typename ParallelIndexSet::const_iterator IndexIterator;
01283 typedef typename GlobalList::const_iterator GlobalIterator;
01284 typedef typename RemoteIndexList::iterator Iterator;
01285 GlobalIterator giter = glist_->begin();
01286 IndexIterator index = indexSet_->begin();
01287
01288 for(Iterator iter=rList_->begin(); iter != end_; ++iter){
01289 while(index->global()<*giter){
01290 ++index;
01291 #ifdef DUNE_ISTL_WITH_CHECKING
01292 if(index == indexSet_.end())
01293 DUNE_THROW(ISTLError, "No such global index in set!");
01294 #endif
01295 }
01296
01297 #ifdef DUNE_ISTL_WITH_CHECKING
01298 if(index->global() != *giter)
01299 DUNE_THROW(ISTLError, "No such global index in set!");
01300 #endif
01301 iter->localIndex_ = &(*index);
01302 }
01303 }
01304 }
01305
01306 template<typename T, bool mode>
01307 inline void RemoteIndexListModifier<T,mode>::insert(const RemoteIndex& index) throw(InvalidPosition)
01308 {
01309 IsTrue<mode>::no();
01310
01311 #ifdef DUNE_ISTL_WITH_CHECKING
01312 if(!first_ && index.localIndexPair().global()<last_)
01313 DUNE_THROW(InvalidPosition, "Modifcation of remote indices have to occur with ascending global index.");
01314 #endif
01315
01316 while(iter_ != end_ && iter_->localIndexPair().global() < index.localIndexPair().global()){
01317 ++iter_;
01318 }
01319
01320
01321 assert(iter_==end_ || iter_->localIndexPair().global() != index.localIndexPair().global());
01322 iter_.insert(index);
01323 last_ = index.localIndexPair().global();
01324 first_ = false;
01325 }
01326
01327 template<typename T, bool mode>
01328 inline void RemoteIndexListModifier<T,mode>::insert(const RemoteIndex& index, const GlobalIndex& global) throw(InvalidPosition)
01329 {
01330 typename IsTrue<mode>::yes();
01331 #ifdef DUNE_ISTL_WITH_CHECKING
01332 if(!first_ && global<last_)
01333 DUNE_THROW(InvalidPosition, "Modification of remote indices have to occur with ascending global index.");
01334 #endif
01335
01336 while(iter_ != end_ && *giter_ < global){
01337 ++giter_;
01338 ++iter_;
01339 }
01340
01341
01342 assert(iter_->localIndexPair().global() != global);
01343 iter_.insert(index);
01344 giter_.insert(global);
01345
01346 last_ = global;
01347 first_ = false;
01348 }
01349
01350 template<typename T, bool mode>
01351 bool RemoteIndexListModifier<T,mode>::remove(const GlobalIndex& global) throw(InvalidPosition)
01352 {
01353 #ifdef DUNE_ISTL_WITH_CHECKING
01354 if(!first_ && global<last_)
01355 DUNE_THROW(InvalidPosition, "Modifcation of remote indices have to occur with ascending global index.");
01356 #endif
01357
01358 bool found= false;
01359
01360 if(MODIFYINDEXSET){
01361
01362 while(iter_!=end_ && *giter_< global){
01363 ++giter_;
01364 ++iter_;
01365 }
01366 if(*giter_ == global){
01367 giter_.remove();
01368 iter_.remove();
01369 found=true;
01370 }
01371 }else{
01372 while(iter_!=end_ && iter_->localIndexPair().global() < global)
01373 ++iter_;
01374
01375 if(iter_->localIndexPair().global()==global){
01376 iter_.remove();
01377 found = true;
01378 }
01379 }
01380
01381 last_ = global;
01382 first_ = false;
01383 return found;
01384 }
01385
01386 template<typename T>
01387 template<bool send>
01388 inline CollectiveIterator<T> RemoteIndices<T>::iterator() const
01389 {
01390 return CollectiveIterator<T>(remoteIndices_, send);
01391 }
01392
01393 template<typename T>
01394 inline MPI_Comm RemoteIndices<T>::communicator() const
01395 {
01396 return comm_;
01397
01398 }
01399
01400 template<typename T>
01401 CollectiveIterator<T>::CollectiveIterator(const RemoteIndexMap& pmap, bool send)
01402 {
01403 typedef typename RemoteIndexMap::const_iterator const_iterator;
01404 typedef typename RemoteIndexMap::iterator iterator;
01405
01406 const const_iterator end=pmap.end();
01407 for(const_iterator process=pmap.begin(); process != end; ++process){
01408 const RemoteIndexList* list = send? process->second.first : process->second.second;
01409 typedef typename RemoteIndexList::const_iterator iterator;
01410 map_.insert(std::make_pair(process->first,
01411 std::pair<iterator, const iterator>(list->begin(), list->end())));
01412 }
01413 }
01414
01415 template<typename T>
01416 inline void CollectiveIterator<T>::advance(const GlobalIndex& index)
01417 {
01418 typedef typename Map::iterator iterator;
01419 typedef typename Map::const_iterator const_iterator;
01420 const const_iterator end = map_.end();
01421
01422 for(iterator iter = map_.begin(); iter != end;){
01423
01424 typename RemoteIndexList::const_iterator current = iter->second.first;
01425 typename RemoteIndexList::const_iterator rend = iter->second.second;
01426 RemoteIndex remoteIndex;
01427 if(current != rend)
01428 remoteIndex = *current;
01429
01430 while(iter->second.first!=iter->second.second && iter->second.first->localIndexPair().global()<index)
01431 ++(iter->second.first);
01432
01433
01434 if(iter->second.first == iter->second.second)
01435 map_.erase(iter++);
01436 else{
01437 ++iter;
01438 }
01439 }
01440 index_=index;
01441 }
01442
01443 template<typename T>
01444 inline bool CollectiveIterator<T>::empty()
01445 {
01446 return map_.empty();
01447 }
01448
01449 template<typename T>
01450 inline typename CollectiveIterator<T>::iterator
01451 CollectiveIterator<T>::begin()
01452 {
01453 return iterator(map_.begin(), map_.end(), index_);
01454 }
01455
01456 template<typename T>
01457 inline typename CollectiveIterator<T>::iterator
01458 CollectiveIterator<T>::end()
01459 {
01460 return iterator(map_.end(), map_.end(), index_);
01461 }
01462
01463 template<typename TG, typename TA>
01464 inline std::ostream& operator<<(std::ostream& os, const RemoteIndex<TG,TA>& index)
01465 {
01466 os<<"[global="<<index.localIndexPair().global()<<",attribute="<<index.attribute()<<"]";
01467 return os;
01468 }
01469
01470 template<typename T>
01471 inline std::ostream& operator<<(std::ostream& os, const RemoteIndices<T>& indices)
01472 {
01473 int rank;
01474 MPI_Comm_rank(indices.comm_, &rank);
01475
01476 typedef typename RemoteIndices<T>::RemoteIndexList RList;
01477 typedef typename std::map<int,std::pair<RList*,RList*> >::const_iterator const_iterator;
01478
01479 const const_iterator rend = indices.remoteIndices_.end();
01480
01481 for(const_iterator rindex = indices.remoteIndices_.begin(); rindex!=rend; ++rindex){
01482 os<<rank<<": Prozess "<<rindex->first<<":";
01483
01484 if(!rindex->second.first->empty()){
01485 os<<" send:";
01486
01487 const typename RList::const_iterator send= rindex->second.first->end();
01488
01489 for(typename RList::const_iterator index = rindex->second.first->begin();
01490 index != send; ++index)
01491 os<<*index<<" ";
01492 os<<std::endl;
01493 }
01494 if(!rindex->second.second->empty()){
01495 os<<rank<<": Prozess "<<rindex->first<<": "<<"receive: ";
01496
01497 const typename RList::const_iterator rend= rindex->second.second->end();
01498
01499 for(typename RList::const_iterator index = rindex->second.second->begin();
01500 index != rend; ++index)
01501 os<<*index<<" ";
01502 }
01503 os<<std::endl<<std::flush;
01504 }
01505 return os;
01506 }
01508 }
01509
01510 #endif
01511 #endif