00001
00002 #ifndef DUNE_COMMUNICATOR
00003 #define DUNE_COMMUNICATOR
00004
00005 #include"remoteindices.hh"
00006 #include"interface.hh"
00007 #include<dune/common/exceptions.hh>
00008 #include<dune/common/typetraits.hh>
00009 namespace Dune
00010 {
00090 struct SizeOne
00091 {};
00092
00098 struct VariableSize
00099 {};
00100
00101
00107 template<class V>
00108 struct CommPolicy
00109 {
00121 typedef V Type;
00122
00128 typedef typename V::value_type IndexedType;
00129
00134 typedef SizeOne IndexedTypeFlag;
00135
00144 static const void* getAddress(const V& v, int index);
00145
00151 static int getSize(const V&, int index);
00152 };
00153
00154 template<class K, int n> class FieldVector;
00155
00156 template<class B, class A> class BlockVector;
00157
00158 template<class K, class A, int n>
00159 struct CommPolicy<BlockVector<FieldVector<K, n>, A> >
00160 {
00161 typedef BlockVector<FieldVector<K, n>, A> Type;
00162
00163 typedef typename Type::block_type IndexedType;
00164
00165 typedef SizeOne IndexedTypeFlag;
00166
00167 static const void* getAddress(const Type& v, int i);
00168
00169 static int getSize(const Type& v, int i);
00170 };
00171
00175 class CommunicationError : public IOError
00176 {};
00177
00181 template<class T>
00182 struct CopyGatherScatter
00183 {
00184 typedef typename CommPolicy<T>::IndexedType IndexedType;
00185
00186 const IndexedType& gather(const T& vec, int i);
00187
00188 void scatter(T& vec, const IndexedType& v, int i);
00189
00190 };
00191
00203 template<typename T>
00204 class DatatypeCommunicator : public InterfaceBuilder<T>
00205 {
00206 public:
00207
00211 typedef T ParallelIndexSet;
00212
00216 typedef RemoteIndices<ParallelIndexSet> RemoteIndices;
00217
00221 typedef typename RemoteIndices::GlobalIndex GlobalIndex;
00222
00226 typedef typename RemoteIndices::Attribute Attribute;
00227
00231 typedef typename RemoteIndices::LocalIndex LocalIndex;
00232
00236 DatatypeCommunicator();
00237
00241 ~DatatypeCommunicator();
00242
00269 template<class T1, class T2, class V>
00270 void build(const RemoteIndices& remoteIndices, const T1& sourceFlags, V& sendData, const T2& destFlags, V& receiveData);
00271
00275 void forward();
00276
00280 void backward();
00281
00285 void free();
00286 private:
00287 enum {
00291 commTag_ = 234
00292 };
00293
00297 const RemoteIndices* remoteIndices_;
00298
00299 typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >
00300 MessageTypeMap;
00301
00305 MessageTypeMap messageTypes;
00306
00310 void* data_;
00311
00312 MPI_Request* requests_[2];
00313
00317 bool created_;
00318
00322 template<class V, bool FORWARD>
00323 void createRequests(V& sendData, V& receiveData);
00324
00328 template<class T1, class T2, class V, bool send>
00329 void createDataTypes(const T1& source, const T2& destination, V& data);
00330
00334 void sendRecv(MPI_Request* req);
00335
00339 struct IndexedTypeInformation
00340 {
00346 void build(int i)
00347 {
00348 length = new int[i];
00349 displ = new MPI_Aint[i];
00350 size = i;
00351 }
00352
00356 void free()
00357 {
00358 delete[] length;
00359 delete[] displ;
00360 }
00362 int* length;
00364 MPI_Aint* displ;
00370 int elements;
00374 int size;
00375 };
00376
00382 template<class V>
00383 struct MPIDatatypeInformation
00384 {
00389 MPIDatatypeInformation(const V& data): data_(data)
00390 {}
00391
00397 void reserve(int proc, int size)
00398 {
00399 information_[proc].build(size);
00400 }
00407 void add(int proc, int local)
00408 {
00409 IndexedTypeInformation& info=information_[proc];
00410 assert(info.elements<info.size);
00411 MPI_Address( const_cast<void*>(CommPolicy<V>::getAddress(data_, local)),
00412 info.displ+info.elements);
00413 info.length[info.elements]=CommPolicy<V>::getSize(data_, local);
00414 info.elements++;
00415 }
00416
00421 std::map<int,IndexedTypeInformation> information_;
00425 const V& data_;
00426
00427 };
00428
00429 };
00430
00440 template<typename T>
00441 class BufferedCommunicator
00442 {
00443 public:
00447 typedef Interface<T> Interface;
00448
00452 typedef typename Interface::GlobalIndex GlobalIndex;
00453
00457 typedef typename Interface::Attribute Attribute;
00458
00462 BufferedCommunicator();
00463
00470 template<class Data>
00471 typename EnableIf<SameType<SizeOne,typename CommPolicy<Data>::IndexedTypeFlag>::value, void>::Type
00472 build(const Interface& interface);
00473
00481 template<class Data>
00482 void build(const Data& source, const Data& target, const Interface& interface);
00483
00512 template<class GatherScatter, class Data>
00513 void forward(const Data& source, Data& dest);
00514
00543 template<class GatherScatter, class Data>
00544 void backward(Data& source, const Data& dest);
00545
00571 template<class GatherScatter, class Data>
00572 void forward(Data& data);
00573
00599 template<class GatherScatter, class Data>
00600 void backward(Data& data);
00601
00605 void free();
00606
00610 ~BufferedCommunicator();
00611
00612 private:
00613
00617 template<class Data, typename IndexedTypeFlag>
00618 struct MessageSizeCalculator
00619 {};
00620
00625 template<class Data>
00626 struct MessageSizeCalculator<Data,SizeOne>
00627 {
00634 inline int operator()(const InterfaceInformation& info) const;
00643 inline int operator()(const Data& data, const InterfaceInformation& info) const;
00644 };
00645
00650 template<class Data>
00651 struct MessageSizeCalculator<Data,VariableSize>
00652 {
00661 inline int operator()(const Data& data, const InterfaceInformation& info) const;
00662 };
00663
00667 template<class Data, class GatherScatter, bool send, typename IndexedTypeFlag>
00668 struct MessageGatherer
00669 {};
00670
00675 template<class Data, class GatherScatter, bool send>
00676 struct MessageGatherer<Data,GatherScatter,send,SizeOne>
00677 {
00679 typedef typename CommPolicy<Data>::IndexedType Type;
00680
00685 typedef GatherScatter Gatherer;
00686
00687 enum{
00693 forward=send
00694 };
00695
00703 inline void operator()(const Interface& interface, const Data& data, Type* buffer, size_t bufferSize) const;
00704 };
00705
00710 template<class Data, class GatherScatter, bool send>
00711 struct MessageGatherer<Data,GatherScatter,send,VariableSize>
00712 {
00714 typedef typename CommPolicy<Data>::IndexedType Type;
00715
00720 typedef GatherScatter Gatherer;
00721
00722 enum{
00728 forward=send
00729 };
00730
00738 inline void operator()(const Interface& interface, const Data& data, Type* buffer, size_t bufferSize) const;
00739 };
00740
00744 template<class Data, class GatherScatter, bool send, typename IndexedTypeFlag>
00745 struct MessageScatterer
00746 {};
00747
00752 template<class Data, class GatherScatter, bool send>
00753 struct MessageScatterer<Data,GatherScatter,send,SizeOne>
00754 {
00756 typedef typename CommPolicy<Data>::IndexedType Type;
00757
00762 typedef GatherScatter Scatterer;
00763
00764 enum{
00770 forward=send
00771 };
00772
00780 inline void operator()(const Interface& interface, Data& data, Type* buffer, const int& proc) const;
00781 };
00786 template<class Data, class GatherScatter, bool send>
00787 struct MessageScatterer<Data,GatherScatter,send,VariableSize>
00788 {
00790 typedef typename CommPolicy<Data>::IndexedType Type;
00791
00796 typedef GatherScatter Scatterer;
00797
00798 enum{
00804 forward=send
00805 };
00806
00814 inline void operator()(const Interface& interface, Data& data, Type* buffer, const int& proc) const;
00815 };
00816
00820 struct MessageInformation
00821 {
00823 MessageInformation()
00824 : start_(0), size_(0)
00825 {}
00826
00834 MessageInformation(size_t start, size_t size)
00835 :start_(start), size_(size)
00836 {}
00840 size_t start_;
00844 size_t size_;
00845 };
00846
00853 typedef std::map<int,std::pair<MessageInformation,MessageInformation> >
00854 InformationMap;
00858 InformationMap messageInformation_;
00862 char* buffers_[2];
00866 size_t bufferSize_[2];
00867
00868 enum{
00872 commTag_
00873 };
00874
00878 const Interface* interface_;
00879
00883 template<class GatherScatter, bool FORWARD, class Data>
00884 void sendRecv(const Data& source, Data& target);
00885
00886 };
00887
00888 template<class V>
00889 inline const void* CommPolicy<V>::getAddress(const V& v, int index)
00890 {
00891 return &(v[index]);
00892 }
00893
00894 template<class V>
00895 inline int CommPolicy<V>::getSize(const V& v, int index)
00896 {
00897 return 1;
00898 }
00899
00900 template<class K, class A, int n>
00901 inline const void* CommPolicy<BlockVector<FieldVector<K, n>, A> >::getAddress(const Type& v, int index)
00902 {
00903 return &(v[index]);
00904 }
00905
00906 template<class K, class A, int n>
00907 inline int CommPolicy<BlockVector<FieldVector<K, n>, A> >::getSize(const Type& v, int index)
00908 {
00909 return 1;
00910 }
00911
00912
00913 template<class T>
00914 inline const typename CopyGatherScatter<T>::IndexedType& CopyGatherScatter<T>::gather(const T& vec, int i)
00915 {
00916 return vec[i];
00917
00918 }
00919
00920 template<class T>
00921 inline void CopyGatherScatter<T>::scatter(T& vec, const IndexedType& v, int i)
00922 {
00923 vec[i]=v;
00924
00925 }
00926 template<typename T>
00927 DatatypeCommunicator<T>::DatatypeCommunicator()
00928 : remoteIndices_(0), created_(false)
00929 {
00930 requests_[0]=0;
00931 requests_[1]=0;
00932 }
00933
00934
00935
00936 template<typename T>
00937 DatatypeCommunicator<T>::~DatatypeCommunicator()
00938 {
00939 free();
00940 }
00941
00942 template<typename T>
00943 template<class T1, class T2, class V>
00944 inline void DatatypeCommunicator<T>::build(const RemoteIndices& remoteIndices,
00945 const T1& source, V& sendData,
00946 const T2& destination, V& receiveData)
00947 {
00948 remoteIndices_ = &remoteIndices;
00949 free();
00950 createDataTypes<T1,T2,V,false>(source,destination, receiveData);
00951 createDataTypes<T1,T2,V,true>(source,destination, sendData);
00952 createRequests<V,true>(sendData, receiveData);
00953 createRequests<V,false>(receiveData, sendData);
00954 created_=true;
00955 }
00956
00957 template<typename T>
00958 void DatatypeCommunicator<T>::free()
00959 {
00960 if(created_){
00961 delete[] requests_[0];
00962 delete[] requests_[1];
00963 typedef MessageTypeMap::iterator iterator;
00964 typedef MessageTypeMap::const_iterator const_iterator;
00965
00966 const const_iterator end=messageTypes.end();
00967
00968 for(iterator process = messageTypes.begin(); process != end; ++process){
00969 MPI_Datatype *type = &(process->second.first);
00970 MPI_Type_free(type);
00971 type = &(process->second.second);
00972 MPI_Type_free(type);
00973 }
00974 messageTypes.clear();
00975
00976 }
00977
00978 }
00979
00980 template<typename T>
00981 template<class T1, class T2, class V, bool send>
00982 void DatatypeCommunicator<T>::createDataTypes(const T1& sourceFlags, const T2& destFlags, V& data)
00983 {
00984
00985 MPIDatatypeInformation<V> dataInfo(data);
00986 this->template buildInterface<T1,T2,MPIDatatypeInformation<V>,send>(*remoteIndices_,sourceFlags, destFlags, dataInfo);
00987
00988 typedef typename RemoteIndices::RemoteIndexMap::const_iterator const_iterator;
00989 const const_iterator end=this->remoteIndices_->end();
00990
00991
00992 for(const_iterator process=this->remoteIndices_->begin(); process != end; ++process){
00993 IndexedTypeInformation& info=dataInfo.information_[process->first];
00994
00995 MPI_Aint base;
00996 MPI_Address(const_cast<void *>(CommPolicy<V>::getAddress(data, 0)), &base);
00997
00998 for(int i=0; i< info.elements; i++){
00999 info.displ[i]-=base;
01000 }
01001
01002
01003 MPI_Datatype* type = &( send ? messageTypes[process->first].first : messageTypes[process->first].second);
01004 MPI_Type_hindexed(info.elements, info.length, info.displ,
01005 MPITraits<typename CommPolicy<V>::IndexedType>::getType(),
01006 type);
01007 MPI_Type_commit(type);
01008
01009 info.free();
01010 }
01011 }
01012
01013 template<typename T>
01014 template<class V, bool createForward>
01015 void DatatypeCommunicator<T>::createRequests(V& sendData, V& receiveData)
01016 {
01017 typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >::const_iterator MapIterator;
01018 int rank;
01019 static int index = createForward?1:0;
01020 int noMessages = messageTypes.size();
01021
01022 requests_[index] = new MPI_Request[2*noMessages];
01023 const MapIterator end = messageTypes.end();
01024 int request=0;
01025 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
01026
01027
01028 for(MapIterator process = messageTypes.begin(); process != end;
01029 ++process, ++request){
01030 MPI_Datatype type = createForward ? process->second.second : process->second.first;
01031 void* address = const_cast<void*>(CommPolicy<V>::getAddress(receiveData,0));
01032 MPI_Recv_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
01033 }
01034
01035
01036
01037 for(MapIterator process = messageTypes.begin(); process != end;
01038 ++process, ++request){
01039 MPI_Datatype type = createForward ? process->second.first : process->second.second;
01040 void* address = const_cast<void*>(CommPolicy<V>::getAddress(sendData, 0));
01041 MPI_Ssend_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
01042 }
01043 }
01044
01045 template<typename T>
01046 void DatatypeCommunicator<T>::forward()
01047 {
01048 sendRecv(requests_[1]);
01049 }
01050
01051 template<typename T>
01052 void DatatypeCommunicator<T>::backward()
01053 {
01054 sendRecv(requests_[0]);
01055 }
01056
01057 template<typename T>
01058 void DatatypeCommunicator<T>::sendRecv(MPI_Request* requests)
01059 {
01060 int noMessages = messageTypes.size();
01061
01062 MPI_Startall(noMessages, requests);
01063
01064 MPI_Startall(noMessages, requests+noMessages);
01065
01066
01067 MPI_Status* status=new MPI_Status[2*noMessages];
01068 for(int i=0; i<2*noMessages; i++)
01069 status[i].MPI_ERROR=MPI_SUCCESS;
01070
01071 int send = MPI_Waitall(noMessages, requests+noMessages, status+noMessages);
01072 int receive = MPI_Waitall(noMessages, requests, status);
01073
01074
01075 int success=1, globalSuccess=0;
01076 if(send==MPI_ERR_IN_STATUS){
01077 int rank;
01078 MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
01079 std::cerr<<rank<<": Error in sending :"<<std::endl;
01080
01081 for(int i=noMessages; i< 2*noMessages; i++)
01082 if(status[i].MPI_ERROR!=MPI_SUCCESS){
01083 char message[300];
01084 int messageLength;
01085 MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
01086 std::cerr<<" source="<<status[i].MPI_SOURCE<<" message: ";
01087 for(int i=0; i< messageLength; i++)
01088 std::cout<<message[i];
01089 }
01090 std::cerr<<std::endl;
01091 success=0;
01092 }
01093
01094 if(receive==MPI_ERR_IN_STATUS){
01095 int rank;
01096 MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
01097 std::cerr<<rank<<": Error in receiving!"<<std::endl;
01098
01099 for(int i=0; i< noMessages; i++)
01100 if(status[i].MPI_ERROR!=MPI_SUCCESS){
01101 char message[300];
01102 int messageLength;
01103 MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
01104 std::cerr<<" source="<<status[i].MPI_SOURCE<<" message: ";
01105 for(int i=0; i< messageLength; i++)
01106 std::cerr<<message[i];
01107 }
01108 std::cerr<<std::endl;
01109 success=0;
01110 }
01111
01112 MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, this->remoteIndices_->communicator());
01113
01114 if(!globalSuccess)
01115 DUNE_THROW(CommunicationError, "A communication error occurred!");
01116
01117
01118 }
01119
01120
01121
01122 template<typename T>
01123 BufferedCommunicator<T>::BufferedCommunicator()
01124 : interface_(0)
01125 {
01126 buffers_[0]=0;
01127 buffers_[1]=0;
01128 bufferSize_[0]=0;
01129 bufferSize_[1]=0;
01130 }
01131
01132
01133 template<typename T>
01134 template<class Data>
01135 typename EnableIf<SameType<SizeOne, typename CommPolicy<Data>::IndexedTypeFlag>::value, void>::Type
01136 BufferedCommunicator<T>::build(const Interface& interface)
01137 {
01138 typedef typename Interface::InformationMap::const_iterator const_iterator;
01139 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
01140 const const_iterator end = interface.interfaces().end();
01141 int lrank;
01142 MPI_Comm_rank(interface.communicator(), &lrank);
01143
01144 bufferSize_[0]=0;
01145 bufferSize_[1]=0;
01146
01147 for(const_iterator interfacePair = interface.interfaces().begin();
01148 interfacePair != end; ++interfacePair){
01149 int noSend = MessageSizeCalculator<Data,Flag>()(interfacePair->second.first);
01150 int noRecv = MessageSizeCalculator<Data,Flag>()(interfacePair->second.second);
01151 messageInformation_.insert(std::make_pair(interfacePair->first,
01152 std::make_pair(MessageInformation(bufferSize_[0],
01153 noSend*sizeof(typename CommPolicy<Data>::IndexedType)),
01154 MessageInformation(bufferSize_[1],
01155 noRecv*sizeof(typename CommPolicy<Data>::IndexedType)))));
01156 bufferSize_[0] += noSend;
01157 bufferSize_[1] += noRecv;
01158 }
01159
01160
01161 bufferSize_[0] *= sizeof(typename CommPolicy<Data>::IndexedType);
01162 bufferSize_[1] *= sizeof(typename CommPolicy<Data>::IndexedType);
01163
01164 buffers_[0] = new char[bufferSize_[0]];
01165 buffers_[1] = new char[bufferSize_[1]];
01166 interface_ = &interface;
01167
01168 }
01169
01170 template<typename T>
01171 template<class Data>
01172 void BufferedCommunicator<T>::build(const Data& source, const Data& dest, const Interface& interface)
01173 {
01174 typedef typename Interface::InformationMap::const_iterator const_iterator;
01175 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
01176 const const_iterator end = interface.interfaces().end();
01177
01178 bufferSize_[0]=0;
01179 bufferSize_[1]=0;
01180
01181 for(const_iterator interfacePair = interface.interfaces().begin();
01182 interfacePair != end; ++interfacePair){
01183 int noSend = MessageSizeCalculator<Data,Flag>()(source, interfacePair->second.first);
01184 int noRecv = MessageSizeCalculator<Data,Flag>()(dest, interfacePair->second.second);
01185
01186 messageInformation_.insert(std::make_pair(interfacePair->first,
01187 std::make_pair(MessageInformation(bufferSize_[0],
01188 noSend*sizeof(typename CommPolicy<Data>::IndexedType)),
01189 MessageInformation(bufferSize_[1],
01190 noRecv*sizeof(typename CommPolicy<Data>::IndexedType)))));
01191 bufferSize_[0] += noSend;
01192 bufferSize_[1] += noRecv;
01193 }
01194
01195 bufferSize_[0] *= sizeof(typename CommPolicy<Data>::IndexedType);
01196 bufferSize_[1] *= sizeof(typename CommPolicy<Data>::IndexedType);
01197
01198 buffers_[0] = new char[bufferSize_[0]];
01199 buffers_[1] = new char[bufferSize_[1]];
01200 interface_ = &interface;
01201 }
01202
01203 template<typename T>
01204 void BufferedCommunicator<T>::free()
01205 {
01206 if(interface_!=0){
01207 messageInformation_.clear();
01208 delete[] buffers_[0];
01209 delete[] buffers_[1];
01210 interface_=0;
01211 }
01212 }
01213
01214 template<typename T>
01215 BufferedCommunicator<T>::~BufferedCommunicator()
01216 {
01217 free();
01218 }
01219
01220 template<typename T>
01221 template<class Data>
01222 inline int BufferedCommunicator<T>::MessageSizeCalculator<Data,SizeOne>::operator()
01223 (const InterfaceInformation& info) const
01224 {
01225 return info.size();
01226 }
01227
01228 template<typename T>
01229 template<class Data>
01230 inline int BufferedCommunicator<T>::MessageSizeCalculator<Data,SizeOne>::operator()
01231 (const Data& data, const InterfaceInformation& info) const
01232 {
01233 return operator()(info);
01234 }
01235
01236 template<typename T>
01237 template<class Data>
01238 inline int BufferedCommunicator<T>::MessageSizeCalculator<Data, VariableSize>::operator()
01239 (const Data& data, const InterfaceInformation& info) const
01240 {
01241 int entries=0;
01242
01243 for(int i=0; i < info.size(); i++)
01244 entries += CommPolicy<Data>::getSize(data,info[i]);
01245
01246 return entries;
01247 }
01248
01249 template<typename T>
01250 template<class Data, class GatherScatter, bool FORWARD>
01251 inline void BufferedCommunicator<T>::MessageGatherer<Data,GatherScatter,FORWARD,VariableSize>::operator()(const Interface& interface,const Data& data, Type* buffer, size_t bufferSize) const
01252 {
01253 typedef typename Interface::InformationMap::const_iterator
01254 const_iterator;
01255
01256 int rank;
01257 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
01258 const const_iterator end = interface.interfaces().end();
01259 size_t index=0;
01260
01261 for(const_iterator interfacePair = interface.interfaces().begin();
01262 interfacePair != end; ++interfacePair){
01263 int size = forward ? interfacePair->second.first.size() :
01264 interfacePair->second.second.size();
01265 int proc = interfacePair->first();
01266
01267 for(int i=0; i < size; i++){
01268 int local = forward ? interfacePair->second->first[i] :
01269 interfacePair->second->second[i];
01270 for(int j=0; j < CommPolicy<Data>::getSize(data, local);j++, index++){
01271
01272 #ifdef DUNE_ISTL_WITH_CHECKING
01273 assert(bufferSize>=(index+1)*sizeof(typename CommPolicy<Data>::IndexedType));
01274 #endif
01275 buffer[index]=GatherScatter::gather(data, local, j);
01276 }
01277
01278 }
01279 }
01280
01281 }
01282
01283 template<typename T>
01284 template<class Data, class GatherScatter, bool FORWARD>
01285 inline void BufferedCommunicator<T>::MessageGatherer<Data,GatherScatter,FORWARD,SizeOne>::operator()(const Interface& interface, const Data& data, Type* buffer, size_t bufferSize)const
01286 {
01287 typedef typename Interface::InformationMap::const_iterator
01288 const_iterator;
01289 const const_iterator end = interface.interfaces().end();
01290 size_t index = 0;
01291
01292 int rank;
01293 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
01294
01295 for(const_iterator interfacePair = interface.interfaces().begin();
01296 interfacePair != end; ++interfacePair){
01297 size_t size = FORWARD ? interfacePair->second.first.size() :
01298 interfacePair->second.second.size();
01299
01300 for(size_t i=0; i < size; i++){
01301
01302 #ifdef DUNE_ISTL_WITH_CHECKING
01303 assert(bufferSize>=(index+1)*sizeof(typename CommPolicy<Data>::IndexedType));
01304 #endif
01305
01306 buffer[index++] = GatherScatter::gather(data, FORWARD ? interfacePair->second.first[i] :
01307 interfacePair->second.second[i]);
01308 }
01309 }
01310
01311 }
01312
01313 template<typename T>
01314 template<class Data, class GatherScatter, bool FORWARD>
01315 inline void BufferedCommunicator<T>::MessageScatterer<Data,GatherScatter,FORWARD,VariableSize>::operator()(const Interface& interface, Data& data, Type* buffer, const int& proc)const
01316 {
01317 typedef typename Interface::Information Information;
01318 const typename Interface::InformationMap::const_iterator infoPair = interface.interfaces().find(proc);
01319
01320 assert(infoPair!=interface.interfaces().end());
01321
01322 const Information& info = FORWARD ? infoPair->second.second :
01323 infoPair->second.first;
01324
01325 for(int i=0, index=0; i < info.size(); i++){
01326 for(int j=0; j < CommPolicy<Data>::getSize(data, info[i]); j++)
01327 GatherScatter::scatter(data, buffer[index++], info[i], j);
01328 }
01329 }
01330
01331 template<typename T>
01332 template<class Data, class GatherScatter, bool FORWARD>
01333 inline void BufferedCommunicator<T>::MessageScatterer<Data,GatherScatter,FORWARD,SizeOne>::operator()(const Interface& interface, Data& data, Type* buffer, const int& proc)const
01334 {
01335 typedef typename Interface::Information Information;
01336 const typename Interface::InformationMap::const_iterator infoPair = interface.interfaces().find(proc);
01337
01338 assert(infoPair!=interface.interfaces().end());
01339
01340 const Information& info = FORWARD ? infoPair->second.second :
01341 infoPair->second.first;
01342
01343 for(size_t i=0; i < info.size(); i++){
01344 GatherScatter::scatter(data, buffer[i], info[i]);
01345 }
01346 }
01347
01348 template<typename T>
01349 template<class GatherScatter,class Data>
01350 void BufferedCommunicator<T>::forward(Data& data)
01351 {
01352 this->template sendRecv<GatherScatter,true>(data, data);
01353 }
01354
01355 template<typename T>
01356 template<class GatherScatter, class Data>
01357 void BufferedCommunicator<T>::backward(Data& data)
01358 {
01359 this->template sendRecv<GatherScatter,false>(data, data);
01360 }
01361
01362 template<typename T>
01363 template<class GatherScatter, class Data>
01364 void BufferedCommunicator<T>::forward(const Data& source, Data& dest)
01365 {
01366 this->template sendRecv<GatherScatter,true>(source, dest);
01367 }
01368
01369 template<typename T>
01370 template<class GatherScatter, class Data>
01371 void BufferedCommunicator<T>::backward(Data& source, const Data& dest)
01372 {
01373 this->template sendRecv<GatherScatter,false>(dest, source);
01374 }
01375
01376 template<typename T>
01377 template<class GatherScatter, bool FORWARD, class Data>
01378 void BufferedCommunicator<T>::sendRecv(const Data& source, Data& dest)
01379 {
01380 int rank, lrank;
01381
01382 MPI_Comm_rank(MPI_COMM_WORLD,&rank);
01383 MPI_Comm_rank(MPI_COMM_WORLD,&lrank);
01384
01385 typedef typename CommPolicy<Data>::IndexedType Type;
01386 Type *sendBuffer, *recvBuffer;
01387 size_t sendBufferSize, recvBufferSize;
01388
01389 if(FORWARD){
01390 sendBuffer = reinterpret_cast<Type*>(buffers_[0]);
01391 sendBufferSize = bufferSize_[0];
01392 recvBuffer = reinterpret_cast<Type*>(buffers_[1]);
01393 recvBufferSize = bufferSize_[1];
01394 }else{
01395 sendBuffer = reinterpret_cast<Type*>(buffers_[1]);
01396 sendBufferSize = bufferSize_[1];
01397 recvBuffer = reinterpret_cast<Type*>(buffers_[0]);
01398 recvBufferSize = bufferSize_[0];
01399 }
01400 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
01401
01402 MessageGatherer<Data,GatherScatter,FORWARD,Flag>()(*interface_, source, sendBuffer, sendBufferSize);
01403
01404 MPI_Request* sendRequests = new MPI_Request[messageInformation_.size()];
01405 MPI_Request* recvRequests = new MPI_Request[messageInformation_.size()];
01406
01407
01408 typedef typename InformationMap::const_iterator const_iterator;
01409
01410 const const_iterator end = messageInformation_.end();
01411 size_t i=0;
01412 int* processMap = new int[messageInformation_.size()];
01413
01414 for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i){
01415 processMap[i]=info->first;
01416 if(FORWARD){
01417 assert(info->second.second.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
01418 MPI_Irecv(recvBuffer+info->second.second.start_, info->second.second.size_,
01419 MPI_BYTE, info->first, commTag_, interface_->communicator(),
01420 recvRequests+i);
01421 }else{
01422 assert(info->second.first.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= recvBufferSize );
01423 MPI_Irecv(recvBuffer+info->second.first.start_, info->second.first.size_,
01424 MPI_BYTE, info->first, commTag_, interface_->communicator(),
01425 recvRequests+i);
01426 }
01427 }
01428
01429
01430 i=0;
01431 for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i)
01432 if(FORWARD){
01433 assert(info->second.first.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= sendBufferSize );
01434 MPI_Issend(sendBuffer+info->second.first.start_, info->second.first.size_,
01435 MPI_BYTE, info->first, commTag_, interface_->communicator(),
01436 sendRequests+i);
01437 }else{
01438 assert(info->second.second.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= sendBufferSize );
01439 MPI_Issend(sendBuffer+info->second.second.start_, info->second.second.size_,
01440 MPI_BYTE, info->first, commTag_, interface_->communicator(),
01441 sendRequests+i);
01442 }
01443
01444
01445 i=0;
01446 int success = 1;
01447 int finished = MPI_UNDEFINED;
01448 MPI_Status status;
01449
01450
01451 for(i=0;i< messageInformation_.size();i++){
01452 MPI_Waitany(messageInformation_.size(), recvRequests, &finished, &status);
01453 assert(finished != MPI_UNDEFINED);
01454
01455 if(status.MPI_ERROR==MPI_SUCCESS){
01456 int& proc = processMap[finished];
01457 typename InformationMap::const_iterator infoIter = messageInformation_.find(proc);
01458 assert(infoIter != messageInformation_.end());
01459
01460 MessageInformation info = (FORWARD)? infoIter->second.second : infoIter->second.first;
01461 assert(info.start_+info.size_ <= recvBufferSize);
01462
01463 MessageScatterer<Data,GatherScatter,FORWARD,Flag>()(*interface_, dest, recvBuffer+info.start_, proc);
01464 }else{
01465 std::cerr<<rank<<": MPI_Error occurred while receiving message from "<<processMap[finished]<<std::endl;
01466 success=0;
01467 }
01468 }
01469
01470 MPI_Status recvStatus;
01471
01472
01473 for(i=0;i< messageInformation_.size();i++)
01474 if(MPI_SUCCESS!=MPI_Wait(sendRequests+i, &recvStatus)){
01475 std::cerr<<rank<<": MPI_Error occurred while sending message to "<<processMap[finished]<<std::endl;
01476 success=0;
01477 }
01478 int globalSuccess;
01479 MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, interface_->communicator());
01480
01481 if(!globalSuccess)
01482 DUNE_THROW(CommunicationError, "A communication error occurred!");
01483
01484 delete[] processMap;
01485 delete[] sendRequests;
01486 delete[] recvRequests;
01487
01488 }
01489
01491 }
01492
01493 #endif