communicator.hh

Go to the documentation of this file.
00001 // $Id: communicator.hh 772 2007-05-08 08:56:30Z christi $
00002 #ifndef DUNE_COMMUNICATOR
00003 #define DUNE_COMMUNICATOR
00004 
00005 #include"remoteindices.hh"
00006 #include"interface.hh"
00007 #include<dune/common/exceptions.hh>
00008 #include<dune/common/typetraits.hh>
00009 namespace Dune
00010 {
00090   struct SizeOne
00091   {};
00092   
00098   struct VariableSize
00099   {};
00100   
00101 
00107   template<class V>
00108   struct CommPolicy
00109   {
00121     typedef V Type;
00122     
00128     typedef typename V::value_type IndexedType;
00129     
00134     typedef SizeOne IndexedTypeFlag;
00135 
00144     static const void* getAddress(const V& v, int index);
00145     
00151     static int getSize(const V&, int index);
00152   };
00153 
00154   template<class K, int n> class FieldVector;
00155   
00156   template<class B, class A> class VariableBlockVector;
00157   
00158   template<class K, class A, int n>
00159   struct CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >
00160   {
00161     typedef VariableBlockVector<FieldVector<K, n>, A> Type;
00162     
00163     typedef typename Type::B IndexedType;
00164     
00165     typedef VariableSize IndexedTypeFlag;
00166 
00167     static const void* getAddress(const Type& v, int i);
00168     
00169     static int getSize(const Type& v, int i);
00170   };
00171     
00175   class CommunicationError : public IOError
00176   {};
00177 
00181   template<class T>
00182   struct CopyGatherScatter
00183   {
00184     typedef typename CommPolicy<T>::IndexedType IndexedType;
00185     
00186     const IndexedType& gather(const T& vec, int i);
00187     
00188     void scatter(T& vec, const IndexedType& v, int i);
00189     
00190   };
00191 
00203   template<typename T>
00204   class DatatypeCommunicator : public InterfaceBuilder<T>
00205   {
00206   public:    
00207 
00211     typedef T ParallelIndexSet;
00212 
00216     typedef RemoteIndices<ParallelIndexSet> RemoteIndices;
00217 
00221     typedef typename RemoteIndices::GlobalIndex GlobalIndex;
00222     
00226     typedef typename RemoteIndices::Attribute Attribute;
00227 
00231     typedef typename RemoteIndices::LocalIndex LocalIndex;
00232     
00236     DatatypeCommunicator();
00237     
00241     ~DatatypeCommunicator();
00242 
00269     template<class T1, class T2, class V>
00270     void build(const RemoteIndices& remoteIndices, const T1& sourceFlags, V& sendData, const T2& destFlags, V& receiveData);
00271     
00275     void forward();
00276     
00280     void backward();
00281    
00285     void free();        
00286   private:
00287     enum { 
00291       commTag_ = 234
00292     };
00293     
00297     const RemoteIndices* remoteIndices_;
00298     
00299     typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> > 
00300     MessageTypeMap;
00301     
00305     MessageTypeMap messageTypes;
00306     
00310     void* data_;
00311 
00312     MPI_Request* requests_[2];
00313 
00317     bool created_;
00318     
00322     template<class V, bool FORWARD>
00323     void createRequests(V& sendData, V& receiveData);
00324     
00328     template<class T1, class T2, class V, bool send>
00329     void createDataTypes(const T1& source, const T2& destination, V& data);
00330 
00334     void sendRecv(MPI_Request* req);
00335     
00339     struct IndexedTypeInformation
00340     {
00346       void build(int i)
00347       {
00348         length = new int[i];
00349         displ  = new MPI_Aint[i];
00350         size = i;
00351       }
00352       
00356       void free()
00357       {
00358         delete[] length;
00359         delete[] displ;
00360       }
00362       int* length;
00364       MPI_Aint* displ;
00370       int elements;
00374       int size;
00375     };
00376     
00382     template<class V>
00383     struct MPIDatatypeInformation
00384     {
00389       MPIDatatypeInformation(const V& data): data_(data)
00390       {}
00391       
00397       void reserve(int proc, int size)
00398       {
00399         information_[proc].build(size);
00400       }
00407       void add(int proc, int local)
00408       {
00409         IndexedTypeInformation& info=information_[proc];
00410         assert(info.elements<info.size);
00411         MPI_Address( const_cast<void*>(CommPolicy<V>::getAddress(data_, local)),
00412                     info.displ+info.elements);
00413         info.length[info.elements]=CommPolicy<V>::getSize(data_, local);
00414         info.elements++;
00415       }
00416       
00421       std::map<int,IndexedTypeInformation> information_;
00425       const V& data_;
00426       
00427     };
00428     
00429   };
00430 
00440   template<typename T>
00441   class BufferedCommunicator
00442   {
00443   public:
00447     typedef Interface<T> Interface;
00448     
00452     typedef typename Interface::GlobalIndex GlobalIndex;
00453     
00457     typedef typename Interface::Attribute Attribute;
00458     
00462     BufferedCommunicator();
00463     
00470     template<class Data>
00471     typename EnableIf<is_same<SizeOne,typename CommPolicy<Data>::IndexedTypeFlag>::value, void>::Type
00472     build(const Interface& interface);
00473 
00481     template<class Data>
00482     void build(const Data& source, const Data& target, const Interface& interface);
00483     
00512     template<class GatherScatter, class Data>
00513     void forward(const Data& source, Data& dest);
00514     
00543     template<class GatherScatter, class Data>
00544     void backward(Data& source, const Data& dest);
00545 
00571     template<class GatherScatter, class Data>
00572     void forward(Data& data);
00573     
00599     template<class GatherScatter, class Data>
00600     void backward(Data& data);
00601     
00605     void free();
00606     
00610     ~BufferedCommunicator();
00611     
00612   private:
00613 
00617     template<class Data, typename IndexedTypeFlag>
00618     struct MessageSizeCalculator
00619     {};
00620     
00625     template<class Data>
00626     struct MessageSizeCalculator<Data,SizeOne>
00627     {
00634       inline int operator()(const InterfaceInformation& info) const;
00643       inline int operator()(const Data& data, const InterfaceInformation& info) const;
00644     };
00645 
00650     template<class Data>
00651     struct MessageSizeCalculator<Data,VariableSize>
00652     {
00661       inline int operator()(const Data& data, const InterfaceInformation& info) const;
00662     };
00663     
00667     template<class Data, class GatherScatter, bool send, typename IndexedTypeFlag>
00668     struct MessageGatherer
00669     {};
00670     
00675     template<class Data, class GatherScatter, bool send>
00676     struct MessageGatherer<Data,GatherScatter,send,SizeOne>
00677     {
00679       typedef typename CommPolicy<Data>::IndexedType Type;
00680       
00685       typedef GatherScatter Gatherer;
00686       
00687       enum{
00693         forward=send
00694       };
00695       
00703       inline void operator()(const Interface& interface, const Data& data, Type* buffer, size_t bufferSize) const;
00704     };
00705 
00710     template<class Data, class GatherScatter, bool send>
00711     struct MessageGatherer<Data,GatherScatter,send,VariableSize>
00712     {
00714       typedef typename CommPolicy<Data>::IndexedType Type;
00715       
00720       typedef GatherScatter Gatherer;
00721       
00722       enum{
00728         forward=send
00729       };
00730       
00738       inline void operator()(const Interface& interface, const Data& data, Type* buffer, size_t bufferSize) const;
00739     };
00740 
00744     template<class Data, class GatherScatter, bool send, typename IndexedTypeFlag>
00745     struct MessageScatterer
00746     {};
00747 
00752     template<class Data, class GatherScatter, bool send>
00753     struct MessageScatterer<Data,GatherScatter,send,SizeOne>
00754     {
00756       typedef typename CommPolicy<Data>::IndexedType Type;
00757       
00762       typedef GatherScatter Scatterer;
00763       
00764       enum{
00770         forward=send
00771       };
00772       
00780       inline void operator()(const Interface& interface, Data& data, Type* buffer, const int& proc) const;
00781     };
00786     template<class Data, class GatherScatter, bool send>
00787     struct MessageScatterer<Data,GatherScatter,send,VariableSize>
00788     {
00790       typedef typename CommPolicy<Data>::IndexedType Type;
00791       
00796       typedef GatherScatter Scatterer;
00797       
00798       enum{
00804         forward=send
00805       };
00806       
00814       inline void operator()(const Interface& interface, Data& data, Type* buffer, const int& proc) const;
00815     };
00816 
00820     struct MessageInformation
00821     {
00823       MessageInformation()
00824         : start_(0), size_(0)
00825       {}
00826       
00834       MessageInformation(size_t start, size_t size)
00835         :start_(start), size_(size)
00836       {}
00840       size_t start_;
00844       size_t size_;
00845     };
00846 
00853     typedef std::map<int,std::pair<MessageInformation,MessageInformation> >
00854     InformationMap;
00858     InformationMap messageInformation_;
00862     char* buffers_[2];
00866     size_t bufferSize_[2];
00867     
00868     enum{
00872       commTag_
00873     };
00874     
00878     const Interface* interface_;
00879 
00883     template<class GatherScatter, bool FORWARD, class Data>
00884     void sendRecv(const Data& source, Data& target);
00885     
00886   };
00887   
00888   template<class V>
00889   inline const void* CommPolicy<V>::getAddress(const V& v, int index)
00890   {
00891     return &(v[index]);
00892   }
00893   
00894   template<class V>
00895   inline int CommPolicy<V>::getSize(const V& v, int index)
00896   {
00897     return 1;
00898   }
00899 
00900   template<class K, class A, int n>
00901   inline const void* CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getAddress(const Type& v, int index)
00902   {
00903     return &(v[index][0]);
00904   }
00905 
00906   template<class K, class A, int n>
00907   inline int CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getSize(const Type& v, int index)
00908   {
00909     return v[index].getsize();
00910   }
00911 
00912   
00913   template<class T>
00914   inline const typename CopyGatherScatter<T>::IndexedType& CopyGatherScatter<T>::gather(const T& vec, int i)
00915   {
00916     return vec[i];
00917     
00918   }
00919   
00920   template<class T>
00921   inline void CopyGatherScatter<T>::scatter(T& vec, const IndexedType& v, int i)
00922   {
00923     vec[i]=v;
00924     
00925   }
00926   template<typename T>
00927   DatatypeCommunicator<T>::DatatypeCommunicator()
00928     : remoteIndices_(0), created_(false)
00929   {
00930     requests_[0]=0;
00931     requests_[1]=0;
00932   }
00933   
00934  
00935 
00936   template<typename T>
00937   DatatypeCommunicator<T>::~DatatypeCommunicator()
00938   {
00939     free();
00940   }
00941   
00942   template<typename T>
00943   template<class T1, class T2, class V>
00944   inline void DatatypeCommunicator<T>::build(const RemoteIndices& remoteIndices,
00945                                                  const T1& source, V& sendData,
00946                                                  const T2& destination, V& receiveData)
00947   {
00948     remoteIndices_ = &remoteIndices;
00949     free();
00950     createDataTypes<T1,T2,V,false>(source,destination, receiveData);
00951     createDataTypes<T1,T2,V,true>(source,destination, sendData);
00952     createRequests<V,true>(sendData, receiveData);
00953     createRequests<V,false>(receiveData, sendData);
00954     created_=true;
00955   }
00956 
00957   template<typename T>
00958   void DatatypeCommunicator<T>::free()
00959   {
00960     if(created_){
00961       delete[] requests_[0];
00962       delete[] requests_[1];
00963       typedef MessageTypeMap::iterator iterator;
00964       typedef MessageTypeMap::const_iterator const_iterator;
00965       
00966       const const_iterator end=messageTypes.end();
00967       
00968       for(iterator process = messageTypes.begin(); process != end; ++process){
00969         MPI_Datatype *type = &(process->second.first);
00970         MPI_Type_free(type);
00971         type = &(process->second.second);
00972         MPI_Type_free(type);
00973       }
00974       messageTypes.clear();
00975       
00976     }
00977     
00978   }
00979   
00980   template<typename T>
00981   template<class T1, class T2, class V, bool send>
00982   void DatatypeCommunicator<T>::createDataTypes(const T1& sourceFlags, const T2& destFlags, V& data)
00983   {
00984 
00985     MPIDatatypeInformation<V>  dataInfo(data);
00986     this->template buildInterface<T1,T2,MPIDatatypeInformation<V>,send>(*remoteIndices_,sourceFlags, destFlags, dataInfo);
00987 
00988     typedef typename RemoteIndices::RemoteIndexMap::const_iterator const_iterator;
00989     const const_iterator end=this->remoteIndices_->end();
00990     
00991     // Allocate MPI_Datatypes and deallocate memory for the type construction.
00992     for(const_iterator process=this->remoteIndices_->begin(); process != end; ++process){
00993       IndexedTypeInformation& info=dataInfo.information_[process->first];
00994       // Shift the displacement
00995       MPI_Aint base;
00996       MPI_Address(const_cast<void *>(CommPolicy<V>::getAddress(data, 0)), &base);
00997       
00998       for(int i=0; i< info.elements; i++){
00999         info.displ[i]-=base;
01000       }
01001       
01002       // Create data type
01003       MPI_Datatype* type = &( send ? messageTypes[process->first].first : messageTypes[process->first].second);
01004       MPI_Type_hindexed(info.elements, info.length, info.displ, 
01005                        MPITraits<typename CommPolicy<V>::IndexedType>::getType(),
01006                        type);
01007       MPI_Type_commit(type);
01008       // Deallocate memory
01009       info.free();
01010     }
01011   }
01012   
01013   template<typename T>
01014   template<class V, bool createForward>
01015   void DatatypeCommunicator<T>::createRequests(V& sendData, V& receiveData)
01016   {
01017     typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >::const_iterator MapIterator;
01018     int rank;
01019     static int index = createForward?1:0;
01020     int noMessages = messageTypes.size();
01021     // allocate request handles
01022     requests_[index] = new MPI_Request[2*noMessages];
01023     const MapIterator end = messageTypes.end();
01024     int request=0;
01025     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
01026     
01027     // Set up the requests for receiving first
01028     for(MapIterator process = messageTypes.begin(); process != end; 
01029         ++process, ++request){
01030       MPI_Datatype type = createForward ? process->second.second : process->second.first;
01031       void* address = const_cast<void*>(CommPolicy<V>::getAddress(receiveData,0));
01032       MPI_Recv_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
01033     }
01034     
01035     // And now the send requests
01036 
01037     for(MapIterator process = messageTypes.begin(); process != end; 
01038         ++process, ++request){
01039       MPI_Datatype type = createForward ? process->second.first : process->second.second;
01040       void* address =  const_cast<void*>(CommPolicy<V>::getAddress(sendData, 0));
01041       MPI_Ssend_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
01042     }
01043   }   
01044         
01045   template<typename T>
01046   void DatatypeCommunicator<T>::forward()
01047   {
01048     sendRecv(requests_[1]);
01049   }
01050   
01051   template<typename T>
01052   void DatatypeCommunicator<T>::backward()
01053   {
01054     sendRecv(requests_[0]);
01055   }
01056 
01057   template<typename T>
01058   void DatatypeCommunicator<T>::sendRecv(MPI_Request* requests)
01059   {
01060     int noMessages = messageTypes.size();
01061     // Start the receive calls first
01062     MPI_Startall(noMessages, requests);
01063     // Now the send calls
01064     MPI_Startall(noMessages, requests+noMessages);
01065     
01066     // Wait for completion of the communication send first then receive
01067     MPI_Status* status=new MPI_Status[2*noMessages];
01068     for(int i=0; i<2*noMessages; i++)
01069       status[i].MPI_ERROR=MPI_SUCCESS;
01070     
01071     int send = MPI_Waitall(noMessages, requests+noMessages, status+noMessages);
01072     int receive = MPI_Waitall(noMessages, requests, status);
01073     
01074     // Error checks
01075     int success=1, globalSuccess=0;
01076     if(send==MPI_ERR_IN_STATUS){
01077       int rank;
01078       MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
01079       std::cerr<<rank<<": Error in sending :"<<std::endl;
01080       // Search for the error
01081       for(int i=noMessages; i< 2*noMessages; i++)
01082         if(status[i].MPI_ERROR!=MPI_SUCCESS){
01083           char message[300];
01084           int messageLength;
01085           MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
01086           std::cerr<<" source="<<status[i].MPI_SOURCE<<" message: ";
01087           for(int i=0; i< messageLength; i++)
01088             std::cout<<message[i];
01089         }
01090       std::cerr<<std::endl;
01091       success=0;
01092     }
01093     
01094     if(receive==MPI_ERR_IN_STATUS){
01095       int rank;
01096       MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
01097       std::cerr<<rank<<": Error in receiving!"<<std::endl;
01098       // Search for the error
01099       for(int i=0; i< noMessages; i++)
01100         if(status[i].MPI_ERROR!=MPI_SUCCESS){
01101           char message[300];
01102           int messageLength;
01103           MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
01104           std::cerr<<" source="<<status[i].MPI_SOURCE<<" message: ";
01105           for(int i=0; i< messageLength; i++)
01106             std::cerr<<message[i];
01107         }
01108       std::cerr<<std::endl;
01109       success=0;
01110     }
01111 
01112     MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, this->remoteIndices_->communicator());
01113 
01114     if(!globalSuccess)
01115       DUNE_THROW(CommunicationError, "A communication error occurred!");
01116     
01117       
01118   }
01119   
01120 
01121 
01122   template<typename T>
01123   BufferedCommunicator<T>::BufferedCommunicator()
01124     : interface_(0)
01125   {
01126     buffers_[0]=0;
01127     buffers_[1]=0;
01128     bufferSize_[0]=0;
01129     bufferSize_[1]=0;
01130   }
01131   
01132   
01133   template<typename T>
01134   template<class Data>
01135   typename EnableIf<is_same<SizeOne, typename CommPolicy<Data>::IndexedTypeFlag>::value, void>::Type
01136   BufferedCommunicator<T>::build(const Interface& interface)
01137   {
01138     typedef typename Interface::InformationMap::const_iterator const_iterator;
01139     typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
01140     const const_iterator end = interface.interfaces().end();
01141     int lrank;
01142     MPI_Comm_rank(interface.communicator(), &lrank);
01143     
01144     bufferSize_[0]=0;
01145     bufferSize_[1]=0;
01146 
01147     for(const_iterator interfacePair = interface.interfaces().begin();
01148         interfacePair != end; ++interfacePair){  
01149       int noSend = MessageSizeCalculator<Data,Flag>()(interfacePair->second.first);
01150       int noRecv = MessageSizeCalculator<Data,Flag>()(interfacePair->second.second);
01151       messageInformation_.insert(std::make_pair(interfacePair->first,
01152                                  std::make_pair(MessageInformation(bufferSize_[0],
01153                                                                    noSend*sizeof(typename CommPolicy<Data>::IndexedType)),
01154                                                 MessageInformation(bufferSize_[1],
01155                                                                    noRecv*sizeof(typename CommPolicy<Data>::IndexedType)))));
01156       bufferSize_[0] += noSend;
01157       bufferSize_[1] += noRecv;
01158     }
01159 
01160     // allocate the buffers
01161     bufferSize_[0] *= sizeof(typename CommPolicy<Data>::IndexedType);
01162     bufferSize_[1] *= sizeof(typename CommPolicy<Data>::IndexedType);
01163     
01164     buffers_[0] = new char[bufferSize_[0]];
01165     buffers_[1] = new char[bufferSize_[1]];
01166     interface_ = &interface;
01167     
01168   }  
01169   
01170   template<typename T>
01171   template<class Data>
01172   void BufferedCommunicator<T>::build(const Data& source, const Data& dest, const Interface& interface)
01173   {
01174     typedef typename Interface::InformationMap::const_iterator const_iterator;
01175     typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
01176     const const_iterator end = interface.interfaces().end();
01177     
01178     bufferSize_[0]=0;
01179     bufferSize_[1]=0;
01180 
01181     for(const_iterator interfacePair = interface.interfaces().begin();
01182         interfacePair != end; ++interfacePair){      
01183       int noSend = MessageSizeCalculator<Data,Flag>()(source, interfacePair->second.first);
01184       int noRecv = MessageSizeCalculator<Data,Flag>()(dest, interfacePair->second.second);
01185       
01186       messageInformation_.insert(std::make_pair(interfacePair->first,
01187                                                 std::make_pair(MessageInformation(bufferSize_[0],
01188                                                                                   noSend*sizeof(typename CommPolicy<Data>::IndexedType)),
01189                                                                MessageInformation(bufferSize_[1],
01190                                                                                   noRecv*sizeof(typename CommPolicy<Data>::IndexedType)))));
01191       bufferSize_[0] += noSend;
01192       bufferSize_[1] += noRecv;
01193     }
01194 
01195     bufferSize_[0] *= sizeof(typename CommPolicy<Data>::IndexedType);
01196     bufferSize_[1] *= sizeof(typename CommPolicy<Data>::IndexedType);
01197     // allocate the buffers
01198     buffers_[0] = new char[bufferSize_[0]];
01199     buffers_[1] = new char[bufferSize_[1]];
01200     interface_ = &interface;
01201   }
01202   
01203   template<typename T>
01204   void BufferedCommunicator<T>::free()
01205   {
01206     if(interface_!=0){
01207       messageInformation_.clear();
01208       delete[] buffers_[0];
01209       delete[] buffers_[1];
01210       interface_=0;
01211     }
01212   }
01213 
01214   template<typename T>
01215   BufferedCommunicator<T>::~BufferedCommunicator()
01216   {
01217     free();
01218   }
01219   
01220   template<typename T>
01221   template<class Data>
01222   inline int BufferedCommunicator<T>::MessageSizeCalculator<Data,SizeOne>::operator()
01223     (const InterfaceInformation& info) const
01224   {
01225     return info.size();
01226   }
01227 
01228   template<typename T>
01229   template<class Data>
01230   inline int BufferedCommunicator<T>::MessageSizeCalculator<Data,SizeOne>::operator()
01231     (const Data& data, const InterfaceInformation& info) const
01232   {
01233     return operator()(info);
01234   }
01235 
01236   template<typename T>
01237   template<class Data>
01238   inline int BufferedCommunicator<T>::MessageSizeCalculator<Data, VariableSize>::operator()
01239     (const Data& data, const InterfaceInformation& info) const
01240   {
01241     int entries=0;
01242 
01243     for(int i=0; i <  info.size(); i++)
01244       entries += CommPolicy<Data>::getSize(data,info[i]);
01245 
01246     return entries;
01247   }
01248   
01249   template<typename T>
01250   template<class Data, class GatherScatter, bool FORWARD>
01251   inline void BufferedCommunicator<T>::MessageGatherer<Data,GatherScatter,FORWARD,VariableSize>::operator()(const Interface& interface,const Data& data, Type* buffer, size_t bufferSize) const
01252   {
01253     typedef typename Interface::InformationMap::const_iterator
01254       const_iterator;
01255 
01256     int rank;
01257     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
01258     const const_iterator end = interface.interfaces().end();
01259     size_t index=0;
01260     
01261     for(const_iterator interfacePair = interface.interfaces().begin();
01262         interfacePair != end; ++interfacePair){
01263       int size = forward ? interfacePair->second.first.size() :
01264         interfacePair->second.second.size();
01265       int proc = interfacePair->first();
01266       
01267       for(int i=0; i < size; i++){  
01268         int local = forward ? interfacePair->second->first[i] :
01269           interfacePair->second->second[i];
01270         for(int j=0; j < CommPolicy<Data>::getSize(data, local);j++, index++){
01271 
01272 #ifdef DUNE_ISTL_WITH_CHECKING
01273           assert(bufferSize>=(index+1)*sizeof(typename CommPolicy<Data>::IndexedType));
01274 #endif
01275           buffer[index]=GatherScatter::gather(data, local, j);
01276         }
01277         
01278       }
01279     }
01280         
01281   }
01282 
01283   template<typename T>
01284   template<class Data, class GatherScatter, bool FORWARD>
01285   inline void BufferedCommunicator<T>::MessageGatherer<Data,GatherScatter,FORWARD,SizeOne>::operator()(const Interface& interface, const Data& data, Type* buffer, size_t bufferSize)const
01286   {
01287     typedef typename Interface::InformationMap::const_iterator
01288       const_iterator;
01289     const const_iterator end = interface.interfaces().end();
01290     size_t index = 0;
01291     
01292     int rank;
01293     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
01294 
01295     for(const_iterator interfacePair = interface.interfaces().begin();
01296         interfacePair != end; ++interfacePair){
01297       size_t size = FORWARD ? interfacePair->second.first.size() :
01298         interfacePair->second.second.size();
01299       
01300       for(size_t i=0; i < size; i++){
01301 
01302 #ifdef DUNE_ISTL_WITH_CHECKING
01303           assert(bufferSize>=(index+1)*sizeof(typename CommPolicy<Data>::IndexedType));
01304 #endif
01305 
01306         buffer[index++] = GatherScatter::gather(data, FORWARD ? interfacePair->second.first[i] :
01307                                                 interfacePair->second.second[i]);
01308       }
01309     }
01310         
01311   }
01312   
01313   template<typename T>
01314   template<class Data, class GatherScatter, bool FORWARD>
01315   inline void BufferedCommunicator<T>::MessageScatterer<Data,GatherScatter,FORWARD,VariableSize>::operator()(const Interface& interface, Data& data, Type* buffer, const int& proc)const
01316   {
01317     typedef typename Interface::Information Information;
01318     const typename Interface::InformationMap::const_iterator infoPair = interface.interfaces().find(proc);
01319 
01320     assert(infoPair!=interface.interfaces().end());
01321     
01322     const Information& info = FORWARD ? infoPair->second.second :
01323        infoPair->second.first;
01324 
01325     for(int i=0, index=0; i < info.size(); i++){
01326         for(int j=0; j < CommPolicy<Data>::getSize(data, info[i]); j++)
01327           GatherScatter::scatter(data, buffer[index++], info[i], j);
01328     }
01329   }
01330 
01331   template<typename T>
01332   template<class Data, class GatherScatter, bool FORWARD>
01333   inline void BufferedCommunicator<T>::MessageScatterer<Data,GatherScatter,FORWARD,SizeOne>::operator()(const Interface& interface, Data& data, Type* buffer, const int& proc)const
01334   {
01335     typedef typename Interface::Information Information;
01336     const typename Interface::InformationMap::const_iterator infoPair = interface.interfaces().find(proc);
01337 
01338     assert(infoPair!=interface.interfaces().end());
01339     
01340     const Information& info = FORWARD ? infoPair->second.second :
01341       infoPair->second.first;
01342     
01343     for(size_t i=0; i < info.size(); i++){
01344       GatherScatter::scatter(data, buffer[i], info[i]);
01345     }
01346   }
01347   
01348   template<typename T>
01349   template<class GatherScatter,class Data>
01350   void BufferedCommunicator<T>::forward(Data& data)
01351   {
01352     this->template sendRecv<GatherScatter,true>(data, data);
01353   }
01354   
01355   template<typename T>
01356   template<class GatherScatter, class Data>
01357   void BufferedCommunicator<T>::backward(Data& data)
01358   {
01359     this->template sendRecv<GatherScatter,false>(data, data);
01360   }
01361 
01362   template<typename T>
01363   template<class GatherScatter, class Data>
01364   void BufferedCommunicator<T>::forward(const Data& source, Data& dest)
01365   {
01366     this->template sendRecv<GatherScatter,true>(source, dest);
01367   }
01368   
01369   template<typename T>
01370   template<class GatherScatter, class Data>
01371   void BufferedCommunicator<T>::backward(Data& source, const Data& dest)
01372   {
01373     this->template sendRecv<GatherScatter,false>(dest, source);
01374   }
01375 
01376   template<typename T>
01377   template<class GatherScatter, bool FORWARD, class Data>
01378   void BufferedCommunicator<T>::sendRecv(const Data& source, Data& dest) 
01379   {
01380     int rank, lrank;
01381     
01382     MPI_Comm_rank(MPI_COMM_WORLD,&rank);
01383     MPI_Comm_rank(MPI_COMM_WORLD,&lrank);
01384     
01385     typedef typename CommPolicy<Data>::IndexedType Type;
01386     Type *sendBuffer, *recvBuffer;
01387     size_t sendBufferSize, recvBufferSize;
01388     
01389     if(FORWARD){
01390       sendBuffer = reinterpret_cast<Type*>(buffers_[0]);
01391       sendBufferSize = bufferSize_[0];
01392       recvBuffer = reinterpret_cast<Type*>(buffers_[1]);
01393       recvBufferSize = bufferSize_[1];
01394     }else{
01395       sendBuffer = reinterpret_cast<Type*>(buffers_[1]);
01396       sendBufferSize = bufferSize_[1];
01397       recvBuffer = reinterpret_cast<Type*>(buffers_[0]);
01398       recvBufferSize = bufferSize_[0];
01399     }
01400     typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
01401 
01402     MessageGatherer<Data,GatherScatter,FORWARD,Flag>()(*interface_, source, sendBuffer, sendBufferSize);
01403     
01404     MPI_Request* sendRequests = new MPI_Request[messageInformation_.size()];
01405     MPI_Request* recvRequests = new MPI_Request[messageInformation_.size()];
01406     
01407     // Setup receive first
01408     typedef typename InformationMap::const_iterator const_iterator;
01409 
01410     const const_iterator end = messageInformation_.end();
01411     size_t i=0;
01412     int* processMap = new int[messageInformation_.size()];
01413     
01414     for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i){
01415           processMap[i]=info->first;
01416           if(FORWARD){
01417             assert(info->second.second.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
01418             MPI_Irecv(recvBuffer+info->second.second.start_, info->second.second.size_,
01419                       MPI_BYTE, info->first, commTag_, interface_->communicator(),
01420                       recvRequests+i);
01421           }else{
01422             assert(info->second.first.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= recvBufferSize );
01423             MPI_Irecv(recvBuffer+info->second.first.start_, info->second.first.size_,
01424                       MPI_BYTE, info->first, commTag_, interface_->communicator(),
01425                       recvRequests+i);
01426           }
01427     }
01428     
01429     // now the send requests
01430     i=0;
01431     for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i)
01432       if(FORWARD){
01433         assert(info->second.first.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= sendBufferSize );
01434         MPI_Issend(sendBuffer+info->second.first.start_, info->second.first.size_,
01435                    MPI_BYTE, info->first, commTag_, interface_->communicator(),
01436                    sendRequests+i);
01437       }else{
01438         assert(info->second.second.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= sendBufferSize );
01439         MPI_Issend(sendBuffer+info->second.second.start_, info->second.second.size_,
01440                    MPI_BYTE, info->first, commTag_, interface_->communicator(),
01441                    sendRequests+i);
01442       }
01443     
01444     // Wait for completion of receive and immediately start scatter
01445     i=0;
01446     int success = 1;
01447     int finished = MPI_UNDEFINED;
01448     MPI_Status status;//[messageInformation_.size()];
01449     //MPI_Waitall(messageInformation_.size(), recvRequests, status);
01450     
01451     for(i=0;i< messageInformation_.size();i++){
01452       MPI_Waitany(messageInformation_.size(), recvRequests, &finished, &status);
01453       assert(finished != MPI_UNDEFINED);
01454       
01455       if(status.MPI_ERROR==MPI_SUCCESS){
01456         int& proc = processMap[finished];
01457         typename InformationMap::const_iterator infoIter = messageInformation_.find(proc);
01458         assert(infoIter != messageInformation_.end());
01459 
01460         MessageInformation info = (FORWARD)? infoIter->second.second : infoIter->second.first;
01461         assert(info.start_+info.size_ <= recvBufferSize);
01462 
01463         MessageScatterer<Data,GatherScatter,FORWARD,Flag>()(*interface_, dest, recvBuffer+info.start_, proc);
01464       }else{
01465         std::cerr<<rank<<": MPI_Error occurred while receiving message from "<<processMap[finished]<<std::endl;
01466         success=0;
01467       }
01468     }
01469 
01470     MPI_Status recvStatus;
01471     
01472     // Wait for completion of sends
01473     for(i=0;i< messageInformation_.size();i++)
01474       if(MPI_SUCCESS!=MPI_Wait(sendRequests+i, &recvStatus)){
01475         std::cerr<<rank<<": MPI_Error occurred while sending message to "<<processMap[finished]<<std::endl;
01476         success=0;
01477       }
01478     int globalSuccess;
01479     MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, interface_->communicator());
01480 
01481     if(!globalSuccess)
01482       DUNE_THROW(CommunicationError, "A communication error occurred!");
01483         
01484     delete[] processMap;
01485     delete[] sendRequests;
01486     delete[] recvRequests;
01487 
01488   }
01489 
01491 }
01492 
01493 #endif

Generated on 9 Apr 2008 with Doxygen (ver 1.5.2) [logfile].