remoteindices.hh

Go to the documentation of this file.
00001 // $Id: remoteindices.hh 762 2007-05-01 17:01:20Z mblatt $
00002 #ifndef DUNE_REMOTEINDICES_HH
00003 #define DUNE_REMOTEINDICES_HH
00004 
00005 #include"indexset.hh"
00006 #include"istlexception.hh"
00007 #include"plocalindex.hh"
00008 #include<dune/common/poolallocator.hh>
00009 #include<dune/common/sllist.hh>
00010 #include<dune/common/helpertemplates.hh>
00011 #include<map>
00012 #include<utility>
00013 #include<iostream>
00014 
00015 #if HAVE_MPI
00016 #include"mpitraits.hh"
00017 #include"mpi.h"
00018 
00019 namespace Dune{
00030 
00031   template<typename TG, typename TA>
00032   class MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >
00033   {
00034   public:
00035     inline static MPI_Datatype getType();
00036   private:
00037     static MPI_Datatype type;
00038   };
00039   
00040   
00041   template<typename T>
00042   class RemoteIndices;
00043   
00044   template<typename T1, typename T2>
00045   class RemoteIndex;
00046   
00047   template<typename T>
00048   class IndicesSyncer;
00049   
00050   template<typename T1, typename T2>
00051   std::ostream& operator<<(std::ostream& os, const RemoteIndex<T1,T2>& index);
00052   
00053 
00054   template<typename T, bool mode>
00055   class RemoteIndexListModifier;
00056   
00060   template<typename T1, typename T2>
00061   class RemoteIndex
00062   {
00063     template<typename T>
00064     friend class IndicesSyncer;
00065     
00066     template<typename T, typename A>
00067     friend void repairLocalIndexPointers(std::map<int,SLList<typename T::GlobalIndex,A> >&, RemoteIndices<T>&,
00068                                          const T&); 
00069 
00070     template<typename T, bool mode>
00071     friend class RemoteIndexListModifier;
00072 
00073   public:    
00078     typedef T1 GlobalIndex;
00087     typedef T2 Attribute;
00088 
00092     typedef IndexPair<GlobalIndex,ParallelLocalIndex<Attribute> >
00093     PairType;
00094     
00099     const Attribute attribute() const;
00100 
00106     const PairType& localIndexPair() const;
00107 
00111     RemoteIndex();
00112     
00113     
00119     RemoteIndex(const T2& attribute, 
00120                 const PairType* local);
00121 
00122     
00128     RemoteIndex(const T2& attribute);
00129 
00130   private:
00132     const PairType* localIndex_;
00133 
00135     char attribute_;
00136   };
00137   
00138   template<class T>
00139   std::ostream& operator<<(std::ostream& os, const RemoteIndices<T>& indices);
00140   
00141   template<typename T>
00142   class InterfaceBuilder;
00143 
00144   template<class T>
00145   class CollectiveIterator;
00146 
00147   template<class T>
00148     class IndicesSyncer;
00149 
00153   template<class T>
00154   class RemoteIndices
00155   {
00156     friend class InterfaceBuilder<T>;
00157     friend class IndicesSyncer<T>;
00158     template<typename T1, typename A>
00159     friend void repairLocalIndexPointers(std::map<int,SLList<typename T1::GlobalIndex,A> >&, 
00160                                          RemoteIndices<T1>&,
00161                                          const T1&);
00162     friend std::ostream& operator<<<>(std::ostream&, const RemoteIndices<T>&);
00163     
00164   public:
00165    
00169     typedef T ParallelIndexSet;
00170  
00174     typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00175     
00176     
00180     typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00181 
00185     typedef typename LocalIndex::Attribute Attribute;
00186  
00190     typedef RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00191 
00192     
00196     typedef PoolAllocator<RemoteIndex,1> Allocator;
00197 
00199     typedef SLList<RemoteIndex,Allocator>
00200     RemoteIndexList;
00201     
00203     typedef std::map<int, std::pair<RemoteIndexList*,RemoteIndexList*> >
00204     RemoteIndexMap;    
00205     
00206     typedef typename RemoteIndexMap::const_iterator const_iterator;
00207     
00218     inline RemoteIndices(const ParallelIndexSet& source, const ParallelIndexSet& destination, 
00219                          const MPI_Comm& comm);
00220 
00221     RemoteIndices();
00222     
00236     void setIndexSets(const ParallelIndexSet& source, const ParallelIndexSet& destination, 
00237                          const MPI_Comm& comm);
00241     ~RemoteIndices();
00242     
00252     template<bool ignorePublic>
00253     void rebuild();
00254 
00262     inline bool isSynced() const;
00263 
00267     inline MPI_Comm communicator() const;
00268 
00278     template<bool mode, bool send>
00279     inline RemoteIndexListModifier<T,mode> getModifier(int process);
00280     
00285     inline const_iterator begin() const;
00286     
00291     inline const_iterator end() const;
00292     
00296     template<bool send>
00297     inline CollectiveIterator<T> iterator() const;
00298 
00302     inline void free();
00303     
00308     inline int neighbours() const;
00309     
00310   private:    
00312     const ParallelIndexSet* source_;
00313 
00315     const ParallelIndexSet* target_;
00316 
00318     MPI_Comm comm_;
00319 
00321     const static int commTag_=333;
00322     
00327     int sourceSeqNo_;
00328     
00333     int destSeqNo_;
00334 
00338     bool publicIgnored;
00339 
00343     bool firstBuild;
00344         
00346     typedef IndexPair<GlobalIndex, LocalIndex> 
00347     PairType;
00348     
00355     RemoteIndexMap remoteIndices_;
00356     
00363     template<bool ignorePublic>
00364     inline void buildRemote();
00365 
00371     inline int noPublic(const ParallelIndexSet& indexSet);
00372     
00384     template<bool ignorePublic>
00385     inline void packEntries(PairType** myPairs, const ParallelIndexSet& indexSet,
00386                             char* p_out, MPI_Datatype type, int bufferSize, 
00387                             int* position, int n);
00388     
00402     inline void unpackIndices(RemoteIndexList& remote, int remoteEntries,
00403                               PairType** local, int localEntries, char* p_in, 
00404                               MPI_Datatype type, int* positon, int bufferSize);
00405 
00406     inline void unpackIndices(RemoteIndexList& send, RemoteIndexList& receive,
00407                               int remoteEntries, PairType** localSource,
00408                               int localSourceEntries, PairType** localDest,
00409                               int localDestEntries, char* p_in,
00410                               MPI_Datatype type, int* position, int bufferSize);
00411 
00412   };
00413   
00431   template<class T, bool mode>
00432   class RemoteIndexListModifier
00433   {
00434   public:
00435     class InvalidPosition : public ISTLError
00436     {};
00437 
00438     enum {
00447       MODIFYINDEXSET=mode
00448     };
00449   
00453     typedef T ParallelIndexSet;
00454 
00458     typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00459 
00463     typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00464     
00468     typedef typename LocalIndex::Attribute Attribute;
00469     
00473     typedef RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00474     
00478     typedef PoolAllocator<RemoteIndex,1> Allocator;
00479 
00481     typedef SLList<RemoteIndex,Allocator>
00482     RemoteIndexList;
00483 
00487     typedef SLListModifyIterator<RemoteIndex,Allocator> ModifyIterator;
00488     
00492     typedef typename RemoteIndexList::const_iterator ConstIterator;
00493     
00507     void insert(const RemoteIndex& index) throw(InvalidPosition);
00508     
00509     
00524     void insert(const RemoteIndex& index, const GlobalIndex& global) throw(InvalidPosition);
00525 
00533     bool remove(const GlobalIndex& global) throw(InvalidPosition);
00534     
00540     RemoteIndexListModifier(const ParallelIndexSet& indexSet,
00541                             RemoteIndexList& rList);
00542 
00543     
00544     RemoteIndexListModifier(const RemoteIndexListModifier<T,mode>&);
00545 
00558     void repairLocalIndexPointers() throw(InvalidIndexSetState);
00559 
00560   private:
00561     
00562     typedef SLList<GlobalIndex,Allocator> GlobalList;
00563     typedef typename GlobalList::ModifyIterator GlobalModifyIterator;
00564     RemoteIndices<ParallelIndexSet>* remoteIndices_;
00565     RemoteIndexList* rList_;
00566     const ParallelIndexSet* indexSet_;
00567     GlobalList* glist_;
00568     ModifyIterator iter_;
00569     GlobalModifyIterator giter_;
00570     ConstIterator end_;
00571     bool first_;
00572     GlobalIndex last_;
00573   };
00574   
00579   template<class T>
00580   class CollectiveIterator
00581   {
00582     
00586     typedef T ParallelIndexSet;
00587 
00591     typedef typename ParallelIndexSet::GlobalIndex GlobalIndex;
00592 
00596     typedef typename ParallelIndexSet::LocalIndex LocalIndex;
00597     
00601     typedef typename LocalIndex::Attribute Attribute;
00602 
00604     typedef RemoteIndex<GlobalIndex,Attribute> RemoteIndex;
00605     
00607     typedef PoolAllocator<RemoteIndex,1> Allocator;
00608 
00610     typedef SLList<RemoteIndex,Allocator> RemoteIndexList;    
00611 
00613     typedef std::map<int,std::pair<typename RemoteIndexList::const_iterator,
00614                                    const typename RemoteIndexList::const_iterator> >
00615     Map;
00616 
00617   public:
00618     
00620     typedef std::map<int, std::pair<RemoteIndexList*,RemoteIndexList*> >
00621     RemoteIndexMap;
00622     
00628     inline CollectiveIterator(const RemoteIndexMap& map_, bool send);
00629     
00638     inline void advance(const GlobalIndex& global);
00639     
00643     inline bool empty();
00644     
00651     class iterator
00652     {
00653     public:
00654       typedef typename Map::iterator RealIterator;
00655       typedef typename Map::iterator ConstRealIterator;
00656       
00657       
00659       iterator(const RealIterator& iter, const ConstRealIterator& end, GlobalIndex& index)
00660         : iter_(iter), end_(end), index_(index)
00661       {
00662         // Move to the first valid entry
00663         while(iter_!=end_ && iter_->second.first->localIndexPair().global()!=index_)
00664           ++iter_;
00665       }
00666       
00668       iterator(const iterator& other)
00669         : iter_(other.iter_), end_(other.end_), index_(other.index_)
00670       { }
00671          
00673       iterator& operator++()
00674       { 
00675         ++iter_;
00676         // If entry is not valid move on
00677         while(iter_!=end_ && iter_->second.first->localIndexPair().global()!=index_)
00678           ++iter_;
00679                   
00680         return *this;
00681       }
00682       
00684       const RemoteIndex& operator*()const
00685       {
00686         return *(iter_->second.first);
00687       }
00688       
00690       int process() const
00691       {
00692         return iter_->first;
00693       }
00694       
00696       const RemoteIndex* operator->()const
00697       {
00698         return iter_->second.first.operator->();
00699       }
00700       
00702       bool operator==(const iterator& other)
00703       {
00704         return other.iter_==iter_;
00705       }
00706 
00708       bool operator!=(const iterator& other)
00709       {
00710         return other.iter_!=iter_;
00711       }
00712 
00713     private:
00714       iterator();
00715       
00716       RealIterator iter_;
00717       RealIterator end_;
00718       GlobalIndex index_;
00719     };
00720     
00721     iterator begin();
00722     
00723     iterator end();
00724     
00725   private:
00726     
00727     Map map_;
00728     GlobalIndex index_;
00729   };
00730     
00731   template<typename TG, typename TA>
00732   MPI_Datatype MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >::getType()
00733   {
00734     if(type==MPI_DATATYPE_NULL){
00735       int length[4];
00736       MPI_Aint disp[4];
00737       MPI_Datatype types[4] = {MPI_LB, MPITraits<TG>::getType(), 
00738                                MPITraits<ParallelLocalIndex<TA> >::getType(), MPI_UB};
00739       IndexPair<TG,ParallelLocalIndex<TA> > rep[2];
00740       length[0]=length[1]=length[2]=length[3]=1;
00741       MPI_Address(rep, disp); // lower bound of the datatype
00742       MPI_Address(&(rep[0].global_), disp+1);
00743       MPI_Address(&(rep[0].local_), disp+2);
00744       MPI_Address(rep+1, disp+3); // upper bound of the datatype
00745       for(int i=3; i >= 0; --i)
00746         disp[i] -= disp[0];
00747       MPI_Type_struct(4, length, disp, types, &type);
00748       MPI_Type_commit(&type);
00749     }
00750     return type;
00751   }
00752 
00753   template<typename TG, typename TA>
00754   MPI_Datatype MPITraits<IndexPair<TG,ParallelLocalIndex<TA> > >::type=MPI_DATATYPE_NULL;
00755 
00756   template<typename T1, typename T2>
00757   RemoteIndex<T1,T2>::RemoteIndex(const T2& attribute, const PairType* local)
00758     : localIndex_(local), attribute_(attribute)
00759   {}
00760 
00761   template<typename T1, typename T2>
00762   RemoteIndex<T1,T2>::RemoteIndex(const T2& attribute)
00763     : localIndex_(0), attribute_(attribute)
00764   {}
00765 
00766   template<typename T1, typename T2>
00767   RemoteIndex<T1,T2>::RemoteIndex()
00768     : localIndex_(0), attribute_()
00769   {}
00770 
00771   template<typename T1, typename T2>
00772   inline const T2 RemoteIndex<T1,T2>::attribute() const
00773   {
00774     return T2(attribute_);
00775   }
00776 
00777   template<typename T1, typename T2>
00778   inline const IndexPair<T1,ParallelLocalIndex<T2> >& RemoteIndex<T1,T2>::localIndexPair() const
00779   {
00780     return *localIndex_;
00781   }
00782   
00783   template<typename T>
00784   inline RemoteIndices<T>::RemoteIndices(const ParallelIndexSet& source,
00785                                              const ParallelIndexSet& destination,
00786                                              const MPI_Comm& comm)
00787     : source_(&source), target_(&destination), comm_(comm), 
00788       sourceSeqNo_(-1), destSeqNo_(-1), publicIgnored(false), firstBuild(true)
00789   {}
00790 
00791   template<typename T>
00792   RemoteIndices<T>::RemoteIndices()
00793     :source_(0), target_(0), sourceSeqNo_(-1), 
00794      destSeqNo_(-1), publicIgnored(false), firstBuild(true)
00795   {}
00796   
00797   template<class T>
00798   void RemoteIndices<T>::setIndexSets(const ParallelIndexSet& source,
00799                                  const ParallelIndexSet& destination,
00800                                  const MPI_Comm& comm)
00801   {
00802     free();
00803     source_ = &source;
00804     target_ = &destination;
00805     comm_ = comm;
00806     firstBuild = true;
00807   }
00808   
00809   template<typename T>
00810   RemoteIndices<T>::~RemoteIndices()
00811   {
00812     free();
00813   }
00814 
00815   template<typename T>
00816   template<bool ignorePublic>
00817   inline void RemoteIndices<T>::packEntries(IndexPair<GlobalIndex,LocalIndex>** pairs,
00818                                                 const ParallelIndexSet& indexSet,
00819                                                 char* p_out, MPI_Datatype type, 
00820                                                 int bufferSize,
00821                                                 int *position, int n)
00822   {
00823     // fill with own indices
00824     typedef typename ParallelIndexSet::const_iterator const_iterator;
00825     typedef IndexPair<GlobalIndex,LocalIndex> PairType;
00826     const const_iterator end = indexSet.end();
00827 
00828     //Now pack the source indices
00829     int i=0;
00830     for(const_iterator index = indexSet.begin(); index != end; ++index)
00831       if(ignorePublic || index->local().isPublic()){
00832         
00833         MPI_Pack(const_cast<PairType*>(&(*index)), 1, 
00834                  type,
00835                  p_out, bufferSize, position, comm_);
00836         pairs[i++] = const_cast<PairType*>(&(*index));
00837       
00838       }
00839     assert(i==n);
00840   }
00841   
00842   template<typename T>
00843   inline int RemoteIndices<T>::noPublic(const ParallelIndexSet& indexSet)
00844   {
00845     typedef typename ParallelIndexSet::const_iterator const_iterator;
00846     
00847     int noPublic=0;
00848     
00849     const const_iterator end=indexSet.end();
00850     for(const_iterator index=indexSet.begin(); index!=end; ++index)
00851       if(index->local().isPublic())
00852         noPublic++;
00853     
00854     return noPublic;
00855     
00856   }
00857     
00858 
00859   template<typename T>
00860   template<bool ignorePublic>
00861   inline void RemoteIndices<T>::buildRemote()
00862   {
00863    
00864     // Processor configuration
00865     int rank, procs;
00866     MPI_Comm_rank(comm_, &rank);
00867     MPI_Comm_size(comm_, &procs);
00868     
00869     // number of local indices to publish
00870     // The indices of the destination will be send.
00871     int sourcePublish, destPublish;
00872     
00873     // Do we need to send two index sets?
00874     char sendTwo = (source_ != target_);
00875 
00876     sourcePublish = (ignorePublic)? source_->size() : noPublic(*source_);
00877     
00878     if(sendTwo)
00879       destPublish = (ignorePublic)? target_->size() : noPublic(*target_);
00880     else  
00881       // we only need to send one set of indices
00882       destPublish = 0;
00883         
00884     int maxPublish, publish=sourcePublish+destPublish;
00885 
00886     // Calucate maximum number of indices send
00887     MPI_Allreduce(&publish, &maxPublish, 1, MPI_INT, MPI_MAX, comm_);
00888 
00889     // allocate buffers
00890     typedef IndexPair<GlobalIndex,LocalIndex> PairType;
00891     
00892     PairType** destPairs;
00893     PairType** sourcePairs = new PairType*[sourcePublish];
00894     
00895     if(sendTwo)
00896       destPairs = new PairType*[destPublish];
00897     else
00898       destPairs=sourcePairs;
00899     
00900     char** buffer = new char*[2];
00901     int bufferSize;    
00902     int position=0;
00903     int intSize;
00904     int charSize;
00905 
00906     // calculate buffer size
00907     MPI_Datatype type = MPITraits<PairType>::getType();
00908     
00909     MPI_Pack_size(maxPublish, type, comm_,
00910                   &bufferSize);
00911     MPI_Pack_size(1, MPI_INT, comm_,
00912                   &intSize);
00913     MPI_Pack_size(1, MPI_CHAR, comm_,
00914                   &charSize);
00915     // Our message will contain the following:
00916     // a bool wether two index sets where sent
00917     // the size of the source and the dest indexset, 
00918     // then the source and destination indices
00919     bufferSize += 2 * intSize + charSize;
00920 
00921     buffer[0] = new char[bufferSize];
00922     buffer[1] = new char[bufferSize];
00923 
00924     // send mesages in ring
00925     for(int proc=0; proc<procs; proc++){
00926       // pointers to the current input and output buffers
00927       char* p_out = buffer[1-(proc%2)];
00928       char* p_in = buffer[proc%2];
00929 
00930       if(proc==0 && (procs>0 || sendTwo)){
00931         if(!sendTwo){
00932           // Write two the output buffer that will be used for processor 1
00933           p_out=buffer[0];
00934           assert(p_in==p_out);
00935         }
00936         
00937         
00938         MPI_Pack(&sendTwo, 1, MPI_CHAR, p_out, bufferSize, &position,
00939                  comm_);
00940         
00941         // The number of indices we send for each index set
00942         MPI_Pack(&sourcePublish, 1, MPI_INT, p_out, bufferSize, &position, 
00943                  comm_);
00944         MPI_Pack(&destPublish, 1, MPI_INT, p_out, bufferSize, &position,
00945                  comm_);
00946         //std::cout<<rank<<": ignore="<<ignorePublic<<" sendTwo="<<(sendTwo?true:false)<<" sending: sourcePublish="<<sourcePublish
00947         //               <<", destPublish="<<destPublish<<std::endl;
00948         
00949         // Now pack the source indices and setup the destination pairs
00950         packEntries<ignorePublic>(sourcePairs, *source_, p_out, type, 
00951                                   bufferSize, &position, sourcePublish);
00952         // If necessary send the dest indices and setup the source pairs
00953         if(sendTwo)
00954           packEntries<ignorePublic>(destPairs, *target_, p_out, type,
00955                                     bufferSize, &position, destPublish);
00956         
00957         //std::cout<<rank<<": Publishing "<<sourcePublish<<" source and "
00958         // <<destPublish<<" destination indices!"<<std::endl<<std::flush;
00959         
00960       }
00961       
00962       MPI_Status status;
00963       if(proc==0){
00964         if(sendTwo)
00965           // Makes only sense if send and destination index sets are not the same.
00966         MPI_Sendrecv(p_out, position, MPI_PACKED, rank, commTag_, p_in, bufferSize, MPI_PACKED, rank, commTag_, 
00967                      comm_, &status);
00968         else
00969           continue;
00970       }else if(rank%2==0){
00971         MPI_Ssend(p_out, position, MPI_PACKED, (rank+1)%procs,
00972                   commTag_, comm_);
00973         MPI_Recv(p_in, bufferSize, MPI_PACKED, (rank+procs-1)%procs,
00974                  commTag_, comm_, &status);
00975       }else{
00976         MPI_Recv(p_in, bufferSize, MPI_PACKED, (rank+procs-1)%procs,
00977                  commTag_, comm_, &status);
00978         MPI_Ssend(p_out, position, MPI_PACKED, (rank+1)%procs,
00979                   commTag_, comm_);
00980       }
00981 
00982       // unpack the number of indices we received
00983       int noRemoteSource=-1, noRemoteDest=-1;
00984       char twoIndexSets=0;
00985       position=0;
00986       // Did we receive two index sets?
00987       MPI_Unpack(p_in, bufferSize, &position, &twoIndexSets, 1, MPI_CHAR, comm_);
00988       // The number of source indices received
00989       MPI_Unpack(p_in, bufferSize, &position, &noRemoteSource, 1, MPI_INT, comm_);
00990       // The number of destination indices received
00991       MPI_Unpack(p_in, bufferSize, &position, &noRemoteDest, 1, MPI_INT, comm_);      
00992 
00993       // The process these indices are from
00994       int remoteProc = (rank+procs-proc)%procs;
00995 
00996       //std::cout<<rank<<": received "<<twoIndexSets<<", "<<((twoIndexSets)?1:0)
00997       //       <<", noSource="<<noRemoteSource<<", noDest="<<noRemoteDest
00998       //       <<" from process "<<remoteProc<<std::endl<<std::flush;
00999             
01000       //typedef SLList<RemoteIndex<GlobalIndex,Attribute> > 
01001       //        RemoteIndexList;
01002 
01003       // Indices for which we receive
01004       RemoteIndexList* receive= new RemoteIndexList();
01005       // Indices for which we send
01006       RemoteIndexList* send=0;
01007 
01008       if(!twoIndexSets){
01009         if(sendTwo){
01010           send = new RemoteIndexList();
01011           // Create both remote index sets simultaneously
01012           unpackIndices(*send, *receive, noRemoteSource, sourcePairs, sourcePublish,
01013                         destPairs, destPublish, p_in, type, &position, bufferSize);
01014         }else{
01015           // we only need one list
01016           unpackIndices(*receive, noRemoteSource, sourcePairs, sourcePublish,
01017                         p_in, type, &position, bufferSize);
01018           send=receive;
01019         }
01020       }else{
01021         // Two index sets received
01022         if(sendTwo){
01023           unpackIndices(*receive, noRemoteSource, destPairs, destPublish,
01024                         p_in, type, &position, bufferSize);
01025         }else{
01026           unpackIndices(*receive, noRemoteSource, sourcePairs, sourcePublish,
01027                         p_in, type, &position, bufferSize);
01028         }
01029         
01030         send = new RemoteIndexList();
01031         unpackIndices(*send, noRemoteDest, sourcePairs, sourcePublish, 
01032                       p_in, type, &position, bufferSize);
01033       }
01034       
01035       if(receive->empty() && send->empty()){
01036         if(send==receive){
01037           delete send;
01038         }else{
01039           delete send;
01040           delete receive;
01041         }
01042       }else{
01043         remoteIndices_.insert(std::make_pair(remoteProc, 
01044                                              std::make_pair(send,receive)));
01045       }
01046     }
01047     MPI_Barrier(comm_);
01048     
01049     // delete allocated memory
01050     if(destPairs!=sourcePairs)
01051       delete[] destPairs;
01052     
01053     delete[] sourcePairs;
01054     delete[] buffer[0];
01055     delete[] buffer[1];
01056     delete[] buffer;
01057   }
01058   
01059   template<typename T>
01060   inline void RemoteIndices<T>::unpackIndices(RemoteIndexList& remote,
01061                                                   int remoteEntries,
01062                                                   PairType** local,
01063                                                   int localEntries,
01064                                                   char* p_in,
01065                                                   MPI_Datatype type,
01066                                                   int* position,
01067                                                   int bufferSize)
01068   {
01069     if(remoteEntries==0)
01070       return;
01071   
01072     PairType index;
01073     MPI_Unpack(p_in, bufferSize, position, &index, 1, 
01074                type, comm_);
01075         
01076     int n_in=0, localIndex=0;
01077         
01078     //Check if we know the global index
01079     while(localIndex<localEntries){
01080       if(local[localIndex]->global()==index.global()){
01081         remote.push_back(RemoteIndex(index.local().attribute(), 
01082                                             local[localIndex]));
01083         localIndex++;
01084         // unpack next remote index
01085         if((++n_in) < remoteEntries){
01086           MPI_Unpack(p_in, bufferSize, position, &index, 1, 
01087                      type, comm_);
01088         }else{
01089           // No more received indices
01090           break;
01091         }
01092       }else if (local[localIndex]->global()<index.global()){
01093         ++localIndex;
01094       }else{
01095         // We do not know the index, unpack next
01096         if((++n_in) < remoteEntries){
01097           MPI_Unpack(p_in, bufferSize, position, &index, 1, 
01098                      type, comm_);
01099         }else
01100           // No more received indices
01101           break;
01102       }
01103     }
01104     
01105     // Unpack the other received indices without doing anything
01106     while(++n_in < remoteEntries)
01107       MPI_Unpack(p_in, bufferSize, position, &index, 1, 
01108                type, comm_);
01109   }
01110   
01111     
01112   template<typename T>
01113   inline void RemoteIndices<T>::unpackIndices(RemoteIndexList& send,
01114                                                   RemoteIndexList& receive,
01115                                                   int remoteEntries,
01116                                                   PairType** localSource,
01117                                                   int localSourceEntries,
01118                                                   PairType** localDest,
01119                                                   int localDestEntries,
01120                                                   char* p_in,
01121                                                   MPI_Datatype type,
01122                                                   int* position,
01123                                                   int bufferSize)
01124   {             
01125     int n_in=0, sourceIndex=0, destIndex=0;
01126         
01127     //Check if we know the global index
01128     while(n_in<remoteEntries && (sourceIndex<localSourceEntries || destIndex<localDestEntries)){
01129       // Unpack next index
01130       PairType index;
01131       MPI_Unpack(p_in, bufferSize, position, &index, 1, 
01132                  type, comm_);
01133       n_in++;
01134       
01135       // Advance until global index in localSource and localDest are >= than the one in the unpacked index
01136       while(sourceIndex<localSourceEntries && localSource[sourceIndex]->global()<index.global())
01137         sourceIndex++;
01138       
01139       while(destIndex<localDestEntries && localDest[destIndex]->global()<index.global())
01140         destIndex++;
01141 
01142       // Add a remote index if we found the global index.
01143       if(sourceIndex<localSourceEntries && localSource[sourceIndex]->global()==index.global())
01144           send.push_back(RemoteIndex(index.local().attribute(), 
01145                                               localSource[sourceIndex]));
01146 
01147       if(destIndex < localDestEntries && localDest[destIndex]->global() == index.global())
01148           receive.push_back(RemoteIndex(index.local().attribute(), 
01149                                         localDest[sourceIndex]));
01150     }
01151     
01152   }
01153     
01154   template<typename T>
01155   inline void RemoteIndices<T>::free()
01156   {
01157     typedef typename RemoteIndexMap::iterator Iterator;
01158     Iterator lend = remoteIndices_.end();
01159     for(Iterator lists=remoteIndices_.begin(); lists != lend; ++lists){
01160       if(lists->second.first==lists->second.second){
01161         // there is only one remote index list.
01162         delete lists->second.first;
01163       }else{
01164         delete lists->second.first;
01165         delete lists->second.second;
01166       }
01167     }
01168     remoteIndices_.clear();
01169   }
01170 
01171   template<typename T>
01172   inline int RemoteIndices<T>::neighbours() const
01173   {
01174     return remoteIndices_.size();
01175   }
01176   
01177   template<typename T>
01178   template<bool ignorePublic>
01179   inline void RemoteIndices<T>::rebuild()
01180   {
01181     // Test wether a rebuild is Needed.
01182     if(firstBuild || 
01183        ignorePublic!=publicIgnored || !
01184        isSynced()){
01185       free();
01186       
01187       buildRemote<ignorePublic>();
01188 
01189       sourceSeqNo_ = source_->seqNo();
01190       destSeqNo_ = target_->seqNo();
01191       firstBuild=false;
01192       publicIgnored=ignorePublic;
01193     }
01194     
01195     
01196   }
01197   
01198   template<typename T>
01199   inline bool RemoteIndices<T>::isSynced() const
01200   {
01201     return sourceSeqNo_==source_->seqNo() && destSeqNo_ ==target_->seqNo();
01202   }
01203 
01204   template<typename T>
01205   template<bool mode, bool send>
01206   RemoteIndexListModifier<T,mode> RemoteIndices<T>::getModifier(int process)
01207   {
01208     typename RemoteIndexMap::iterator found = remoteIndices_.find(process);
01209     
01210     if(found == remoteIndices_.end())
01211       if(source_ != target_)
01212         remoteIndices_.insert(std::make_pair(process,
01213                                              std::make_pair(new RemoteIndexList(), 
01214                                                             new RemoteIndexList())));
01215       else{
01216         RemoteIndexList* rlist = new RemoteIndexList();
01217         remoteIndices_.insert(std::make_pair(process,
01218                                              std::make_pair(rlist, rlist)));
01219         
01220         found = remoteIndices_.find(process);
01221       }
01222     
01223     firstBuild = false;
01224     
01225     if(send)
01226       return RemoteIndexListModifier<T,mode>(*source_, *(found->second.first));
01227     else
01228       return RemoteIndexListModifier<T,mode>(*target_, *(found->second.second));
01229   }
01230   
01231   template<typename T>
01232   inline typename std::map<int, std::pair<SLList<typename RemoteIndices<T>::RemoteIndex,
01233                                                  PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*,
01234                                           SLList<typename RemoteIndices<T>::RemoteIndex,
01235                                                  PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*> >::const_iterator
01236   RemoteIndices<T>::begin() const
01237   {
01238     return remoteIndices_.begin();
01239   }
01240   
01241   template<typename T>
01242   inline typename std::map<int, std::pair<SLList<typename RemoteIndices<T>::RemoteIndex,
01243                                                  PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*,
01244                                           SLList<typename RemoteIndices<T>::RemoteIndex,
01245                                                  PoolAllocator<typename RemoteIndices<T>::RemoteIndex,1> >*> >::const_iterator
01246   RemoteIndices<T>::end() const
01247   {
01248     return remoteIndices_.end();
01249   }
01250 
01251   template<class T, bool mode>
01252   RemoteIndexListModifier<T,mode>::RemoteIndexListModifier(const ParallelIndexSet& indexSet,
01253                                                            RemoteIndexList& rList)
01254     : rList_(&rList), indexSet_(&indexSet), glist_(new GlobalList()), iter_(rList.beginModify()), end_(rList.end()), first_(true)
01255   {
01256     if(MODIFYINDEXSET){
01257       assert(indexSet_);
01258       for(ConstIterator iter=iter_; iter != end_; ++iter)
01259         glist_->push_back(iter->localIndexPair().global());
01260       giter_ = glist_->beginModify();
01261     }
01262   }
01263 
01264   template<typename T, bool mode>
01265   RemoteIndexListModifier<T,mode>::RemoteIndexListModifier(const RemoteIndexListModifier<T,mode>& other)
01266     : rList_(other.rList_), indexSet_(other.indexSet_), 
01267       glist_(other.glist_), iter_(other.iter_), giter_(other.giter_), end_(other.end_), 
01268       first_(other.first_), last_(other.last_)
01269   {}
01270     
01271   template<typename T, bool mode>
01272   inline void RemoteIndexListModifier<T,mode>::repairLocalIndexPointers() throw(InvalidIndexSetState)
01273   {
01274     IsTrue<mode>::yes();
01275     
01276     if(MODIFYINDEXSET){
01277       // repair pointers to local index set.
01278 #ifdef DUNE_ISTL_WITH_CHECKING
01279       if(indexSet_->state()!=GROUND)
01280         DUNE_THROW(InvalidIndexSetState, "Index has to be in ground mode for repairing pointers to indices");
01281 #endif
01282       typedef typename ParallelIndexSet::const_iterator IndexIterator;
01283       typedef typename GlobalList::const_iterator GlobalIterator;
01284       typedef typename RemoteIndexList::iterator Iterator;
01285       GlobalIterator giter = glist_->begin();
01286       IndexIterator index = indexSet_->begin();
01287       
01288       for(Iterator iter=rList_->begin(); iter != end_; ++iter){
01289         while(index->global()<*giter){
01290           ++index;
01291 #ifdef DUNE_ISTL_WITH_CHECKING
01292           if(index == indexSet_.end())
01293             DUNE_THROW(ISTLError, "No such global index in set!");
01294 #endif
01295         }
01296 
01297 #ifdef DUNE_ISTL_WITH_CHECKING
01298           if(index->global() != *giter)
01299             DUNE_THROW(ISTLError, "No such global index in set!");
01300 #endif
01301           iter->localIndex_ = &(*index);
01302       }
01303     }
01304   }
01305   
01306   template<typename T, bool mode>
01307   inline void RemoteIndexListModifier<T,mode>::insert(const RemoteIndex& index) throw(InvalidPosition)
01308   {
01309     IsTrue<mode>::no();
01310     
01311 #ifdef DUNE_ISTL_WITH_CHECKING
01312     if(!first_ && index.localIndexPair().global()<last_)
01313       DUNE_THROW(InvalidPosition, "Modifcation of remote indices have to occur with ascending global index.");
01314 #endif
01315     // Move to the correct position
01316     while(iter_ != end_ && iter_->localIndexPair().global() < index.localIndexPair().global()){
01317       ++iter_;
01318     }
01319     
01320     // No duplicate entries allowed
01321     assert(iter_==end_ || iter_->localIndexPair().global() != index.localIndexPair().global());
01322     iter_.insert(index);
01323     last_ = index.localIndexPair().global();
01324     first_ = false;
01325   }
01326   
01327   template<typename T, bool mode>
01328   inline void RemoteIndexListModifier<T,mode>::insert(const RemoteIndex& index, const GlobalIndex& global) throw(InvalidPosition)
01329   {
01330     typename IsTrue<mode>::yes();
01331 #ifdef DUNE_ISTL_WITH_CHECKING
01332     if(!first_ && global<last_)
01333       DUNE_THROW(InvalidPosition, "Modification of remote indices have to occur with ascending global index.");
01334 #endif
01335     // Move to the correct position
01336     while(iter_ != end_ && *giter_ < global){
01337       ++giter_;
01338       ++iter_;
01339     }
01340     
01341     // No duplicate entries allowed
01342     assert(iter_->localIndexPair().global() != global);
01343     iter_.insert(index);
01344     giter_.insert(global);
01345     
01346     last_ = global;
01347     first_ = false;
01348   }
01349   
01350   template<typename T, bool mode>
01351   bool RemoteIndexListModifier<T,mode>::remove(const GlobalIndex& global) throw(InvalidPosition)
01352   {
01353 #ifdef DUNE_ISTL_WITH_CHECKING
01354     if(!first_ && global<last_)
01355       DUNE_THROW(InvalidPosition, "Modifcation of remote indices have to occur with ascending global index.");
01356 #endif
01357 
01358     bool found= false;
01359     
01360     if(MODIFYINDEXSET){
01361     // Move to the correct position
01362       while(iter_!=end_ && *giter_< global){
01363         ++giter_;
01364         ++iter_;
01365       }
01366       if(*giter_ == global){
01367         giter_.remove();
01368         iter_.remove();
01369         found=true;
01370       }
01371     }else{
01372       while(iter_!=end_ && iter_->localIndexPair().global() < global)
01373         ++iter_;
01374         
01375       if(iter_->localIndexPair().global()==global){
01376         iter_.remove();
01377         found = true;
01378       }
01379     }
01380     
01381     last_ = global;
01382     first_ = false;
01383     return found;
01384   }
01385 
01386   template<typename T>
01387   template<bool send>
01388   inline CollectiveIterator<T> RemoteIndices<T>::iterator() const
01389   {
01390     return CollectiveIterator<T>(remoteIndices_, send);
01391   }
01392 
01393   template<typename T>
01394   inline MPI_Comm RemoteIndices<T>::communicator() const
01395   {
01396     return comm_;
01397     
01398   }
01399   
01400   template<typename T>
01401   CollectiveIterator<T>::CollectiveIterator(const RemoteIndexMap& pmap, bool send)
01402   {
01403     typedef typename RemoteIndexMap::const_iterator const_iterator;
01404     typedef typename RemoteIndexMap::iterator iterator;
01405     
01406     const const_iterator end=pmap.end();
01407     for(const_iterator process=pmap.begin(); process != end; ++process){
01408       const RemoteIndexList* list = send? process->second.first : process->second.second;
01409       typedef typename RemoteIndexList::const_iterator iterator;
01410       map_.insert(std::make_pair(process->first, 
01411                                  std::pair<iterator, const iterator>(list->begin(), list->end())));
01412     }
01413   }
01414 
01415   template<typename T>
01416   inline void CollectiveIterator<T>::advance(const GlobalIndex& index)
01417   {
01418     typedef typename Map::iterator iterator;
01419     typedef typename Map::const_iterator const_iterator;
01420     const const_iterator end = map_.end();
01421     
01422     for(iterator iter = map_.begin(); iter != end;){
01423       // Step the iterator until we are >= index
01424       typename RemoteIndexList::const_iterator current = iter->second.first;
01425       typename RemoteIndexList::const_iterator rend = iter->second.second;
01426       RemoteIndex remoteIndex;
01427       if(current != rend)
01428         remoteIndex = *current;
01429       
01430       while(iter->second.first!=iter->second.second && iter->second.first->localIndexPair().global()<index)
01431         ++(iter->second.first);
01432       
01433       // erase from the map if there are no more entries.
01434       if(iter->second.first == iter->second.second)
01435         map_.erase(iter++);
01436       else{
01437         ++iter;
01438       }
01439     }
01440     index_=index;
01441   }
01442   
01443   template<typename T>
01444   inline bool CollectiveIterator<T>::empty()
01445   {
01446     return map_.empty();
01447   }
01448     
01449   template<typename T>
01450   inline typename CollectiveIterator<T>::iterator
01451   CollectiveIterator<T>::begin()
01452   {
01453     return iterator(map_.begin(), map_.end(), index_);
01454   }
01455    
01456   template<typename T>
01457   inline typename CollectiveIterator<T>::iterator
01458   CollectiveIterator<T>::end()
01459   {
01460     return iterator(map_.end(), map_.end(), index_);
01461   }
01462   
01463   template<typename TG, typename TA>
01464   inline std::ostream& operator<<(std::ostream& os, const RemoteIndex<TG,TA>& index)
01465   {
01466     os<<"[global="<<index.localIndexPair().global()<<",attribute="<<index.attribute()<<"]";
01467     return os;
01468   }
01469   
01470   template<typename T>
01471   inline std::ostream& operator<<(std::ostream& os, const RemoteIndices<T>& indices)
01472   {
01473     int rank;
01474     MPI_Comm_rank(indices.comm_, &rank);
01475 
01476     typedef typename RemoteIndices<T>::RemoteIndexList RList;
01477     typedef typename std::map<int,std::pair<RList*,RList*> >::const_iterator const_iterator;
01478 
01479     const const_iterator rend = indices.remoteIndices_.end();
01480 
01481     for(const_iterator rindex = indices.remoteIndices_.begin(); rindex!=rend; ++rindex){
01482       os<<rank<<": Prozess "<<rindex->first<<":";
01483       
01484       if(!rindex->second.first->empty()){
01485         os<<" send:";
01486 
01487         const typename RList::const_iterator send= rindex->second.first->end();
01488       
01489         for(typename RList::const_iterator index = rindex->second.first->begin(); 
01490             index != send; ++index)
01491           os<<*index<<" ";
01492         os<<std::endl;
01493       }
01494       if(!rindex->second.second->empty()){
01495         os<<rank<<": Prozess "<<rindex->first<<": "<<"receive: ";
01496         
01497         const typename RList::const_iterator rend= rindex->second.second->end();
01498         
01499         for(typename RList::const_iterator index = rindex->second.second->begin(); 
01500             index != rend; ++index)
01501           os<<*index<<" ";
01502       }
01503       os<<std::endl<<std::flush;
01504     }
01505     return os;
01506   }
01508 }
01509 
01510 #endif
01511 #endif

Generated on 9 Apr 2008 with Doxygen (ver 1.5.2) [logfile].