owneroverlapcopy.hh

Go to the documentation of this file.
00001 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
00002 // vi: set ts=4 sw=2 et sts=2:
00003 // $Id$
00004 #ifndef DUNE_OWNEROVERLAPCOPY_HH
00005 #define DUNE_OWNEROVERLAPCOPY_HH
00006 
00007 #include<new>
00008 #include<iostream>
00009 #include<vector>
00010 #include<list>
00011 #include<map>
00012 #include<set>
00013 
00014 #include"cmath"
00015 
00016 // MPI header
00017 #if HAVE_MPI
00018 #include<mpi.h>
00019 #endif
00020 
00021 
00022 #include<dune/common/tuples.hh>
00023 #include<dune/common/enumset.hh>
00024 
00025 #if HAVE_MPI
00026 #include <dune/common/parallel/indexset.hh>
00027 #include <dune/common/parallel/communicator.hh>
00028 #include <dune/common/parallel/remoteindices.hh>
00029 #include<dune/common/mpicollectivecommunication.hh>
00030 #endif
00031 
00032 #include"solvercategory.hh"
00033 #include"istlexception.hh"
00034 #include<dune/common/collectivecommunication.hh>
00035 
00036 template<int dim, template<class,class> class Comm>
00037 void testRedistributed(int s);
00038 
00039 
00040 namespace Dune {
00041    
00057   struct OwnerOverlapCopyAttributeSet
00058   {
00059         enum AttributeSet { 
00060           owner=1, overlap=2, copy=3 };
00061   };
00062 
00074   template <class G, class L>
00075   class IndexInfoFromGrid
00076   {
00077   public:
00079     typedef G GlobalIdType;
00080 
00082     typedef L LocalIdType;
00083     
00090         typedef tuple<GlobalIdType,LocalIdType,int> IndexTripel;
00097         typedef tuple<int,GlobalIdType,int> RemoteIndexTripel;
00098 
00104         void addLocalIndex (const IndexTripel& x)
00105         {
00106           if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 
00107                   get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 
00108                   get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00109                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00110           localindices.insert(x);
00111         }
00112 
00118         void addRemoteIndex (const RemoteIndexTripel& x)
00119         {
00120           if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 
00121                   get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 
00122                   get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00123                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00124           remoteindices.insert(x);
00125         }
00126 
00131         const std::set<IndexTripel>& localIndices () const
00132         {
00133           return localindices;
00134         }
00135 
00140         const std::set<RemoteIndexTripel>& remoteIndices () const
00141         {
00142           return remoteindices;
00143         }
00144 
00148         void clear ()
00149         {
00150           localindices.clear();
00151           remoteindices.clear();
00152         }
00153 
00154   private:
00156         std::set<IndexTripel> localindices;
00158         std::set<RemoteIndexTripel> remoteindices;
00159   };
00160 
00161 
00162 #if HAVE_MPI
00163 
00170   template <class GlobalIdType, class LocalIdType=int>
00171   class OwnerOverlapCopyCommunication
00172   {    
00173         // used types
00174         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel;
00175         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel;
00176         typedef typename std::set<IndexTripel>::const_iterator localindex_iterator;
00177         typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator;
00178     typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet;
00179         typedef Dune::ParallelLocalIndex<AttributeSet> LI;
00180   public:
00181         typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS;
00182         typedef Dune::RemoteIndices<PIS> RI;
00183     typedef Dune::RemoteIndexListModifier<PIS,typename RI::Allocator,false> RILM;
00184         typedef typename RI::RemoteIndex RX;
00185         typedef Dune::BufferedCommunicator BC;
00186         typedef Dune::Interface IF;
00187     typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet;
00188     typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopySet;
00189     typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00190     typedef Dune::AllSet<AttributeSet> AllSet;
00191   protected:
00192 
00193     
00195         template<typename T>
00196         struct CopyGatherScatter
00197         {
00198           typedef typename CommPolicy<T>::IndexedType V;
00199 
00200           static V gather(const T& a, std::size_t i)
00201           {
00202                 return a[i];
00203           }
00204           
00205           static void scatter(T& a, V v, std::size_t i)
00206           {
00207                 a[i] = v;
00208           }
00209         };
00210         template<typename T>
00211         struct AddGatherScatter
00212         {
00213           typedef typename CommPolicy<T>::IndexedType V;
00214 
00215           static V gather(const T& a, std::size_t i)
00216           {
00217                 return a[i];
00218           }
00219           
00220           static void scatter(T& a, V v, std::size_t i)
00221           {
00222                 a[i] += v;
00223           }
00224         };
00225 
00226         void buildOwnerOverlapToAllInterface () const
00227         {
00228           if (OwnerOverlapToAllInterfaceBuilt)
00229                 OwnerOverlapToAllInterface.free();
00230           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00231           typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00232           OwnerOverlapSet sourceFlags;
00233           AllSet destFlags;
00234           OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags);
00235           OwnerOverlapToAllInterfaceBuilt = true;
00236         }
00237 
00238         void buildOwnerToAllInterface () const
00239         {
00240           if (OwnerToAllInterfaceBuilt)
00241                 OwnerToAllInterface.free();
00242           OwnerSet sourceFlags;
00243           AllSet destFlags;
00244           OwnerToAllInterface.build(ri,sourceFlags,destFlags);
00245           OwnerToAllInterfaceBuilt = true;
00246         }
00247 
00248         void buildOwnerCopyToAllInterface () const
00249         {
00250           if (OwnerCopyToAllInterfaceBuilt)
00251                 OwnerCopyToAllInterface.free();
00252           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet;
00253           typedef Combine<OwnerCopySet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> AllSet;
00254           OwnerCopySet sourceFlags;
00255           AllSet destFlags;
00256           OwnerCopyToAllInterface.build(ri,sourceFlags,destFlags);
00257           OwnerCopyToAllInterfaceBuilt = true;
00258         }
00259 
00260         void buildOwnerCopyToOwnerCopyInterface () const
00261         {
00262           if (OwnerCopyToOwnerCopyInterfaceBuilt)
00263                 OwnerCopyToOwnerCopyInterface.free();
00264           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet;
00265           OwnerCopySet sourceFlags;
00266           OwnerCopySet destFlags;
00267           OwnerCopyToOwnerCopyInterface.build(ri,sourceFlags,destFlags);
00268           OwnerCopyToOwnerCopyInterfaceBuilt = true;
00269         }
00270 
00271   public:
00272 
00277     void setSolverCategory (SolverCategory set) {
00278       category = set;
00279     }
00280 
00286     const SolverCategory::Category getSolverCategory () const {
00287       return category;
00288     }
00289 
00290     const CollectiveCommunication<MPI_Comm>& communicator() const
00291     {
00292       return cc;
00293     }
00294     
00301         template<class T>
00302         void copyOwnerToAll (const T& source, T& dest) const
00303         {
00304           if (!OwnerToAllInterfaceBuilt)
00305                 buildOwnerToAllInterface ();
00306           BC communicator;
00307           communicator.template build<T>(OwnerToAllInterface);
00308           communicator.template forward<CopyGatherScatter<T> >(source,dest);
00309           communicator.free();
00310         }
00311 
00312 
00319         template<class T>
00320         void addOwnerOverlapToAll (const T& source, T& dest) const
00321         {
00322           if (!OwnerOverlapToAllInterfaceBuilt)
00323                 buildOwnerOverlapToAllInterface ();
00324           BC communicator;
00325           communicator.template build<T>(OwnerOverlapToAllInterface);
00326           communicator.template forward<AddGatherScatter<T> >(source,dest);
00327           communicator.free();
00328         }
00329 
00336         template<class T>
00337         void addOwnerCopyToAll (const T& source, T& dest) const
00338     {
00339           if (!OwnerCopyToAllInterfaceBuilt)
00340                 buildOwnerCopyToAllInterface ();
00341           BC communicator;
00342           communicator.template build<T>(OwnerCopyToAllInterface);
00343           communicator.template forward<AddGatherScatter<T> >(source,dest);
00344           communicator.free();
00345         }
00346 
00353         template<class T>
00354         void addOwnerCopyToOwnerCopy (const T& source, T& dest) const
00355         {
00356           if (!OwnerCopyToOwnerCopyInterfaceBuilt)
00357                 buildOwnerCopyToOwnerCopyInterface ();
00358           BC communicator;
00359           communicator.template build<T>(OwnerCopyToOwnerCopyInterface);
00360           communicator.template forward<AddGatherScatter<T> >(source,dest);
00361           communicator.free();
00362         }
00363 
00364 
00372         template<class T1, class T2>
00373         void dot (const T1& x, const T1& y, T2& result) const
00374         {
00375           // set up mask vector
00376           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00377                 {
00378                   mask.resize(x.size());
00379                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00380                     mask[i] = 1;
00381                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00382                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00383                           mask[i->local().local()] = 0;
00384                 }
00385           result = 0;
00386 
00387           for (typename T1::size_type i=0; i<x.size(); i++)
00388                 result += x[i]*(y[i])*mask[i];
00389           result = cc.sum(result);
00390           return;
00391         }
00392 
00399         template<class T1>
00400         double norm (const T1& x) const
00401         {
00402           // set up mask vector
00403           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00404                 {
00405                   mask.resize(x.size());
00406                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00407                     mask[i] = 1;
00408                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00409                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00410                           mask[i->local().local()] = 0;
00411                 }
00412           double result = 0;
00413           for (typename T1::size_type i=0; i<x.size(); i++)
00414                 result += x[i].two_norm2()*mask[i];
00415           return sqrt(cc.sum(result));
00416         }
00417 
00418     typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags;
00419     
00421     typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet;
00422 
00424     typedef Dune::RemoteIndices<PIS> RemoteIndices;
00425 
00428     typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet;
00429 
00434     const ParallelIndexSet& indexSet() const
00435     {
00436       return pis;
00437     }
00438     
00443     const RemoteIndices& remoteIndices() const
00444     {
00445       return ri;
00446     }
00447     
00452     ParallelIndexSet& indexSet()
00453     {
00454       return pis;
00455     }
00456     
00457 
00462     RemoteIndices& remoteIndices()
00463     {
00464       return ri;
00465     }
00466 
00467     void buildGlobalLookup()
00468     {
00469       if(globalLookup_){
00470           if(pis.seqNo()==oldseqNo)
00471             // Nothing changed!
00472             return;
00473           delete globalLookup_;
00474       }
00475       
00476       globalLookup_ = new GlobalLookupIndexSet(pis);
00477       oldseqNo = pis.seqNo();
00478     }
00479     
00480     void buildGlobalLookup(std::size_t size)
00481     {
00482       if(globalLookup_){
00483           if(pis.seqNo()==oldseqNo)
00484             // Nothing changed!
00485             return;
00486           delete globalLookup_;
00487       }
00488       globalLookup_ = new GlobalLookupIndexSet(pis, size);
00489       oldseqNo = pis.seqNo();
00490     }
00491 
00492     void freeGlobalLookup()
00493     {
00494       delete globalLookup_;
00495       globalLookup_=0;
00496     }    
00497 
00498     const GlobalLookupIndexSet& globalLookup() const
00499     {
00500       assert(globalLookup_ != 0);
00501       return *globalLookup_;
00502     }
00503     
00509         template<class T1>
00510         void project (T1& x) const
00511         {
00512           for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00513                 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy)
00514                   x[i->local().local()] = 0;
00515         }
00516 
00524     OwnerOverlapCopyCommunication (MPI_Comm comm_, SolverCategory::Category cat = SolverCategory::overlapping)
00525       : cc(comm_), pis(), ri(pis,pis,comm_), 
00526         OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 
00527         OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 
00528         globalLookup_(0), category(cat)
00529     {}
00530     
00537     OwnerOverlapCopyCommunication (SolverCategory::Category cat = SolverCategory::overlapping)
00538       : cc(MPI_COMM_WORLD), pis(), ri(pis,pis,MPI_COMM_WORLD), 
00539         OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 
00540         OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 
00541         globalLookup_(0), category(cat)
00542     {}
00543 
00549         OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType, LocalIdType>& indexinfo, MPI_Comm comm_, SolverCategory::Category cat = SolverCategory::overlapping)
00550           : cc(comm_), OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 
00551         OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 
00552         globalLookup_(0), category(cat)
00553         {
00554           // set up an ISTL index set
00555           pis.beginResize();
00556           for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i)
00557                 {
00558                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00559                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true));
00560                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00561                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true));
00562                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00563                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true));
00564 //                std::cout << cc.rank() << ": adding index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl;
00565                 }
00566           pis.endResize();
00567 
00568           // build remote indices WITHOUT communication
00569 //        std::cout << cc.rank() << ": build remote indices" << std::endl;
00570           ri.setIndexSets(pis,pis,cc);
00571           if (indexinfo.remoteIndices().size()>0)
00572                 {
00573                   remoteindex_iterator i=indexinfo.remoteIndices().begin();
00574                   int p = get<0>(*i);
00575                   RILM modifier = ri.template getModifier<false,true>(p);
00576                   typename PIS::const_iterator pi=pis.begin();
00577                   for ( ; i!=indexinfo.remoteIndices().end(); ++i)
00578                         {
00579                           // handle processor change
00580                           if (p!=get<0>(*i))
00581                                 {
00582                                   p = get<0>(*i);
00583                                   modifier = ri.template getModifier<false,true>(p);
00584                                   pi=pis.begin();
00585                                 }
00586                           
00587                           // position to correct entry in parallel index set
00588                           while (pi->global()!=get<1>(*i) && pi!=pis.end())
00589                                 ++pi;
00590                           if (pi==pis.end())
00591                                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00592                           
00593                           // insert entry
00594 //                        std::cout << cc.rank() << ": adding remote index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl;
00595                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00596                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi)));
00597                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00598                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi)));
00599                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00600                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi)));
00601                         }
00602                 }else{
00603                   // Force remote indices to be synced!
00604                   ri.template getModifier<false,true>(0);
00605                 }
00606         }
00607 
00608         // destructor: free memory in some objects
00609         ~OwnerOverlapCopyCommunication ()
00610         {
00611           ri.free();
00612           if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free();
00613           if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free();
00614           if (OwnerCopyToAllInterfaceBuilt) OwnerCopyToAllInterface.free();
00615           if (OwnerCopyToOwnerCopyInterfaceBuilt) OwnerCopyToOwnerCopyInterface.free();
00616           if (globalLookup_) delete globalLookup_;
00617         }
00618 
00619   private:
00620     OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&)
00621     {}
00622         CollectiveCommunication<MPI_Comm> cc;
00623         PIS pis;
00624         RI ri;
00625         mutable IF OwnerToAllInterface;
00626         mutable bool OwnerToAllInterfaceBuilt;
00627         mutable IF OwnerOverlapToAllInterface;
00628         mutable bool OwnerOverlapToAllInterfaceBuilt;
00629         mutable IF OwnerCopyToAllInterface;
00630         mutable bool OwnerCopyToAllInterfaceBuilt;
00631     mutable IF OwnerCopyToOwnerCopyInterface;
00632         mutable bool OwnerCopyToOwnerCopyInterfaceBuilt;
00633         mutable std::vector<double> mask;
00634     int oldseqNo;
00635     GlobalLookupIndexSet* globalLookup_;
00636     SolverCategory::Category category;
00637   };
00638 
00639 #endif
00640 
00641  
00644 } // end namespace
00645 
00646 #endif

Generated on Fri Apr 29 2011 with Doxygen (ver 1.7.1) [doxygen-log,error-log].