owneroverlapcopy.hh

Go to the documentation of this file.
00001 // $Id: owneroverlapcopy.hh 803 2007-08-15 14:06:45Z mblatt $
00002 #ifndef DUNE_OWNEROVERLAPCOPY_HH
00003 #define DUNE_OWNEROVERLAPCOPY_HH
00004 
00005 #include<new>
00006 #include<iostream>
00007 #include<vector>
00008 #include<list>
00009 #include<map>
00010 #include<set>
00011 
00012 #include"math.h"
00013 
00014 // MPI header
00015 #if HAVE_MPI
00016 #include<mpi.h>
00017 #endif
00018 
00019 
00020 #include<dune/common/tripel.hh>
00021 #include<dune/common/enumset.hh>
00022 
00023 #if HAVE_MPI
00024 #include"indexset.hh"
00025 #include"communicator.hh"
00026 #include"remoteindices.hh"
00027 #include<dune/common/mpicollectivecommunication.hh>
00028 #endif
00029 
00030 #include"solvercategory.hh"
00031 #include"istlexception.hh"
00032 #include<dune/common/collectivecommunication.hh>
00033 
00034 namespace Dune {
00035    
00051   struct OwnerOverlapCopyAttributeSet
00052   {
00053         enum AttributeSet { 
00054           owner=1, overlap=2, copy=0 };
00055   };
00056 
00068   template <class G, class L>
00069   class IndexInfoFromGrid
00070   {
00071   public:
00073     typedef G GlobalIdType;
00074 
00076     typedef L LocalIdType;
00077     
00084         typedef tripel<GlobalIdType,LocalIdType,int> IndexTripel;
00091         typedef tripel<int,GlobalIdType,int> RemoteIndexTripel;
00092 
00098         void addLocalIndex (const IndexTripel& x)
00099         {
00100           if (x.third!=OwnerOverlapCopyAttributeSet::owner && 
00101                   x.third!=OwnerOverlapCopyAttributeSet::overlap && 
00102                   x.third!=OwnerOverlapCopyAttributeSet::copy)
00103                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00104           localindices.insert(x);
00105         }
00106 
00112         void addRemoteIndex (const RemoteIndexTripel& x)
00113         {
00114           if (x.third!=OwnerOverlapCopyAttributeSet::owner && 
00115                   x.third!=OwnerOverlapCopyAttributeSet::overlap && 
00116                   x.third!=OwnerOverlapCopyAttributeSet::copy)
00117                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00118           remoteindices.insert(x);
00119         }
00120 
00125         const std::set<IndexTripel>& localIndices () const
00126         {
00127           return localindices;
00128         }
00129 
00134         const std::set<RemoteIndexTripel>& remoteIndices () const
00135         {
00136           return remoteindices;
00137         }
00138 
00142         void clear ()
00143         {
00144           localindices.clear();
00145           remoteindices.clear();
00146         }
00147 
00148   private:
00150         std::set<IndexTripel> localindices;
00152         std::set<RemoteIndexTripel> remoteindices;
00153   };
00154 
00155 
00156 #if HAVE_MPI
00157 
00164   template <class GlobalIdType, class LocalIdType>
00165   class OwnerOverlapCopyCommunication
00166   {
00167         // used types
00168         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel;
00169         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel;
00170         typedef typename std::set<IndexTripel>::const_iterator localindex_iterator;
00171         typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator;
00172     typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet;
00173         typedef ParallelLocalIndex<AttributeSet> LI;
00174         typedef ParallelIndexSet<GlobalIdType,LI,512> PIS;
00175         typedef RemoteIndices<PIS> RI;
00176         typedef RemoteIndexListModifier<PIS,false> RILM;
00177         typedef typename RI::RemoteIndex RX;
00178         typedef BufferedCommunicator<PIS> BC;
00179         typedef Interface<PIS> IF;
00180 
00181     
00183         template<typename T>
00184         struct CopyGatherScatter
00185         {
00186           typedef typename CommPolicy<T>::IndexedType V;
00187 
00188           static V gather(const T& a, int i)
00189           {
00190                 return a[i];
00191           }
00192           
00193           static void scatter(T& a, V v, int i)
00194           {
00195                 a[i] = v;
00196           }
00197         };
00198         template<typename T>
00199         struct AddGatherScatter
00200         {
00201           typedef typename CommPolicy<T>::IndexedType V;
00202 
00203           static V gather(const T& a, int i)
00204           {
00205                 return a[i];
00206           }
00207           
00208           static void scatter(T& a, V v, int i)
00209           {
00210                 a[i] += v;
00211           }
00212         };
00213 
00214         void buildOwnerOverlapToAllInterface () const
00215         {
00216           if (OwnerOverlapToAllInterfaceBuilt)
00217                 OwnerOverlapToAllInterface.free();
00218           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00219           typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00220           OwnerOverlapSet sourceFlags;
00221           AllSet destFlags;
00222           OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags);
00223           OwnerOverlapToAllInterfaceBuilt = true;
00224         }
00225 
00226         void buildOwnerToAllInterface () const
00227         {
00228           if (OwnerToAllInterfaceBuilt)
00229                 OwnerToAllInterface.free();
00230           typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet;
00231           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00232           typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00233           OwnerOverlapSet sourceFlags;
00234           AllSet destFlags;
00235           OwnerToAllInterface.build(ri,sourceFlags,destFlags);
00236           OwnerToAllInterfaceBuilt = true;
00237         }
00238 
00239   public:
00240       enum{
00241         category = SolverCategory::overlapping
00242           };
00243 
00244     const CollectiveCommunication<MPI_Comm>& communicator() const
00245     {
00246       return cc;
00247     }
00248     
00255         template<class T>
00256         void copyOwnerToAll (const T& source, T& dest) const
00257         {
00258           if (!OwnerToAllInterfaceBuilt)
00259                 buildOwnerToAllInterface ();
00260           BC communicator;
00261           communicator.template build<T>(OwnerToAllInterface);
00262           communicator.template forward<CopyGatherScatter<T> >(source,dest);
00263           communicator.free();
00264         }
00265 
00266     
00273         template<class T>
00274         void addOwnerOverlapToAll (const T& source, T& dest) const
00275         {
00276           if (!OwnerOverlapToAllInterfaceBuilt)
00277                 buildOwnerOverlapToAllInterface ();
00278           BC communicator;
00279           communicator.template build<T>(OwnerOverlapToAllInterface);
00280           communicator.template forward<AddGatherScatter<T> >(source,dest);
00281           communicator.free();
00282         }
00283 
00291         template<class T1, class T2>
00292         void dot (const T1& x, const T1& y, T2& result) const
00293         {
00294           // set up mask vector
00295           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00296                 {
00297                   mask.resize(x.size());
00298                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00299                     mask[i] = 1;
00300                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00301                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00302                           mask[i->local().local()] = 0;
00303                 }
00304           result = 0;
00305           for (int i=0; i<x.size(); i++)
00306                 result += x[i]*(y[i])*mask[i];
00307           result = cc.sum(result);
00308           return;
00309         }
00310 
00317         template<class T1>
00318         double norm (const T1& x) const
00319         {
00320           // set up mask vector
00321           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00322                 {
00323                   mask.resize(x.size());
00324                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00325                     mask[i] = 1;
00326                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00327                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00328                           mask[i->local().local()] = 0;
00329                 }
00330           double result = 0;
00331           for (int i=0; i<x.size(); i++)
00332                 result += x[i].two_norm2()*mask[i];
00333           return sqrt(cc.sum(result));
00334         }
00335 
00336         typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags;
00337     
00339     typedef ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet;
00340 
00342     typedef RemoteIndices<PIS> RemoteIndices;
00343 
00346     typedef GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet;
00347 
00352     const ParallelIndexSet& indexSet() const
00353     {
00354       return pis;
00355     }
00356     
00361     const RemoteIndices& remoteIndices() const
00362     {
00363       return ri;
00364     }
00365     
00370     ParallelIndexSet& indexSet()
00371     {
00372       return pis;
00373     }
00374     
00375 
00380     RemoteIndices& remoteIndices()
00381     {
00382       return ri;
00383     }
00384 
00385     void buildGlobalLookup(std::size_t size)
00386     {
00387       assert(!globalLookup_);
00388       globalLookup_ = new GlobalLookupIndexSet(pis, size);
00389     }
00390 
00391     void freeGlobalLookup()
00392     {
00393       delete globalLookup_;
00394       globalLookup_=0;
00395     }    
00396 
00397     const GlobalLookupIndexSet& globalLookup() const
00398     {
00399       assert(globalLookup_ != 0);
00400       return *globalLookup_;
00401     }
00402     
00408         template<class T1>
00409         void project (T1& x) const
00410         {
00411           for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00412                 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy)
00413                   x[i->local().local()] = 0;
00414         }
00415 
00416 
00424     OwnerOverlapCopyCommunication (MPI_Comm comm_)
00425       : cc(comm_), pis(), ri(pis,pis,comm_), 
00426         OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00427     {}
00428     
00434         OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType,LocalIdType>& indexinfo, MPI_Comm comm_)
00435           : cc(comm_),OwnerToAllInterfaceBuilt(false),OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00436         {
00437           // set up an ISTL index set
00438           pis.beginResize();
00439           for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i)
00440                 {
00441                   if (i->third==OwnerOverlapCopyAttributeSet::owner)
00442                         pis.add(i->first,LI(i->second,OwnerOverlapCopyAttributeSet::owner,true));
00443                   if (i->third==OwnerOverlapCopyAttributeSet::overlap)
00444                         pis.add(i->first,LI(i->second,OwnerOverlapCopyAttributeSet::overlap,true));
00445                   if (i->third==OwnerOverlapCopyAttributeSet::copy)
00446                         pis.add(i->first,LI(i->second,OwnerOverlapCopyAttributeSet::copy,true));
00447 //                std::cout << cc.rank() << ": adding index " << i->first << " " << i->second << " " << i->third << std::endl;
00448                 }
00449           pis.endResize();
00450 
00451           // build remote indices WITHOUT communication
00452 //        std::cout << cc.rank() << ": build remote indices" << std::endl;
00453           ri.setIndexSets(pis,pis,cc);
00454           if (indexinfo.remoteIndices().size()>0)
00455                 {
00456                   remoteindex_iterator i=indexinfo.remoteIndices().begin();
00457                   int p = i->first;
00458                   RILM modifier = ri.template getModifier<false,true>(p);
00459                   typename PIS::const_iterator pi=pis.begin();
00460                   for ( ; i!=indexinfo.remoteIndices().end(); ++i)
00461                         {
00462                           // handle processor change
00463                           if (p!=i->first)
00464                                 {
00465                                   p = i->first;
00466                                   modifier = ri.template getModifier<false,true>(p);
00467                                   pi=pis.begin();
00468                                 }
00469                           
00470                           // position to correct entry in parallel index set
00471                           while (pi->global()!=i->second && pi!=pis.end())
00472                                 ++pi;
00473                           if (pi==pis.end())
00474                                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00475                           
00476                           // insert entry
00477 //                        std::cout << cc.rank() << ": adding remote index " << i->first << " " << i->second << " " << i->third << std::endl;
00478                           if (i->third==OwnerOverlapCopyAttributeSet::owner)
00479                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi)));
00480                           if (i->third==OwnerOverlapCopyAttributeSet::overlap)
00481                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi)));
00482                           if (i->third==OwnerOverlapCopyAttributeSet::copy)
00483                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi)));
00484                         }
00485                 }
00486         }
00487 
00488         // destructor: free memory in some objects
00489         ~OwnerOverlapCopyCommunication ()
00490         {
00491           ri.free();
00492           if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free();
00493           if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free();
00494         }
00495 
00496   private:
00497     OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&)
00498     {}
00499         CollectiveCommunication<MPI_Comm> cc;
00500         PIS pis;
00501         RI ri;
00502         mutable IF OwnerToAllInterface;
00503         mutable bool OwnerToAllInterfaceBuilt;
00504         mutable IF OwnerOverlapToAllInterface;
00505         mutable bool OwnerOverlapToAllInterfaceBuilt;
00506         mutable std::vector<double> mask;
00507     GlobalLookupIndexSet* globalLookup_;
00508   };
00509 
00510 #endif
00511 
00512  
00515 } // end namespace
00516 
00517 #endif

Generated on 12 Dec 2007 with Doxygen (ver 1.5.1)