dune-istl
2.1.1
|
00001 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- 00002 // vi: set ts=4 sw=2 et sts=2: 00003 // $Id: owneroverlapcopy.hh 1516 2012-01-03 14:54:45Z mblatt $ 00004 #ifndef DUNE_OWNEROVERLAPCOPY_HH 00005 #define DUNE_OWNEROVERLAPCOPY_HH 00006 00007 #include<new> 00008 #include<iostream> 00009 #include<vector> 00010 #include<list> 00011 #include<map> 00012 #include<set> 00013 00014 #include"cmath" 00015 00016 // MPI header 00017 #if HAVE_MPI 00018 #include<mpi.h> 00019 #endif 00020 00021 00022 #include<dune/common/tuples.hh> 00023 #include<dune/common/enumset.hh> 00024 00025 #if HAVE_MPI 00026 #include <dune/common/parallel/indexset.hh> 00027 #include <dune/common/parallel/communicator.hh> 00028 #include <dune/common/parallel/remoteindices.hh> 00029 #include<dune/common/mpicollectivecommunication.hh> 00030 #endif 00031 00032 #include"solvercategory.hh" 00033 #include"istlexception.hh" 00034 #include<dune/common/collectivecommunication.hh> 00035 00036 template<int dim, template<class,class> class Comm> 00037 void testRedistributed(int s); 00038 00039 00040 namespace Dune { 00041 00057 struct OwnerOverlapCopyAttributeSet 00058 { 00059 enum AttributeSet { 00060 owner=1, overlap=2, copy=3 }; 00061 }; 00062 00074 template <class G, class L> 00075 class IndexInfoFromGrid 00076 { 00077 public: 00079 typedef G GlobalIdType; 00080 00082 typedef L LocalIdType; 00083 00090 typedef tuple<GlobalIdType,LocalIdType,int> IndexTripel; 00097 typedef tuple<int,GlobalIdType,int> RemoteIndexTripel; 00098 00104 void addLocalIndex (const IndexTripel& x) 00105 { 00106 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 00107 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 00108 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy) 00109 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set"); 00110 localindices.insert(x); 00111 } 00112 00118 void addRemoteIndex (const RemoteIndexTripel& x) 00119 { 00120 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 00121 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 00122 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy) 00123 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set"); 00124 remoteindices.insert(x); 00125 } 00126 00131 const std::set<IndexTripel>& localIndices () const 00132 { 00133 return localindices; 00134 } 00135 00140 const std::set<RemoteIndexTripel>& remoteIndices () const 00141 { 00142 return remoteindices; 00143 } 00144 00148 void clear () 00149 { 00150 localindices.clear(); 00151 remoteindices.clear(); 00152 } 00153 00154 private: 00156 std::set<IndexTripel> localindices; 00158 std::set<RemoteIndexTripel> remoteindices; 00159 }; 00160 00161 00162 #if HAVE_MPI 00163 00170 template <class GlobalIdType, class LocalIdType=int> 00171 class OwnerOverlapCopyCommunication 00172 { 00173 // used types 00174 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel; 00175 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel; 00176 typedef typename std::set<IndexTripel>::const_iterator localindex_iterator; 00177 typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator; 00178 typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet; 00179 typedef Dune::ParallelLocalIndex<AttributeSet> LI; 00180 public: 00181 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS; 00182 typedef Dune::RemoteIndices<PIS> RI; 00183 typedef Dune::RemoteIndexListModifier<PIS,typename RI::Allocator,false> RILM; 00184 typedef typename RI::RemoteIndex RX; 00185 typedef Dune::BufferedCommunicator BC; 00186 typedef Dune::Interface IF; 00187 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet; 00188 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopySet; 00189 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet; 00190 typedef Dune::AllSet<AttributeSet> AllSet; 00191 protected: 00192 00193 00195 template<typename T> 00196 struct CopyGatherScatter 00197 { 00198 typedef typename CommPolicy<T>::IndexedType V; 00199 00200 static V gather(const T& a, std::size_t i) 00201 { 00202 return a[i]; 00203 } 00204 00205 static void scatter(T& a, V v, std::size_t i) 00206 { 00207 a[i] = v; 00208 } 00209 }; 00210 template<typename T> 00211 struct AddGatherScatter 00212 { 00213 typedef typename CommPolicy<T>::IndexedType V; 00214 00215 static V gather(const T& a, std::size_t i) 00216 { 00217 return a[i]; 00218 } 00219 00220 static void scatter(T& a, V v, std::size_t i) 00221 { 00222 a[i] += v; 00223 } 00224 }; 00225 00226 void buildOwnerOverlapToAllInterface () const 00227 { 00228 if (OwnerOverlapToAllInterfaceBuilt) 00229 OwnerOverlapToAllInterface.free(); 00230 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet; 00231 typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet; 00232 OwnerOverlapSet sourceFlags; 00233 AllSet destFlags; 00234 OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags); 00235 OwnerOverlapToAllInterfaceBuilt = true; 00236 } 00237 00238 void buildOwnerToAllInterface () const 00239 { 00240 if (OwnerToAllInterfaceBuilt) 00241 OwnerToAllInterface.free(); 00242 OwnerSet sourceFlags; 00243 AllSet destFlags; 00244 OwnerToAllInterface.build(ri,sourceFlags,destFlags); 00245 OwnerToAllInterfaceBuilt = true; 00246 } 00247 00248 void buildOwnerCopyToAllInterface () const 00249 { 00250 if (OwnerCopyToAllInterfaceBuilt) 00251 OwnerCopyToAllInterface.free(); 00252 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet; 00253 typedef Combine<OwnerCopySet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> AllSet; 00254 OwnerCopySet sourceFlags; 00255 AllSet destFlags; 00256 OwnerCopyToAllInterface.build(ri,sourceFlags,destFlags); 00257 OwnerCopyToAllInterfaceBuilt = true; 00258 } 00259 00260 void buildOwnerCopyToOwnerCopyInterface () const 00261 { 00262 if (OwnerCopyToOwnerCopyInterfaceBuilt) 00263 OwnerCopyToOwnerCopyInterface.free(); 00264 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet; 00265 OwnerCopySet sourceFlags; 00266 OwnerCopySet destFlags; 00267 OwnerCopyToOwnerCopyInterface.build(ri,sourceFlags,destFlags); 00268 OwnerCopyToOwnerCopyInterfaceBuilt = true; 00269 } 00270 00271 public: 00272 00277 void setSolverCategory (SolverCategory set) { 00278 category = set; 00279 } 00280 00286 const SolverCategory::Category getSolverCategory () const { 00287 return category; 00288 } 00289 00290 const CollectiveCommunication<MPI_Comm>& communicator() const 00291 { 00292 return cc; 00293 } 00294 00301 template<class T> 00302 void copyOwnerToAll (const T& source, T& dest) const 00303 { 00304 if (!OwnerToAllInterfaceBuilt) 00305 buildOwnerToAllInterface (); 00306 BC communicator; 00307 communicator.template build<T>(OwnerToAllInterface); 00308 communicator.template forward<CopyGatherScatter<T> >(source,dest); 00309 communicator.free(); 00310 } 00311 00312 00319 template<class T> 00320 void addOwnerOverlapToAll (const T& source, T& dest) const 00321 { 00322 if (!OwnerOverlapToAllInterfaceBuilt) 00323 buildOwnerOverlapToAllInterface (); 00324 BC communicator; 00325 communicator.template build<T>(OwnerOverlapToAllInterface); 00326 communicator.template forward<AddGatherScatter<T> >(source,dest); 00327 communicator.free(); 00328 } 00329 00336 template<class T> 00337 void addOwnerCopyToAll (const T& source, T& dest) const 00338 { 00339 if (!OwnerCopyToAllInterfaceBuilt) 00340 buildOwnerCopyToAllInterface (); 00341 BC communicator; 00342 communicator.template build<T>(OwnerCopyToAllInterface); 00343 communicator.template forward<AddGatherScatter<T> >(source,dest); 00344 communicator.free(); 00345 } 00346 00353 template<class T> 00354 void addOwnerCopyToOwnerCopy (const T& source, T& dest) const 00355 { 00356 if (!OwnerCopyToOwnerCopyInterfaceBuilt) 00357 buildOwnerCopyToOwnerCopyInterface (); 00358 BC communicator; 00359 communicator.template build<T>(OwnerCopyToOwnerCopyInterface); 00360 communicator.template forward<AddGatherScatter<T> >(source,dest); 00361 communicator.free(); 00362 } 00363 00364 00372 template<class T1, class T2> 00373 void dot (const T1& x, const T1& y, T2& result) const 00374 { 00375 // set up mask vector 00376 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size())) 00377 { 00378 mask.resize(x.size()); 00379 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 00380 mask[i] = 1; 00381 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i) 00382 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner) 00383 mask[i->local().local()] = 0; 00384 } 00385 result = 0; 00386 00387 for (typename T1::size_type i=0; i<x.size(); i++) 00388 result += x[i]*(y[i])*mask[i]; 00389 result = cc.sum(result); 00390 return; 00391 } 00392 00399 template<class T1> 00400 double norm (const T1& x) const 00401 { 00402 // set up mask vector 00403 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size())) 00404 { 00405 mask.resize(x.size()); 00406 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 00407 mask[i] = 1; 00408 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i) 00409 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner) 00410 mask[i->local().local()] = 0; 00411 } 00412 double result = 0; 00413 for (typename T1::size_type i=0; i<x.size(); i++) 00414 result += x[i].two_norm2()*mask[i]; 00415 return sqrt(cc.sum(result)); 00416 } 00417 00418 typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags; 00419 00421 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet; 00422 00424 typedef Dune::RemoteIndices<PIS> RemoteIndices; 00425 00428 typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet; 00429 00434 const ParallelIndexSet& indexSet() const 00435 { 00436 return pis; 00437 } 00438 00443 const RemoteIndices& remoteIndices() const 00444 { 00445 return ri; 00446 } 00447 00452 ParallelIndexSet& indexSet() 00453 { 00454 return pis; 00455 } 00456 00457 00462 RemoteIndices& remoteIndices() 00463 { 00464 return ri; 00465 } 00466 00467 void buildGlobalLookup() 00468 { 00469 if(globalLookup_){ 00470 if(pis.seqNo()==oldseqNo) 00471 // Nothing changed! 00472 return; 00473 delete globalLookup_; 00474 } 00475 00476 globalLookup_ = new GlobalLookupIndexSet(pis); 00477 oldseqNo = pis.seqNo(); 00478 } 00479 00480 void buildGlobalLookup(std::size_t size) 00481 { 00482 if(globalLookup_){ 00483 if(pis.seqNo()==oldseqNo) 00484 // Nothing changed! 00485 return; 00486 delete globalLookup_; 00487 } 00488 globalLookup_ = new GlobalLookupIndexSet(pis, size); 00489 oldseqNo = pis.seqNo(); 00490 } 00491 00492 void freeGlobalLookup() 00493 { 00494 delete globalLookup_; 00495 globalLookup_=0; 00496 } 00497 00498 const GlobalLookupIndexSet& globalLookup() const 00499 { 00500 assert(globalLookup_ != 0); 00501 return *globalLookup_; 00502 } 00503 00509 template<class T1> 00510 void project (T1& x) const 00511 { 00512 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i) 00513 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy) 00514 x[i->local().local()] = 0; 00515 } 00516 00526 OwnerOverlapCopyCommunication (MPI_Comm comm_, 00527 SolverCategory::Category cat_ = SolverCategory::overlapping, 00528 bool freecomm_ = false) 00529 : comm(comm_), cc(comm_), pis(), ri(pis,pis,comm_), 00530 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 00531 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 00532 globalLookup_(0), category(cat_), 00533 freecomm(freecomm_) 00534 {} 00535 00544 OwnerOverlapCopyCommunication (SolverCategory::Category cat_ = SolverCategory::overlapping) 00545 : comm(MPI_COMM_WORLD), cc(MPI_COMM_WORLD), pis(), ri(pis,pis,MPI_COMM_WORLD), 00546 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 00547 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 00548 globalLookup_(0), category(cat_), freecomm(false) 00549 {} 00550 00558 OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType, LocalIdType>& indexinfo, 00559 MPI_Comm comm_, 00560 SolverCategory::Category cat_ = SolverCategory::overlapping, 00561 bool freecomm_ = false) 00562 : cc(comm_), OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 00563 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 00564 globalLookup_(0), category(cat_), freecomm(freecomm_) 00565 { 00566 // set up an ISTL index set 00567 pis.beginResize(); 00568 for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i) 00569 { 00570 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner) 00571 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true)); 00572 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap) 00573 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true)); 00574 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy) 00575 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true)); 00576 // std::cout << cc.rank() << ": adding index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl; 00577 } 00578 pis.endResize(); 00579 00580 // build remote indices WITHOUT communication 00581 // std::cout << cc.rank() << ": build remote indices" << std::endl; 00582 ri.setIndexSets(pis,pis,cc); 00583 if (indexinfo.remoteIndices().size()>0) 00584 { 00585 remoteindex_iterator i=indexinfo.remoteIndices().begin(); 00586 int p = get<0>(*i); 00587 RILM modifier = ri.template getModifier<false,true>(p); 00588 typename PIS::const_iterator pi=pis.begin(); 00589 for ( ; i!=indexinfo.remoteIndices().end(); ++i) 00590 { 00591 // handle processor change 00592 if (p!=get<0>(*i)) 00593 { 00594 p = get<0>(*i); 00595 modifier = ri.template getModifier<false,true>(p); 00596 pi=pis.begin(); 00597 } 00598 00599 // position to correct entry in parallel index set 00600 while (pi->global()!=get<1>(*i) && pi!=pis.end()) 00601 ++pi; 00602 if (pi==pis.end()) 00603 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set"); 00604 00605 // insert entry 00606 // std::cout << cc.rank() << ": adding remote index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl; 00607 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner) 00608 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi))); 00609 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap) 00610 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi))); 00611 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy) 00612 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi))); 00613 } 00614 }else{ 00615 // Force remote indices to be synced! 00616 ri.template getModifier<false,true>(0); 00617 } 00618 } 00619 00620 // destructor: free memory in some objects 00621 ~OwnerOverlapCopyCommunication () 00622 { 00623 ri.free(); 00624 if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free(); 00625 if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free(); 00626 if (OwnerCopyToAllInterfaceBuilt) OwnerCopyToAllInterface.free(); 00627 if (OwnerCopyToOwnerCopyInterfaceBuilt) OwnerCopyToOwnerCopyInterface.free(); 00628 if (globalLookup_) delete globalLookup_; 00629 if (freecomm==true) 00630 if(comm!=MPI_COMM_NULL) 00631 MPI_Comm_free(&comm); 00632 } 00633 00634 private: 00635 OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&) 00636 {} 00637 MPI_Comm comm; 00638 CollectiveCommunication<MPI_Comm> cc; 00639 PIS pis; 00640 RI ri; 00641 mutable IF OwnerToAllInterface; 00642 mutable bool OwnerToAllInterfaceBuilt; 00643 mutable IF OwnerOverlapToAllInterface; 00644 mutable bool OwnerOverlapToAllInterfaceBuilt; 00645 mutable IF OwnerCopyToAllInterface; 00646 mutable bool OwnerCopyToAllInterfaceBuilt; 00647 mutable IF OwnerCopyToOwnerCopyInterface; 00648 mutable bool OwnerCopyToOwnerCopyInterfaceBuilt; 00649 mutable std::vector<double> mask; 00650 int oldseqNo; 00651 GlobalLookupIndexSet* globalLookup_; 00652 SolverCategory::Category category; 00653 bool freecomm; 00654 }; 00655 00656 #endif 00657 00658 00661 } // end namespace 00662 00663 #endif