- Home
- About DUNE
- Download
- Documentation
- Community
- Development
00001 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- 00002 // vi: set ts=4 sw=2 et sts=2: 00003 // $Id$ 00004 #ifndef DUNE_OWNEROVERLAPCOPY_HH 00005 #define DUNE_OWNEROVERLAPCOPY_HH 00006 00007 #include<new> 00008 #include<iostream> 00009 #include<vector> 00010 #include<list> 00011 #include<map> 00012 #include<set> 00013 00014 #include"cmath" 00015 00016 // MPI header 00017 #if HAVE_MPI 00018 #include<mpi.h> 00019 #endif 00020 00021 00022 #include<dune/common/tuples.hh> 00023 #include<dune/common/enumset.hh> 00024 00025 #if HAVE_MPI 00026 #include <dune/common/parallel/indexset.hh> 00027 #include <dune/common/parallel/communicator.hh> 00028 #include <dune/common/parallel/remoteindices.hh> 00029 #include<dune/common/mpicollectivecommunication.hh> 00030 #endif 00031 00032 #include"solvercategory.hh" 00033 #include"istlexception.hh" 00034 #include<dune/common/collectivecommunication.hh> 00035 00036 template<int dim, template<class,class> class Comm> 00037 void testRedistributed(int s); 00038 00039 00040 namespace Dune { 00041 00057 struct OwnerOverlapCopyAttributeSet 00058 { 00059 enum AttributeSet { 00060 owner=1, overlap=2, copy=3 }; 00061 }; 00062 00074 template <class G, class L> 00075 class IndexInfoFromGrid 00076 { 00077 public: 00079 typedef G GlobalIdType; 00080 00082 typedef L LocalIdType; 00083 00090 typedef tuple<GlobalIdType,LocalIdType,int> IndexTripel; 00097 typedef tuple<int,GlobalIdType,int> RemoteIndexTripel; 00098 00104 void addLocalIndex (const IndexTripel& x) 00105 { 00106 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 00107 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 00108 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy) 00109 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set"); 00110 localindices.insert(x); 00111 } 00112 00118 void addRemoteIndex (const RemoteIndexTripel& x) 00119 { 00120 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 00121 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 00122 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy) 00123 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set"); 00124 remoteindices.insert(x); 00125 } 00126 00131 const std::set<IndexTripel>& localIndices () const 00132 { 00133 return localindices; 00134 } 00135 00140 const std::set<RemoteIndexTripel>& remoteIndices () const 00141 { 00142 return remoteindices; 00143 } 00144 00148 void clear () 00149 { 00150 localindices.clear(); 00151 remoteindices.clear(); 00152 } 00153 00154 private: 00156 std::set<IndexTripel> localindices; 00158 std::set<RemoteIndexTripel> remoteindices; 00159 }; 00160 00161 00162 #if HAVE_MPI 00163 00170 template <class GlobalIdType, class LocalIdType=int> 00171 class OwnerOverlapCopyCommunication 00172 { 00173 // used types 00174 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel; 00175 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel; 00176 typedef typename std::set<IndexTripel>::const_iterator localindex_iterator; 00177 typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator; 00178 typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet; 00179 typedef Dune::ParallelLocalIndex<AttributeSet> LI; 00180 public: 00181 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS; 00182 typedef Dune::RemoteIndices<PIS> RI; 00183 typedef Dune::RemoteIndexListModifier<PIS,typename RI::Allocator,false> RILM; 00184 typedef typename RI::RemoteIndex RX; 00185 typedef Dune::BufferedCommunicator BC; 00186 typedef Dune::Interface IF; 00187 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet; 00188 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopySet; 00189 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet; 00190 typedef Dune::AllSet<AttributeSet> AllSet; 00191 protected: 00192 00193 00195 template<typename T> 00196 struct CopyGatherScatter 00197 { 00198 typedef typename CommPolicy<T>::IndexedType V; 00199 00200 static V gather(const T& a, std::size_t i) 00201 { 00202 return a[i]; 00203 } 00204 00205 static void scatter(T& a, V v, std::size_t i) 00206 { 00207 a[i] = v; 00208 } 00209 }; 00210 template<typename T> 00211 struct AddGatherScatter 00212 { 00213 typedef typename CommPolicy<T>::IndexedType V; 00214 00215 static V gather(const T& a, std::size_t i) 00216 { 00217 return a[i]; 00218 } 00219 00220 static void scatter(T& a, V v, std::size_t i) 00221 { 00222 a[i] += v; 00223 } 00224 }; 00225 00226 void buildOwnerOverlapToAllInterface () const 00227 { 00228 if (OwnerOverlapToAllInterfaceBuilt) 00229 OwnerOverlapToAllInterface.free(); 00230 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet; 00231 typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet; 00232 OwnerOverlapSet sourceFlags; 00233 AllSet destFlags; 00234 OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags); 00235 OwnerOverlapToAllInterfaceBuilt = true; 00236 } 00237 00238 void buildOwnerToAllInterface () const 00239 { 00240 if (OwnerToAllInterfaceBuilt) 00241 OwnerToAllInterface.free(); 00242 OwnerSet sourceFlags; 00243 AllSet destFlags; 00244 OwnerToAllInterface.build(ri,sourceFlags,destFlags); 00245 OwnerToAllInterfaceBuilt = true; 00246 } 00247 00248 void buildOwnerCopyToAllInterface () const 00249 { 00250 if (OwnerCopyToAllInterfaceBuilt) 00251 OwnerCopyToAllInterface.free(); 00252 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet; 00253 typedef Combine<OwnerCopySet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> AllSet; 00254 OwnerCopySet sourceFlags; 00255 AllSet destFlags; 00256 OwnerCopyToAllInterface.build(ri,sourceFlags,destFlags); 00257 OwnerCopyToAllInterfaceBuilt = true; 00258 } 00259 00260 void buildOwnerCopyToOwnerCopyInterface () const 00261 { 00262 if (OwnerCopyToOwnerCopyInterfaceBuilt) 00263 OwnerCopyToOwnerCopyInterface.free(); 00264 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet; 00265 OwnerCopySet sourceFlags; 00266 OwnerCopySet destFlags; 00267 OwnerCopyToOwnerCopyInterface.build(ri,sourceFlags,destFlags); 00268 OwnerCopyToOwnerCopyInterfaceBuilt = true; 00269 } 00270 00271 public: 00272 00277 void setSolverCategory (SolverCategory set) { 00278 category = set; 00279 } 00280 00286 const SolverCategory::Category getSolverCategory () const { 00287 return category; 00288 } 00289 00290 const CollectiveCommunication<MPI_Comm>& communicator() const 00291 { 00292 return cc; 00293 } 00294 00301 template<class T> 00302 void copyOwnerToAll (const T& source, T& dest) const 00303 { 00304 if (!OwnerToAllInterfaceBuilt) 00305 buildOwnerToAllInterface (); 00306 BC communicator; 00307 communicator.template build<T>(OwnerToAllInterface); 00308 communicator.template forward<CopyGatherScatter<T> >(source,dest); 00309 communicator.free(); 00310 } 00311 00312 00319 template<class T> 00320 void addOwnerOverlapToAll (const T& source, T& dest) const 00321 { 00322 if (!OwnerOverlapToAllInterfaceBuilt) 00323 buildOwnerOverlapToAllInterface (); 00324 BC communicator; 00325 communicator.template build<T>(OwnerOverlapToAllInterface); 00326 communicator.template forward<AddGatherScatter<T> >(source,dest); 00327 communicator.free(); 00328 } 00329 00336 template<class T> 00337 void addOwnerCopyToAll (const T& source, T& dest) const 00338 { 00339 if (!OwnerCopyToAllInterfaceBuilt) 00340 buildOwnerCopyToAllInterface (); 00341 BC communicator; 00342 communicator.template build<T>(OwnerCopyToAllInterface); 00343 communicator.template forward<AddGatherScatter<T> >(source,dest); 00344 communicator.free(); 00345 } 00346 00353 template<class T> 00354 void addOwnerCopyToOwnerCopy (const T& source, T& dest) const 00355 { 00356 if (!OwnerCopyToOwnerCopyInterfaceBuilt) 00357 buildOwnerCopyToOwnerCopyInterface (); 00358 BC communicator; 00359 communicator.template build<T>(OwnerCopyToOwnerCopyInterface); 00360 communicator.template forward<AddGatherScatter<T> >(source,dest); 00361 communicator.free(); 00362 } 00363 00364 00372 template<class T1, class T2> 00373 void dot (const T1& x, const T1& y, T2& result) const 00374 { 00375 // set up mask vector 00376 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size())) 00377 { 00378 mask.resize(x.size()); 00379 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 00380 mask[i] = 1; 00381 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i) 00382 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner) 00383 mask[i->local().local()] = 0; 00384 } 00385 result = 0; 00386 00387 for (typename T1::size_type i=0; i<x.size(); i++) 00388 result += x[i]*(y[i])*mask[i]; 00389 result = cc.sum(result); 00390 return; 00391 } 00392 00399 template<class T1> 00400 double norm (const T1& x) const 00401 { 00402 // set up mask vector 00403 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size())) 00404 { 00405 mask.resize(x.size()); 00406 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 00407 mask[i] = 1; 00408 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i) 00409 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner) 00410 mask[i->local().local()] = 0; 00411 } 00412 double result = 0; 00413 for (typename T1::size_type i=0; i<x.size(); i++) 00414 result += x[i].two_norm2()*mask[i]; 00415 return sqrt(cc.sum(result)); 00416 } 00417 00418 typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags; 00419 00421 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet; 00422 00424 typedef Dune::RemoteIndices<PIS> RemoteIndices; 00425 00428 typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet; 00429 00434 const ParallelIndexSet& indexSet() const 00435 { 00436 return pis; 00437 } 00438 00443 const RemoteIndices& remoteIndices() const 00444 { 00445 return ri; 00446 } 00447 00452 ParallelIndexSet& indexSet() 00453 { 00454 return pis; 00455 } 00456 00457 00462 RemoteIndices& remoteIndices() 00463 { 00464 return ri; 00465 } 00466 00467 void buildGlobalLookup() 00468 { 00469 if(globalLookup_){ 00470 if(pis.seqNo()==oldseqNo) 00471 // Nothing changed! 00472 return; 00473 delete globalLookup_; 00474 } 00475 00476 globalLookup_ = new GlobalLookupIndexSet(pis); 00477 oldseqNo = pis.seqNo(); 00478 } 00479 00480 void buildGlobalLookup(std::size_t size) 00481 { 00482 if(globalLookup_){ 00483 if(pis.seqNo()==oldseqNo) 00484 // Nothing changed! 00485 return; 00486 delete globalLookup_; 00487 } 00488 globalLookup_ = new GlobalLookupIndexSet(pis, size); 00489 oldseqNo = pis.seqNo(); 00490 } 00491 00492 void freeGlobalLookup() 00493 { 00494 delete globalLookup_; 00495 globalLookup_=0; 00496 } 00497 00498 const GlobalLookupIndexSet& globalLookup() const 00499 { 00500 assert(globalLookup_ != 0); 00501 return *globalLookup_; 00502 } 00503 00509 template<class T1> 00510 void project (T1& x) const 00511 { 00512 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i) 00513 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy) 00514 x[i->local().local()] = 0; 00515 } 00516 00524 OwnerOverlapCopyCommunication (MPI_Comm comm_, SolverCategory::Category cat = SolverCategory::overlapping) 00525 : cc(comm_), pis(), ri(pis,pis,comm_), 00526 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 00527 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 00528 globalLookup_(0), category(cat) 00529 {} 00530 00537 OwnerOverlapCopyCommunication (SolverCategory::Category cat = SolverCategory::overlapping) 00538 : cc(MPI_COMM_WORLD), pis(), ri(pis,pis,MPI_COMM_WORLD), 00539 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 00540 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 00541 globalLookup_(0), category(cat) 00542 {} 00543 00549 OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType, LocalIdType>& indexinfo, MPI_Comm comm_, SolverCategory::Category cat = SolverCategory::overlapping) 00550 : cc(comm_), OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), 00551 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false), 00552 globalLookup_(0), category(cat) 00553 { 00554 // set up an ISTL index set 00555 pis.beginResize(); 00556 for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i) 00557 { 00558 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner) 00559 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true)); 00560 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap) 00561 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true)); 00562 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy) 00563 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true)); 00564 // std::cout << cc.rank() << ": adding index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl; 00565 } 00566 pis.endResize(); 00567 00568 // build remote indices WITHOUT communication 00569 // std::cout << cc.rank() << ": build remote indices" << std::endl; 00570 ri.setIndexSets(pis,pis,cc); 00571 if (indexinfo.remoteIndices().size()>0) 00572 { 00573 remoteindex_iterator i=indexinfo.remoteIndices().begin(); 00574 int p = get<0>(*i); 00575 RILM modifier = ri.template getModifier<false,true>(p); 00576 typename PIS::const_iterator pi=pis.begin(); 00577 for ( ; i!=indexinfo.remoteIndices().end(); ++i) 00578 { 00579 // handle processor change 00580 if (p!=get<0>(*i)) 00581 { 00582 p = get<0>(*i); 00583 modifier = ri.template getModifier<false,true>(p); 00584 pi=pis.begin(); 00585 } 00586 00587 // position to correct entry in parallel index set 00588 while (pi->global()!=get<1>(*i) && pi!=pis.end()) 00589 ++pi; 00590 if (pi==pis.end()) 00591 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set"); 00592 00593 // insert entry 00594 // std::cout << cc.rank() << ": adding remote index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl; 00595 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner) 00596 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi))); 00597 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap) 00598 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi))); 00599 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy) 00600 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi))); 00601 } 00602 }else{ 00603 // Force remote indices to be synced! 00604 ri.template getModifier<false,true>(0); 00605 } 00606 } 00607 00608 // destructor: free memory in some objects 00609 ~OwnerOverlapCopyCommunication () 00610 { 00611 ri.free(); 00612 if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free(); 00613 if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free(); 00614 if (OwnerCopyToAllInterfaceBuilt) OwnerCopyToAllInterface.free(); 00615 if (OwnerCopyToOwnerCopyInterfaceBuilt) OwnerCopyToOwnerCopyInterface.free(); 00616 if (globalLookup_) delete globalLookup_; 00617 } 00618 00619 private: 00620 OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&) 00621 {} 00622 CollectiveCommunication<MPI_Comm> cc; 00623 PIS pis; 00624 RI ri; 00625 mutable IF OwnerToAllInterface; 00626 mutable bool OwnerToAllInterfaceBuilt; 00627 mutable IF OwnerOverlapToAllInterface; 00628 mutable bool OwnerOverlapToAllInterfaceBuilt; 00629 mutable IF OwnerCopyToAllInterface; 00630 mutable bool OwnerCopyToAllInterfaceBuilt; 00631 mutable IF OwnerCopyToOwnerCopyInterface; 00632 mutable bool OwnerCopyToOwnerCopyInterfaceBuilt; 00633 mutable std::vector<double> mask; 00634 int oldseqNo; 00635 GlobalLookupIndexSet* globalLookup_; 00636 SolverCategory::Category category; 00637 }; 00638 00639 #endif 00640 00641 00644 } // end namespace 00645 00646 #endif
Generated on Fri Apr 29 2011 with Doxygen (ver 1.7.1) [doxygen-log,error-log].