5#ifndef DUNE_ISTL_MATRIXREDISTRIBUTE_HH
6#define DUNE_ISTL_MATRIXREDISTRIBUTE_HH
12#include <dune/istl/paamg/pinfo.hh>
21 struct RedistributeInformation
28 void redistribute([[maybe_unused]]
const D& from, [[maybe_unused]] D& to)
const
32 void redistributeBackward([[maybe_unused]] D& from, [[maybe_unused]]
const D& to)
const
38 void setNoRows([[maybe_unused]] std::size_t
size)
41 void setNoCopyRows([[maybe_unused]] std::size_t
size)
44 void setNoBackwardsCopyRows([[maybe_unused]] std::size_t
size)
47 std::size_t getRowSize([[maybe_unused]] std::size_t index)
const
52 std::size_t getCopyRowSize([[maybe_unused]] std::size_t index)
const
57 std::size_t getBackwardsCopyRowSize([[maybe_unused]] std::size_t index)
const
65 template<
typename T,
typename T1>
66 class RedistributeInformation<OwnerOverlapCopyCommunication<T,T1> >
69 typedef OwnerOverlapCopyCommunication<T,T1> Comm;
71 RedistributeInformation()
72 : interface(), setup_(false)
75 RedistributeInterface& getInterface()
80 void checkInterface(
const IS& source,
81 const IS& target, MPI_Comm comm)
83 auto ri = std::make_unique<RemoteIndices<IS> >(source, target, comm);
84 ri->template rebuild<true>();
86 typename OwnerOverlapCopyCommunication<int>::OwnerSet flags;
88 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
90 inf.build(*ri, flags, flags);
96 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
98 std::cout<<
"Interfaces do not match!"<<std::endl;
99 std::cout<<rank<<
": redist interface new :"<<inf<<std::endl;
100 std::cout<<rank<<
": redist interface :"<<interface<<std::endl;
117 template<
class GatherScatter,
class D>
118 void redistribute(
const D& from, D& to)
const
120 BufferedCommunicator communicator;
121 communicator.template build<D>(from,to, interface);
122 communicator.template forward<GatherScatter>(from, to);
125 template<
class GatherScatter,
class D>
126 void redistributeBackward(D& from,
const D& to)
const
129 BufferedCommunicator communicator;
130 communicator.template build<D>(from,to, interface);
131 communicator.template backward<GatherScatter>(from, to);
136 void redistribute(
const D& from, D& to)
const
138 redistribute<CopyGatherScatter<D> >(from,to);
141 void redistributeBackward(D& from,
const D& to)
const
143 redistributeBackward<CopyGatherScatter<D> >(from,to);
150 void reserve(std::size_t
size)
153 std::size_t& getRowSize(std::size_t index)
155 return rowSize[index];
158 std::size_t getRowSize(std::size_t index)
const
160 return rowSize[index];
163 std::size_t& getCopyRowSize(std::size_t index)
165 return copyrowSize[index];
168 std::size_t getCopyRowSize(std::size_t index)
const
170 return copyrowSize[index];
173 std::size_t& getBackwardsCopyRowSize(std::size_t index)
175 return backwardscopyrowSize[index];
178 std::size_t getBackwardsCopyRowSize(std::size_t index)
const
180 return backwardscopyrowSize[index];
183 void setNoRows(std::size_t rows)
185 rowSize.resize(rows, 0);
188 void setNoCopyRows(std::size_t rows)
190 copyrowSize.resize(rows, 0);
193 void setNoBackwardsCopyRows(std::size_t rows)
195 backwardscopyrowSize.resize(rows, 0);
199 std::vector<std::size_t> rowSize;
200 std::vector<std::size_t> copyrowSize;
201 std::vector<std::size_t> backwardscopyrowSize;
202 RedistributeInterface interface;
214 template<
class M,
class RI>
218 typedef typename M::size_type value_type;
219 typedef typename M::size_type size_type;
227 : matrix(m_), rowsize(rowsize_)
243 template<
class M,
class I>
246 typedef typename M::size_type size_type;
255 : matrix(m_), idxset(idxset_), aggidxset(aggidxset_), rowsize()
266 const std::vector<typename M::size_type>& rowsize_)
267 : matrix(m_), idxset(idxset_), aggidxset(aggidxset_), sparsity(aggidxset_.
size()), rowsize(&rowsize_)
285 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
287 for(IIter i= aggidxset.begin(), end=aggidxset.end(); i!=end; ++i) {
288 if(!OwnerSet::contains(i->local().attribute())) {
290 std::cout<<rank<<
" Inserting diagonal for"<<i->local()<<std::endl;
292 sparsity[i->local()].insert(i->local());
295 nnz+=sparsity[i->local()].size();
297 assert( aggidxset.size()==sparsity.size());
300 m.setSize(aggidxset.size(), aggidxset.size(), nnz);
301 m.setBuildMode(M::row_wise);
302 typename M::CreateIterator citer=m.createbegin();
308 typedef typename std::vector<std::set<size_type> >::const_iterator Iter;
309 for(Iter i=sparsity.begin(), end=sparsity.end(); i!=end; ++i, ++citer)
311 typedef typename std::set<size_type>::const_iterator SIter;
312 for(SIter si=i->begin(), send=i->end(); si!=send; ++si)
315 if(i->find(idx)==i->end()) {
316 const typename I::IndexPair* gi=global.
pair(idx);
318 std::cout<<rank<<
": row "<<idx<<
" is missing a diagonal entry! global="<<gi->global()<<
" attr="<<gi->local().attribute()<<
" "<<
319 OwnerSet::contains(gi->local().attribute())<<
320 " row size="<<i->size()<<std::endl;
342 for (
unsigned int i = 0; i != sparsity.size(); ++i) {
343 if (add_sparsity[i].
size() != 0) {
344 typedef std::set<size_type> Set;
346 std::insert_iterator<Set> tmp_insert (tmp_set, tmp_set.begin());
347 std::set_union(add_sparsity[i].begin(), add_sparsity[i].end(),
348 sparsity[i].begin(), sparsity[i].end(), tmp_insert);
349 sparsity[i].swap(tmp_set);
358 std::vector<std::set<size_type> > sparsity;
359 const std::vector<size_type>* rowsize;
362 template<
class M,
class I>
363 struct CommPolicy<CommMatrixSparsityPattern<M,I> >
365 typedef CommMatrixSparsityPattern<M,I>
Type;
376 static typename M::size_type
getSize(
const Type& t, std::size_t i)
379 return t.matrix[i].size();
382 assert((*t.rowsize)[i]>0);
383 return (*t.rowsize)[i];
394 template<
class M,
class I>
413 std::vector<size_t>& rowsize_)
427 if(!OwnerSet::contains(i->local().attribute())) {
429 typedef typename M::ColIterator CIter;
430 for(CIter c=
matrix[i->local()].begin(), cend=
matrix[i->local()].end();
434 if(c.index()==i->local()) {
435 auto setDiagonal = [](
auto&& scalarOrMatrix,
const auto& value) {
436 auto&& matrixView = Dune::Impl::asMatrix(scalarOrMatrix);
437 for (
auto rowIt = matrixView.begin(); rowIt != matrixView.end(); ++rowIt)
438 (*rowIt)[rowIt.index()] = value;
455 template<
class M,
class I>
464 typedef std::pair<typename I::GlobalIndex,typename M::block_type>
IndexedType;
469 static std::size_t
getSize(
const Type& t, std::size_t i)
472 return t.matrix[i].size();
475 assert((*t.rowsize)[i]>0);
476 return (*t.rowsize)[i];
481 template<
class M,
class I,
class RI>
482 struct MatrixRowSizeGatherScatter
484 typedef CommMatrixRowSize<M,RI> Container;
486 static const typename M::size_type gather(
const Container& cont, std::size_t i)
488 return cont.matrix[i].size();
490 static void scatter(Container& cont,
const typename M::size_type& rowsize, std::size_t i)
493 cont.rowsize.getRowSize(i)=rowsize;
498 template<
class M,
class I,
class RI>
499 struct MatrixCopyRowSizeGatherScatter
501 typedef CommMatrixRowSize<M,RI> Container;
503 static const typename M::size_type gather(
const Container& cont, std::size_t i)
505 return cont.matrix[i].size();
507 static void scatter(Container& cont,
const typename M::size_type& rowsize, std::size_t i)
510 if (rowsize > cont.rowsize.getCopyRowSize(i))
511 cont.rowsize.getCopyRowSize(i)=rowsize;
516 template<
class M,
class I>
517 struct MatrixSparsityPatternGatherScatter
519 typedef typename I::GlobalIndex GlobalIndex;
520 typedef CommMatrixSparsityPattern<M,I> Container;
521 typedef typename M::ConstColIterator ColIter;
524 static GlobalIndex numlimits;
526 static const GlobalIndex& gather(
const Container& cont, std::size_t i, std::size_t j)
529 col=cont.matrix[i].begin();
530 else if (col!=cont.matrix[i].end())
537 if (col==cont.matrix[i].end()) {
542 const typename I::IndexPair* index=cont.idxset.pair(col.index());
545 if ( index->local().attribute() != 2)
546 return index->global();
553 static void scatter(Container& cont,
const GlobalIndex& gi, std::size_t i, [[maybe_unused]] std::size_t j)
557 const typename I::IndexPair& ip=cont.aggidxset.at(gi);
558 assert(ip.global()==gi);
559 std::size_t column = ip.local();
560 cont.sparsity[i].insert(column);
563 if(!OwnerSet::contains(ip.local().attribute()))
565 cont.sparsity[column].insert(i);
571 typedef typename Container::LookupIndexSet GlobalLookup;
572 typedef typename GlobalLookup::IndexPair IndexPair;
575 GlobalLookup lookup(cont.aggidxset);
576 const IndexPair* pi=lookup.pair(i);
578 if(OwnerSet::contains(pi->local().attribute())) {
580 MPI_Comm_rank(MPI_COMM_WORLD,&rank);
581 std::cout<<rank<<cont.aggidxset<<std::endl;
582 std::cout<<rank<<
": row "<<i<<
" (global="<<gi <<
") not in index set for owner index "<<pi->global()<<std::endl;
590 template<
class M,
class I>
591 typename MatrixSparsityPatternGatherScatter<M,I>::ColIter MatrixSparsityPatternGatherScatter<M,I>::col;
593 template<
class M,
class I>
594 typename MatrixSparsityPatternGatherScatter<M,I>::GlobalIndex MatrixSparsityPatternGatherScatter<M,I>::numlimits;
597 template<
class M,
class I>
598 struct MatrixRowGatherScatter
600 typedef typename I::GlobalIndex GlobalIndex;
601 typedef CommMatrixRow<M,I> Container;
602 typedef typename M::ConstColIterator ColIter;
603 typedef typename std::pair<GlobalIndex,typename M::block_type> Data;
605 static Data datastore;
606 static GlobalIndex numlimits;
608 static const Data& gather(
const Container& cont, std::size_t i, std::size_t j)
611 col=cont.matrix[i].begin();
612 else if (col!=cont.matrix[i].end())
618 if (col==cont.matrix[i].end()) {
620 datastore = Data(numlimits,*col);
625 const typename I::IndexPair* index=cont.idxset.pair(col.index());
629 if ( index->local().attribute() != 2)
630 datastore = Data(index->global(),*col);
633 datastore = Data(numlimits,*col);
638 static void scatter(Container& cont,
const Data& data, std::size_t i, [[maybe_unused]] std::size_t j)
642 typename M::size_type column=cont.aggidxset.at(data.first).local();
643 cont.matrix[i][column]=data.second;
653 template<
class M,
class I>
654 typename MatrixRowGatherScatter<M,I>::ColIter MatrixRowGatherScatter<M,I>::col;
656 template<
class M,
class I>
657 typename MatrixRowGatherScatter<M,I>::Data MatrixRowGatherScatter<M,I>::datastore;
659 template<
class M,
class I>
660 typename MatrixRowGatherScatter<M,I>::GlobalIndex MatrixRowGatherScatter<M,I>::numlimits;
662 template<
typename M,
typename C>
663 void redistributeSparsityPattern(M& origMatrix, M& newMatrix, C& origComm, C& newComm,
664 RedistributeInformation<C>& ri)
666 typename C::CopySet copyflags;
667 typename C::OwnerSet ownerflags;
668 typedef typename C::ParallelIndexSet IndexSet;
669 typedef RedistributeInformation<C> RI;
670 std::vector<typename M::size_type> rowsize(newComm.indexSet().size(), 0);
671 std::vector<typename M::size_type> copyrowsize(newComm.indexSet().size(), 0);
672 std::vector<typename M::size_type> backwardscopyrowsize(origComm.indexSet().size(), 0);
675 CommMatrixRowSize<M,RI> commRowSize(origMatrix, ri);
676 ri.template redistribute<MatrixRowSizeGatherScatter<M,IndexSet,RI> >(commRowSize,commRowSize);
678 origComm.buildGlobalLookup();
680 for (std::size_t i=0; i < newComm.indexSet().size(); i++) {
681 rowsize[i] = ri.getRowSize(i);
684 CommMatrixSparsityPattern<M,IndexSet>
685 origsp(origMatrix, origComm.globalLookup(), newComm.indexSet());
686 CommMatrixSparsityPattern<M,IndexSet>
687 newsp(origMatrix, origComm.globalLookup(), newComm.indexSet(), rowsize);
689 ri.template redistribute<MatrixSparsityPatternGatherScatter<M,IndexSet> >(origsp,newsp);
693 RemoteIndices<IndexSet> *ris =
new RemoteIndices<IndexSet>(origComm.indexSet(),
695 origComm.communicator());
696 ris->template rebuild<true>();
698 ri.getInterface().free();
699 ri.getInterface().build(*ris,copyflags,ownerflags);
702 CommMatrixRowSize<M,RI> commRowSize_copy(origMatrix, ri);
703 ri.template redistribute<MatrixCopyRowSizeGatherScatter<M,IndexSet,RI> >(commRowSize_copy,
706 for (std::size_t i=0; i < newComm.indexSet().size(); i++) {
707 copyrowsize[i] = ri.getCopyRowSize(i);
710 ri.redistributeBackward(backwardscopyrowsize,copyrowsize);
711 for (std::size_t i=0; i < origComm.indexSet().size(); i++) {
712 ri.getBackwardsCopyRowSize(i) = backwardscopyrowsize[i];
716 CommMatrixSparsityPattern<M,IndexSet> origsp_copy(origMatrix,
717 origComm.globalLookup(),
719 backwardscopyrowsize);
720 CommMatrixSparsityPattern<M,IndexSet> newsp_copy(origMatrix, origComm.globalLookup(),
721 newComm.indexSet(), copyrowsize);
722 ri.template redistribute<MatrixSparsityPatternGatherScatter<M,IndexSet> >(origsp_copy,
725 newsp.completeSparsityPattern(newsp_copy.sparsity);
726 newsp.storeSparsityPattern(newMatrix);
729 newsp.storeSparsityPattern(newMatrix);
731#ifdef DUNE_ISTL_WITH_CHECKING
734 typedef typename M::ConstRowIterator RIter;
735 for(RIter row=newMatrix.begin(), rend=newMatrix.end(); row != rend; ++row) {
736 typedef typename M::ConstColIterator CIter;
737 for(CIter col=row->begin(), cend=row->end(); col!=cend; ++col)
740 newMatrix[col.index()][row.index()];
742 std::cerr<<newComm.communicator().rank()<<
": entry ("
743 <<col.index()<<
","<<row.index()<<
") missing! for symmetry!"<<std::endl;
752 DUNE_THROW(ISTLError,
"Matrix not symmetric!");
756 template<
typename M,
typename C>
757 void redistributeMatrixEntries(M& origMatrix, M& newMatrix, C& origComm, C& newComm,
758 RedistributeInformation<C>& ri)
760 typedef typename C::ParallelIndexSet IndexSet;
761 typename C::OwnerSet ownerflags;
762 std::vector<typename M::size_type> rowsize(newComm.indexSet().size(), 0);
763 std::vector<typename M::size_type> copyrowsize(newComm.indexSet().size(), 0);
764 std::vector<typename M::size_type> backwardscopyrowsize(origComm.indexSet().size(), 0);
766 for (std::size_t i=0; i < newComm.indexSet().
size(); i++) {
767 rowsize[i] = ri.getRowSize(i);
769 copyrowsize[i] = ri.getCopyRowSize(i);
773 for (std::size_t i=0; i < origComm.indexSet().
size(); i++)
775 backwardscopyrowsize[i] = ri.getBackwardsCopyRowSize(i);
780 CommMatrixRow<M,IndexSet> origrow_copy(origMatrix, origComm.globalLookup(),
781 newComm.indexSet(), backwardscopyrowsize);
782 CommMatrixRow<M,IndexSet> newrow_copy(newMatrix, origComm.globalLookup(),
783 newComm.indexSet(),copyrowsize);
784 ri.template redistribute<MatrixRowGatherScatter<M,IndexSet> >(origrow_copy,
786 ri.getInterface().free();
787 RemoteIndices<IndexSet> *ris =
new RemoteIndices<IndexSet>(origComm.indexSet(),
789 origComm.communicator());
790 ris->template rebuild<true>();
791 ri.getInterface().build(*ris,ownerflags,ownerflags);
794 CommMatrixRow<M,IndexSet>
795 origrow(origMatrix, origComm.globalLookup(), newComm.indexSet());
796 CommMatrixRow<M,IndexSet>
797 newrow(newMatrix, origComm.globalLookup(), newComm.indexSet(),rowsize);
798 ri.template redistribute<MatrixRowGatherScatter<M,IndexSet> >(origrow,newrow);
800 newrow.setOverlapRowsToDirichlet();
819 template<
typename M,
typename C>
821 RedistributeInformation<C>& ri)
823 ri.setNoRows(newComm.indexSet().size());
824 ri.setNoCopyRows(newComm.indexSet().size());
825 ri.setNoBackwardsCopyRows(origComm.indexSet().size());
826 redistributeSparsityPattern(origMatrix, newMatrix, origComm, newComm, ri);
827 redistributeMatrixEntries(origMatrix, newMatrix, origComm, newComm, ri);
832 void redistributeMatrixEntries(M& origMatrix, M& newMatrix,
833 Dune::Amg::SequentialInformation& origComm,
834 Dune::Amg::SequentialInformation& newComm,
835 RedistributeInformation<Dune::Amg::SequentialInformation>& ri)
837 DUNE_THROW(InvalidStateException,
"Trying to redistribute in sequential program!");
841 Dune::Amg::SequentialInformation& origComm,
842 Dune::Amg::SequentialInformation& newComm,
843 RedistributeInformation<Dune::Amg::SequentialInformation>& ri)
845 DUNE_THROW(InvalidStateException,
"Trying to redistribute in sequential program!");
A constant random access iterator for the Dune::ArrayList class.
Definition: arraylist.hh:368
A set consisting only of one item.
Definition: enumset.hh:61
Decorates an index set with the possibility to find a global index that is mapped to a specific local...
Definition: indexset.hh:507
derive error class from the base class in common
Definition: istlexception.hh:19
Default exception class for range errors.
Definition: exceptions.hh:254
A few common exception classes.
Provides a map between global and local indices.
Classes providing communication interfaces for overlapping Schwarz methods.
const IndexPair * pair(const std::size_t &local) const
Get the index pair corresponding to a local index.
#define DUNE_THROW(E, m)
Definition: exceptions.hh:218
constexpr auto max
Function object that returns the greater of the given values.
Definition: hybridutilities.hh:484
Dune namespace.
Definition: alignedallocator.hh:13
constexpr std::integral_constant< std::size_t, sizeof...(II)> size(std::integer_sequence< T, II... >)
Return the size of the sequence.
Definition: integersequence.hh:75
void redistributeMatrix(M &origMatrix, M &newMatrix, C &origComm, C &newComm, RedistributeInformation< C > &ri)
Redistribute a matrix according to given domain decompositions.
Definition: matrixredistribute.hh:820
Functionality for redistributing a parallel index set using graph partitioning.
Utility class to communicate and set the row sizes of a redistributed matrix.
Definition: matrixredistribute.hh:216
CommMatrixRowSize(const M &m_, RI &rowsize_)
Constructor.
Definition: matrixredistribute.hh:226
Utility class for comunicating the matrix entries.
Definition: matrixredistribute.hh:396
std::vector< size_t > * rowsize
row size information for the receiving side.
Definition: matrixredistribute.hh:452
M & matrix
The matrix to communicate the values of.
Definition: matrixredistribute.hh:446
CommMatrixRow(M &m_, const Dune::GlobalLookupIndexSet< I > &idxset_, const I &aggidxset_, std::vector< size_t > &rowsize_)
Constructor.
Definition: matrixredistribute.hh:412
const Dune::GlobalLookupIndexSet< I > & idxset
Index set for the original matrix.
Definition: matrixredistribute.hh:448
void setOverlapRowsToDirichlet()
Sets the non-owner rows correctly as Dirichlet boundaries.
Definition: matrixredistribute.hh:421
const I & aggidxset
Index set for the redistributed matrix.
Definition: matrixredistribute.hh:450
CommMatrixRow(M &m_, const Dune::GlobalLookupIndexSet< I > &idxset_, const I &aggidxset_)
Constructor.
Definition: matrixredistribute.hh:405
Utility class to communicate and build the sparsity pattern of a redistributed matrix.
Definition: matrixredistribute.hh:245
void storeSparsityPattern(M &m)
Creates and stores the sparsity pattern of the redistributed matrix.
Definition: matrixredistribute.hh:276
void completeSparsityPattern(std::vector< std::set< size_type > > add_sparsity)
Completes the sparsity pattern of the redistributed matrix with data from copy rows for the novlp cas...
Definition: matrixredistribute.hh:340
CommMatrixSparsityPattern(const M &m_, const Dune::GlobalLookupIndexSet< I > &idxset_, const I &aggidxset_)
Constructor for the original side.
Definition: matrixredistribute.hh:254
CommMatrixSparsityPattern(const M &m_, const Dune::GlobalLookupIndexSet< I > &idxset_, const I &aggidxset_, const std::vector< typename M::size_type > &rowsize_)
Constructor for the redistruted side.
Definition: matrixredistribute.hh:265
Default policy used for communicating an indexed type.
Definition: communicator.hh:128
V::value_type IndexedType
The type we get at each index with operator[].
Definition: communicator.hh:147
static int getSize(const V &, int index)
Get the number of primitive elements at that index.
SizeOne IndexedTypeFlag
Whether the indexed type has variable size or there is always one value at each index.
Definition: communicator.hh:153
V Type
The type the policy is for.
Definition: communicator.hh:140
@ nonoverlapping
Category for non-overlapping solvers.
Definition: solvercategory.hh:27
static Category category(const OP &op, decltype(op.category()) *=nullptr)
Helperfunction to extract the solver category either from an enum, or from the newly introduced virtu...
Definition: solvercategory.hh:34
Flag for marking indexed data structures where the data at each index may be a variable multiple of a...
Definition: communicator.hh:118