1#ifndef DUNE_FEM_CACHED_COMMUNICATION_MANAGER_HH
2#define DUNE_FEM_CACHED_COMMUNICATION_MANAGER_HH
15#include <dune/common/math.hh>
16#include <dune/common/timer.hh>
17#include <dune/common/visibility.hh>
20#include <dune/grid/common/grid.hh>
21#include <dune/grid/common/datahandleif.hh>
22#include <dune/grid/utility/entitycommhelper.hh>
26#include <dune/alugrid/3d/alu3dinclude.hh>
46 template<
class DiscreteFunctionSpace >
67 template<
class BlockMapper >
73 typedef BlockMapper BlockMapperType;
80 typedef std::vector< IndexMapType > IndexMapVectorType;
83 typedef std :: set< int > LinkStorageType;
86 typedef ALU3DSPACE ObjectStream ObjectStreamType;
89 typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
91 typedef ALU3DSPACE MpAccessMPI MPAccessImplType;
94 typedef std :: vector< ObjectStreamType > ObjectStreamVectorType;
97 const InterfaceType interface_;
98 const CommunicationDirection dir_;
100 LinkStorageType linkStorage_;
102 IndexMapVectorType recvIndexMap_;
103 IndexMapVectorType sendIndexMap_;
106 std::unique_ptr< MPAccessInterfaceType > mpAccess_;
109 double exchangeTime_;
116 int nonBlockingObjects_ ;
119 template<
class Communication,
class LinkStorage,
120 class IndexMapVector, InterfaceType CommInterface >
127 class NonBlockingCommunication
129 typedef DependencyCache < BlockMapper > DependencyCacheType;
132 typedef MPAccessInterfaceType :: NonBlockingExchange NonBlockingExchange;
134 template <
class DiscreteFunction>
135 class Pack :
public NonBlockingExchange :: DataHandleIF
138 NonBlockingCommunication& commObj_;
139 const DiscreteFunction& discreteFunction_;
142 Pack( NonBlockingCommunication& commObj,
const DiscreteFunction& df )
143 : commObj_( commObj ), discreteFunction_( df )
146 void pack(
const int link, ObjectStreamType& buffer )
148 commObj_.pack( link, buffer, discreteFunction_ );
151 void unpack(
const int link, ObjectStreamType& buffer )
153 DUNE_THROW(InvalidStateException,
"Pack::unpack should not be called!");
157 template <
class DiscreteFunction,
class Operation>
158 class Unpack :
public NonBlockingExchange :: DataHandleIF
161 NonBlockingCommunication& commObj_;
162 DiscreteFunction& discreteFunction_;
165 const Operation operation_;
168 Unpack( NonBlockingCommunication& commObj, DiscreteFunction& df )
169 : commObj_( commObj ), discreteFunction_( df ), operation_()
172 void pack(
const int link, ObjectStreamType& buffer )
174 DUNE_THROW(InvalidStateException,
"Unpack::pack should not be called!");
177 void unpack(
const int link, ObjectStreamType& buffer )
179 commObj_.unpack( link, buffer, discreteFunction_, operation_ );
183 typedef int NonBlockingExchange;
187 DUNE_EXPORT
static int getMessageTag()
189 enum { initial = 665 };
190 static int tagCounter = initial ;
192 int messageTag = tagCounter ;
197 messageTag = initial ;
198 tagCounter = initial ;
204 template <
class Space>
205 NonBlockingCommunication(
const Space& space,
206 DependencyCacheType& dependencyCache )
207 : dependencyCache_( dependencyCache ),
208 nonBlockingExchange_(),
210 exchangeTime_( 0.0 ),
211 mySize_( space.gridPart().comm().size() )
214 dependencyCache_.rebuild( space );
217 dependencyCache_.attachComm();
221 NonBlockingCommunication(
const NonBlockingCommunication& other )
222 : dependencyCache_( other.dependencyCache_ ),
223 nonBlockingExchange_(),
225 exchangeTime_( 0.0 ),
226 mySize_( other.mySize_ )
229 dependencyCache_.attachComm();
232 ~NonBlockingCommunication()
235 assert( ! nonBlockingExchange_ );
237 dependencyCache_.detachComm() ;
240 template <
class DiscreteFunctionSpace >
241 void send(
const PetscDiscreteFunction< DiscreteFunctionSpace >& discreteFunction )
246 template <
class DiscreteFunction >
247 void send(
const DiscreteFunction& discreteFunction )
250 assert( ! nonBlockingExchange_ );
253 if( mySize_ <= 1 )
return;
256 Dune::Timer sendTimer ;
259 const int nLinks = dependencyCache_.nlinks();
262 buffer_.resize( nLinks );
266 nonBlockingExchange_.reset( dependencyCache_.mpAccess().nonBlockingExchange( getMessageTag() ) );
269 Pack< DiscreteFunction > packData( *
this, discreteFunction );
272 nonBlockingExchange_->send( buffer_, packData );
275 for(
int link = 0; link < nLinks; ++link )
276 pack( link, buffer_[ link ], discreteFunction );
280 exchangeTime_ = sendTimer.elapsed();
284 template <
class DiscreteFunctionSpace,
class Operation >
285 double receive( PetscDiscreteFunction< DiscreteFunctionSpace >& discreteFunction,
286 const Operation& operation )
289 Dune::Timer exchTimer;
292 discreteFunction.dofVector().communicateNow( operation );
294 return exchTimer.elapsed();
298 template <
class DiscreteFunction,
class Operation >
299 double receive( DiscreteFunction& discreteFunction,
const Operation& operation )
302 if( mySize_ <= 1 )
return 0.0;
305 Dune::Timer recvTimer ;
309 Unpack< DiscreteFunction, Operation > unpackData( *
this, discreteFunction );
312 nonBlockingExchange_->receive( unpackData );
315 buffer_ = dependencyCache_.mpAccess().exchange( buffer_ );
318 const int nLinks = buffer_.size();
321 for(
int link = 0; link < nLinks; ++link )
322 unpack( link, buffer_[ link ], discreteFunction, operation );
326 exchangeTime_ += recvTimer.elapsed();
330 nonBlockingExchange_.reset();
332 return exchangeTime_;
336 template <
class DiscreteFunction >
337 double receive( DiscreteFunction& discreteFunction )
340 typedef typename DiscreteFunction :: DiscreteFunctionSpaceType
341 :: template CommDataHandle< DiscreteFunction > :: OperationType DefaultOperationType;
342 DefaultOperationType operation;
343 return receive( discreteFunction, operation );
347 template <
class DiscreteFunction>
348 void pack(
const int link, ObjectStreamType& buffer,
const DiscreteFunction& discreteFunction )
353 dependencyCache_.writeBuffer( link, buffer, discreteFunction );
356 template <
class DiscreteFunction,
class Operation>
357 void unpack(
const int link, ObjectStreamType& buffer,
358 DiscreteFunction& discreteFunction,
const Operation& operation )
361 dependencyCache_.readBuffer( link, buffer, discreteFunction, operation );
365 DependencyCacheType& dependencyCache_;
366 std::unique_ptr< NonBlockingExchange > nonBlockingExchange_ ;
367 ObjectStreamVectorType buffer_;
368 double exchangeTime_ ;
373 typedef NonBlockingCommunication NonBlockingCommunicationType;
376 template <
class Space>
377 NonBlockingCommunicationType nonBlockingCommunication(
const Space& space )
380 return NonBlockingCommunicationType( space, *
this );
387 DependencyCache(
const int nProcs,
const InterfaceType interface,
const CommunicationDirection dir )
388 : interface_( interface ),
391 recvIndexMap_( nProcs ),
392 sendIndexMap_( nProcs ),
394 exchangeTime_( 0.0 ),
397 nonBlockingObjects_( 0 )
401 template <
class Communication>
402 void init(
const Communication& comm )
406 mpAccess_.reset(
new MPAccessImplType( comm ) );
411 DependencyCache(
const DependencyCache & ) =
delete;
414 InterfaceType communicationInterface()
const
420 CommunicationDirection communicationDirection()
const
426 double buildTime()
const
432 double exchangeTime()
const
434 return exchangeTime_;
440 ++nonBlockingObjects_;
446 --nonBlockingObjects_;
447 assert( nonBlockingObjects_ >= 0 );
450 bool noOpenCommunications()
const
457 template <
class Space >
458 inline void buildMaps(
const Space& space );
461 inline void checkConsistency();
463 template<
class Space,
class Comm,
class LS,
class IMV, InterfaceType CI >
464 inline void buildMaps(
const Space& space, LinkBuilder< Comm, LS, IMV, CI > &handle );
468 inline int dest(
const int link )
const
470 return mpAccess().dest()[ link ];
474 inline int nlinks()
const
476 return mpAccess().nlinks();
482 template <
class Space>
483 inline void rebuild(
const Space& space )
485 const auto& comm = space.gridPart().comm();
486 const int spcSequence = space.sequence();
489 if( comm.size() <= 1 )
return;
492 assert( noOpenCommunications() );
496 int willRebuild = (sequence_ != spcSequence) ? 1 : 0;
497 const int myRebuild = willRebuild;
500 comm.broadcast( &willRebuild, 1 , 0);
502 assert( willRebuild == myRebuild );
506 if( sequence_ != spcSequence )
509 Dune::Timer buildTime;
514 sequence_ = spcSequence;
517 buildTime_ = buildTime.elapsed();
522 template<
class DiscreteFunction,
class Operation >
523 inline void exchange( DiscreteFunction &discreteFunction,
const Operation& operation );
526 template<
class DiscreteFunction >
527 inline void writeBuffer( ObjectStreamVectorType &osv,
const DiscreteFunction &discreteFunction )
const;
530 template<
class DiscreteFunctionType,
class Operation >
531 inline void readBuffer( ObjectStreamVectorType &osv,
532 DiscreteFunctionType &discreteFunction,
533 const Operation& operation )
const;
536 inline MPAccessInterfaceType &mpAccess()
543 inline const MPAccessInterfaceType &mpAccess()
const
551 template<
class DiscreteFunctionSpace >
552 inline void writeBuffer(
const int link,
553 ObjectStreamType &str,
554 const PetscDiscreteFunction< DiscreteFunctionSpace > &discreteFunction )
const
556 DUNE_THROW(NotImplemented,
"writeBuffer not implemented for PetscDiscteteFunction" );
561 template<
class DiscreteFunction >
562 inline void writeBuffer(
const int link,
563 ObjectStreamType &str,
564 const DiscreteFunction &discreteFunction )
const
566 assert( sequence_ == discreteFunction.space().sequence() );
567 const auto &indexMap = sendIndexMap_[ dest( link ) ];
568 const int size = indexMap.size();
570 typedef typename DiscreteFunction :: DofType DofType;
573 typename DiscreteFunction::DiscreteFunctionSpaceType::LocalBlockIndices localBlockIndices;
574 str.reserve( size * Hybrid::size( localBlockIndices ) *
sizeof( DofType ) );
575 for(
int i = 0; i < size; ++i )
577 const auto &block = discreteFunction.dofVector()[ indexMap[ i ] ];
578 Hybrid::forEach( localBlockIndices, [ &str, &block ] (
auto &&k ) { str.writeUnchecked( block[ k ] ); } );
584 template<
class DiscreteFunctionSpace,
class Operation >
585 inline void readBuffer(
const int link,
586 ObjectStreamType &str,
587 PetscDiscreteFunction< DiscreteFunctionSpace > &discreteFunction,
588 const Operation& )
const
590 DUNE_THROW(NotImplemented,
"readBuffer not implemented for PetscDiscteteFunction" );
595 template<
class DiscreteFunction,
class Operation >
596 inline void readBuffer(
const int link,
597 ObjectStreamType &str,
598 DiscreteFunction &discreteFunction,
599 const Operation& operation )
const
601 static_assert( ! std::is_pointer< Operation > :: value,
602 "DependencyCache::readBuffer: Operation needs to be a reference!");
604 assert( sequence_ == discreteFunction.space().sequence() );
605 typedef typename DiscreteFunction :: DofType DofType;
608 const auto &indexMap = recvIndexMap_[ dest( link ) ];
610 const int size = indexMap.size();
612 typename DiscreteFunction::DiscreteFunctionSpaceType::LocalBlockIndices localBlockIndices;
613 assert(
static_cast< std::size_t
>( size * Hybrid::size( localBlockIndices ) *
sizeof( DofType ) ) <=
static_cast< std::size_t
>( str.size() ) );
614 for(
int i = 0; i < size; ++i )
616 auto &&block = discreteFunction.dofVector()[ indexMap[ i ] ];
617 Hybrid::forEach( localBlockIndices, [ &str, &operation, &block ] (
auto &&k ) {
620 str.readUnchecked( value );
625 operation( value, block[ k ] );
632 template<
class BlockMapper >
633 template<
class Communication,
class LinkStorage,
class IndexMapVector, InterfaceType CommInterface >
634 class DependencyCache< BlockMapper > :: LinkBuilder
635 :
public CommDataHandleIF
636 < LinkBuilder< Communication, LinkStorage, IndexMapVector, CommInterface >,
637 typename BlockMapper :: GlobalKeyType >
640 typedef Communication CommunicationType;
641 typedef BlockMapper BlockMapperType;
643 typedef typename BlockMapperType :: GlobalKeyType GlobalKeyType;
645 typedef LinkStorage LinkStorageType;
646 typedef IndexMapVector IndexMapVectorType;
648 typedef GlobalKeyType DataType;
651 const CommunicationType& comm_;
652 const BlockMapperType &blockMapper_;
654 const GlobalKeyType myRank_;
655 const GlobalKeyType mySize_;
657 LinkStorageType &linkStorage_;
659 IndexMapVectorType &sendIndexMap_;
660 IndexMapVectorType &recvIndexMap_;
664 LinkBuilder(
const CommunicationType& comm,
665 const BlockMapperType& blockMapper,
666 LinkStorageType &linkStorage,
667 IndexMapVectorType &sendIdxMap,
668 IndexMapVectorType &recvIdxMap )
670 blockMapper_( blockMapper ),
671 myRank_( comm.rank() ),
672 mySize_( comm.size() ),
673 linkStorage_( linkStorage ),
674 sendIndexMap_( sendIdxMap ),
675 recvIndexMap_( recvIdxMap )
679 void sendBackSendMaps()
682 MPAccessImplType mpAccess( comm_ );
685 mpAccess.removeLinkage();
687 mpAccess.insertRequestSymetric( linkStorage_ );
689 std::vector<int> dest = mpAccess.dest();
691 const int nlinks = mpAccess.nlinks();
694 ObjectStreamVectorType osv( nlinks );
704 for(
int link=0; link<nlinks; ++link)
705 sendIndexMap_[ dest[link] ].writeToBuffer( osv[link] );
708 osv = mpAccess.exchange( osv );
711 for(
int link=0; link<nlinks; ++link)
712 sendIndexMap_[ dest[link] ].readFromBuffer( osv[link] );
723 bool contains(
int dim,
int codim )
const
725 return blockMapper_.contains( codim );
729 bool fixedSize(
int dim,
int codim )
const
735 template<
class MessageBuffer,
class Entity >
736 void gather( MessageBuffer &buffer,
const Entity &entity )
const
739 const auto myPartitionType = entity.partitionType();
740 const bool send = EntityCommHelper< CommInterface > :: send( myPartitionType );
746 buffer.write( myRank_ );
748 const int numDofs = blockMapper_.numEntityDofs( entity );
750 typedef std::vector< GlobalKeyType > IndicesType ;
751 IndicesType indices( numDofs );
754 blockMapper_.mapEachEntityDof( entity, AssignFunctor< IndicesType >( indices ) );
757 for(
int i = 0; i < numDofs; ++i )
758 buffer.write( indices[ i ] );
763 template<
class MessageBuffer,
class Entity >
764 void scatter( MessageBuffer &buffer,
const Entity &entity,
const size_t dataSize )
772 assert( (rank >= 0) && (rank < mySize_) );
775 const auto myPartitionType = entity.partitionType();
776 const bool receive = EntityCommHelper< CommInterface > :: receive( myPartitionType );
779 linkStorage_.insert( rank );
782 typedef std::vector< GlobalKeyType > IndicesType ;
783 IndicesType indices( dataSize - 1 );
784 for(
size_t i=0; i<dataSize-1; ++i)
785 buffer.read( indices[i] );
800 sendIndexMap_[ rank ].insert( indices );
803 const int numDofs = blockMapper_.numEntityDofs( entity );
804 indices.resize( numDofs );
807 blockMapper_.mapEachEntityDof( entity, AssignFunctor< IndicesType >( indices ) );
810 recvIndexMap_[ rank ].insert( indices );
816 template<
class Entity >
817 size_t size(
const Entity &entity )
const
819 const PartitionType myPartitionType = entity.partitionType();
820 const bool send = EntityCommHelper< CommInterface > :: send( myPartitionType );
821 return (send) ? (blockMapper_.numEntityDofs( entity ) + 1) : 0;
827 template<
class BlockMapper >
828 template<
class Space >
829 inline void DependencyCache< BlockMapper > :: buildMaps(
const Space& space )
831 typedef typename Space::GridPartType::CollectiveCommunicationType CommunicationType;
832 if( interface_ == InteriorBorder_All_Interface )
834 LinkBuilder< CommunicationType, LinkStorageType, IndexMapVectorType,
835 InteriorBorder_All_Interface >
836 handle( space.gridPart().comm(),
838 linkStorage_, sendIndexMap_, recvIndexMap_ );
839 buildMaps( space, handle );
841 else if( interface_ == InteriorBorder_InteriorBorder_Interface )
843 LinkBuilder< CommunicationType, LinkStorageType, IndexMapVectorType,
844 InteriorBorder_InteriorBorder_Interface >
845 handle( space.gridPart().comm(),
847 linkStorage_, sendIndexMap_, recvIndexMap_ );
848 buildMaps( space, handle );
850 else if( interface_ == All_All_Interface )
852 LinkBuilder< CommunicationType, LinkStorageType, IndexMapVectorType, All_All_Interface >
853 handle( space.gridPart().comm(),
855 linkStorage_, sendIndexMap_, recvIndexMap_ );
856 buildMaps( space, handle );
859 DUNE_THROW( NotImplemented,
"DependencyCache for the given interface has not been implemented, yet." );
867 template<
class BlockMapper >
868 template<
class Space,
class Comm,
class LS,
class IMV, InterfaceType CI >
869 inline void DependencyCache< BlockMapper >
870 :: buildMaps(
const Space& space, LinkBuilder< Comm, LS, IMV, CI > &handle )
872 linkStorage_.clear();
873 const size_t size = recvIndexMap_.size();
874 for(
size_t i = 0; i < size; ++i )
876 recvIndexMap_[ i ].clear();
877 sendIndexMap_[ i ].clear();
881 space.gridPart().communicate( handle, All_All_Interface , ForwardCommunication );
884 mpAccess().removeLinkage();
886 mpAccess().insertRequestSymetric( linkStorage_ );
889 template<
class BlockMapper >
890 inline void DependencyCache< BlockMapper > :: checkConsistency()
892 const int nLinks = nlinks();
894 ObjectStreamVectorType buffer( nLinks );
897 for(
int l=0; l<nLinks; ++l)
900 const int sendSize = sendIndexMap_[ dest( l ) ].size();
901 buffer[l].write( sendSize );
902 for(
int i=0; i<sendSize; ++i)
903 buffer[l].write( i );
907 buffer = mpAccess().exchange( buffer );
910 for(
int l=0; l<nLinks; ++l)
912 const int recvSize = recvIndexMap_[ dest( l ) ].size();
914 buffer[l].read( sendedSize );
917 if( recvSize != sendedSize )
919 DUNE_THROW(InvalidStateException,
"Sizes do not match!" << sendedSize <<
" o|r " << recvSize);
922 for(
int i=0; i<recvSize; ++i)
925 buffer[l].read( idx );
930 DUNE_THROW(InvalidStateException,
"Wrong ordering of send and recv maps!");
936 template<
class BlockMapper >
937 template<
class DiscreteFunction,
class Operation >
938 inline void DependencyCache< BlockMapper >
939 :: exchange( DiscreteFunction &discreteFunction,
const Operation& operation )
941 const auto& space = discreteFunction.space();
944 if( space.gridPart().comm().size() <= 1 )
return;
947 NonBlockingCommunicationType nbc( space, *
this );
950 nbc.send( discreteFunction );
953 exchangeTime_ = nbc.receive( discreteFunction, operation );
956 template<
class BlockMapper >
957 template<
class DiscreteFunction >
958 inline void DependencyCache< BlockMapper >
959 :: writeBuffer( ObjectStreamVectorType &osv,
960 const DiscreteFunction &discreteFunction )
const
962 const int numLinks = nlinks();
963 for(
int link = 0; link < numLinks; ++link )
964 writeBuffer( link, osv[ link ], discreteFunction );
967 template<
class BlockMapper >
968 template<
class DiscreteFunction,
class Operation >
969 inline void DependencyCache< BlockMapper >
970 :: readBuffer( ObjectStreamVectorType &osv,
971 DiscreteFunction &discreteFunction,
972 const Operation& operation )
const
974 const int numLinks = nlinks();
975 for(
int link = 0; link < numLinks; ++link )
976 readBuffer( link, osv[ link ], discreteFunction, operation );
980 template <
class BlockMapper >
981 class CommManagerSingletonKey
983 const BlockMapper& blockMapper_;
984 const InterfaceType interface_;
985 const CommunicationDirection dir_;
989 CommManagerSingletonKey(
const int pSize,
990 const BlockMapper& blockMapper,
991 const InterfaceType interface,
992 const CommunicationDirection dir)
993 : blockMapper_( blockMapper ),
994 interface_(interface), dir_(dir), pSize_( pSize )
998 CommManagerSingletonKey(
const CommManagerSingletonKey & org) =
default;
1001 bool operator == (
const CommManagerSingletonKey & otherKey)
const
1004 return (&(blockMapper_) == &(otherKey.blockMapper_) );
1008 InterfaceType interface()
const
1014 CommunicationDirection direction()
const
1020 int pSize ()
const {
return pSize_; }
1025 template <
class KeyImp,
class ObjectImp>
1026 class CommManagerFactory
1030 static ObjectImp * createObject(
const KeyImp & key )
1032 return new ObjectImp(key.pSize(), key.interface(), key.direction());
1036 static void deleteObject( ObjectImp * obj )
1043 template <
class SpaceImp>
1044 class CommunicationManager
1046 typedef CommunicationManager<SpaceImp>
ThisType;
1048 typedef typename SpaceImp::BlockMapperType BlockMapperType;
1051 typedef DependencyCache< BlockMapperType > DependencyCacheType;
1053 typedef CommManagerSingletonKey< BlockMapperType > KeyType;
1054 typedef CommManagerFactory<KeyType, DependencyCacheType> FactoryType;
1056 typedef SingletonList< KeyType , DependencyCacheType , FactoryType > CommunicationProviderType;
1061 typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
1064 std::unique_ptr< DependencyCacheType, typename CommunicationProviderType::Deleter > cache_;
1067 CommunicationManager(
const ThisType& org) =
delete;
1073 CommunicationManager(
const SpaceType& space,
1074 const InterfaceType interface,
1075 const CommunicationDirection dir)
1078 , cache_( &CommunicationProviderType::getObject(
1079 KeyType( space.gridPart().comm().size(),
space_.blockMapper(), interface,dir) ) )
1082 cache().init( space.gridPart().comm() );
1088 CommunicationManager(
const SpaceType& space)
1092 DependencyCacheType& cache ()
const { assert( cache_ );
return *cache_; }
1097 return cache().communicationInterface();
1103 return cache().communicationDirection();
1109 return cache().buildTime();
1115 return cache().exchangeTime();
1118 MPAccessInterfaceType& mpAccess()
1120 return cache().mpAccess();
1126 return cache().nonBlockingCommunication(
space_ );
1131 template <
class DiscreteFunctionType>
1132 void exchange(DiscreteFunctionType & df)
const
1135 typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType
1136 :: template CommDataHandle< DiscreteFunctionType > :: OperationType DefaultOperationType;
1139 DefaultOperationType operation;
1146 template <
class DiscreteFunctionType,
class Operation>
1147 void exchange(DiscreteFunctionType & df,
const Operation& operation )
const
1149 cache().exchange( df, operation );
1153 template <
class ObjectStreamVectorType,
class DiscreteFunctionType>
1154 void writeBuffer(ObjectStreamVectorType& osv,
const DiscreteFunctionType & df)
const
1156 cache().writeBuffer( osv, df );
1160 template <
class ObjectStreamVectorType,
class DiscreteFunctionType>
1161 void readBuffer(ObjectStreamVectorType& osv, DiscreteFunctionType & df)
const
1163 typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType
1164 :: template CommDataHandle<DiscreteFunctionType> :: OperationType OperationType;
1167 OperationType operation;
1169 readBuffer( osv, df, operation );
1173 template <
class ObjectStreamVectorType,
class DiscreteFunctionType,
class OperationType>
1174 void readBuffer(ObjectStreamVectorType& osv, DiscreteFunctionType & df,
const OperationType& operation)
const
1176 cache().readBuffer( osv, df , operation);
1182 cache().rebuild(
space_ );
1187 class CommunicationManagerList
1190 template <
class MPAccessType,
class ObjectStreamVectorType>
1191 class DiscreteFunctionCommunicatorInterface
1200 virtual MPAccessType& mpAccess() = 0;
1201 virtual void writeBuffer(ObjectStreamVectorType&)
const = 0;
1202 virtual void readBuffer(ObjectStreamVectorType&) = 0;
1203 virtual void rebuildCache() = 0;
1205 virtual bool handles ( IsDiscreteFunction &df )
const = 0;
1211 template <
class DiscreteFunctionImp,
1213 class ObjectStreamVectorType,
1214 class OperationType >
1215 class DiscreteFunctionCommunicator
1216 :
public DiscreteFunctionCommunicatorInterface<MPAccessType,ObjectStreamVectorType>
1218 typedef DiscreteFunctionImp DiscreteFunctionType;
1219 typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType DiscreteFunctionSpaceType;
1221 typedef CommunicationManager<DiscreteFunctionSpaceType> CommunicationManagerType;
1224 DiscreteFunctionType& df_;
1226 CommunicationManagerType comm_;
1229 const OperationType operation_;
1232 DiscreteFunctionCommunicator(DiscreteFunctionType& df,
const OperationType& op )
1233 : df_(df), comm_(df_.space()), operation_( op )
1237 virtual MPAccessType& mpAccess()
1239 return comm_.mpAccess();
1243 virtual void writeBuffer(ObjectStreamVectorType& osv)
const
1245 comm_.writeBuffer(osv,df_);
1249 virtual void readBuffer(ObjectStreamVectorType& osv)
1251 comm_.readBuffer(osv, df_, operation_ );
1255 virtual void rebuildCache()
1257 comm_.rebuildCache();
1260 virtual bool handles ( IsDiscreteFunction &df )
const {
return (&
static_cast< IsDiscreteFunction &
>( df_ ) == &df); }
1264 typedef ALU3DSPACE ObjectStream ObjectStreamType;
1267 typedef std::vector< ObjectStreamType > ObjectStreamVectorType;
1270 typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
1273 typedef DiscreteFunctionCommunicatorInterface<MPAccessInterfaceType,ObjectStreamVectorType>
1274 CommObjInterfaceType;
1277 typedef std::list < std::unique_ptr< CommObjInterfaceType > > CommObjListType;
1278 CommObjListType objList_;
1285 template <
class CombinedObjectType>
1290 cObj.addToList(*
this);
1301 template <
class DiscreteFunctionImp,
class Operation>
1302 void addToList(DiscreteFunctionImp &df,
const Operation& operation )
1305 typedef DiscreteFunctionCommunicator<DiscreteFunctionImp,
1306 MPAccessInterfaceType,
1307 ObjectStreamVectorType,
1308 Operation > CommObj;
1309 CommObj * obj =
new CommObj(df, operation);
1310 objList_.push_back( std::unique_ptr< CommObjInterfaceType > (obj) );
1316 MPAccessInterfaceType& mpAccess = objList_.front()->mpAccess();
1319 mySize_ = mpAccess.psize();
1324 template <
class DiscreteFunctionImp>
1327 DFCommunicationOperation::Copy operation;
1331 template<
class DiscreteFunction >
1334 const auto handles = [ &df ] (
const std::unique_ptr< CommObjInterfaceType > &commObj ) { assert( commObj );
return commObj->handles( df ); };
1335 CommObjListType::reverse_iterator pos = std::find_if( objList_.rbegin(), objList_.rend(), handles );
1336 if( pos != objList_.rend() )
1337 objList_.erase( --pos.base() );
1339 DUNE_THROW( RangeError,
"Trying to remove discrete function that was never added" );
1347 if( mySize_ <= 1 ) return ;
1350 if(objList_.size() > 0)
1353 for(
auto& elem : objList_)
1354 elem->rebuildCache();
1357 auto& mpAccess = objList_.front()->mpAccess();
1360 ObjectStreamVectorType osv( mpAccess.nlinks() );
1363 for(
auto& elem : objList_)
1364 elem->writeBuffer(osv);
1367 osv = mpAccess.exchange(osv);
1370 for(
auto& elem : objList_)
1371 elem->readBuffer(osv);
const SpaceType & space_
Definition: communicationmanager.hh:165
double buildTime() const
return time needed for last build
Definition: communicationmanager.hh:203
InterfaceType communicationInterface() const
return communication interface
Definition: communicationmanager.hh:189
DefaultCommunicationManager< SpaceImp > ThisType
Definition: communicationmanager.hh:82
virtual bool handles(IsDiscreteFunction &df) const =0
DiscreteFunctionCommunicatorInterface()=default
SpaceImp SpaceType
Definition: communicationmanager.hh:79
void exchange() const
Definition: communicationmanager.hh:377
void exchange(DiscreteFunction &discreteFunction) const
exchange data for a discrete function using the copy operation
Definition: communicationmanager.hh:225
CommunicationDirection communicationDirection() const
return communication direction
Definition: communicationmanager.hh:194
NonBlockingCommunication NonBlockingCommunicationType
Definition: communicationmanager.hh:173
NonBlockingCommunicationType nonBlockingCommunication() const
return object for non-blocking communication
Definition: communicationmanager.hh:215
bool handles(IsDiscreteFunction &df) const
Definition: communicationmanager.hh:329
double exchangeTime() const
return time needed for last exchange of data
Definition: communicationmanager.hh:209
void addToList(DiscreteFunctionImp &df, const Operation &operation)
add discrete function to communication list
Definition: communicationmanager.hh:349
void removeFromList(DiscreteFunction &df)
Definition: communicationmanager.hh:365
CommunicationManagerList()=default
virtual ~DiscreteFunctionCommunicatorInterface()=default
Definition: bindguard.hh:11
base class for determing whether a class is a discrete function or not
Definition: common/discretefunction.hh:53
Definition: cachedcommmanager.hh:47
Definition: commindexmap.hh:16