datawriter.hh

#ifndef DUNE_FEM_DATAWRITER_HH
#define DUNE_FEM_DATAWRITER_HH
 
#include <string>
#include <tuple>
#include <limits>
#include <memory>
 
#include <dune/fem/io/file/asciiparser.hh>
#include <dune/fem/io/file/iointerface.hh>
#include <dune/fem/io/file/iotuple.hh>
#include <dune/fem/io/parameter.hh>
#include <dune/fem/space/common/loadbalancer.hh>
#include <dune/fem/gridpart/adaptiveleafgridpart.hh>
#include <dune/fem/space/lagrange.hh>
#include <dune/fem/function/adaptivefunction.hh>
#include <dune/fem/quadrature/cachingquadrature.hh>
#include <dune/fem/io/file/persistencemanager.hh>
#include <dune/fem/io/file/dataoutput.hh>
#include <dune/fem/misc/gridname.hh>
#include <dune/grid/common/backuprestore.hh>
 
#if USE_GRAPE
#include <dune/grid/io/visual/grapedatadisplay.hh>
#endif
 
namespace Dune
{
 
  namespace Fem
  {
 
    struct DataWriterParameters : public DataOutputParameters
    {
      explicit DataWriterParameters ( std::string keyPrefix, const ParameterReader &parameter = Parameter::container() )
        : DataOutputParameters( keyPrefix, parameter )
      {}
 
      explicit DataWriterParameters ( const ParameterReader &parameter = Parameter::container() )
        : DataOutputParameters( parameter )
      {}
 
      virtual std::string macroGridName (const int dim) const
      {
        return parameter().getValue< std::string >( IOInterface::defaultGridKey( dim, parameter() ) );
      }
 
      virtual bool separateRankPath () const
      {
        return false;
      }
    };
 
    template <class GridImp,
              class DataImp>
    class DataWriter : public DataOutput< GridImp, DataImp >
    {
    protected:
      typedef GridImp GridType;
      typedef DataImp OutPutDataType;
 
      typedef DataWriter< GridImp, DataImp > ThisType;
 
      typedef DataOutput< GridImp, DataImp > BaseType;
 
      using BaseType :: grid_;
      using BaseType :: data_;
 
      using BaseType :: path_;
      using BaseType :: datapref_;
      using BaseType :: writeStep_;
      using BaseType :: outputFormat_ ;
 
      friend class DataOutput< GridImp, DataImp >;
      mutable std::stringstream macroGrid_;
      const bool separateRankPath_ ;
 
    public:
 
      DataWriter(const GridType & grid,
                 OutPutDataType& data,
                 const DataWriterParameters& parameter = DataWriterParameters() )
        : BaseType( grid, data, parameter ),
          separateRankPath_( parameter.separateRankPath() )
      {
      }
 
      DataWriter(const GridType & grid,
                 OutPutDataType& data,
                 const TimeProviderBase& tp,
                 const DataWriterParameters& parameter = DataWriterParameters() )
        : BaseType( grid, data, tp, parameter ),
          separateRankPath_( parameter.separateRankPath() )
      {
      }
 
      virtual ~DataWriter() {}
 
    protected:
      virtual const char* myClassName() const { return "DataWriter"; }
 
      virtual void writeBinaryData(const double sequenceStamp) const
      {
        writeMyBinaryData( sequenceStamp, writeStep_ , data_ );
      }
 
      template< class OutputTuple >
      std::string writeMyBinaryData ( const double sequenceStamp, const int step,
                                      OutputTuple &data ) const
      {
        // create new path for time step output
        std::string timeStepPath = IOInterface::createPath( grid_.comm(), path_, datapref_, step, separateRankPath_ );
 
        typedef IOTuple< OutputTuple > IOTupleType ;
 
        // if data is given, use BinaryDataIO to write it
        if( IOTupleType :: length > 0 )
        {
          // call output of IOTuple
          IOTupleType :: output( grid_, sequenceStamp, timeStepPath, datapref_, data );
        }
 
        return timeStepPath;
      }
    }; // end class DataWriter
 
    //
    //  Checkpointer
    //
 
    struct CheckPointerParameters : public DataWriterParameters
    {
    protected:
      bool writeMode_;
 
    public:
      CheckPointerParameters( const bool writeMode, const std::string keyPrefix = "fem.io." ) :
        DataWriterParameters( keyPrefix, Parameter::container() ),
        writeMode_( writeMode )
      {}
 
      explicit CheckPointerParameters( const std::string keyPrefix = "fem.io." ) :
        DataWriterParameters( keyPrefix, Parameter::container() ),
        writeMode_( true )
      {}
 
      virtual std::string prefix () const
      {
        return checkPointPrefix();
      }
 
      virtual int checkPointStep() const
      {
        return parameter().getValue< int > ( keyPrefix_ + "checkpointstep", 500 );
      }
 
      virtual int maxNumberOfCheckPoints() const
      {
        return parameter().getValue< int > ( keyPrefix_ + "checkpointmax", 2 );
      }
 
      virtual std::string checkPointPrefix() const
      {
        return parameter().getValue< std::string > ( keyPrefix_ +  "checkpointfile", "checkpoint" );
      }
 
      virtual bool writeMode() const
      {
        return writeMode_;
      }
 
      virtual bool separateRankPath () const
      {
        return false;
      }
 
      virtual int outputformat() const
      {
        return 5; // i.e. none
      }
    };
 
    template< class GridImp >
    class CheckPointer
    : public DataWriter< GridImp, std::tuple<> >
    {
      typedef std::tuple<> DataImp;
    protected:
      typedef GridImp GridType;
 
      typedef DofManager< GridType > DofManagerType ;
 
      struct GridPersistentObject : public PersistentObject
      {
        const GridType& grid_ ;
        const std::string name_;
 
        GridPersistentObject( const GridType& grid )
          : grid_( grid ),
            name_( Fem :: gridName( grid_ ) )
        {
          // we need to push the grid at the
          // front position of the PersistenceManager list
          // since we have to read it first on restore
          const bool pushFront = true ;
          // add grid at first position
          PersistenceManager::insert( *this, pushFront );
        }
 
        ~GridPersistentObject()
        {
          // remove myself
          PersistenceManager::remove( *this );
        }
 
        virtual void backup() const
        {
          // this feature is available in dune-grid 2.3.x and later
 
          // try backup using stream method first
          try
          {
            std::string gridBackup ;
            // get backup stream from grid facility
            {
              std::stringstream gridBackupStream;
              Dune::BackupRestoreFacility< GridType > :: backup( grid_, gridBackupStream );
              gridBackup = gridBackupStream.str();
            }
            PersistenceManager :: backupStream() << gridBackup;
          }
          catch ( const Dune :: NotImplemented& )
          {
#ifndef NDEBUG
            if( Parameter :: verbose () )
              std::cerr << "GridPersistentObject::backup: cannot use stream backup." << std::endl;
#endif
 
            // try method given a filename
            try {
              std::string filename( PersistenceManager :: uniqueFileName( name_ ) );
              Dune::BackupRestoreFacility< GridType > :: backup( grid_, filename );
            }
            catch ( const Dune :: NotImplemented& )
            {
              std::cerr << "ERROR: GridPersistentObject::backup: not possible!" << std::endl;
            }
          }
 
          // backup dof manager
          DofManagerType :: instance( grid_ ).backup();
        }
 
        virtual void restore ()
        {
        }
      };
 
      typedef DataWriter<GridImp,DataImp> BaseType;
 
      using BaseType :: grid_;
      using BaseType :: data_;
 
      using BaseType :: path_;
      using BaseType :: datapref_;
      using BaseType :: writeStep_;
      using BaseType :: outputFormat_ ;
      using BaseType :: grapeDisplay_;
      using BaseType :: separateRankPath_;
 
      typedef CheckPointer< GridImp > ThisType;
 
      typedef DataImp OutPutDataType;
      OutPutDataType fakeData_; // empty tuple
 
      typedef GridPersistentObject PersistentGridObjectType;
      std::unique_ptr< PersistentGridObjectType > persistentGridObject_ ;
 
      OutPutDataType* dataPtr_ ;
 
      const int checkPointStep_;
      const int maxCheckPointNumber_;
      int myRank_;
 
      std::string checkPointFile_;
 
      bool takeCareOfPersistenceManager_;
 
    public:
      CheckPointer(const GridType & grid,
                   const CheckPointerParameters& parameter = CheckPointerParameters() )
        : BaseType(grid, fakeData_, parameter)
        , persistentGridObject_( new PersistentGridObjectType( grid_ ) )
        , dataPtr_ ( 0 )
        , checkPointStep_( parameter.checkPointStep() )
        , maxCheckPointNumber_( parameter.maxNumberOfCheckPoints() )
        , myRank_( grid.comm().rank() )
        , takeCareOfPersistenceManager_( true )
      {
        initialize( parameter );
      }
 
      CheckPointer(const GridType & grid,
                   const TimeProviderBase& tp,
                   const CheckPointerParameters& parameter = CheckPointerParameters() )
        : BaseType(grid, fakeData_,tp,parameter)
        , persistentGridObject_( new PersistentGridObjectType( grid_ ) )
        , checkPointStep_( parameter.checkPointStep() )
        , maxCheckPointNumber_( parameter.maxNumberOfCheckPoints() )
        , myRank_( grid.comm().rank() )
        , takeCareOfPersistenceManager_( true )
      {
        initialize( parameter );
      }
 
    protected:
      void initialize( const CheckPointerParameters& parameter )
      {
        // output format can only be binary
        outputFormat_ = BaseType :: binary;
        // do not display
        grapeDisplay_ = false ;
 
        checkPointFile_ = path_;
        checkPointFile_ += "/";
        checkPointFile_ += parameter.prefix();
 
        // write parameter file
        Parameter::write("parameter.log");
      }
 
    protected:
      CheckPointer(const GridType & grid,
                   const int myRank,
                   const char * checkFile,
                   const bool takeCareOfPersistenceManager = true,
                   const int writeStep = 0 )
        : BaseType(grid, fakeData_, CheckPointerParameters( checkFile == 0 ) ) // checkFile != 0 means read mode
        , persistentGridObject_( ) // do not create a persistent object here, since we are in read mode
        , checkPointStep_( 0 )
        , maxCheckPointNumber_( writeStep + 1 )
        , myRank_( myRank )
        , takeCareOfPersistenceManager_( takeCareOfPersistenceManager )
      {
        // output format can oinly be binary
        outputFormat_ = BaseType :: binary;
        // do not display
        grapeDisplay_ = false ;
 
        CheckPointerParameters parameter( checkFile == 0 );
        datapref_ = parameter.checkPointPrefix();
 
        if( checkFile )
        {
          // try to read given check point file
          checkPointFile_ = checkFile;
 
          // read last counter, don't issue warning
          bool ok = readCheckPoint( false );
 
          // if check point couldn't be opened, try again with default
          if(!ok)
          {
 
            // read name of check point file
            checkPointFile_ = path_;
            checkPointFile_ += "/";
            checkPointFile_ += parameter.checkPointPrefix();
 
            const bool warn = (myRank == 0);
            if( warn )
            {
              std::cerr << "WARNING: Coudn't open file `" << checkFile << "' trying file `" << checkPointFile_ << "' instead!" << std::endl;
            }
 
            ok = readCheckPoint();
            if( ! ok )
            {
              std::cerr <<"ERROR: unable to open checkpoint file! \n";
              exit(1);
            }
          }
        }
        else
        {
          initialize( CheckPointerParameters( checkFile == 0 ) );
        }
 
        // set write step counter to value given in constructor
        writeStep_ = writeStep ;
      }
 
    public:
      static GridType* restoreGrid(const std::string checkFile,
                                   const int givenRank = -1,
                                   const CheckPointerParameters& parameter = CheckPointerParameters() )
      {
        const int rank = ( givenRank < 0 ) ? MPIManager :: rank() : givenRank ;
        std::string datapref( parameter.checkPointPrefix() );
        std::string path;
 
        const bool verbose = (rank == 0);
 
        int checkPointNumber = 0;
        // if given checkpointfile is not valid use default checkpoint file
        if( ! readParameter(checkFile,"LastCheckPoint",checkPointNumber, verbose, false ) )
        {
          // read default path
          path = IOInterface::readPath();
          // set checkpointfile
          std::string checkPointFile = path;
          // try out default checkpoint file
          checkPointFile += "/";
          checkPointFile += parameter.checkPointPrefix();
          if ( verbose )
          {
            std::cerr << "WARNING: Coudn't open file `" << checkFile << "' trying file `" << checkPointFile << "' instead!" << std::endl;
          }
          readParameter(checkPointFile,"LastCheckPoint",checkPointNumber, verbose);
        }
        else
        {
          if( ! readParameter(checkFile,"RecoverPath",path,verbose) )
          {
            // read default path
            path = IOInterface::readPath();
          }
        }
 
        // now add timestamp (and rank)
        path = IOInterface::createRecoverPath(
            path, rank, datapref, checkPointNumber, parameter.separateRankPath() );
 
        // initialize PersistenceManager
        PersistenceManager :: startRestore ( path );
 
        GridType* grid = 0;
        // this is only available in dune-grid 2.3.x and later
        try
        {
          std::string gridData;
          PersistenceManager :: restoreStream() >> gridData;
 
          // copy data to stream
          std::stringstream gridStream( gridData );
          // clear grid data
          gridData = std::string();
 
          // perform restore using grid stream only
          grid = Dune::BackupRestoreFacility< GridType > :: restore( gridStream );
        }
        catch ( const Dune :: NotImplemented& )
        {
#ifndef NDEBUG
          if( Parameter :: verbose () )
            std::cerr << "GridPersistentObject::restore: cannot use stream restore." << std::endl;
#endif
          try {
            std::string name ( Fem :: gridName( *grid ) );
            std::string filename( PersistenceManager :: uniqueFileName( name ) );
            grid = Dune::BackupRestoreFacility< GridType > :: restore( filename );
          }
          catch ( const Dune :: NotImplemented& )
          {
            std::cerr << "ERROR: GridPersistentObject::restore: not possible!" << std::endl;
          }
        }
 
        if( grid == 0 )
        {
          DUNE_THROW(InvalidStateException,"Could not recover grid");
        }
 
        return grid;
      }
 
      static inline
      void restoreData ( const GridType &grid, const std::string checkFile, const int rank=-1  )
      {
        // make rank exchangable
        const int myRank = ( rank < 0 ) ? grid.comm().rank() : rank ;
 
        // check that check point is not empty
        if( checkFile == "" )
        {
          DUNE_THROW(InvalidStateException,"Checkpoint file empty!");
        }
 
        // create temporary check pointer
        CheckPointer<GridType> checker( grid, myRank, checkFile.c_str() );
 
        // restore data
        checker.restoreData();
      }
 
    protected:
      std::string restorePersistentData()
      {
        // now add timestamp and rank
        std::string path = IOInterface::createRecoverPath(
            path_, myRank_ , datapref_, writeStep_, separateRankPath_ );
 
        // if true also restore PersistenceManager
        if( takeCareOfPersistenceManager_ )
        {
          // restore all persistent values kept by PersistenceManager
          PersistenceManager::restore( path );
        }
 
        return path;
      }
 
      template< class InputTuple >
      void restoreUserData ( InputTuple &data )
      {
        // restore of the grid is done in method restoreGrid
        // here we only need to restore the DofManager since now
        // all spaces and index sets have been created
        DofManagerType :: instance( grid_ ).restore();
 
        // restore persistent data
        std::string path = restorePersistentData( );
 
        // make data consecutive at the end of the restore process
        // and communicate data
        DofManagerType :: instance( grid_ ).compress();
      }
 
      void restoreData( )
      {
        restoreUserData( data_ );
      }
 
    public:
      virtual const char* myClassName() const { return "CheckPointer"; }
 
      using BaseType :: willWrite ;
      bool willWrite(const TimeProviderBase& tp) const
      {
        const int timestep = tp.timeStep();
        // only write data time > saveTime
        return ( (checkPointStep_ > 0) && (((timestep % checkPointStep_) == 0) && timestep > 0) );
      }
 
      static void writeSingleCheckPoint(const GridType& grid,
                                        const double time,
                                        const bool storePersistenceManager,
                                        const int writeStep = 0 )
      {
        CheckPointer< GridType > checkPointer( grid );
        checkPointer.writeBinaryData( time );
      }
 
      virtual void writeBinaryData(const double time) const
      {
        // reset writeStep_ when maxCheckPointNumber_ is reached
        if( writeStep_ >= maxCheckPointNumber_ ) writeStep_ = 0;
 
        // write data
        std::string path = this->writeMyBinaryData( time, writeStep_, data_ );
 
        // if true also backup PersistenceManager
        if( takeCareOfPersistenceManager_ )
        {
          // backup all persistent values kept by PersistenceManager
          PersistenceManager::backup( path );
        }
 
        // write checkpoint info
        writeCheckPoint(path, time,
                        writeStep_ );
 
        return;
      }
 
    protected:
      bool readCheckPoint(const bool warn = true)
      {
        const bool verbose = Parameter::verbose();
 
        // read Checkpiont file
        if( readParameter(checkPointFile_,"LastCheckPoint",writeStep_, verbose, warn ) )
        {
          std::string recoverPath;
          // try to read recover path
          if( ! readParameter(checkPointFile_,"RecoverPath", recoverPath, verbose) )
          {
            // default value is output path
            recoverPath = path_;
          }
 
          int storedPersistentManager = 0;
          if( readParameter(checkPointFile_,"PersistenceManager", storedPersistentManager, verbose) )
          {
            takeCareOfPersistenceManager_ = ( storedPersistentManager > 0 );
          }
 
          // overwrite path with recover path
          path_ = recoverPath;
 
          return true;
        }
        return false;
      }
 
      // write some info for checkpointing
      void writeCheckPoint (const std::string& path,
                            const double time,
                            const int savestep ) const
      {
        // write some needed informantion to current checkpoint file
        // only proc 0 writes the global checkpoint file
        if( myRank_ <= 0 )
        {
          std::string checkpointstr ;
          {
            std::stringstream checkpoint;
            checkpoint << "LastCheckPoint: " << savestep << std::endl;
            checkpoint.precision( 16 );
            checkpoint << "Time: " << std::scientific << time << std::endl;
            checkpoint << "SaveCount: " << savestep << std::endl;
            checkpoint << "PersistenceManager: " << takeCareOfPersistenceManager_ << std::endl;
            checkpoint << "NumberProcessors: " << grid_.comm().size() << std::endl;
            checkpoint << "# RecoverPath can be edited by hand if data has been moved!" << std::endl;
            checkpoint << "RecoverPath: " << path_ << std::endl;
            checkpointstr = checkpoint.str();
          }
 
          // overwrite the last checkpoint file
          {
            std::ofstream file (checkPointFile_.c_str());
            if( file.is_open() )
            {
              file << checkpointstr;
            }
          }
 
          // write check point file for this checkpoint
          {
            std::string checkPointStepFile( path );
            checkPointStepFile += "/" + datapref_;
 
            std::ofstream file ( checkPointStepFile.c_str() );
            if( file.is_open() )
            {
              file << checkpointstr;
            }
          }
        }
      }
 
    }; // end class CheckPointer
 
  } // end namespace Fem
 
} // namespace Dune
 
#endif // #ifndef DUNE_FEM_DATAWRITER_HH