Tsqr_TpetraMessenger.hpp

// @HEADER
// ***********************************************************************
//
//                 Anasazi: Block Eigensolvers Package
//                 Copyright (2010) Sandia Corporation
//
// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
// license for use of this work by or on behalf of the U.S. Government.
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
// USA
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ***********************************************************************
// @HEADER

#ifndef __TSQR_Trilinos_TpetraMessenger_hpp
#define __TSQR_Trilinos_TpetraMessenger_hpp

#include "Teuchos_CommHelpers.hpp"
#include "Tsqr_MessengerBase.hpp"
#include <algorithm>


namespace TSQR { 
  namespace Trilinos {
    template< class Datum >
    class TpetraMessenger : public MessengerBase< Datum > {
    public:
      typedef Teuchos::RCP< const Teuchos::Comm<int> > comm_ptr;

      TpetraMessenger (const comm_ptr& pComm) : pComm_ (pComm) {}

      virtual void 
      send (const Datum sendData[], 
      const int sendCount, 
      const int destProc, 
      const int tag) 
      {
  Teuchos::send (*pComm_, sendCount, sendData, destProc);
      }

      virtual void 
      recv (Datum recvData[], 
      const int recvCount, 
      const int srcProc, 
      const int tag) 
      {
  Teuchos::receive (*pComm_, srcProc, recvCount, recvData);
      }

      virtual void 
      swapData (const Datum sendData[], 
    Datum recvData[], 
    const int sendRecvCount, 
    const int destProc, 
    const int tag)
      {
  if (destProc == rank())
    std::copy (sendData, sendData+sendRecvCount, recvData);
  else
    {
      using Teuchos::RCP;
      using Teuchos::ArrayRCP;
      using Teuchos::CommRequest;

      const int srcProc = Teuchos::rank (*pComm_);

      // FIXME (mfh 14 June 2010, 09 Jul 2010) It would be nice
      // if Teuchos had a sendRecv() routine... as it is, we
      // have to do a send and then a receive.  We could do an
      // isend and an ireceive in order to exploit potential
      // overlap of the two messages.  That works if sendData
      // and recvData don't alias one another.  We only do a
      // partial check for aliasing here (sendData == recvData).
      if (sendData == recvData)
        {
    // The smaller-rank process sends first, and the
    // larger-rank process receives first.
    //
    // Teuchos::send() and Teuchos::recv() are blocking,
    // so we may safely write to recvBuf even if it
    // aliases sendBuf.
    if (srcProc < destProc)
      {
        Teuchos::send (*pComm_, sendRecvCount, sendData, destProc);
        Teuchos::receive (*pComm_, destProc, sendRecvCount, recvData);
      }
    else 
      {
        Teuchos::receive (*pComm_, destProc, sendRecvCount, recvData);
        Teuchos::send (*pComm_, sendRecvCount, sendData, destProc);
      }
        }
      else
        {
    ArrayRCP< const Datum > sendBuf (sendData, 0, sendRecvCount, false);
    ArrayRCP< Datum > recvBuf (recvData, 0, sendRecvCount, false);

    RCP< CommRequest > sendReq, recvReq;
    if (srcProc < destProc)
      {
        sendReq = Teuchos::isend (*pComm_, sendBuf, destProc);
        recvReq = Teuchos::ireceive (*pComm_, recvBuf, destProc);
      }
    else
      {
        recvReq = Teuchos::ireceive (*pComm_, recvBuf, destProc);
        sendReq = Teuchos::isend (*pComm_, sendBuf, destProc);
      }
    // Wait on both the send and the receive to complete.  The
    // two can happen independently, because sendBuf and recvBuf
    // are different.  (We assert no aliasing of buffers here,
    // and we've also checked above that destProc != rank().)
    Teuchos::waitAll (*pComm_, Teuchos::tuple (sendReq, recvReq));
        }
    }
      }

      virtual Datum 
      globalSum (const Datum& inDatum) 
      {
  Datum outDatum;
  Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_SUM, inDatum, 
          Teuchos::outArg(outDatum));
  return outDatum;
      }

      virtual Datum 
      globalMin (const Datum& inDatum)
      {
  Datum outDatum;
  Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_MIN, inDatum, 
          Teuchos::outArg(outDatum));
  return outDatum;
      }

      virtual Datum 
      globalMax (const Datum& inDatum)
      {
  Datum outDatum;
  Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_MAX, inDatum, 
          Teuchos::outArg(outDatum));
  return outDatum;
      }

      virtual void
      globalVectorSum (const Datum inData[], 
           Datum outData[], 
           const int count) 
      {
  Teuchos::reduceAll (*pComm_, Teuchos::REDUCE_SUM, count, 
          inData, outData);
      }

      virtual void
      broadcast (Datum data[], 
     const int count,
     const int root)
      {
  // Assumes that Datum has value semantics.
  Teuchos::broadcast (*pComm_, root, count, data);
      }

      virtual int rank () const { return Teuchos::rank (*pComm_); }

      virtual int size () const { return Teuchos::size (*pComm_); }

      virtual void barrier () const { Teuchos::barrier (*pComm_); }

    private:
      comm_ptr pComm_;
    };
  } // namespace Trilinos
} // namespace TSQR

#endif // __TSQR_Trilinos_TpetraMessenger_hpp