Tpetra Matrix/Vector Services: Tpetra_DistObject.hpp Source File

00001 // @HEADER
00002 // ***********************************************************************
00003 // 
00004 //          Tpetra: Templated Linear Algebra Services Package
00005 //                 Copyright (2008) Sandia Corporation
00006 // 
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 // 
00010 // This library is free software; you can redistribute it and/or modify
00011 // it under the terms of the GNU Lesser General Public License as
00012 // published by the Free Software Foundation; either version 2.1 of the
00013 // License, or (at your option) any later version.
00014 //  
00015 // This library is distributed in the hope that it will be useful, but
00016 // WITHOUT ANY WARRANTY; without even the implied warranty of
00017 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018 // Lesser General Public License for more details.
00019 //  
00020 // You should have received a copy of the GNU Lesser General Public
00021 // License along with this library; if not, write to the Free Software
00022 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
00023 // USA
00024 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 
00025 // 
00026 // ***********************************************************************
00027 // @HEADER
00028 
00029 #ifndef TPETRA_DISTOBJECT_HPP
00030 #define TPETRA_DISTOBJECT_HPP
00031 
00032 #include "Tpetra_ConfigDefs.hpp"
00033 #include "Tpetra_Map.hpp"
00034 #include "Tpetra_Import.hpp"
00035 #include "Tpetra_Export.hpp"
00036 #include "Tpetra_Distributor.hpp"
00037 
00038 #include <Kokkos_DefaultNode.hpp>
00039 #include <Teuchos_Describable.hpp>
00040 
00041 namespace Tpetra {
00042 
00044 
00064   template <class Packet, class LocalOrdinal = int, class GlobalOrdinal = LocalOrdinal, class Node = Kokkos::DefaultNode::DefaultNodeType>
00065   class DistObject : virtual public Teuchos::Describable {
00066 
00067   public:
00068 
00070 
00071 
00073     explicit DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > &map);
00074 
00076     DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source);
00077 
00079     virtual ~DistObject();
00080 
00082 
00084 
00085 
00087     void doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source, 
00088                   const Import<LocalOrdinal,GlobalOrdinal,Node> &importer, CombineMode CM);
00089 
00091     void doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest, 
00092                   const Export<LocalOrdinal,GlobalOrdinal,Node> &exporter, CombineMode CM);
00093 
00095     void doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00096                   const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter, CombineMode CM);
00097 
00099     void doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &dest,
00100                   const Import<LocalOrdinal,GlobalOrdinal,Node>& importer, CombineMode CM);
00101 
00103 
00105 
00106 
00108     inline bool isDistributed() const;
00109 
00111     inline const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & getMap() const { return map_; }
00112 
00114 
00116 
00117 
00119 
00120     void print(std::ostream &os) const;
00121 
00123 
00124   protected:
00125 
00127     enum ReverseOption {
00128       DoForward, //*!< Indicates that the transfer should be performed in forward mode.
00129       DoReverse  //*!< Indicates that the transfer should be performed in reverse mode.
00130     };
00131 
00133     virtual void doTransfer(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> &source,
00134                             CombineMode CM,
00135                             size_t numSameIDs,
00136                             const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00137                             const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00138                             const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,
00139                             const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00140                             Distributor &distor,
00141                             ReverseOption revOp);
00142 
00143     // The following four methods must be implemented by the derived class
00144 
00146 
00147     virtual bool checkSizes(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source) = 0;
00148 
00150 
00166     virtual void copyAndPermute(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00167                                 size_t numSameIDs,
00168                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs,
00169                                 const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs) = 0;
00170 
00172 
00190     virtual void packAndPrepare(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00191                                 const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00192                                 Teuchos::Array<Packet> &exports,
00193                                 const Teuchos::ArrayView<size_t> & numPacketsPerLID,
00194                                 size_t& constantNumPackets,
00195                                 Distributor &distor) = 0;
00196 
00198 
00215     virtual void unpackAndCombine(const Teuchos::ArrayView<const LocalOrdinal> &importLIDs,
00216                                   const Teuchos::ArrayView<const Packet> &imports,
00217                                   const Teuchos::ArrayView<size_t> &numPacketsPerLID,
00218                                   size_t constantNumPackets,
00219                                   Distributor &distor,
00220                                   CombineMode CM) = 0;
00221 
00222     virtual void createViews() const {}
00223     virtual void createViewsNonConst(Kokkos::ReadWriteOption rwo) {}
00224     virtual void releaseViews() const {}
00225 
00226   private:
00227 
00228     Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > map_;
00229     // buffers into which packed data is imported
00230     Teuchos::Array<Packet> imports_;
00231     Teuchos::Array<size_t> numImportPacketsPerLID_;
00232     // buffers from which packed data is exported
00233     Teuchos::Array<Packet> exports_;
00234     Teuchos::Array<size_t> numExportPacketsPerLID_;
00235 
00236   }; // class DistObject
00237 
00238   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00239   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const Teuchos::RCP<const Map<LocalOrdinal,GlobalOrdinal,Node> > & map)
00240   : map_(map)
00241   {}
00242 
00243   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00244   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::DistObject(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source)
00245   : map_(source.map_)
00246   {}
00247 
00248   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00249   DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::~DistObject() 
00250   {}
00251 
00252   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00253   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doImport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00254                                             const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) 
00255   {
00256     TEST_FOR_EXCEPTION( getMap() != importer.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00257     TEST_FOR_EXCEPTION( A.getMap() != importer.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00258     size_t numSameIDs = importer.getNumSameIDs();
00259     const Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getExportLIDs();
00260     const Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getRemoteLIDs();
00261     const Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteToLIDs();
00262     const Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteFromLIDs();
00263     this->doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00264                      importer.getDistributor(), DoForward);
00265   }
00266 
00267   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00268   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doExport(const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A, 
00269                                             const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, CombineMode CM) 
00270   {
00271     TEST_FOR_EXCEPTION( getMap() != exporter.getTargetMap(), std::runtime_error, "Target Maps don't match.");
00272     TEST_FOR_EXCEPTION( A.getMap() != exporter.getSourceMap(), std::runtime_error, "Source Maps don't match.");
00273     size_t numSameIDs = exporter.getNumSameIDs();
00274     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getExportLIDs();
00275     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getRemoteLIDs();
00276     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteToLIDs();
00277     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteFromLIDs();
00278     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00279                exporter.getDistributor(), DoForward);
00280   }
00281 
00282   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00283   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doImport(
00284                                                 const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00285                                                 const Export<LocalOrdinal,GlobalOrdinal,Node> & exporter, 
00286                                                 CombineMode CM) {
00287     TEST_FOR_EXCEPTION( getMap() != exporter.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00288     TEST_FOR_EXCEPTION( A.getMap() != exporter.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00289     size_t numSameIDs = exporter.getNumSameIDs();
00290     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = exporter.getRemoteLIDs();
00291     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = exporter.getExportLIDs();
00292     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = exporter.getPermuteFromLIDs();
00293     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = exporter.getPermuteToLIDs();
00294     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00295                exporter.getDistributor(), DoReverse);
00296   }
00297 
00298   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00299   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doExport(
00300                                                 const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & A,
00301                                                 const Import<LocalOrdinal,GlobalOrdinal,Node> & importer, CombineMode CM) {
00302     TEST_FOR_EXCEPTION( getMap() != importer.getSourceMap(), std::runtime_error, "Target Maps don't match.");
00303     TEST_FOR_EXCEPTION( A.getMap() != importer.getTargetMap(), std::runtime_error, "Source Maps don't match.");
00304     size_t numSameIDs = importer.getNumSameIDs();
00305     Teuchos::ArrayView<const LocalOrdinal> exportLIDs      = importer.getRemoteLIDs();
00306     Teuchos::ArrayView<const LocalOrdinal> remoteLIDs      = importer.getExportLIDs();
00307     Teuchos::ArrayView<const LocalOrdinal> permuteToLIDs   = importer.getPermuteFromLIDs();
00308     Teuchos::ArrayView<const LocalOrdinal> permuteFromLIDs = importer.getPermuteToLIDs();
00309     doTransfer(A, CM, numSameIDs, permuteToLIDs, permuteFromLIDs, remoteLIDs, exportLIDs,
00310                importer.getDistributor(), DoReverse);
00311   }
00312 
00313   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00314   bool DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::isDistributed() const {
00315     return map_->isDistributed();
00316   }
00317 
00318   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00319   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::doTransfer(
00320       const DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node> & source,
00321       CombineMode CM,
00322       size_t numSameIDs, 
00323       const Teuchos::ArrayView<const LocalOrdinal> &permuteToLIDs, 
00324       const Teuchos::ArrayView<const LocalOrdinal> &permuteFromLIDs,
00325       const Teuchos::ArrayView<const LocalOrdinal> &remoteLIDs,    
00326       const Teuchos::ArrayView<const LocalOrdinal> &exportLIDs,
00327       Distributor &distor, ReverseOption revOp) 
00328   {
00329     TEST_FOR_EXCEPTION( checkSizes(source) == false, std::runtime_error, 
00330         "Tpetra::DistObject::doTransfer(): checkSizes() indicates that DistOjbects are not size-compatible.");
00331     Kokkos::ReadWriteOption rwo = Kokkos::ReadWrite;
00332     if (CM == INSERT || CM == REPLACE) {
00333       if (numSameIDs + permuteToLIDs.size() + remoteLIDs.size() == this->getMap()->getNodeNumElements()) {
00334         // we are overwriting, and everything will be overwritten
00335         rwo = Kokkos::WriteOnly;
00336       }
00337     }
00338     source.createViews();
00339     this->createViewsNonConst(rwo); 
00340     if (numSameIDs + permuteToLIDs.size()) {
00341       copyAndPermute(source,numSameIDs,permuteToLIDs,permuteFromLIDs);
00342     }
00343     size_t constantNumPackets = 0;
00344     numExportPacketsPerLID_.resize(exportLIDs.size());
00345     numImportPacketsPerLID_.resize(remoteLIDs.size());
00346     packAndPrepare(source,exportLIDs,exports_,numExportPacketsPerLID_(),constantNumPackets,distor);
00347     source.releaseViews();
00348     if (constantNumPackets != 0) {
00349       size_t rbufLen = remoteLIDs.size()*constantNumPackets;
00350       imports_.resize(rbufLen);
00351     }
00352     if ((isDistributed() && revOp == DoReverse) || (source.isDistributed() && revOp == DoForward)) 
00353     {
00354       // call one of the doPostsAndWaits functions
00355       if (revOp == DoReverse) {
00356         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00357           distor.doReversePostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00358                                         numImportPacketsPerLID_());
00359           size_t totalImportPackets = 0;
00360           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00361             totalImportPackets += numImportPacketsPerLID_[i];
00362           }
00363           imports_.resize(totalImportPackets);
00364           distor.doReversePostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00365                                         imports_(), numImportPacketsPerLID_());
00366         }
00367         else {
00368           distor.doReversePostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00369         }
00370       }
00371       else {
00372         if (constantNumPackets == 0) { //variable num-packets-per-LID:
00373           distor.doPostsAndWaits(numExportPacketsPerLID_().getConst(), 1,
00374                                  numImportPacketsPerLID_());
00375           size_t totalImportPackets = 0;
00376           for(Array_size_type i=0; i<numImportPacketsPerLID_.size(); ++i) {
00377             totalImportPackets += numImportPacketsPerLID_[i];
00378           }
00379           imports_.resize(totalImportPackets);
00380           distor.doPostsAndWaits(exports_().getConst(),numExportPacketsPerLID_(),
00381                                  imports_(), numImportPacketsPerLID_());
00382         }
00383         else {
00384           distor.doPostsAndWaits(exports_().getConst(),constantNumPackets,imports_());
00385         }
00386       }
00387       unpackAndCombine(remoteLIDs,imports_(),numImportPacketsPerLID_(), constantNumPackets, distor,CM);
00388     }
00389     this->releaseViews();
00390   }
00391 
00392   template <class Packet, class LocalOrdinal, class GlobalOrdinal, class Node>
00393   void DistObject<Packet,LocalOrdinal,GlobalOrdinal,Node>::print(std::ostream &os) const
00394   {
00395     using std::endl;
00396     os << "Tpetra::DistObject" << endl
00397        << " export buffer size: " << exports_.size() << endl
00398        << " import buffer size: " << imports_.size() << endl
00399        << "Map:" << endl
00400        << map_;
00401   }
00402 
00403 } // namespace Tpetra
00404 
00405 #endif /* TPETRA_DISTOBJECT_HPP */