Tpetra parallel linear algebra Version of the Day
Loading...
Searching...
No Matches
Tpetra_Distributor.hpp
1// @HEADER
2// ***********************************************************************
3//
4// Tpetra: Templated Linear Algebra Services Package
5// Copyright (2008) Sandia Corporation
6//
7// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8// the U.S. Government retains certain rights in this software.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// ************************************************************************
38// @HEADER
39
40#ifndef TPETRA_DISTRIBUTOR_HPP
41#define TPETRA_DISTRIBUTOR_HPP
42
43#include "Tpetra_Details_DistributorActor.hpp"
44#include "Tpetra_Details_DistributorPlan.hpp"
45
46#include "Tpetra_Util.hpp"
47#include "Teuchos_as.hpp"
48#include "Teuchos_Describable.hpp"
49#include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
50#include "Teuchos_VerboseObject.hpp"
52
53#include "KokkosCompat_View.hpp"
54#include "Kokkos_Core.hpp"
55#include "Kokkos_TeuchosCommAdapters.hpp"
56#include <memory>
57#include <sstream>
58#include <type_traits>
59
60namespace Tpetra {
61
68 Teuchos::Array<std::string> distributorSendTypes ();
69
133 public Teuchos::Describable,
134 public Teuchos::ParameterListAcceptorDefaultBase {
135 public:
137
138
147 explicit Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm);
148
160 Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
161 const Teuchos::RCP<Teuchos::FancyOStream>& out);
162
176 Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
177 const Teuchos::RCP<Teuchos::ParameterList>& plist);
178
195 Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
196 const Teuchos::RCP<Teuchos::FancyOStream>& out,
197 const Teuchos::RCP<Teuchos::ParameterList>& plist);
198
200 Distributor (const Distributor& distributor);
201
206 virtual ~Distributor () = default;
207
213 void swap (Distributor& rhs);
214
216
218
223 void setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist);
224
229 Teuchos::RCP<const Teuchos::ParameterList> getValidParameters () const;
230
232
234
254 size_t createFromSends (const Teuchos::ArrayView<const int>& exportProcIDs);
255
289 template <class Ordinal>
290 void
291 createFromRecvs (const Teuchos::ArrayView<const Ordinal>& remoteIDs,
292 const Teuchos::ArrayView<const int>& remoteProcIDs,
293 Teuchos::Array<Ordinal>& exportIDs,
294 Teuchos::Array<int>& exportProcIDs);
295
303 void
304 createFromSendsAndRecvs (const Teuchos::ArrayView<const int>& exportProcIDs,
305 const Teuchos::ArrayView<const int>& remoteProcIDs);
306
308
310
314 size_t getNumReceives() const;
315
319 size_t getNumSends() const;
320
322 bool hasSelfMessage() const;
323
325 size_t getMaxSendLength() const;
326
328 size_t getTotalReceiveLength() const;
329
334 Teuchos::ArrayView<const int> getProcsFrom() const;
335
340 Teuchos::ArrayView<const int> getProcsTo() const;
341
349 Teuchos::ArrayView<const size_t> getLengthsFrom() const;
350
358 Teuchos::ArrayView<const size_t> getLengthsTo() const;
359
365 return plan_.howInitialized();
366 }
367
369
371
382 Teuchos::RCP<Distributor> getReverse(bool create=true) const;
383
385
387
394 void doWaits ();
395
402 void doReverseWaits ();
403
424 template <class ExpView, class ImpView>
425 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
427 const ExpView &exports,
428 size_t numPackets,
429 const ImpView &imports);
430
452 template <class ExpView, class ImpView>
453 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
454 doPostsAndWaits (const ExpView &exports,
455 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
456 const ImpView &imports,
457 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
458
483 template <class ExpView, class ImpView>
484 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
485 doPosts (const ExpView &exports,
486 size_t numPackets,
487 const ImpView &imports);
488
507 template <class ExpView, class ImpView>
508 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
509 doPosts (const ExpView &exports,
510 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
511 const ImpView &imports,
512 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
513
518 template <class ExpView, class ImpView>
519 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
520 doReversePostsAndWaits (const ExpView &exports,
521 size_t numPackets,
522 const ImpView &imports);
523
528 template <class ExpView, class ImpView>
529 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
530 doReversePostsAndWaits (const ExpView &exports,
531 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
532 const ImpView &imports,
533 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
534
539 template <class ExpView, class ImpView>
540 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
541 doReversePosts (const ExpView &exports,
542 size_t numPackets,
543 const ImpView &imports);
544
549 template <class ExpView, class ImpView>
550 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
551 doReversePosts (const ExpView &exports,
552 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
553 const ImpView &imports,
554 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
555
557
559
561 std::string description() const;
562
584 void
585 describe (Teuchos::FancyOStream& out,
586 const Teuchos::EVerbosityLevel verbLevel =
587 Teuchos::Describable::verbLevel_default) const;
589
594 const Details::DistributorPlan& getPlan() const { return plan_; }
595 private:
597 Details::DistributorActor actor_;
598
600
601
603 static bool getVerbose();
604
609 std::unique_ptr<std::string>
610 createPrefix(const char methodName[]) const;
611
613 bool verbose_ = getVerbose();
615
620 mutable Teuchos::RCP<Distributor> reverseDistributor_;
621
634 template <class Ordinal>
635 void computeSends (const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
636 const Teuchos::ArrayView<const int> &remoteProcIDs,
637 Teuchos::Array<Ordinal> &exportGIDs,
638 Teuchos::Array<int> &exportProcIDs);
639
641 void createReverseDistributor() const;
642
643
648 std::string
649 localDescribeToString (const Teuchos::EVerbosityLevel vl) const;
650 }; // class Distributor
651
652 template <class ExpView, class ImpView>
653 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
655 doPostsAndWaits (const ExpView& exports,
656 size_t numPackets,
657 const ImpView& imports)
658 {
659 actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
660 }
661
662 template <class ExpView, class ImpView>
663 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
665 doPostsAndWaits(const ExpView& exports,
666 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
667 const ImpView& imports,
668 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
669 {
670 actor_.doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
671 }
672
673
674 template <class ExpView, class ImpView>
675 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
677 doPosts (const ExpView &exports,
678 size_t numPackets,
679 const ImpView &imports)
680 {
681 actor_.doPosts(plan_, exports, numPackets, imports);
682 }
683
684 template <class ExpView, class ImpView>
685 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
687 doPosts (const ExpView &exports,
688 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
689 const ImpView &imports,
690 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
691 {
692 actor_.doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
693 }
694
695 template <class ExpView, class ImpView>
696 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
698 doReversePostsAndWaits (const ExpView& exports,
699 size_t numPackets,
700 const ImpView& imports)
701 {
702 doReversePosts (exports, numPackets, imports);
703 doReverseWaits ();
704 }
705
706 template <class ExpView, class ImpView>
707 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
709 doReversePostsAndWaits (const ExpView& exports,
710 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
711 const ImpView& imports,
712 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
713 {
714 doReversePosts (exports, numExportPacketsPerLID, imports,
715 numImportPacketsPerLID);
716 doReverseWaits ();
717 }
718
719 template <class ExpView, class ImpView>
720 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
722 doReversePosts (const ExpView &exports,
723 size_t numPackets,
724 const ImpView &imports)
725 {
726 // FIXME (mfh 29 Mar 2012) WHY?
727 TEUCHOS_TEST_FOR_EXCEPTION(
728 ! plan_.getIndicesTo().is_null(), std::runtime_error,
729 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
730 "reverse communication when original data are blocked by process.");
731 if (reverseDistributor_.is_null ()) {
732 createReverseDistributor ();
733 }
734 reverseDistributor_->doPosts (exports, numPackets, imports);
735 }
736
737 template <class ExpView, class ImpView>
738 typename std::enable_if<(Kokkos::is_view<ExpView>::value && Kokkos::is_view<ImpView>::value)>::type
740 doReversePosts (const ExpView &exports,
741 const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
742 const ImpView &imports,
743 const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
744 {
745 // FIXME (mfh 29 Mar 2012) WHY?
746 TEUCHOS_TEST_FOR_EXCEPTION(
747 ! plan_.getIndicesTo().is_null(), std::runtime_error,
748 "Tpetra::Distributor::doReversePosts(3 args): Can only do "
749 "reverse communication when original data are blocked by process.");
750 if (reverseDistributor_.is_null ()) {
751 createReverseDistributor ();
752 }
753 reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
754 imports, numImportPacketsPerLID);
755 }
756
757 template <class OrdinalType>
758 void Distributor::
759 computeSends(const Teuchos::ArrayView<const OrdinalType>& importGIDs,
760 const Teuchos::ArrayView<const int>& importProcIDs,
761 Teuchos::Array<OrdinalType>& exportGIDs,
762 Teuchos::Array<int>& exportProcIDs)
763 {
764 // NOTE (mfh 19 Apr 2012): There was a note on this code saying:
765 // "assumes that size_t >= Ordinal". The code certainly does
766 // assume that sizeof(size_t) >= sizeof(OrdinalType) as well as
767 // sizeof(size_t) >= sizeof(int). This is because it casts the
768 // OrdinalType elements of importGIDs (along with their
769 // corresponding process IDs, as int) to size_t, and does a
770 // doPostsAndWaits<size_t>() to send the packed data.
771 using Teuchos::ArrayView;
772 using std::endl;
773 using size_type = typename ArrayView<const OrdinalType>::size_type;
774 const char errPrefix[] = "Tpetra::Distributor::computeSends: ";
775 const char suffix[] =
776 " Please report this bug to the Tpetra developers.";
777
778 const int myRank = plan_.getComm()->getRank ();
779
780 TEUCHOS_TEST_FOR_EXCEPTION
781 (importGIDs.size () != importProcIDs.size (),
782 std::invalid_argument, errPrefix << "On Process " << myRank
783 << ": importProcIDs.size()=" << importProcIDs.size()
784 << " != importGIDs.size()=" << importGIDs.size() << ".");
785
786 const size_type numImports = importProcIDs.size();
787 Kokkos::View<size_t*, Kokkos::HostSpace> importObjs("importObjs", 2*numImports);
788 // Pack pairs (importGIDs[i], my process ID) to send into importObjs.
789 for (size_type i = 0; i < numImports; ++i) {
790 importObjs[2*i] = static_cast<size_t>(importGIDs[i]);
791 importObjs[2*i+1] = static_cast<size_t>(myRank);
792 }
793 //
794 // Use a temporary Distributor to send the (importGIDs[i], myRank)
795 // pairs to importProcIDs[i].
796 //
797 Distributor tempPlan(plan_.getComm());
798 // mfh 20 Mar 2014: An extra-cautious cast from unsigned to
799 // signed, in order to forestall any possible causes for Bug 6069.
800 const size_t numExportsAsSizeT =
801 tempPlan.createFromSends(importProcIDs);
802 const size_type numExports =
803 static_cast<size_type>(numExportsAsSizeT);
804 TEUCHOS_TEST_FOR_EXCEPTION
805 (numExports < 0, std::logic_error, errPrefix <<
806 "tempPlan.createFromSends() returned numExports="
807 << numExportsAsSizeT << " as a size_t, which overflows to "
808 << numExports << " when cast to " <<
809 Teuchos::TypeNameTraits<size_type>::name () << "." << suffix);
810 TEUCHOS_TEST_FOR_EXCEPTION
811 (size_type(tempPlan.getTotalReceiveLength()) != numExports,
812 std::logic_error, errPrefix << "tempPlan.getTotalReceiveLength()="
813 << tempPlan.getTotalReceiveLength () << " != numExports="
814 << numExports << "." << suffix);
815
816 if (numExports > 0) {
817 exportGIDs.resize(numExports);
818 exportProcIDs.resize(numExports);
819 }
820
821 // exportObjs: Packed receive buffer. (exportObjs[2*i],
822 // exportObjs[2*i+1]) will give the (GID, PID) pair for export i,
823 // after tempPlan.doPostsAndWaits(...) finishes below.
824 //
825 // FIXME (mfh 19 Mar 2014) This only works if OrdinalType fits in
826 // size_t. This issue might come up, for example, on a 32-bit
827 // machine using 64-bit global indices. I will add a check here
828 // for that case.
829 static_assert(sizeof(size_t) >= sizeof(OrdinalType),
830 "Tpetra::Distributor::computeSends: "
831 "sizeof(size_t) < sizeof(OrdinalType).");
832
833 TEUCHOS_TEST_FOR_EXCEPTION
834 (tempPlan.getTotalReceiveLength () < size_t(numExports),
835 std::logic_error,
836 errPrefix << "tempPlan.getTotalReceiveLength()="
837 << tempPlan.getTotalReceiveLength() << " < numExports="
838 << numExports << "." << suffix);
839
840 Kokkos::View<size_t*, Kokkos::HostSpace> exportObjs("exportObjs", tempPlan.getTotalReceiveLength() * 2);
841 tempPlan.doPostsAndWaits(importObjs, 2, exportObjs);
842
843 // Unpack received (GID, PID) pairs into exportIDs resp. exportProcIDs.
844 for (size_type i = 0; i < numExports; ++i) {
845 exportGIDs[i] = static_cast<OrdinalType> (exportObjs[2*i]);
846 exportProcIDs[i] = static_cast<int> (exportObjs[2*i+1]);
847 }
848 }
849
850 template <class OrdinalType>
851 void Distributor::
852 createFromRecvs (const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
853 const Teuchos::ArrayView<const int> &remoteProcIDs,
854 Teuchos::Array<OrdinalType> &exportGIDs,
855 Teuchos::Array<int> &exportProcIDs)
856 {
857 using std::endl;
858 const char errPrefix[] = "Tpetra::Distributor::createFromRecvs: ";
859 const int myRank = plan_.getComm()->getRank();
860
861 std::unique_ptr<std::string> prefix;
862 if (verbose_) {
863 prefix = createPrefix("createFromRecvs");
864 std::ostringstream os;
865 os << *prefix << "Start" << endl;
866 std::cerr << os.str();
867 }
868
869 const bool debug = Details::Behavior::debug("Distributor");
870 if (debug) {
871 using Teuchos::outArg;
872 using Teuchos::REDUCE_MAX;
873 using Teuchos::reduceAll;
874 // In debug mode, first test locally, then do an all-reduce to
875 // make sure that all processes passed.
876 const int errProc =
877 (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
878 int maxErrProc = -1;
879 reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
880 TEUCHOS_TEST_FOR_EXCEPTION
881 (maxErrProc != -1, std::runtime_error, errPrefix << "Lists "
882 "of remote IDs and remote process IDs must have the same "
883 "size on all participating processes. Maximum process ID "
884 "with error: " << maxErrProc << ".");
885 }
886 else { // in non-debug mode, just test locally
887 // NOTE (mfh 13 Feb 2020) This needs to throw std::runtime_error
888 // in order to make an existing Distributor unit test pass.
889 TEUCHOS_TEST_FOR_EXCEPTION
890 (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
891 errPrefix << "On Process " << myRank << ": "
892 "remoteGIDs.size()=" << remoteGIDs.size() <<
893 " != remoteProcIDs.size()=" << remoteProcIDs.size() << ".");
894 }
895
896 computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
897
898 plan_.createFromRecvs(remoteProcIDs);
899
900 if (verbose_) {
901 std::ostringstream os;
902 os << *prefix << "Done" << endl;
903 std::cerr << os.str();
904 }
905 }
906
907} // namespace Tpetra
908
909#endif // TPETRA_DISTRIBUTOR_HPP
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Stand-alone utility functions and macros.
static bool debug()
Whether Tpetra is in debug mode.
Sets up and executes a communication plan for a Tpetra DistObject.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a forward plan, but do not execute the waits yet.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doPostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the (forward) communication plan.
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
size_t getNumReceives() const
The number of processes from which we will receive data.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePostsAndWaits(const ExpView &exports, size_t numPackets, const ImpView &imports)
Execute the reverse communication plan.
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
virtual ~Distributor()=default
Destructor (virtual for memory safety).
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
std::string description() const
Return a one-line description of this object.
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
std::enable_if<(Kokkos::is_view< ExpView >::value &&Kokkos::is_view< ImpView >::value)>::type doReversePosts(const ExpView &exports, size_t numPackets, const ImpView &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
size_t getNumSends() const
The number of processes to which we will send data.
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.