46#ifndef MUELU_ZOLTANINTERFACE_DEF_HPP
47#define MUELU_ZOLTANINTERFACE_DEF_HPP
50#if defined(HAVE_MUELU_ZOLTAN) && defined(HAVE_MPI)
52#include <Teuchos_Utils.hpp>
53#include <Teuchos_DefaultMpiComm.hpp>
54#include <Teuchos_OpaqueWrapper.hpp>
59#include "MueLu_Utilities.hpp"
63 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
65 RCP<ParameterList> validParamList = rcp(
new ParameterList());
67 validParamList->set< RCP<const FactoryBase> >(
"A", Teuchos::null,
"Factory of the matrix A");
68 validParamList->set< RCP<const FactoryBase> >(
"number of partitions", Teuchos::null,
"Instance of RepartitionHeuristicFactory.");
69 validParamList->set< RCP<const FactoryBase> >(
"Coordinates", Teuchos::null,
"Factory of the coordinates");
71 return validParamList;
75 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
77 Input(currentLevel,
"A");
78 Input(currentLevel,
"number of partitions");
79 Input(currentLevel,
"Coordinates");
82 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
86 RCP<Matrix> A = Get< RCP<Matrix> > (level,
"A");
87 RCP<BlockedCrsMatrix> bA = Teuchos::rcp_dynamic_cast<BlockedCrsMatrix>(A);
88 RCP<const Map> rowMap;
89 if(bA != Teuchos::null) {
91 RCP<const Map> bArowMap = bA->getRowMap();
92 RCP<const BlockedMap> bRowMap = Teuchos::rcp_dynamic_cast<const BlockedMap>(bArowMap);
93 rowMap = bRowMap->getFullMap();
95 rowMap = A->getRowMap();
98 typedef Xpetra::MultiVector<typename Teuchos::ScalarTraits<Scalar>::magnitudeType,
LocalOrdinal,
GlobalOrdinal,
Node> double_multivector_type;
99 RCP<double_multivector_type> Coords = Get< RCP<double_multivector_type> >(level,
"Coordinates");
100 size_t dim = Coords->getNumVectors();
101 int numParts = Get<int>(level,
"number of partitions");
103 if (numParts == 1 || numParts == -1) {
105 RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap,
true);
106 Set(level,
"Partition", decomposition);
108 }
else if (numParts == -1) {
110 RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = Teuchos::null;
111 Set(level,
"Partition", decomposition);
115 float zoltanVersion_;
116 Zoltan_Initialize(0, NULL, &zoltanVersion_);
118 RCP<const Teuchos::MpiComm<int> > dupMpiComm = rcp_dynamic_cast<const Teuchos::MpiComm<int> >(rowMap->getComm()->duplicate());
119 RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > zoltanComm = dupMpiComm->getRawMpiComm();
121 RCP<Zoltan> zoltanObj_ = rcp(
new Zoltan((*zoltanComm)()));
122 if (zoltanObj_ == Teuchos::null)
129 if ((rv = zoltanObj_->Set_Param(
"num_gid_entries",
"1")) != ZOLTAN_OK)
130 throw Exceptions::RuntimeError(
"MueLu::Zoltan::Setup : setting parameter 'num_gid_entries' returned error code " + Teuchos::toString(rv));
131 if ((rv = zoltanObj_->Set_Param(
"num_lid_entries",
"0") ) != ZOLTAN_OK)
132 throw Exceptions::RuntimeError(
"MueLu::Zoltan::Setup : setting parameter 'num_lid_entries' returned error code " + Teuchos::toString(rv));
133 if ((rv = zoltanObj_->Set_Param(
"obj_weight_dim",
"1") ) != ZOLTAN_OK)
134 throw Exceptions::RuntimeError(
"MueLu::Zoltan::Setup : setting parameter 'obj_weight_dim' returned error code " + Teuchos::toString(rv));
136 if (GetVerbLevel() &
Statistics1) zoltanObj_->Set_Param(
"debug_level",
"1");
137 else zoltanObj_->Set_Param(
"debug_level",
"0");
139 zoltanObj_->Set_Param(
"num_global_partitions",
toString(numParts));
141 zoltanObj_->Set_Num_Obj_Fn(GetLocalNumberOfRows, (
void *) A.getRawPtr());
142 zoltanObj_->Set_Obj_List_Fn(GetLocalNumberOfNonzeros, (
void *) A.getRawPtr());
143 zoltanObj_->Set_Num_Geom_Fn(GetProblemDimension, (
void *) &dim);
144 zoltanObj_->Set_Geom_Multi_Fn(GetProblemGeometry, (
void *) Coords.get());
147 ZOLTAN_ID_PTR import_gids = NULL;
148 ZOLTAN_ID_PTR import_lids = NULL;
149 int *import_procs = NULL;
150 int *import_to_part = NULL;
151 ZOLTAN_ID_PTR export_gids = NULL;
152 ZOLTAN_ID_PTR export_lids = NULL;
153 int *export_procs = NULL;
154 int *export_to_part = NULL;
163 rv = zoltanObj_->LB_Partition(newDecomp, num_gid_entries, num_lid_entries,
164 num_imported, import_gids, import_lids, import_procs, import_to_part,
165 num_exported, export_gids, export_lids, export_procs, export_to_part);
166 if (rv == ZOLTAN_FATAL)
172 RCP<Xpetra::Vector<GO, LO, GO, NO> > decomposition;
174 decomposition = Xpetra::VectorFactory<GO, LO, GO, NO>::Build(rowMap,
false);
175 ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
177 int mypid = rowMap->getComm()->getRank();
178 for (
typename ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
181 LO blockSize = A->GetFixedBlockSize();
182 for (
int i = 0; i < num_exported; ++i) {
185 LO localEl = rowMap->getLocalElement(export_gids[i]);
186 int partNum = export_to_part[i];
187 for (LO j = 0; j < blockSize; ++j)
188 decompEntries[localEl + j] = partNum;
192 Set(level,
"Partition", decomposition);
194 zoltanObj_->LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_to_part);
195 zoltanObj_->LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_to_part);
203 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
206 *ierr = ZOLTAN_FATAL;
209 Matrix *A = (Matrix*) data;
212 LO blockSize = A->GetFixedBlockSize();
215 return A->getRowMap()->getLocalNumElements() / blockSize;
222 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
225 ZOLTAN_ID_PTR ,
int ,
float *weights,
int *ierr) {
226 if (data == NULL || NumGidEntries < 1) {
227 *ierr = ZOLTAN_FATAL;
233 Matrix *A = (Matrix*) data;
234 RCP<const Map> map = A->getRowMap();
236 LO blockSize = A->GetFixedBlockSize();
239 size_t numElements = map->getLocalNumElements();
240 ArrayView<const GO> mapGIDs = map->getLocalElementList();
242 if (blockSize == 1) {
243 for (
size_t i = 0; i < numElements; i++) {
244 gids[i] = as<ZOLTAN_ID_TYPE>(mapGIDs[i]);
245 weights[i] = A->getNumEntriesInLocalRow(i);
249 LO numBlockElements = numElements / blockSize;
251 for (LO i = 0; i < numBlockElements; i++) {
254 gids[i] = as<ZOLTAN_ID_TYPE>(mapGIDs[i*blockSize]);
256 for (LO j = 0; j < blockSize; j++)
257 weights[i] += A->getNumEntriesInLocalRow(i*blockSize+j);
267 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
271 int dim = *((
int*)data);
281 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
284 ZOLTAN_ID_PTR , ZOLTAN_ID_PTR ,
int dim,
double *coordinates,
int *ierr)
287 *ierr = ZOLTAN_FATAL;
291 typedef Xpetra::MultiVector<typename Teuchos::ScalarTraits<Scalar>::coordinateType,
LocalOrdinal,
GlobalOrdinal,
Node> double_multivector_type;
292 double_multivector_type *Coords = (double_multivector_type*) data;
294 if (dim != Teuchos::as<int>(Coords->getNumVectors())) {
296 *ierr = ZOLTAN_FATAL;
300 TEUCHOS_TEST_FOR_EXCEPTION(numObjectIDs != Teuchos::as<int>(Coords->getLocalLength()),
Exceptions::Incompatible,
"Length of coordinates must be the same as the number of objects");
302 ArrayRCP<ArrayRCP<const typename Teuchos::ScalarTraits<Scalar>::coordinateType> > CoordsData(dim);
303 for (
int j = 0; j < dim; ++j)
304 CoordsData[j] = Coords->getData(j);
306 size_t numElements = Coords->getLocalLength();
307 for (
size_t i = 0; i < numElements; ++i)
308 for (
int j = 0; j < dim; ++j)
309 coordinates[i*dim+j] = (
double) CoordsData[j][i];
MueLu::DefaultLocalOrdinal LocalOrdinal
MueLu::DefaultGlobalOrdinal GlobalOrdinal
Exception throws to report incompatible objects (like maps).
Exception throws to report errors in the internal logical of the program.
Timer to be used in factories. Similar to Monitor but with additional timers.
Class that holds all level-specific information.
Timer to be used in factories. Similar to SubMonitor but adds a timer level by level.
static void GetLocalNumberOfNonzeros(void *data, int NumGidEntries, int NumLidEntries, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int wgtDim, float *weights, int *ierr)
static int GetLocalNumberOfRows(void *data, int *ierr)
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.
void Build(Level &level) const
Build an object with this factory.
void DeclareInput(Level &level) const
Specifies the data that this class needs, and the factories that generate that data.
static void GetProblemGeometry(void *data, int numGIDEntries, int numLIDEntries, int numObjectIDs, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int dim, double *coordinates, int *ierr)
static int GetProblemDimension(void *data, int *ierr)
Namespace for MueLu classes and methods.
@ Statistics1
Print more statistics.
std::string toString(const T &what)
Little helper function to convert non-string types to strings.