Teuchos Package Browser (Single Doxygen Collection) Version of the Day
Loading...
Searching...
No Matches
Teuchos_GlobalMPISession.cpp
Go to the documentation of this file.
1// @HEADER
2// ***********************************************************************
3//
4// Teuchos: Common Tools Package
5// Copyright (2004) Sandia Corporation
6//
7// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8// license for use of this work by or on behalf of the U.S. Government.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38//
39// ***********************************************************************
40// @HEADER
41
43#include "Teuchos_Assert.hpp"
44
45// The header file does not at all depend on MPI routines or types,
46// so we can defer inclusion of mpi.h to here. This also fixes Bug
47// 5631: https://software.sandia.gov/bugzilla/show_bug.cgi?id=5631
48#ifdef HAVE_MPI
49# include "mpi.h"
50#endif
51
52#ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
53# include "Kokkos_Core.hpp"
54#endif // HAVE_TEUCHOSCORE_KOKKOSCORE
55
56
57
58namespace Teuchos {
59
60
65
66#ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
67
68// We have to invoke the std::vector's constructor here,
69// because it's a class (static) variable.
70std::vector<std::string> GlobalMPISession::argvCopy_;
71
72#endif // HAVE_TEUCHOSCORE_KOKKOSCORE
73
74
75GlobalMPISession::GlobalMPISession( int* argc, char*** argv, std::ostream *out )
76{
77 std::ostringstream oss;
78
79 // Above is used to create all output before sending to *out to avoid
80 // jumbled parallel output between processors
81
82#ifdef HAVE_MPI
83
84 int mpierr = 0;
85
86 // Assert that MPI is not already initialized
87 int mpiHasBeenStarted = 0;
88 MPI_Initialized(&mpiHasBeenStarted);
89 if (mpiHasBeenStarted) {
90 if (out) {
91 *out << "GlobalMPISession(): Error, MPI_Intialized() return true,"
92 << " calling std::terminate()!\n"
93 << std::flush;
94 }
95 std::terminate();
96 }
97
98 // Initialize MPI
99 mpierr = ::MPI_Init(argc, (char ***) argv);
100 if (mpierr != 0) {
101 if (out) {
102 *out << "GlobalMPISession(): Error, MPI_Init() returned error code="
103 << mpierr << "!=0, calling std::terminate()!\n"
104 << std::flush;
105 }
106 std::terminate();
107 }
108
109 initialize(out); // Get NProc_ and rank_
110
111 int nameLen;
112 char procName[MPI_MAX_PROCESSOR_NAME];
113 mpierr = ::MPI_Get_processor_name(procName, &nameLen);
114 if (mpierr != 0) {
115 if (out) {
116 *out << "GlobalMPISession(): Error, MPI_Get_processor_name() error code="
117 << mpierr << "!=0, calling std::terminate()!\n"
118 << std::flush;
119 }
120 std::terminate();
121 }
122
123 oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started processor with name "
124 << procName << " and rank " << rank_ << "!" << std::endl;
125
126#else
127
128 oss << "Teuchos::GlobalMPISession::GlobalMPISession(): started serial run"
129 << std::endl;
130
131#endif
132
133#ifndef TEUCHOS_SUPPRESS_PROC_STARTUP_BANNER
134
135 // See if we should suppress the startup banner
136 bool printStartupBanner = true;
137 const std::string suppress_option("--teuchos-suppress-startup-banner");
138 for ( int opt_i = 0; opt_i < *argc; ++opt_i ) {
139 if ( suppress_option == (*argv)[opt_i] ) {
140 // We are suppressing the output!
141 printStartupBanner = false;
142 // Remove this option!
143 // Note that (*argv)[*argc]==0 but convention so we copy it too!
144 for( int i = opt_i; i < *argc; ++i )
145 (*argv)[i] = (*argv)[i+1];
146 --*argc;
147 }
148 }
149 if (out && printStartupBanner) {
150 *out << oss.str() << std::flush;
151 }
152
153#endif
154
155#ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
156 // mfh 15/16 Apr 2016: This is the one chance we get to save the
157 // command-line arguments, so that we can (later) initialize Kokkos
158 // with the correct number of threads as specified by (e.g.,) the
159 // --kokkos-threads command-line argument. We won't attempt to
160 // initialize Kokkos now, because not all applications want Kokkos.
161 // Some applications may also prefer to initialize Kokkos with their
162 // own thread count.
163 //
164 // NOTE (mfh 15/16 Apr 2016): While static variables are not thread
165 // safe in general, and this is not thread safe in particular, it
166 // only makes sense to GlobalMPISession's constructor on a single
167 // thread per MPI process anyway, because MPI_Init has the same
168 // requirement.
169
170 const int numArgs = *argc;
171 argvCopy_.resize (numArgs);
172 for (int c = 0; c < numArgs; ++c) {
173 argvCopy_[c] = std::string ((*argv)[c]); // deep copy
174 }
175#endif // HAVE_TEUCHOSCORE_KOKKOSCORE
176}
177
178
179#ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
180std::vector<std::string> GlobalMPISession::getArgv ()
181{
182 return argvCopy_;
183}
184#endif // HAVE_TEUCHOSCORE_KOKKOSCORE
185
186
188{
189
190#ifdef HAVE_TEUCHOSCORE_KOKKOSCORE
191 try {
192 if (Kokkos::is_initialized())
193 Kokkos::finalize();
194 }
195 catch (const std::runtime_error& e) {
196 std::cerr << "Kokkos::finalize failed:\n"
197 << e.what() << "\n";
198 }
199#endif
200
201 haveMPIState_ = false;
202#ifdef HAVE_MPI
203 const int mpierr = ::MPI_Finalize();
204 mpiIsFinalized_ = (mpierr == 0);
205 if (mpierr != 0)
206 std::cerr << "Error code " << mpierr << " returned from MPI_Finalize()\n";
207#else
208 mpiIsFinalized_ = true;
209#endif
210}
211
214 #ifdef HAVE_MPI
215 MPI_Abort(MPI_COMM_WORLD, MPI_ERR_UNKNOWN);
216 #else
217 std::abort();
218 #endif
219}
220
223 return haveMPIState_;
224}
225
226
228{
229 return mpiIsFinalized_;
230}
231
232
234{
236 return rank_;
237}
238
239
242 return nProc_;
243}
244
245
247{
249#ifdef HAVE_MPI
250 MPI_Barrier(MPI_COMM_WORLD);
251#endif
252}
253
254
255int GlobalMPISession::sum(int localVal)
256{
258#ifdef HAVE_MPI
259 int globalSum = -1;
260 MPI_Allreduce(&localVal, &globalSum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
261 return globalSum;
262#else
263 return localVal;
264#endif
265}
266
267
268void GlobalMPISession::allGather(int localVal, const ArrayView<int> &allVals)
269{
272#ifdef HAVE_MPI
273 MPI_Allgather( &localVal, 1, MPI_INT, allVals.getRawPtr(), 1, MPI_INT,
274 MPI_COMM_WORLD);
275#else
276 allVals[0] = localVal;
277#endif
278}
279
280
281// private
282
283
284void GlobalMPISession::initialize( std::ostream *out )
285{
286#ifdef HAVE_MPI
287
288 if(mpiIsFinalized_) {
289 // MPI has aleady been finalized so we have a serial machine again!
290 rank_ = 0;
291 nProc_ = 1;
292 return;
293 }
294
295 if(haveMPIState_) {
296 return; // We already have what we need!
297 }
298
299 // We don't have the state of MPI so the constructor for this class must not
300 // have been called. However, if MPI has been called in another way we
301 // can still get the state of MPI_COMM_WORLD here.
302
303 int mpiHasBeenStarted = 0;
304 MPI_Initialized(&mpiHasBeenStarted);
305
306 if(!mpiHasBeenStarted)
307 return; // We have to give up and just leave NProc_ and rank_ at the default values.
308
309 // Get the state of MPI
310 // Don't throw exceptions here since this part of the code
311 // is used by TEUCHOS_STANDARD_CATCH_STATEMENTS().
312 // See bug #6192 <https://software.sandia.gov/bugzilla/show_bug.cgi?id=6192>.
313 int mpierr = 0;
314 mpierr = ::MPI_Comm_rank( MPI_COMM_WORLD, &rank_ );
315 if (mpierr != 0) {
316 *out << "Error code=" << mpierr << " detected in MPI_Comm_rank()"
317 << std::endl;
318 }
319
320 mpierr = ::MPI_Comm_size( MPI_COMM_WORLD, &nProc_ );
321 if (mpierr != 0) {
322 *out << "Error code=" << mpierr << " detected in MPI_Comm_size()"
323 << std::endl;
324 }
325
326 haveMPIState_ = true;
327 mpiIsFinalized_ = false;
328
329#endif // HAVE_MPI
330
331}
332
333
335{
336 if(!haveMPIState_)
337 initialize(&std::cerr);
338}
339
340
341} // namespace Teuchos
A MPI utilities class, providing methods for initializing, finalizing, and querying the global MPI se...
Nonowning array view.
size_type size() const
The total number of items in the managed array.
T * getRawPtr() const
Return a raw pointer to beginning of array or NULL if unsized.
static int sum(int localVal)
Sum a set of integers across processes.
static void abort()
abort the program
static void barrier()
Call MPI_Barrier() on MPI_COMM_WORLD.
static int getRank()
The rank of the calling process in MPI_COMM_WORLD.
GlobalMPISession(int *argc, char ***argv, std::ostream *out=&std::cout)
Calls MPI_Init() if MPI is enabled.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
static bool mpiIsInitialized()
Return whether MPI was initialized.
static bool mpiIsFinalized()
Return whether MPI was already finalized.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
~GlobalMPISession()
Call MPI_Finalize() if MPI is enabled.
static void initialize(std::ostream *out)
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...