Stokhos Package Browser (Single Doxygen Collection) Version of the Day
Loading...
Searching...
No Matches
MPAssembly/TestAssembly.cpp
Go to the documentation of this file.
1// @HEADER
2// ***********************************************************************
3//
4// Stokhos Package
5// Copyright (2009) Sandia Corporation
6//
7// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8// license for use of this work by or on behalf of the U.S. Government.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// Questions? Contact Eric T. Phipps (etphipp@sandia.gov).
38//
39// ***********************************************************************
40// @HEADER
41
42#include <iostream>
43
44// Tests
45#include "TestAssembly.hpp"
46
47// Devices
48#include "Kokkos_Core.hpp"
49
50// Utilities
51#include "Teuchos_DefaultComm.hpp"
52#include "Teuchos_CommandLineProcessor.hpp"
53#include "Teuchos_StandardCatchMacros.hpp"
54#ifdef KOKKOS_ENABLE_CUDA
55#include "cuda_runtime_api.h"
56#endif
57
58template <typename Storage>
59void mainHost(const Teuchos::RCP<const Teuchos::Comm<int> >& comm ,
60 const int use_print ,
61 const int use_trials ,
62 const int use_nodes[] ,
63 const bool check ,
65#ifdef __MIC__
66 const int entry_min = 8;
67 const int entry_max = 48;
68 const int entry_step = 8;
69#else
70 const int entry_min = 4;
71 const int entry_max = 32;
72 const int entry_step = 4;
73 // const int entry_min = 16;
74 // const int entry_max = 16;
75 // const int entry_step = 16;
76#endif
77
78 performance_test_driver<Storage,entry_min,entry_max,entry_step>(
79 comm, use_print, use_trials, use_nodes, check, dev_config);
80}
81
82template <typename Storage>
83void mainCuda(const Teuchos::RCP<const Teuchos::Comm<int> >& comm ,
84 const int use_print ,
85 const int use_trials ,
86 const int use_nodes[] ,
87 const bool check ,
89 const int entry_min = 16;
90 const int entry_max = 64;
91 const int entry_step = 16;
92 performance_test_driver<Storage,entry_min,entry_max,entry_step>(
93 comm, use_print, use_trials, use_nodes, check, dev_config);
94}
95
96int main(int argc, char *argv[])
97{
98 bool success = true;
99 bool verbose = false;
100 try {
101
102 Teuchos::oblackholestream blackHole;
103 Teuchos::GlobalMPISession mpiSession (&argc, &argv, &blackHole);
104
105 Teuchos::RCP<const Teuchos::Comm<int> > comm =
106 Teuchos::DefaultComm<int>::getComm();
107
108 const size_t num_sockets = Kokkos::hwloc::get_available_numa_count();
109 const size_t num_cores_per_socket =
110 Kokkos::hwloc::get_available_cores_per_numa();
111 const size_t num_threads_per_core =
112 Kokkos::hwloc::get_available_threads_per_core();
113
114 // Setup command line options
115 Teuchos::CommandLineProcessor CLP;
116 CLP.setDocString(
117 "This test performance of MP::Vector FEM assembly.\n");
118 int nGrid = 32;
119 CLP.setOption("n", &nGrid, "Number of mesh points in the each direction");
120 int nIter = 10;
121 CLP.setOption("ni", &nIter, "Number of assembly iterations");
122 bool print = false;
123 CLP.setOption("print", "no-print", &print, "Print debugging output");
124 bool check = false;
125 int num_cores = num_cores_per_socket * num_sockets;
126 CLP.setOption("cores", &num_cores,
127 "Number of CPU cores to use (defaults to all)");
128 int num_hyper_threads = num_threads_per_core;
129 CLP.setOption("hyperthreads", &num_hyper_threads,
130 "Number of hyper threads per core to use (defaults to all)");
131 int threads_per_vector = 1;
132 CLP.setOption("threads_per_vector", &threads_per_vector,
133 "Number of threads to use within each vector");
134 CLP.setOption("check", "no-check", &check, "Check correctness");
135#ifdef KOKKOS_ENABLE_SERIAL
136 bool serial = true;
137 CLP.setOption("serial", "no-serial", &serial, "Enable Serial device");
138#endif
139#ifdef KOKKOS_ENABLE_THREADS
140 bool threads = true;
141 CLP.setOption("threads", "no-threads", &threads, "Enable Threads device");
142#endif
143#ifdef KOKKOS_ENABLE_OPENMP
144 bool openmp = true;
145 CLP.setOption("openmp", "no-openmp", &openmp, "Enable OpenMP device");
146#endif
147#ifdef KOKKOS_ENABLE_CUDA
148 bool cuda = true;
149 CLP.setOption("cuda", "no-cuda", &cuda, "Enable Cuda device");
150 int cuda_threads_per_vector = 16;
151 CLP.setOption("cuda_threads_per_vector", &cuda_threads_per_vector,
152 "Number of Cuda threads to use within each vector");
153 int cuda_block_size = 256;
154 CLP.setOption("cuda_block_size", &cuda_block_size,
155 "Cuda block size");
156 int num_cuda_blocks = 0;
157 CLP.setOption("num_cuda_blocks", &num_cuda_blocks,
158 "Number of Cuda blocks (0 implies the default choice)");
159 int device_id = -1;
160 CLP.setOption("device", &device_id, "CUDA device ID. Set to default of -1 to use the default device as determined by the local node MPI rank and --ngpus");
161 int ngpus = 1;
162 CLP.setOption("ngpus", &ngpus, "Number of GPUs per node for multi-GPU runs via MPI");
163#endif
164 CLP.parse( argc, argv );
165
166 int use_nodes[3];
167 use_nodes[0] = nGrid; use_nodes[1] = nGrid; use_nodes[2] = nGrid;
168
169 typedef int Ordinal;
170 typedef double Scalar;
171
172#ifdef KOKKOS_ENABLE_SERIAL
173 if (serial) {
174 typedef Kokkos::Serial Device;
176
177 Kokkos::initialize();
178
179 if (comm->getRank() == 0)
180 std::cout << std::endl
181 << "Serial performance with " << comm->getSize()
182 << " MPI ranks" << std::endl;
183
184 Kokkos::Example::FENL::DeviceConfig dev_config(1, 1, 1);
185
186 mainHost<Storage>(comm, print, nIter, use_nodes, check,
187 dev_config);
188
189 Kokkos::finalize();
190 }
191#endif
192
193#ifdef KOKKOS_ENABLE_THREADS
194 if (threads) {
195 typedef Kokkos::Threads Device;
197
198 Kokkos::InitializationSettings init_args;
199 init_args.set_num_threads(num_cores*num_hyper_threads);
200 Kokkos::initialize( init_args );
201
202 if (comm->getRank() == 0)
203 std::cout << std::endl
204 << "Threads performance with " << comm->getSize()
205 << " MPI ranks and " << num_cores*num_hyper_threads
206 << " threads per rank:" << std::endl;
207
209 threads_per_vector,
210 num_hyper_threads / threads_per_vector);
211
212 mainHost<Storage>(comm, print, nIter, use_nodes, check,
213 dev_config);
214
215 Kokkos::finalize();
216 }
217#endif
218
219#ifdef KOKKOS_ENABLE_OPENMP
220 if (openmp) {
221 typedef Kokkos::OpenMP Device;
223
224 Kokkos::InitializationSettings init_args;
225 init_args.set_num_threads(num_cores*num_hyper_threads);
226 Kokkos::initialize( init_args );
227
228 if (comm->getRank() == 0)
229 std::cout << std::endl
230 << "OpenMP performance with " << comm->getSize()
231 << " MPI ranks and " << num_cores*num_hyper_threads
232 << " threads per rank:" << std::endl;
233
235 threads_per_vector,
236 num_hyper_threads / threads_per_vector);
237
238 mainHost<Storage>(comm, print, nIter, use_nodes, check,
239 dev_config);
240
241 Kokkos::finalize();
242 }
243#endif
244
245#ifdef KOKKOS_ENABLE_CUDA
246 if (cuda) {
247 typedef Kokkos::Cuda Device;
249
250 if (device_id == -1) {
251 int local_rank = 0;
252 char *str;
253 if ((str = std::getenv("SLURM_LOCALID")))
254 local_rank = std::atoi(str);
255 else if ((str = std::getenv("MV2_COMM_WORLD_LOCAL_RANK")))
256 local_rank = std::atoi(str);
257 else if ((str = getenv("OMPI_COMM_WORLD_LOCAL_RANK")))
258 local_rank = std::atoi(str);
259 device_id = local_rank % ngpus;
260
261 // Check device is valid
262 int num_device; cudaGetDeviceCount(&num_device);
263 TEUCHOS_TEST_FOR_EXCEPTION(
264 device_id >= num_device, std::logic_error,
265 "Invalid device ID " << device_id << ". You probably are trying" <<
266 " to run with too many GPUs per node");
267 }
268
269 Kokkos::InitializationSettings init_args;
270 init_args.set_device_id(device_id);
271 Kokkos::initialize( init_args );
272
273 cudaDeviceProp deviceProp;
274 cudaGetDeviceProperties(&deviceProp, device_id);
275 if (comm->getRank() == 0)
276 std::cout << std::endl
277 << "CUDA performance performance with " << comm->getSize()
278 << " MPI ranks and device " << device_id << " ("
279 << deviceProp.name << "):"
280 << std::endl;
281
283 num_cuda_blocks,
284 cuda_threads_per_vector,
285 cuda_threads_per_vector == 0 ? 0 : cuda_block_size / cuda_threads_per_vector);
286
287 mainCuda<Storage>(comm, print, nIter, use_nodes, check,
288 dev_config);
289
290 Kokkos::finalize();
291 }
292#endif
293
294 }
295 TEUCHOS_STANDARD_CATCH_STATEMENTS(verbose, std::cerr, success);
296
297 if (success)
298 return 0;
299 return -1;
300}
int main(int argc, char *argv[])
void mainCuda(const Teuchos::RCP< const Teuchos::Comm< int > > &comm, const int use_print, const int use_trials, const int use_nodes[], const bool check, Kokkos::Example::FENL::DeviceConfig dev_config)
void mainHost(const Teuchos::RCP< const Teuchos::Comm< int > > &comm, const int use_print, const int use_trials, const int use_nodes[], const bool check, Kokkos::Example::FENL::DeviceConfig dev_config)
Statically allocated storage class.
int check(Epetra_CrsGraph &A, int NumMyRows1, int NumGlobalRows1, int NumMyNonzeros1, int NumGlobalNonzeros1, int *MyGlobalElements, bool verbose)
Stokhos::StandardStorage< int, double > Storage