43#ifndef DOMI_MDVECTOR_HPP
44#define DOMI_MDVECTOR_HPP
52#include "Domi_ConfigDefs.hpp"
53#include "Domi_MDMap.hpp"
54#include "Domi_MDArrayRCP.hpp"
57#include "Teuchos_DataAccess.hpp"
58#include "Teuchos_Describable.hpp"
59#include "Teuchos_ScalarTraitsDecl.hpp"
60#include "Teuchos_Comm.hpp"
61#include "Teuchos_CommHelpers.hpp"
64#include "Epetra_IntVector.h"
65#include "Epetra_Vector.h"
69#include "Tpetra_Vector.hpp"
80struct ompi_datatype_t {};
174template<
class Scalar >
251 Teuchos::DataAccess
access = Teuchos::View);
267 Teuchos::ParameterList &
plist);
282 Teuchos::ParameterList &
plist);
331 const Teuchos::ArrayView< Slice > &
slices,
332 const Teuchos::ArrayView< int > &
bndryPad =
333 Teuchos::ArrayView< int >());
353 inline const Teuchos::RCP< const MDMap >
370 inline Teuchos::RCP< const Teuchos::Comm< int > >
getTeuchosComm()
const;
765 Teuchos::RCP< Tpetra::Vector< Scalar, LocalOrdinal, GlobalOrdinal > >
787 Teuchos::RCP< Tpetra::MultiVector<
Scalar,
799 Teuchos::RCP< Tpetra::Vector< Scalar, LocalOrdinal, GlobalOrdinal > >
816 Teuchos::RCP< Tpetra::MultiVector<
Scalar,
884 typename Teuchos::ScalarTraits< Scalar >::magnitudeType
norm1()
const;
888 typename Teuchos::ScalarTraits< Scalar >::magnitudeType
norm2()
const;
892 typename Teuchos::ScalarTraits< Scalar >::magnitudeType
normInf()
const;
898 typename Teuchos::ScalarTraits< Scalar >::magnitudeType
923 const Teuchos::EVerbosityLevel
verbLevel =
924 Teuchos::Describable::verbLevel_default)
const;
1058 Teuchos::RCP< const Teuchos::Comm< int > > _teuchosComm;
1062 Teuchos::RCP< const MDMap > _mdMap;
1083 Teuchos::Array< MPI_Request >
_requests;
1096 Teuchos::RCP< MPI_Datatype >
datatype;
1110 Teuchos::Array< Teuchos::Tuple< MessageInfo, 2 > > _sendMessages;
1115 Teuchos::Array< Teuchos::Tuple< MessageInfo, 2 > > _recvMessages;
1119 void initializeMessages();
1131 Teuchos::Array< dim_type > fileShape;
1132 Teuchos::Array< dim_type > bufferShape;
1133 Teuchos::Array< dim_type > dataShape;
1134 Teuchos::Array< dim_type > fileStart;
1135 Teuchos::Array< dim_type > dataStart;
1137 Teuchos::RCP< MPI_Datatype >
filetype;
1138 Teuchos::RCP< MPI_Datatype >
datatype;
1146 mutable Teuchos::RCP< FileInfo > _fileInfo;
1152 mutable Teuchos::RCP< FileInfo > _fileInfoWithBndry;
1160 Teuchos::RCP< FileInfo > & computeFileInfo(
bool includeBndryPad)
const;
1170template<
class Scalar >
1174 _teuchosComm(
mdMap->getTeuchosComm()),
1188 int numDims = _mdMap->numDims();
1190 for (
int axis = 0; axis <
numDims; ++axis)
1191 dims[axis] = _mdMap->getLocalDim(axis,
true);
1195 _mdArrayView = _mdArrayRcp();
1200template<
class Scalar >
1206 _teuchosComm(
mdMap->getTeuchosComm()),
1220 int numDims = _mdMap->numDims();
1222 for (
int axis = 0; axis <
numDims; ++axis)
1223 dims[axis] = _mdMap->getLocalDim(axis,
true);
1227 _mdArrayView = _mdArrayRcp();
1232template<
class Scalar >
1238 _mdArrayView(_mdArrayRcp()),
1247 int numDims = _mdMap->numDims();
1251 "MDMap and source array do not have the same number of dimensions");
1253 for (
int axis = 0; axis <
numDims; ++axis)
1256 _mdMap->getLocalDim(axis) != _mdArrayRcp.
dimension(axis),
1258 "Axis " << axis <<
": MDMap dimension = " << _mdMap->getLocalDim(axis)
1259 <<
", MDArray dimension = " << _mdArrayRcp.
dimension(axis));
1265template<
class Scalar >
1271 _mdArrayView(_mdArrayRcp()),
1279#ifdef DOMI_MDVECTOR_VERBOSE
1280 cout <<
"_mdArrayRcp = " << _mdArrayRcp << endl;
1281 cout <<
"_mdArrayView.getRawPtr() = " << _mdArrayView.
getRawPtr()
1282 <<
" (constructor)" << endl;
1283 cout <<
"_mdArrayView = " << _mdArrayView << endl;
1286 int numDims = _mdMap->numDims();
1290 "MDMap and source array do not have the same number of dimensions");
1292 for (
int axis = 0; axis <
numDims; ++axis)
1295 _mdMap->getLocalDim(axis) != _mdArrayRcp.
dimension(axis),
1297 "Axis " << axis <<
": MDMap dimension = " << _mdMap->getLocalDim(axis)
1298 <<
", MDArray dimension = " << _mdArrayRcp.
dimension(axis));
1304template<
class Scalar >
1307 Teuchos::DataAccess
access) :
1308 _teuchosComm(
source.getMDMap()->getTeuchosComm()),
1309 _mdMap(
source.getMDMap()),
1310 _mdArrayRcp(
source._mdArrayRcp),
1311 _mdArrayView(
source._mdArrayView),
1321 if (
access == Teuchos::Copy)
1323#ifdef DOMI_MDVECTOR_VERBOSE
1324 cout <<
"Inside MDVector copy constructor with copy access" << endl;
1327 int numDims = _mdMap->numDims();
1329 for (
int axis = 0; axis <
numDims; ++axis)
1330 dims[axis] = _mdMap->getLocalDim(axis,
true);
1334 _mdArrayView = _mdArrayRcp();
1346#ifdef DOMI_MDVECTOR_VERBOSE
1349 cout <<
"Inside MDVector copy constructor with view access"
1357template<
class Scalar >
1360 Teuchos::ParameterList &
plist) :
1384 _mdMap = Teuchos::rcp(
myMdMap);
1387 int numDims = _mdMap->numDims();
1389 for (
int axis = 0; axis <
numDims; ++axis)
1390 dims[axis] = _mdMap->getLocalDim(axis,
true);
1394 _mdArrayView = _mdArrayRcp();
1399template<
class Scalar >
1402 Teuchos::ParameterList &
plist) :
1403 _teuchosComm(
mdComm->getTeuchosComm()),
1426 _mdMap = Teuchos::rcp(
myMdMap);
1429 int numDims = _mdMap->numDims();
1431 for (
int axis = 0; axis <
numDims; ++axis)
1432 dims[axis] = _mdMap->getLocalDim(axis,
true);
1436 _mdArrayView = _mdArrayRcp();
1441template<
class Scalar >
1446 _teuchosComm(
parent._teuchosComm),
1448 _mdArrayRcp(
parent._mdArrayRcp),
1449 _mdArrayView(
parent._mdArrayView),
1468 if (_mdMap->onSubcommunicator())
1491 _mdArrayRcp.
clear();
1498template<
class Scalar >
1506 _mdArrayRcp(
parent._mdArrayRcp),
1507 _mdArrayView(
parent._mdArrayView),
1515#ifdef DOMI_MDVECTOR_VERBOSE
1516 cout <<
"slice axis " << axis << endl;
1517 cout <<
" _mdArrayRcp @ " << _mdArrayRcp.
getRawPtr() << endl;
1518 cout <<
" _mdArrayView @ " << _mdArrayView.
getRawPtr() << endl;
1531 _teuchosComm = _mdMap->getTeuchosComm();
1534 if (_mdMap->onSubcommunicator())
1570 _mdArrayRcp.
clear();
1573#ifdef DOMI_MDVECTOR_VERBOSE
1574 cout <<
" _mdArrayView @ " << _mdArrayView.
getRawPtr() << endl;
1582template<
class Scalar >
1585 const Teuchos::ArrayView< Slice > &
slices,
1586 const Teuchos::ArrayView< int > &
bndryPad)
1597 "number of slices = " <<
slices.
size() <<
" != parent MDVector number of "
1602 for (
int axis = 0; axis <
numDims; ++axis)
1616template<
class Scalar >
1621 _teuchosComm =
source._teuchosComm;
1623 _mdArrayRcp =
source._mdArrayRcp;
1624 _mdArrayView =
source._mdArrayView;
1625 _nextAxis =
source._nextAxis;
1629 _sendMessages =
source._sendMessages;
1630 _recvMessages =
source._recvMessages;
1636template<
class Scalar >
1643template<
class Scalar >
1644const Teuchos::RCP< const MDMap >
1653template<
class Scalar >
1658 return _mdMap->onSubcommunicator();
1663template<
class Scalar >
1664Teuchos::RCP< const Teuchos::Comm< int > >
1668 return _mdMap->getTeuchosComm();
1673template<
class Scalar >
1683template<
class Scalar >
1688 return _mdMap->getCommDim(axis);
1693template<
class Scalar >
1698 return _mdMap->isPeriodic(axis);
1703template<
class Scalar >
1708 return _mdMap->getCommIndex(axis);
1713template<
class Scalar >
1718 return _mdMap->getLowerNeighbor(axis);
1723template<
class Scalar >
1728 return _mdMap->getUpperNeighbor(axis);
1733template<
class Scalar >
1743template<
class Scalar >
1753template<
class Scalar >
1758 return _mdMap->getGlobalRankBounds(axis,
withBndryPad);
1763template<
class Scalar >
1773template<
class Scalar >
1774Teuchos::ArrayView< const Slice >
1778 return _mdMap->getLocalBounds();
1783template<
class Scalar >
1793template<
class Scalar >
1798 return _mdMap->getLocalInteriorBounds(axis);
1803template<
class Scalar >
1808 return _mdMap->hasPadding();
1813template<
class Scalar >
1818 return _mdMap->getLowerPadSize(axis);
1823template<
class Scalar >
1828 return _mdMap->getUpperPadSize(axis);
1833template<
class Scalar >
1838 return _mdMap->getCommPadSize(axis);
1843template<
class Scalar >
1848 return _mdMap->getLowerBndryPad(axis);
1853template<
class Scalar >
1858 return _mdMap->getUpperBndryPad(axis);
1863template<
class Scalar >
1868 return _mdMap->getBndryPadSize(axis);
1873template<
class Scalar >
1885template<
class Scalar >
1897template<
class Scalar >
1902 return _mdMap->isReplicatedBoundary(axis);
1907template<
class Scalar >
1912 return _mdMap->getLayout();
1917template<
class Scalar >
1922 return _mdMap->isContiguous();
1936template<
class Scalar >
1937Teuchos::RCP< Epetra_IntVector >
1943 typeid(
Scalar) !=
typeid(
int),
1945 "MDVector is of scalar type '" <<
typeid(
Scalar).
name() <<
"', but "
1946 "Epetra_IntVector requires scalar type 'int'");
1952 "This MDVector's MDMap is non-contiguous. This can happen when you take "
1953 "a slice of a parent MDVector.");
1956 Teuchos::RCP< const Epetra_Map >
epetraMap = _mdMap->getEpetraMap(
true);
1961 void * buffer = (
void*) _mdArrayView.
getRawPtr();
1976template<
class Scalar >
1977Teuchos::RCP< Epetra_Vector >
1979getEpetraVectorView()
const
1983 typeid(
Scalar) !=
typeid(
double),
1985 "MDVector is of scalar type '" <<
typeid(
Scalar).
name() <<
"', but "
1986 "Epetra_Vector requires scalar type 'double'");
1991 MDMapNoncontiguousError,
1992 "This MDVector's MDMap is non-contiguous. This can happen when you take "
1993 "a slice of a parent MDVector.");
1996 Teuchos::RCP< const Epetra_Map >
epetraMap = _mdMap->getEpetraMap(
true);
2001 void * buffer = (
void*) _mdArrayView.
getRawPtr();
2016template<
class Scalar >
2017Teuchos::RCP< Epetra_MultiVector >
2019getEpetraMultiVectorView()
const
2023 typeid(
Scalar) !=
typeid(
double),
2025 "MDVector is of scalar type '" <<
typeid(
Scalar).
name() <<
"', but "
2026 "Epetra_Vector requires scalar type 'double'");
2035 Teuchos::RCP< const MDMap >
newMdMap;
2045 MDMapNoncontiguousError,
2046 "This MDVector's MDMap is non-contiguous. This can happen when you take "
2047 "a slice of a parent MDVector.");
2053 for (
int axis = 1; axis <
newMdMap->numDims(); ++axis)
2068 void * buffer = (
void*) _mdArrayView.getRawPtr();
2078template<
class Scalar >
2079Teuchos::RCP< Epetra_IntVector >
2081getEpetraIntVectorCopy()
const
2083 typedef typename MDArrayView< const Scalar >::iterator iterator;
2086 Teuchos::RCP< const Epetra_Map >
epetraMap = _mdMap->getEpetraMap(
true);
2089 Teuchos::RCP< Epetra_IntVector >
result =
2095 for (iterator
it = _mdArrayView.begin();
it != _mdArrayView.end(); ++
it)
2104template<
class Scalar >
2105Teuchos::RCP< Epetra_Vector >
2107getEpetraVectorCopy()
const
2109 typedef typename MDArrayView< const Scalar >::iterator iterator;
2112 Teuchos::RCP< const Epetra_Map >
epetraMap = _mdMap->getEpetraMap(
true);
2115 Teuchos::RCP< Epetra_Vector >
result =
2121 for (iterator
it = _mdArrayView.begin();
it != _mdArrayView.end(); ++
it)
2130template<
class Scalar >
2131Teuchos::RCP< Epetra_MultiVector >
2133getEpetraMultiVectorCopy()
const
2136 typedef typename MDArrayView< const Scalar >::iterator
citerator;
2145 Teuchos::RCP< const MDMap >
newMdMap;
2158 Teuchos::RCP< Epetra_MultiVector >
result =
2166 for (
citerator it = _mdArrayView.begin();
it != _mdArrayView.end(); ++
it)
2190template<
class Scalar >
2191template<
class LocalOrdinal,
2192 class GlobalOrdinal >
2193Teuchos::RCP< Tpetra::Vector< Scalar, LocalOrdinal, GlobalOrdinal > >
2195getTpetraVectorView()
const
2200 MDMapNoncontiguousError,
2201 "This MDVector's MDMap is non-contiguous. This can happen when you take "
2202 "a slice of a parent MDVector.");
2210 return Teuchos::rcp(
new Tpetra::Vector<
Scalar,
2213 _mdArrayView.arrayView()));
2218template<
class Scalar >
2219template<
class LocalOrdinal,
2220 class GlobalOrdinal >
2221Teuchos::RCP< Tpetra::Vector< Scalar, LocalOrdinal, GlobalOrdinal > >
2223getTpetraVectorCopy()
const
2225 typedef typename MDArrayView< const Scalar >::iterator iterator;
2233 Teuchos::RCP< Tpetra::Vector<
Scalar,
2236 Teuchos::rcp(
new Tpetra::Vector<
Scalar,
2244 for (iterator
it = _mdArrayView.begin();
it != _mdArrayView.end(); ++
it)
2253template<
class Scalar >
2254template <
class LocalOrdinal,
2255 class GlobalOrdinal >
2256Teuchos::RCP< Tpetra::MultiVector< Scalar,
2260getTpetraMultiVectorView()
const
2269 Teuchos::RCP< const MDMap >
newMdMap;
2279 MDMapNoncontiguousError,
2280 "This MDVector's MDMap is non-contiguous. This can happen when you take "
2281 "a slice of a parent MDVector.");
2287 for (
int axis = 1; axis <
newMdMap->numDims(); ++axis)
2290 (
long long int)(
stride*
numVectors) > Teuchos::OrdinalTraits<GlobalOrdinal>::max(),
2302 return Teuchos::rcp(
new Tpetra::MultiVector<
Scalar,
2305 _mdArrayView.arrayView(),
2312template<
class Scalar >
2313template <
class LocalOrdinal,
2314 class GlobalOrdinal >
2315Teuchos::RCP< Tpetra::MultiVector< Scalar,
2319getTpetraMultiVectorCopy()
const
2322 typedef typename MDArrayView< const Scalar >::iterator
citerator;
2331 Teuchos::RCP< const MDMap >
newMdMap;
2346 Teuchos::RCP< Tpetra::MultiVector<
Scalar,
2349 Teuchos::rcp(
new Tpetra::MultiVector<
Scalar,
2359 for (
citerator it = _mdArrayView.begin();
it != _mdArrayView.end(); ++
it)
2383template<
class Scalar >
2388#ifdef DOMI_MDVECTOR_VERBOSE
2389 cout <<
"_mdArrayView.getRawPtr() = " << _mdArrayView.
getRawPtr()
2391 cout <<
"_mdArrayView = " << _mdArrayView << endl;
2394 return _mdArrayView;
2396 for (
int axis = 0; axis <
numDims(); ++axis)
2398 int lo = getLowerPadSize(axis);
2399 int hi = getLocalDim(axis,
true) - getUpperPadSize(axis);
2407template<
class Scalar >
2413 return _mdArrayView.getConst();
2415 for (
int axis = 0; axis <
numDims(); ++axis)
2417 int lo = getLowerPadSize(axis);
2418 int hi = getLocalDim(axis,
true) - getUpperPadSize(axis);
2426template<
class Scalar >
2433 Slice(getLowerPadSize(axis)));
2439template<
class Scalar >
2446 Slice(getLowerPadSize(axis)));
2452template<
class Scalar >
2457 dim_type
n = getLocalDim(axis,
true);
2458 int pad = getUpperPadSize(axis);
2470template<
class Scalar >
2475 dim_type
n = getLocalDim(axis,
true);
2476 int pad = getUpperPadSize(axis);
2488template<
class Scalar >
2496 ! _mdMap->isCompatible(*(
a._mdMap)),
2498 "MDMap of calling MDVector and argument 'a' are incompatible");
2503 for (iterator
it = _mdArrayView.
begin();
it != _mdArrayView.
end();
2507 Teuchos::reduceAll(*_teuchosComm,
2508 Teuchos::REDUCE_SUM,
2517template<
class Scalar >
2518typename Teuchos::ScalarTraits< Scalar >::magnitudeType
2522 typedef typename Teuchos::ScalarTraits< Scalar >::magnitudeType
mag;
2526 for (iterator
it = _mdArrayView.
begin();
it != _mdArrayView.
end(); ++
it)
2529 Teuchos::reduceAll(*_teuchosComm,
2530 Teuchos::REDUCE_SUM,
2539template<
class Scalar >
2540typename Teuchos::ScalarTraits< Scalar >::magnitudeType
2544 typedef typename Teuchos::ScalarTraits< Scalar >::magnitudeType
mag;
2545 mag norm2 = dot(*
this);
2546 return Teuchos::ScalarTraits<mag>::squareroot(norm2);
2551template<
class Scalar >
2552typename Teuchos::ScalarTraits< Scalar >::magnitudeType
2556 typedef typename Teuchos::ScalarTraits< Scalar >::magnitudeType
mag;
2560 for (iterator
it = _mdArrayView.
begin();
it != _mdArrayView.
end(); ++
it)
2563 Teuchos::reduceAll(*_teuchosComm,
2564 Teuchos::REDUCE_MAX,
2573template<
class Scalar >
2574typename Teuchos::ScalarTraits< Scalar >::magnitudeType
2578 typedef typename Teuchos::ScalarTraits< Scalar >::magnitudeType
mag;
2582 ! _mdMap->isCompatible(*(
weights._mdMap)),
2584 "MDMap of calling MDVector and argument 'weights' are incompatible");
2589 for (iterator
it = _mdArrayView.
begin();
it != _mdArrayView.
end();
2593 Teuchos::reduceAll(*_teuchosComm,
2594 Teuchos::REDUCE_SUM,
2602 if (
n == 0)
return 0;
2603 return Teuchos::ScalarTraits<mag>::squareroot(
global_wNorm /
n);
2608template<
class Scalar >
2613 typedef typename Teuchos::ScalarTraits< Scalar >::magnitudeType
mag;
2617 for (iterator
it = _mdArrayView.
begin();
it != _mdArrayView.
end(); ++
it)
2620 Teuchos::reduceAll(*_teuchosComm,
2621 Teuchos::REDUCE_SUM,
2629 if (
n == 0)
return 0;
2635template<
class Scalar >
2640 using Teuchos::TypeNameTraits;
2643 for (
int axis = 0; axis <
numDims(); ++axis)
2644 dims[axis] = getGlobalDim(axis,
true);
2646 std::ostringstream
oss;
2647 oss <<
"\"Domi::MDVector\": {"
2648 <<
"Template parameters: {"
2653 oss <<
"Global dimensions: " <<
dims <<
" }";
2659template<
class Scalar >
2663 const Teuchos::EVerbosityLevel
verbLevel)
const
2666 using Teuchos::Comm;
2668 using Teuchos::TypeNameTraits;
2669 using Teuchos::VERB_DEFAULT;
2670 using Teuchos::VERB_NONE;
2671 using Teuchos::VERB_LOW;
2672 using Teuchos::VERB_MEDIUM;
2673 using Teuchos::VERB_HIGH;
2674 using Teuchos::VERB_EXTREME;
2676 const Teuchos::EVerbosityLevel
vl =
2680 Teuchos::RCP< const Teuchos::Comm< int > >
comm =
mdMap.getTeuchosComm();
2689 out <<
"\"Domi::MDVector\":" << endl;
2694 out <<
"Template parameters:";
2705 for (
int axis = 0; axis <
numDims(); ++axis)
2719 for (
int axis = 0; axis <
numDims(); ++axis)
2720 localDims[axis] = getLocalDim(axis,
true);
2734template<
class Scalar >
2749template<
class Scalar >
2756 Teuchos::ScalarTraits< Scalar >::seedrandom(
time(
NULL));
2757 for (iterator
it = _mdArrayView.
begin();
it != _mdArrayView.
end(); ++
it)
2758 *
it = Teuchos::ScalarTraits< Scalar >::random();
2763template<
class Scalar >
2768 for (
int axis = 0; axis <
numDims(); ++axis)
2770 updateCommPad(axis);
2776template<
class Scalar >
2781 startUpdateCommPad(axis);
2782 endUpdateCommPad(axis);
2787template<
class Scalar >
2796 if (_sendMessages.empty()) initializeMessages();
2799 int rank = _teuchosComm->getRank();
2800 int numProc = _teuchosComm->getSize();
2806 Teuchos::RCP< const Teuchos::MpiComm< int > >
mpiComm =
2807 Teuchos::rcp_dynamic_cast< const Teuchos::MpiComm< int > >(_teuchosComm);
2808 const Teuchos::OpaqueWrapper< MPI_Comm > &
communicator =
2820#ifdef DOMI_MDVECTOR_OUTPUT_UPDATECOMMPAD
2821 cout <<
rank <<
": post send for axis " << axis <<
", boundary "
2823 <<
message.proc <<
", tag = " <<
tag << endl;
2833 throw std::runtime_error(
"Domi::MDVector: Error in MPI_Isend");
2846#ifdef DOMI_MDVECTOR_OUTPUT_UPDATECOMMPAD
2847 cout <<
rank <<
": post recv for axis " << axis <<
", boundary "
2849 <<
message.proc <<
", tag = " <<
tag << endl;
2859 throw std::runtime_error(
"Domi::MDVector: Error in MPI_Irecv");
2867 if (isPeriodic(axis))
2890template<
class Scalar >
2902 throw std::runtime_error(
"Domi::MDVector: Error in MPI_Waitall");
2910template<
class Scalar >
2926template<
class Scalar >
2942template<
class Scalar >
2955 _sendMessages.resize(
ndims);
2956 _recvMessages.resize(
ndims);
2958#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
2959 std::stringstream
msg;
2960 int rank = _teuchosComm->getRank();
2972 for (
int axis = 0; axis <
ndims; ++axis)
2983 int proc = getLowerNeighbor(
msgAxis);
2985#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
2986 msg << endl <<
"P" <<
rank <<
": axis " <<
msgAxis <<
", lower neighbor = "
2995 messageInfo.buffer = (
void*) getData().getRawPtr();
3007#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
3009 <<
", Lower receive message:" << endl <<
" ndims = " <<
ndims
3010 << endl <<
" sizes = " <<
sizes << endl <<
" subsizes = "
3012 <<
" order = " <<
order << endl;
3026 for (
int axis = 0; axis <
numDims(); ++axis)
3050#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
3052 <<
", Lower send message:" << endl <<
" ndims = " <<
ndims
3053 << endl <<
" sizes = " <<
sizes << endl <<
" subsizes = "
3055 <<
" order = " <<
order << endl;
3069 for (
int axis = 0; axis <
numDims(); ++axis)
3085 proc = getUpperNeighbor(
msgAxis);
3087#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
3088 msg << endl <<
"P" <<
rank <<
": axis " <<
msgAxis <<
", upper neighbor = "
3105#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
3107 <<
", Upper receive message:" << endl <<
" ndims = " <<
ndims
3108 << endl <<
" sizes = " <<
sizes << endl <<
" subsizes = "
3110 <<
" order = " <<
order << endl;
3124 for (
int axis = 0; axis <
numDims(); ++axis)
3140 if (isReplicatedBoundary(
msgAxis) &&
3148#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
3150 <<
", Upper send message:" << endl <<
" ndims = " <<
ndims
3151 << endl <<
" sizes = " <<
sizes << endl <<
" subsizes = "
3153 <<
" order = " <<
order << endl;
3167 for (
int axis = 0; axis <
numDims(); ++axis)
3180#ifdef DOMI_MDVECTOR_MESSAGE_INITIALIZE
3181 for (
int proc = 0; proc < _teuchosComm->getSize(); ++proc)
3188 _teuchosComm->barrier();
3196template<
class Scalar >
3207 int pid = _teuchosComm->getRank();
3213 _teuchosComm->barrier();
3229 Teuchos::RCP< const Teuchos::MpiComm< int > >
mpiComm =
3230 Teuchos::rcp_dynamic_cast< const Teuchos::MpiComm< int > >(_teuchosComm);
3231 const Teuchos::OpaqueWrapper< MPI_Comm > &
communicator =
3285template<
class Scalar >
3305 Teuchos::RCP< const Teuchos::MpiComm< int > >
mpiComm =
3306 Teuchos::rcp_dynamic_cast< const Teuchos::MpiComm< int > >(_teuchosComm);
3307 const Teuchos::OpaqueWrapper< MPI_Comm > &
communicator =
3348 Teuchos::Array< Ordinal > index(3);
3349 for (
int axis = 0; axis <
ndims; ++axis)
3350 index[axis] =
fileInfo->dataStart[axis];
3368template<
class Scalar >
3369Teuchos::RCP< typename MDVector< Scalar >::FileInfo > &
3375 Teuchos::RCP< MDVector< Scalar >::FileInfo > &
fileInfo =
3383 fileInfo.reset(
new MDVector< Scalar >::FileInfo);
3391 for (
int axis = 0; axis <
ndims; ++axis)
3396 fileInfo->bufferShape[axis] = getLocalDim(axis,
true );
3397 fileInfo->dataShape[axis] = getLocalDim(axis,
false);
3399 fileInfo->dataStart[axis] = getLocalBounds(axis).start();
3403 int commIndex = _mdMap->getCommIndex(axis);
3406 int pad = getLowerBndryPad(axis);
3410 if (
commIndex == _mdMap->getCommDim(axis)-1)
3412 fileInfo->dataShape[axis] += getUpperBndryPad(axis);
3417#ifdef DOMI_MDVECTOR_DEBUG_IO
3418 cout <<
pid <<
": fileShape = " <<
fileInfo->fileShape() << endl;
3419 cout <<
pid <<
": bufferShape = " <<
fileInfo->bufferShape() << endl;
3420 cout <<
pid <<
": dataShape = " <<
fileInfo->dataShape() << endl;
3421 cout <<
pid <<
": fileStart = " <<
fileInfo->fileStart() << endl;
3422 cout <<
pid <<
": dataStart = " <<
fileInfo->dataStart() << endl;
Invalid argument exception type.
Definition Domi_Exceptions.hpp:54
Memory-safe templated multi-dimensional array class.
Definition Domi_MDArray.hpp:287
const Teuchos::Array< dim_type > & dimensions() const
Return an array of dimensions.
Definition Domi_MDArray.hpp:1016
const T * getRawPtr() const
Return a const raw pointer to the beginning of the MDArray or NULL if unsized.
Definition Domi_MDArray.hpp:1714
void clear()
Clear the MDArray
Definition Domi_MDArray.hpp:1628
MDArray()
Default constructor.
Definition Domi_MDArray.hpp:915
void resize(const Teuchos::ArrayView< dim_type > &dims)
Resize the MDArray based on the given dimensions.
Definition Domi_MDArray.hpp:1660
void assign(const T &value)
Assign a value to all elements of the MDArray
Definition Domi_MDArray.hpp:1569
size_type size() const
Return the total size of the MDArray
Definition Domi_MDArray.hpp:1037
int numDims() const
Return the number of dimensions.
Definition Domi_MDArray.hpp:1007
dim_type dimension(int axis) const
Return the dimension of the given axis.
Definition Domi_MDArray.hpp:1025
iterator end()
Return the ending iterator.
Definition Domi_MDArray.hpp:1082
iterator begin()
Return the beginning iterator.
Definition Domi_MDArray.hpp:1073
MDArrayView< T > mdArrayView()
Perform an explicit conversion to a non-const MDArrayView<T>
Definition Domi_MDArray.hpp:1168
int numDims() const
Return the number of dimensions.
Definition Domi_MDArrayRCP.hpp:999
const_pointer getRawPtr() const
Return a const raw pointer to the beginning of the MDArrayRCP or NULL if unsized.
Definition Domi_MDArrayRCP.hpp:1718
void clear()
Clear the MDArrayRCP
Definition Domi_MDArrayRCP.hpp:1652
void resize(const Teuchos::ArrayView< dim_type > &dims)
Resize the MDArrayRCP based on the given dimensions.
Definition Domi_MDArrayRCP.hpp:1684
dim_type dimension(int axis) const
Return the dimension of the given axis.
Definition Domi_MDArrayRCP.hpp:1017
const T * getRawPtr() const
Return a const raw pointer to the beginning of the MDArrayView or NULL if unsized.
Definition Domi_MDArrayView.hpp:1521
iterator end()
Return the ending iterator.
Definition Domi_MDArrayView.hpp:969
iterator begin()
Return the beginning iterator.
Definition Domi_MDArrayView.hpp:960
Iterator class suitable for multi-dimensional arrays.
Definition Domi_MDIterator.hpp:102
MDMap Error exception type.
Definition Domi_Exceptions.hpp:103
Multi-dimensional map.
Definition Domi_MDMap.hpp:146
MDMap Error exception type.
Definition Domi_Exceptions.hpp:115
Multi-dimensional distributed vector.
Definition Domi_MDVector.hpp:176
bool onSubcommunicator() const
Query whether this processor is on the sub-communicator.
Definition Domi_MDVector.hpp:1656
int getLowerPadSize(int axis) const
Get the size of the lower padding along the given axis.
Definition Domi_MDVector.hpp:1816
Scalar meanValue() const
Compute the mean (average) value of this MDVector.
Definition Domi_MDVector.hpp:2611
Slice getGlobalRankBounds(int axis, bool withBndryPad=false) const
Get the global loop bounds on this processor along the specified axis.
Definition Domi_MDVector.hpp:1756
void randomize()
Set all values in the multivector to pseudorandom numbers.
Definition Domi_MDVector.hpp:2752
int getLowerNeighbor(int axis) const
Get the rank of the lower neighbor.
Definition Domi_MDVector.hpp:1716
bool hasPadding() const
Return true if there is any padding stored locally.
Definition Domi_MDVector.hpp:1806
bool isContiguous() const
True if there are no stride gaps on any processor.
Definition Domi_MDVector.hpp:1920
Teuchos::ScalarTraits< Scalar >::magnitudeType normWeighted(const MDVector< Scalar > &weights) const
Compute the weighted norm of this.
Definition Domi_MDVector.hpp:2576
bool isPeriodic(int axis) const
Return the periodic flag for the given axis.
Definition Domi_MDVector.hpp:1696
void setUpperPad(int axis, const Scalar value)
Assign all elements of the upper pad a constant value.
Definition Domi_MDVector.hpp:1888
int getUpperBndryPad(int axis) const
Get the size of the upper boundary padding along the given axis.
Definition Domi_MDVector.hpp:1856
MDArrayView< const Scalar > getData(bool includePadding=true) const
Get a const view of the data as an MDArrayView.
Definition Domi_MDVector.hpp:2410
int getCommDim(int axis) const
Get the communicator size along the given axis.
Definition Domi_MDVector.hpp:1686
Teuchos::ArrayView< const Slice > getLocalBounds() const
Get the local loop bounds along every axis.
Definition Domi_MDVector.hpp:1776
MDArrayView< const Scalar > getLowerPadData(int axis) const
Get a const view of the lower padding data along the given axis as an MDArrayView.
Definition Domi_MDVector.hpp:2442
Teuchos::RCP< const Teuchos::Comm< int > > getTeuchosComm() const
Get the Teuchos communicator.
Definition Domi_MDVector.hpp:1666
void readBinary(const std::string &filename, bool includeBndryPad=false)
Read the MDVector from a binary file.
Definition Domi_MDVector.hpp:3288
void putScalar(const Scalar &value, bool includePadding=true)
Set all values in the multivector with the given value.
Definition Domi_MDVector.hpp:2737
dim_type getGlobalDim(int axis, bool withBndryPad=false) const
Get the global dimension along the specified axis.
Definition Domi_MDVector.hpp:1736
Scalar dot(const MDVector< Scalar > &a) const
Compute the dot product of this MDVector and MDVector a.
Definition Domi_MDVector.hpp:2491
MDArrayView< const Scalar > getUpperPadData(int axis) const
Get a const view of the upper padding data along the given axis as an MDArrayView.
Definition Domi_MDVector.hpp:2473
MDArrayView< Scalar > getLowerPadDataNonConst(int axis)
Get a non-const view of the lower padding data along the given axis as an MDArrayView.
Definition Domi_MDVector.hpp:2429
virtual ~MDVector()
Destructor.
Definition Domi_MDVector.hpp:1637
const Teuchos::RCP< const MDMap > getMDMap() const
MDMap accessor method.
Definition Domi_MDVector.hpp:1646
MDVector< Scalar > & operator=(const MDVector< Scalar > &source)
Assignment operator.
Definition Domi_MDVector.hpp:1619
int getCommIndex(int axis) const
Get the axis rank of this processor.
Definition Domi_MDVector.hpp:1706
MDVector(const Teuchos::RCP< const MDMap > &mdMap, bool zeroOut=true)
Main constructor.
Definition Domi_MDVector.hpp:1172
Teuchos::ScalarTraits< Scalar >::magnitudeType norm2() const
Compute the 2-norm of this MDVector.
Definition Domi_MDVector.hpp:2542
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Print the object with some verbosity level to a FancyOStream.
Definition Domi_MDVector.hpp:2662
MDVector< Scalar > operator[](dim_type index) const
Sub-vector access operator.
Definition Domi_MDVector.hpp:2913
void writeBinary(const std::string &filename, bool includeBndryPad=false) const
Write the MDVector to a binary file.
Definition Domi_MDVector.hpp:3199
int numDims() const
Get the number of dimensions.
Definition Domi_MDVector.hpp:1676
int getBndryPadSize(int axis) const
Get the boundary padding size along the given axis.
Definition Domi_MDVector.hpp:1866
bool isReplicatedBoundary(int axis) const
Return whether the given axis supports a replicated boundary.
Definition Domi_MDVector.hpp:1900
Slice getLocalInteriorBounds(int axis) const
Get the local interior looping bounds along the specified axis.
Definition Domi_MDVector.hpp:1796
void setLowerPad(int axis, const Scalar value)
Assign all elements of the lower pad a constant value.
Definition Domi_MDVector.hpp:1876
dim_type getLocalDim(int axis, bool withCommPad=false) const
Get the local dimension along the specified axis.
Definition Domi_MDVector.hpp:1766
virtual std::string description() const
A simple one-line description of this MDVector.
Definition Domi_MDVector.hpp:2638
Teuchos::ScalarTraits< Scalar >::magnitudeType norm1() const
Compute the 1-norm of this MDVector.
Definition Domi_MDVector.hpp:2520
int getUpperNeighbor(int axis) const
Get the rank of the upper neighbor.
Definition Domi_MDVector.hpp:1726
Teuchos::ScalarTraits< Scalar >::magnitudeType normInf() const
Compute the infinity-norm of this MDVector.
Definition Domi_MDVector.hpp:2554
MDArrayView< Scalar > getDataNonConst(bool includePadding=true)
Get a non-const view of the data as an MDArrayView.
Definition Domi_MDVector.hpp:2386
Layout getLayout() const
Get the storage order.
Definition Domi_MDVector.hpp:1910
int getCommPadSize(int axis) const
Get the communication padding size along the given axis.
Definition Domi_MDVector.hpp:1836
MDArrayView< Scalar > getUpperPadDataNonConst(int axis)
Get a non-const view of the upper padding data along the given axis as an MDArrayView.
Definition Domi_MDVector.hpp:2455
int getUpperPadSize(int axis) const
Get the size of the upper padding along the given axis.
Definition Domi_MDVector.hpp:1826
void updateCommPad()
The simplest method for updating the communication padding.
Definition Domi_MDVector.hpp:2766
void startUpdateCommPad(int axis)
Start an asyncronous update of the communication padding.
Definition Domi_MDVector.hpp:2790
void endUpdateCommPad(int axis)
Complete an asyncronous update of the communication padding.
Definition Domi_MDVector.hpp:2893
Slice getGlobalBounds(int axis, bool withBndryPadding=false) const
Get the bounds of the global problem.
Definition Domi_MDVector.hpp:1746
int getLowerBndryPad(int axis) const
Get the size of the lower boundary padding along the given axis.
Definition Domi_MDVector.hpp:1846
Type Error exception type.
Definition Domi_Exceptions.hpp:127
A Slice contains a start, stop, and step index, describing a subset of an ordered container.
Definition Domi_Slice.hpp:138
const dim_type start() const
Start index.
Definition Domi_Slice.hpp:431
const dim_type stop() const
Stop index.
Definition Domi_Slice.hpp:438