Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

StokesFO Kokkos Refactor and Boundary Layout Update #654

Merged
merged 25 commits into from
Feb 25, 2021
Merged

StokesFO Kokkos Refactor and Boundary Layout Update #654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
a88bd09
Updated StokesFO boundary layouts to new collapsed layouts
mcarlson801 Nov 9, 2020
8470ba9
Kokkos port of all velocity sideset evaluators except for BasalFricti…
mcarlson801 Nov 10, 2020
f5a3883
Kokkos refactor of GatherVerticallyContractedSolution, still needs so…
mcarlson801 Jan 4, 2021
2b770a5
Kokkos refactor of QuadPointsToCellInterpolation, not actually a side…
mcarlson801 Jan 4, 2021
c87161e
Added temporary boolean so that StokesFOBase changes don't break othe…
mcarlson801 Jan 4, 2021
1ef91af
Removed a leftover debug print statement
mcarlson801 Jan 5, 2021
97627e5
GatherVerticallyContracedSolution: Removed scratch comments, moved qu…
mcarlson801 Jan 5, 2021
fb1e819
GatherVerticallyContractedSolution: Had to move quad weight initializ…
mcarlson801 Jan 5, 2021
dfff8ed
Added a bugfix to pass tests, localDOFViews need to be initializaed f…
mcarlson801 Jan 5, 2021
f9aaaf0
Various fixes for issues arising during rebase
mcarlson801 Jan 6, 2021
dd8fd1f
Converting remaining stokesFO evaluators to new sideset layout. Fixed…
mcarlson801 Jan 19, 2021
79c1adc
Updating more evaluators using old sideset layout, all unit tests now…
mcarlson801 Jan 25, 2021
0cbd332
A little code cleanup
mcarlson801 Jan 26, 2021
35ff8e5
Merge remote-tracking branch 'origin/master' into ali-velocity-kokkos…
mcarlson801 Jan 26, 2021
018df42
Kokkos port for BasalFrictionCoefficient and BasalFrictionCoefficient…
mcarlson801 Jan 29, 2021
1bc7a63
Fixed typo, removed unnecessary side/cell variables, added scatter/ga…
mcarlson801 Feb 1, 2021
0dda5e8
Added numLayers to phalanx setup data, moved quadweights initializati…
mcarlson801 Feb 8, 2021
bb1bdc5
Moved quadweights initialization into function to reduce repeated cod…
mcarlson801 Feb 16, 2021
cd5f0a3
Reverted exception handling in getLocalDOFViews temporarily
mcarlson801 Feb 16, 2021
42ad9f9
Changed init_disc_params input type and reorganized localDOFView cons…
mcarlson801 Feb 16, 2021
85f4742
Getting number of layers in Albany::CreateDiscretization instead of i…
mcarlson801 Feb 17, 2021
2d5925c
Fixed a few minor things in response evaluators
mcarlson801 Feb 17, 2021
b97c6c0
Fixed GPU hang in STKDiscretization, fixes to FieldFrobeniusNorm, Sav…
mcarlson801 Feb 20, 2021
f3ccfb1
Merge branch 'master' into ali-velocity-kokkos-refactor
mperego Feb 21, 2021
62d16a1
Added atomic_adds to Basal and Lateral residual to avoid race conditi…
mcarlson801 Feb 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions src/Albany_Application.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,24 +160,25 @@ Application::initialSetUp(const RCP<Teuchos::ParameterList>& params)
} catch (...) {
tangent_deriv_dim = 1;
}
// Initialize Phalanx postRegistration setup
phxSetup = Teuchos::rcp(new PHAL::Setup());
phxSetup->init_problem_params(problemParams);

// Pull the number of solution vectors out of the problem and send them to the
// discretization list, if the user specifies this in the problem
Teuchos::ParameterList& discParams = params->sublist("Discretization");
Teuchos::RCP<Teuchos::ParameterList> discParams = Teuchos::sublist(params, "Discretization", true);

// Initialize Phalanx postRegistration setup
phxSetup = Teuchos::rcp(new PHAL::Setup());
phxSetup->init_problem_params(problemParams);

// Set in Albany_AbstractProblem constructor or in siblings
num_time_deriv = problemParams->get<int>("Number Of Time Derivatives");

// Possibly set in the Discretization list in the input file - this overrides
// the above if set
int num_time_deriv_from_input =
discParams.get<int>("Number Of Time Derivatives", -1);
discParams->get<int>("Number Of Time Derivatives", -1);
if (num_time_deriv_from_input <
0) // Use the value from the problem by default
discParams.set<int>("Number Of Time Derivatives", num_time_deriv);
discParams->set<int>("Number Of Time Derivatives", num_time_deriv);
else
num_time_deriv = num_time_deriv_from_input;

Expand Down Expand Up @@ -545,6 +546,12 @@ Application::createDiscretization()
problem->getNullSpace());
// The following is for Aeras problems.
explicit_scheme = disc->isExplicitScheme();
// For extruded meshes, we need the number of layers in postRegistrationSetup
Teuchos::RCP<LayeredMeshNumbering<GO>> layeredMeshNumbering = disc->getLayeredMeshNumbering();
if (!layeredMeshNumbering.is_null()) {
int numLayers = layeredMeshNumbering->numLayers;
phxSetup->set_num_layers(numLayers);
}
}

void
Expand Down Expand Up @@ -3092,6 +3099,7 @@ Application::loadWorksetSidesetInfo(PHAL::Workset& workset, const int ws)
{
workset.sideSets = Teuchos::rcpFromRef(disc->getSideSets(ws));
workset.sideSetViews = Teuchos::rcpFromRef(disc->getSideSetViews(ws));
workset.localDOFViews = Teuchos::rcpFromRef(disc->getLocalDOFViews(ws));
}

void
Expand Down
11 changes: 3 additions & 8 deletions src/Albany_StateManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,14 +576,9 @@ Albany::StateManager::registerSideSetStateVariable_collapsed(
else if (dl->rank() == 4) // node tensor
nodalDataBase->registerVectorState(
stateName, stateRef.dim[1] * stateRef.dim[2]);
} else {

TEUCHOS_TEST_FOR_EXCEPTION(
true,
std::logic_error,
"Error! The given layout does not appear to be that of a collapsed side set "
"field.\n");

}
else {
mcarlson801 marked this conversation as resolved.
Show resolved Hide resolved
dl->dimensions(stateRef.dim);
}
stateRef.output = outputToExodus;
stateRef.responseIDtoRequire = responseIDtoRequire;
Expand Down
132 changes: 132 additions & 0 deletions src/LandIce/evaluators/LandIce_BasalFrictionCoefficient.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,15 @@ class BasalFrictionCoefficient : public PHX::EvaluatorWithBaseImpl<Traits>,

double rho_i, rho_w; // [kg m^{-3}]

ParamScalarT mu;
ParamScalarT lambda;
ParamScalarT power;

unsigned int numNodes;
unsigned int numQPs;
unsigned int dim;

bool useCollapsedSidesets;

bool logParameters;
bool distributedLambda;
Expand All @@ -106,6 +113,131 @@ class BasalFrictionCoefficient : public PHX::EvaluatorWithBaseImpl<Traits>,
BETA_TYPE beta_type;

PHAL::MDFieldMemoizer<Traits> memoizer;

Albany::LocalSideSetInfo sideSet;

public:

typedef Kokkos::View<int***, PHX::Device>::execution_space ExecutionSpace;

struct GivenFieldParam_Tag{};
struct GivenField_Tag{};
struct PowerLaw_DistributedMu_Tag{};
struct PowerLaw_Tag{};
struct RegularizedCoulomb_DistributedLambda_DistributedMu_Tag{};
struct RegularizedCoulomb_DistributedLambda_Tag{};
struct RegularizedCoulomb_DistributedMu_Tag{};
struct RegularizedCoulomb_Tag{};
struct ExpGivenFieldParam_Nodal_Tag{};
struct ExpGivenFieldParam_Tag{};
struct ExpGivenField_Nodal_Tag{};
struct ExpGivenField_Tag{};
struct StereographicMapCorrection_Tag{};

struct Side_GivenFieldParam_Tag{};
struct Side_GivenField_Tag{};
struct Side_PowerLaw_DistributedMu_Tag{};
struct Side_PowerLaw_Tag{};
struct Side_RegularizedCoulomb_DistributedLambda_DistributedMu_Tag{};
struct Side_RegularizedCoulomb_DistributedLambda_Tag{};
struct Side_RegularizedCoulomb_DistributedMu_Tag{};
struct Side_RegularizedCoulomb_Tag{};
struct Side_ExpGivenFieldParam_Nodal_Tag{};
struct Side_ExpGivenFieldParam_Tag{};
struct Side_ExpGivenField_Nodal_Tag{};
struct Side_ExpGivenField_Tag{};
struct Side_ZeroOnFloatingParam_Tag{};
struct Side_ZeroOnFloating_Tag{};
struct Side_StereographicMapCorrection_Tag{};

typedef Kokkos::RangePolicy<ExecutionSpace,GivenFieldParam_Tag> GivenFieldParam_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,GivenField_Tag> GivenField_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,PowerLaw_DistributedMu_Tag> PowerLaw_DistributedMu_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,PowerLaw_Tag> PowerLaw_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,RegularizedCoulomb_DistributedLambda_DistributedMu_Tag> RegularizedCoulomb_DistributedLambda_DistributedMu_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,RegularizedCoulomb_DistributedLambda_Tag> RegularizedCoulomb_DistributedLambda_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,RegularizedCoulomb_DistributedMu_Tag> RegularizedCoulomb_DistributedMu_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,RegularizedCoulomb_Tag> RegularizedCoulomb_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,ExpGivenFieldParam_Nodal_Tag> ExpGivenFieldParam_Nodal_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,ExpGivenFieldParam_Tag> ExpGivenFieldParam_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,ExpGivenField_Nodal_Tag> ExpGivenField_Nodal_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,ExpGivenField_Tag> ExpGivenField_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,StereographicMapCorrection_Tag> StereographicMapCorrection_Policy;

typedef Kokkos::RangePolicy<ExecutionSpace,Side_GivenFieldParam_Tag> Side_GivenFieldParam_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_GivenField_Tag> Side_GivenField_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_PowerLaw_DistributedMu_Tag> Side_PowerLaw_DistributedMu_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_PowerLaw_Tag> Side_PowerLaw_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_RegularizedCoulomb_DistributedLambda_DistributedMu_Tag> Side_RegularizedCoulomb_DistributedLambda_DistributedMu_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_RegularizedCoulomb_DistributedLambda_Tag> Side_RegularizedCoulomb_DistributedLambda_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_RegularizedCoulomb_DistributedMu_Tag> Side_RegularizedCoulomb_DistributedMu_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_RegularizedCoulomb_Tag> Side_RegularizedCoulomb_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_ExpGivenFieldParam_Nodal_Tag> Side_ExpGivenFieldParam_Nodal_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_ExpGivenFieldParam_Tag> Side_ExpGivenFieldParam_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_ExpGivenField_Nodal_Tag> Side_ExpGivenField_Nodal_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_ExpGivenField_Tag> Side_ExpGivenField_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_ZeroOnFloatingParam_Tag> Side_ZeroOnFloatingParam_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_ZeroOnFloating_Tag> Side_ZeroOnFloating_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,Side_StereographicMapCorrection_Tag> Side_StereographicMapCorrection_Policy;

KOKKOS_INLINE_FUNCTION
void operator() (const GivenFieldParam_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const GivenField_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const PowerLaw_DistributedMu_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const PowerLaw_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const RegularizedCoulomb_DistributedLambda_DistributedMu_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const RegularizedCoulomb_DistributedLambda_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const RegularizedCoulomb_DistributedMu_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const RegularizedCoulomb_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const ExpGivenFieldParam_Nodal_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const ExpGivenFieldParam_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const ExpGivenField_Nodal_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const ExpGivenField_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const StereographicMapCorrection_Tag& tag, const int& i) const;

KOKKOS_INLINE_FUNCTION
void operator() (const Side_GivenFieldParam_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_GivenField_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_PowerLaw_DistributedMu_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_PowerLaw_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_RegularizedCoulomb_DistributedLambda_DistributedMu_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_RegularizedCoulomb_DistributedLambda_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_RegularizedCoulomb_DistributedMu_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_RegularizedCoulomb_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_ExpGivenFieldParam_Nodal_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_ExpGivenFieldParam_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_ExpGivenField_Nodal_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_ExpGivenField_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_ZeroOnFloatingParam_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_ZeroOnFloating_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const Side_StereographicMapCorrection_Tag& tag, const int& i) const;

};

} // Namespace LandIce
Expand Down
51 changes: 41 additions & 10 deletions src/LandIce/evaluators/LandIce_BasalFrictionCoefficientGradient.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,24 +53,27 @@ class BasalFrictionCoefficientGradient : public PHX::EvaluatorWithBaseImpl<Trait
typedef typename EvalT::ParamScalarT ParamScalarT;

// Input:
PHX::MDField<const RealType,Cell,Side,Node> given_field;
PHX::MDField<const ParamScalarT,Cell,Side,Node> given_field_param;
PHX::MDField<const MeshScalarT,Cell,Side,Node,QuadPoint,Dim> GradBF;
PHX::MDField<const ParamScalarT,Cell,Side,QuadPoint> N;
PHX::MDField<const ScalarT,Cell,Side,QuadPoint,Dim> U;
PHX::MDField<const ScalarT,Cell,Side,QuadPoint,Dim> gradN;
PHX::MDField<const ScalarT,Cell,Side,QuadPoint,Dim,Dim> gradU;
PHX::MDField<const ScalarT,Cell,Side,QuadPoint> u_norm;
PHX::MDField<const MeshScalarT,Cell,Side,QuadPoint,Dim> coordVec;
// TODO: restore layout template arguments when removing old sideset layout
PHX::MDField<const RealType> given_field; // Side, Node
PHX::MDField<const ParamScalarT> given_field_param; // Side, Node
PHX::MDField<const MeshScalarT> GradBF; // Side, Node, QuadPoint, Dim
PHX::MDField<const ParamScalarT> N; // Side, QuadPoint
PHX::MDField<const ScalarT> U; // Side, QuadPoint, Dim
PHX::MDField<const ScalarT> gradN; // Side, QuadPoint, Dim
PHX::MDField<const ScalarT> gradU; // Side, QuadPoint, Dim, Dim
PHX::MDField<const ScalarT> u_norm; // Side, QuadPoint
PHX::MDField<const MeshScalarT> coordVec; // Side, QuadPoint, Dim

PHX::MDField<const ScalarT,Dim> lambdaParam;
PHX::MDField<const ScalarT,Dim> muParam;
PHX::MDField<const ScalarT,Dim> powerParam;

// Output:
PHX::MDField<ScalarT,Cell,Side,QuadPoint,Dim> grad_beta;
PHX::MDField<ScalarT> grad_beta; // Side, QuadPoint, Dim

std::string basalSideName;
bool useCollapsedSidesets;
Albany::LocalSideSetInfo sideSet;

unsigned int numSideNodes;
unsigned int numSideQPs;
Expand All @@ -82,13 +85,41 @@ class BasalFrictionCoefficientGradient : public PHX::EvaluatorWithBaseImpl<Trait
double y_0;
double R2;

ScalarT lambda;
ScalarT mu;
ScalarT power;

bool use_stereographic_map;
bool is_given_field_param;

enum BETA_TYPE {INVALID, GIVEN_CONSTANT, GIVEN_FIELD, REGULARIZED_COULOMB};
BETA_TYPE beta_type;

PHAL::MDFieldMemoizer<Traits> memoizer;

public:

typedef Kokkos::View<int***, PHX::Device>::execution_space ExecutionSpace;

struct GivenFieldParam_Tag{};
struct GivenField_Tag{};
struct RegularizedCoulomb_Tag{};
struct StereographicMapCorrection_Tag{};

typedef Kokkos::RangePolicy<ExecutionSpace,GivenFieldParam_Tag> GivenFieldParam_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,GivenField_Tag> GivenField_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,RegularizedCoulomb_Tag> RegularizedCoulomb_Policy;
typedef Kokkos::RangePolicy<ExecutionSpace,StereographicMapCorrection_Tag> StereographicMapCorrection_Policy;

KOKKOS_INLINE_FUNCTION
void operator() (const GivenFieldParam_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const GivenField_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const RegularizedCoulomb_Tag& tag, const int& i) const;
KOKKOS_INLINE_FUNCTION
void operator() (const StereographicMapCorrection_Tag& tag, const int& i) const;

};

} // Namespace LandIce
Expand Down
Loading