Skip to content

Add vertical advection #328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
brian-oneill wants to merge 18 commits into
base: develop
Choose a base branch
from
Draft

Add vertical advection #328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
5af4160
VertAdv module definition; creation, destruction, retrieval methods
brian-oneill Dec 27, 2025
f09b3ae
Add optional per-team scratch space to parallelForOuter
brian-oneill Dec 27, 2025
ff5fa57
Add computeVerticalVelocity
brian-oneill Dec 27, 2025
d9deb07
Add computeThicknessVAdvTend
brian-oneill Dec 29, 2025
93876a3
Add computeVelocityVAdvTend
brian-oneill Dec 29, 2025
43fa0c8
Add computeTracerVAdvTend dispatch function
brian-oneill Dec 29, 2025
56ca686
Add computeVerticalFluxes
brian-oneill Jan 1, 2026
42609bf
Add computeStdVAdvTend
brian-oneill Jan 1, 2026
bd522d7
Add computeFCTVAdvTend
brian-oneill Jan 2, 2026
421ab50
Add ProvThickness to auxvars (follow-up restructuring needed)
brian-oneill Jan 6, 2026
725210c
Add new config variables to default config
brian-oneill Jan 8, 2026
980fef8
Add VertAdv unit tests
brian-oneill Jan 8, 2026
9e9d606
Init some previously missed pointers and add Field for ProvThickness
brian-oneill Jan 9, 2026
d859c84
Comments, bug fix, and add Fields for VertAdv arrays
brian-oneill Jan 9, 2026
f811db1
Some bug fixes for running on GPU
brian-oneill Jan 9, 2026
5b40aa4
Bug fixes: BottomDepth no longer in HorzMesh and typo fixes
brian-oneill Jan 10, 2026
9bdf05d
Make host copies of VertAdv arrays
brian-oneill Jan 11, 2026
7f8f0ef
Linting fix
brian-oneill Jan 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions components/omega/configs/Default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@ Omega:
State:
NTimeLevels: 2
Advection:
Coef3rdOrder: 0.25
FluxThicknessType: Center
FluxTracerType: Center
VerticalTracerFluxLimiterEnabled: true
Copy link

@cbegeman cbegeman Jan 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
VerticalTracerFluxLimiterEnabled: true
VerticalTracerFluxLimiterEnable: true

I think this is more consistent with other config options

VerticalTracerFluxOrder: 3
WindStress:
InterpType: Isotropic
VertCoord:
Expand All @@ -50,6 +53,9 @@ Omega:
EddyDiff4: 0.0
UseCustomTendency: false
ManufacturedSolutionTendency: false
ThicknessVertAdvTendencyEnable: true
VelocityVertAdvTendencyEnable: true
TracerVertAdvTendencyEnable: true
Tracers:
Base: [Temperature, Salinity]
Debug: [Debug1, Debug2, Debug3]
Expand Down
17 changes: 14 additions & 3 deletions components/omega/src/infra/OmegaKokkos.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ using ScratchMemSpace = ExecSpace::scratch_memory_space;
using Kokkos::MemoryUnmanaged;
using Kokkos::PerTeam;
using Kokkos::TeamThreadRange;
using RealScratchArray =
Kokkos::View<Real *, ScratchMemSpace, Kokkos::MemoryUnmanaged>;

/// team_size for hierarchical parallelism
#ifdef OMEGA_TARGET_DEVICE
Expand Down Expand Up @@ -321,7 +323,8 @@ KOKKOS_INLINE_FUNCTION void teamBarrier(const TeamMember &Team) {
// parallelForOuter: with label
template <int N, class F>
inline void parallelForOuter(const std::string &Label,
const int (&UpperBounds)[N], F &&Functor) {
const int (&UpperBounds)[N], F &&Functor,
int ScratchValsPerTeam = 0) {

auto LinFunctor = LinearIdxWrapper{std::forward<F>(Functor), UpperBounds};
int LinBound = 1;
Expand All @@ -330,6 +333,12 @@ inline void parallelForOuter(const std::string &Label,
}

auto Policy = TeamPolicy(LinBound, OMEGA_TEAMSIZE);

if (ScratchValsPerTeam > 0) {
Policy.set_scratch_size(
0, Kokkos::PerTeam(ScratchValsPerTeam * sizeof(Real)));
}

Kokkos::parallel_for(
Label, Policy, KOKKOS_LAMBDA(const TeamMember &Team) {
const int TeamId = Team.league_rank();
Expand All @@ -339,8 +348,10 @@ inline void parallelForOuter(const std::string &Label,

// parallelForOuter: without label
template <int N, class F>
inline void parallelForOuter(const int (&UpperBounds)[N], F &&Functor) {
parallelForOuter("", UpperBounds, std::forward<F>(Functor));
inline void parallelForOuter(const int (&UpperBounds)[N], F &&Functor,
int ScratchValsPerTeam = 0) {
parallelForOuter("", UpperBounds, std::forward<F>(Functor),
ScratchValsPerTeam);
}

// parallelForInner
Expand Down
5 changes: 3 additions & 2 deletions components/omega/src/ocn/AuxiliaryState.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,9 @@ void AuxiliaryState::computeMomAux(const OceanState *State, int ThickTimeLevel,

parallelForInner(
Team, KRange, INNER_LAMBDA(int KChunk) {
LocLayerThicknessAux.computeVarsOnCells(ICell, KChunk,
LayerThickCell);
LocLayerThicknessAux.computeVarsOnCells(
ICell, KChunk, LayerThickCell, NormalVelEdge, 0._Real);
// TODO: make timestep available to this call
});
});
Pacer::stop("AuxState:cellAuxState3", 2);
Expand Down
5 changes: 3 additions & 2 deletions components/omega/src/ocn/CustomTendencyTerms.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "Config.h"
#include "GlobalConstants.h"
#include "TimeStepper.h"
#include "VertCoord.h"

namespace OMEGA {

Expand Down Expand Up @@ -76,8 +77,8 @@ void ManufacturedSolution::init() {
/// This test case assumes that the restingThickness is horizontally uniform
/// and that only one vertical level is used so only one set of indices is
/// used here.
HorzMesh *DefHorzMesh = HorzMesh::getDefault();
R8 H0 = DefHorzMesh->BottomDepthH(0);
VertCoord *DefVCoord = VertCoord::getDefault();
R8 H0 = DefVCoord->BottomDepthH(0);

// Define and compute common constants
R8 Kx = TwoPi / WavelengthX; // Wave in X-dir
Expand Down
10 changes: 0 additions & 10 deletions components/omega/src/ocn/HorzMesh.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,6 @@ HorzMesh::HorzMesh(const std::string &Name, //< [in] Name for new mesh
// Read x/y/z and lon/lat coordinates for cells, edges, and vertices
readCoordinates();

// Read the cell-centered bottom depth
readBottomDepth();

// Read the mesh areas, lengths, and angles
readMeasurements();

Expand Down Expand Up @@ -439,12 +436,6 @@ void HorzMesh::readCoordinates() {

} // end readCoordinates

//------------------------------------------------------------------------------
// Read the cell-centered bottom depth
void HorzMesh::readBottomDepth() {
readCellArray(BottomDepthH, "bottomDepth");
} // end readDepth

//------------------------------------------------------------------------------
// Read the mesh areas (cell, triangle, and kite),
// lengths (between centers and vertices), and edge angles
Expand Down Expand Up @@ -600,7 +591,6 @@ void HorzMesh::copyToDevice() {
AngleEdge = createDeviceMirrorCopy(AngleEdgeH);
WeightsOnEdge = createDeviceMirrorCopy(WeightsOnEdgeH);
FVertex = createDeviceMirrorCopy(FVertexH);
BottomDepth = createDeviceMirrorCopy(BottomDepthH);
FEdge = createDeviceMirrorCopy(FEdgeH);
XCell = createDeviceMirrorCopy(XCellH);
YCell = createDeviceMirrorCopy(YCellH);
Expand Down
9 changes: 0 additions & 9 deletions components/omega/src/ocn/HorzMesh.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,12 @@ class HorzMesh {

void readCoordinates();

void readBottomDepth();

void readMeasurements();

void readWeights();

void readCoriolis();

// void computeEdgeSign();

void copyToDevice();

// int computeMesh();
Expand Down Expand Up @@ -232,11 +228,6 @@ class HorzMesh {
Array1DReal FVertex; ///< Coriolis parameter at vertices (radians s^-1)
HostArray1DReal FVertexH; ///< Coriolis parameter at vertices (radians s^-1)

// Depth

Array1DReal BottomDepth; ///< Depth of the bottom of the ocean (m)
HostArray1DReal BottomDepthH; ///< Depth of the bottom of the ocean (m)

// Edge sign

Array2DReal EdgeSignOnCell; ///< Sign of vector connecting cells
Expand Down
Loading
Loading