mod_mpi_flow Module

MPI parallelization and domain decomposition for the flow solver.

This module manages the parallel execution of the hydrodynamic solver using a Replicated Mesh strategy. In this approach, every MPI rank holds the full topological structure of the mesh and globally sized field arrays, but performs physical updates (fluxes, source terms) only on owned cells or faces whose owner cell is owned locally.

Halo exchanges synchronize off-rank neighbor cells and shared face fluxes for stencil operations. Global reductions compute diagnostics and PCG dot products; gather routines are used for root-only output/diagnostics where needed.


Uses


Variables

Type Visibility Attributes Name Initial
logical, private :: mpi_started_here = .false.

Derived Types

type, public ::  flow_mpi_t

MPI context for hydrodynamic operations.

Read more…

Components

Type Visibility Attributes Name Initial
integer, public :: cell_halo_max_components = 4
integer, public, allocatable :: cell_owner(:)
integer, public, allocatable :: cell_recv_cells(:)
integer, public, allocatable :: cell_recv_counts(:)
integer, public, allocatable :: cell_recv_displs(:)
integer, public, allocatable :: cell_recv_ranks(:)
real(kind=rk), public, allocatable :: cell_recvbuf(:)
type(MPI_Request), public, allocatable :: cell_requests(:)
integer, public, allocatable :: cell_send_cells(:)
integer, public, allocatable :: cell_send_counts(:)
integer, public, allocatable :: cell_send_displs(:)
integer, public, allocatable :: cell_send_ranks(:)
real(kind=rk), public, allocatable :: cell_sendbuf(:)
type(MPI_Comm), public :: comm = MPI_COMM_NULL
integer, public, allocatable :: face_recv_counts(:)
integer, public, allocatable :: face_recv_displs(:)
integer, public, allocatable :: face_recv_faces(:)
integer, public, allocatable :: face_recv_ranks(:)
real(kind=rk), public, allocatable :: face_recvbuf(:)
type(MPI_Request), public, allocatable :: face_requests(:)
integer, public, allocatable :: face_send_counts(:)
integer, public, allocatable :: face_send_displs(:)
integer, public, allocatable :: face_send_faces(:)
integer, public, allocatable :: face_send_ranks(:)
real(kind=rk), public, allocatable :: face_sendbuf(:)
integer, public :: first_cell = 0
integer, public, allocatable :: gather_counts(:)
integer, public, allocatable :: gather_displs(:)
integer, public, allocatable :: gather_firsts(:)
integer, public, allocatable :: gather_matrix_counts(:)
integer, public, allocatable :: gather_matrix_displs(:)
real(kind=rk), public, allocatable :: gather_matrix_recvbuf(:)
real(kind=rk), public, allocatable :: gather_matrix_sendbuf(:)
integer, public :: gather_max_components = 0
real(kind=rk), public, allocatable :: gather_recvbuf(:)
real(kind=rk), public, allocatable :: gather_sendbuf(:)
integer, public, allocatable :: ghost_cells(:)
integer, public :: last_cell = -1
integer, public :: ncell_recv_ranks = 0
integer, public :: ncell_send_ranks = 0
integer, public :: nface_recv_ranks = 0
integer, public :: nface_send_ranks = 0
integer, public :: nlocal = 0
integer, public :: nprocs = 0
logical, public, allocatable :: owned(:)
integer, public, allocatable :: owned_faces(:)
integer, public :: rank = -1

Functions

public function flow_global_dot_owned(flow, a, b) result(dot)

Computes the global dot product of two vectors over owned cells.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a(:)
real(kind=rk), intent(in) :: b(:)

Return Value real(kind=rk)

public function flow_global_max_owned(flow, a) result(global_max)

Computes the global maximum magnitude of a field over owned cells.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a(:)

Return Value real(kind=rk)

public function flow_global_sum_owned(flow, a) result(total)

Computes the global sum of a field over owned cells.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a(:)

Return Value real(kind=rk)

private function mesh_neighbor_for_cell(mesh, face_id, cell_id) result(nb)

Returns the mesh neighbor, using stored periodic links when present.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
integer, intent(in) :: face_id
integer, intent(in) :: cell_id

Return Value integer


Subroutines

public subroutine flow_allgather_owned_matrix(flow, local_global, global)

Gathers locally-updated matrix cell values and broadcasts to the global mesh.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_global(:,:)
real(kind=rk), intent(out) :: global(:,:)

public subroutine flow_allgather_owned_matrix_inplace(flow, field)

In-place variant for fields that already hold owned-cell updates.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: field(:,:)

public subroutine flow_allgather_owned_scalar(flow, local_global, global)

Gathers locally-updated cell values and broadcasts to the global mesh.

Read more…

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_global(:)
real(kind=rk), intent(out) :: global(:)

public subroutine flow_allgather_owned_v4(flow, local_v, local_s, global_v, global_s)

Gathers 4-component cell values (e.g., Velocity + Scalar) in one call.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_v(:,:)
real(kind=rk), intent(in) :: local_s(:)
real(kind=rk), intent(out) :: global_v(:,:)
real(kind=rk), intent(out) :: global_s(:)

public subroutine flow_allgather_owned_vector(flow, local_global, global)

Gathers locally-updated 3D vector cell values and broadcasts to the global mesh.

Read more…

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_global(:,:)
real(kind=rk), intent(out) :: global(:,:)

public subroutine flow_allreduce_global_scalar(flow, local_values, global_values)

Sum-Allreduce for a global scalar field.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: local_values(:)
real(kind=rk), intent(out) :: global_values(:)

public subroutine flow_allreduce_global_vector(flow, local_values, global_values)

Sum-Allreduce for a 3D global vector field.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: local_values(:,:)
real(kind=rk), intent(out) :: global_values(:,:)

public subroutine flow_exchange_cell_matrix(flow, field)

Exchanges owned cell matrix values to ranks that keep them as ghosts.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: field(:,:)

public subroutine flow_exchange_cell_scalar(flow, field)

Exchanges owned cell scalar values to ranks that keep them as ghosts.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: field(:)

public subroutine flow_exchange_cell_scalars(flow, field1, field2, field3, field4)

Exchanges two to four related cell scalars in one halo transaction.

Read more…

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: field1(:)
real(kind=rk), intent(inout), optional :: field2(:)
real(kind=rk), intent(inout), optional :: field3(:)
real(kind=rk), intent(inout), optional :: field4(:)

public subroutine flow_exchange_face_scalar(flow, face_field)

Exchanges owner-computed face scalar values to ranks owning the neighbor cell.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: face_field(:)

public subroutine flow_gather_owned_matrix_root(flow, field, root_field)

Gathers owned matrix cell values to rank 0 only.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: field(:,:)
real(kind=rk), intent(inout) :: root_field(:,:)

public subroutine flow_gather_owned_scalar_root(flow, field, root_field)

Gathers owned scalar cell values to rank 0 only.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: field(:)
real(kind=rk), intent(inout) :: root_field(:)

public subroutine flow_global_dots_owned(flow, n_dots, a, b, results)

Computes multiple global dot products in a single MPI_Allreduce.

Read more…

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
integer, intent(in) :: n_dots
real(kind=rk), intent(in) :: a(:,:)
real(kind=rk), intent(in) :: b(:,:)
real(kind=rk), intent(out) :: results(:)

public subroutine flow_global_two_dots_owned(flow, a1, b1, a2, b2, results)

Computes two global dot products without constructing temporary full-size batches.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a1(:)
real(kind=rk), intent(in) :: b1(:)
real(kind=rk), intent(in) :: a2(:)
real(kind=rk), intent(in) :: b2(:)
real(kind=rk), intent(out) :: results(2)

public subroutine flow_mpi_finalize(flow)

Releases all MPI resources and buffers.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow

public subroutine flow_mpi_initialize(mesh, flow, comm_parent, max_gather_components)

Sets up domain decomposition for a given mesh.

Read more…

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh

The mesh to decompose.
type(flow_mpi_t), intent(inout) :: flow

The MPI context to populate.
type(MPI_Comm), intent(in) :: comm_parent

The parent communicator (usually MPI_COMM_WORLD).
integer, intent(in), optional :: max_gather_components

public subroutine mpi_flow_shutdown()

Shuts down the MPI environment if it was started by this module.

Arguments

None

public subroutine mpi_flow_startup()

Initializes the MPI environment if not already active.

Arguments

None

private subroutine check_mpi(ierr, where)

Internal helper for MPI error checking.

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: ierr
character(len=*), intent(in) :: where

private subroutine pack_rank_metadata(counts_all, displs_all, nactive, ranks, counts, displs)

Packs full per-rank metadata down to active communication partners.

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: counts_all(:)
integer, intent(in) :: displs_all(:)
integer, intent(out) :: nactive
integer, intent(out), allocatable :: ranks(:)
integer, intent(out), allocatable :: counts(:)
integer, intent(out), allocatable :: displs(:)

private subroutine prefix_counts(counts, displs)

Converts per-rank counts to zero-based displacements.

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: counts(:)
integer, intent(out) :: displs(:)

private subroutine prepare_matrix_gather(flow, ncomp, ncells, nlocal_comp)

Updates cached count/displacement arrays for a packed component count.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
integer, intent(in) :: ncomp
integer, intent(in) :: ncells
integer, intent(out) :: nlocal_comp

private subroutine setup_cell_halo(mesh, flow)

Builds cell ghost send/receive metadata for one-ring neighbor stencils.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_cell_owners(mesh, flow)

Initializes contiguous-decomposition owner lookup.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_face_halo(mesh, flow)

Builds face-flux halo metadata for shared internal faces.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_owned_faces(mesh, flow)

Caches faces whose owner cell belongs to this rank.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_owned_gather(mesh, flow, max_gather_components)

Pre-calculates MPI gather offsets and counts for allgather operations.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow
integer, intent(in), optional :: max_gather_components