mod_mpi_flow Module

MPI parallelization and domain decomposition for the flow solver.

This module manages the parallel execution of the hydrodynamic solver using a Replicated Mesh strategy. In this approach, every MPI rank holds the full topological structure of the mesh, but performs physical updates (fluxes, source terms) only on a specific subset of "owned" cells.

Collective communication (Allreduce, Allgather) is then used to synchronize the full field across all processors. This simplifies the implementation of complex unstructured operators while allowing multi-CPU acceleration.


Uses

  • module~~mod_mpi_flow~~UsesGraph module~mod_mpi_flow mod_mpi_flow module~mod_kinds mod_kinds module~mod_mpi_flow->module~mod_kinds module~mod_mesh_types mod_mesh_types module~mod_mpi_flow->module~mod_mesh_types mpi_f08 mpi_f08 module~mod_mpi_flow->mpi_f08 iso_fortran_env iso_fortran_env module~mod_kinds->iso_fortran_env module~mod_mesh_types->module~mod_kinds

Used by

  • module~~mod_mpi_flow~~UsedByGraph module~mod_mpi_flow mod_mpi_flow module~mod_energy mod_energy module~mod_energy->module~mod_mpi_flow module~mod_flow_projection mod_flow_projection module~mod_flow_projection->module~mod_mpi_flow module~mod_transport_properties mod_transport_properties module~mod_flow_projection->module~mod_transport_properties module~mod_output mod_output module~mod_output->module~mod_mpi_flow module~mod_output->module~mod_energy module~mod_output->module~mod_flow_projection module~mod_species mod_species module~mod_output->module~mod_species module~mod_output->module~mod_transport_properties module~mod_species->module~mod_mpi_flow module~mod_species->module~mod_flow_projection module~mod_species->module~mod_transport_properties module~mod_transport_properties->module~mod_mpi_flow program~lowmach_react_hex lowmach_react_hex program~lowmach_react_hex->module~mod_mpi_flow program~lowmach_react_hex->module~mod_energy program~lowmach_react_hex->module~mod_flow_projection program~lowmach_react_hex->module~mod_output program~lowmach_react_hex->module~mod_species program~lowmach_react_hex->module~mod_transport_properties

Variables

Type Visibility Attributes Name Initial
logical, private :: mpi_started_here = .false.

Derived Types

type, public ::  flow_mpi_t

MPI context for hydrodynamic operations.

Read more…

Components

Type Visibility Attributes Name Initial
integer, public :: cell_halo_max_components = 4
integer, public, allocatable :: cell_owner(:)
integer, public, allocatable :: cell_recv_cells(:)
integer, public, allocatable :: cell_recv_counts(:)
integer, public, allocatable :: cell_recv_displs(:)
integer, public, allocatable :: cell_recv_ranks(:)
real(kind=rk), public, allocatable :: cell_recvbuf(:)
type(MPI_Request), public, allocatable :: cell_requests(:)
integer, public, allocatable :: cell_send_cells(:)
integer, public, allocatable :: cell_send_counts(:)
integer, public, allocatable :: cell_send_displs(:)
integer, public, allocatable :: cell_send_ranks(:)
real(kind=rk), public, allocatable :: cell_sendbuf(:)
type(MPI_Comm), public :: comm = MPI_COMM_NULL
integer, public, allocatable :: face_recv_counts(:)
integer, public, allocatable :: face_recv_displs(:)
integer, public, allocatable :: face_recv_faces(:)
integer, public, allocatable :: face_recv_ranks(:)
real(kind=rk), public, allocatable :: face_recvbuf(:)
type(MPI_Request), public, allocatable :: face_requests(:)
integer, public, allocatable :: face_send_counts(:)
integer, public, allocatable :: face_send_displs(:)
integer, public, allocatable :: face_send_faces(:)
integer, public, allocatable :: face_send_ranks(:)
real(kind=rk), public, allocatable :: face_sendbuf(:)
integer, public :: first_cell = 0
integer, public, allocatable :: gather_counts(:)
integer, public, allocatable :: gather_displs(:)
integer, public, allocatable :: gather_firsts(:)
integer, public, allocatable :: gather_matrix_counts(:)
integer, public, allocatable :: gather_matrix_displs(:)
real(kind=rk), public, allocatable :: gather_matrix_recvbuf(:)
real(kind=rk), public, allocatable :: gather_matrix_sendbuf(:)
integer, public :: gather_max_components = 0
real(kind=rk), public, allocatable :: gather_recvbuf(:)
real(kind=rk), public, allocatable :: gather_sendbuf(:)
integer, public, allocatable :: ghost_cells(:)
integer, public :: last_cell = -1
integer, public :: ncell_recv_ranks = 0
integer, public :: ncell_send_ranks = 0
integer, public :: nface_recv_ranks = 0
integer, public :: nface_send_ranks = 0
integer, public :: nlocal = 0
integer, public :: nprocs = 0
logical, public, allocatable :: owned(:)
integer, public, allocatable :: owned_faces(:)
integer, public :: rank = -1

Functions

public function flow_global_dot_owned(flow, a, b) result(dot)

Computes the global dot product of two vectors over owned cells.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a(:)
real(kind=rk), intent(in) :: b(:)

Return Value real(kind=rk)

public function flow_global_max_owned(flow, a) result(global_max)

Computes the global maximum magnitude of a field over owned cells.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a(:)

Return Value real(kind=rk)

public function flow_global_sum_owned(flow, a) result(total)

Computes the global sum of a field over owned cells.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a(:)

Return Value real(kind=rk)

private function mesh_neighbor_for_cell(mesh, face_id, cell_id) result(nb)

Returns the mesh neighbor, using stored periodic links when present.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
integer, intent(in) :: face_id
integer, intent(in) :: cell_id

Return Value integer


Subroutines

public subroutine flow_allgather_owned_matrix(flow, local_global, global)

Gathers locally-updated matrix cell values and broadcasts to the global mesh.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_global(:,:)
real(kind=rk), intent(out) :: global(:,:)

public subroutine flow_allgather_owned_matrix_inplace(flow, field)

In-place variant for fields that already hold owned-cell updates.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: field(:,:)

public subroutine flow_allgather_owned_scalar(flow, local_global, global)

Gathers locally-updated cell values and broadcasts to the global mesh.

Read more…

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_global(:)
real(kind=rk), intent(out) :: global(:)

public subroutine flow_allgather_owned_v4(flow, local_v, local_s, global_v, global_s)

Gathers 4-component cell values (e.g., Velocity + Scalar) in one call.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_v(:,:)
real(kind=rk), intent(in) :: local_s(:)
real(kind=rk), intent(out) :: global_v(:,:)
real(kind=rk), intent(out) :: global_s(:)

public subroutine flow_allgather_owned_vector(flow, local_global, global)

Gathers locally-updated 3D vector cell values and broadcasts to the global mesh.

Read more…

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: local_global(:,:)
real(kind=rk), intent(out) :: global(:,:)

public subroutine flow_allreduce_global_scalar(flow, local_values, global_values)

Sum-Allreduce for a global scalar field.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: local_values(:)
real(kind=rk), intent(out) :: global_values(:)

public subroutine flow_allreduce_global_vector(flow, local_values, global_values)

Sum-Allreduce for a 3D global vector field.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: local_values(:,:)
real(kind=rk), intent(out) :: global_values(:,:)

public subroutine flow_exchange_cell_matrix(flow, field)

Exchanges owned cell matrix values to ranks that keep them as ghosts.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: field(:,:)

public subroutine flow_exchange_cell_scalar(flow, field)

Exchanges owned cell scalar values to ranks that keep them as ghosts.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: field(:)

public subroutine flow_exchange_face_scalar(flow, face_field)

Exchanges owner-computed face scalar values to ranks owning the neighbor cell.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(inout) :: face_field(:)

public subroutine flow_gather_owned_matrix_root(flow, field, root_field)

Gathers owned matrix cell values to rank 0 only.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: field(:,:)
real(kind=rk), intent(inout) :: root_field(:,:)

public subroutine flow_gather_owned_scalar_root(flow, field, root_field)

Gathers owned scalar cell values to rank 0 only.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
real(kind=rk), intent(in) :: field(:)
real(kind=rk), intent(inout) :: root_field(:)

public subroutine flow_global_dots_owned(flow, n_dots, a, b, results)

Computes multiple global dot products in a single MPI_Allreduce.

Read more…

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
integer, intent(in) :: n_dots
real(kind=rk), intent(in) :: a(:,:)
real(kind=rk), intent(in) :: b(:,:)
real(kind=rk), intent(out) :: results(:)

public subroutine flow_global_two_dots_owned(flow, a1, b1, a2, b2, results)

Computes two global dot products without constructing temporary full-size batches.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(in) :: flow
real(kind=rk), intent(in) :: a1(:)
real(kind=rk), intent(in) :: b1(:)
real(kind=rk), intent(in) :: a2(:)
real(kind=rk), intent(in) :: b2(:)
real(kind=rk), intent(out) :: results(2)

public subroutine flow_mpi_finalize(flow)

Releases all MPI resources and buffers.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow

public subroutine flow_mpi_initialize(mesh, flow, comm_parent, max_gather_components)

Sets up domain decomposition for a given mesh.

Read more…

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh

The mesh to decompose.
type(flow_mpi_t), intent(inout) :: flow

The MPI context to populate.
type(MPI_Comm), intent(in) :: comm_parent

The parent communicator (usually MPI_COMM_WORLD).
integer, intent(in), optional :: max_gather_components

public subroutine mpi_flow_shutdown()

Shuts down the MPI environment if it was started by this module.

Arguments

None

public subroutine mpi_flow_startup()

Initializes the MPI environment if not already active.

Arguments

None

private subroutine check_mpi(ierr, where)

Internal helper for MPI error checking.

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: ierr
character(len=*), intent(in) :: where

private subroutine pack_rank_metadata(counts_all, displs_all, nactive, ranks, counts, displs)

Packs full per-rank metadata down to active communication partners.

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: counts_all(:)
integer, intent(in) :: displs_all(:)
integer, intent(out) :: nactive
integer, intent(out), allocatable :: ranks(:)
integer, intent(out), allocatable :: counts(:)
integer, intent(out), allocatable :: displs(:)

private subroutine prefix_counts(counts, displs)

Converts per-rank counts to zero-based displacements.

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: counts(:)
integer, intent(out) :: displs(:)

private subroutine prepare_matrix_gather(flow, ncomp, ncells, nlocal_comp)

Updates cached count/displacement arrays for a packed component count.

Arguments

Type IntentOptional Attributes Name
type(flow_mpi_t), intent(inout) :: flow
integer, intent(in) :: ncomp
integer, intent(in) :: ncells
integer, intent(out) :: nlocal_comp

private subroutine setup_cell_halo(mesh, flow)

Builds cell ghost send/receive metadata for one-ring neighbor stencils.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_cell_owners(mesh, flow)

Initializes contiguous-decomposition owner lookup.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_face_halo(mesh, flow)

Builds face-flux halo metadata for shared internal faces.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_owned_faces(mesh, flow)

Caches faces whose owner cell belongs to this rank.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow

private subroutine setup_owned_gather(mesh, flow, max_gather_components)

Pre-calculates MPI gather offsets and counts for allgather operations.

Arguments

Type IntentOptional Attributes Name
type(mesh_t), intent(in) :: mesh
type(flow_mpi_t), intent(inout) :: flow
integer, intent(in), optional :: max_gather_components