#!/bin/bash
# BEGIN_ICS_COPYRIGHT8 ****************************************
#
# Copyright (c) 2015-2022, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# END_ICS_COPYRIGHT8   ****************************************

#[ICS VERSION STRING: unknown]

# PURPOSE:
#
# This file sets environment variables for an IntelMPI job.
# Note that there are many, many such variables.
#
# SYNTAX:
#
# This file must be a valid BASH script. In general, anything that's valid
# in BASH is valid here. To pass variables to IntelMPI, they
# may take either of these forms:
#
# export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv VARIABLE_NAME variablevalue"
# export VARIABLE_NAME=variablevalue
#
# This script generally both exports the variable and provides it via -genv
# As such MPI_CMD_ARGS shown in the log of running the MPI app will explicitly
# show the variables used.

# SAMPLE Tuning variables:
#
# Uncomment the following lines to enable them.
#

export MPI_CMD_ARGS=

# By default, the run_* scripts use a round-robin method to assign MPI
# processes to hosts. (For example, given 4 hosts, named A, B, C, and D, by
# default 8 processes will be laid out as A, B, C, D, A, B, C, D.) Changing
# PROCS_PER_NODE will change how this is done. For example, PROCS_PER_NODE=2
# results in processes being laid out as A, A, B, B, C, C, D, D.
#export PROCS_PER_NODE=1

# It is recommended to use the OFI interface over PSM3 within IntelMPI
# These variables are passed to the Intel mpirun command.
export I_MPI_FABRICS=shm:ofi

# For Intel MPI v2021.6.0 it's recommended to use FI_PROVIDER
# for other releases I_MPI_OFI_PROVIDER is recommended
export I_MPI_OFI_PROVIDER=psm3
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv FI_PROVIDER=psm3"

# This defines the order in which PSM interfaces are tested for
# reachability to another rank of an MPI job. You will not normally need
# to change this.
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_DEVICES self,shm,nic"

# The PSM RDMA mode controls how data transfers are done.
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_RDMA 1"

# When running single-rail you can specify the RDMA NIC to use via
# PSM3_NIC. PSM3_NIC selects by the device name
# (as listed alphabetically in /sys/class/infiniband) or by device number,
# with the first device listed in /sys/class/infiniband being device 0,
# the second device being device 1, and so on. By default,
# PSM3_NIC=-1 (select first available NIC).
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_NIC irdma0"
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_NIC 2"

# These values can enable and control PSM Multi-Rail
# In most cases the default automatic selections will be sufficient
# The sample shown is for Dual NIC server with one port per NIC connected
# and all PSM processes using both NICs
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_MULTIRAIL 1 -genv PSM3_MULTIRAIL_MAP 0,1"

# If the network is routed so that NICs with different IP subnets can still
# talk to eachother, this causes PSM to treat all IP addresses as part of the
# same network.
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_ALLOW_ROUTERS 1"

# Use this to explicitly specify a MTU
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_MTU 4096"

# PSM at job launch can output basic identifying information including it's
# version, location, NIC(s) selected, etc
# The 1st example outputs this for all ranks, the second outputs only for rank 0
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_IDENTIFY 1"
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_IDENTIFY 1:"

# PSM can output performance statistics periodically to a separate file per
# rank.  This controls the frequency of the output (in seconds)
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_PRINT_STATS 1"

# PSM has the capability to dump its own backtrace on a crash, independent of
# the existing coredump facility. Enabling this feature should cause a
# backtrace to be logged to stderr and to a file in the current working
# directory.
#export MPI_CMD_ARGS="$MPI_CMD_ARGS -genv PSM3_BACKTRACE 1"

# This variable is used by the OPA run_* scripts to force benchmarks to run
# on selected CPU cores.
#export MPI_TASKSET="${MPI_TASKSET:- -c 1-7}"
