#!/bin/bash
#
# dssgckdisks - check performance of enclosures disks
#
# Copyright 2017-2022 Lenovo. All rights reserved.
# License: Subject to terms of Lenovo License Agreement
#

readonly N=$(basename "$0")            # script name
readonly T=$(date "+%Y-%m-%d.%H%M%S")  # time stamp
readonly F="$N"."$T"."$$"              # unique filename
readonly L=/var/log/dssg/"$F"          # log basename

readonly out=$(test -e /dev/tty && echo /dev/tty || echo /dev/stdout)

log()   { cat <<< "$@"; }
run()   { log "RUN: $@"; eval "$@"; }
WARN()  { log "${dssgcolorwarning}Warning: ${@}${dssgcolornormal}"; }
ERROR() { local retval="$?"; [ "$retval"=0 ] && retval=1; log "${dssgcolorerror}ERROR: $@ (exit status $retval)${dssgcolornormal}"; exit "$retval"; }
QUIT()  { log; log "$@"; log; exit 0; }

abort() {
  trap '' INT TERM
  ###pkill -g 0 2> /dev/null   # kill all processes in the current process group
  {
    local nodes=( $(test -f "$L".allnodes.top && cat "$L".allnodes.top | cut -d: -f1 | sort -u) )
    (( "${#nodes[@]}" )) && echo && dssgxdsh  "$(sed 's/ /,/g' <<< ${nodes[@]})"  'while(pkill gpfsperf); do echo "killing gpfsperf"; sleep 1; done'   # kill remote gpfsperf commands
    #rm -f "$L".*   # remove logs
    echo
    ERROR "User abort"
  } > "$out"
  trap - INT TERM
}
cleanup() { rm -f "$L".*; }
trap abort INT TERM

dssgfuncs=$(dirname $(readlink -f "$0"))/dssgfuncs.sh
. "$dssgfuncs" || ERROR "Cannot source $dssgfuncs"

dssgisroot || exit 2  # not root
dssgsetcolors

mkdir -p /var/log/dssg || ERROR "Cannot create directory /var/log/dssg"


#####
main() {
# usage
local -r purpose="
Purpose: Check the relative performance of the external or NVMe storage drives
"

local -r usage="
Usage:
$N [-h] [-m char] [-t int] [-e string] nodes
  -h, --help   show this help
  -m char      profiling mode: sequential (r|w) or random (R|W) read or write (default: r)
  -t int       run time in seconds per drive path (default: 10)
  -e string    comma-separated list of enclosures S/N to test against
  nodes        noderange/group or comma-separated node list
"

# init options
local mode="r"
local runtime=10
local enclosures=

# parse options
# getopt reorders the command-line: options -- arguments
local cmdline;  # do not assign; getopt's return status is otherwise masked out by local's one
cmdline=$(getopt -n "${dssgcolorwarning}Warning" -o 'hm:b:t:e:' --long help -- "$@") || { echo "${dssgcolornormal}$usage"; ERROR "Invalid and/or missing argument(s)"; }
eval set -- "$cmdline";  # replace command line
log "Parsing options: $@"
while true ; do
  case "$1" in
    -h|--help) echo "${purpose}$usage"; exit 255 ;;
    -m) case "$2" in
          r|R|w|W) mode="$2";;
                *) WARN "Unknown argument '$2' for option $1";;
        esac; shift 2 ;;
    -b) WARN "Option to change the batch size is deprecated - ignoring"; shift 2 ;;
    -t) if test "$2" -eq "$2" 2> /dev/null ; then
          runtime="$2"
        else
          WARN "Non-integer argument '$2' for option $1"
        fi; shift 2 ;;
    -e) enclosures="$2"; shift 2 ;;
    --) shift; break ;;
     *) break ;;
  esac
done
cmdline="$@"

# nodes are either a noderange/group or a comma-separated node list
(( "$#" > 1 )) && WARN "Ignoring extra parameter(s): ${cmdline#* }"

# write mode: ask for confirmation
case "$mode" in
  w|W)
    log
    WARN "Write mode will OVERWRITE all storage drives resulting in potential data loss!"
    printf "Proceed anyway? [yN] "
    local answer
    read answer
    case "$answer" in
      *y|*Y)  echo "$answer" ;;
      "")     answer="<default>" ;&  # fallthrough
      *n|*N)  echo "$answer"; QUIT "Not profiling any drive" ;;
    esac
    ;;
esac

# obtain configurations
dssgconfigs "$1" "$L" 3 2 0 1  # node list, log basename, allowed configs: 0=skipcheck 1=G100 2=G2xy 3=both 4=exclusive_or 5=check server moddels only, check for node count, check for active mmsfd, obtain 1=topologies 2=configs only
local -r g2xy=( $(cat "$L".G2xy.nodes) )
local -r g100=( $(cat "$L".G100.nodes) )


log
log "Profiling drives with mode=${mode} and runtime=${runtime}s per path..."

# start progress status
progress &
local -r progresspid="$!"

# process all DSS-G configurations in parallel
rm -f "$L".allnodes.batches
local -A pid
for nodelist in "${g2xy[@]}" "${g100[@]}" ; do
  if [[ " ${g100[@]} " =~ " $nodelist " ]] ; then
    local config=G100
    local regex=$(tr ',' '|' <<< "$nodelist")
  else
    local config="$nodelist"
    local regex="${nodelist/,/: |}: "
    grep -E "$regex" "$L".G2xy.top > "$L"."$config".top
  fi
  grep -E "$regex" "$L".allnodes.topsum > "$L"."$config".topsum
  log "Start profiling for ${config/G100/G100 nodes}"
  doconfig "$nodelist" &
  pid["$config"]="$!"
done
local -A profstatus
for p in "${!pid[@]}" ; do
  wait "${pid[$p]}"  # wait for doconfig to complete
  profstatus["$p"]="$?"
done

# stop progress status
sleep 1
kill -s USR1 "$progresspid" 2> /dev/null
wait "$progresspid" 2> /dev/null  # supress termination message
log

# profiling status
local warn=0
for nodelist in "${g2xy[@]}" "${g100[@]}" ; do
  config="$nodelist"
  [[ " ${g100[@]} " =~ " $nodelist " ]] && config=G100
  if (( "${profstatus[$config]}" )) ; then
    WARN "Profiling issues were found for ${config/G100/G100 nodes}; please check ${L}.${config}.prof"
    let warn++
  else
    log "Done with profiling for ${config/G100/G100 nodes}: ${L}.${config}.prof"
  fi
done

sync
local errors=0
local configs=0

# process each DSS-G2xy building block
for pair in "${g2xy[@]}" ; do
  local warn=0

  log
  log
  log "********** Processing building block with $pair **********"
  log
  log "Summary of the configuration found:"
  grep -E 'GNR.*enclosures|GNR.*topo|GNR.*conf' "$L"."$pair".topsum | dssgxcoll -n

  log "Checking for number of servers..."
  local servers=( $(tr ',' ' ' <<< "$pair") )
  if (( "${#servers[@]}" < 2 )) ; then
    WARN "No matching node for $pair"
    let warn++
  else
    log "Success: found ${#servers[@]} servers"
  fi

  log
  checkdrives     || let warn+="$?"
  checksizes      || let warn+="$?"

  log "Reporting drive profiling..."
  if test -s "$L"."$pair".prof ; then
    log "Maximum data rate obtained over all drive paths:"
    displayall "$L"."$pair" || let warn++
  else
    echo "No drives selected" | tee "$L"."$pair".prof
    #let warn++
  fi

  log
  log "********** $( ((warn)) && echo "${dssgcolorwarning}Found $warn type(s) of issue" || echo "${dssgcolorgood}Done" ) with $pair${dssgcolornormal} **********"
  let errors+=warn
  ((warn)) && let configs++
done  # building block

# process ECE configuration
for ece in "${g100[@]}"; do
  local warn=0
  local pair=G100  # pair must be defined for the check* functions below

  log
  log
  log "********** Processing DSS-G100 configuration **********"
  log
  log "Summary of the configuration found:"
  local regex=$(tr ',' '|' <<< "$ece")
  grep -E "$regex" "$L".allnodes.topsum > "$L"."$pair".topsum
  grep -E 'GNR.*enclosures|GNR.*topo|GNR.*conf' "$L"."$pair".topsum | dssgxcoll -n

  log "Checking for number of servers..."
  local servers=( $(tr ',' ' ' <<< "$ece") )
  if (( "${#servers[@]}" < 6 )) ; then
    WARN "at least 6 nodes should be specified for a DSS-G100 configuration regarding performance and server fault tolerance; found ${#servers[@]}"
    let warn++
  else
    log "Success: found ${#servers[@]} servers"
  fi

  log
  log "Checking for number of NVMe drives..."
  local drives=( $(grep "sees.*disk" "$L"."$pair".topsum | sed 's,.*sees\s*\([0-9]\+\)\s*.*,\1,') )
  local ndrives=0
  for i in "${drives[@]}"; do let ndrives+="$i"; done
  if (( ndrives < 12 )) ; then
    WARN "A total of at least 12 NMVe drives is required; found $ndrives"
    let warn++
  else
    log "Success: found a total of $ndrives drives"
  fi

  log
  log "Checking for a balanced configuration..."
  local unique=( $(printf "%d\n" "${drives[@]}" | sort -u) )
  if (( "${#unique[@]}" > 1 )); then
    WARN "Unbalanced drive configuration: $(echo ${unique[@]} | tr ' ' ,)"
    let warn++
  else
    log "Success: found ${unique[@]} drive(s) per server"
  fi

  log
  checkdrives     || let warn+="$?"
  checksizes      || let warn+="$?"

  log "Reporting drive profiling..."
  if test -s "$L"."$pair".prof ; then
    log "Maximum data rate obtained over all drive paths:"
    displayall "$L"."$pair" || let warn++
  else
    echo "No drives selected" | tee "$L"."$pair".prof
    #let warn++
  fi
  log

  log "********** $( ((warn)) && echo "${dssgcolorwarning}Found $warn type(s) of issue" || echo "${dssgcolorgood}Done" ) with DSS-G100 configuration${dssgcolornormal} **********"
  let errors+=warn
  ((warn)) && let configs++
done

log
((errors)) && ERROR "Found $errors type(s) of issue to be resolved in $configs configuration(s)"
log "${dssgcolorgood}All done${dssgcolornormal} - see ${L}.log"

return 0
}  # main

#####
checkdrives() {
  local warn=0

  log "Checking for drive issues..."

  grep -i "Location.*but should" "$L"."$pair".topsum  > "$L"."$pair".drives
  grep -i "Location.*disk size"  "$L"."$pair".topsum >> "$L"."$pair".drives

  if test -s "$L"."$pair".drives ; then
    let warn++
    WARN "The following problems related to drive issues were detected:"
    cat "$L"."$pair".drives | sort -k3 | dssgxcoll -n
  else
    log "Success: no drive issue detected"
    log
  fi

  return "$warn"
}

#####
checksizes() {
  local warn=0

  log "Checking for drive capacities..."

  # sequential array of drives with capacity, e.g.
  #   1,38Y0KBA,1-19,naa.5000C50083E027E3,HDD=6000
  #   1,38Y0KBA,1-1,naa.5002538A475B6460,SSD=800
  #   1,38Y0KBA,1-20,naa.5000C50084EA669B,HDD=6000
  servers="$pair"  # for displaydrive
  local -r drives=( $(grep ': [^:]*:0:' "$L"."$pair".top | grep -v RAID | awk -F: '
    {
      loc=(length($18) ? $18 : $1)   # enclosure: S/N (=external) or server name (=internal)
      slot=$20                       # drive slot
      split($5, s, /\|/)
      type=(s[7]+0 ? "HDD" : "SSD")  # drive RPM value (zero for SSD)
      print loc "," slot "," $6 "," type "=" int($15/1e10)*10  # number,(enclosure_sn|server),slot,wwn,type=capacity (rounded to 10GB then 1GB)
    }' | sed "$(displaydrive)" | sort -u) )

  # arrays of SSD and HDD capacities
  local -r ssdcap=( $(printf "%s\n" "${drives[@]}" | grep SSD | cut -d= -f2 | sort -u -n) )
  local -r hddcap=( $(printf "%s\n" "${drives[@]}" | grep HDD | cut -d= -f2 | sort -u -n) )
  local -r logtipbackup="^1,.*(00|23|1-01|2-14).*SSD"  # logTipBackup SSD locations in first enclosure

  # DSS-G2xy: either HDDs and 2+ SSDs (logTipBackup and/or hybrids) or SSDs only of the same capacity (DSS-G20y, no logTipBackup)
  # DSS-G100: SSDs only of the same capacity
  ssdwarn=0 logtipbackupwarn=0
  if (( "${#ssdcap[@]}" > 2 )); then  # more than 2 different SSD capacities
    ssdwarn=1  # definitely an issue
  elif (( "${#ssdcap[@]}" > 1 ));  then  # likely a hybrid, look for logTipBackup SSDs in first enclosure
    local found=0
    for cap in "${ssdcap[@]}"; do
      local ndrives=$(printf "%s\n" "${drives[@]}" | grep -E "$logtipbackup=$cap" | wc -l)
      ((ndrives == 2)) && found=1 && break  # found logTipBackup SSDs
    done
    ((!found)) && ssdwarn=1 && logtipbackupwarn=1  # looks like logTipBackup SSDs are of different capacities
  fi

  # found discrepancy
  if (( "${#hddcap[@]}" > 1 || ssdwarn )); then
    let warn++
    local majhdd=0 majhddcap=0 majssd=0 majssdcap=0
    WARN "Found non-uniform capacities for SSDs and/or HDDs; see details below"

    # determine majority of HDDs / SSDs and their capacity
    local colsright=$(column -tR1 <<< "" 2> /dev/null && echo "-R2,6")  # column supports -R option
    {
      for cap in "${hddcap[@]}"; do
        local ndrives=$(printf "%s\n" "${drives[@]}" | grep HDD="$cap" | wc -l)
        log "Found $ndrives HDD(s) of size $cap GB"  # display capacity
        (( ndrives >= majhdd )) && majhdd="$ndrives" && majhddcap="$cap"
      done
      for cap in "${ssdcap[@]}"; do
        local ndrives=$(printf "%s\n" "${drives[@]}" | grep SSD="$cap" | wc -l)
        log "Found $ndrives SSD(s) of size $cap GB"  # display capacity
        (( ndrives >= majssd )) && majssd="$ndrives" && majssdcap="$cap"
      done
    } > >(column -t -o' ' $colsright)  # do not use pipe
    sync

    # display deviating drives
    colsright=$(column -tR1 <<< "" 2> /dev/null && echo "-R2")  # column supports -R option
    if (( "${#hddcap[@]}" > 1 )); then
      log "-------------------------------------------------------------------------------"
      log "The following HDD(s) deviate from the prevailing $majhddcap GB capacity:"
      {
        log "#drive(encl_number,encl_SN,slot,wwn,type)=size(GB)"
        printf "%s\n" "${drives[@]}" | grep HDD | grep -Ev "=$majhddcap"  # BEWARE: requires non-null majhddcap
      } | column -s= -t $colsright
    fi
    if ((ssdwarn)); then
      log "-------------------------------------------------------------------------------"
      log "The following SSD(s) deviate from the prevailing $majssdcap GB capacity:"
      {
        log "#drive(encl_number,encl_SN,slot,wwn,type)=size(GB)"
        printf "%s\n" "${drives[@]}" | grep SSD | grep -Ev "=$majssdcap|$logtipbackup"  # BEWARE: requires non-null majssdcap
      } | column -s= -t $colsright
    fi
    if ((logtipbackupwarn)); then
      log "-------------------------------------------------------------------------------"
      log "The logTipBackup SSDs in the first enclosure have different capacities:"
      {
        log "#drive(encl_number,encl_SN,slot,wwn,type)=size(GB)"
        printf "%s\n" "${drives[@]}" | grep -E "$logtipbackup"
      } | column -s= -t $colsright
    fi
  else
    log "Success: no discrepancy detected"
  fi
  log

  return "$warn"
}

##### progress thread
progress() {
  rm -f "$L".allnodes.allpaths

  # wait for all DSS-G configurations to have their number of batches available
  local -r nconfigs=$( { test -f "$L".G2xy.nodes && cat "$L".G2xy.nodes; test -f "$L".G100.nodes && cat "$L".G100.nodes; } | wc -l )
  local i=0
  while (( i < nconfigs )) ; do
    sync
    test -s "$L".allnodes.batches && i=$(sed -n '$=' "$L".allnodes.batches)  # count lines
    sleep 0.1
  done

  # estimate remaining time to run
  local -r npaths=$(( $(cat "$L".allnodes.paths | awk 'BEGIN{s=0}{s+=0+$1}END{print s}') ))
  local -r maxbatches=$(cat "$L".allnodes.batches | LC_ALL=C sort -n | tail -n 1)
  local -r maxenclosures=$(cat "$L".allnodes.enclosures | awk '{print NF}' | LC_ALL=C sort -n | tail -n 1)
  local drivepaths=4
  local overhead=$(( maxbatches + maxenclosures ))  # empirical overhead
  if test ! -f "$L".G2xy.top ; then  # there a no G2xy configurations
    drivepaths=1
    overhead=2
  fi
  local -r eta=$(( maxbatches*drivepaths*runtime + overhead ))  # estimated runtime in seconds
  local p=0 elapsed=0 total="$eta"

  # log final progress status to stdout / log file and return
  trap 'printf "\r"; p=100; total="$elapsed"; displayprogress; return' USR1

  # display status every second
  displayprogress  # print initial progress status to stdout / log file
  while true ; do
    sync
    local ended=0
    test -f "$L".allnodes.allpaths && ended=$(sed -n '$=' "$L".allnodes.allpaths)  # count lines
    local remain=$(( npaths - ended ))
    local oldtotal="$total"
    ((npaths)) && total=$(( eta*remain/npaths + elapsed )) || total=0  # estimated total runtime
    (( elapsed < oldtotal && elapsed % runtime )) && total="$oldtotal"  # update total time at regular intervals
    ((npaths)) && p=$(( 100*ended/npaths )) || p=100
    (( p > 100 )) && p=100
    { printf "\r"; displayprogress; } > "$out"  # do not print to stdout / log file when possible
    sleep 1
    (( elapsed++ ))
  done
}

##### display progress line
displayprogress() {
  printf "Profiling... %3d%% (%d:%02d:%02d/%d:%02d:%02d)" \
    $p \
    $(( elapsed/3600 )) $(( (elapsed/60) % 60 )) $(( elapsed % 60 )) \
    $(( total  /3600 )) $(( (total  /60) % 60 )) $(( total   % 60 ))
}

##### perform disk profiling for a DSS-G configuration (comma-separated server list) passed as argument
doconfig() {
  if [[ " ${g100[@]} " =~ " $1 " ]] ; then
    local -r servers=G100
    local -r batchsize=10  # up to 10 NVMe drives per server
    local -r ssdweight=1
  else
    local -r servers="$1"
    local -r batchsize=12  # number of drives profiled at once per enclosure
    local -r ssdweight=4   # each SSD counts as multiple HDDs in a batch
    grep ',' <<< "$servers" > /dev/null 2>&1 || WARN "No matching node for $servers"
  fi

  # note: all fields in the combined topology (.top) and topsummary (.topsum) files are offset by +1 since lines are prefixed with "node: "

  # rationale:
  #   The drive profiling is done for all (selected) enclosures in parallel.
  #
  # DSS-G2xy configurations:
  #   Each enclosure IOM is attached with one cable to a given server: PCIe 3.0 x4 ~ 4 GB/s.
  #   Batch size of 12 HDDs per enclosure: 4 GB/s / 12 ~ supports max 333 MB/s sustained transfer rate per HDD.
  #   Batch size of up to 3 SSDs (batchsize/ssdweight) per enclosure: 4 GB/s / 3 ~ supports max 1333 MB/s sustained transfer rate per SSD.
  #   All SSDs in an enclosure are profiled first.
  #
  # DSS-G100/ECE configuration:
  #   Only one drive path exists to each NVMe SSD in a G100 server (=internal enclosure) with either PCIe 3.0 x4 ~ 4 GB/s (SR630) or PCIe 4.0 x4 ~ 8 GB/s (SR630 V2).
  #   Configurations from 1 up to 8 (SR630) or 10 (SR630 V2) NVMe drives are supported, and all drives are evaluated in parallel.

  # Comma-separated enclosure selection with optional slot, e.g. AA:1-12,BB:08,CC,:12
  # Remove leading zero in slot definition. Note: the slot will be the last field taken from the topology.
  # For instance, the enclosure selection:
  #   :10,AA:0,AA:01,BB:1-01,BB:2-9,BB:2-14,CC:01,CC:2,CC:23,X:0,X:00,:5,WW:SSD,:HDD
  # gives the extended regular expression:
  #   :.*:10$|AA:.*:0$|AA:.*:1$|BB:.*:1-1$|BB:.*:2-9$|BB:.*:2-14$|CC:.*:1$|CC:.*:2$|CC:.*:23$|X:.*:0$|X:.*:0$|:.*:5$|WW:.*\|0:|:.*\|[^0\|][0-9]*:
  local -r encregex=$(sed \
    -e 's/:/:.*/g'                                 `# add .* after the colon`                          \
    -e 's/\*\([0-9]\)-\([1-9][0-9]*\)/*:\1-\2$/g'  `# replace e.g. "*2-14" with "*:2-14$"`             \
    -e 's/\*\([0-9]\)-0\([0-9]\)/*:\1-\2$/g'       `# replace e.g. "*1-01" with "*:1-1$"`              \
    -e 's/\*\([0-9]\)\(,\|$\)/*0\1\2/g'            `# replace e.g. "*5" or "*5," with "*05" or "*05,"` \
    -e 's/\*\([1-9][0-9]*\)/*:\1$/g'               `# replace e.g. "*23" with "*:23$"`                 \
    -e 's/\*\(0\)*\([0-9]\)/*:\2$/g'               `# replace e.g. "*04" with "*:4$"`                  \
    -e 's/\*SSD/*\\|0:/g'                          `# select all SSDs`                                 \
    -e 's/\*HDD/*\\|[^0\\|][0-9]*:/g'              `# select all HDDs`                                 \
    -e 's/,\{2,\}/,/g; s/^,//; s/,$//; s/,/|/g'    `# replace commas with pipe`                        \
    <<< "$enclosures")

  # topology data from selected drives
  local -r drives=$(cat "$L"."$servers".top | grep ': [^:]*:0:' | grep -v RAID | cut -d: -f1-20 | grep -E "$encregex")
  if test -z "$drives" ; then
    WARN "No drive to get profiling data from in ${servers/G100/G100 nodes}"
    echo  0 >> "$L".allnodes.batches  # required to start the progress status
    echo  0 >> "$L".allnodes.paths
    echo "" >> "$L".allnodes.enclosures
    return 0
  fi

  # unique drive paths as "storage[enclosure,slot,wwn,type]=srv1:dev1,srv1:dev2,srv2:dev1,srv2:dev2" where type is HDD or SSD
  local -r elements=$(printf "$drives\n" | sed '/^\s*$/d' | awk -F: '
    {
      loc=(length($18) ? $18 : $1)   # enclosure: S/N (=external) or server name (=internal)
      slot=$20                       # drive slot
      split($5, s, /\|/)
      type=(s[7]+0 ? "HDD" : "SSD")  # drive RPM value (zero for SSD)
      id=(loc "," slot "," $6 "," type)  # drive id: (enclosure|server),slot,wwn,type
      a[id]=a[id] (a[id] ? "," : "") $1 ":" gensub(",", ","$1":", "g", $4)  # append srv:dev for each device
    }
    END{for(i in a) print i "=" a[i]}' \
  )
  declare -A storage
  for e in $elements ; do
    storage["${e%%=*}"]="${e##*=}"
  done
  log "Selecting ${#storage[@]} drives for profiling from ${servers/G100/G100 nodes}"
  echo "${#storage[@]}" > "$L"."$servers".drivecount

  # let the progress thread read the number of drive paths
  local npaths=0
  for id in "${!storage[@]}" ; do
    local paths="${storage[$id]}"  # DSS-G2xy: each drive should have 4 paths (2 per server)
    local p=0
    for dpath in ${paths//,/ }; do ((p++)); done
    if test x"$servers" != x"G100"; then
      if ((p < 4)) ; then
        local drive=$(sed "$(displaydrive)" <<< "$id")
        WARN "${servers}: drive $drive sees $p paths out of 4"
      fi
    fi
    ((npaths += p))
  done
  echo "$npaths" >> "$L".allnodes.paths

  # let the progress thread read the highest number of batches from all selected enclosures
  local enc
  local -a gnrenc
  if test x"$servers" = x"G100"; then
    gnrenc=( $(tr ',' ' ' <<< "${g100[@]}") )
    nbatches=1
  else
    gnrenc=( $(cat "$L"."$servers".top | grep ': [^:]*:13:' | awk -F: '{print $13}' | LC_ALL=C sort -u) )  # enclosure serial numbers
    local maxdrives=0
    for enc in "${gnrenc[@]}" ; do  # filter drives from this enclosure, sorting by field 5 (includes spin) and 6 (drive wwn) and counting unique entries; SSD counts as multiple HDDs except if NVMe
      local ndrives=$(printf "$drives\n" | sed '/^\s*$/d' | grep "$enc" | cut -d: -f5,6 | sort -u | awk -F: -v w="$ssdweight" 'BEGIN{n=0}{split($1, s, /\|/); n+=(s[7]+0?1:w)}END{print n}')
      ((ndrives > maxdrives)) && maxdrives="$ndrives"
    done
    local nbatches=$(( ( maxdrives + batchsize - 1 ) / batchsize ))  # round up number of batches
  fi
  echo "$nbatches" >> "$L".allnodes.batches

  # let the progress thread read the selected enclosures
  local -a selectedenc
  for enc in "${gnrenc[@]}" ; do
    grep -E "$encregex" <<< "$enc" > /dev/null && selectedenc+=("$enc")
  done
  echo "${selectedenc[@]}" >> "$L".allnodes.enclosures

  # profile all enclosures in parallel
  cat /dev/null > "$L"."$servers".prof.tmp
  local -a pidenc
  for enc in "${gnrenc[@]}" ; do  # do not use ${selectedenc[@]} that may be incomplete when selecting drive slots only
    doenclosure "$enc" &  # appends each drive path to tmp file
    pidenc+=( "$!" )
  done
  wait "${pidenc[@]}"
  sync

  # format output
  # from the gpfsperf help:
  #   -nolabels - Produce a single line of output containing all parameters and the measured data rate.
  #   Reported data rate is in units of 1000 bytes/second.
  # note: nDone is an undocumented size reporting "totalXferCounts*RecordSize"
  local -r fields="#id iter node op pattern fn recordSize nBytes fileSize nProcs nThreads strideRecs inv dio shm fsync cycle reltoken aio osync datarate oprate latency util nDone"
  {
    echo "$fields"  # header
    LC_ALL=C sort -n < "$L"."$servers".prof.tmp
    echo "$fields"  # footer
  } | column -t > "$L"."$servers".prof
  sync
  rm -f "$L"."$servers".prof.tmp

  # gpfsperf did encounter error(s)
  if [ $(cat "$L"."$servers".prof | grep -v '^#id' | awk '{if($4 != "read" && $4 != "write") print $0}' | wc -l) != 0 ]; then
    return 2
  fi

  return 0
}

##### profile all drives from the enclosure serial passed as argument
doenclosure() {
  local -r enc="$1"
  local id

  # split SSDs and HDDs in two pools
  declare -a SSDs HDDs
  for id in "${!storage[@]}" ; do
    grep "$enc" > /dev/null <<< "$id" || continue   # skip drive from another enclosure
    local ssd=$(! grep ',SSD$' > /dev/null <<< "$id"; echo "$?")
    ((ssd)) && SSDs+=("$id") || HDDs+=("$id")
  done

  # profile SSDs then HDDs
  local -r display=$(displaydrive)
  local d=0  # number of devices in the current batch
  for id in "${SSDs[@]}" "${HDDs[@]}" ; do
    local i=0  # iteration number
    local ssd=$(! grep ',SSD$' > /dev/null <<< "$id"; echo "$?")
    local paths="${storage[$id]}"   # each drive should have 4 paths (2 per server) for DSS-G2xy configs or 1 path for DSS-G100 / ECE configs
    local displayid=$(sed "$display" <<< "$id")

    # build up profiling command(s) for that drive
    local cmdseq=":"
    local dpath
    for dpath in ${paths//,/ }; do  # perform all drive paths in sequence
      local srv="${dpath%%:*}"
      local dev="${dpath##*:}"
      # run the profiling command for the current $dev on the $srv node
      # stdout is prefixed with drive id and iteration number; the colon is removed from the node name
      # the value of the gpfsperf -r parameter is tweaked for SSDs
      local r
      ((ssd)) && r="-r 100M"
      cmdseq+="; printf '%d %s ' $i $srv; "
      case "$mode" in
        w) cmdseq+="gpfsperf write seq  $dev -r 10M -n ${runtime}0G -th 1  -dio -nolabels -millis ${runtime}000 -nongpfs" ;;
        W) cmdseq+="gpfsperf write rand $dev -r 10M -n ${runtime}0G -th 16 -dio -nolabels -millis ${runtime}000 -nongpfs" ;;
        r) cmdseq+="gpfsperf read  seq  $dev -r 10M -n ${runtime}0G -th 1  -dio -nolabels -millis ${runtime}000 -nongpfs" ;;
        R) cmdseq+="gpfsperf read  rand $dev -r 10M -n ${runtime}0G -th 16 -dio -nolabels -millis ${runtime}000 -nongpfs" ;;
      esac
      cmdseq+=" 2>&1; echo $dev >&2"
      ((i++))
    done

    # run profiling for that drive in background
    dssgxdsh "$srv" "eval $cmdseq" 2>> "$L".allnodes.allpaths | sed -Eu 's@^[^:]*: (.*)$@'"$displayid "'\1@' >> "$L"."$servers".prof.tmp &

    ((ssd)) && ((d += ssdweight)) || ((d++))  # each SSD counts as multiple HDDs except for NVMe
    ((d >= batchsize)) && d=0 && wait         # wait for all drives of current batch
  done
  wait
}

##### map enclosure S/N with its number and reformat drive slot; output sed commands
displaydrive() {
  local e

  # build "enclosure_serial=enclosure_number" for all enclosures from combined topsummary
  local -r enclosures=$(cat "$L"."$servers".topsum | grep -E 'Enclosure.*number' | sed 's,.*Enclosure\s*\([^\s]*\)\s\+(.*number\s*\([0-9]*\).*,\1=\2,' | LC_ALL=C sort -u)
  declare -A encnum
  if test x"$enclosures" = x"internal=1" ; then  # DSS-G100
    local i=1
    for e in $(tr ',' ' ' <<< "${g100[@]}") ; do
      encnum["$e"]=$(awk '{printf "%02d",$1}' <<< "$i")  # encnum[node]=index
      let i++
    done
  else
    for e in $enclosures ; do
      encnum[${e%%=*}]=${e##*=}  # encnum[enclosure_serial]=enclosure_number
    done
  fi

  # replace "encl_serial,slot,wwn,type" with "encl_number,encl_serial,slot2digits,wwn,type"
  for e in "${!encnum[@]}"; do
    printf "s/$e,\([0-9]\)-\([0-9]\),/$e,\\\1-0\\\2,/ ; "  # e.g. "1-4" becomes "1-04" (D3284)
    printf "s/$e,\([0-9]\),/$e,0\\\1,/ ; "                 # e.g. "2" becomes "02" (D1224)
    printf "s/$e/${encnum[$e]},$e/ ; "                     # prefix enclosure serial with its number
  done
}

##### display to stdout one line per drive with the highest measured data rate
displayall() {
  local -r prof="$1".prof  # profiling file
  local -r count="$1".drivecount  # expected drive count file
  local retval=0

  cat "$prof" | grep -v "^#id" | awk '
  {
    if($4 != "read" && $4 != "write")
      a[$1]=$1" ? ? ? ERROR"
    else {
      if(maxr[$1] < 0+$21)  maxr[$1]=0+$21
      if(! index(a[$1],"ERROR"))  a[$1]=sprintf("%s %d %s %d", $1, int($9/1e10)*10, $4, maxr[$1]/1e3)  # 3rd param: round size to 10GB then 1GB
    }
  }
  END{for(i in a) print a[i]}' | LC_ALL=C sort -n > "$prof".disp

  if test -s "$prof".disp; then
    local colsright=$(column -tR1 <<< "" 2> /dev/null && echo "-R2,4")  # column supports -R option
    { echo "#drive(encl_number,encl_SN,slot,wwn,type) size(GB) mode datarate(MB/s)"; cat "$prof".disp; } | column -t $colsright
    local drivecount=$(cat "$prof".disp | wc -l)
    local expected=$(cat "$count" 2> /dev/null)
    echo "#drives=$drivecount"
    (( drivecount != expected )) && log && WARN "Drive count mismatch between actually profiled/expected: $drivecount/$expected" && retval=2
  else
    WARN "Unable to display statistics; please check ${L}.${servers}.prof"
    retval=1
  fi

  rm -f "$prof".disp

  return "$retval"
}

#####
main "$@" 2>&1 | tee >(dssgstripcolors | tr '\r' '\n' > "$L".log)  # strip colors and convert CR to LF in log file

retval="${PIPESTATUS[0]}"

((retval==255)) && cleanup && retval=0
exit "$retval"