See ceph#980 (comment)

Signed-off-by: Alexander Indenbaum <[email protected]>

tests/ha/4gws_create_delete.sh

@@ -133,7 +133,8 @@ validate_all_active() {
 NUM_SUBSYSTEMS=2
 NUM_GATEWAYS=4
 FAILING_GATEWAYS=2
-NUM_OPTIMIZED=1
+NUM_OPTIMIZED_FAILOVER=2
+NUM_OPTIMIZED_REBALANCE=1
 #
 # Step 1 validate all gateways are optimized for one of ANA group
 # and all groups are unique
@@ -156,11 +157,11 @@ for i in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do
   echo  📫 nvme-gw delete gateway: \'$gw_name\' pool: \'$POOL\', group: \'\' \(empty string\)
   docker compose exec -T ceph ceph nvme-gw delete $gw_name $POOL ''
 done
-sleep 100 # wait for scale down rebalance complete
+
 docker ps
 
-# expect remaining gws to have 1 optimized groups each because
-# due to scale down rebalance 2 deleted gws and 2 ANA groups were removed from the monitor's database
+# expect remaining gws to have two optimized groups each initially
+# till rebalance kicks and we should expect a single optimized group
 for i in $(seq 4); do
   found=0
   for j in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do
@@ -174,12 +175,26 @@ for i in $(seq 4); do
   # if gw is a healthy one
   if [ "$found" -eq "0" ]; then
      echo "ℹ️ Check healthy gw gw=$i"
-     for s in $(seq $NUM_SUBSYSTEMS); do
-       NQN="nqn.2016-06.io.spdk:cnode$s"
-       GW_OPTIMIZED=$(expect_optimized "$(gw_name $i)" "$NUM_OPTIMIZED" "$NQN")
-     done
+
+    (
+      for s in $(seq $NUM_SUBSYSTEMS); do
+        (
+          NQN="nqn.2016-06.io.spdk:cnode$s"
+          GW_OPTIMIZED=$(expect_optimized "$(gw_name $i)" "$NUM_OPTIMIZED_FAILOVER" "$NQN")
+          echo "✅ failover gw gw=$i nqn=$NQN"
+          GW_OPTIMIZED=$(expect_optimized "$(gw_name $i)" "$NUM_OPTIMIZED_REBALANCE" "$NQN")
+          echo "✅ rebalance gw gw=$i nqn=$NQN"
+        ) &
+      done
+      wait # Wait for all subsystem checks to complete for this gateway
+      echo "✅ failover rebalance gw=$i all subsystems"
+    ) &
   fi
 done
+
+# Wait for all gateway checks to complete
+wait
+echo "✅ failover rebalance all gateways"
 
 #
 # Step 3 failback