set_qos test

Signed-off-by: Alexander Indenbaum <[email protected]>

.github/workflows/build-container.yml

@@ -575,7 +575,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["sanity", "no_huge", "ns_lb_change", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster"]
+        test: ["sanity", "no_huge", "ns_lb_change", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster", "set_qos", "librbd_qos"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 1024  # 4 spdk instances

tests/ceph-nvmeof.librbd_qos.conf

@@ -0,0 +1,104 @@
+#
+#  Copyright (c) 2021 International Business Machines
+#  All rights reserved.
+#
+#  SPDX-License-Identifier: LGPL-3.0-or-later
+#
+#  Authors: [email protected], [email protected]
+#
+
+[gateway]
+name =
+group =
+addr = 0.0.0.0
+port = 5500
+enable_auth = False
+state_update_notify = True
+state_update_timeout_in_msec = 2000
+state_update_interval_sec = 5
+enable_spdk_discovery_controller = False
+encryption_key = /etc/ceph/encryption.key
+rebalance_period_sec = 7
+max_gws_in_grp = 16
+max_ns_to_change_lb_grp = 8
+#omap_file_lock_duration = 20
+#omap_file_lock_retries = 30
+#omap_file_lock_retry_sleep_interval = 1.0
+#omap_file_update_reloads = 10
+#enable_prometheus_exporter = True
+#prometheus_exporter_ssl = True
+#prometheus_port = 10008
+#prometheus_bdev_pools = rbd
+#prometheus_stats_interval = 10
+#verify_nqns = True
+#verify_keys = True
+#verify_listener_ip = True
+#allowed_consecutive_spdk_ping_failures = 1
+#spdk_ping_interval_in_seconds = 2.0
+#max_hosts_per_namespace = 8
+#max_namespaces_with_netmask = 1000
+#max_subsystems = 128
+#max_hosts = 2048
+#max_namespaces = 1024
+#max_namespaces_per_subsystem = 256
+#max_hosts_per_subsystem = 32
+
+[gateway-logs]
+log_level=debug
+#log_files_enabled = True
+#log_files_rotation_enabled = True
+#verbose_log_messages = True
+#max_log_file_size_in_mb=10
+#max_log_files_count=20
+#max_log_directory_backups=10
+#
+# Notice that if you change the log directory the log files will only be visible inside the container
+#
+#log_directory = /var/log/ceph/
+
+[discovery]
+addr = 0.0.0.0
+port = 8009
+
+[ceph]
+pool = rbd
+config_file = /etc/ceph/ceph.conf
+
+[mtls]
+server_key = ./server.key
+client_key = ./client.key
+server_cert = ./server.crt
+client_cert = ./client.crt
+
+[spdk]
+# Support multiple cluster allocation strategies
+# Legacy strategy, per ANA grp, max bdevs_per_cluster
+bdevs_per_cluster = 32
+# Flat bdevs per cluster, ignore ANA grp id
+# flat_bdevs_per_cluster = 32
+# Cluster pool
+# cluster_pool_size = 32
+tgt_path = /usr/local/bin/nvmf_tgt
+#rpc_socket_dir = /var/tmp/
+#rpc_socket_name = spdk.sock
+#tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va
+timeout = 60.0
+#log_level =
+#protocol_log_level = WARNING
+#log_file_dir =
+
+# Example value: -m 0x3 -L all
+tgt_cmd_extra_args = -m 0x3
+# partition cores between librbd and spdk reactor
+librbd_core_mask=0xC
+# transports = tcp
+
+# Example value: {"max_queue_depth" : 16, "max_io_size" : 4194304, "io_unit_size" : 1048576, "zcopy" : false}
+transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr" : 7}
+
+# Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168}
+# iobuf_options =
+
+[monitor]
+#timeout = 1.0
+#log_file_dir =

tests/ha/librbd_qos.sh

@@ -0,0 +1 @@
+set_qos.sh

tests/ha/no_setup.sh

@@ -0,0 +1,2 @@
+set -e
+echo ℹ️  Skipping setup for this test

tests/ha/set_qos.sh

@@ -0,0 +1,14 @@
+set -xe
+
+GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}')
+GW1_IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$GW1_NAME")"
+NQN="nqn.2016-06.io.spdk:cnode17"
+NS_COUNT=400
+MAX_NS=1024
+cephnvmf="docker compose run --rm nvmeof-cli --server-address $GW1_IP --server-port 5500"
+
+$cephnvmf subsystem add --subsystem $NQN --max-namespaces 1024
+for i in $(seq  $NS_COUNT); do
+    $cephnvmf namespace add -n $NQN --rbd-pool rbd --rbd-image image$i --rbd-create-image --size 1MB
+    $cephnvmf namespace set_qos -n $NQN --nsid $i --rw-ios-per-second 1000 --rw-megabytes-per-second 19 --r-megabytes-per-second 19 --w-megabytes-per-second 19
+done

tests/ha/setup_librbd_qos.sh

@@ -0,0 +1 @@
+no_setup.sh

tests/ha/setup_main_exit.sh

@@ -0,0 +1 @@
+no_setup.sh

tests/ha/setup_set_qos.sh

@@ -0,0 +1 @@
+no_setup.sh

tests/ha/start_up_librbd_qos.sh

@@ -0,0 +1,20 @@
+# Check if GITHUB_WORKSPACE is defined
+if [ -n "$GITHUB_WORKSPACE" ]; then
+    test_dir="$GITHUB_WORKSPACE/tests/ha"
+else
+    test_dir=$(dirname $0)
+fi
+
+# Get the number of available CPU cores
+CPU_CORES=$(grep -c ^processor /proc/cpuinfo)
+
+# Check if there are at least 4 cores
+if [ "$CPU_CORES" -ge 4 ]; then
+    echo "Sufficient CPU cores available: $CPU_CORES"
+else
+    echo "Insufficient CPU cores: Only $CPU_CORES cores available."
+    exit 1
+fi
+
+export NVMEOF_CONFIG=./tests/ceph-nvmeof.librbd_qos.conf
+$test_dir/start_up.sh 2