test: add a stress test for the https outcalls feature

rs/rust_canisters/proxy_canister/BUILD.bazel

@@ -26,6 +26,7 @@ rust_canister(
         ":lib",
         "//rs/types/management_canister_types",
         "@crate_index//:candid",
+        "@crate_index//:futures",
         "@crate_index//:ic-cdk",
     ],
 )

rs/rust_canisters/proxy_canister/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 candid = { workspace = true }
+futures = { workspace = true }
 ic-cdk = { workspace = true }
 ic-cdk-macros = { workspace = true }
 ic-management-canister-types-private = { path = "../../types/management_canister_types" }

rs/rust_canisters/proxy_canister/src/lib.rs

@@ -16,6 +16,13 @@ pub struct RemoteHttpRequest {
     pub cycles: u64,
 }
 
+#[derive(Clone, Debug, CandidType, Deserialize)]
+pub struct RemoteHttpStressRequest {
+    pub request: RemoteHttpRequest,
+    /// Number of requests to send concurrently.
+    pub count: u64,
+}
+
 /// We create a custom type instead of reusing [`ic_management_canister_types_private::CanisterHttpRequestArgs`]
 /// as we don't want the body to be deserialized as a bounded vec.
 /// This allows us to test sending headers that are longer than the default limit and test.
@@ -52,6 +59,12 @@ pub struct RemoteHttpResponse {
     pub body: String,
 }
 
+#[derive(Clone, Debug, CandidType, Deserialize)]
+pub struct RemoteHttpStressResponse {
+    pub response: RemoteHttpResponse,
+    pub duration_ns: u64,
+}
+
 impl RemoteHttpResponse {
     pub fn new(status: u128, headers: Vec<(String, String)>, body: String) -> Self {
         Self {

rs/rust_canisters/proxy_canister/src/main.rs

@@ -6,13 +6,17 @@
 //! otherwise errors out.
 //!
 use candid::Principal;
+use futures::future::join_all;
 use ic_cdk::api::call::RejectionCode;
-use ic_cdk::caller;
+use ic_cdk::api::time;
+use ic_cdk::{caller, spawn};
 use ic_cdk_macros::{query, update};
 use ic_management_canister_types_private::{
     CanisterHttpResponsePayload, HttpHeader, Payload, TransformArgs,
 };
-use proxy_canister::{RemoteHttpRequest, RemoteHttpResponse};
+use proxy_canister::{
+    RemoteHttpRequest, RemoteHttpResponse, RemoteHttpStressRequest, RemoteHttpStressResponse,
+};
 use std::cell::RefCell;
 use std::collections::HashMap;
 
@@ -21,6 +25,92 @@ thread_local! {
     pub static REMOTE_CALLS: RefCell<HashMap<String, Result<RemoteHttpResponse, (RejectionCode, String)>>>  = RefCell::new(HashMap::new());
 }
 
+#[update]
+async fn send_requests_in_parallel(
+    request: RemoteHttpStressRequest,
+) -> Result<RemoteHttpStressResponse, (RejectionCode, String)> {
+    let start = time();
+    if request.count == 0 {
+        return Err((
+            RejectionCode::CanisterError,
+            "Count cannot be 0".to_string(),
+        ));
+    }
+
+    // This is the maximum size of the queue of canister messages. In our case, it's the highest number of requests we can send in parallel.
+    const MAX_CONCURRENCY: usize = 500;
+
+    let mut all_results: Vec<Result<RemoteHttpResponse, (RejectionCode, String)>> = Vec::new();
+
+    let indices: Vec<u64> = (0..request.count).collect();
+    for chunk in indices.chunks(MAX_CONCURRENCY) {
+        let futures_iter = chunk.iter().map(|_| send_request(request.request.clone()));
+        let chunk_results = join_all(futures_iter).await;
+        all_results.extend(chunk_results);
+    }
+
+    let mut response = None;
+
+    for result in all_results {
+        match result {
+            Ok(rsp) => response = Some(rsp),
+            Err(err) => {
+                return Err(err);
+            }
+        }
+    }
+    let duration_ns = time() - start;
+    Ok(RemoteHttpStressResponse {
+        response: response.unwrap(),
+        duration_ns,
+    })
+}
+
+#[update]
+pub fn start_continuous_requests(
+    request: RemoteHttpRequest,
+) -> Result<RemoteHttpResponse, (RejectionCode, String)> {
+    spawn(async move {
+        run_continuous_request_loop(request).await;
+    });
+
+    Ok(RemoteHttpResponse::new(
+        200,
+        vec![],
+        "Started non-stop sending.".to_string(),
+    ))
+}
+
+// TODO: instead of sequentially awaiting on each batch, try to send the next requests anyway, with backoff.
+// This should improve the overall qps, as the canister message queue is the bottleneck, and it's not being saturated.
+async fn run_continuous_request_loop(request: RemoteHttpRequest) {
+    const BATCH_SIZE: usize = 500;
+    let futures_iter = (0..BATCH_SIZE).map(|_| send_request(request.clone()));
+    let results = join_all(futures_iter).await;
+
+    let mut successes = 0;
+    let mut errors = 0;
+    for result in results {
+        match result {
+            Ok(_resp) => {
+                successes += 1;
+            }
+            Err((rejection_code, msg)) => {
+                errors += 1;
+                println!("Request failed: {:?} - {}", rejection_code, msg);
+            }
+        }
+    }
+    println!(
+        "Finished batch of {} requests => successes: {}, errors: {}",
+        BATCH_SIZE, successes, errors
+    );
+
+    spawn(async move {
+        run_continuous_request_loop(request).await;
+    });
+}
+
 #[update]
 async fn send_request(
     request: RemoteHttpRequest,

rs/tests/networking/BUILD.bazel

@@ -50,6 +50,35 @@ system_test_nns(
     deps = CANISTER_HTTP_BASE_DEPS + ["//rs/rust_canisters/canister_test"],
 )
 
+system_test_nns(
+    name = "canister_http_stress_test",
+    env = {
+        "PROXY_WASM_PATH": "$(rootpath //rs/rust_canisters/proxy_canister:proxy_canister)",
+    },
+    extra_head_nns_tags = [],  # don't run the head_nns variant on nightly since it aleady runs on long_test.
+    tags = [
+        "dynamic_testnet",
+        "manual",
+    ],
+    target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
+    test_timeout = "eternal",
+    runtime_deps =
+        GUESTOS_RUNTIME_DEPS +
+        UNIVERSAL_VM_RUNTIME_DEPS +
+        BOUNDARY_NODE_GUESTOS_RUNTIME_DEPS +
+        CANISTER_HTTP_RUNTIME_DEPS + PROXY_CANISTER_RUNTIME_DEPS,
+    deps = CANISTER_HTTP_BASE_DEPS + [
+        "//rs/registry/subnet_features",
+        "//rs/registry/subnet_type",
+        "//rs/rust_canisters/canister_test",
+        "//rs/types/types",
+        "@crate_index//:futures",
+        "@crate_index//:serde",
+        "@crate_index//:serde_json",
+        "@crate_index//:tokio",
+    ],
+)
+
 system_test_nns(
     name = "canister_http_socks_test",
     env = {

rs/tests/networking/Cargo.toml

@@ -54,6 +54,10 @@ path = "canister_http_fault_tolerance_test.rs"
 name = "ic-systest-canister-http-socks"
 path = "canister_http_socks_test.rs"
 
+[[bin]]
+name = "ic-systest-canister-http-stress"
+path = "canister_http_stress_test.rs"
+
 [[bin]]
 name = "ic-systest-canister-http"
 path = "canister_http_test.rs"

rs/tests/networking/canister_http/canister_http.rs

@@ -12,8 +12,8 @@ use ic_system_test_driver::driver::{
     test_env::{TestEnv, TestEnvAttribute},
     test_env_api::*,
 };
-use ic_system_test_driver::util::{self, create_and_install};
-pub use ic_types::{CanisterId, PrincipalId};
+use ic_system_test_driver::util::{self, create_and_install, create_and_install_with_cycles};
+pub use ic_types::{CanisterId, Cycles, PrincipalId};
 use slog::info;
 use std::env;
 use std::net::{Ipv4Addr, Ipv6Addr};
@@ -54,7 +54,7 @@ pub fn install_nns_canisters(env: &TestEnv) {
 
 pub fn setup(env: TestEnv) {
     std::thread::scope(|s| {
-        // Set up IC with 1 system subnet and 4 application subnets
+        // Set up IC with 1 system subnet with one node, and one application subnet with 4 nodes.
         s.spawn(|| {
             InternetComputer::new()
                 .add_subnet(Subnet::new(SubnetType::System).add_nodes(1))
@@ -187,6 +187,13 @@ pub fn get_node_snapshots(env: &TestEnv) -> Box<dyn Iterator<Item = IcNodeSnapsh
         .nodes()
 }
 
+pub fn get_all_application_subnets(env: &TestEnv) -> Vec<SubnetSnapshot> {
+    env.topology_snapshot()
+        .subnets()
+        .filter(|s| s.subnet_type() == SubnetType::Application)
+        .collect()
+}
+
 pub fn get_system_subnet_node_snapshots(env: &TestEnv) -> Box<dyn Iterator<Item = IcNodeSnapshot>> {
     env.topology_snapshot()
         .subnets()
@@ -227,6 +234,42 @@ pub fn create_proxy_canister_with_name<'a>(
 
     Canister::new(runtime, CanisterId::unchecked_from_principal(principal_id))
 }
+
+pub fn create_proxy_canister_with_name_and_cycles<'a>(
+    env: &TestEnv,
+    runtime: &'a Runtime,
+    node: &IcNodeSnapshot,
+    canister_name: &str,
+    cycles: Cycles,
+) -> Canister<'a> {
+    info!(
+        &env.logger(),
+        "Installing proxy_canister with a custom cycle amount ({cycles:?})."
+    );
+
+    let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime.");
+    let proxy_canister_id = rt.block_on(create_and_install_with_cycles(
+        &node.build_default_agent(),
+        node.effective_canister_id(),
+        &load_wasm(
+            env::var("PROXY_WASM_PATH").expect("Environment variable PROXY_WASM_PATH not set"),
+        ),
+        cycles,
+    ));
+
+    info!(
+        &env.logger(),
+        "Proxy canister {:?} installed with cycles {:?}", proxy_canister_id, cycles
+    );
+
+    let principal_id = PrincipalId::from(proxy_canister_id);
+
+    env.write_json_object(canister_name, &principal_id)
+        .expect("Could not write proxy canister ID to TestEnv.");
+
+    Canister::new(runtime, CanisterId::unchecked_from_principal(principal_id))
+}
+
 pub fn create_proxy_canister<'a>(
     env: &TestEnv,
     runtime: &'a Runtime,
@@ -235,6 +278,15 @@ pub fn create_proxy_canister<'a>(
     create_proxy_canister_with_name(env, runtime, node, PROXY_CANISTER_ID_PATH)
 }
 
+pub fn create_proxy_canister_with_cycles<'a>(
+    env: &TestEnv,
+    runtime: &'a Runtime,
+    node: &IcNodeSnapshot,
+    cycles: Cycles,
+) -> Canister<'a> {
+    create_proxy_canister_with_name_and_cycles(env, runtime, node, PROXY_CANISTER_ID_PATH, cycles)
+}
+
 pub fn get_proxy_canister_id_with_name(env: &TestEnv, name: &str) -> PrincipalId {
     env.read_json_object(name)
         .expect("Proxy canister should should .")

rs/tests/networking/canister_http_correctness_test.rs

@@ -1233,6 +1233,7 @@ fn test_large_maximum_response_size(env: TestEnv) {
             cycles: 500_000_000_000,
         },
     ));
+
     assert_matches!(
         response,
         Err(RejectResponse {