From 757eacfe1257662c4e463d258edccf0cfb4404a8 Mon Sep 17 00:00:00 2001 From: Maria Kuklina <101095419+kmd-fl@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:55:40 +0200 Subject: [PATCH] fix(vm-network): fix iptables rules (#2388) --- Cargo.lock | 91 +++++++++++++++++++-- crates/peer-metrics/src/info.rs | 1 + crates/server-config/src/node_config.rs | 2 + crates/server-config/src/resolved_config.rs | 2 + crates/vm-network-utils/Cargo.toml | 7 +- crates/vm-network-utils/bin/main.rs | 37 --------- crates/vm-network-utils/src/lib.rs | 76 +++++++---------- crates/workers/src/workers.rs | 7 +- nox/src/node.rs | 48 +++++++---- 9 files changed, 152 insertions(+), 119 deletions(-) delete mode 100644 crates/vm-network-utils/bin/main.rs diff --git a/Cargo.lock b/Cargo.lock index f57c704ed5..085dc5a43e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2371,6 +2371,17 @@ dependencies = [ "syn 2.0.46", ] +[[package]] +name = "dlopen2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b4f5f101177ff01b8ec4ecc81eead416a8aa42819a2869311b3420fa114ffa" +dependencies = [ + "libc", + "once_cell", + "winapi", +] + [[package]] name = "dlv-list" version = "0.3.0" @@ -3689,7 +3700,7 @@ dependencies = [ "ipnet", "log", "rtnetlink", - "system-configuration", + "system-configuration 0.5.1", "tokio", "windows 0.51.1", ] @@ -3827,9 +3838,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "iptables" @@ -5662,6 +5673,23 @@ dependencies = [ "tempfile", ] +[[package]] +name = "netdev" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f901362e84cd407be6f8cd9d3a46bccf09136b095792785401ea7d283c79b91d" +dependencies = [ + "dlopen2", + "ipnet", + "libc", + "netlink-packet-core 0.7.0", + "netlink-packet-route 0.17.1", + "netlink-sys", + "once_cell", + "system-configuration 0.6.1", + "windows-sys 0.52.0", +] + [[package]] name = "netlink-packet-core" version = "0.4.2" @@ -5674,6 +5702,17 @@ dependencies = [ "netlink-packet-utils", ] +[[package]] +name = "netlink-packet-core" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72724faf704479d67b388da142b186f916188505e7e0b26719019c525882eda4" +dependencies = [ + "anyhow", + "byteorder", + "netlink-packet-utils", +] + [[package]] name = "netlink-packet-route" version = "0.12.0" @@ -5684,7 +5723,21 @@ dependencies = [ "bitflags 1.3.2", "byteorder", "libc", - "netlink-packet-core", + "netlink-packet-core 0.4.2", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-route" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053998cea5a306971f88580d0829e90f270f940befd7cf928da179d4187a5a66" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "byteorder", + "libc", + "netlink-packet-core 0.7.0", "netlink-packet-utils", ] @@ -5709,7 +5762,7 @@ dependencies = [ "bytes", "futures", "log", - "netlink-packet-core", + "netlink-packet-core 0.4.2", "netlink-sys", "thiserror", "tokio", @@ -7241,7 +7294,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tower-service", @@ -7358,7 +7411,7 @@ checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0" dependencies = [ "futures", "log", - "netlink-packet-route", + "netlink-packet-route 0.12.0", "netlink-proto", "nix 0.24.3", "thiserror", @@ -8382,7 +8435,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.4.1", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -8395,6 +8459,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-interface" version = "0.26.1" @@ -9238,6 +9312,7 @@ name = "vm-network-utils" version = "0.1.0" dependencies = [ "iptables", + "netdev", "thiserror", "tracing", ] diff --git a/crates/peer-metrics/src/info.rs b/crates/peer-metrics/src/info.rs index 9921eaa007..3c9136c5b2 100644 --- a/crates/peer-metrics/src/info.rs +++ b/crates/peer-metrics/src/info.rs @@ -87,6 +87,7 @@ pub struct SystemInfo { #[derive(Default, Debug, Clone, Hash, Eq, PartialEq, EncodeLabelSet)] pub struct VmInfo { pub allow_gpu: u8, + pub interface: String, pub public_ip: String, pub host_ssh_port: u16, pub vm_ssh_port: u16, diff --git a/crates/server-config/src/node_config.rs b/crates/server-config/src/node_config.rs index e0eb773229..6e54d11d93 100644 --- a/crates/server-config/src/node_config.rs +++ b/crates/server-config/src/node_config.rs @@ -647,6 +647,8 @@ pub struct VmConfig { #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Eq)] pub struct VmNetworkConfig { + #[serde(default)] + pub interface: Option, #[serde(default = "default_bridge_name")] pub bridge_name: String, pub public_ip: Ipv4Addr, diff --git a/crates/server-config/src/resolved_config.rs b/crates/server-config/src/resolved_config.rs index 1c93e6aaca..dade1884ff 100644 --- a/crates/server-config/src/resolved_config.rs +++ b/crates/server-config/src/resolved_config.rs @@ -861,6 +861,7 @@ mod tests { libvirt_uri = "qemu:///system" allow_gpu = true [vm.network] + interface = "eth0" bridge_name = "br422442" public_ip = "1.1.1.1" vm_ip = "2.2.2.2" @@ -883,6 +884,7 @@ mod tests { libvirt_uri: "qemu:///system".to_string(), allow_gpu: true, network: VmNetworkConfig { + interface: Some("eth0".to_string()), bridge_name: "br422442".to_string(), public_ip: Ipv4Addr::new(1, 1, 1, 1), vm_ip: Ipv4Addr::new(2, 2, 2, 2), diff --git a/crates/vm-network-utils/Cargo.toml b/crates/vm-network-utils/Cargo.toml index 6ac110882b..9f35f704f3 100644 --- a/crates/vm-network-utils/Cargo.toml +++ b/crates/vm-network-utils/Cargo.toml @@ -3,13 +3,10 @@ name = "vm-network-utils" version = "0.1.0" edition = "2021" -[[bin]] -name = 'vm-network-utils' -path = "bin/main.rs" - [target.'cfg(target_os = "linux")'.dependencies] iptables = "0.5.2" [dependencies] thiserror = { workspace = true } -tracing = { workspace = true } \ No newline at end of file +tracing = { workspace = true } +netdev = "0.31.0" diff --git a/crates/vm-network-utils/bin/main.rs b/crates/vm-network-utils/bin/main.rs deleted file mode 100644 index 18b1e733fe..0000000000 --- a/crates/vm-network-utils/bin/main.rs +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Nox Fluence Peer - * - * Copyright (C) 2024 Fluence DAO - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation version 3 of the - * License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -use std::net::Ipv4Addr; -use std::str::FromStr; -use vm_network_utils::{clear_network, NetworkSettings}; - -// bin for local tests -fn main() { - let ns = NetworkSettings { - public_ip: Ipv4Addr::from_str("1.1.1.1").unwrap(), - vm_ip: Ipv4Addr::from_str("2.2.2.2").unwrap(), - bridge_name: "br0".to_string(), - port_range: (1000, 65535), - host_ssh_port: 2222, - vm_ssh_port: 22, - }; - //let result = setup_network(&ns, "test"); - let result = clear_network(&ns, "12D3KooWAb7dquiiyrxZEchx"); - println!("{result:?}"); -} diff --git a/crates/vm-network-utils/src/lib.rs b/crates/vm-network-utils/src/lib.rs index 1bd332b34a..0951dd0cf7 100644 --- a/crates/vm-network-utils/src/lib.rs +++ b/crates/vm-network-utils/src/lib.rs @@ -30,7 +30,10 @@ pub use non_linux_mocks::*; mod linux; mod non_linux_mocks; +#[derive(Debug, Clone)] pub struct NetworkSettings { + // Network interface via which the server can be reached by the `public_ip` + pub interface: String, pub public_ip: Ipv4Addr, pub vm_ip: Ipv4Addr, pub bridge_name: String, @@ -53,6 +56,16 @@ pub enum NetworkSetupError { Init { message: String }, #[error("error cleaning the rules for the chain {chain_name}: {message}")] Clean { chain_name: String, message: String }, + #[error("error getting default network interface for VM settings: {message}. You may set the correct network interface manually in the configuration file")] + Interface { message: String }, +} + +pub fn get_default_interface() -> Result { + netdev::get_default_interface() + .map(|i| i.name) + .map_err(|err| NetworkSetupError::Interface { + message: err.to_string(), + }) } type IpTablesError = Box; @@ -106,12 +119,6 @@ struct IpTablesRules { // iptables -A SNAT-${VM_NAME} // -s ${VM_IP} // -p tcp -m tcp -// --dport ${RNG_START}:${RNG_END} -// -j SNAT -// --to-source ${PUBLIC_IP} -// iptables -A SNAT-${VM_NAME} -// -s ${VM_IP} -// -p tcp -m tcp // --dport ${RNG_START}:${RNG_END} // -d ${VM_IP} // -j MASQUERADE @@ -119,33 +126,18 @@ struct IpTablesRules { // iptables -A SNAT-${VM_NAME} // -s ${VM_IP} // -p tcp -m tcp -// --dport ${MAP_VM} -// -j SNAT -// --to-source ${PUBLIC_IP} -// iptables -A SNAT-${VM_NAME} -// -s ${VM_IP} -// -p tcp -m tcp // --dport ${MAP_HOST} // -d ${VM_IP} // -j MASQUERADE // ``` fn snat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { - let public_ip = network_settings.public_ip; let vm_ip = network_settings.vm_ip; - let vm_ssh_port = network_settings.vm_ssh_port; let host_ssh_port = network_settings.host_ssh_port; let port_start = network_settings.port_range.0; let port_end = network_settings.port_range.1; let name = cut_chain_name(format!("SNAT-{name}")); - let ports_rule = format!( - "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j SNAT --to-source {public_ip}" - ); - - let ssh_ports_rule = - format!("-s {vm_ip} -p tcp -m tcp --dport {vm_ssh_port} -j SNAT --to-source {public_ip}"); - let masquerade_ports_rule = format!( "-s {vm_ip} -p tcp -m tcp --dport {port_start}:{port_end} -d {vm_ip} -j MASQUERADE" ); @@ -156,12 +148,7 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { let append_rules = IpTablesRules { table_name: "nat", chain_name: name.clone(), - rules: vec![ - ports_rule, - ssh_ports_rule, - masquerade_ports_rule, - masquerade_ssh_ports_rule, - ], + rules: vec![masquerade_ports_rule, masquerade_ssh_ports_rule], }; let postrouting_rule = IpTablesRules { @@ -181,7 +168,7 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { // iptables -t nat -N DNAT-${VM_NAME} // # Map the port range // iptables -A DNAT-${VM_NAME} # --append chain -// -d ${PUBLIC_IP} # --destination +// -i ${INTERFACE_NAME} // -p tcp # --protocol // -m tcp # --match // --dport ${RNG_START}:${RNG_END} # --destination-port (I don't have it my manual) @@ -189,7 +176,7 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { // -j DNAT # --jump // # Map the SSH ports // iptables -A DNAT-${VM_NAME} -// -d ${PUBLIC_IP} +// -i ${INTERFACE_NAME} // -p tcp -m tcp // --dport ${MAP_HOST} // --to-destination ${VM_IP}:${MAP_VM} @@ -197,15 +184,15 @@ fn snat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { // // iptables -t nat // -I OUTPUT -// -d ${PUBLIC_IP} +// -i ${INTERFACE_NAME} // -j DNAT-${VM_NAME} // iptables -t nat // -I PREROUTING -// -d ${PUBLIC_IP} +// -i ${INTERFACE_NAME} // -j DNAT-${VM_NAME} // ``` fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { - let public_ip = network_settings.public_ip; + let interface = &network_settings.interface; let vm_ip = network_settings.vm_ip; let vm_ssh_port = network_settings.vm_ssh_port; let host_ssh_port = network_settings.host_ssh_port; @@ -215,12 +202,12 @@ fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { let name = cut_chain_name(format!("DNAT-{name}")); let port_rule = format!( - "-d {public_ip} -p tcp -m tcp --dport {port_start}:{port_end} -j DNAT \ + "-i {interface} -p tcp -m tcp --dport {port_start}:{port_end} -j DNAT \ --to-destination {vm_ip}:{port_start}-{port_end}", ); let ssh_port_rule = format!( - "-d {public_ip} -p tcp -m tcp --dport {host_ssh_port} -j DNAT \ + "-i {interface} -p tcp -m tcp --dport {host_ssh_port} -j DNAT \ --to-destination {vm_ip}:{vm_ssh_port}", ); let append_rules = IpTablesRules { @@ -232,13 +219,13 @@ fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { let prerouting_rule = IpTablesRules { table_name: "nat", chain_name: "PREROUTING".to_string(), - rules: vec![format!("-d {public_ip} -j {name}")], + rules: vec![format!("-i {interface} -j {name}")], }; let output_rule = IpTablesRules { table_name: "nat", chain_name: "OUTPUT".to_string(), - rules: vec![format!("-d {public_ip} -j {name}")], + rules: vec![format!("-o {interface} -j {name}")], }; RulesSet { @@ -251,12 +238,6 @@ fn dnat_rules(network_settings: &NetworkSettings, name: &str) -> RulesSet { // iptables -t filter -N FWD-${VM_NAME} // iptables -A FWD-${VM_NAME} // -d ${VM_IP} -// -o ${BRIDGE_NAME} # --out-interface -// -p tcp -m tcp -// --dport ${RNG_START}:${RNG_END} -// -j ACCEPT -// iptables -A FWD-${VM_NAME} -// -d ${VM_IP} // -o ${BRIDGE_NAME} // -p tcp -m tcp // --dport ${MAP_VM} @@ -344,6 +325,7 @@ fn test() { } let ns = NetworkSettings { + interface: "eth0".to_string(), public_ip: Ipv4Addr::from_str("1.1.1.1").unwrap(), vm_ip: Ipv4Addr::from_str("2.2.2.2").unwrap(), bridge_name: "br0".to_string(), @@ -366,10 +348,10 @@ fn test() { let test = dnat_rules(&ns, "test"); let result = to_string(&test); let expected = r#"-t nat -N DNAT-test --t nat -A DNAT-test -d 1.1.1.1 -p tcp -m tcp --dport 1000:65535 -j DNAT --to-destination 2.2.2.2:1000-65535 --t nat -A DNAT-test -d 1.1.1.1 -p tcp -m tcp --dport 2222 -j DNAT --to-destination 2.2.2.2:22 --t nat -I OUTPUT -d 1.1.1.1 -j DNAT-test --t nat -I PREROUTING -d 1.1.1.1 -j DNAT-test"#; +-t nat -A DNAT-test -i eth0 -p tcp -m tcp --dport 1000:65535 -j DNAT --to-destination 2.2.2.2:1000-65535 +-t nat -A DNAT-test -i eth0 -p tcp -m tcp --dport 2222 -j DNAT --to-destination 2.2.2.2:22 +-t nat -I OUTPUT -o eth0 -j DNAT-test +-t nat -I PREROUTING -i eth0 -j DNAT-test"#; assert_eq!(expected, result); } @@ -377,8 +359,6 @@ fn test() { let test = snat_rules(&ns, "test"); let result = to_string(&test); let expected = r#"-t nat -N SNAT-test --t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 1000:65535 -j SNAT --to-source 1.1.1.1 --t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 22 -j SNAT --to-source 1.1.1.1 -t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 1000:65535 -d 2.2.2.2 -j MASQUERADE -t nat -A SNAT-test -s 2.2.2.2 -p tcp -m tcp --dport 2222 -d 2.2.2.2 -j MASQUERADE -t nat -I POSTROUTING -s 2.2.2.2 -d 2.2.2.2 -j SNAT-test"#; diff --git a/crates/workers/src/workers.rs b/crates/workers/src/workers.rs index ea7605a28d..bed30aec83 100644 --- a/crates/workers/src/workers.rs +++ b/crates/workers/src/workers.rs @@ -120,11 +120,12 @@ impl WorkersConfig { } } +#[derive(Debug, Clone)] pub struct VmConfig { /// Uri to the libvirt API - libvirt_uri: String, - allow_gpu: bool, - network: NetworkSettings, + pub libvirt_uri: String, + pub allow_gpu: bool, + pub network: NetworkSettings, } impl VmConfig { diff --git a/nox/src/node.rs b/nox/src/node.rs index 220adf9b0b..cedc013522 100644 --- a/nox/src/node.rs +++ b/nox/src/node.rs @@ -199,16 +199,25 @@ impl Node { key_storage.clone(), ); - let workers_config = WorkersConfig::new( - config.node_config.workers_queue_buffer, - config.node_config.vm.clone().map(|conf| { - VmConfig::new( - conf.libvirt_uri, - conf.allow_gpu, - to_vm_network_settings(conf.network), - ) - }), - ); + let vm = if let Some(vm_config) = &config.node_config.vm { + let vm_config = vm_config.clone(); + let network_interface = if let Some(interface) = &vm_config.network.interface { + interface.clone() + } else { + vm_network_utils::get_default_interface()? + }; + log::info!("Using default interface for VM traffic routing: {network_interface}"); + + Some(VmConfig::new( + vm_config.libvirt_uri, + vm_config.allow_gpu, + to_vm_network_settings(vm_config.network, network_interface), + )) + } else { + None + }; + let workers_config = + WorkersConfig::new(config.node_config.workers_queue_buffer, vm.clone()); let (workers, worker_events) = Workers::from_path( workers_config, @@ -410,17 +419,17 @@ impl Node { air_version: air_interpreter_wasm::VERSION, spell_version: spell_version.clone(), allowed_binaries, - vm_info: config.node_config.vm.as_ref().map(|vm| VmInfo { + vm_info: vm.as_ref().map(|vm| VmInfo { ip: vm.network.public_ip.to_string(), default_ssh_port: vm.network.host_ssh_port, forwarded_ports: vec![PortInfo::Range( - vm.network.port_range.start, - vm.network.port_range.end, + vm.network.port_range.0, + vm.network.port_range.1, )], }), }; if let Some(m) = metrics_registry.as_mut() { - let nox_info = to_nox_info_metrics(&config, &node_info, peer_id.to_string()); + let nox_info = to_nox_info_metrics(&config, &node_info, &vm, peer_id.to_string()); peer_metrics::add_info_metrics(m, nox_info); } custom_service_functions.extend_one(make_peer_builtin(node_info)); @@ -585,8 +594,10 @@ impl Node { fn to_vm_network_settings( config: server_config::VmNetworkConfig, + network_interface: String, ) -> vm_network_utils::NetworkSettings { vm_network_utils::NetworkSettings { + interface: network_interface, public_ip: config.public_ip, vm_ip: config.vm_ip, bridge_name: config.bridge_name, @@ -842,6 +853,7 @@ fn services_wasm_backend_config(config: &ResolvedConfig) -> WasmBackendConfig { fn to_nox_info_metrics( config: &NodeConfig, node_info: &NodeInfo, + vm_config: &Option, peer_id: String, ) -> peer_metrics::NoxInfo { use peer_metrics::*; @@ -870,16 +882,16 @@ fn to_nox_info_metrics( spell_version: node_info.spell_version.to_string(), }; - let vm_info = config - .vm + let vm_info = vm_config .as_ref() .map(|vm| VmInfo { allow_gpu: if vm.allow_gpu { 1 } else { 0 }, + interface: vm.network.interface.clone(), public_ip: vm.network.public_ip.to_string(), host_ssh_port: vm.network.host_ssh_port, vm_ssh_port: vm.network.vm_ssh_port, - port_range_start: vm.network.port_range.start, - port_range_end: vm.network.port_range.end, + port_range_start: vm.network.port_range.0, + port_range_end: vm.network.port_range.0, }) .unwrap_or_default();