Compare commits

..
Author SHA1 Message Date
fedor a540299219 fixed build 2024-03-07 07:25:01 -05:00
fedor 16864f38f0 Allow traffic to private networks 2024-03-07 05:57:53 -05:00
15 changed files with 640 additions and 1599 deletions
-6
View File
@@ -1,6 +0,0 @@
version: 2
updates:
- package-ecosystem: "cargo"
directory: "/"
schedule:
interval: "weekly"
Generated
+567 -1254
View File
File diff suppressed because it is too large Load Diff
+16 -20
View File
@@ -12,23 +12,19 @@ inherits = "release"
debug = true
[dependencies]
smoltcp = "0"
libc = "0"
polling = "3"
dhcproto = { git = "https://github.com/bluecatengineering/dhcproto.git", branch = "master" }
vmnet = "0"
clap = { version = "4", features = ["derive"] }
mac_address = "1"
privdrop = "0"
anyhow = { version = "1", features = ["backtrace"] }
ip_network = "0"
uzers = "0"
system-configuration = "0"
num_enum = "0"
sentry = { version = "0", features = ["debug-images"] }
sentry-anyhow = { version = "0", features = ["backtrace"] }
nix = { version = "0", features = ["signal"] }
prefix-trie = "0"
ipnet = "2"
oslog = "0.2.0"
log = "0.4.22"
smoltcp = "0.8.1"
libc = "0.2.126"
polling = { git = "https://github.com/smol-rs/polling.git" }
dhcproto = "0.7.0"
vmnet = "0.1.1"
clap = { version = "3.1.18", features = ["derive"] }
mac_address = "1.1.3"
privdrop = "0.5.2"
anyhow = { version = "1.0.66", features = ["backtrace"] }
ip_network = "0.4.1"
uzers = "0.11.3"
system-configuration = "0.5.0"
num_enum = "0.5.7"
sentry = { version = "0.29.1", features = ["debug-images"] }
sentry-anyhow = { version = "0.29.1", features = ["backtrace"] }
nix = "0.26.2"
+2 -2
View File
@@ -8,9 +8,9 @@ Please check out [this blog post](https://cirrus-ci.org/blog/2022/07/07/isolatin
Softnet solves two problems:
1. VM network isolation
* [`VZNATNetworkDeviceAttachment`](https://developer.apple.com/documentation/virtualization/vznatnetworkdeviceattachment) (the default networking in Tart) enables [vmnet's bridge isolation](https://developer.apple.com/documentation/vmnet/vmnet_enable_isolation_key) by default and prevents cross-VM traffic, however it's still possible for any VM to spoof the host's ARP-table and capture other VMs traffic by using tools that enable conducting the [ARP spoofing attacks](https://en.wikipedia.org/wiki/ARP_spoofing) (e.g. [arpspoof](https://www.monkey.org/~dugsong/dsniff/), [arpoison](http://www.arpoison.net/) and so on)
* [`VZNATNetworkDeviceAttachment`](https://developer.apple.com/documentation/virtualization/vznatnetworkdeviceattachment) (the default networking in Tart) enables [vmnet's bridge isolation](https://developer.apple.com/documentation/vmnet/vmnet_enable_isolation_key) by default and prevents cross-VM traffic, however it's still possible for any VM to spoof the host's ARP-table and capture other VMs traffic
2. DHCP exhaustion
* macOS built-in DHCP-server allocates a `/24` subnet with 86400 seconds lease time by default, which only allows for ~253 VMs a day (or 1 VM every ~6 minutes) to be spawned without causing a denial-of-service, which is pretty limiting for CI services like Cirrus CI
* macOS built-in DHCP-server allocates a `/24` subnet with 86400 seconds lease time by default, which only allows for ~253 VMs a day (or 1 VM every ~6 minutes) to be spawned without causing a denial-of-service, which is pretty limiting for CI services like Cirrus CI
And assumes that:
+5 -13
View File
@@ -26,14 +26,14 @@ impl DhcpSnooper {
};
let dns_ips = match message.opts().get(OptionCode::DomainNameServer) {
Some(DhcpOption::DomainNameServer(dns_ips)) => {
HashSet::from_iter(dns_ips.iter().cloned())
}
Some(DhcpOption::DomainNameServer(dns_ips)) => HashSet::from_iter(
dns_ips.iter().map(|dns_ip| Ipv4Address(dns_ip.octets())),
),
_ => HashSet::new(),
};
self.vm_lease = Some(Lease::new(
message.yiaddr(),
message.yiaddr().into(),
Duration::from_secs(*lease_time as u64),
dns_ips,
))
@@ -73,15 +73,7 @@ impl Lease {
}
}
pub fn address(&self) -> Ipv4Address {
self.address
}
pub fn valid(&self) -> bool {
Instant::now() < self.valid_until
}
pub fn valid_ip_source(&self, address: Ipv4Address) -> bool {
self.address == address && self.valid()
self.address == address && Instant::now() < self.valid_until
}
}
+10 -51
View File
@@ -1,6 +1,5 @@
use clap::ArgEnum;
use anyhow::{anyhow, Context, Result};
use clap::ValueEnum;
use log::info;
use std::net::Ipv4Addr;
use std::os::unix::io::{AsRawFd, RawFd};
use std::os::unix::net::UnixDatagram;
@@ -8,10 +7,9 @@ use std::str::FromStr;
use std::sync::mpsc::{sync_channel, SyncSender};
use vmnet::mode::Mode;
use vmnet::parameters::{Parameter, ParameterKind};
use vmnet::port_forwarding::{AddressFamily, Protocol};
use vmnet::{Events, Options};
#[derive(ValueEnum, Clone, Debug)]
#[derive(ArgEnum, Clone, Debug)]
pub enum NetType {
/// Shared network
///
@@ -33,7 +31,7 @@ pub struct Host {
}
impl Host {
pub fn new(vm_net_type: NetType, enable_isolation: bool) -> Result<Host> {
pub fn new(vm_net_type: NetType) -> Result<Host> {
// Initialize a vmnet.framework NAT or Host interface with isolation enabled
let mut interface = vmnet::Interface::new(
match vm_net_type {
@@ -41,30 +39,22 @@ impl Host {
NetType::Host => Mode::Host(Default::default()),
},
Options {
enable_isolation: Some(enable_isolation),
enable_isolation: Some(true),
..Default::default()
},
)
.context("failed to initialize vmnet interface")?;
// Retrieve first IP (gateway) used for this interface
let Some(Parameter::StartAddress(gateway_ip)) =
interface.parameters().get(ParameterKind::StartAddress)
else {
return Err(anyhow!(
"failed to retrieve vmnet's interface start address"
));
let Some(Parameter::StartAddress(gateway_ip)) = interface.parameters().get(ParameterKind::StartAddress) else {
return Err(anyhow!("failed to retrieve vmnet's interface start address"));
};
let gateway_ip = Ipv4Addr::from_str(&gateway_ip)
.context("failed to parse vmnet's interface start address")?;
// Retrieve max packet size for this interface
let Some(Parameter::MaxPacketSize(max_packet_size)) =
interface.parameters().get(ParameterKind::MaxPacketSize)
else {
return Err(anyhow!(
"failed to retrieve vmnet's interface max packet size"
));
let Some(Parameter::MaxPacketSize(max_packet_size)) = interface.parameters().get(ParameterKind::MaxPacketSize) else {
return Err(anyhow!("failed to retrieve vmnet's interface max packet size"));
};
// Set up a socketpair() to emulate polling of the vmnet interface
@@ -76,9 +66,7 @@ impl Host {
interface
.set_event_callback(Events::PACKETS_AVAILABLE, move |_mask, _params| {
// Send a dummy datagram to make the other end of socketpair() readable
// and ignore the error as this merely a signalling channel to wake up
// the poller
new_packets_tx.send(&[0; 1]).ok();
new_packets_tx.send(&[0; 1]).unwrap();
// Wait for the permission to continue to avoid
// wasting CPU cycles or in case of termination,
@@ -94,7 +82,7 @@ impl Host {
interface,
new_packets_rx,
callback_can_continue_tx,
gateway_ip,
gateway_ip: gateway_ip.into(),
max_packet_size,
finalized: false,
})
@@ -102,35 +90,6 @@ impl Host {
}
impl Host {
pub fn port_forwarding_add_rule(
&mut self,
external_port: u16,
internal_addr: Ipv4Addr,
internal_port: u16,
) -> Result<()> {
let details = format!("external_port={external_port}, internal_addr={internal_addr}, internal_port={internal_port}");
self.interface
.port_forwarding_rule_add(
AddressFamily::Ipv4,
Protocol::Tcp,
external_port,
internal_addr.into(),
internal_port,
)
.map(|_| info!("added port forwarding rule {details}"))
.map_err(|err| anyhow!("failed to add port forwarding rule {details}: {err}"))
}
pub fn port_forwarding_remove_rule(&mut self, external_port: u16) -> Result<()> {
let details = format!("external_port={external_port}");
self.interface
.port_forwarding_rule_remove(AddressFamily::Ipv4, Protocol::Tcp, external_port)
.map(|_| info!("removed port forwarding rule {details}"))
.map_err(|err| anyhow!("failed to remove port forwarding rule {details}: {err}"))
}
pub fn read(&mut self, buf: &mut [u8]) -> vmnet::Result<usize> {
// Dequeue dummy datagram from the socket (if any)
// to free up buffer space and reduce false-positives
+16 -18
View File
@@ -2,15 +2,14 @@ use anyhow::Result;
use num_enum::IntoPrimitive;
use polling::os::kqueue::PollerKqueueExt;
use polling::PollMode;
use std::os::fd::{AsRawFd, BorrowedFd};
use std::os::unix::io::RawFd;
use std::time::Duration;
pub struct Poller<'poller> {
pub struct Poller {
poller: polling::Poller,
events: polling::Events,
vm_fd: BorrowedFd<'poller>,
host_fd: BorrowedFd<'poller>,
events: Vec<polling::Event>,
vm_fd: RawFd,
host_fd: RawFd,
}
#[derive(IntoPrimitive)]
@@ -21,25 +20,22 @@ enum EventKey {
Interrupt,
}
impl Poller<'_> {
pub fn new<'poller>(vm_fd: RawFd, host_fd: RawFd) -> Result<Poller<'poller>> {
impl Poller {
pub fn new(vm_fd: RawFd, host_fd: RawFd) -> Result<Poller> {
let poller = polling::Poller::new()?;
Ok(Poller {
poller,
events: polling::Events::new(),
vm_fd: unsafe { BorrowedFd::borrow_raw(vm_fd) },
host_fd: unsafe { BorrowedFd::borrow_raw(host_fd) },
events: Vec::new(),
vm_fd,
host_fd,
})
}
pub fn arm(&self) -> Result<()> {
unsafe {
self.poller
.add(self.vm_fd.as_raw_fd(), self.vm_interest())?;
self.poller
.add(self.host_fd.as_raw_fd(), self.host_interest())?;
}
self.poller.add(self.vm_fd as RawFd, self.vm_interest())?;
self.poller
.add(self.host_fd as RawFd, self.host_interest())?;
let interrupt_signal = polling::os::kqueue::Signal(libc::SIGINT);
self.poller
@@ -56,8 +52,10 @@ impl Poller<'_> {
pub fn rearm(&mut self) -> Result<()> {
self.events.clear();
self.poller.modify(self.vm_fd, self.vm_interest())?;
self.poller.modify(self.host_fd, self.host_interest())?;
self.poller
.modify(self.vm_fd as RawFd, self.vm_interest())?;
self.poller
.modify(self.host_fd as RawFd, self.host_interest())?;
let interrupt_signal = polling::os::kqueue::Signal(libc::SIGINT);
self.poller.modify_filter(
-47
View File
@@ -1,47 +0,0 @@
use anyhow::{anyhow, Context, Error};
use std::str::FromStr;
#[derive(Debug, Clone, Copy, Default)]
pub struct ExposedPort {
pub external_port: u16,
pub internal_port: u16,
}
impl FromStr for ExposedPort {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let splits: Vec<&str> = s.split(':').collect();
match splits.len() {
2 => Ok(ExposedPort {
external_port: splits[0]
.parse()
.context(format!("invalid external port {:?}", splits[0]))?,
internal_port: splits[1]
.parse()
.context(format!("invalid internal port {:?}", splits[1]))?,
}),
_ => Err(anyhow!(
"invalid exposed port specification {:?}, the format should be EXTERNAL:INTERNAL",
s
)),
}
}
}
#[cfg(test)]
mod tests {
use crate::proxy::exposed_port::ExposedPort;
#[test]
fn exposed_port() {
assert_eq!(
ExposedPort {
external_port: 2222,
internal_port: 22
},
"2222:22".parse().unwrap()
);
}
}
+2 -2
View File
@@ -3,7 +3,7 @@ use crate::proxy::Proxy;
use anyhow::{Context, Result};
use smoltcp::wire::{EthernetFrame, EthernetProtocol, Ipv4Packet, UdpPacket};
impl Proxy<'_> {
impl Proxy {
pub(crate) fn process_frame_from_host(&mut self, frame: &EthernetFrame<&[u8]>) -> Result<()> {
if self.allowed_from_host(frame).is_none() {
// Block packet by not forwarding it to the VM
@@ -58,7 +58,7 @@ impl Proxy<'_> {
return;
}
if ipv4_pkt.next_header() != smoltcp::wire::IpProtocol::Udp {
if ipv4_pkt.protocol() != smoltcp::wire::IpProtocol::Udp {
return;
}
+5 -27
View File
@@ -1,6 +1,4 @@
mod exposed_port;
mod host;
mod port_forwarder;
mod udp_packet_helper;
mod vm;
@@ -10,36 +8,24 @@ use crate::host::NetType;
use crate::poller::Poller;
use crate::vm::VM;
use anyhow::Result;
pub use exposed_port::ExposedPort;
use ipnet::Ipv4Net;
use mac_address::MacAddress;
use port_forwarder::PortForwarder;
use prefix_trie::{Prefix, PrefixSet};
use smoltcp::wire::EthernetFrame;
use std::io::ErrorKind;
use std::os::unix::io::{AsRawFd, RawFd};
pub struct Proxy<'proxy> {
pub struct Proxy {
vm: VM,
host: Host,
poller: Poller<'proxy>,
poller: Poller,
vm_mac_address: smoltcp::wire::EthernetAddress,
dhcp_snooper: DhcpSnooper,
allow: PrefixSet<Ipv4Net>,
enobufs_encountered: bool,
port_forwarder: PortForwarder,
}
impl Proxy<'_> {
pub fn new<'proxy>(
vm_fd: RawFd,
vm_mac_address: MacAddress,
vm_net_type: NetType,
allow: PrefixSet<Ipv4Net>,
exposed_ports: Vec<ExposedPort>,
) -> Result<Proxy<'proxy>> {
impl Proxy {
pub fn new(vm_fd: RawFd, vm_mac_address: MacAddress, vm_net_type: NetType) -> Result<Proxy> {
let vm = VM::new(vm_fd)?;
let host = Host::new(vm_net_type, !allow.contains(&Ipv4Net::zero()))?;
let host = Host::new(vm_net_type)?;
let poller = Poller::new(vm.as_raw_fd(), host.as_raw_fd())?;
Ok(Proxy {
@@ -48,9 +34,7 @@ impl Proxy<'_> {
poller,
vm_mac_address: smoltcp::wire::EthernetAddress(vm_mac_address.bytes()),
dhcp_snooper: Default::default(),
allow,
enobufs_encountered: false,
port_forwarder: PortForwarder::new(exposed_ports),
})
}
@@ -75,12 +59,6 @@ impl Proxy<'_> {
return Ok(());
}
// Timeout
if !vm_readable && !host_readable && !interrupt {
self.port_forwarder
.tick(&mut self.host, self.dhcp_snooper.lease());
}
self.poller.rearm()?;
}
}
-99
View File
@@ -1,99 +0,0 @@
use crate::dhcp_snooper::Lease;
use crate::host::Host;
use crate::proxy::exposed_port::ExposedPort;
use anyhow::Result;
use log::error;
use std::net::Ipv4Addr;
#[derive(Default)]
pub struct PortForwarder {
port_forwardings: Vec<PortForwarding>,
failed: bool,
}
#[derive(Debug, Clone, Copy, Default)]
struct PortForwarding {
exposed_port: ExposedPort,
forwarding_to_addr: Option<Ipv4Addr>,
}
impl PortForwarder {
pub fn new(exposed_ports: Vec<ExposedPort>) -> PortForwarder {
let port_forwardings = exposed_ports
.into_iter()
.map(|exposed_port| PortForwarding {
exposed_port,
..Default::default()
})
.collect();
PortForwarder {
port_forwardings,
..Default::default()
}
}
pub fn tick(&mut self, host: &mut Host, lease: &Option<Lease>) {
if self.failed {
return;
}
if let Err(err) = self.tick_inner(host, lease) {
error!("port-forwarding failed: {}", err);
self.failed = true;
}
}
fn tick_inner(&mut self, host: &mut Host, lease: &Option<Lease>) -> Result<()> {
if let Some(lease) = lease {
// Lease exists, but is not valid, remove all port forwardings
if !lease.valid() {
self.remove_all_port_forwardings(host)?;
return Ok(());
}
// Lease exists and is valid, install/re-install port forwardings
for port_forwarding in &mut self.port_forwardings {
if let Some(installed_addr) = port_forwarding.forwarding_to_addr {
// Port forwarding already installed, perhaps it's outdated?
if installed_addr == lease.address() {
// Nope, the port forwarding is up to date
continue;
}
// Remove port forwarding since the lease address had changed
host.port_forwarding_remove_rule(port_forwarding.exposed_port.external_port)?;
port_forwarding.forwarding_to_addr = None;
}
// Install new port forwarding
host.port_forwarding_add_rule(
port_forwarding.exposed_port.external_port,
lease.address(),
port_forwarding.exposed_port.internal_port,
)?;
port_forwarding.forwarding_to_addr = Some(lease.address());
}
} else {
// Lease does not exist, remove all port forwardings
self.remove_all_port_forwardings(host)?;
}
Ok(())
}
fn remove_all_port_forwardings(&mut self, host: &mut Host) -> Result<()> {
for port_forwarding in &mut self.port_forwardings {
if port_forwarding.forwarding_to_addr.is_none() {
continue;
}
host.port_forwarding_remove_rule(port_forwarding.exposed_port.external_port)?;
port_forwarding.forwarding_to_addr = None;
}
Ok(())
}
}
+12 -20
View File
@@ -2,13 +2,12 @@ use crate::proxy::udp_packet_helper::UdpPacketHelper;
use crate::proxy::Proxy;
use anyhow::Context;
use anyhow::Result;
use ipnet::Ipv4Net;
use smoltcp::wire::{
ArpPacket, EthernetFrame, EthernetProtocol, IpProtocol, Ipv4Packet, UdpPacket,
};
use std::net::Ipv4Addr;
impl Proxy<'_> {
impl Proxy {
pub(crate) fn process_frame_from_vm(&mut self, frame: EthernetFrame<&[u8]>) -> Result<()> {
if self.allowed_from_vm(&frame).is_none() {
// Block packet by not forwarding it to the host
@@ -48,7 +47,7 @@ impl Proxy<'_> {
let source_protocol_addr = Ipv4Addr::from(source_protocol_addr);
if let Some(lease) = self.dhcp_snooper.lease() {
if lease.valid_ip_source(source_protocol_addr) {
if lease.valid_ip_source(source_protocol_addr.into()) {
return Some(());
}
} else if source_protocol_addr.is_unspecified() {
@@ -59,24 +58,17 @@ impl Proxy<'_> {
}
fn allowed_from_vm_ipv4(&self, ipv4_pkt: Ipv4Packet<&[u8]>) -> Option<()> {
// Have we learned the VM's IP from the DHCP snooping?
// Once we've learned the VM's IP from the DHCP snooping,
// allow all global traffic for that VM's IP
if let Some(lease) = &self.dhcp_snooper.lease() {
// If so, allow all global traffic
let dst_addr = ipv4_pkt.dst_addr();
let dst_is_global = ip_network::IpNetwork::from(dst_addr).is_global();
let ip_net = &ip_network::IpNetwork::from(Ipv4Addr::from(ipv4_pkt.dst_addr().0));
let dst_is_global_or_private =
match &ip_net {
ip_network::IpNetwork::V4(ip_net) => ip_net.is_global() || ip_net.is_private(),
ip_network::IpNetwork::V6(ip_net) => ip_net.is_global(),
};
if lease.valid_ip_source(ipv4_pkt.src_addr()) && dst_is_global {
return Some(());
}
// Also allow all traffic to the user-specified CIDRs
let dst_net = Ipv4Net::from(dst_addr);
// Use get_lpm() instead of get_spm() to work around prefix-trie
// not handling prefixes like 0.0.0.0/0 correctly[1]
//
// [1]: https://github.com/tiborschneider/prefix-trie/issues/8
if self.allow.get_lpm(&dst_net).is_some() {
if lease.valid_ip_source(ipv4_pkt.src_addr()) && dst_is_global_or_private {
return Some(());
}
}
@@ -86,7 +78,7 @@ impl Proxy<'_> {
return Some(());
}
if ipv4_pkt.next_header() == IpProtocol::Udp {
if ipv4_pkt.protocol() == IpProtocol::Udp {
let udp_pkt = UdpPacket::new_checked(ipv4_pkt.payload()).ok()?;
// Allow DNS communication with the DNS-servers provided by DHCP
+1
View File
@@ -0,0 +1 @@
nightly
-2
View File
@@ -1,2 +0,0 @@
[toolchain]
channel = "nightly"
+4 -38
View File
@@ -1,14 +1,9 @@
use anyhow::{anyhow, Context};
use clap::Parser;
use ipnet::Ipv4Net;
use log::LevelFilter;
use nix::sys::signal::{signal, SigHandler, Signal};
use oslog::OsLogger;
use prefix_trie::PrefixSet;
use privdrop::PrivDrop;
use softnet::proxy::ExposedPort;
use softnet::proxy::Proxy;
use softnet::NetType;
use softnet::proxy::Proxy;
use std::borrow::Cow;
use std::env;
use std::os::raw::c_int;
@@ -34,7 +29,7 @@ struct Args {
#[clap(long, help = "MAC address to enforce for the VM")]
vm_mac_address: mac_address::MacAddress,
#[clap(long, value_enum, help = "type of network to use for the VM", default_value_t=NetType::Nat)]
#[clap(long, arg_enum, help = "type of network to use for the VM", default_value_t=NetType::Nat)]
vm_net_type: NetType,
#[clap(
@@ -50,24 +45,6 @@ struct Args {
#[clap(long, help = "group name to drop privileges to")]
group: Option<String>,
#[clap(
long,
help = "comma-separated list of CIDRs to allow the traffic to (e.g. --allow=192.168.0.0/24)",
value_name = "comma-separated CIDRs",
use_value_delimiter = true,
action = clap::ArgAction::Set
)]
allow: Vec<Ipv4Net>,
#[clap(
long,
help = "comma-separated list of TCP ports to expose (e.g. --expose 2222:22,8080:80)",
value_name = "comma-separated port specifications",
use_value_delimiter = true,
action = clap::ArgAction::Set
)]
expose: Vec<ExposedPort>,
#[clap(long, hide = true)]
sudo_escalation_probing: bool,
@@ -112,11 +89,6 @@ fn main() -> ExitCode {
}
fn try_main() -> anyhow::Result<()> {
// Initialize logger
OsLogger::new("org.cirruslabs.softnet")
.level_filter(LevelFilter::Info)
.init()?;
// The default signal(3)[1] action for SIGINT is to interrupt program,
// but we want to handle SIGINT ourselves, so we ignore it. The kqueue(2)'s[2]
// EVFILT_SIGNAL will receive it anyways, because it has lower precedence.
@@ -172,14 +144,8 @@ fn try_main() -> anyhow::Result<()> {
set_bootpd_lease_time(args.bootpd_lease_time);
// Initialize the proxy while still having the root privileges
let mut proxy = Proxy::new(
args.vm_fd as RawFd,
args.vm_mac_address,
args.vm_net_type,
PrefixSet::from_iter(args.allow),
args.expose,
)
.context("failed to initialize proxy")?;
let mut proxy = Proxy::new(args.vm_fd as RawFd, args.vm_mac_address, args.vm_net_type)
.context("failed to initialize proxy")?;
// Drop effective privileges to the user
// and group which have had invoked us