1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
use std::collections::HashMap;
use std::io::ErrorKind;
use std::net::SocketAddr;
use std::sync::{Arc, Weak};

use async_trait::async_trait;
use tokio::sync::{watch, Mutex};
use util::sync::RwLock;
use util::{Conn, Error};

mod udp_mux_conn;
pub use udp_mux_conn::{UDPMuxConn, UDPMuxConnParams, UDPMuxWriter};

#[cfg(test)]
mod udp_mux_test;

mod socket_addr_ext;

use stun::attributes::ATTR_USERNAME;
use stun::message::{is_message as is_stun_message, Message as STUNMessage};

use crate::candidate::RECEIVE_MTU;

/// Normalize a target socket addr for sending over a given local socket addr. This is useful when
/// a dual stack socket is used, in which case an IPv4 target needs to be mapped to an IPv6
/// address.
fn normalize_socket_addr(target: &SocketAddr, socket_addr: &SocketAddr) -> SocketAddr {
    match (target, socket_addr) {
        (SocketAddr::V4(target_ipv4), SocketAddr::V6(_)) => {
            let ipv6_mapped = target_ipv4.ip().to_ipv6_mapped();

            SocketAddr::new(std::net::IpAddr::V6(ipv6_mapped), target_ipv4.port())
        }
        // This will fail later if target is IPv6 and socket is IPv4, we ignore it here
        (_, _) => *target,
    }
}

#[async_trait]
pub trait UDPMux {
    /// Close the muxing.
    async fn close(&self) -> Result<(), Error>;

    /// Get the underlying connection for a given ufrag.
    async fn get_conn(self: Arc<Self>, ufrag: &str) -> Result<Arc<dyn Conn + Send + Sync>, Error>;

    /// Remove the underlying connection for a given ufrag.
    async fn remove_conn_by_ufrag(&self, ufrag: &str);
}

pub struct UDPMuxParams {
    conn: Box<dyn Conn + Send + Sync>,
}

impl UDPMuxParams {
    pub fn new<C>(conn: C) -> Self
    where
        C: Conn + Send + Sync + 'static,
    {
        Self {
            conn: Box::new(conn),
        }
    }
}

pub struct UDPMuxDefault {
    /// The params this instance is configured with.
    /// Contains the underlying UDP socket in use
    params: UDPMuxParams,

    /// Maps from ufrag to the underlying connection.
    conns: Mutex<HashMap<String, UDPMuxConn>>,

    /// Maps from ip address to the underlying connection.
    address_map: RwLock<HashMap<SocketAddr, UDPMuxConn>>,

    // Close sender
    closed_watch_tx: Mutex<Option<watch::Sender<()>>>,

    /// Close receiver
    closed_watch_rx: watch::Receiver<()>,
}

impl UDPMuxDefault {
    pub fn new(params: UDPMuxParams) -> Arc<Self> {
        let (closed_watch_tx, closed_watch_rx) = watch::channel(());

        let mux = Arc::new(Self {
            params,
            conns: Mutex::default(),
            address_map: RwLock::default(),
            closed_watch_tx: Mutex::new(Some(closed_watch_tx)),
            closed_watch_rx: closed_watch_rx.clone(),
        });

        let cloned_mux = Arc::clone(&mux);
        cloned_mux.start_conn_worker(closed_watch_rx);

        mux
    }

    pub async fn is_closed(&self) -> bool {
        self.closed_watch_tx.lock().await.is_none()
    }

    /// Create a muxed connection for a given ufrag.
    fn create_muxed_conn(self: &Arc<Self>, ufrag: &str) -> Result<UDPMuxConn, Error> {
        let local_addr = self.params.conn.local_addr()?;

        let params = UDPMuxConnParams {
            local_addr,
            key: ufrag.into(),
            udp_mux: Arc::downgrade(self) as Weak<dyn UDPMuxWriter + Send + Sync>,
        };

        Ok(UDPMuxConn::new(params))
    }

    async fn conn_from_stun_message(&self, buffer: &[u8], addr: &SocketAddr) -> Option<UDPMuxConn> {
        let (result, message) = {
            let mut m = STUNMessage::new();

            (m.unmarshal_binary(buffer), m)
        };

        match result {
            Err(err) => {
                log::warn!("Failed to handle decode ICE from {}: {}", addr, err);
                None
            }
            Ok(_) => {
                let (attr, found) = message.attributes.get(ATTR_USERNAME);
                if !found {
                    log::warn!("No username attribute in STUN message from {}", &addr);
                    return None;
                }

                let s = match String::from_utf8(attr.value) {
                    // Per the RFC this shouldn't happen
                    // https://datatracker.ietf.org/doc/html/rfc5389#section-15.3
                    Err(err) => {
                        log::warn!(
                            "Failed to decode USERNAME from STUN message as UTF-8: {}",
                            err
                        );
                        return None;
                    }
                    Ok(s) => s,
                };

                let conns = self.conns.lock().await;
                let conn = s
                    .split(':')
                    .next()
                    .and_then(|ufrag| conns.get(ufrag))
                    .cloned();

                conn
            }
        }
    }

    fn start_conn_worker(self: Arc<Self>, mut closed_watch_rx: watch::Receiver<()>) {
        tokio::spawn(async move {
            let mut buffer = [0u8; RECEIVE_MTU];

            loop {
                let loop_self = Arc::clone(&self);
                let conn = &loop_self.params.conn;

                tokio::select! {
                    res = conn.recv_from(&mut buffer) => {
                        match res {
                            Ok((len, addr)) => {
                                // Find connection based on previously having seen this source address
                                let conn = {
                                    let address_map = loop_self
                                        .address_map
                                        .read();

                                    address_map.get(&addr).cloned()
                                };

                                let conn = match conn {
                                    // If we couldn't find the connection based on source address, see if
                                    // this is a STUN message and if so if we can find the connection based on ufrag.
                                    None if is_stun_message(&buffer) => {
                                        loop_self.conn_from_stun_message(&buffer, &addr).await
                                    }
                                    s @ Some(_) => s,
                                    _ => None,
                                };

                                match conn {
                                    None => {
                                        log::trace!("Dropping packet from {}", &addr);
                                    }
                                    Some(conn) => {
                                        if let Err(err) = conn.write_packet(&buffer[..len], addr).await {
                                            log::error!("Failed to write packet: {}", err);
                                        }
                                    }
                                }
                            }
                            Err(Error::Io(err)) if err.0.kind() == ErrorKind::TimedOut => continue,
                            Err(err) => {
                                log::error!("Could not read udp packet: {}", err);
                                break;
                            }
                        }
                    }
                    _ = closed_watch_rx.changed() => {
                        return;
                    }
                }
            }
        });
    }
}

#[async_trait]
impl UDPMux for UDPMuxDefault {
    async fn close(&self) -> Result<(), Error> {
        if self.is_closed().await {
            return Err(Error::ErrAlreadyClosed);
        }

        let mut closed_tx = self.closed_watch_tx.lock().await;

        if let Some(tx) = closed_tx.take() {
            let _ = tx.send(());
            drop(closed_tx);

            let old_conns = {
                let mut conns = self.conns.lock().await;

                std::mem::take(&mut (*conns))
            };

            // NOTE: We don't wait for these closure to complete
            for (_, conn) in old_conns {
                conn.close();
            }

            {
                let mut address_map = self.address_map.write();

                // NOTE: This is important, we need to drop all instances of `UDPMuxConn` to
                // avoid a retain cycle due to the use of [`std::sync::Arc`] on both sides.
                let _ = std::mem::take(&mut (*address_map));
            }
        }

        Ok(())
    }

    async fn get_conn(self: Arc<Self>, ufrag: &str) -> Result<Arc<dyn Conn + Send + Sync>, Error> {
        if self.is_closed().await {
            return Err(Error::ErrUseClosedNetworkConn);
        }

        {
            let mut conns = self.conns.lock().await;
            if let Some(conn) = conns.get(ufrag) {
                // UDPMuxConn uses `Arc` internally so it's cheap to clone, but because
                // we implement `Conn` we need to further wrap it in an `Arc` here.
                return Ok(Arc::new(conn.clone()) as Arc<dyn Conn + Send + Sync>);
            }

            let muxed_conn = self.create_muxed_conn(ufrag)?;
            let mut close_rx = muxed_conn.close_rx();
            let cloned_self = Arc::clone(&self);
            let cloned_ufrag = ufrag.to_string();
            tokio::spawn(async move {
                let _ = close_rx.changed().await;

                // Arc needed
                cloned_self.remove_conn_by_ufrag(&cloned_ufrag).await;
            });

            conns.insert(ufrag.into(), muxed_conn.clone());

            Ok(Arc::new(muxed_conn) as Arc<dyn Conn + Send + Sync>)
        }
    }

    async fn remove_conn_by_ufrag(&self, ufrag: &str) {
        // Pion's ice implementation has both `RemoveConnByFrag` and `RemoveConn`, but since `conns`
        // is keyed on `ufrag` their implementation is equivalent.

        let removed_conn = {
            let mut conns = self.conns.lock().await;
            conns.remove(ufrag)
        };

        if let Some(conn) = removed_conn {
            let mut address_map = self.address_map.write();

            for address in conn.get_addresses() {
                address_map.remove(&address);
            }
        }
    }
}

#[async_trait]
impl UDPMuxWriter for UDPMuxDefault {
    async fn register_conn_for_address(&self, conn: &UDPMuxConn, addr: SocketAddr) {
        if self.is_closed().await {
            return;
        }

        let key = conn.key();
        {
            let mut addresses = self.address_map.write();

            addresses
                .entry(addr)
                .and_modify(|e| {
                    if e.key() != key {
                        e.remove_address(&addr);
                        *e = conn.clone();
                    }
                })
                .or_insert_with(|| conn.clone());
        }

        log::debug!("Registered {} for {}", addr, key);
    }

    async fn send_to(&self, buf: &[u8], target: &SocketAddr) -> Result<usize, Error> {
        self.params
            .conn
            .send_to(buf, *target)
            .await
            .map_err(Into::into)
    }
}