solana_perf/
sigverify.rs

1//! The `sigverify` module provides digital signature verification functions.
2//! By default, signatures are verified in parallel using all available CPU
3//! cores.  When perf-libs are available signature verification is offloaded
4//! to the GPU.
5//!
6use {
7    crate::{
8        cuda_runtime::PinnedVec,
9        packet::{Packet, PacketBatch, PacketFlags, PACKET_DATA_SIZE},
10        perf_libs,
11        recycler::Recycler,
12    },
13    rayon::{prelude::*, ThreadPool},
14    solana_rayon_threadlimit::get_thread_count,
15    solana_sdk::{
16        hash::Hash,
17        message::{MESSAGE_HEADER_LENGTH, MESSAGE_VERSION_PREFIX},
18        pubkey::Pubkey,
19        signature::Signature,
20    },
21    solana_short_vec::decode_shortu16_len,
22    std::{convert::TryFrom, mem::size_of},
23};
24
25// Representing key tKeYE4wtowRb8yRroZShTipE18YVnqwXjsSAoNsFU6g
26const TRACER_KEY_BYTES: [u8; 32] = [
27    13, 37, 180, 170, 252, 137, 36, 194, 183, 143, 161, 193, 201, 207, 211, 23, 189, 93, 33, 110,
28    155, 90, 30, 39, 116, 115, 238, 38, 126, 21, 232, 133,
29];
30const TRACER_KEY: Pubkey = Pubkey::new_from_array(TRACER_KEY_BYTES);
31const TRACER_KEY_OFFSET_IN_TRANSACTION: usize = 69;
32// Empirically derived to constrain max verify latency to ~8ms at lower packet counts
33pub const VERIFY_PACKET_CHUNK_SIZE: usize = 128;
34
35lazy_static! {
36    static ref PAR_THREAD_POOL: ThreadPool = rayon::ThreadPoolBuilder::new()
37        .num_threads(get_thread_count())
38        .thread_name(|i| format!("solSigVerify{i:02}"))
39        .build()
40        .unwrap();
41}
42
43pub type TxOffset = PinnedVec<u32>;
44
45type TxOffsets = (TxOffset, TxOffset, TxOffset, TxOffset, Vec<Vec<u32>>);
46
47#[derive(Debug, PartialEq, Eq)]
48struct PacketOffsets {
49    pub sig_len: u32,
50    pub sig_start: u32,
51    pub msg_start: u32,
52    pub pubkey_start: u32,
53    pub pubkey_len: u32,
54}
55
56impl PacketOffsets {
57    pub fn new(
58        sig_len: u32,
59        sig_start: u32,
60        msg_start: u32,
61        pubkey_start: u32,
62        pubkey_len: u32,
63    ) -> Self {
64        Self {
65            sig_len,
66            sig_start,
67            msg_start,
68            pubkey_start,
69            pubkey_len,
70        }
71    }
72}
73
74#[derive(Debug, PartialEq, Eq)]
75pub enum PacketError {
76    InvalidLen,
77    InvalidPubkeyLen,
78    InvalidShortVec,
79    InvalidSignatureLen,
80    MismatchSignatureLen,
81    PayerNotWritable,
82    InvalidProgramIdIndex,
83    InvalidProgramLen,
84    UnsupportedVersion,
85}
86
87impl std::convert::From<std::boxed::Box<bincode::ErrorKind>> for PacketError {
88    fn from(_e: std::boxed::Box<bincode::ErrorKind>) -> PacketError {
89        PacketError::InvalidShortVec
90    }
91}
92
93impl std::convert::From<std::num::TryFromIntError> for PacketError {
94    fn from(_e: std::num::TryFromIntError) -> Self {
95        Self::InvalidLen
96    }
97}
98
99pub fn init() {
100    if let Some(api) = perf_libs::api() {
101        unsafe {
102            (api.ed25519_set_verbose)(true);
103            assert!((api.ed25519_init)(), "ed25519_init() failed");
104            (api.ed25519_set_verbose)(false);
105        }
106    }
107}
108
109/// Returns true if the signatrue on the packet verifies.
110/// Caller must do packet.set_discard(true) if this returns false.
111#[must_use]
112fn verify_packet(packet: &mut Packet, reject_non_vote: bool) -> bool {
113    // If this packet was already marked as discard, drop it
114    if packet.meta().discard() {
115        return false;
116    }
117
118    let packet_offsets = get_packet_offsets(packet, 0, reject_non_vote);
119    let mut sig_start = packet_offsets.sig_start as usize;
120    let mut pubkey_start = packet_offsets.pubkey_start as usize;
121    let msg_start = packet_offsets.msg_start as usize;
122
123    if packet_offsets.sig_len == 0 {
124        return false;
125    }
126
127    if packet.meta().size <= msg_start {
128        return false;
129    }
130
131    for _ in 0..packet_offsets.sig_len {
132        let pubkey_end = pubkey_start.saturating_add(size_of::<Pubkey>());
133        let Some(sig_end) = sig_start.checked_add(size_of::<Signature>()) else {
134            return false;
135        };
136        let Some(Ok(signature)) = packet.data(sig_start..sig_end).map(Signature::try_from) else {
137            return false;
138        };
139        let Some(pubkey) = packet.data(pubkey_start..pubkey_end) else {
140            return false;
141        };
142        let Some(message) = packet.data(msg_start..) else {
143            return false;
144        };
145        if !signature.verify(pubkey, message) {
146            return false;
147        }
148        pubkey_start = pubkey_end;
149        sig_start = sig_end;
150    }
151    true
152}
153
154pub fn count_packets_in_batches(batches: &[PacketBatch]) -> usize {
155    batches.iter().map(|batch| batch.len()).sum()
156}
157
158pub fn count_valid_packets(
159    batches: &[PacketBatch],
160    mut process_valid_packet: impl FnMut(&Packet),
161) -> usize {
162    batches
163        .iter()
164        .map(|batch| {
165            batch
166                .iter()
167                .filter(|p| {
168                    let should_keep = !p.meta().discard();
169                    if should_keep {
170                        process_valid_packet(p);
171                    }
172                    should_keep
173                })
174                .count()
175        })
176        .sum()
177}
178
179pub fn count_discarded_packets(batches: &[PacketBatch]) -> usize {
180    batches
181        .iter()
182        .map(|batch| batch.iter().filter(|p| p.meta().discard()).count())
183        .sum()
184}
185
186// internal function to be unit-tested; should be used only by get_packet_offsets
187fn do_get_packet_offsets(
188    packet: &Packet,
189    current_offset: usize,
190) -> Result<PacketOffsets, PacketError> {
191    // should have at least 1 signature and sig lengths
192    let _ = 1usize
193        .checked_add(size_of::<Signature>())
194        .filter(|v| *v <= packet.meta().size)
195        .ok_or(PacketError::InvalidLen)?;
196
197    // read the length of Transaction.signatures (serialized with short_vec)
198    let (sig_len_untrusted, sig_size) = packet
199        .data(..)
200        .and_then(|bytes| decode_shortu16_len(bytes).ok())
201        .ok_or(PacketError::InvalidShortVec)?;
202    // Using msg_start_offset which is based on sig_len_untrusted introduces uncertainty.
203    // Ultimately, the actual sigverify will determine the uncertainty.
204    let msg_start_offset = sig_len_untrusted
205        .checked_mul(size_of::<Signature>())
206        .and_then(|v| v.checked_add(sig_size))
207        .ok_or(PacketError::InvalidLen)?;
208
209    // Determine the start of the message header by checking the message prefix bit.
210    let msg_header_offset = {
211        // Packet should have data for prefix bit
212        if msg_start_offset >= packet.meta().size {
213            return Err(PacketError::InvalidSignatureLen);
214        }
215
216        // next byte indicates if the transaction is versioned. If the top bit
217        // is set, the remaining bits encode a version number. If the top bit is
218        // not set, this byte is the first byte of the message header.
219        let message_prefix = *packet
220            .data(msg_start_offset)
221            .ok_or(PacketError::InvalidSignatureLen)?;
222        if message_prefix & MESSAGE_VERSION_PREFIX != 0 {
223            let version = message_prefix & !MESSAGE_VERSION_PREFIX;
224            match version {
225                0 => {
226                    // header begins immediately after prefix byte
227                    msg_start_offset
228                        .checked_add(1)
229                        .ok_or(PacketError::InvalidLen)?
230                }
231
232                // currently only v0 is supported
233                _ => return Err(PacketError::UnsupportedVersion),
234            }
235        } else {
236            msg_start_offset
237        }
238    };
239
240    let msg_header_offset_plus_one = msg_header_offset
241        .checked_add(1)
242        .ok_or(PacketError::InvalidLen)?;
243
244    // Packet should have data at least for MessageHeader and 1 byte for Message.account_keys.len
245    let _ = msg_header_offset_plus_one
246        .checked_add(MESSAGE_HEADER_LENGTH)
247        .filter(|v| *v <= packet.meta().size)
248        .ok_or(PacketError::InvalidSignatureLen)?;
249
250    // read MessageHeader.num_required_signatures (serialized with u8)
251    let sig_len_maybe_trusted = *packet
252        .data(msg_header_offset)
253        .ok_or(PacketError::InvalidSignatureLen)?;
254    let message_account_keys_len_offset = msg_header_offset
255        .checked_add(MESSAGE_HEADER_LENGTH)
256        .ok_or(PacketError::InvalidSignatureLen)?;
257
258    // This reads and compares the MessageHeader num_required_signatures and
259    // num_readonly_signed_accounts bytes. If num_required_signatures is not larger than
260    // num_readonly_signed_accounts, the first account is not debitable, and cannot be charged
261    // required transaction fees.
262    let readonly_signer_offset = msg_header_offset_plus_one;
263    if sig_len_maybe_trusted
264        <= *packet
265            .data(readonly_signer_offset)
266            .ok_or(PacketError::InvalidSignatureLen)?
267    {
268        return Err(PacketError::PayerNotWritable);
269    }
270
271    if usize::from(sig_len_maybe_trusted) != sig_len_untrusted {
272        return Err(PacketError::MismatchSignatureLen);
273    }
274
275    // read the length of Message.account_keys (serialized with short_vec)
276    let (pubkey_len, pubkey_len_size) = packet
277        .data(message_account_keys_len_offset..)
278        .and_then(|bytes| decode_shortu16_len(bytes).ok())
279        .ok_or(PacketError::InvalidShortVec)?;
280    let pubkey_start = message_account_keys_len_offset
281        .checked_add(pubkey_len_size)
282        .ok_or(PacketError::InvalidPubkeyLen)?;
283
284    let _ = pubkey_len
285        .checked_mul(size_of::<Pubkey>())
286        .and_then(|v| v.checked_add(pubkey_start))
287        .filter(|v| *v <= packet.meta().size)
288        .ok_or(PacketError::InvalidPubkeyLen)?;
289
290    if pubkey_len < sig_len_untrusted {
291        return Err(PacketError::InvalidPubkeyLen);
292    }
293
294    let sig_start = current_offset
295        .checked_add(sig_size)
296        .ok_or(PacketError::InvalidLen)?;
297    let msg_start = current_offset
298        .checked_add(msg_start_offset)
299        .ok_or(PacketError::InvalidLen)?;
300    let pubkey_start = current_offset
301        .checked_add(pubkey_start)
302        .ok_or(PacketError::InvalidLen)?;
303
304    Ok(PacketOffsets::new(
305        u32::try_from(sig_len_untrusted)?,
306        u32::try_from(sig_start)?,
307        u32::try_from(msg_start)?,
308        u32::try_from(pubkey_start)?,
309        u32::try_from(pubkey_len)?,
310    ))
311}
312
313pub fn check_for_tracer_packet(packet: &mut Packet) -> bool {
314    let first_pubkey_start: usize = TRACER_KEY_OFFSET_IN_TRANSACTION;
315    let Some(first_pubkey_end) = first_pubkey_start.checked_add(size_of::<Pubkey>()) else {
316        return false;
317    };
318    // Check for tracer pubkey
319    match packet.data(first_pubkey_start..first_pubkey_end) {
320        Some(pubkey) if pubkey == TRACER_KEY.as_ref() => {
321            packet.meta_mut().set_tracer(true);
322            true
323        }
324        _ => false,
325    }
326}
327
328fn get_packet_offsets(
329    packet: &mut Packet,
330    current_offset: usize,
331    reject_non_vote: bool,
332) -> PacketOffsets {
333    let unsanitized_packet_offsets = do_get_packet_offsets(packet, current_offset);
334    if let Ok(offsets) = unsanitized_packet_offsets {
335        check_for_simple_vote_transaction(packet, &offsets, current_offset).ok();
336        if !reject_non_vote || packet.meta().is_simple_vote_tx() {
337            return offsets;
338        }
339    }
340    // force sigverify to fail by returning zeros
341    PacketOffsets::new(0, 0, 0, 0, 0)
342}
343
344fn check_for_simple_vote_transaction(
345    packet: &mut Packet,
346    packet_offsets: &PacketOffsets,
347    current_offset: usize,
348) -> Result<(), PacketError> {
349    // vote could have 1 or 2 sigs; zero sig has already been excluded at
350    // do_get_packet_offsets.
351    if packet_offsets.sig_len > 2 {
352        return Err(PacketError::InvalidSignatureLen);
353    }
354
355    // simple vote should only be legacy message
356    let msg_start = (packet_offsets.msg_start as usize)
357        .checked_sub(current_offset)
358        .ok_or(PacketError::InvalidLen)?;
359    let message_prefix = *packet.data(msg_start).ok_or(PacketError::InvalidLen)?;
360    if message_prefix & MESSAGE_VERSION_PREFIX != 0 {
361        return Ok(());
362    }
363
364    let pubkey_start = (packet_offsets.pubkey_start as usize)
365        .checked_sub(current_offset)
366        .ok_or(PacketError::InvalidLen)?;
367
368    let instructions_len_offset = (packet_offsets.pubkey_len as usize)
369        .checked_mul(size_of::<Pubkey>())
370        .and_then(|v| v.checked_add(pubkey_start))
371        .and_then(|v| v.checked_add(size_of::<Hash>()))
372        .ok_or(PacketError::InvalidLen)?;
373
374    // Packet should have at least 1 more byte for instructions.len
375    let _ = instructions_len_offset
376        .checked_add(1usize)
377        .filter(|v| *v <= packet.meta().size)
378        .ok_or(PacketError::InvalidLen)?;
379
380    let (instruction_len, instruction_len_size) = packet
381        .data(instructions_len_offset..)
382        .and_then(|bytes| decode_shortu16_len(bytes).ok())
383        .ok_or(PacketError::InvalidLen)?;
384    // skip if has more than 1 instruction
385    if instruction_len != 1 {
386        return Err(PacketError::InvalidProgramLen);
387    }
388
389    let instruction_start = instructions_len_offset
390        .checked_add(instruction_len_size)
391        .ok_or(PacketError::InvalidLen)?;
392
393    // Packet should have at least 1 more byte for one instructions_program_id
394    let _ = instruction_start
395        .checked_add(1usize)
396        .filter(|v| *v <= packet.meta().size)
397        .ok_or(PacketError::InvalidLen)?;
398
399    let instruction_program_id_index: usize = usize::from(
400        *packet
401            .data(instruction_start)
402            .ok_or(PacketError::InvalidLen)?,
403    );
404
405    if instruction_program_id_index >= packet_offsets.pubkey_len as usize {
406        return Err(PacketError::InvalidProgramIdIndex);
407    }
408
409    let instruction_program_id_start = instruction_program_id_index
410        .checked_mul(size_of::<Pubkey>())
411        .and_then(|v| v.checked_add(pubkey_start))
412        .ok_or(PacketError::InvalidLen)?;
413    let instruction_program_id_end = instruction_program_id_start
414        .checked_add(size_of::<Pubkey>())
415        .ok_or(PacketError::InvalidLen)?;
416
417    if packet
418        .data(instruction_program_id_start..instruction_program_id_end)
419        .ok_or(PacketError::InvalidLen)?
420        == solana_sdk::vote::program::id().as_ref()
421    {
422        packet.meta_mut().flags |= PacketFlags::SIMPLE_VOTE_TX;
423    }
424    Ok(())
425}
426
427pub fn generate_offsets(
428    batches: &mut [PacketBatch],
429    recycler: &Recycler<TxOffset>,
430    reject_non_vote: bool,
431) -> TxOffsets {
432    debug!("allocating..");
433    let mut signature_offsets: PinnedVec<_> = recycler.allocate("sig_offsets");
434    signature_offsets.set_pinnable();
435    let mut pubkey_offsets: PinnedVec<_> = recycler.allocate("pubkey_offsets");
436    pubkey_offsets.set_pinnable();
437    let mut msg_start_offsets: PinnedVec<_> = recycler.allocate("msg_start_offsets");
438    msg_start_offsets.set_pinnable();
439    let mut msg_sizes: PinnedVec<_> = recycler.allocate("msg_size_offsets");
440    msg_sizes.set_pinnable();
441    let mut current_offset: usize = 0;
442    let offsets = batches
443        .iter_mut()
444        .map(|batch| {
445            batch
446                .iter_mut()
447                .map(|packet| {
448                    let packet_offsets =
449                        get_packet_offsets(packet, current_offset, reject_non_vote);
450
451                    trace!("pubkey_offset: {}", packet_offsets.pubkey_start);
452
453                    let mut pubkey_offset = packet_offsets.pubkey_start;
454                    let mut sig_offset = packet_offsets.sig_start;
455                    let msg_size = current_offset.saturating_add(packet.meta().size) as u32;
456                    for _ in 0..packet_offsets.sig_len {
457                        signature_offsets.push(sig_offset);
458                        sig_offset = sig_offset.saturating_add(size_of::<Signature>() as u32);
459
460                        pubkey_offsets.push(pubkey_offset);
461                        pubkey_offset = pubkey_offset.saturating_add(size_of::<Pubkey>() as u32);
462
463                        msg_start_offsets.push(packet_offsets.msg_start);
464
465                        let msg_size = msg_size.saturating_sub(packet_offsets.msg_start);
466                        msg_sizes.push(msg_size);
467                    }
468                    current_offset = current_offset.saturating_add(size_of::<Packet>());
469                    packet_offsets.sig_len
470                })
471                .collect()
472        })
473        .collect();
474    (
475        signature_offsets,
476        pubkey_offsets,
477        msg_start_offsets,
478        msg_sizes,
479        offsets,
480    )
481}
482
483//inplace shrink a batch of packets
484pub fn shrink_batches(batches: &mut Vec<PacketBatch>) {
485    let mut valid_batch_ix = 0;
486    let mut valid_packet_ix = 0;
487    let mut last_valid_batch = 0;
488    for batch_ix in 0..batches.len() {
489        for packet_ix in 0..batches[batch_ix].len() {
490            if batches[batch_ix][packet_ix].meta().discard() {
491                continue;
492            }
493            last_valid_batch = batch_ix.saturating_add(1);
494            let mut found_spot = false;
495            while valid_batch_ix < batch_ix && !found_spot {
496                while valid_packet_ix < batches[valid_batch_ix].len() {
497                    if batches[valid_batch_ix][valid_packet_ix].meta().discard() {
498                        batches[valid_batch_ix][valid_packet_ix] =
499                            batches[batch_ix][packet_ix].clone();
500                        batches[batch_ix][packet_ix].meta_mut().set_discard(true);
501                        last_valid_batch = valid_batch_ix.saturating_add(1);
502                        found_spot = true;
503                        break;
504                    }
505                    valid_packet_ix = valid_packet_ix.saturating_add(1);
506                }
507                if valid_packet_ix >= batches[valid_batch_ix].len() {
508                    valid_packet_ix = 0;
509                    valid_batch_ix = valid_batch_ix.saturating_add(1);
510                }
511            }
512        }
513    }
514    batches.truncate(last_valid_batch);
515}
516
517pub fn ed25519_verify_cpu(batches: &mut [PacketBatch], reject_non_vote: bool, packet_count: usize) {
518    debug!("CPU ECDSA for {}", packet_count);
519    PAR_THREAD_POOL.install(|| {
520        batches.par_iter_mut().flatten().for_each(|packet| {
521            if !packet.meta().discard() && !verify_packet(packet, reject_non_vote) {
522                packet.meta_mut().set_discard(true);
523            }
524        });
525    });
526}
527
528pub fn ed25519_verify_disabled(batches: &mut [PacketBatch]) {
529    let packet_count = count_packets_in_batches(batches);
530    debug!("disabled ECDSA for {}", packet_count);
531    batches.into_par_iter().for_each(|batch| {
532        batch
533            .par_iter_mut()
534            .for_each(|p| p.meta_mut().set_discard(false))
535    });
536}
537
538pub fn copy_return_values<I, T>(sig_lens: I, out: &PinnedVec<u8>, rvs: &mut [Vec<u8>])
539where
540    I: IntoIterator<Item = T>,
541    T: IntoIterator<Item = u32>,
542{
543    debug_assert!(rvs.iter().flatten().all(|&rv| rv == 0u8));
544    let mut offset = 0usize;
545    let rvs = rvs.iter_mut().flatten();
546    for (k, rv) in sig_lens.into_iter().flatten().zip(rvs) {
547        let out = out[offset..].iter().take(k as usize).all(|&x| x == 1u8);
548        *rv = u8::from(k != 0u32 && out);
549        offset = offset.saturating_add(k as usize);
550    }
551}
552
553// return true for success, i.e ge unpacks and !ge.is_small_order()
554pub fn check_packed_ge_small_order(ge: &[u8; 32]) -> bool {
555    if let Some(api) = perf_libs::api() {
556        unsafe {
557            // Returns 1 == fail, 0 == success
558            let res = (api.ed25519_check_packed_ge_small_order)(ge.as_ptr());
559
560            return res == 0;
561        }
562    }
563    false
564}
565
566pub fn get_checked_scalar(scalar: &[u8; 32]) -> Result<[u8; 32], PacketError> {
567    let mut out = [0u8; 32];
568    if let Some(api) = perf_libs::api() {
569        unsafe {
570            let res = (api.ed25519_get_checked_scalar)(out.as_mut_ptr(), scalar.as_ptr());
571            if res == 0 {
572                return Ok(out);
573            } else {
574                return Err(PacketError::InvalidLen);
575            }
576        }
577    }
578    Ok(out)
579}
580
581pub fn mark_disabled(batches: &mut [PacketBatch], r: &[Vec<u8>]) {
582    for (batch, v) in batches.iter_mut().zip(r) {
583        for (pkt, f) in batch.iter_mut().zip(v) {
584            if !pkt.meta().discard() {
585                pkt.meta_mut().set_discard(*f == 0);
586            }
587        }
588    }
589}
590
591pub fn ed25519_verify(
592    batches: &mut [PacketBatch],
593    recycler: &Recycler<TxOffset>,
594    recycler_out: &Recycler<PinnedVec<u8>>,
595    reject_non_vote: bool,
596    valid_packet_count: usize,
597) {
598    let Some(api) = perf_libs::api() else {
599        return ed25519_verify_cpu(batches, reject_non_vote, valid_packet_count);
600    };
601    let total_packet_count = count_packets_in_batches(batches);
602    // micro-benchmarks show GPU time for smallest batch around 15-20ms
603    // and CPU speed for 64-128 sigverifies around 10-20ms. 64 is a nice
604    // power-of-two number around that accounting for the fact that the CPU
605    // may be busy doing other things while being a real validator
606    // TODO: dynamically adjust this crossover
607    let maybe_valid_percentage = 100usize
608        .wrapping_mul(valid_packet_count)
609        .checked_div(total_packet_count);
610    let Some(valid_percentage) = maybe_valid_percentage else {
611        return;
612    };
613    if valid_percentage < 90 || valid_packet_count < 64 {
614        ed25519_verify_cpu(batches, reject_non_vote, valid_packet_count);
615        return;
616    }
617
618    let (signature_offsets, pubkey_offsets, msg_start_offsets, msg_sizes, sig_lens) =
619        generate_offsets(batches, recycler, reject_non_vote);
620
621    debug!("CUDA ECDSA for {}", valid_packet_count);
622    debug!("allocating out..");
623    let mut out = recycler_out.allocate("out_buffer");
624    out.set_pinnable();
625    let mut elems = Vec::new();
626    let mut rvs = Vec::new();
627
628    let mut num_packets: usize = 0;
629    for batch in batches.iter() {
630        elems.push(perf_libs::Elems {
631            elems: batch.as_ptr().cast::<u8>(),
632            num: batch.len() as u32,
633        });
634        let v = vec![0u8; batch.len()];
635        rvs.push(v);
636        num_packets = num_packets.saturating_add(batch.len());
637    }
638    out.resize(signature_offsets.len(), 0);
639    trace!("Starting verify num packets: {}", num_packets);
640    trace!("elem len: {}", elems.len() as u32);
641    trace!("packet sizeof: {}", size_of::<Packet>() as u32);
642    trace!("len offset: {}", PACKET_DATA_SIZE as u32);
643    const USE_NON_DEFAULT_STREAM: u8 = 1;
644    unsafe {
645        let res = (api.ed25519_verify_many)(
646            elems.as_ptr(),
647            elems.len() as u32,
648            size_of::<Packet>() as u32,
649            num_packets as u32,
650            signature_offsets.len() as u32,
651            msg_sizes.as_ptr(),
652            pubkey_offsets.as_ptr(),
653            signature_offsets.as_ptr(),
654            msg_start_offsets.as_ptr(),
655            out.as_mut_ptr(),
656            USE_NON_DEFAULT_STREAM,
657        );
658        if res != 0 {
659            trace!("RETURN!!!: {}", res);
660        }
661    }
662    trace!("done verify");
663    copy_return_values(sig_lens, &out, &mut rvs);
664    mark_disabled(batches, &rvs);
665}
666
667#[cfg(test)]
668#[allow(clippy::arithmetic_side_effects)]
669mod tests {
670    use {
671        super::*,
672        crate::{
673            packet::{to_packet_batches, Packet, PacketBatch, PACKETS_PER_BATCH},
674            sigverify::{self, PacketOffsets},
675            test_tx::{new_test_vote_tx, test_multisig_tx, test_tx},
676        },
677        bincode::{deserialize, serialize},
678        curve25519_dalek::{edwards::CompressedEdwardsY, scalar::Scalar},
679        rand::{thread_rng, Rng},
680        solana_sdk::{
681            instruction::CompiledInstruction,
682            message::{Message, MessageHeader},
683            signature::{Keypair, Signature, Signer},
684            transaction::Transaction,
685        },
686        std::{
687            iter::repeat_with,
688            sync::atomic::{AtomicU64, Ordering},
689        },
690    };
691
692    const SIG_OFFSET: usize = 1;
693
694    pub fn memfind<A: Eq>(a: &[A], b: &[A]) -> Option<usize> {
695        assert!(a.len() >= b.len());
696        let end = a.len() - b.len() + 1;
697        (0..end).find(|&i| a[i..i + b.len()] == b[..])
698    }
699
700    #[test]
701    fn test_copy_return_values() {
702        let mut rng = rand::thread_rng();
703        let sig_lens: Vec<Vec<u32>> = {
704            let size = rng.gen_range(0..64);
705            repeat_with(|| {
706                let size = rng.gen_range(0..16);
707                repeat_with(|| rng.gen_range(0..5)).take(size).collect()
708            })
709            .take(size)
710            .collect()
711        };
712        let out: Vec<Vec<Vec<bool>>> = sig_lens
713            .iter()
714            .map(|sig_lens| {
715                sig_lens
716                    .iter()
717                    .map(|&size| repeat_with(|| rng.gen()).take(size as usize).collect())
718                    .collect()
719            })
720            .collect();
721        let expected: Vec<Vec<u8>> = out
722            .iter()
723            .map(|out| {
724                out.iter()
725                    .map(|out| u8::from(!out.is_empty() && out.iter().all(|&k| k)))
726                    .collect()
727            })
728            .collect();
729        let out =
730            PinnedVec::<u8>::from_vec(out.into_iter().flatten().flatten().map(u8::from).collect());
731        let mut rvs: Vec<Vec<u8>> = sig_lens
732            .iter()
733            .map(|sig_lens| vec![0u8; sig_lens.len()])
734            .collect();
735        copy_return_values(sig_lens, &out, &mut rvs);
736        assert_eq!(rvs, expected);
737    }
738
739    #[test]
740    fn test_mark_disabled() {
741        let batch_size = 1;
742        let mut batch = PacketBatch::with_capacity(batch_size);
743        batch.resize(batch_size, Packet::default());
744        let mut batches: Vec<PacketBatch> = vec![batch];
745        mark_disabled(&mut batches, &[vec![0]]);
746        assert!(batches[0][0].meta().discard());
747        batches[0][0].meta_mut().set_discard(false);
748        mark_disabled(&mut batches, &[vec![1]]);
749        assert!(!batches[0][0].meta().discard());
750    }
751
752    #[test]
753    fn test_layout() {
754        let tx = test_tx();
755        let tx_bytes = serialize(&tx).unwrap();
756        let packet = serialize(&tx).unwrap();
757        assert_matches!(memfind(&packet, &tx_bytes), Some(0));
758        assert_matches!(memfind(&packet, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), None);
759    }
760
761    #[test]
762    fn test_system_transaction_layout() {
763        let tx = test_tx();
764        let tx_bytes = serialize(&tx).unwrap();
765        let message_data = tx.message_data();
766        let mut packet = Packet::from_data(None, tx.clone()).unwrap();
767
768        let packet_offsets = sigverify::get_packet_offsets(&mut packet, 0, false);
769
770        assert_eq!(
771            memfind(&tx_bytes, tx.signatures[0].as_ref()),
772            Some(SIG_OFFSET)
773        );
774        assert_eq!(
775            memfind(&tx_bytes, tx.message().account_keys[0].as_ref()),
776            Some(packet_offsets.pubkey_start as usize)
777        );
778        assert_eq!(
779            memfind(&tx_bytes, &message_data),
780            Some(packet_offsets.msg_start as usize)
781        );
782        assert_eq!(
783            memfind(&tx_bytes, tx.signatures[0].as_ref()),
784            Some(packet_offsets.sig_start as usize)
785        );
786        assert_eq!(packet_offsets.sig_len, 1);
787    }
788
789    fn packet_from_num_sigs(required_num_sigs: u8, actual_num_sigs: usize) -> Packet {
790        let message = Message {
791            header: MessageHeader {
792                num_required_signatures: required_num_sigs,
793                num_readonly_signed_accounts: 12,
794                num_readonly_unsigned_accounts: 11,
795            },
796            account_keys: vec![],
797            recent_blockhash: Hash::default(),
798            instructions: vec![],
799        };
800        let mut tx = Transaction::new_unsigned(message);
801        tx.signatures = vec![Signature::default(); actual_num_sigs];
802        Packet::from_data(None, tx).unwrap()
803    }
804
805    #[test]
806    fn test_untrustworthy_sigs() {
807        let required_num_sigs = 14;
808        let actual_num_sigs = 5;
809
810        let packet = packet_from_num_sigs(required_num_sigs, actual_num_sigs);
811
812        let unsanitized_packet_offsets = sigverify::do_get_packet_offsets(&packet, 0);
813
814        assert_eq!(
815            unsanitized_packet_offsets,
816            Err(PacketError::MismatchSignatureLen)
817        );
818    }
819
820    #[test]
821    fn test_small_packet() {
822        let tx = test_tx();
823        let mut packet = Packet::from_data(None, tx).unwrap();
824
825        packet.buffer_mut()[0] = 0xff;
826        packet.buffer_mut()[1] = 0xff;
827        packet.meta_mut().size = 2;
828
829        let res = sigverify::do_get_packet_offsets(&packet, 0);
830        assert_eq!(res, Err(PacketError::InvalidLen));
831    }
832
833    #[test]
834    fn test_pubkey_too_small() {
835        solana_logger::setup();
836        let mut tx = test_tx();
837        let sig = tx.signatures[0];
838        const NUM_SIG: usize = 18;
839        tx.signatures = vec![sig; NUM_SIG];
840        tx.message.account_keys = vec![];
841        tx.message.header.num_required_signatures = NUM_SIG as u8;
842        let mut packet = Packet::from_data(None, tx).unwrap();
843
844        let res = sigverify::do_get_packet_offsets(&packet, 0);
845        assert_eq!(res, Err(PacketError::InvalidPubkeyLen));
846
847        assert!(!verify_packet(&mut packet, false));
848
849        packet.meta_mut().set_discard(false);
850        let mut batches = generate_packet_batches(&packet, 1, 1);
851        ed25519_verify(&mut batches);
852        assert!(batches[0][0].meta().discard());
853    }
854
855    #[test]
856    fn test_pubkey_len() {
857        // See that the verify cannot walk off the end of the packet
858        // trying to index into the account_keys to access pubkey.
859        solana_logger::setup();
860
861        const NUM_SIG: usize = 17;
862        let keypair1 = Keypair::new();
863        let pubkey1 = keypair1.pubkey();
864        let mut message = Message::new(&[], Some(&pubkey1));
865        message.account_keys.push(pubkey1);
866        message.account_keys.push(pubkey1);
867        message.header.num_required_signatures = NUM_SIG as u8;
868        message.recent_blockhash = Hash::new_from_array(pubkey1.to_bytes());
869        let mut tx = Transaction::new_unsigned(message);
870
871        info!("message: {:?}", tx.message_data());
872        info!("tx: {:?}", tx);
873        let sig = keypair1.try_sign_message(&tx.message_data()).unwrap();
874        tx.signatures = vec![sig; NUM_SIG];
875
876        let mut packet = Packet::from_data(None, tx).unwrap();
877
878        let res = sigverify::do_get_packet_offsets(&packet, 0);
879        assert_eq!(res, Err(PacketError::InvalidPubkeyLen));
880
881        assert!(!verify_packet(&mut packet, false));
882
883        packet.meta_mut().set_discard(false);
884        let mut batches = generate_packet_batches(&packet, 1, 1);
885        ed25519_verify(&mut batches);
886        assert!(batches[0][0].meta().discard());
887    }
888
889    #[test]
890    fn test_large_sig_len() {
891        let tx = test_tx();
892        let mut packet = Packet::from_data(None, tx).unwrap();
893
894        // Make the signatures len huge
895        packet.buffer_mut()[0] = 0x7f;
896
897        let res = sigverify::do_get_packet_offsets(&packet, 0);
898        assert_eq!(res, Err(PacketError::InvalidSignatureLen));
899    }
900
901    #[test]
902    fn test_really_large_sig_len() {
903        let tx = test_tx();
904        let mut packet = Packet::from_data(None, tx).unwrap();
905
906        // Make the signatures len huge
907        packet.buffer_mut()[0] = 0xff;
908        packet.buffer_mut()[1] = 0xff;
909        packet.buffer_mut()[2] = 0xff;
910        packet.buffer_mut()[3] = 0xff;
911
912        let res = sigverify::do_get_packet_offsets(&packet, 0);
913        assert_eq!(res, Err(PacketError::InvalidShortVec));
914    }
915
916    #[test]
917    fn test_invalid_pubkey_len() {
918        let tx = test_tx();
919        let mut packet = Packet::from_data(None, tx).unwrap();
920
921        let res = sigverify::do_get_packet_offsets(&packet, 0);
922
923        // make pubkey len huge
924        packet.buffer_mut()[res.unwrap().pubkey_start as usize - 1] = 0x7f;
925
926        let res = sigverify::do_get_packet_offsets(&packet, 0);
927        assert_eq!(res, Err(PacketError::InvalidPubkeyLen));
928    }
929
930    #[test]
931    fn test_fee_payer_is_debitable() {
932        let message = Message {
933            header: MessageHeader {
934                num_required_signatures: 1,
935                num_readonly_signed_accounts: 1,
936                num_readonly_unsigned_accounts: 1,
937            },
938            account_keys: vec![],
939            recent_blockhash: Hash::default(),
940            instructions: vec![],
941        };
942        let mut tx = Transaction::new_unsigned(message);
943        tx.signatures = vec![Signature::default()];
944        let packet = Packet::from_data(None, tx).unwrap();
945        let res = sigverify::do_get_packet_offsets(&packet, 0);
946
947        assert_eq!(res, Err(PacketError::PayerNotWritable));
948    }
949
950    #[test]
951    fn test_unsupported_version() {
952        let tx = test_tx();
953        let mut packet = Packet::from_data(None, tx).unwrap();
954
955        let res = sigverify::do_get_packet_offsets(&packet, 0);
956
957        // set message version to 1
958        packet.buffer_mut()[res.unwrap().msg_start as usize] = MESSAGE_VERSION_PREFIX + 1;
959
960        let res = sigverify::do_get_packet_offsets(&packet, 0);
961        assert_eq!(res, Err(PacketError::UnsupportedVersion));
962    }
963
964    #[test]
965    fn test_versioned_message() {
966        let tx = test_tx();
967        let mut packet = Packet::from_data(None, tx).unwrap();
968
969        let mut legacy_offsets = sigverify::do_get_packet_offsets(&packet, 0).unwrap();
970
971        // set message version to 0
972        let msg_start = legacy_offsets.msg_start as usize;
973        let msg_bytes = packet.data(msg_start..).unwrap().to_vec();
974        packet.buffer_mut()[msg_start] = MESSAGE_VERSION_PREFIX;
975        packet.meta_mut().size += 1;
976        let msg_end = packet.meta().size;
977        packet.buffer_mut()[msg_start + 1..msg_end].copy_from_slice(&msg_bytes);
978
979        let offsets = sigverify::do_get_packet_offsets(&packet, 0).unwrap();
980        let expected_offsets = {
981            legacy_offsets.pubkey_start += 1;
982            legacy_offsets
983        };
984
985        assert_eq!(expected_offsets, offsets);
986    }
987
988    #[test]
989    fn test_system_transaction_data_layout() {
990        let mut tx0 = test_tx();
991        tx0.message.instructions[0].data = vec![1, 2, 3];
992        let message0a = tx0.message_data();
993        let tx_bytes = serialize(&tx0).unwrap();
994        assert!(tx_bytes.len() <= PACKET_DATA_SIZE);
995        assert_eq!(
996            memfind(&tx_bytes, tx0.signatures[0].as_ref()),
997            Some(SIG_OFFSET)
998        );
999        let tx1 = deserialize(&tx_bytes).unwrap();
1000        assert_eq!(tx0, tx1);
1001        assert_eq!(tx1.message().instructions[0].data, vec![1, 2, 3]);
1002
1003        tx0.message.instructions[0].data = vec![1, 2, 4];
1004        let message0b = tx0.message_data();
1005        assert_ne!(message0a, message0b);
1006    }
1007
1008    // Just like get_packet_offsets, but not returning redundant information.
1009    fn get_packet_offsets_from_tx(tx: Transaction, current_offset: u32) -> PacketOffsets {
1010        let mut packet = Packet::from_data(None, tx).unwrap();
1011        let packet_offsets =
1012            sigverify::get_packet_offsets(&mut packet, current_offset as usize, false);
1013        PacketOffsets::new(
1014            packet_offsets.sig_len,
1015            packet_offsets.sig_start - current_offset,
1016            packet_offsets.msg_start - packet_offsets.sig_start,
1017            packet_offsets.pubkey_start - packet_offsets.msg_start,
1018            packet_offsets.pubkey_len,
1019        )
1020    }
1021
1022    #[test]
1023    fn test_get_packet_offsets() {
1024        assert_eq!(
1025            get_packet_offsets_from_tx(test_tx(), 0),
1026            PacketOffsets::new(1, 1, 64, 4, 2)
1027        );
1028        assert_eq!(
1029            get_packet_offsets_from_tx(test_tx(), 100),
1030            PacketOffsets::new(1, 1, 64, 4, 2)
1031        );
1032
1033        // Ensure we're not indexing packet by the `current_offset` parameter.
1034        assert_eq!(
1035            get_packet_offsets_from_tx(test_tx(), 1_000_000),
1036            PacketOffsets::new(1, 1, 64, 4, 2)
1037        );
1038
1039        // Ensure we're returning sig_len, not sig_size.
1040        assert_eq!(
1041            get_packet_offsets_from_tx(test_multisig_tx(), 0),
1042            PacketOffsets::new(2, 1, 128, 4, 4)
1043        );
1044    }
1045
1046    fn generate_packet_batches_random_size(
1047        packet: &Packet,
1048        max_packets_per_batch: usize,
1049        num_batches: usize,
1050    ) -> Vec<PacketBatch> {
1051        // generate packet vector
1052        let batches: Vec<_> = (0..num_batches)
1053            .map(|_| {
1054                let num_packets_per_batch = thread_rng().gen_range(1..max_packets_per_batch);
1055                let mut packet_batch = PacketBatch::with_capacity(num_packets_per_batch);
1056                for _ in 0..num_packets_per_batch {
1057                    packet_batch.push(packet.clone());
1058                }
1059                assert_eq!(packet_batch.len(), num_packets_per_batch);
1060                packet_batch
1061            })
1062            .collect();
1063        assert_eq!(batches.len(), num_batches);
1064
1065        batches
1066    }
1067
1068    fn generate_packet_batches(
1069        packet: &Packet,
1070        num_packets_per_batch: usize,
1071        num_batches: usize,
1072    ) -> Vec<PacketBatch> {
1073        // generate packet vector
1074        let batches: Vec<_> = (0..num_batches)
1075            .map(|_| {
1076                let mut packet_batch = PacketBatch::with_capacity(num_packets_per_batch);
1077                for _ in 0..num_packets_per_batch {
1078                    packet_batch.push(packet.clone());
1079                }
1080                assert_eq!(packet_batch.len(), num_packets_per_batch);
1081                packet_batch
1082            })
1083            .collect();
1084        assert_eq!(batches.len(), num_batches);
1085
1086        batches
1087    }
1088
1089    fn test_verify_n(n: usize, modify_data: bool) {
1090        let tx = test_tx();
1091        let mut packet = Packet::from_data(None, tx).unwrap();
1092
1093        // jumble some data to test failure
1094        if modify_data {
1095            packet.buffer_mut()[20] = packet.data(20).unwrap().wrapping_add(10);
1096        }
1097
1098        let mut batches = generate_packet_batches(&packet, n, 2);
1099
1100        // verify packets
1101        ed25519_verify(&mut batches);
1102
1103        // check result
1104        let should_discard = modify_data;
1105        assert!(batches
1106            .iter()
1107            .flat_map(|batch| batch.iter())
1108            .all(|p| p.meta().discard() == should_discard));
1109    }
1110
1111    fn ed25519_verify(batches: &mut [PacketBatch]) {
1112        let recycler = Recycler::default();
1113        let recycler_out = Recycler::default();
1114        let packet_count = sigverify::count_packets_in_batches(batches);
1115        sigverify::ed25519_verify(batches, &recycler, &recycler_out, false, packet_count);
1116    }
1117
1118    #[test]
1119    fn test_verify_tampered_sig_len() {
1120        let mut tx = test_tx();
1121        // pretend malicious leader dropped a signature...
1122        tx.signatures.pop();
1123        let packet = Packet::from_data(None, tx).unwrap();
1124
1125        let mut batches = generate_packet_batches(&packet, 1, 1);
1126
1127        // verify packets
1128        ed25519_verify(&mut batches);
1129        assert!(batches
1130            .iter()
1131            .flat_map(|batch| batch.iter())
1132            .all(|p| p.meta().discard()));
1133    }
1134
1135    #[test]
1136    fn test_verify_zero() {
1137        test_verify_n(0, false);
1138    }
1139
1140    #[test]
1141    fn test_verify_one() {
1142        test_verify_n(1, false);
1143    }
1144
1145    #[test]
1146    fn test_verify_seventy_one() {
1147        test_verify_n(71, false);
1148    }
1149
1150    #[test]
1151    fn test_verify_medium_pass() {
1152        test_verify_n(VERIFY_PACKET_CHUNK_SIZE, false);
1153    }
1154
1155    #[test]
1156    fn test_verify_large_pass() {
1157        test_verify_n(VERIFY_PACKET_CHUNK_SIZE * get_thread_count(), false);
1158    }
1159
1160    #[test]
1161    fn test_verify_medium_fail() {
1162        test_verify_n(VERIFY_PACKET_CHUNK_SIZE, true);
1163    }
1164
1165    #[test]
1166    fn test_verify_large_fail() {
1167        test_verify_n(VERIFY_PACKET_CHUNK_SIZE * get_thread_count(), true);
1168    }
1169
1170    #[test]
1171    fn test_verify_multisig() {
1172        solana_logger::setup();
1173
1174        let tx = test_multisig_tx();
1175        let mut packet = Packet::from_data(None, tx).unwrap();
1176
1177        let n = 4;
1178        let num_batches = 3;
1179        let mut batches = generate_packet_batches(&packet, n, num_batches);
1180
1181        packet.buffer_mut()[40] = packet.data(40).unwrap().wrapping_add(8);
1182
1183        batches[0].push(packet);
1184
1185        // verify packets
1186        ed25519_verify(&mut batches);
1187
1188        // check result
1189        let ref_ans = 1u8;
1190        let mut ref_vec = vec![vec![ref_ans; n]; num_batches];
1191        ref_vec[0].push(0u8);
1192        assert!(batches
1193            .iter()
1194            .flat_map(|batch| batch.iter())
1195            .zip(ref_vec.into_iter().flatten())
1196            .all(|(p, discard)| {
1197                if discard == 0 {
1198                    p.meta().discard()
1199                } else {
1200                    !p.meta().discard()
1201                }
1202            }));
1203    }
1204
1205    #[test]
1206    fn test_verify_fuzz() {
1207        solana_logger::setup();
1208
1209        let tx = test_multisig_tx();
1210        let packet = Packet::from_data(None, tx).unwrap();
1211
1212        let recycler = Recycler::default();
1213        let recycler_out = Recycler::default();
1214        for _ in 0..50 {
1215            let num_batches = thread_rng().gen_range(2..30);
1216            let mut batches = generate_packet_batches_random_size(&packet, 128, num_batches);
1217
1218            let num_modifications = thread_rng().gen_range(0..5);
1219            for _ in 0..num_modifications {
1220                let batch = thread_rng().gen_range(0..batches.len());
1221                let packet = thread_rng().gen_range(0..batches[batch].len());
1222                let offset = thread_rng().gen_range(0..batches[batch][packet].meta().size);
1223                let add = thread_rng().gen_range(0..255);
1224                batches[batch][packet].buffer_mut()[offset] = batches[batch][packet]
1225                    .data(offset)
1226                    .unwrap()
1227                    .wrapping_add(add);
1228            }
1229
1230            let batch_to_disable = thread_rng().gen_range(0..batches.len());
1231            for p in batches[batch_to_disable].iter_mut() {
1232                p.meta_mut().set_discard(true);
1233            }
1234
1235            // verify from GPU verification pipeline (when GPU verification is enabled) are
1236            // equivalent to the CPU verification pipeline.
1237            let mut batches_cpu = batches.clone();
1238            let packet_count = sigverify::count_packets_in_batches(&batches);
1239            sigverify::ed25519_verify(&mut batches, &recycler, &recycler_out, false, packet_count);
1240            ed25519_verify_cpu(&mut batches_cpu, false, packet_count);
1241
1242            // check result
1243            batches
1244                .iter()
1245                .flat_map(|batch| batch.iter())
1246                .zip(batches_cpu.iter().flat_map(|batch| batch.iter()))
1247                .for_each(|(p1, p2)| assert_eq!(p1, p2));
1248        }
1249    }
1250
1251    #[test]
1252    fn test_verify_fail() {
1253        test_verify_n(5, true);
1254    }
1255
1256    #[test]
1257    fn test_get_checked_scalar() {
1258        solana_logger::setup();
1259        if perf_libs::api().is_none() {
1260            return;
1261        }
1262
1263        let passed_g = AtomicU64::new(0);
1264        let failed_g = AtomicU64::new(0);
1265        (0..4).into_par_iter().for_each(|_| {
1266            let mut input = [0u8; 32];
1267            let mut passed = 0;
1268            let mut failed = 0;
1269            for _ in 0..1_000_000 {
1270                thread_rng().fill(&mut input);
1271                let ans = get_checked_scalar(&input);
1272                let ref_ans = Scalar::from_canonical_bytes(input).into_option();
1273                if let Some(ref_ans) = ref_ans {
1274                    passed += 1;
1275                    assert_eq!(ans.unwrap(), ref_ans.to_bytes());
1276                } else {
1277                    failed += 1;
1278                    assert!(ans.is_err());
1279                }
1280            }
1281            passed_g.fetch_add(passed, Ordering::Relaxed);
1282            failed_g.fetch_add(failed, Ordering::Relaxed);
1283        });
1284        info!(
1285            "passed: {} failed: {}",
1286            passed_g.load(Ordering::Relaxed),
1287            failed_g.load(Ordering::Relaxed)
1288        );
1289    }
1290
1291    #[test]
1292    fn test_ge_small_order() {
1293        solana_logger::setup();
1294        if perf_libs::api().is_none() {
1295            return;
1296        }
1297
1298        let passed_g = AtomicU64::new(0);
1299        let failed_g = AtomicU64::new(0);
1300        (0..4).into_par_iter().for_each(|_| {
1301            let mut input = [0u8; 32];
1302            let mut passed = 0;
1303            let mut failed = 0;
1304            for _ in 0..1_000_000 {
1305                thread_rng().fill(&mut input);
1306                let ans = check_packed_ge_small_order(&input);
1307                let ref_ge = CompressedEdwardsY::from_slice(&input).unwrap();
1308                if let Some(ref_element) = ref_ge.decompress() {
1309                    if ref_element.is_small_order() {
1310                        assert!(!ans);
1311                    } else {
1312                        assert!(ans);
1313                    }
1314                } else {
1315                    assert!(!ans);
1316                }
1317                if ans {
1318                    passed += 1;
1319                } else {
1320                    failed += 1;
1321                }
1322            }
1323            passed_g.fetch_add(passed, Ordering::Relaxed);
1324            failed_g.fetch_add(failed, Ordering::Relaxed);
1325        });
1326        info!(
1327            "passed: {} failed: {}",
1328            passed_g.load(Ordering::Relaxed),
1329            failed_g.load(Ordering::Relaxed)
1330        );
1331    }
1332
1333    #[test]
1334    fn test_is_simple_vote_transaction() {
1335        solana_logger::setup();
1336        let mut rng = rand::thread_rng();
1337
1338        // tansfer tx is not
1339        {
1340            let mut tx = test_tx();
1341            tx.message.instructions[0].data = vec![1, 2, 3];
1342            let mut packet = Packet::from_data(None, tx).unwrap();
1343            let packet_offsets = do_get_packet_offsets(&packet, 0).unwrap();
1344            check_for_simple_vote_transaction(&mut packet, &packet_offsets, 0).ok();
1345            assert!(!packet.meta().is_simple_vote_tx());
1346        }
1347
1348        // single legacy vote tx is
1349        {
1350            let mut tx = new_test_vote_tx(&mut rng);
1351            tx.message.instructions[0].data = vec![1, 2, 3];
1352            let mut packet = Packet::from_data(None, tx).unwrap();
1353            let packet_offsets = do_get_packet_offsets(&packet, 0).unwrap();
1354            check_for_simple_vote_transaction(&mut packet, &packet_offsets, 0).ok();
1355            assert!(packet.meta().is_simple_vote_tx());
1356        }
1357
1358        // single versioned vote tx is not
1359        {
1360            let mut tx = new_test_vote_tx(&mut rng);
1361            tx.message.instructions[0].data = vec![1, 2, 3];
1362            let mut packet = Packet::from_data(None, tx).unwrap();
1363
1364            // set messager version to v0
1365            let mut packet_offsets = do_get_packet_offsets(&packet, 0).unwrap();
1366            let msg_start = packet_offsets.msg_start as usize;
1367            let msg_bytes = packet.data(msg_start..).unwrap().to_vec();
1368            packet.buffer_mut()[msg_start] = MESSAGE_VERSION_PREFIX;
1369            packet.meta_mut().size += 1;
1370            let msg_end = packet.meta().size;
1371            packet.buffer_mut()[msg_start + 1..msg_end].copy_from_slice(&msg_bytes);
1372
1373            packet_offsets = do_get_packet_offsets(&packet, 0).unwrap();
1374            check_for_simple_vote_transaction(&mut packet, &packet_offsets, 0).ok();
1375            assert!(!packet.meta().is_simple_vote_tx());
1376        }
1377
1378        // multiple mixed tx is not
1379        {
1380            let key = Keypair::new();
1381            let key1 = Pubkey::new_unique();
1382            let key2 = Pubkey::new_unique();
1383            let tx = Transaction::new_with_compiled_instructions(
1384                &[&key],
1385                &[key1, key2],
1386                Hash::default(),
1387                vec![solana_vote_program::id(), Pubkey::new_unique()],
1388                vec![
1389                    CompiledInstruction::new(3, &(), vec![0, 1]),
1390                    CompiledInstruction::new(4, &(), vec![0, 2]),
1391                ],
1392            );
1393            let mut packet = Packet::from_data(None, tx).unwrap();
1394            let packet_offsets = do_get_packet_offsets(&packet, 0).unwrap();
1395            check_for_simple_vote_transaction(&mut packet, &packet_offsets, 0).ok();
1396            assert!(!packet.meta().is_simple_vote_tx());
1397        }
1398
1399        // single legacy vote tx with extra (invalid) signature is not
1400        {
1401            let mut tx = new_test_vote_tx(&mut rng);
1402            tx.signatures.push(Signature::default());
1403            tx.message.header.num_required_signatures = 3;
1404            tx.message.instructions[0].data = vec![1, 2, 3];
1405            let mut packet = Packet::from_data(None, tx).unwrap();
1406            let packet_offsets = do_get_packet_offsets(&packet, 0).unwrap();
1407            assert_eq!(
1408                Err(PacketError::InvalidSignatureLen),
1409                check_for_simple_vote_transaction(&mut packet, &packet_offsets, 0)
1410            );
1411            assert!(!packet.meta().is_simple_vote_tx());
1412        }
1413    }
1414
1415    #[test]
1416    fn test_is_simple_vote_transaction_with_offsets() {
1417        solana_logger::setup();
1418        let mut rng = rand::thread_rng();
1419
1420        // batch of legacy messages
1421        {
1422            let mut current_offset = 0usize;
1423            let mut batch = PacketBatch::default();
1424            batch.push(Packet::from_data(None, test_tx()).unwrap());
1425            let tx = new_test_vote_tx(&mut rng);
1426            batch.push(Packet::from_data(None, tx).unwrap());
1427            batch.iter_mut().enumerate().for_each(|(index, packet)| {
1428                let packet_offsets = do_get_packet_offsets(packet, current_offset).unwrap();
1429                check_for_simple_vote_transaction(packet, &packet_offsets, current_offset).ok();
1430                if index == 1 {
1431                    assert!(packet.meta().is_simple_vote_tx());
1432                } else {
1433                    assert!(!packet.meta().is_simple_vote_tx());
1434                }
1435
1436                current_offset = current_offset.saturating_add(size_of::<Packet>());
1437            });
1438        }
1439
1440        // batch of mixed legacy messages and versioned vote tx, which won't be flagged as
1441        // simple_vote_tx
1442        {
1443            let mut current_offset = 0usize;
1444            let mut batch = PacketBatch::default();
1445            batch.push(Packet::from_data(None, test_tx()).unwrap());
1446            // versioned vote tx
1447            let tx = new_test_vote_tx(&mut rng);
1448            let mut packet = Packet::from_data(None, tx).unwrap();
1449            let packet_offsets = do_get_packet_offsets(&packet, 0).unwrap();
1450            let msg_start = packet_offsets.msg_start as usize;
1451            let msg_bytes = packet.data(msg_start..).unwrap().to_vec();
1452            packet.buffer_mut()[msg_start] = MESSAGE_VERSION_PREFIX;
1453            packet.meta_mut().size += 1;
1454            let msg_end = packet.meta().size;
1455            packet.buffer_mut()[msg_start + 1..msg_end].copy_from_slice(&msg_bytes);
1456            batch.push(packet);
1457
1458            batch.iter_mut().for_each(|packet| {
1459                let packet_offsets = do_get_packet_offsets(packet, current_offset).unwrap();
1460                check_for_simple_vote_transaction(packet, &packet_offsets, current_offset).ok();
1461                assert!(!packet.meta().is_simple_vote_tx());
1462
1463                current_offset = current_offset.saturating_add(size_of::<Packet>());
1464            });
1465        }
1466    }
1467
1468    #[test]
1469    fn test_shrink_fuzz() {
1470        for _ in 0..5 {
1471            let mut batches = to_packet_batches(
1472                &(0..PACKETS_PER_BATCH * 3)
1473                    .map(|_| test_tx())
1474                    .collect::<Vec<_>>(),
1475                PACKETS_PER_BATCH,
1476            );
1477            batches.iter_mut().for_each(|b| {
1478                b.iter_mut()
1479                    .for_each(|p| p.meta_mut().set_discard(thread_rng().gen()))
1480            });
1481            //find all the non discarded packets
1482            let mut start = vec![];
1483            batches.iter_mut().for_each(|b| {
1484                b.iter_mut()
1485                    .filter(|p| !p.meta().discard())
1486                    .for_each(|p| start.push(p.clone()))
1487            });
1488            start.sort_by(|a, b| a.data(..).cmp(&b.data(..)));
1489
1490            let packet_count = count_valid_packets(&batches, |_| ());
1491            shrink_batches(&mut batches);
1492
1493            //make sure all the non discarded packets are the same
1494            let mut end = vec![];
1495            batches.iter_mut().for_each(|b| {
1496                b.iter_mut()
1497                    .filter(|p| !p.meta().discard())
1498                    .for_each(|p| end.push(p.clone()))
1499            });
1500            end.sort_by(|a, b| a.data(..).cmp(&b.data(..)));
1501            let packet_count2 = count_valid_packets(&batches, |_| ());
1502            assert_eq!(packet_count, packet_count2);
1503            assert_eq!(start, end);
1504        }
1505    }
1506
1507    #[test]
1508    fn test_shrink_empty() {
1509        const PACKET_COUNT: usize = 1024;
1510        const BATCH_COUNT: usize = PACKET_COUNT / PACKETS_PER_BATCH;
1511
1512        // No batches
1513        // truncate of 1 on len 0 is a noop
1514        shrink_batches(&mut Vec::new());
1515        // One empty batch
1516        {
1517            let mut batches = vec![PacketBatch::with_capacity(0)];
1518            shrink_batches(&mut batches);
1519            assert_eq!(batches.len(), 0);
1520        }
1521        // Many empty batches
1522        {
1523            let mut batches = (0..BATCH_COUNT)
1524                .map(|_| PacketBatch::with_capacity(0))
1525                .collect::<Vec<_>>();
1526            shrink_batches(&mut batches);
1527            assert_eq!(batches.len(), 0);
1528        }
1529    }
1530
1531    #[test]
1532    fn test_shrink_vectors() {
1533        const PACKET_COUNT: usize = 1024;
1534        const BATCH_COUNT: usize = PACKET_COUNT / PACKETS_PER_BATCH;
1535
1536        let set_discards = [
1537            // contiguous
1538            // 0
1539            // No discards
1540            |_, _| false,
1541            // All discards
1542            |_, _| true,
1543            // single partitions
1544            // discard last half of packets
1545            |b, p| ((b * PACKETS_PER_BATCH) + p) >= (PACKET_COUNT / 2),
1546            // discard first half of packets
1547            |b, p| ((b * PACKETS_PER_BATCH) + p) < (PACKET_COUNT / 2),
1548            // discard last half of each batch
1549            |_, p| p >= (PACKETS_PER_BATCH / 2),
1550            // 5
1551            // discard first half of each batch
1552            |_, p| p < (PACKETS_PER_BATCH / 2),
1553            // uniform sparse
1554            // discard even packets
1555            |b, p| ((b * PACKETS_PER_BATCH) + p) % 2 == 0,
1556            // discard odd packets
1557            |b, p| ((b * PACKETS_PER_BATCH) + p) % 2 == 1,
1558            // discard even batches
1559            |b, _| b % 2 == 0,
1560            // discard odd batches
1561            |b, _| b % 2 == 1,
1562            // edges
1563            // 10
1564            // discard first batch
1565            |b, _| b == 0,
1566            // discard last batch
1567            |b, _| b == BATCH_COUNT - 1,
1568            // discard first and last batches
1569            |b, _| b == 0 || b == BATCH_COUNT - 1,
1570            // discard all but first and last batches
1571            |b, _| b != 0 && b != BATCH_COUNT - 1,
1572            // discard first packet
1573            |b, p| ((b * PACKETS_PER_BATCH) + p) == 0,
1574            // 15
1575            // discard all but first packet
1576            |b, p| ((b * PACKETS_PER_BATCH) + p) != 0,
1577            // discard last packet
1578            |b, p| ((b * PACKETS_PER_BATCH) + p) == PACKET_COUNT - 1,
1579            // discard all but last packet
1580            |b, p| ((b * PACKETS_PER_BATCH) + p) != PACKET_COUNT - 1,
1581            // discard first packet of each batch
1582            |_, p| p == 0,
1583            // discard all but first packet of each batch
1584            |_, p| p != 0,
1585            // 20
1586            // discard last packet of each batch
1587            |_, p| p == PACKETS_PER_BATCH - 1,
1588            // discard all but last packet of each batch
1589            |_, p| p != PACKETS_PER_BATCH - 1,
1590            // discard first and last packet of each batch
1591            |_, p| p == 0 || p == PACKETS_PER_BATCH - 1,
1592            // discard all but first and last packet of each batch
1593            |_, p| p != 0 && p != PACKETS_PER_BATCH - 1,
1594            // discard all after first packet in second to last batch
1595            |b, p| (b == BATCH_COUNT - 2 && p > 0) || b == BATCH_COUNT - 1,
1596            // 25
1597        ];
1598
1599        let expect_valids = [
1600            // (expected_batches, expected_valid_packets)
1601            //
1602            // contiguous
1603            // 0
1604            (BATCH_COUNT, PACKET_COUNT),
1605            (0, 0),
1606            // single partitions
1607            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1608            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1609            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1610            // 5
1611            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1612            // uniform sparse
1613            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1614            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1615            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1616            (BATCH_COUNT / 2, PACKET_COUNT / 2),
1617            // edges
1618            // 10
1619            (BATCH_COUNT - 1, PACKET_COUNT - PACKETS_PER_BATCH),
1620            (BATCH_COUNT - 1, PACKET_COUNT - PACKETS_PER_BATCH),
1621            (BATCH_COUNT - 2, PACKET_COUNT - 2 * PACKETS_PER_BATCH),
1622            (2, 2 * PACKETS_PER_BATCH),
1623            (BATCH_COUNT, PACKET_COUNT - 1),
1624            // 15
1625            (1, 1),
1626            (BATCH_COUNT, PACKET_COUNT - 1),
1627            (1, 1),
1628            (
1629                (BATCH_COUNT * (PACKETS_PER_BATCH - 1) + PACKETS_PER_BATCH) / PACKETS_PER_BATCH,
1630                (PACKETS_PER_BATCH - 1) * BATCH_COUNT,
1631            ),
1632            (
1633                (BATCH_COUNT + PACKETS_PER_BATCH) / PACKETS_PER_BATCH,
1634                BATCH_COUNT,
1635            ),
1636            // 20
1637            (
1638                (BATCH_COUNT * (PACKETS_PER_BATCH - 1) + PACKETS_PER_BATCH) / PACKETS_PER_BATCH,
1639                (PACKETS_PER_BATCH - 1) * BATCH_COUNT,
1640            ),
1641            (
1642                (BATCH_COUNT + PACKETS_PER_BATCH) / PACKETS_PER_BATCH,
1643                BATCH_COUNT,
1644            ),
1645            (
1646                (BATCH_COUNT * (PACKETS_PER_BATCH - 2) + PACKETS_PER_BATCH) / PACKETS_PER_BATCH,
1647                (PACKETS_PER_BATCH - 2) * BATCH_COUNT,
1648            ),
1649            (
1650                (2 * BATCH_COUNT + PACKETS_PER_BATCH) / PACKETS_PER_BATCH,
1651                PACKET_COUNT - (PACKETS_PER_BATCH - 2) * BATCH_COUNT,
1652            ),
1653            (BATCH_COUNT - 1, PACKET_COUNT - 2 * PACKETS_PER_BATCH + 1),
1654            // 25
1655        ];
1656
1657        let test_cases = set_discards.iter().zip(&expect_valids).enumerate();
1658        for (i, (set_discard, (expect_batch_count, expect_valid_packets))) in test_cases {
1659            debug!("test_shrink case: {}", i);
1660            let mut batches = to_packet_batches(
1661                &(0..PACKET_COUNT).map(|_| test_tx()).collect::<Vec<_>>(),
1662                PACKETS_PER_BATCH,
1663            );
1664            assert_eq!(batches.len(), BATCH_COUNT);
1665            assert_eq!(count_valid_packets(&batches, |_| ()), PACKET_COUNT);
1666            batches.iter_mut().enumerate().for_each(|(i, b)| {
1667                b.iter_mut()
1668                    .enumerate()
1669                    .for_each(|(j, p)| p.meta_mut().set_discard(set_discard(i, j)))
1670            });
1671            assert_eq!(count_valid_packets(&batches, |_| ()), *expect_valid_packets);
1672            debug!("show valid packets for case {}", i);
1673            batches.iter_mut().enumerate().for_each(|(i, b)| {
1674                b.iter_mut().enumerate().for_each(|(j, p)| {
1675                    if !p.meta().discard() {
1676                        trace!("{} {}", i, j)
1677                    }
1678                })
1679            });
1680            debug!("done show valid packets for case {}", i);
1681            shrink_batches(&mut batches);
1682            let shrunken_batch_count = batches.len();
1683            debug!("shrunk batch test {} count: {}", i, shrunken_batch_count);
1684            assert_eq!(shrunken_batch_count, *expect_batch_count);
1685            assert_eq!(count_valid_packets(&batches, |_| ()), *expect_valid_packets);
1686        }
1687    }
1688}