cranelift_codegen/isa/x64/inst/
args.rs

1//! Instruction operand sub-components (aka "parts"): definitions and printing.
2
3use super::regs::{self};
4use crate::ir::condcodes::{FloatCC, IntCC};
5use crate::ir::types::*;
6use crate::ir::MemFlags;
7use crate::isa::x64::inst::regs::pretty_print_reg;
8use crate::isa::x64::inst::Inst;
9use crate::machinst::*;
10use smallvec::{smallvec, SmallVec};
11use std::fmt;
12use std::string::String;
13
14pub use crate::isa::x64::lower::isle::generated_code::DivSignedness;
15
16/// An extension trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.
17pub trait ToWritableReg {
18    /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.
19    fn to_writable_reg(&self) -> Writable<Reg>;
20}
21
22/// An extension trait for converting `Writable<Reg>` to `Writable{Xmm,Gpr}`.
23pub trait FromWritableReg: Sized {
24    /// Convert `Writable<Reg>` to `Writable{Xmm,Gpr}`.
25    fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;
26}
27
28/// A macro for defining a newtype of `Reg` that enforces some invariant about
29/// the wrapped `Reg` (such as that it is of a particular register class).
30macro_rules! newtype_of_reg {
31    (
32        $newtype_reg:ident,
33        $newtype_writable_reg:ident,
34        $newtype_option_writable_reg:ident,
35        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
36        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
37        $newtype_imm8_reg:ident,
38        |$check_reg:ident| $check:expr
39    ) => {
40        /// A newtype wrapper around `Reg`.
41        #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
42        pub struct $newtype_reg(Reg);
43
44        impl PartialEq<Reg> for $newtype_reg {
45            fn eq(&self, other: &Reg) -> bool {
46                self.0 == *other
47            }
48        }
49
50        impl From<$newtype_reg> for Reg {
51            fn from(r: $newtype_reg) -> Self {
52                r.0
53            }
54        }
55
56        impl $newtype_reg {
57            /// Create this newtype from the given register, or return `None` if the register
58            /// is not a valid instance of this newtype.
59            pub fn new($check_reg: Reg) -> Option<Self> {
60                if $check {
61                    Some(Self($check_reg))
62                } else {
63                    None
64                }
65            }
66
67            /// Like `Self::new(r).unwrap()` but with a better panic message on
68            /// failure.
69            pub fn unwrap_new($check_reg: Reg) -> Self {
70                if $check {
71                    Self($check_reg)
72                } else {
73                    panic!(
74                        "cannot construct {} from register {:?} with register class {:?}",
75                        stringify!($newtype_reg),
76                        $check_reg,
77                        $check_reg.class(),
78                    )
79                }
80            }
81
82            /// Get this newtype's underlying `Reg`.
83            pub fn to_reg(self) -> Reg {
84                self.0
85            }
86        }
87
88        // Convenience impl so that people working with this newtype can use it
89        // "just like" a plain `Reg`.
90        //
91        // NB: We cannot implement `DerefMut` because that would let people do
92        // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the
93        // invariants that `Gpr` provides.
94        impl std::ops::Deref for $newtype_reg {
95            type Target = Reg;
96
97            fn deref(&self) -> &Reg {
98                &self.0
99            }
100        }
101
102        /// If you know what you're doing, you can explicitly mutably borrow the
103        /// underlying `Reg`. Don't make it point to the wrong type of register
104        /// please.
105        impl AsMut<Reg> for $newtype_reg {
106            fn as_mut(&mut self) -> &mut Reg {
107                &mut self.0
108            }
109        }
110
111        /// Writable Gpr.
112        pub type $newtype_writable_reg = Writable<$newtype_reg>;
113
114        #[allow(dead_code)] // Used by some newtypes and not others.
115        /// Optional writable Gpr.
116        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
117
118        impl ToWritableReg for $newtype_writable_reg {
119            fn to_writable_reg(&self) -> Writable<Reg> {
120                Writable::from_reg(self.to_reg().to_reg())
121            }
122        }
123
124        impl FromWritableReg for $newtype_writable_reg {
125            fn from_writable_reg(w: Writable<Reg>) -> Option<Self> {
126                Some(Writable::from_reg($newtype_reg::new(w.to_reg())?))
127            }
128        }
129
130        $(
131            /// A newtype wrapper around `RegMem` for general-purpose registers.
132            #[derive(Clone, Debug)]
133            pub struct $newtype_reg_mem(RegMem);
134
135            impl From<$newtype_reg_mem> for RegMem {
136                fn from(rm: $newtype_reg_mem) -> Self {
137                    rm.0
138                }
139            }
140            impl<'a> From<&'a $newtype_reg_mem> for &'a RegMem {
141                fn from(rm: &'a $newtype_reg_mem) -> &'a RegMem {
142                    &rm.0
143                }
144            }
145
146            impl From<$newtype_reg> for $newtype_reg_mem {
147                fn from(r: $newtype_reg) -> Self {
148                    $newtype_reg_mem(RegMem::reg(r.into()))
149                }
150            }
151
152            impl $newtype_reg_mem {
153                /// Construct a `RegMem` newtype from the given `RegMem`, or return
154                /// `None` if the `RegMem` is not a valid instance of this `RegMem`
155                /// newtype.
156                pub fn new(rm: RegMem) -> Option<Self> {
157                    match rm {
158                        RegMem::Mem { addr } => {
159                            let mut _allow = true;
160                            $(
161                                if $aligned {
162                                    _allow = addr.aligned();
163                                }
164                            )?
165                            if _allow {
166                                Some(Self(RegMem::Mem { addr }))
167                            } else {
168                                None
169                            }
170                        }
171                        RegMem::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
172                    }
173                }
174
175                /// Like `Self::new(rm).unwrap()` but with better panic messages
176                /// in case of failure.
177                pub fn unwrap_new(rm: RegMem) -> Self {
178                    match rm {
179                        RegMem::Mem { addr } => {
180                            $(
181                                if $aligned && !addr.aligned() {
182                                    panic!(
183                                        "cannot create {} from an unaligned memory address: {addr:?}",
184                                        stringify!($newtype_reg_mem),
185                                    );
186                                }
187                            )?
188                            Self(RegMem::Mem { addr })
189                        }
190                        RegMem::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
191                    }
192                }
193
194                /// Convert this newtype into its underlying `RegMem`.
195                pub fn to_reg_mem(self) -> RegMem {
196                    self.0
197                }
198
199                #[allow(dead_code)] // Used by some newtypes and not others.
200                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
201                    self.0.get_operands(collector);
202                }
203            }
204            impl PrettyPrint for $newtype_reg_mem {
205                fn pretty_print(&self, size: u8) -> String {
206                    self.0.pretty_print(size)
207                }
208            }
209        )*
210
211        $(
212            /// A newtype wrapper around `RegMemImm`.
213            #[derive(Clone, Debug)]
214            pub struct $newtype_reg_mem_imm(RegMemImm);
215
216            impl From<$newtype_reg_mem_imm> for RegMemImm {
217                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
218                    rmi.0
219                }
220            }
221            impl<'a> From<&'a $newtype_reg_mem_imm> for &'a RegMemImm {
222                fn from(rmi: &'a $newtype_reg_mem_imm) -> &'a RegMemImm {
223                    &rmi.0
224                }
225            }
226
227            impl From<$newtype_reg> for $newtype_reg_mem_imm {
228                fn from(r: $newtype_reg) -> Self {
229                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
230                }
231            }
232
233            impl $newtype_reg_mem_imm {
234                /// Construct this newtype from the given `RegMemImm`, or return
235                /// `None` if the `RegMemImm` is not a valid instance of this
236                /// newtype.
237                pub fn new(rmi: RegMemImm) -> Option<Self> {
238                    match rmi {
239                        RegMemImm::Imm { .. } => Some(Self(rmi)),
240                        RegMemImm::Mem { addr } => {
241                            let mut _allow = true;
242                            $(
243                                if $aligned_imm {
244                                    _allow = addr.aligned();
245                                }
246                            )?
247                            if _allow {
248                                Some(Self(RegMemImm::Mem { addr }))
249                            } else {
250                                None
251                            }
252                        }
253                        RegMemImm::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
254                    }
255                }
256
257                /// Like `Self::new(rmi).unwrap()` but with better panic
258                /// messages in case of failure.
259                pub fn unwrap_new(rmi: RegMemImm) -> Self {
260                    match rmi {
261                        RegMemImm::Imm { .. } => Self(rmi),
262                        RegMemImm::Mem { addr } => {
263                            $(
264                                if $aligned_imm && !addr.aligned() {
265                                    panic!(
266                                        "cannot construct {} from unaligned memory address: {:?}",
267                                        stringify!($newtype_reg_mem_imm),
268                                        addr,
269                                    );
270                                }
271                            )?
272                            Self(RegMemImm::Mem { addr })
273
274                        }
275                        RegMemImm::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
276                    }
277                }
278
279                /// Convert this newtype into its underlying `RegMemImm`.
280                #[allow(dead_code)] // Used by some newtypes and not others.
281                pub fn to_reg_mem_imm(self) -> RegMemImm {
282                    self.0
283                }
284
285                #[allow(dead_code)] // Used by some newtypes and not others.
286                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
287                    self.0.get_operands(collector);
288                }
289            }
290
291            impl PrettyPrint for $newtype_reg_mem_imm {
292                fn pretty_print(&self, size: u8) -> String {
293                    self.0.pretty_print(size)
294                }
295            }
296        )*
297
298        /// A newtype wrapper around `Imm8Reg`.
299        #[derive(Clone, Debug)]
300        #[allow(dead_code)] // Used by some newtypes and not others.
301        pub struct $newtype_imm8_reg(Imm8Reg);
302
303        impl From<$newtype_reg> for $newtype_imm8_reg {
304            fn from(r: $newtype_reg) -> Self {
305                Self(Imm8Reg::Reg { reg: r.to_reg() })
306            }
307        }
308
309        impl $newtype_imm8_reg {
310            /// Construct this newtype from the given `Imm8Reg`, or return
311            /// `None` if the `Imm8Reg` is not a valid instance of this newtype.
312            #[allow(dead_code)] // Used by some newtypes and not others.
313            pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
314                match imm8_reg {
315                    Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
316                    Imm8Reg::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
317                }
318            }
319
320            /// Like `Self::new(imm8_reg).unwrap()` but with better panic
321            /// messages on failure.
322            pub fn unwrap_new(imm8_reg: Imm8Reg) -> Self {
323                match imm8_reg {
324                    Imm8Reg::Imm8 { .. } => Self(imm8_reg),
325                    Imm8Reg::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
326                }
327            }
328
329            /// Borrow this newtype as its underlying `Imm8Reg`.
330            #[allow(dead_code)] // Used by some newtypes and not others.
331            pub fn as_imm8_reg(&self) -> &Imm8Reg {
332                &self.0
333            }
334
335            /// Borrow this newtype as its underlying `Imm8Reg`.
336            #[allow(dead_code)] // Used by some newtypes and not others.
337            pub fn as_imm8_reg_mut(&mut self) -> &mut Imm8Reg {
338                &mut self.0
339            }
340        }
341    };
342}
343
344// Define a newtype of `Reg` for general-purpose registers.
345newtype_of_reg!(
346    Gpr,
347    WritableGpr,
348    OptionWritableGpr,
349    reg_mem: (GprMem),
350    reg_mem_imm: (GprMemImm),
351    Imm8Gpr,
352    |reg| reg.class() == RegClass::Int
353);
354
355// Define a newtype of `Reg` for XMM registers.
356newtype_of_reg!(
357    Xmm,
358    WritableXmm,
359    OptionWritableXmm,
360    reg_mem: (XmmMem, XmmMemAligned aligned:true),
361    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
362    Imm8Xmm,
363    |reg| reg.class() == RegClass::Float
364);
365
366// N.B.: `Amode` is defined in `inst.isle`. We add some convenience
367// constructors here.
368
369// Re-export the type from the ISLE generated code.
370pub use crate::isa::x64::lower::isle::generated_code::Amode;
371
372impl Amode {
373    /// Create an immediate sign-extended and register addressing mode.
374    pub fn imm_reg(simm32: i32, base: Reg) -> Self {
375        debug_assert!(base.class() == RegClass::Int);
376        Self::ImmReg {
377            simm32,
378            base,
379            flags: MemFlags::trusted(),
380        }
381    }
382
383    /// Create a sign-extended-32-to-64 with register and shift addressing mode.
384    pub fn imm_reg_reg_shift(simm32: i32, base: Gpr, index: Gpr, shift: u8) -> Self {
385        debug_assert!(base.class() == RegClass::Int);
386        debug_assert!(index.class() == RegClass::Int);
387        debug_assert!(shift <= 3);
388        Self::ImmRegRegShift {
389            simm32,
390            base,
391            index,
392            shift,
393            flags: MemFlags::trusted(),
394        }
395    }
396
397    pub(crate) fn rip_relative(target: MachLabel) -> Self {
398        Self::RipRelative { target }
399    }
400
401    /// Set the specified [MemFlags] to the [Amode].
402    pub fn with_flags(&self, flags: MemFlags) -> Self {
403        match self {
404            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {
405                simm32,
406                base,
407                flags,
408            },
409            &Self::ImmRegRegShift {
410                simm32,
411                base,
412                index,
413                shift,
414                ..
415            } => Self::ImmRegRegShift {
416                simm32,
417                base,
418                index,
419                shift,
420                flags,
421            },
422            _ => panic!("Amode {self:?} cannot take memflags"),
423        }
424    }
425
426    /// Add the registers mentioned by `self` to `collector`.
427    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
428        match self {
429            Amode::ImmReg { base, .. } => {
430                if *base != regs::rbp() && *base != regs::rsp() {
431                    collector.reg_use(base);
432                }
433            }
434            Amode::ImmRegRegShift { base, index, .. } => {
435                debug_assert_ne!(base.to_reg(), regs::rbp());
436                debug_assert_ne!(base.to_reg(), regs::rsp());
437                collector.reg_use(base);
438                debug_assert_ne!(index.to_reg(), regs::rbp());
439                debug_assert_ne!(index.to_reg(), regs::rsp());
440                collector.reg_use(index);
441            }
442            Amode::RipRelative { .. } => {
443                // RIP isn't involved in regalloc.
444            }
445        }
446    }
447
448    /// Same as `get_operands`, but add the registers in the "late" phase.
449    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
450        match self {
451            Amode::ImmReg { base, .. } => {
452                collector.reg_late_use(base);
453            }
454            Amode::ImmRegRegShift { base, index, .. } => {
455                collector.reg_late_use(base);
456                collector.reg_late_use(index);
457            }
458            Amode::RipRelative { .. } => {
459                // RIP isn't involved in regalloc.
460            }
461        }
462    }
463
464    pub(crate) fn get_flags(&self) -> MemFlags {
465        match self {
466            Amode::ImmReg { flags, .. } | Amode::ImmRegRegShift { flags, .. } => *flags,
467            Amode::RipRelative { .. } => MemFlags::trusted(),
468        }
469    }
470
471    /// Offset the amode by a fixed offset.
472    pub(crate) fn offset(&self, offset: i32) -> Self {
473        let mut ret = self.clone();
474        match &mut ret {
475            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
476            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
477            _ => panic!("Cannot offset amode: {self:?}"),
478        }
479        ret
480    }
481
482    pub(crate) fn aligned(&self) -> bool {
483        self.get_flags().aligned()
484    }
485}
486
487impl PrettyPrint for Amode {
488    fn pretty_print(&self, _size: u8) -> String {
489        match self {
490            Amode::ImmReg { simm32, base, .. } => {
491                // Note: size is always 8; the address is 64 bits,
492                // even if the addressed operand is smaller.
493                format!("{}({})", *simm32, pretty_print_reg(*base, 8))
494            }
495            Amode::ImmRegRegShift {
496                simm32,
497                base,
498                index,
499                shift,
500                ..
501            } => format!(
502                "{}({},{},{})",
503                *simm32,
504                pretty_print_reg(base.to_reg(), 8),
505                pretty_print_reg(index.to_reg(), 8),
506                1 << shift
507            ),
508            Amode::RipRelative { target } => format!("label{}(%rip)", target.as_u32()),
509        }
510    }
511}
512
513/// A Memory Address. These denote a 64-bit value only.
514/// Used for usual addressing modes as well as addressing modes used during compilation, when the
515/// moving SP offset is not known.
516#[derive(Clone, Debug)]
517pub enum SyntheticAmode {
518    /// A real amode.
519    Real(Amode),
520
521    /// A (virtual) offset into the incoming argument area.
522    IncomingArg {
523        /// The downward offset from the start of the incoming argument area.
524        offset: u32,
525    },
526
527    /// A (virtual) offset to the slot area of the function frame, which lies just above the
528    /// outgoing arguments.
529    SlotOffset {
530        /// The offset into the slot area.
531        simm32: i32,
532    },
533
534    /// A virtual offset to a constant that will be emitted in the constant section of the buffer.
535    ConstantOffset(VCodeConstant),
536}
537
538impl SyntheticAmode {
539    /// Create a real addressing mode.
540    pub fn real(amode: Amode) -> Self {
541        Self::Real(amode)
542    }
543
544    pub(crate) fn slot_offset(simm32: i32) -> Self {
545        SyntheticAmode::SlotOffset { simm32 }
546    }
547
548    /// Add the registers mentioned by `self` to `collector`.
549    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
550        match self {
551            SyntheticAmode::Real(addr) => addr.get_operands(collector),
552            SyntheticAmode::IncomingArg { .. } => {
553                // Nothing to do; the base is known and isn't involved in regalloc.
554            }
555            SyntheticAmode::SlotOffset { .. } => {
556                // Nothing to do; the base is SP and isn't involved in regalloc.
557            }
558            SyntheticAmode::ConstantOffset(_) => {}
559        }
560    }
561
562    /// Same as `get_operands`, but add the register in the "late" phase.
563    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
564        match self {
565            SyntheticAmode::Real(addr) => addr.get_operands_late(collector),
566            SyntheticAmode::IncomingArg { .. } => {
567                // Nothing to do; the base is known and isn't involved in regalloc.
568            }
569            SyntheticAmode::SlotOffset { .. } => {
570                // Nothing to do; the base is SP and isn't involved in regalloc.
571            }
572            SyntheticAmode::ConstantOffset(_) => {}
573        }
574    }
575
576    pub(crate) fn finalize(&self, frame: &FrameLayout, buffer: &mut MachBuffer<Inst>) -> Amode {
577        match self {
578            SyntheticAmode::Real(addr) => addr.clone(),
579            SyntheticAmode::IncomingArg { offset } => {
580                // NOTE: this could be made relative to RSP by adding additional
581                // offsets from the frame_layout.
582                let args_max_fp_offset = frame.tail_args_size + frame.setup_area_size;
583                Amode::imm_reg(
584                    i32::try_from(args_max_fp_offset - offset).unwrap(),
585                    regs::rbp(),
586                )
587            }
588            SyntheticAmode::SlotOffset { simm32 } => {
589                let off = *simm32 as i64 + i64::from(frame.outgoing_args_size);
590                Amode::imm_reg(off.try_into().expect("invalid sp offset"), regs::rsp())
591            }
592            SyntheticAmode::ConstantOffset(c) => {
593                Amode::rip_relative(buffer.get_label_for_constant(*c))
594            }
595        }
596    }
597
598    pub(crate) fn aligned(&self) -> bool {
599        match self {
600            SyntheticAmode::Real(addr) => addr.aligned(),
601            &SyntheticAmode::IncomingArg { .. }
602            | SyntheticAmode::SlotOffset { .. }
603            | SyntheticAmode::ConstantOffset { .. } => true,
604        }
605    }
606}
607
608impl Into<SyntheticAmode> for Amode {
609    fn into(self) -> SyntheticAmode {
610        SyntheticAmode::Real(self)
611    }
612}
613
614impl Into<SyntheticAmode> for VCodeConstant {
615    fn into(self) -> SyntheticAmode {
616        SyntheticAmode::ConstantOffset(self)
617    }
618}
619
620impl PrettyPrint for SyntheticAmode {
621    fn pretty_print(&self, _size: u8) -> String {
622        match self {
623            // See note in `Amode` regarding constant size of `8`.
624            SyntheticAmode::Real(addr) => addr.pretty_print(8),
625            &SyntheticAmode::IncomingArg { offset } => {
626                format!("rbp(stack args max - {offset})")
627            }
628            SyntheticAmode::SlotOffset { simm32 } => {
629                format!("rsp({} + virtual offset)", *simm32)
630            }
631            SyntheticAmode::ConstantOffset(c) => format!("const({})", c.as_u32()),
632        }
633    }
634}
635
636/// An operand which is either an integer Register, a value in Memory or an Immediate.  This can
637/// denote an 8, 16, 32 or 64 bit value.  For the Immediate form, in the 8- and 16-bit case, only
638/// the lower 8 or 16 bits of `simm32` is relevant.  In the 64-bit case, the value denoted by
639/// `simm32` is its sign-extension out to 64 bits.
640#[derive(Clone, Debug)]
641pub enum RegMemImm {
642    /// A register operand.
643    Reg {
644        /// The underlying register.
645        reg: Reg,
646    },
647    /// A memory operand.
648    Mem {
649        /// The memory address.
650        addr: SyntheticAmode,
651    },
652    /// An immediate operand.
653    Imm {
654        /// The immediate value.
655        simm32: u32,
656    },
657}
658
659impl RegMemImm {
660    /// Create a register operand.
661    pub fn reg(reg: Reg) -> Self {
662        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
663        Self::Reg { reg }
664    }
665
666    /// Create a memory operand.
667    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
668        Self::Mem { addr: addr.into() }
669    }
670
671    /// Create an immediate operand.
672    pub fn imm(simm32: u32) -> Self {
673        Self::Imm { simm32 }
674    }
675
676    /// Asserts that in register mode, the reg class is the one that's expected.
677    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
678        if let Self::Reg { reg } = self {
679            debug_assert_eq!(reg.class(), expected_reg_class);
680        }
681    }
682
683    /// Add the regs mentioned by `self` to `collector`.
684    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
685        match self {
686            Self::Reg { reg } => collector.reg_use(reg),
687            Self::Mem { addr } => addr.get_operands(collector),
688            Self::Imm { .. } => {}
689        }
690    }
691}
692
693impl From<RegMem> for RegMemImm {
694    fn from(rm: RegMem) -> RegMemImm {
695        match rm {
696            RegMem::Reg { reg } => RegMemImm::Reg { reg },
697            RegMem::Mem { addr } => RegMemImm::Mem { addr },
698        }
699    }
700}
701
702impl From<Reg> for RegMemImm {
703    fn from(reg: Reg) -> Self {
704        RegMemImm::Reg { reg }
705    }
706}
707
708impl PrettyPrint for RegMemImm {
709    fn pretty_print(&self, size: u8) -> String {
710        match self {
711            Self::Reg { reg } => pretty_print_reg(*reg, size),
712            Self::Mem { addr } => addr.pretty_print(size),
713            Self::Imm { simm32 } => format!("${}", *simm32 as i32),
714        }
715    }
716}
717
718/// An operand which is either an 8-bit integer immediate or a register.
719#[derive(Clone, Debug)]
720pub enum Imm8Reg {
721    /// 8-bit immediate operand.
722    Imm8 {
723        /// The 8-bit immediate value.
724        imm: u8,
725    },
726    /// A register operand.
727    Reg {
728        /// The underlying register.
729        reg: Reg,
730    },
731}
732
733impl From<u8> for Imm8Reg {
734    fn from(imm: u8) -> Self {
735        Self::Imm8 { imm }
736    }
737}
738
739impl From<Reg> for Imm8Reg {
740    fn from(reg: Reg) -> Self {
741        Self::Reg { reg }
742    }
743}
744
745/// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,
746/// 32, 64, or 128 bit value.
747#[derive(Clone, Debug)]
748pub enum RegMem {
749    /// A register operand.
750    Reg {
751        /// The underlying register.
752        reg: Reg,
753    },
754    /// A memory operand.
755    Mem {
756        /// The memory address.
757        addr: SyntheticAmode,
758    },
759}
760
761impl RegMem {
762    /// Create a register operand.
763    pub fn reg(reg: Reg) -> Self {
764        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
765        Self::Reg { reg }
766    }
767
768    /// Create a memory operand.
769    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
770        Self::Mem { addr: addr.into() }
771    }
772    /// Asserts that in register mode, the reg class is the one that's expected.
773    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
774        if let Self::Reg { reg } = self {
775            debug_assert_eq!(reg.class(), expected_reg_class);
776        }
777    }
778    /// Add the regs mentioned by `self` to `collector`.
779    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
780        match self {
781            RegMem::Reg { reg } => collector.reg_use(reg),
782            RegMem::Mem { addr, .. } => addr.get_operands(collector),
783        }
784    }
785}
786
787impl From<Reg> for RegMem {
788    fn from(reg: Reg) -> RegMem {
789        RegMem::Reg { reg }
790    }
791}
792
793impl From<Writable<Reg>> for RegMem {
794    fn from(r: Writable<Reg>) -> Self {
795        RegMem::reg(r.to_reg())
796    }
797}
798
799impl PrettyPrint for RegMem {
800    fn pretty_print(&self, size: u8) -> String {
801        match self {
802            RegMem::Reg { reg } => pretty_print_reg(*reg, size),
803            RegMem::Mem { addr, .. } => addr.pretty_print(size),
804        }
805    }
806}
807
808/// Some basic ALU operations.
809#[derive(Copy, Clone, PartialEq)]
810pub enum AluRmiROpcode {
811    /// Add operation.
812    Add,
813    /// Add with carry.
814    Adc,
815    /// Integer subtraction.
816    Sub,
817    /// Integer subtraction with borrow.
818    Sbb,
819    /// Bitwise AND operation.
820    And,
821    /// Bitwise inclusive OR.
822    Or,
823    /// Bitwise exclusive OR.
824    Xor,
825}
826
827impl fmt::Debug for AluRmiROpcode {
828    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
829        let name = match self {
830            AluRmiROpcode::Add => "add",
831            AluRmiROpcode::Adc => "adc",
832            AluRmiROpcode::Sub => "sub",
833            AluRmiROpcode::Sbb => "sbb",
834            AluRmiROpcode::And => "and",
835            AluRmiROpcode::Or => "or",
836            AluRmiROpcode::Xor => "xor",
837        };
838        write!(fmt, "{name}")
839    }
840}
841
842impl fmt::Display for AluRmiROpcode {
843    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
844        fmt::Debug::fmt(self, f)
845    }
846}
847
848pub use crate::isa::x64::lower::isle::generated_code::AluRmROpcode;
849
850impl AluRmROpcode {
851    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
852        match self {
853            AluRmROpcode::Andn => smallvec![InstructionSet::BMI1],
854            AluRmROpcode::Sarx | AluRmROpcode::Shrx | AluRmROpcode::Shlx | AluRmROpcode::Bzhi => {
855                smallvec![InstructionSet::BMI2]
856            }
857        }
858    }
859}
860
861impl fmt::Display for AluRmROpcode {
862    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
863        f.write_str(&format!("{self:?}").to_lowercase())
864    }
865}
866
867#[derive(Clone, PartialEq)]
868/// Unary operations requiring register or memory and register operands.
869pub enum UnaryRmROpcode {
870    /// Bit-scan reverse.
871    Bsr,
872    /// Bit-scan forward.
873    Bsf,
874    /// Counts leading zeroes (Leading Zero CouNT).
875    Lzcnt,
876    /// Counts trailing zeroes (Trailing Zero CouNT).
877    Tzcnt,
878    /// Counts the number of ones (POPulation CouNT).
879    Popcnt,
880}
881
882impl UnaryRmROpcode {
883    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
884        match self {
885            UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => smallvec![],
886            UnaryRmROpcode::Lzcnt => smallvec![InstructionSet::Lzcnt],
887            UnaryRmROpcode::Tzcnt => smallvec![InstructionSet::BMI1],
888            UnaryRmROpcode::Popcnt => smallvec![InstructionSet::Popcnt],
889        }
890    }
891}
892
893impl fmt::Debug for UnaryRmROpcode {
894    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
895        match self {
896            UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
897            UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
898            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
899            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
900            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
901        }
902    }
903}
904
905impl fmt::Display for UnaryRmROpcode {
906    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
907        fmt::Debug::fmt(self, f)
908    }
909}
910
911pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRVexOpcode;
912
913impl UnaryRmRVexOpcode {
914    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
915        match self {
916            UnaryRmRVexOpcode::Blsi | UnaryRmRVexOpcode::Blsmsk | UnaryRmRVexOpcode::Blsr => {
917                smallvec![InstructionSet::BMI1]
918            }
919        }
920    }
921}
922
923impl fmt::Display for UnaryRmRVexOpcode {
924    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
925        f.write_str(&format!("{self:?}").to_lowercase())
926    }
927}
928
929pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRImmVexOpcode;
930
931impl UnaryRmRImmVexOpcode {
932    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
933        match self {
934            UnaryRmRImmVexOpcode::Rorx => {
935                smallvec![InstructionSet::BMI2]
936            }
937        }
938    }
939}
940
941impl fmt::Display for UnaryRmRImmVexOpcode {
942    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
943        f.write_str(&format!("{self:?}").to_lowercase())
944    }
945}
946
947#[derive(Clone, Copy, PartialEq)]
948/// Comparison operations.
949pub enum CmpOpcode {
950    /// CMP instruction: compute `a - b` and set flags from result.
951    Cmp,
952    /// TEST instruction: compute `a & b` and set flags from result.
953    Test,
954}
955
956#[derive(Debug)]
957pub(crate) enum InstructionSet {
958    SSE,
959    SSE2,
960    CMPXCHG16b,
961    SSSE3,
962    SSE41,
963    SSE42,
964    Popcnt,
965    Lzcnt,
966    BMI1,
967    #[allow(dead_code)] // never constructed (yet).
968    BMI2,
969    FMA,
970    AVX,
971    AVX2,
972    AVX512BITALG,
973    AVX512DQ,
974    AVX512F,
975    AVX512VBMI,
976    AVX512VL,
977}
978
979/// Some SSE operations requiring 2 operands r/m and r.
980#[derive(Clone, Copy, PartialEq)]
981#[allow(dead_code)] // some variants here aren't used just yet
982#[allow(missing_docs)]
983pub enum SseOpcode {
984    Addps,
985    Addpd,
986    Addss,
987    Addsd,
988    Andps,
989    Andpd,
990    Andnps,
991    Andnpd,
992    Blendvpd,
993    Blendvps,
994    Comiss,
995    Comisd,
996    Cmpps,
997    Cmppd,
998    Cmpss,
999    Cmpsd,
1000    Cvtdq2ps,
1001    Cvtdq2pd,
1002    Cvtpd2ps,
1003    Cvtps2pd,
1004    Cvtsd2ss,
1005    Cvtsd2si,
1006    Cvtsi2ss,
1007    Cvtsi2sd,
1008    Cvtss2si,
1009    Cvtss2sd,
1010    Cvttpd2dq,
1011    Cvttps2dq,
1012    Cvttss2si,
1013    Cvttsd2si,
1014    Divps,
1015    Divpd,
1016    Divss,
1017    Divsd,
1018    Insertps,
1019    Maxps,
1020    Maxpd,
1021    Maxss,
1022    Maxsd,
1023    Minps,
1024    Minpd,
1025    Minss,
1026    Minsd,
1027    Movaps,
1028    Movapd,
1029    Movd,
1030    Movdqa,
1031    Movdqu,
1032    Movlhps,
1033    Movmskps,
1034    Movmskpd,
1035    Movq,
1036    Movss,
1037    Movsd,
1038    Movups,
1039    Movupd,
1040    Mulps,
1041    Mulpd,
1042    Mulss,
1043    Mulsd,
1044    Orps,
1045    Orpd,
1046    Pabsb,
1047    Pabsw,
1048    Pabsd,
1049    Packssdw,
1050    Packsswb,
1051    Packusdw,
1052    Packuswb,
1053    Paddb,
1054    Paddd,
1055    Paddq,
1056    Paddw,
1057    Paddsb,
1058    Paddsw,
1059    Paddusb,
1060    Paddusw,
1061    Palignr,
1062    Pand,
1063    Pandn,
1064    Pavgb,
1065    Pavgw,
1066    Pblendvb,
1067    Pcmpeqb,
1068    Pcmpeqw,
1069    Pcmpeqd,
1070    Pcmpeqq,
1071    Pcmpgtb,
1072    Pcmpgtw,
1073    Pcmpgtd,
1074    Pcmpgtq,
1075    Pextrb,
1076    Pextrw,
1077    Pextrd,
1078    Pextrq,
1079    Pinsrb,
1080    Pinsrw,
1081    Pinsrd,
1082    Pmaddubsw,
1083    Pmaddwd,
1084    Pmaxsb,
1085    Pmaxsw,
1086    Pmaxsd,
1087    Pmaxub,
1088    Pmaxuw,
1089    Pmaxud,
1090    Pminsb,
1091    Pminsw,
1092    Pminsd,
1093    Pminub,
1094    Pminuw,
1095    Pminud,
1096    Pmovmskb,
1097    Pmovsxbd,
1098    Pmovsxbw,
1099    Pmovsxbq,
1100    Pmovsxwd,
1101    Pmovsxwq,
1102    Pmovsxdq,
1103    Pmovzxbd,
1104    Pmovzxbw,
1105    Pmovzxbq,
1106    Pmovzxwd,
1107    Pmovzxwq,
1108    Pmovzxdq,
1109    Pmuldq,
1110    Pmulhw,
1111    Pmulhuw,
1112    Pmulhrsw,
1113    Pmulld,
1114    Pmullw,
1115    Pmuludq,
1116    Por,
1117    Pshufb,
1118    Pshufd,
1119    Psllw,
1120    Pslld,
1121    Psllq,
1122    Psraw,
1123    Psrad,
1124    Psrlw,
1125    Psrld,
1126    Psrlq,
1127    Psubb,
1128    Psubd,
1129    Psubq,
1130    Psubw,
1131    Psubsb,
1132    Psubsw,
1133    Psubusb,
1134    Psubusw,
1135    Ptest,
1136    Punpckhbw,
1137    Punpckhwd,
1138    Punpcklbw,
1139    Punpcklwd,
1140    Pxor,
1141    Rcpss,
1142    Roundps,
1143    Roundpd,
1144    Roundss,
1145    Roundsd,
1146    Rsqrtss,
1147    Shufps,
1148    Sqrtps,
1149    Sqrtpd,
1150    Sqrtss,
1151    Sqrtsd,
1152    Subps,
1153    Subpd,
1154    Subss,
1155    Subsd,
1156    Ucomiss,
1157    Ucomisd,
1158    Unpcklps,
1159    Unpcklpd,
1160    Unpckhps,
1161    Xorps,
1162    Xorpd,
1163    Phaddw,
1164    Phaddd,
1165    Punpckhdq,
1166    Punpckldq,
1167    Punpckhqdq,
1168    Punpcklqdq,
1169    Pshuflw,
1170    Pshufhw,
1171    Pblendw,
1172    Movddup,
1173}
1174
1175impl SseOpcode {
1176    /// Which `InstructionSet` is the first supporting this opcode?
1177    pub(crate) fn available_from(&self) -> InstructionSet {
1178        use InstructionSet::*;
1179        match self {
1180            SseOpcode::Addps
1181            | SseOpcode::Addss
1182            | SseOpcode::Andps
1183            | SseOpcode::Andnps
1184            | SseOpcode::Comiss
1185            | SseOpcode::Cmpps
1186            | SseOpcode::Cmpss
1187            | SseOpcode::Cvtsi2ss
1188            | SseOpcode::Cvtss2si
1189            | SseOpcode::Cvttss2si
1190            | SseOpcode::Divps
1191            | SseOpcode::Divss
1192            | SseOpcode::Maxps
1193            | SseOpcode::Maxss
1194            | SseOpcode::Minps
1195            | SseOpcode::Minss
1196            | SseOpcode::Movaps
1197            | SseOpcode::Movlhps
1198            | SseOpcode::Movmskps
1199            | SseOpcode::Movss
1200            | SseOpcode::Movups
1201            | SseOpcode::Mulps
1202            | SseOpcode::Mulss
1203            | SseOpcode::Orps
1204            | SseOpcode::Rcpss
1205            | SseOpcode::Rsqrtss
1206            | SseOpcode::Shufps
1207            | SseOpcode::Sqrtps
1208            | SseOpcode::Sqrtss
1209            | SseOpcode::Subps
1210            | SseOpcode::Subss
1211            | SseOpcode::Ucomiss
1212            | SseOpcode::Unpcklps
1213            | SseOpcode::Unpckhps
1214            | SseOpcode::Xorps => SSE,
1215
1216            SseOpcode::Addpd
1217            | SseOpcode::Addsd
1218            | SseOpcode::Andpd
1219            | SseOpcode::Andnpd
1220            | SseOpcode::Cmppd
1221            | SseOpcode::Cmpsd
1222            | SseOpcode::Comisd
1223            | SseOpcode::Cvtdq2ps
1224            | SseOpcode::Cvtdq2pd
1225            | SseOpcode::Cvtpd2ps
1226            | SseOpcode::Cvtps2pd
1227            | SseOpcode::Cvtsd2ss
1228            | SseOpcode::Cvtsd2si
1229            | SseOpcode::Cvtsi2sd
1230            | SseOpcode::Cvtss2sd
1231            | SseOpcode::Cvttpd2dq
1232            | SseOpcode::Cvttps2dq
1233            | SseOpcode::Cvttsd2si
1234            | SseOpcode::Divpd
1235            | SseOpcode::Divsd
1236            | SseOpcode::Maxpd
1237            | SseOpcode::Maxsd
1238            | SseOpcode::Minpd
1239            | SseOpcode::Minsd
1240            | SseOpcode::Movapd
1241            | SseOpcode::Movd
1242            | SseOpcode::Movmskpd
1243            | SseOpcode::Movq
1244            | SseOpcode::Movsd
1245            | SseOpcode::Movupd
1246            | SseOpcode::Movdqa
1247            | SseOpcode::Movdqu
1248            | SseOpcode::Mulpd
1249            | SseOpcode::Mulsd
1250            | SseOpcode::Orpd
1251            | SseOpcode::Packssdw
1252            | SseOpcode::Packsswb
1253            | SseOpcode::Packuswb
1254            | SseOpcode::Paddb
1255            | SseOpcode::Paddd
1256            | SseOpcode::Paddq
1257            | SseOpcode::Paddw
1258            | SseOpcode::Paddsb
1259            | SseOpcode::Paddsw
1260            | SseOpcode::Paddusb
1261            | SseOpcode::Paddusw
1262            | SseOpcode::Pand
1263            | SseOpcode::Pandn
1264            | SseOpcode::Pavgb
1265            | SseOpcode::Pavgw
1266            | SseOpcode::Pcmpeqb
1267            | SseOpcode::Pcmpeqw
1268            | SseOpcode::Pcmpeqd
1269            | SseOpcode::Pcmpgtb
1270            | SseOpcode::Pcmpgtw
1271            | SseOpcode::Pcmpgtd
1272            | SseOpcode::Pextrw
1273            | SseOpcode::Pinsrw
1274            | SseOpcode::Pmaddwd
1275            | SseOpcode::Pmaxsw
1276            | SseOpcode::Pmaxub
1277            | SseOpcode::Pminsw
1278            | SseOpcode::Pminub
1279            | SseOpcode::Pmovmskb
1280            | SseOpcode::Pmulhw
1281            | SseOpcode::Pmulhuw
1282            | SseOpcode::Pmullw
1283            | SseOpcode::Pmuludq
1284            | SseOpcode::Por
1285            | SseOpcode::Pshufd
1286            | SseOpcode::Psllw
1287            | SseOpcode::Pslld
1288            | SseOpcode::Psllq
1289            | SseOpcode::Psraw
1290            | SseOpcode::Psrad
1291            | SseOpcode::Psrlw
1292            | SseOpcode::Psrld
1293            | SseOpcode::Psrlq
1294            | SseOpcode::Psubb
1295            | SseOpcode::Psubd
1296            | SseOpcode::Psubq
1297            | SseOpcode::Psubw
1298            | SseOpcode::Psubsb
1299            | SseOpcode::Psubsw
1300            | SseOpcode::Psubusb
1301            | SseOpcode::Psubusw
1302            | SseOpcode::Punpckhbw
1303            | SseOpcode::Punpckhwd
1304            | SseOpcode::Punpcklbw
1305            | SseOpcode::Punpcklwd
1306            | SseOpcode::Pxor
1307            | SseOpcode::Sqrtpd
1308            | SseOpcode::Sqrtsd
1309            | SseOpcode::Subpd
1310            | SseOpcode::Subsd
1311            | SseOpcode::Ucomisd
1312            | SseOpcode::Xorpd
1313            | SseOpcode::Punpckldq
1314            | SseOpcode::Punpckhdq
1315            | SseOpcode::Punpcklqdq
1316            | SseOpcode::Punpckhqdq
1317            | SseOpcode::Pshuflw
1318            | SseOpcode::Pshufhw
1319            | SseOpcode::Unpcklpd => SSE2,
1320
1321            SseOpcode::Pabsb
1322            | SseOpcode::Pabsw
1323            | SseOpcode::Pabsd
1324            | SseOpcode::Palignr
1325            | SseOpcode::Pmulhrsw
1326            | SseOpcode::Pshufb
1327            | SseOpcode::Phaddw
1328            | SseOpcode::Phaddd
1329            | SseOpcode::Pmaddubsw
1330            | SseOpcode::Movddup => SSSE3,
1331
1332            SseOpcode::Blendvpd
1333            | SseOpcode::Blendvps
1334            | SseOpcode::Insertps
1335            | SseOpcode::Packusdw
1336            | SseOpcode::Pblendvb
1337            | SseOpcode::Pcmpeqq
1338            | SseOpcode::Pextrb
1339            | SseOpcode::Pextrd
1340            | SseOpcode::Pextrq
1341            | SseOpcode::Pinsrb
1342            | SseOpcode::Pinsrd
1343            | SseOpcode::Pmaxsb
1344            | SseOpcode::Pmaxsd
1345            | SseOpcode::Pmaxuw
1346            | SseOpcode::Pmaxud
1347            | SseOpcode::Pminsb
1348            | SseOpcode::Pminsd
1349            | SseOpcode::Pminuw
1350            | SseOpcode::Pminud
1351            | SseOpcode::Pmovsxbd
1352            | SseOpcode::Pmovsxbw
1353            | SseOpcode::Pmovsxbq
1354            | SseOpcode::Pmovsxwd
1355            | SseOpcode::Pmovsxwq
1356            | SseOpcode::Pmovsxdq
1357            | SseOpcode::Pmovzxbd
1358            | SseOpcode::Pmovzxbw
1359            | SseOpcode::Pmovzxbq
1360            | SseOpcode::Pmovzxwd
1361            | SseOpcode::Pmovzxwq
1362            | SseOpcode::Pmovzxdq
1363            | SseOpcode::Pmuldq
1364            | SseOpcode::Pmulld
1365            | SseOpcode::Ptest
1366            | SseOpcode::Roundps
1367            | SseOpcode::Roundpd
1368            | SseOpcode::Roundss
1369            | SseOpcode::Roundsd
1370            | SseOpcode::Pblendw => SSE41,
1371
1372            SseOpcode::Pcmpgtq => SSE42,
1373        }
1374    }
1375
1376    /// Returns the src operand size for an instruction.
1377    pub(crate) fn src_size(&self) -> u8 {
1378        match self {
1379            SseOpcode::Movd => 4,
1380            _ => 8,
1381        }
1382    }
1383
1384    /// Is `src2` with this opcode a scalar, as for lane insertions?
1385    pub(crate) fn has_scalar_src2(self) -> bool {
1386        match self {
1387            SseOpcode::Pinsrb | SseOpcode::Pinsrw | SseOpcode::Pinsrd => true,
1388            SseOpcode::Pmovsxbw
1389            | SseOpcode::Pmovsxbd
1390            | SseOpcode::Pmovsxbq
1391            | SseOpcode::Pmovsxwd
1392            | SseOpcode::Pmovsxwq
1393            | SseOpcode::Pmovsxdq => true,
1394            SseOpcode::Pmovzxbw
1395            | SseOpcode::Pmovzxbd
1396            | SseOpcode::Pmovzxbq
1397            | SseOpcode::Pmovzxwd
1398            | SseOpcode::Pmovzxwq
1399            | SseOpcode::Pmovzxdq => true,
1400            _ => false,
1401        }
1402    }
1403}
1404
1405impl fmt::Debug for SseOpcode {
1406    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1407        let name = match self {
1408            SseOpcode::Addps => "addps",
1409            SseOpcode::Addpd => "addpd",
1410            SseOpcode::Addss => "addss",
1411            SseOpcode::Addsd => "addsd",
1412            SseOpcode::Andpd => "andpd",
1413            SseOpcode::Andps => "andps",
1414            SseOpcode::Andnps => "andnps",
1415            SseOpcode::Andnpd => "andnpd",
1416            SseOpcode::Blendvpd => "blendvpd",
1417            SseOpcode::Blendvps => "blendvps",
1418            SseOpcode::Cmpps => "cmpps",
1419            SseOpcode::Cmppd => "cmppd",
1420            SseOpcode::Cmpss => "cmpss",
1421            SseOpcode::Cmpsd => "cmpsd",
1422            SseOpcode::Comiss => "comiss",
1423            SseOpcode::Comisd => "comisd",
1424            SseOpcode::Cvtdq2ps => "cvtdq2ps",
1425            SseOpcode::Cvtdq2pd => "cvtdq2pd",
1426            SseOpcode::Cvtpd2ps => "cvtpd2ps",
1427            SseOpcode::Cvtps2pd => "cvtps2pd",
1428            SseOpcode::Cvtsd2ss => "cvtsd2ss",
1429            SseOpcode::Cvtsd2si => "cvtsd2si",
1430            SseOpcode::Cvtsi2ss => "cvtsi2ss",
1431            SseOpcode::Cvtsi2sd => "cvtsi2sd",
1432            SseOpcode::Cvtss2si => "cvtss2si",
1433            SseOpcode::Cvtss2sd => "cvtss2sd",
1434            SseOpcode::Cvttpd2dq => "cvttpd2dq",
1435            SseOpcode::Cvttps2dq => "cvttps2dq",
1436            SseOpcode::Cvttss2si => "cvttss2si",
1437            SseOpcode::Cvttsd2si => "cvttsd2si",
1438            SseOpcode::Divps => "divps",
1439            SseOpcode::Divpd => "divpd",
1440            SseOpcode::Divss => "divss",
1441            SseOpcode::Divsd => "divsd",
1442            SseOpcode::Insertps => "insertps",
1443            SseOpcode::Maxps => "maxps",
1444            SseOpcode::Maxpd => "maxpd",
1445            SseOpcode::Maxss => "maxss",
1446            SseOpcode::Maxsd => "maxsd",
1447            SseOpcode::Minps => "minps",
1448            SseOpcode::Minpd => "minpd",
1449            SseOpcode::Minss => "minss",
1450            SseOpcode::Minsd => "minsd",
1451            SseOpcode::Movaps => "movaps",
1452            SseOpcode::Movapd => "movapd",
1453            SseOpcode::Movd => "movd",
1454            SseOpcode::Movdqa => "movdqa",
1455            SseOpcode::Movdqu => "movdqu",
1456            SseOpcode::Movlhps => "movlhps",
1457            SseOpcode::Movmskps => "movmskps",
1458            SseOpcode::Movmskpd => "movmskpd",
1459            SseOpcode::Movq => "movq",
1460            SseOpcode::Movss => "movss",
1461            SseOpcode::Movsd => "movsd",
1462            SseOpcode::Movups => "movups",
1463            SseOpcode::Movupd => "movupd",
1464            SseOpcode::Mulps => "mulps",
1465            SseOpcode::Mulpd => "mulpd",
1466            SseOpcode::Mulss => "mulss",
1467            SseOpcode::Mulsd => "mulsd",
1468            SseOpcode::Orpd => "orpd",
1469            SseOpcode::Orps => "orps",
1470            SseOpcode::Pabsb => "pabsb",
1471            SseOpcode::Pabsw => "pabsw",
1472            SseOpcode::Pabsd => "pabsd",
1473            SseOpcode::Packssdw => "packssdw",
1474            SseOpcode::Packsswb => "packsswb",
1475            SseOpcode::Packusdw => "packusdw",
1476            SseOpcode::Packuswb => "packuswb",
1477            SseOpcode::Paddb => "paddb",
1478            SseOpcode::Paddd => "paddd",
1479            SseOpcode::Paddq => "paddq",
1480            SseOpcode::Paddw => "paddw",
1481            SseOpcode::Paddsb => "paddsb",
1482            SseOpcode::Paddsw => "paddsw",
1483            SseOpcode::Paddusb => "paddusb",
1484            SseOpcode::Paddusw => "paddusw",
1485            SseOpcode::Palignr => "palignr",
1486            SseOpcode::Pand => "pand",
1487            SseOpcode::Pandn => "pandn",
1488            SseOpcode::Pavgb => "pavgb",
1489            SseOpcode::Pavgw => "pavgw",
1490            SseOpcode::Pblendvb => "pblendvb",
1491            SseOpcode::Pcmpeqb => "pcmpeqb",
1492            SseOpcode::Pcmpeqw => "pcmpeqw",
1493            SseOpcode::Pcmpeqd => "pcmpeqd",
1494            SseOpcode::Pcmpeqq => "pcmpeqq",
1495            SseOpcode::Pcmpgtb => "pcmpgtb",
1496            SseOpcode::Pcmpgtw => "pcmpgtw",
1497            SseOpcode::Pcmpgtd => "pcmpgtd",
1498            SseOpcode::Pcmpgtq => "pcmpgtq",
1499            SseOpcode::Pextrb => "pextrb",
1500            SseOpcode::Pextrw => "pextrw",
1501            SseOpcode::Pextrd => "pextrd",
1502            SseOpcode::Pextrq => "pextrq",
1503            SseOpcode::Pinsrb => "pinsrb",
1504            SseOpcode::Pinsrw => "pinsrw",
1505            SseOpcode::Pinsrd => "pinsrd",
1506            SseOpcode::Pmaddubsw => "pmaddubsw",
1507            SseOpcode::Pmaddwd => "pmaddwd",
1508            SseOpcode::Pmaxsb => "pmaxsb",
1509            SseOpcode::Pmaxsw => "pmaxsw",
1510            SseOpcode::Pmaxsd => "pmaxsd",
1511            SseOpcode::Pmaxub => "pmaxub",
1512            SseOpcode::Pmaxuw => "pmaxuw",
1513            SseOpcode::Pmaxud => "pmaxud",
1514            SseOpcode::Pminsb => "pminsb",
1515            SseOpcode::Pminsw => "pminsw",
1516            SseOpcode::Pminsd => "pminsd",
1517            SseOpcode::Pminub => "pminub",
1518            SseOpcode::Pminuw => "pminuw",
1519            SseOpcode::Pminud => "pminud",
1520            SseOpcode::Pmovmskb => "pmovmskb",
1521            SseOpcode::Pmovsxbd => "pmovsxbd",
1522            SseOpcode::Pmovsxbw => "pmovsxbw",
1523            SseOpcode::Pmovsxbq => "pmovsxbq",
1524            SseOpcode::Pmovsxwd => "pmovsxwd",
1525            SseOpcode::Pmovsxwq => "pmovsxwq",
1526            SseOpcode::Pmovsxdq => "pmovsxdq",
1527            SseOpcode::Pmovzxbd => "pmovzxbd",
1528            SseOpcode::Pmovzxbw => "pmovzxbw",
1529            SseOpcode::Pmovzxbq => "pmovzxbq",
1530            SseOpcode::Pmovzxwd => "pmovzxwd",
1531            SseOpcode::Pmovzxwq => "pmovzxwq",
1532            SseOpcode::Pmovzxdq => "pmovzxdq",
1533            SseOpcode::Pmuldq => "pmuldq",
1534            SseOpcode::Pmulhw => "pmulhw",
1535            SseOpcode::Pmulhuw => "pmulhuw",
1536            SseOpcode::Pmulhrsw => "pmulhrsw",
1537            SseOpcode::Pmulld => "pmulld",
1538            SseOpcode::Pmullw => "pmullw",
1539            SseOpcode::Pmuludq => "pmuludq",
1540            SseOpcode::Por => "por",
1541            SseOpcode::Pshufb => "pshufb",
1542            SseOpcode::Pshufd => "pshufd",
1543            SseOpcode::Psllw => "psllw",
1544            SseOpcode::Pslld => "pslld",
1545            SseOpcode::Psllq => "psllq",
1546            SseOpcode::Psraw => "psraw",
1547            SseOpcode::Psrad => "psrad",
1548            SseOpcode::Psrlw => "psrlw",
1549            SseOpcode::Psrld => "psrld",
1550            SseOpcode::Psrlq => "psrlq",
1551            SseOpcode::Psubb => "psubb",
1552            SseOpcode::Psubd => "psubd",
1553            SseOpcode::Psubq => "psubq",
1554            SseOpcode::Psubw => "psubw",
1555            SseOpcode::Psubsb => "psubsb",
1556            SseOpcode::Psubsw => "psubsw",
1557            SseOpcode::Psubusb => "psubusb",
1558            SseOpcode::Psubusw => "psubusw",
1559            SseOpcode::Ptest => "ptest",
1560            SseOpcode::Punpckhbw => "punpckhbw",
1561            SseOpcode::Punpckhwd => "punpckhwd",
1562            SseOpcode::Punpcklbw => "punpcklbw",
1563            SseOpcode::Punpcklwd => "punpcklwd",
1564            SseOpcode::Pxor => "pxor",
1565            SseOpcode::Rcpss => "rcpss",
1566            SseOpcode::Roundps => "roundps",
1567            SseOpcode::Roundpd => "roundpd",
1568            SseOpcode::Roundss => "roundss",
1569            SseOpcode::Roundsd => "roundsd",
1570            SseOpcode::Rsqrtss => "rsqrtss",
1571            SseOpcode::Shufps => "shufps",
1572            SseOpcode::Sqrtps => "sqrtps",
1573            SseOpcode::Sqrtpd => "sqrtpd",
1574            SseOpcode::Sqrtss => "sqrtss",
1575            SseOpcode::Sqrtsd => "sqrtsd",
1576            SseOpcode::Subps => "subps",
1577            SseOpcode::Subpd => "subpd",
1578            SseOpcode::Subss => "subss",
1579            SseOpcode::Subsd => "subsd",
1580            SseOpcode::Ucomiss => "ucomiss",
1581            SseOpcode::Ucomisd => "ucomisd",
1582            SseOpcode::Unpcklps => "unpcklps",
1583            SseOpcode::Unpckhps => "unpckhps",
1584            SseOpcode::Xorps => "xorps",
1585            SseOpcode::Xorpd => "xorpd",
1586            SseOpcode::Phaddw => "phaddw",
1587            SseOpcode::Phaddd => "phaddd",
1588            SseOpcode::Punpckldq => "punpckldq",
1589            SseOpcode::Punpckhdq => "punpckhdq",
1590            SseOpcode::Punpcklqdq => "punpcklqdq",
1591            SseOpcode::Punpckhqdq => "punpckhqdq",
1592            SseOpcode::Pshuflw => "pshuflw",
1593            SseOpcode::Pshufhw => "pshufhw",
1594            SseOpcode::Pblendw => "pblendw",
1595            SseOpcode::Movddup => "movddup",
1596            SseOpcode::Unpcklpd => "unpcklpd",
1597        };
1598        write!(fmt, "{name}")
1599    }
1600}
1601
1602impl fmt::Display for SseOpcode {
1603    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1604        fmt::Debug::fmt(self, f)
1605    }
1606}
1607
1608pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;
1609
1610impl AvxOpcode {
1611    /// Which `InstructionSet`s support the opcode?
1612    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1613        match self {
1614            AvxOpcode::Vfmadd213ss
1615            | AvxOpcode::Vfmadd213sd
1616            | AvxOpcode::Vfmadd213ps
1617            | AvxOpcode::Vfmadd213pd
1618            | AvxOpcode::Vfmadd132ss
1619            | AvxOpcode::Vfmadd132sd
1620            | AvxOpcode::Vfmadd132ps
1621            | AvxOpcode::Vfmadd132pd
1622            | AvxOpcode::Vfnmadd213ss
1623            | AvxOpcode::Vfnmadd213sd
1624            | AvxOpcode::Vfnmadd213ps
1625            | AvxOpcode::Vfnmadd213pd
1626            | AvxOpcode::Vfnmadd132ss
1627            | AvxOpcode::Vfnmadd132sd
1628            | AvxOpcode::Vfnmadd132ps
1629            | AvxOpcode::Vfnmadd132pd
1630            | AvxOpcode::Vfmsub213ss
1631            | AvxOpcode::Vfmsub213sd
1632            | AvxOpcode::Vfmsub213ps
1633            | AvxOpcode::Vfmsub213pd
1634            | AvxOpcode::Vfmsub132ss
1635            | AvxOpcode::Vfmsub132sd
1636            | AvxOpcode::Vfmsub132ps
1637            | AvxOpcode::Vfmsub132pd
1638            | AvxOpcode::Vfnmsub213ss
1639            | AvxOpcode::Vfnmsub213sd
1640            | AvxOpcode::Vfnmsub213ps
1641            | AvxOpcode::Vfnmsub213pd
1642            | AvxOpcode::Vfnmsub132ss
1643            | AvxOpcode::Vfnmsub132sd
1644            | AvxOpcode::Vfnmsub132ps
1645            | AvxOpcode::Vfnmsub132pd => smallvec![InstructionSet::FMA],
1646            AvxOpcode::Vminps
1647            | AvxOpcode::Vminpd
1648            | AvxOpcode::Vmaxps
1649            | AvxOpcode::Vmaxpd
1650            | AvxOpcode::Vandnps
1651            | AvxOpcode::Vandnpd
1652            | AvxOpcode::Vpandn
1653            | AvxOpcode::Vcmpps
1654            | AvxOpcode::Vcmppd
1655            | AvxOpcode::Vpsrlw
1656            | AvxOpcode::Vpsrld
1657            | AvxOpcode::Vpsrlq
1658            | AvxOpcode::Vpaddb
1659            | AvxOpcode::Vpaddw
1660            | AvxOpcode::Vpaddd
1661            | AvxOpcode::Vpaddq
1662            | AvxOpcode::Vpaddsb
1663            | AvxOpcode::Vpaddsw
1664            | AvxOpcode::Vpaddusb
1665            | AvxOpcode::Vpaddusw
1666            | AvxOpcode::Vpsubb
1667            | AvxOpcode::Vpsubw
1668            | AvxOpcode::Vpsubd
1669            | AvxOpcode::Vpsubq
1670            | AvxOpcode::Vpsubsb
1671            | AvxOpcode::Vpsubsw
1672            | AvxOpcode::Vpsubusb
1673            | AvxOpcode::Vpsubusw
1674            | AvxOpcode::Vpavgb
1675            | AvxOpcode::Vpavgw
1676            | AvxOpcode::Vpand
1677            | AvxOpcode::Vandps
1678            | AvxOpcode::Vandpd
1679            | AvxOpcode::Vpor
1680            | AvxOpcode::Vorps
1681            | AvxOpcode::Vorpd
1682            | AvxOpcode::Vpxor
1683            | AvxOpcode::Vxorps
1684            | AvxOpcode::Vxorpd
1685            | AvxOpcode::Vpmullw
1686            | AvxOpcode::Vpmulld
1687            | AvxOpcode::Vpmulhw
1688            | AvxOpcode::Vpmulhd
1689            | AvxOpcode::Vpmulhrsw
1690            | AvxOpcode::Vpmulhuw
1691            | AvxOpcode::Vpmuldq
1692            | AvxOpcode::Vpmuludq
1693            | AvxOpcode::Vpunpckhwd
1694            | AvxOpcode::Vpunpcklwd
1695            | AvxOpcode::Vunpcklps
1696            | AvxOpcode::Vunpckhps
1697            | AvxOpcode::Vaddps
1698            | AvxOpcode::Vaddpd
1699            | AvxOpcode::Vsubps
1700            | AvxOpcode::Vsubpd
1701            | AvxOpcode::Vmulps
1702            | AvxOpcode::Vmulpd
1703            | AvxOpcode::Vdivps
1704            | AvxOpcode::Vdivpd
1705            | AvxOpcode::Vpcmpeqb
1706            | AvxOpcode::Vpcmpeqw
1707            | AvxOpcode::Vpcmpeqd
1708            | AvxOpcode::Vpcmpeqq
1709            | AvxOpcode::Vpcmpgtb
1710            | AvxOpcode::Vpcmpgtw
1711            | AvxOpcode::Vpcmpgtd
1712            | AvxOpcode::Vpcmpgtq
1713            | AvxOpcode::Vblendvps
1714            | AvxOpcode::Vblendvpd
1715            | AvxOpcode::Vpblendvb
1716            | AvxOpcode::Vmovlhps
1717            | AvxOpcode::Vpminsb
1718            | AvxOpcode::Vpminsw
1719            | AvxOpcode::Vpminsd
1720            | AvxOpcode::Vpminub
1721            | AvxOpcode::Vpminuw
1722            | AvxOpcode::Vpminud
1723            | AvxOpcode::Vpmaxsb
1724            | AvxOpcode::Vpmaxsw
1725            | AvxOpcode::Vpmaxsd
1726            | AvxOpcode::Vpmaxub
1727            | AvxOpcode::Vpmaxuw
1728            | AvxOpcode::Vpmaxud
1729            | AvxOpcode::Vpunpcklbw
1730            | AvxOpcode::Vpunpckhbw
1731            | AvxOpcode::Vpacksswb
1732            | AvxOpcode::Vpackssdw
1733            | AvxOpcode::Vpackuswb
1734            | AvxOpcode::Vpackusdw
1735            | AvxOpcode::Vpalignr
1736            | AvxOpcode::Vpinsrb
1737            | AvxOpcode::Vpinsrw
1738            | AvxOpcode::Vpinsrd
1739            | AvxOpcode::Vpinsrq
1740            | AvxOpcode::Vpmaddwd
1741            | AvxOpcode::Vpmaddubsw
1742            | AvxOpcode::Vinsertps
1743            | AvxOpcode::Vpshufb
1744            | AvxOpcode::Vshufps
1745            | AvxOpcode::Vpsllw
1746            | AvxOpcode::Vpslld
1747            | AvxOpcode::Vpsllq
1748            | AvxOpcode::Vpsraw
1749            | AvxOpcode::Vpsrad
1750            | AvxOpcode::Vpmovsxbw
1751            | AvxOpcode::Vpmovzxbw
1752            | AvxOpcode::Vpmovsxwd
1753            | AvxOpcode::Vpmovzxwd
1754            | AvxOpcode::Vpmovsxdq
1755            | AvxOpcode::Vpmovzxdq
1756            | AvxOpcode::Vaddss
1757            | AvxOpcode::Vaddsd
1758            | AvxOpcode::Vmulss
1759            | AvxOpcode::Vmulsd
1760            | AvxOpcode::Vsubss
1761            | AvxOpcode::Vsubsd
1762            | AvxOpcode::Vdivss
1763            | AvxOpcode::Vdivsd
1764            | AvxOpcode::Vpabsb
1765            | AvxOpcode::Vpabsw
1766            | AvxOpcode::Vpabsd
1767            | AvxOpcode::Vminss
1768            | AvxOpcode::Vminsd
1769            | AvxOpcode::Vmaxss
1770            | AvxOpcode::Vmaxsd
1771            | AvxOpcode::Vsqrtps
1772            | AvxOpcode::Vsqrtpd
1773            | AvxOpcode::Vroundpd
1774            | AvxOpcode::Vroundps
1775            | AvxOpcode::Vcvtdq2pd
1776            | AvxOpcode::Vcvtdq2ps
1777            | AvxOpcode::Vcvtpd2ps
1778            | AvxOpcode::Vcvtps2pd
1779            | AvxOpcode::Vcvttpd2dq
1780            | AvxOpcode::Vcvttps2dq
1781            | AvxOpcode::Vphaddw
1782            | AvxOpcode::Vphaddd
1783            | AvxOpcode::Vpunpckldq
1784            | AvxOpcode::Vpunpckhdq
1785            | AvxOpcode::Vpunpcklqdq
1786            | AvxOpcode::Vpunpckhqdq
1787            | AvxOpcode::Vpshuflw
1788            | AvxOpcode::Vpshufhw
1789            | AvxOpcode::Vpshufd
1790            | AvxOpcode::Vmovss
1791            | AvxOpcode::Vmovsd
1792            | AvxOpcode::Vmovups
1793            | AvxOpcode::Vmovupd
1794            | AvxOpcode::Vmovdqu
1795            | AvxOpcode::Vpextrb
1796            | AvxOpcode::Vpextrw
1797            | AvxOpcode::Vpextrd
1798            | AvxOpcode::Vpextrq
1799            | AvxOpcode::Vpblendw
1800            | AvxOpcode::Vmovddup
1801            | AvxOpcode::Vbroadcastss
1802            | AvxOpcode::Vmovd
1803            | AvxOpcode::Vmovq
1804            | AvxOpcode::Vmovmskps
1805            | AvxOpcode::Vmovmskpd
1806            | AvxOpcode::Vpmovmskb
1807            | AvxOpcode::Vcvtsi2ss
1808            | AvxOpcode::Vcvtsi2sd
1809            | AvxOpcode::Vcvtss2sd
1810            | AvxOpcode::Vcvtsd2ss
1811            | AvxOpcode::Vsqrtss
1812            | AvxOpcode::Vsqrtsd
1813            | AvxOpcode::Vroundss
1814            | AvxOpcode::Vroundsd
1815            | AvxOpcode::Vunpcklpd
1816            | AvxOpcode::Vptest
1817            | AvxOpcode::Vucomiss
1818            | AvxOpcode::Vucomisd => {
1819                smallvec![InstructionSet::AVX]
1820            }
1821
1822            AvxOpcode::Vpbroadcastb | AvxOpcode::Vpbroadcastw | AvxOpcode::Vpbroadcastd => {
1823                smallvec![InstructionSet::AVX2]
1824            }
1825        }
1826    }
1827
1828    /// Is the opcode known to be commutative?
1829    ///
1830    /// Note that this method is not exhaustive, and there may be commutative
1831    /// opcodes that we don't recognize as commutative.
1832    pub(crate) fn is_commutative(&self) -> bool {
1833        match *self {
1834            AvxOpcode::Vpaddb
1835            | AvxOpcode::Vpaddw
1836            | AvxOpcode::Vpaddd
1837            | AvxOpcode::Vpaddq
1838            | AvxOpcode::Vpaddsb
1839            | AvxOpcode::Vpaddsw
1840            | AvxOpcode::Vpaddusb
1841            | AvxOpcode::Vpaddusw
1842            | AvxOpcode::Vpand
1843            | AvxOpcode::Vandps
1844            | AvxOpcode::Vandpd
1845            | AvxOpcode::Vpor
1846            | AvxOpcode::Vorps
1847            | AvxOpcode::Vorpd
1848            | AvxOpcode::Vpxor
1849            | AvxOpcode::Vxorps
1850            | AvxOpcode::Vxorpd
1851            | AvxOpcode::Vpmuldq
1852            | AvxOpcode::Vpmuludq
1853            | AvxOpcode::Vaddps
1854            | AvxOpcode::Vaddpd
1855            | AvxOpcode::Vmulps
1856            | AvxOpcode::Vmulpd
1857            | AvxOpcode::Vpcmpeqb
1858            | AvxOpcode::Vpcmpeqw
1859            | AvxOpcode::Vpcmpeqd
1860            | AvxOpcode::Vpcmpeqq
1861            | AvxOpcode::Vaddss
1862            | AvxOpcode::Vaddsd
1863            | AvxOpcode::Vmulss
1864            | AvxOpcode::Vmulsd => true,
1865            _ => false,
1866        }
1867    }
1868}
1869
1870impl fmt::Display for AvxOpcode {
1871    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1872        format!("{self:?}").to_lowercase().fmt(f)
1873    }
1874}
1875
1876#[derive(Copy, Clone, PartialEq)]
1877#[allow(missing_docs)]
1878pub enum Avx512TupleType {
1879    Full,
1880    FullMem,
1881    Mem128,
1882}
1883
1884pub use crate::isa::x64::lower::isle::generated_code::Avx512Opcode;
1885
1886impl Avx512Opcode {
1887    /// Which `InstructionSet`s support the opcode?
1888    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1889        match self {
1890            Avx512Opcode::Vcvtudq2ps
1891            | Avx512Opcode::Vpabsq
1892            | Avx512Opcode::Vpsraq
1893            | Avx512Opcode::VpsraqImm => {
1894                smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL]
1895            }
1896            Avx512Opcode::Vpermi2b => {
1897                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512VBMI]
1898            }
1899            Avx512Opcode::Vpmullq => smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512DQ],
1900            Avx512Opcode::Vpopcntb => {
1901                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512BITALG]
1902            }
1903        }
1904    }
1905
1906    /// What is the "TupleType" of this opcode, which affects the scaling factor
1907    /// for 8-bit displacements when this instruction uses memory operands.
1908    ///
1909    /// This can be found in the encoding table for each instruction and is
1910    /// interpreted according to Table 2-34 and 2-35 in the Intel instruction
1911    /// manual.
1912    pub fn tuple_type(&self) -> Avx512TupleType {
1913        use Avx512Opcode::*;
1914        use Avx512TupleType::*;
1915
1916        match self {
1917            Vcvtudq2ps | Vpabsq | Vpmullq | VpsraqImm => Full,
1918            Vpermi2b | Vpopcntb => FullMem,
1919            Vpsraq => Mem128,
1920        }
1921    }
1922}
1923
1924impl fmt::Display for Avx512Opcode {
1925    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1926        let s = format!("{self:?}");
1927        f.write_str(&s.to_lowercase())
1928    }
1929}
1930
1931/// This defines the ways a value can be extended: either signed- or zero-extension, or none for
1932/// types that are not extended. Contrast with [ExtMode], which defines the widths from and to which
1933/// values can be extended.
1934#[allow(dead_code)]
1935#[derive(Clone, PartialEq)]
1936pub enum ExtKind {
1937    /// No extension.
1938    None,
1939    /// Sign-extend.
1940    SignExtend,
1941    /// Zero-extend.
1942    ZeroExtend,
1943}
1944
1945/// These indicate ways of extending (widening) a value, using the Intel
1946/// naming: B(yte) = u8, W(ord) = u16, L(ong)word = u32, Q(uad)word = u64
1947#[derive(Clone, PartialEq)]
1948pub enum ExtMode {
1949    /// Byte -> Longword.
1950    BL,
1951    /// Byte -> Quadword.
1952    BQ,
1953    /// Word -> Longword.
1954    WL,
1955    /// Word -> Quadword.
1956    WQ,
1957    /// Longword -> Quadword.
1958    LQ,
1959}
1960
1961impl ExtMode {
1962    /// Calculate the `ExtMode` from passed bit lengths of the from/to types.
1963    pub(crate) fn new(from_bits: u16, to_bits: u16) -> Option<ExtMode> {
1964        match (from_bits, to_bits) {
1965            (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),
1966            (1, 64) | (8, 64) => Some(ExtMode::BQ),
1967            (16, 32) => Some(ExtMode::WL),
1968            (16, 64) => Some(ExtMode::WQ),
1969            (32, 64) => Some(ExtMode::LQ),
1970            _ => None,
1971        }
1972    }
1973
1974    /// Return the source register size in bytes.
1975    pub(crate) fn src_size(&self) -> u8 {
1976        match self {
1977            ExtMode::BL | ExtMode::BQ => 1,
1978            ExtMode::WL | ExtMode::WQ => 2,
1979            ExtMode::LQ => 4,
1980        }
1981    }
1982
1983    /// Return the destination register size in bytes.
1984    pub(crate) fn dst_size(&self) -> u8 {
1985        match self {
1986            ExtMode::BL | ExtMode::WL => 4,
1987            ExtMode::BQ | ExtMode::WQ | ExtMode::LQ => 8,
1988        }
1989    }
1990
1991    /// Source size, as an integer type.
1992    pub(crate) fn src_type(&self) -> Type {
1993        match self {
1994            ExtMode::BL | ExtMode::BQ => I8,
1995            ExtMode::WL | ExtMode::WQ => I16,
1996            ExtMode::LQ => I32,
1997        }
1998    }
1999}
2000
2001impl fmt::Debug for ExtMode {
2002    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2003        let name = match self {
2004            ExtMode::BL => "bl",
2005            ExtMode::BQ => "bq",
2006            ExtMode::WL => "wl",
2007            ExtMode::WQ => "wq",
2008            ExtMode::LQ => "lq",
2009        };
2010        write!(fmt, "{name}")
2011    }
2012}
2013
2014impl fmt::Display for ExtMode {
2015    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2016        fmt::Debug::fmt(self, f)
2017    }
2018}
2019
2020/// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
2021#[derive(Clone, Copy)]
2022pub enum ShiftKind {
2023    /// Left shift.
2024    ShiftLeft,
2025    /// Inserts zeros in the most significant bits.
2026    ShiftRightLogical,
2027    /// Replicates the sign bit in the most significant bits.
2028    ShiftRightArithmetic,
2029    /// Left rotation.
2030    RotateLeft,
2031    /// Right rotation.
2032    RotateRight,
2033}
2034
2035impl fmt::Debug for ShiftKind {
2036    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2037        let name = match self {
2038            ShiftKind::ShiftLeft => "shl",
2039            ShiftKind::ShiftRightLogical => "shr",
2040            ShiftKind::ShiftRightArithmetic => "sar",
2041            ShiftKind::RotateLeft => "rol",
2042            ShiftKind::RotateRight => "ror",
2043        };
2044        write!(fmt, "{name}")
2045    }
2046}
2047
2048impl fmt::Display for ShiftKind {
2049    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2050        fmt::Debug::fmt(self, f)
2051    }
2052}
2053
2054/// These indicate condition code tests.  Not all are represented since not all are useful in
2055/// compiler-generated code.
2056#[derive(Copy, Clone, PartialEq, Eq)]
2057#[repr(u8)]
2058pub enum CC {
2059    ///  overflow
2060    O = 0,
2061    /// no overflow
2062    NO = 1,
2063
2064    /// < unsigned
2065    B = 2,
2066    /// >= unsigned
2067    NB = 3,
2068
2069    /// zero
2070    Z = 4,
2071    /// not-zero
2072    NZ = 5,
2073
2074    /// <= unsigned
2075    BE = 6,
2076    /// > unsigned
2077    NBE = 7,
2078
2079    /// negative
2080    S = 8,
2081    /// not-negative
2082    NS = 9,
2083
2084    /// < signed
2085    L = 12,
2086    /// >= signed
2087    NL = 13,
2088
2089    /// <= signed
2090    LE = 14,
2091    /// > signed
2092    NLE = 15,
2093
2094    /// parity
2095    P = 10,
2096
2097    /// not parity
2098    NP = 11,
2099}
2100
2101impl CC {
2102    pub(crate) fn from_intcc(intcc: IntCC) -> Self {
2103        match intcc {
2104            IntCC::Equal => CC::Z,
2105            IntCC::NotEqual => CC::NZ,
2106            IntCC::SignedGreaterThanOrEqual => CC::NL,
2107            IntCC::SignedGreaterThan => CC::NLE,
2108            IntCC::SignedLessThanOrEqual => CC::LE,
2109            IntCC::SignedLessThan => CC::L,
2110            IntCC::UnsignedGreaterThanOrEqual => CC::NB,
2111            IntCC::UnsignedGreaterThan => CC::NBE,
2112            IntCC::UnsignedLessThanOrEqual => CC::BE,
2113            IntCC::UnsignedLessThan => CC::B,
2114        }
2115    }
2116
2117    pub(crate) fn invert(&self) -> Self {
2118        match self {
2119            CC::O => CC::NO,
2120            CC::NO => CC::O,
2121
2122            CC::B => CC::NB,
2123            CC::NB => CC::B,
2124
2125            CC::Z => CC::NZ,
2126            CC::NZ => CC::Z,
2127
2128            CC::BE => CC::NBE,
2129            CC::NBE => CC::BE,
2130
2131            CC::S => CC::NS,
2132            CC::NS => CC::S,
2133
2134            CC::L => CC::NL,
2135            CC::NL => CC::L,
2136
2137            CC::LE => CC::NLE,
2138            CC::NLE => CC::LE,
2139
2140            CC::P => CC::NP,
2141            CC::NP => CC::P,
2142        }
2143    }
2144
2145    pub(crate) fn get_enc(self) -> u8 {
2146        self as u8
2147    }
2148}
2149
2150impl fmt::Debug for CC {
2151    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2152        let name = match self {
2153            CC::O => "o",
2154            CC::NO => "no",
2155            CC::B => "b",
2156            CC::NB => "nb",
2157            CC::Z => "z",
2158            CC::NZ => "nz",
2159            CC::BE => "be",
2160            CC::NBE => "nbe",
2161            CC::S => "s",
2162            CC::NS => "ns",
2163            CC::L => "l",
2164            CC::NL => "nl",
2165            CC::LE => "le",
2166            CC::NLE => "nle",
2167            CC::P => "p",
2168            CC::NP => "np",
2169        };
2170        write!(fmt, "{name}")
2171    }
2172}
2173
2174impl fmt::Display for CC {
2175    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2176        fmt::Debug::fmt(self, f)
2177    }
2178}
2179
2180/// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`,
2181/// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS
2182/// whereas [FcmpImm] is used as an immediate.
2183#[derive(Clone, Copy)]
2184pub enum FcmpImm {
2185    /// Equal comparison.
2186    Equal = 0x00,
2187    /// Less than comparison.
2188    LessThan = 0x01,
2189    /// Less than or equal comparison.
2190    LessThanOrEqual = 0x02,
2191    /// Unordered.
2192    Unordered = 0x03,
2193    /// Not equal comparison.
2194    NotEqual = 0x04,
2195    /// Unordered of greater than or equal comparison.
2196    UnorderedOrGreaterThanOrEqual = 0x05,
2197    /// Unordered or greater than comparison.
2198    UnorderedOrGreaterThan = 0x06,
2199    /// Ordered.
2200    Ordered = 0x07,
2201}
2202
2203impl FcmpImm {
2204    pub(crate) fn encode(self) -> u8 {
2205        self as u8
2206    }
2207}
2208
2209impl From<FloatCC> for FcmpImm {
2210    fn from(cond: FloatCC) -> Self {
2211        match cond {
2212            FloatCC::Equal => FcmpImm::Equal,
2213            FloatCC::LessThan => FcmpImm::LessThan,
2214            FloatCC::LessThanOrEqual => FcmpImm::LessThanOrEqual,
2215            FloatCC::Unordered => FcmpImm::Unordered,
2216            FloatCC::NotEqual => FcmpImm::NotEqual,
2217            FloatCC::UnorderedOrGreaterThanOrEqual => FcmpImm::UnorderedOrGreaterThanOrEqual,
2218            FloatCC::UnorderedOrGreaterThan => FcmpImm::UnorderedOrGreaterThan,
2219            FloatCC::Ordered => FcmpImm::Ordered,
2220            _ => panic!("unable to create comparison predicate for {cond}"),
2221        }
2222    }
2223}
2224
2225/// Encode the rounding modes used as part of the Rounding Control field.
2226/// Note, these rounding immediates only consider the rounding control field
2227/// (i.e. the rounding mode) which only take up the first two bits when encoded.
2228/// However the rounding immediate which this field helps make up, also includes
2229/// bits 3 and 4 which define the rounding select and precision mask respectively.
2230/// These two bits are not defined here and are implicitly set to zero when encoded.
2231#[derive(Clone, Copy)]
2232pub enum RoundImm {
2233    /// Round to nearest mode.
2234    RoundNearest = 0x00,
2235    /// Round down mode.
2236    RoundDown = 0x01,
2237    /// Round up mode.
2238    RoundUp = 0x02,
2239    /// Round to zero mode.
2240    RoundZero = 0x03,
2241}
2242
2243impl RoundImm {
2244    pub(crate) fn encode(self) -> u8 {
2245        self as u8
2246    }
2247}
2248
2249/// An operand's size in bits.
2250#[derive(Clone, Copy, PartialEq)]
2251pub enum OperandSize {
2252    /// 8-bit.
2253    Size8,
2254    /// 16-bit.
2255    Size16,
2256    /// 32-bit.
2257    Size32,
2258    /// 64-bit.
2259    Size64,
2260}
2261
2262impl OperandSize {
2263    pub(crate) fn from_bytes(num_bytes: u32) -> Self {
2264        match num_bytes {
2265            1 => OperandSize::Size8,
2266            2 => OperandSize::Size16,
2267            4 => OperandSize::Size32,
2268            8 => OperandSize::Size64,
2269            _ => unreachable!("Invalid OperandSize: {}", num_bytes),
2270        }
2271    }
2272
2273    // Computes the OperandSize for a given type.
2274    // For vectors, the OperandSize of the lanes is returned.
2275    pub(crate) fn from_ty(ty: Type) -> Self {
2276        Self::from_bytes(ty.lane_type().bytes())
2277    }
2278
2279    // Check that the value of self is one of the allowed sizes.
2280    pub(crate) fn is_one_of(&self, sizes: &[Self]) -> bool {
2281        sizes.iter().any(|val| *self == *val)
2282    }
2283
2284    pub(crate) fn to_bytes(&self) -> u8 {
2285        match self {
2286            Self::Size8 => 1,
2287            Self::Size16 => 2,
2288            Self::Size32 => 4,
2289            Self::Size64 => 8,
2290        }
2291    }
2292
2293    pub(crate) fn to_bits(&self) -> u8 {
2294        self.to_bytes() * 8
2295    }
2296
2297    pub(crate) fn to_type(&self) -> Type {
2298        match self {
2299            Self::Size8 => I8,
2300            Self::Size16 => I16,
2301            Self::Size32 => I32,
2302            Self::Size64 => I64,
2303        }
2304    }
2305}
2306
2307/// An x64 memory fence kind.
2308#[derive(Clone)]
2309#[allow(dead_code)]
2310pub enum FenceKind {
2311    /// `mfence` instruction ("Memory Fence")
2312    MFence,
2313    /// `lfence` instruction ("Load Fence")
2314    LFence,
2315    /// `sfence` instruction ("Store Fence")
2316    SFence,
2317}
cranelift_codegen/isa/x64/inst/args.rs

cranelift_codegen/isa/x64/inst/
args.rs
