Skip to main content

rustyn64_cpu/
pipeline.rs

1//! The VR4300's five-stage pipeline (ADR 0007).
2//!
3//! `IC` → `RF` → `EX` → `DC` → `WB` (VR4300 User's Manual §4.1, Figure 4-1):
4//! Instruction Cache fetch, Register Fetch, Execution, Data Cache fetch, Write
5//! Back. In-order, single-issue, with one architectural delay slot. At least 5
6//! `PCycle`s are required to execute an instruction, and up to five are in flight
7//! at once when the pipe flows.
8//!
9//! # Latches, not stages
10//!
11//! Five stages have four boundaries, and the state lives on the **boundaries**.
12//! [`Latch`] is what travels with an instruction as it advances.
13//!
14//! `in_delay_slot` riding in the latch rather than in a global CPU flag is
15//! load-bearing: a multi-cycle stall between a branch and its delay slot
16//! desynchronizes a global flag, and that is the classic bug in this area. With
17//! the flag attached to the instruction, `Cause.BD` and `EPC` come out right for
18//! free. `delay_slot_flag_survives_a_multi_cycle_stall` pins it.
19//!
20//! # Reverse step order is the latching
21//!
22//! [`Pipeline::advance`] runs **WB → DC → EX → RF → IC**. Each stage reads its
23//! input latch and writes its output latch, so running downstream-first means a
24//! stage's input still holds the *previous* cycle's value when it is read. No
25//! value can therefore propagate two stages in one cycle, and **no double
26//! buffering is needed** — the reverse order *is* the latching.
27//!
28//! This is a load-bearing invariant, not a style choice. Reversing it silently
29//! makes the pipeline one-cycle-too-fast; `a_value_advances_exactly_one_stage_per_cycle`
30//! is the guard.
31//!
32//! # Status
33//!
34//! **Structure only.** The stages move latches and account for time; they do not
35//! decode or execute yet (T-11-002 onward). What is real here is the shape, the
36//! stall/interlock mechanism, the delay-slot carriage, and the interrupt gate —
37//! the parts that cannot be retrofitted later without rewriting every consumer.
38
39use crate::Bus;
40use crate::alu::HiLo;
41use crate::cop0::Cop0;
42use crate::cop1::Cop1Control;
43use crate::decode::{Decoded, decode};
44use crate::exception;
45use crate::exec::{Cop0Access, Cop1Access, MemOp, Redirect, TlbOp, WriteBack, execute};
46use crate::fpr::Fpr;
47use crate::mem;
48use crate::regs::Regs;
49use crate::softfloat;
50use crate::tlb::Tlb;
51use serde::{Deserialize, Serialize};
52
53/// The five pipeline stages, in hardware order (UM §4.1, Figure 4-1).
54///
55/// Note the names: **IC** and **DC**, not `IF`/`DF`. The manual's whole interlock
56/// and exception taxonomy is stated stage-relative, so these spellings are what
57/// make a citation resolvable.
58#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
59pub enum Stage {
60    /// Instruction Cache fetch.
61    Ic,
62    /// Register Fetch.
63    Rf,
64    /// Execution.
65    Ex,
66    /// Data Cache fetch — the bus access, and where interrupts are sampled.
67    Dc,
68    /// Write Back.
69    Wb,
70}
71
72/// An aborting condition travelling down the pipe.
73///
74/// Deliberately **not** called `Fault`. UM §4.5 defines a *fault* as the union of
75/// interlocks and exceptions (Figure 4-11: Faults = Interlocks ∪ Exceptions,
76/// split into Stalls vs Abort), and CEN64 follows that wider usage. What rides in
77/// a latch here is only the aborting subset, so it carries the narrower name.
78#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
79pub enum Exception {
80    /// An interrupt was accepted (`Cause.IP` unmasked, `IE` set, `EXL`/`ERL` clear).
81    Interrupt,
82    /// Address error on an instruction fetch or data access.
83    ///
84    /// Carries the direction because the architecture does: `AdEL` (4) and
85    /// `AdES` (5) are **different** `ExcCode` values (UM Table 6-2, p. 172), and
86    /// a handler distinguishes them. An instruction fetch is a load.
87    AddressError {
88        /// The faulting access was a store.
89        store: bool,
90    },
91    /// Integer overflow (`ADD`, `ADDI`, `SUB`, `DADD`, …).
92    Overflow,
93    /// `SYSCALL`.
94    Syscall,
95    /// `BREAK`.
96    Breakpoint,
97    /// A conditional trap (`TGE`, `TEQ`, `TNEI`, …) whose condition held.
98    Trap,
99    /// A reserved / unimplemented opcode.
100    ReservedInstruction,
101    /// A reserved encoding **within a coprocessor's** instruction space.
102    ///
103    /// Reports the same `ExcCode` as [`Exception::ReservedInstruction`] but
104    /// also sets `Cause.CE` to the coprocessor number. A plain
105    /// `ReservedInstruction` leaves `CE` at zero, so the two are not
106    /// interchangeable — n64-systemtest checks the whole `Cause` register, and
107    /// `DCFC2` with `CU2` set expects `CE = 2`.
108    CoprocessorReserved {
109        /// Which coprocessor, for `Cause.CE`.
110        unit: u8,
111    },
112    /// A coprocessor instruction with that unit disabled in `Status.CU`.
113    CoprocessorUnusable {
114        /// Which coprocessor, for `Cause.CE`.
115        unit: u8,
116    },
117    /// **Non-maskable interrupt** (UM §6.4.6).
118    ///
119    /// Not one of the `Cause.ExcCode` exceptions: it writes no `Cause`, saves to
120    /// `ErrorEPC` rather than `EPC`, sets `ERL` rather than `EXL`, and is taken
121    /// *"regardless of the settings of the EXL, ERL, and the IE bits"*. It rides
122    /// in this enum anyway so it can reuse the pipeline-flush machinery every
123    /// other exception uses; [`crate::exception::dispatch`] intercepts it before
124    /// any of the general path runs.
125    Nmi,
126    /// A TLB refill — no entry matched. Takes the **refill** vector.
127    TlbRefill {
128        /// The faulting access was a store (`TLBS` rather than `TLBL`).
129        store: bool,
130        /// The access was made with **64-bit addressing** enabled for its mode
131        /// (`Status.KX`/`SX`/`UX`), which selects the **XTLB** refill vector at
132        /// offset `0x080` instead of the 32-bit one at `0x000`.
133        ///
134        /// Carried on the exception rather than re-derived at dispatch because
135        /// by then `EXL` is set, which forces `Pipeline::access_mode` to
136        /// report Kernel and would answer for the handler's mode instead of the
137        /// faulting access's.
138        wide: bool,
139    },
140    /// A TLB entry matched but was invalid. Takes the **general** vector, with
141    /// the same `ExcCode` as a refill — the vector is the only difference, which
142    /// is why they are separate variants rather than one with a flag nobody
143    /// reads.
144    TlbInvalid {
145        /// The faulting access was a store.
146        store: bool,
147    },
148    /// A store to a valid but non-writable page.
149    TlbModified,
150    /// A floating-point operation raised a condition whose `FCSR.Enable` bit is
151    /// set.
152    ///
153    /// Carries nothing: which condition fired is reported in `FCSR.Cause`, not
154    /// in COP0 `Cause`, and the handler reads it from there. Adding a field
155    /// here would duplicate — and could contradict — the architectural record.
156    FloatingPoint,
157}
158
159/// What a COP1 operation writes when it does not trap.
160///
161/// Named rather than a `(u64, bool)` pair because the destinations are of
162/// genuinely different kinds — two FPR widths and a single `FCSR` bit — and a
163/// flag pair makes "write the condition to `fd`" representable.
164#[derive(Clone, Copy, Debug, Eq, PartialEq)]
165enum FpCommit {
166    /// A 32-bit result into `fd`'s low half: `.S` values and `.W` integers.
167    Single(u32),
168    /// A 64-bit result into `fd` through the `FR` view: `.D` values and `.L`
169    /// integers.
170    Double(u64),
171    /// `FCSR.C`. Only `C.cond.fmt` produces this, and it writes no FPR at all.
172    Condition(bool),
173}
174
175/// The documented interlocks (UM Table 4-3).
176///
177/// Held as a named enum rather than a bare cycle count so a stall is always
178/// attributable — "why did this stall" is answerable from a trace.
179#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
180pub enum Interlock {
181    /// Load interlock — 1 cycle (UM §4.6.5).
182    ///
183    /// Deliberately **imprecise**, matching hardware: it fires when the next
184    /// instruction's `rs` *or* `rt` field equals the load's `rt`, whether or not
185    /// that field is actually used as a source. See [`load_interlocks`].
186    Ldi,
187    /// Data cache busy — a cached store keeps the cache busy for its `DC` *and*
188    /// `WB` stages, so a following cache access stalls 1 cycle (UM §4.6.7).
189    Dcb,
190    /// Data cache miss — the fill cost is `8..=9 + M` `PCycle`s (UM Table 11-1).
191    Dcm,
192    /// Instruction cache busy (UM §4.6.3).
193    Icb,
194    /// Instruction micro-TLB miss — 3 `PCycle`s (UM §4.6.2).
195    Itm,
196    /// Multi-cycle interlock: `MULT`/`DIV`/FPU stall the whole pipeline for the
197    /// documented count (UM Tables 3-12, 7-14).
198    Mci,
199    /// Cache operation (UM Table 4-3).
200    Cop,
201    /// CP0 bypass interlock — **1 `PCycle`** (UM §4.6.9, p. 113).
202    ///
203    /// **Named but never raised.** On hardware it fires when an instruction that
204    /// caused an exception reaches `WB` while the next instruction in `DC` reads
205    /// any CP0 register. Here, taking an exception already flushes the pipeline
206    /// and stalls for the 2-`PCycle` epilogue, so the two triggers overlap and
207    /// no case distinguishes them — charging this on top would invent a cycle
208    /// the hardware may not spend. Separating them needs the timing set
209    /// n64-systemtest ships default-off, the same instrument ledger C-1 waits on.
210    ///
211    /// The variant stays because [`Interlock`] enumerates the documented
212    /// taxonomy, and a gap in a taxonomy is worth more than a silently missing
213    /// row. This doc previously said it *fires*, in the present tense, which is
214    /// the comment-is-not-an-implementation hazard `docs/engineering-lessons.md`
215    /// §3.3c describes -- the cost was itself recorded as undocumented in three
216    /// files while sitting in the very paragraph they cited (§3.3b).
217    Cp0i,
218    /// Taking an exception — **2 `PCycle`s** (UM §4.7, p. 114).
219    ///
220    /// Not strictly one of Table 4-3's eight interlocks: the pipeline stalls
221    /// while the epilogue runs and the aborted instructions drain. Named here so
222    /// a trace can attribute the cycles rather than showing an unexplained gap.
223    Exception,
224}
225
226/// A stall request: how long, and what caused it.
227///
228/// ADR 0007 describes an interlock as `(cycles, resume_stage)`. The `resume`
229/// half is **deliberately absent** until it can be load-bearing. Today
230/// [`Pipeline::advance`] always runs the full cascade when not stalled, so a
231/// stored `resume` would be read by nothing — and a field that looks like it
232/// carries information while carrying none is the exact hazard
233/// `docs/engineering-lessons.md` §3.2 is about. `Bus::poll_irq_at_phase` was
234/// removed for the same reason rather than left in place looking wired.
235///
236/// It lands with T-11-002, when stages can stall independently and a partial
237/// resume becomes meaningful.
238#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
239pub struct Stall {
240    /// `PCycle`s remaining.
241    pub cycles: u32,
242    /// Which documented interlock caused it.
243    pub cause: Interlock,
244}
245
246/// State carried across one inter-stage boundary — what travels *with* an
247/// instruction as it advances.
248#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
249pub struct Latch {
250    /// Is an instruction present at this boundary? A bubble is `false`.
251    pub occupied: bool,
252    /// The PC of the instruction in flight.
253    pub pc: u64,
254    /// The raw instruction word (decode is T-11-002).
255    pub word: u32,
256    /// Is this instruction in a branch delay slot?
257    ///
258    /// Travels with the instruction, never as global CPU state — that is the
259    /// whole point (see the module docs).
260    pub in_delay_slot: bool,
261    /// An aborting condition stamped into this latch and every latch upstream.
262    pub abort: Option<Exception>,
263    /// The decoded instruction, filled at `IC`.
264    pub decoded: Decoded,
265    /// `rs` value, read at `RF`.
266    pub rs_val: u64,
267    /// `rt` value, read at `RF`.
268    pub rt_val: u64,
269    /// What `EX` computed, committed at `WB`.
270    pub write_back: WriteBack,
271    /// A memory access `EX` resolved for `DC` to perform.
272    pub mem: Option<MemOp>,
273    /// A COP0 access `EX` resolved: read performed in `DC`, write in `WB`.
274    ///
275    /// Split across those two stages because UM §4.6.9 defines the CP0 bypass
276    /// interlock in terms of a write reaching `WB` while the next instruction
277    /// reads in `DC` — a rule that cannot be expressed if both happen in `EX`.
278    pub cop0: Option<Cop0Access>,
279}
280
281// The inter-stage latch is **copied four times per emulated CPU cycle**, so its size is
282// a performance fact and not merely a layout detail: at ~1.56 M steps a frame those
283// copies are ~15% of the frame (`docs/performance.md` §"The latch copies, anatomized").
284//
285// This pins it. `repr(Rust)` layout is not stable across compiler versions, and the
286// measured breakdown in that document — 120 bytes, with the six scalars occupying
287// exactly their naive sum, so no padding is wasted — would otherwise decay silently
288// into a claim about a toolchain nobody is using any more.
289//
290// **If this fires, do not just change the number.** Either a field was added, in which
291// case re-measure and update the breakdown, or the layout algorithm moved, in which
292// case the "no padding is wasted" conclusion needs re-deriving before it is re-quoted.
293//
294// **120 is not a universal fact**, and the distinction was learned in review. Every
295// field is fixed-width — no `usize`, no references, no pointers — but that does *not*
296// make the layout width-independent, because it is `u64` **alignment** that varies, not
297// pointer size:
298//
299// | target | `size_of::<Latch>()` |
300// | --- | --- |
301// | `x86_64`, `thumbv7em-none-eabihf`, `wasm32-unknown-unknown`, `armv7-*` | 120 |
302// | `i686-*` (32-bit x86, `u64` aligns to 4) | 108 |
303//
304// `armv7` is in that first row by measurement, not by assumption: it is 32-bit with an
305// 8-byte-aligned `u64`, which is the combination the implication below is keyed on, so
306// it is the case that would fail first if the keying were wrong.
307//
308// So there are two assertions rather than one, and neither breaks a cross-compile:
309//
310// 1. **No padding**, universally. This is the property the breakdown actually rests on
311//    — that no field ordering would make the struct smaller — and it holds on every
312//    target above, `i686` included.
313// 2. **120 where a `u64` aligns to 8**, written as an implication rather than a
314//    `#[cfg]`. It pins the documented figure on every ABI the figure describes, and
315//    makes no claim on the ones it does not.
316//
317// `#[repr(C)]` is not the alternative either, and this was measured rather than
318// reasoned: `repr(C)` lays fields out in declaration order, which costs **128 bytes**
319// instead of 120 because the two `bool`s can no longer sit in alignment gaps. On a
320// struct copied four times per emulated cycle that adds ~1.2 ms a frame, to exactly
321// the copies `docs/performance.md` is trying to shrink.
322const _: () = {
323    // The scalars: `occupied` + `pc` + `word` + `in_delay_slot` + `rs_val` + `rt_val`.
324    // Written as `size_of` of each field's type rather than as literals, so a field
325    // whose type changes updates this term instead of silently desynchronizing it —
326    // which would mask exactly the padding this is meant to detect.
327    let scalars = core::mem::size_of::<bool>()
328        + core::mem::size_of::<u64>()
329        + core::mem::size_of::<u32>()
330        + core::mem::size_of::<bool>()
331        + core::mem::size_of::<u64>()
332        + core::mem::size_of::<u64>();
333    let parts = core::mem::size_of::<Decoded>()
334        + core::mem::size_of::<Option<Exception>>()
335        + core::mem::size_of::<WriteBack>()
336        + core::mem::size_of::<Option<MemOp>>()
337        + core::mem::size_of::<Option<Cop0Access>>()
338        + scalars;
339    assert!(
340        core::mem::size_of::<Latch>() == parts,
341        "Latch has acquired padding; docs/performance.md's copy-cost breakdown assumes none"
342    );
343    // And the documented figure itself, conditioned on the property that actually
344    // determines it. Writing it as an implication rather than a `#[cfg]` keeps it
345    // compilable everywhere while still pinning 120 on every ABI the number describes:
346    // on 32-bit x86 a `u64` aligns to 4 and `Latch` is 108, which is not a defect and
347    // not something the document claims.
348    assert!(
349        core::mem::align_of::<u64>() != 8 || core::mem::size_of::<Latch>() == 120,
350        "Latch is no longer 120 bytes where a u64 aligns to 8; re-measure docs/performance.md"
351    );
352};
353
354/// Does a load into `load_rt` interlock with the following instruction?
355///
356/// `rs` / `rt` are the *raw encoded fields* of the next instruction,
357/// deliberately named for the encoding rather than for operands — the hardware
358/// checks the fields whether or not they are used as sources, and naming them
359/// `next_rs`/`next_operand` would imply a semantics the check does not have.
360///
361/// Reproduces the hardware's **imprecision**, which is the specification here —
362/// emulating precise behavior is the bug. From
363/// `n64brew_wiki/markdown/VR4300.md` § Microarchitecture → Load Delay Interlock:
364///
365/// - Matches the load's `rt` against the next instruction's `rs` **or** `rt`
366///   field, "whether or not they are actually used as a source". So a load
367///   followed by `LUI` into the same register stalls, and two consecutive loads
368///   into the same register stall.
369/// - A load into `$zero` never interlocks.
370/// - GPR loads interlock only with non-float instructions, and FPR loads only
371///   with float instructions.
372#[must_use]
373pub const fn load_interlocks(load_rt: u8, rs: u8, rt: u8, same_reg_file: bool) -> bool {
374    // The zero register is exempt: a load into $zero is discarded, so nothing
375    // downstream can depend on it.
376    if load_rt == 0 {
377        return false;
378    }
379    if !same_reg_file {
380        return false;
381    }
382    load_rt == rs || load_rt == rt
383}
384
385/// `Status.FR` — whether the FP register file presents 32 independent 64-bit
386/// registers (set) or 16 built from FGR pairs (clear).
387fn fr_of(cop0: &Cop0) -> bool {
388    cop0.read(crate::cop0::reg::STATUS) & (1 << 26) != 0
389}
390
391/// An exception captured at its raising site, with the context the epilogue
392/// needs.
393#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
394struct Pending {
395    /// What happened.
396    exc: Exception,
397    /// The faulting instruction's address.
398    pc: u64,
399    /// Was the faulting instruction in a branch delay slot?
400    in_delay_slot: bool,
401    /// The offending address, for the exceptions that write `BadVAddr`.
402    bad_vaddr: u64,
403}
404
405/// The instruction-granular execution path (ADR 0013), behind `fast-exec`.
406///
407/// A child module so it can reach `Pipeline`'s private fields — privacy extends to
408/// descendants — without widening the surface for anyone else. Absent entirely
409/// with the feature off, so the default build gains nothing (ADR 0011 §1).
410#[cfg(feature = "fast-exec")]
411mod fastexec;
412
413/// How a retired instruction's commit was classified, for
414/// [`Pipeline::commit_census`]. The order is the reporting order.
415#[cfg(feature = "work-counters")]
416pub mod commit_class {
417    /// Commit is a GPR write or nothing: no COP0/COP1 access, no memory op, no
418    /// abort. **This is the set a direct commit path could serve** without a
419    /// second implementation of the subtle cases.
420    pub const SIMPLE: usize = 0;
421    /// Has a memory operation (`DC` work).
422    pub const MEM: usize = 1;
423    /// Touches COP0 or COP1 — the cases whose semantics live in `wb_stage` and
424    /// must not be duplicated.
425    pub const COP: usize = 2;
426    /// Writes `HI`/`LO` (every multiply and divide).
427    pub const HILO: usize = 3;
428    /// Reserved for future instruction classes (aborted instructions never reach retirement).
429    pub const OTHER: usize = 4;
430    /// Number of classes.
431    pub const COUNT: usize = 5;
432}
433
434/// The four inter-stage latches plus the pipeline control state.
435#[derive(Clone, Debug, Default, Serialize, Deserialize)]
436// The bools are independent hardware lines and latches -- `prev_was_run`,
437// `flush_pending`, `nmi_pending`, `ll_bit`. Clippy suggests folding them into a
438// state machine; they do not form one, because any combination is reachable and
439// each is set and cleared by a different part of the machine. Naming them is
440// what makes the exception and interrupt rules readable.
441#[allow(clippy::struct_excessive_bools)]
442pub struct Pipeline {
443    /// Retired-instruction census by commit class, indexed by [`commit_class`].
444    ///
445    /// A **counter**, not a schedule input (ADR 0006). It exists to answer one
446    /// question before any code is written against it: what fraction of executed
447    /// instructions could a direct commit path actually serve? Sizing a change on
448    /// an assumed fraction is how the last program produced four reverts.
449    ///
450    /// `#[serde(skip)]` because it is diagnostic and must not enter the
451    /// save-state layout (ADR 0011 §4).
452    #[cfg(feature = "work-counters")]
453    #[serde(skip)]
454    commit_census: [u64; commit_class::COUNT],
455    /// The **whole** of COP2: one 64-bit latch.
456    ///
457    /// COP2 is not populated on the VR4300, and what remains is a single
458    /// value that every `MTC2`/`DMTC2` writes and every `MFC2`/`DMFC2` reads,
459    /// with the register index ignored. See the `EX` stage and ledger C-20.
460    pub cop2_latch: u64,
461    /// `IC` → `RF`.
462    pub ic_rf: Latch,
463    /// `RF` → `EX`.
464    pub rf_ex: Latch,
465    /// `EX` → `DC`.
466    pub ex_dc: Latch,
467    /// `DC` → `WB`.
468    pub dc_wb: Latch,
469    /// Remaining stall cycles; while non-zero the pipeline does not advance.
470    stall: Option<Stall>,
471    /// Was the *previous* `PCycle` a run cycle (not a stall)?
472    ///
473    /// UM §4.7.1: *"NMI and interrupt exception requests are accepted only if the
474    /// previous `PCycle` was a run cycle."* This is the gate, and it is the
475    /// reason the flag exists at all.
476    prev_was_run: bool,
477    /// The NMI line, latched until an instruction boundary lets it be taken.
478    ///
479    /// A level rather than an edge here: the hardware signal is the falling edge
480    /// of the NMI pin (or bit 6 of the internal interrupt register written over
481    /// `SysAD`), and the edge is what `signal_nmi` represents. Holding it means
482    /// an NMI raised during a stall is taken when the stall drains rather than
483    /// being dropped -- the same reason the timer latches `IP7`.
484    nmi_pending: bool,
485    /// An abort was raised this cycle, so `IC` must fetch a bubble rather than a
486    /// live instruction — the wrong-path fetch would otherwise escape the flush.
487    ///
488    /// Cleared at the end of each [`Pipeline::advance`].
489    flush_pending: bool,
490    /// Instructions retired at `WB` — a work tally, not a time position.
491    pub retired: u64,
492    /// An exception raised this cycle, awaiting dispatch at the end of it.
493    ///
494    /// Captured where it is *raised* rather than reconstructed afterwards,
495    /// because the epilogue needs the faulting instruction's PC and delay-slot
496    /// flag — and by the end of the cycle the reverse cascade has moved every
497    /// latch, so the faulting instruction is no longer where it was.
498    pending: Option<Pending>,
499    /// The virtual address of the access that faulted this cycle.
500    ///
501    /// Recorded where the fault is *detected*, because `BadVAddr` needs the
502    /// address and `Self::access` reports only which exception. Reconstructing
503    /// it at dispatch time is impossible — the `MemOp` has been consumed.
504    fault_vaddr: u64,
505    /// COP1 **control** registers (T-12-006).
506    pub cop1: Cop1Control,
507    /// The floating-point register file (T-13-001).
508    pub fpr: Fpr,
509    /// The joint TLB and its instruction micro-TLB (T-12-004).
510    pub tlb: Tlb,
511    /// The 16 KiB primary instruction cache (T-11-003).
512    pub icache: crate::cache::Icache,
513    /// The PC of a recognized self-branch idle loop, or `None` (ADR 0013 mode only).
514    ///
515    /// `beq $0, $0, -1` followed by a `nop` is a branch to itself: after both
516    /// instructions the PC is back where it started and no register, no memory
517    /// word and no COP0 field other than `Count` and `Random` has changed. Super
518    /// Mario 64 and Mario Kart 64 spend roughly **90% of every instruction they
519    /// retire** in exactly that loop — it is the N64 idle thread — so recognizing
520    /// it and charging its cost without fetching, decoding or executing it is
521    /// worth 1.65x and 1.93x respectively (`docs/performance.md`).
522    ///
523    /// **Why caching this is as correct as the hardware.** The N64 does not snoop
524    /// DMA against the I-cache: software that overwrites code must issue a `CACHE`
525    /// instruction itself, and until it does the CPU keeps executing the stale
526    /// cached line. So a recognition keyed to the I-cache's own lifetime cannot
527    /// diverge from the machine — [`Pipeline::cache_op`] clears this, which covers
528    /// every invalidate, fill and tag write, and so does taking any exception.
529    ///
530    /// Kept on `Pipeline` rather than in `fastexec` so it lives with the `icache`
531    /// whose validity it borrows; it is simply never set on the accurate path.
532    pub(crate) idle_pc: Option<u64>,
533    /// The 8 KiB primary write-back data cache (T-11-003).
534    pub dcache: crate::cache::Dcache,
535    /// The COP0 register file (T-12-001).
536    ///
537    /// Public because exception dispatch, the TLB and the interrupt path all
538    /// read it, and they land in separate tickets.
539    pub cop0: Cop0,
540    /// The link bit, `LLbit`.
541    ///
542    /// *"set by the LL instruction, cleared by an ERET, and tested by the SC
543    /// instruction"* (UM §3.1). Note what is **absent** from that list: `SC`
544    /// itself does not clear it, and neither does an intervening load or store.
545    /// Clearing it in `SC` is the natural-looking mistake, and it makes a
546    /// retried `LL`/`SC` loop fail forever on the second iteration.
547    ///
548    /// `ERET` is the other half and lands with the exception model (Sprint 2);
549    /// until then nothing clears this, which is correct-so-far rather than
550    /// finished — recorded as such in `docs/cpu.md`.
551    ll_bit: bool,
552}
553
554/// `FCSR.C`, the compare condition — bit 23, above the `Cause` field.
555///
556/// Written only by `C.cond.fmt` and read only by the `BC1` family, so it is
557/// deliberately outside the `Cause`/`Flags` bookkeeping that every other FP
558/// operation touches.
559const FCSR_C: u32 = 1 << 23;
560
561impl Pipeline {
562    /// A fresh, empty pipeline.
563    #[must_use]
564    pub const fn new() -> Self {
565        const EMPTY: Latch = Latch {
566            occupied: false,
567            pc: 0,
568            word: 0,
569            in_delay_slot: false,
570            abort: None,
571            cop0: None,
572            decoded: Decoded {
573                op: crate::decode::Op::Reserved,
574                rs: 0,
575                rt: 0,
576                rd: 0,
577                sa: 0,
578                imm: 0,
579                dest: 0,
580                target: 0,
581            },
582            rs_val: 0,
583            rt_val: 0,
584            write_back: WriteBack::None,
585            mem: None,
586        };
587        Self {
588            #[cfg(feature = "work-counters")]
589            commit_census: [0; commit_class::COUNT],
590            // Power-on value: a documented zero (ADR 0004).
591            cop2_latch: 0,
592            ic_rf: EMPTY,
593            rf_ex: EMPTY,
594            ex_dc: EMPTY,
595            dc_wb: EMPTY,
596            stall: None,
597            prev_was_run: false,
598            nmi_pending: false,
599            flush_pending: false,
600            retired: 0,
601            pending: None,
602            fault_vaddr: 0,
603            cop1: Cop1Control::new(),
604            fpr: Fpr::new(),
605            tlb: Tlb::new(),
606            icache: crate::cache::Icache::new(),
607            idle_pc: None,
608            dcache: crate::cache::Dcache::new(),
609            cop0: Cop0::new(),
610            ll_bit: false,
611        }
612    }
613
614    /// The interlock currently stalling the pipeline, if any.
615    #[must_use]
616    pub const fn stalled_by(&self) -> Option<Interlock> {
617        match self.stall {
618            Some(s) => Some(s.cause),
619            None => None,
620        }
621    }
622
623    /// Was the previous `PCycle` a run cycle? Gates interrupt acceptance.
624    #[must_use]
625    pub const fn prev_cycle_was_run(&self) -> bool {
626        self.prev_was_run
627    }
628
629    /// The link bit, as `SC` would test it.
630    ///
631    /// Exposed for the COP0 / `ERET` work in Sprint 2, which must clear it.
632    #[must_use]
633    pub const fn ll_bit(&self) -> bool {
634        self.ll_bit
635    }
636
637    /// `LLAddr` (COP0 register 17): `PA(31:4)` of the most recent `LL`.
638    ///
639    /// Reads straight out of the COP0 file. `LL` writes it there and nowhere
640    /// else — there is deliberately no second copy, because two stores of one
641    /// architectural value drift, and `MFC0 $rt, $17` would then disagree with
642    /// the CPU's own idea of the link address.
643    #[must_use]
644    pub const fn ll_addr(&self) -> u64 {
645        self.cop0.read(crate::cop0::reg::LL_ADDR)
646    }
647
648    /// Request a stall of `cycles` `PCycle`s.
649    ///
650    /// A zero-cycle request is **not** a stall and is ignored. Recording it would
651    /// still consume a cycle in [`Pipeline::advance`] and mark it as not-a-run
652    /// cycle, which silently inserts a bubble *and* suppresses interrupt
653    /// acceptance on the following cycle (UM §4.7.1) — a one-cycle timing error
654    /// with no visible cause.
655    pub const fn stall_for(&mut self, cycles: u32, cause: Interlock) {
656        if cycles == 0 {
657            return;
658        }
659        self.stall = Some(Stall { cycles, cause });
660    }
661
662    /// Assert the **NMI** line.
663    ///
664    /// Latched, then taken at the next instruction boundary. Nothing in the core
665    /// calls this yet: on hardware the source is the console's reset button,
666    /// which reaches the CPU as PRENMI followed by NMI, and the frontend owns
667    /// that button (Phase 6). It exists now because the exception itself is
668    /// Phase 1 work and belongs with the rest of the exception model — the wire
669    /// from the button is a separate, later concern.
670    pub const fn signal_nmi(&mut self) {
671        self.nmi_pending = true;
672    }
673
674    /// Stamp an abort into `at` **and every latch upstream of it** — the
675    /// kill-younger-instructions step. Instructions older than `at` have already
676    /// passed and are unaffected.
677    ///
678    /// # Ordering contract
679    ///
680    /// **A stage must call this BEFORE it moves its latch.** The instruction
681    /// executing in stage S this cycle sits in S's *input* latch until the move,
682    /// so stamping first is what makes the abort travel with the instruction that
683    /// caused it. Calling it after the move stamps the abort onto the *younger*
684    /// instruction instead, and the causing one escapes — a misalignment that no
685    /// single-cycle assertion catches. `an_abort_survives_the_cascade` advances
686    /// the pipeline to verify it, rather than checking latch state in place.
687    ///
688    /// The abort also raises an internal pending-flush flag, so the instruction
689    /// fetched later in the same cycle is a bubble rather than a live
690    /// wrong-path fetch.
691    pub const fn abort_from(&mut self, at: Stage, exc: Exception) {
692        self.abort_with(at, exc, 0);
693    }
694
695    /// [`Pipeline::abort_from`], additionally recording the offending address
696    /// for the exceptions that write `BadVAddr`.
697    pub const fn abort_with(&mut self, at: Stage, exc: Exception, bad_vaddr: u64) {
698        // Capture the faulting instruction's context NOW. The latch holding it
699        // is the one this stage is reading -- `ex_dc` for DC, `rf_ex` for EX --
700        // and by the end of the cycle the cascade will have moved it on.
701        //
702        // Priority: an exception already pending this cycle came from a LATER
703        // stage (the cascade runs WB first), and UM §4.7.2 gives a later stage
704        // precedence over an earlier one. So the first capture wins.
705        if self.pending.is_none() {
706            let src = match at {
707                Stage::Wb => &self.dc_wb,
708                Stage::Dc => &self.ex_dc,
709                Stage::Ex => &self.rf_ex,
710                Stage::Rf | Stage::Ic => &self.ic_rf,
711            };
712            self.pending = Some(Pending {
713                exc,
714                pc: src.pc,
715                in_delay_slot: src.in_delay_slot,
716                bad_vaddr,
717            });
718        }
719        self.flush_pending = true;
720        match at {
721            Stage::Wb => {
722                self.dc_wb.abort = Some(exc);
723                self.ex_dc.abort = Some(exc);
724                self.rf_ex.abort = Some(exc);
725                self.ic_rf.abort = Some(exc);
726            }
727            Stage::Dc => {
728                self.ex_dc.abort = Some(exc);
729                self.rf_ex.abort = Some(exc);
730                self.ic_rf.abort = Some(exc);
731            }
732            Stage::Ex => {
733                self.rf_ex.abort = Some(exc);
734                self.ic_rf.abort = Some(exc);
735            }
736            Stage::Rf | Stage::Ic => self.ic_rf.abort = Some(exc),
737        }
738    }
739
740    /// Advance the pipeline by exactly one `PCycle`.
741    ///
742    /// Stages run **WB → DC → EX → RF → IC**. Because each stage reads its input
743    /// latch before any upstream stage writes it, no value moves two stages in
744    /// one cycle and no double buffering is required. Do not reorder this.
745    ///
746    /// Hot path: allocation-free.
747    pub fn advance<B: Bus>(&mut self, bus: &mut B, regs: &mut Regs, next_pc: &mut u64) {
748        // The timeline is HELD, not advanced. `Count` runs at half PClock, so
749        // bumping it once per PClock here would run the timer at double rate --
750        // and inventing a parity bit to halve it would be a second incremented
751        // counter, which is what ADR 0006 exists to forbid.
752        //
753        // Holding is the honest option: with no scheduler attached there is no
754        // timeline, so `Count` does not move. Anything exercising `Count` or
755        // `Compare` must call `advance_at` and supply the position.
756        self.advance_at(bus, regs, next_pc, self.cop0.count_now());
757    }
758
759    /// [`Pipeline::advance`], with the scheduler's `Count` timeline supplied.
760    ///
761    /// `Count` is **derived** from the master clock (ADR 0006), so the position
762    /// is passed in rather than incremented here. This is the path the scheduler
763    /// uses ([`crate::Cpu::tick_at`]); [`Pipeline::advance`] is a convenience
764    /// for callers with no scheduler, and **holds** the timeline rather than
765    /// guessing at it.
766    pub fn advance_at<B: Bus>(
767        &mut self,
768        bus: &mut B,
769        regs: &mut Regs,
770        next_pc: &mut u64,
771        count_now: u64,
772    ) {
773        self.cop0.set_now(count_now);
774        // A stall consumes the cycle. The pipeline holds its state, and the cycle
775        // is recorded as NOT a run cycle so an interrupt cannot be accepted on the
776        // cycle following it (UM §4.7.1).
777        if let Some(mut s) = self.stall {
778            s.cycles = s.cycles.saturating_sub(1);
779            self.stall = if s.cycles == 0 { None } else { Some(s) };
780            self.prev_was_run = false;
781            // The timer belongs to the clock, not to the pipeline. `Count` keeps
782            // advancing through the stall, so `Compare` can be reached -- and
783            // passed -- entirely inside one. Latching `IP7` here does not accept
784            // the interrupt (that still needs a run cycle, guarded by
785            // `prev_was_run` below); it only records in `Cause` what the hardware
786            // records, so the handler can run once the pipeline resumes.
787            return;
788        }
789
790        self.wb_stage(regs);
791        self.dc_stage(bus);
792        self.ex_stage(regs, next_pc);
793        self.rf_stage(regs);
794        self.ic_stage(bus, next_pc);
795
796        self.prev_was_run = true;
797        self.flush_pending = false;
798
799        // Dispatch AFTER the cascade, so every stage has seen the abort and the
800        // pipeline is drained, and exactly once per cycle regardless of how many
801        // stages raised.
802        if let Some(p) = self.pending.take() {
803            let d = exception::dispatch(&mut self.cop0, p.exc, p.pc, p.in_delay_slot, p.bad_vaddr);
804            *next_pc = d.vector;
805            self.stall_for(d.stall_cycles, Interlock::Exception);
806        }
807    }
808
809    /// The retired-instruction census by commit class ([`commit_class`]).
810    #[cfg(feature = "work-counters")]
811    #[must_use]
812    pub const fn commit_census(&self) -> &[u64; commit_class::COUNT] {
813        &self.commit_census
814    }
815
816    /// Classify one retired instruction for the census.
817    ///
818    /// Takes the pieces rather than the assembled `Latch`, so it can be called
819    /// from the fast path *before* the latch is built — the whole point being to
820    /// find out how often building it is avoidable.
821    #[cfg(feature = "work-counters")]
822    pub(crate) fn count_commit(
823        &mut self,
824        cop0: Option<crate::exec::Cop0Access>,
825        mem: Option<crate::exec::MemOp>,
826        write_back: crate::exec::WriteBack,
827    ) {
828        use crate::exec::WriteBack as W;
829        // Order matters: an instruction with BOTH a memory op and a COP0 access
830        // is not simple, and must not be counted as if the cheaper class applied.
831        let class = if cop0.is_some() {
832            commit_class::COP
833        } else if mem.is_some() {
834            commit_class::MEM
835        } else {
836            match write_back {
837                W::None | W::Gpr { .. } => commit_class::SIMPLE,
838                W::HiLo(_) | W::Hi(_) | W::Lo(_) => commit_class::HILO,
839                #[allow(unreachable_patterns, reason = "future WriteBack variants land here")]
840                _ => commit_class::OTHER,
841            }
842        };
843        self.commit_census[class] = self.commit_census[class].saturating_add(1);
844    }
845
846    /// `WB` — commit the result and retire the instruction.
847    fn wb_stage(&mut self, regs: &mut Regs) {
848        if self.dc_wb.occupied && self.dc_wb.abort.is_none() {
849            // The COP0 WRITE lands here (UM §4.6.9). A `Read` in this latch was
850            // already performed in DC and left its value in `write_back`.
851            // The TLB instructions land in WB, with the COP0 write: they are
852            // CP0 operations and `TLBR`/`TLBP` write COP0 registers, so doing
853            // them earlier would let a following `MFC0` read the result a cycle
854            // before hardware produces it.
855            if let Some(Cop0Access::Cop1(Cop1Access::WriteFpr { dest, value, wide })) =
856                self.dc_wb.cop0
857            {
858                // DMTC1 mirrors DMFC1: the FR view, not the physical register.
859                if wide {
860                    let fr = fr_of(&self.cop0);
861                    self.fpr.write_d(dest, fr, value);
862                } else {
863                    self.fpr.write_s(dest, fr_of(&self.cop0), value as u32);
864                }
865            }
866            if let Some(Cop0Access::Cop1(Cop1Access::WriteControl { dest, value })) =
867                self.dc_wb.cop0
868            {
869                self.cop1.ctc1(dest, value);
870                // **`CTC1` can raise on its own.** Writing `FCSR` with a Cause
871                // bit whose Enable is also set meets the trap condition
872                // immediately -- no arithmetic required. n64-systemtest writes
873                // `enable_overflow | cause_overflow` in one go and expects the
874                // FP exception to report the `CTC1` itself as `ExceptPC`.
875                //
876                // Bit 17 (Unimplemented) is unmaskable and traps regardless,
877                // which is why it is tested outside the enable comparison.
878                if self.fcsr_traps_now() {
879                    self.abort_from(Stage::Wb, Exception::FloatingPoint);
880                    self.dc_wb.occupied = false;
881                    return;
882                }
883            }
884            if let Some(Cop0Access::Cop1(Cop1Access::Arith {
885                fmt,
886                funct,
887                ft,
888                fs,
889                fd,
890            })) = self.dc_wb.cop0
891            {
892                if self.fp_arith(fmt, funct, ft, fs, fd) {
893                    // Trapped. The instruction does **not** complete, so it must
894                    // not reach the retirement tail below: `Random` "decrements
895                    // as each instruction executes" (UM §5.4.2), and one that
896                    // took an exception did not execute.
897                    //
898                    // No delay is charged either. A trapped operation abandons
899                    // its result, and the exception's own 2-PCycle epilogue is
900                    // the cost that applies.
901                    self.dc_wb.occupied = false;
902                    return;
903                }
904                // The multi-cycle FPU operations stall the pipeline for their
905                // documented rate (UM Table 7-14). The manual's "latency = rate
906                // + 1 for a dependent consumer" falls out of this rather than
907                // being added: the stall holds every stage, so a consumer spends
908                // its own cycle once the stall drains.
909                self.stall_for(crate::fpu::stall_cycles(funct, fmt), Interlock::Mci);
910            }
911            if let Some(Cop0Access::Tlb(op)) = self.dc_wb.cop0 {
912                match op {
913                    TlbOp::Read => {
914                        let i = (self.cop0.read(crate::cop0::reg::INDEX) & 0x3F) as usize;
915                        self.tlb.read_entry(i, &mut self.cop0);
916                    }
917                    TlbOp::WriteIndexed => {
918                        let i = (self.cop0.read(crate::cop0::reg::INDEX) & 0x3F) as usize;
919                        // TLBWI CAN overwrite a wired entry; only TLBWR cannot
920                        // (UM §5.4.4, p. 150). Guarding both is a natural-looking
921                        // mistake that makes wired entries unwritable at all.
922                        self.tlb.write_entry(i, &self.cop0);
923                    }
924                    TlbOp::WriteRandom => {
925                        // `Random` never goes below `Wired`, so the wired entries
926                        // are protected by the counter's range rather than by a
927                        // check here -- which is how hardware does it.
928                        let i = (self.cop0.read(crate::cop0::reg::RANDOM) & 0x3F) as usize;
929                        self.tlb.write_entry(i, &self.cop0);
930                    }
931                    TlbOp::Probe => self.tlb.probe(&mut self.cop0),
932                }
933            }
934            if let Some(Cop0Access::Write { dest, value, wide }) = self.dc_wb.cop0 {
935                if wide {
936                    self.cop0.dmtc0(dest, value);
937                } else {
938                    self.cop0.mtc0(dest, value);
939                }
940            }
941            match self.dc_wb.write_back {
942                WriteBack::None => {}
943                // `Regs::write` discards `$zero`, so no guard is needed here --
944                // and must not be added, or the rule lives in two places.
945                WriteBack::Gpr { dest, value } => regs.write(dest, value),
946                WriteBack::HiLo(hl) => {
947                    regs.hi = hl.hi;
948                    regs.lo = hl.lo;
949                }
950                WriteBack::Hi(v) => regs.hi = v,
951                WriteBack::Lo(v) => regs.lo = v,
952            }
953            self.retired = self.retired.wrapping_add(1);
954            #[cfg(feature = "work-counters")]
955            self.count_commit(self.dc_wb.cop0, self.dc_wb.mem, self.dc_wb.write_back);
956            // "Random decrements as each instruction executes" (UM §5.4.2,
957            // p. 147) -- advanced HERE, at retirement, so it counts executed
958            // instructions rather than cycles.
959            //
960            // This was implemented and then never called from the pipeline, so
961            // `Random` sat at 31 forever and **every `TLBWR` overwrote the same
962            // entry**. A refill handler that needs more than one mapping live at
963            // once therefore destroys its previous entry on each miss and faults
964            // again immediately -- an infinite refill loop, which is exactly what
965            // n64-systemtest hit. A stuck counter is invisible to any test that
966            // calls `tick_random` itself.
967            self.cop0.tick_random();
968        }
969        self.dc_wb.occupied = false;
970    }
971
972    /// Perform the COP0/COP1 **reads** an instruction resolved, leaving the value
973    /// in the latch's `write_back` for the commit to store.
974    ///
975    /// The fourth latch-independent primitive (`docs/cpu.md`), and the one PR #231
976    /// deliberately left alone: at that point it had a single caller, and the shape
977    /// of a seam guessed without a second caller is a guess. `fast-exec`'s
978    /// sequential path is that caller, so the extraction is now driven rather than
979    /// speculative.
980    ///
981    /// It takes the latch **by reference** rather than the fields by value because
982    /// the three reads each replace `write_back` wholesale, and threading four
983    /// values in and one out would be a worse signature than the one the accurate
984    /// path already has.
985    ///
986    /// A **read** happens here, in `DC`; a **write** happens at `WB`. They are
987    /// split across the two stages because UM §4.6.9 defines the CP0 bypass
988    /// interlock in terms of a write reaching `WB` while the next instruction reads
989    /// in `DC` — a rule that cannot be expressed if both happen in one stage.
990    fn apply_cop0_read(&self, out: &mut Latch) {
991        if out.occupied
992            && out.abort.is_none()
993            && let Some(Cop0Access::Cop1(Cop1Access::ReadFpr { src, dest, wide })) = out.cop0
994        {
995            out.write_back = WriteBack::Gpr {
996                dest,
997                // DMFC1 applies the FR view -- it does NOT move the physical
998                // register. UM Ch. 17's pseudocode is explicit:
999                //
1000                //   if FR = 1        then data <- FGR[fs]
1001                //   else if fs0 = 0  then data <- FGR[fs+1] || FGR[fs]
1002                //   else                  data <- undefined
1003                //
1004                // So with FR = 0 and an even `fs` it reads the PAIR, exactly like
1005                // LDC1. Only an odd `fs` with FR = 0 is undefined -- and it is
1006                // *undefined*, not a Reserved Instruction exception.
1007                //
1008                // MFC1 moves the low word, sign-extended, in both modes.
1009                value: if wide {
1010                    self.fpr.read_d(src, fr_of(&self.cop0))
1011                } else {
1012                    crate::alu::sext32(self.fpr.read_s(src, fr_of(&self.cop0)))
1013                },
1014            };
1015        }
1016        if out.occupied
1017            && out.abort.is_none()
1018            && let Some(Cop0Access::Cop1(Cop1Access::ReadControl { src, dest })) = out.cop0
1019        {
1020            out.write_back = WriteBack::Gpr {
1021                dest,
1022                // CFC1 is a 32-bit move, so the result is sign-extended into the
1023                // 64-bit GPR exactly as MFC0's is.
1024                value: crate::alu::sext32(self.cop1.cfc1(src)),
1025            };
1026        }
1027        if out.occupied
1028            && out.abort.is_none()
1029            && let Some(Cop0Access::Read { src, dest, wide }) = out.cop0
1030        {
1031            let value = if wide {
1032                self.cop0.dmfc0(src)
1033            } else {
1034                self.cop0.mfc0(src)
1035            };
1036            out.write_back = WriteBack::Gpr { dest, value };
1037        }
1038    }
1039
1040    /// Update the `Cause.IP` lines from the hardware: the RCP's aggregate
1041    /// interrupt and the `Count == Compare` timer edge.
1042    ///
1043    /// **Asserting a line is not recognizing an interrupt**, and separating the
1044    /// two is the point. The `IP` bits track what the hardware is *asserting*
1045    /// regardless of masks; recognition then applies `IE`/`EXL`/`ERL`/`IM` and is
1046    /// the caller's business. Folding them together would make a masked interrupt
1047    /// invisible to `MFC0 Cause`, which software polls.
1048    ///
1049    /// Latch-independent, like [`Self::fetch_word`] and [`Self::ex_gate`]: it
1050    /// stamps nothing and names no [`Stage`]. Asserting a line happens on
1051    /// hardware whether or not the CPU is in a position to take the interrupt, so
1052    /// the instruction-granular path (ADR 0013) needs exactly this and nothing
1053    /// more.
1054    ///
1055    /// `IP2` is the RCP's aggregate line from the MI (libdragon `cop0.h`:
1056    /// `C0_INTERRUPT_RCP = C0_INTERRUPT_2`; ledger U-4). `IP3` is CART, `IP4`
1057    /// PRENMI, `IP7` the timer; the rest are unused on this board.
1058    fn sample_interrupt_lines<B: Bus>(&mut self, bus: &mut B) {
1059        self.cop0.set_ip(2, bus.poll_irq());
1060        // IP7 is LATCHED on the match and stays set until `Compare` is written
1061        // (UM §6.4.18, p. 200) -- note the one-way `if`, with no `else` clearing
1062        // it. Modeling it as a level tied to `Count == Compare` looks tidier
1063        // and silently DROPS any timer interrupt that fires while `EXL` is set,
1064        // because the equality holds for one tick and the handler never sees it.
1065        //
1066        // The trigger is the rising EDGE of the match, not the standing
1067        // equality: both `Count` and `Compare` reset to zero, so an equality
1068        // test latches `IP7` before a single instruction retires. See
1069        // `Cop0::timer_edge`.
1070        if self.cop0.timer_edge() {
1071            self.cop0.set_ip(7, true);
1072        }
1073    }
1074
1075    /// `DC` — the data-cache access, and the interrupt sampling point.
1076    ///
1077    /// The stage placement is documented, not inherited from a reference
1078    /// implementation: UM Figure 4-12 puts `INTR` in the `DC` column and §4.7.6
1079    /// "DC-Stage Interlock and Exception Priorities" lists the interrupt
1080    /// exception among them.
1081    fn dc_stage<B: Bus>(&mut self, bus: &mut B) {
1082        // Sample the lines once per PCycle here. `sample_interrupt_lines` carries
1083        // why asserting and recognizing are two different steps; what follows is
1084        // the recognition half.
1085        self.sample_interrupt_lines(bus);
1086
1087        // Accepted only if the previous PCycle was a run cycle (UM §4.7.1). This
1088        // is the ONLY interrupt recognition predicate in the tree -- carrying two
1089        // subtly different ones is a known source of one-cycle discrepancies in
1090        // other emulators.
1091
1092        // NMI is checked FIRST and without consulting `interrupt_pending`.
1093        // *"Unlike all other interrupts, this interrupt is not maskable; it
1094        // occurs regardless of the settings of the EXL, ERL, and the IE bits"*
1095        // (UM SS6.4.6). Only the run-cycle gate applies, because NMI is still
1096        // *"taken only at instruction boundaries"* -- which is exactly what
1097        // `prev_was_run` expresses (UM SS4.7.1).
1098        if self.prev_was_run && self.nmi_pending {
1099            self.nmi_pending = false;
1100            self.abort_from(Stage::Dc, Exception::Nmi);
1101        } else if self.prev_was_run
1102            && self.cop0.interrupt_pending()
1103            && self.interrupt_has_an_instruction_to_charge()
1104        {
1105            self.abort_from(Stage::Dc, Exception::Interrupt);
1106        }
1107        // The memory access. This is the point the scheduler interleaves the RCP
1108        // around -- the whole reason the pipeline is modeled at all (ADR 0007).
1109        let mut out = self.ex_dc;
1110        if out.occupied
1111            && out.abort.is_none()
1112            && let Some(op) = out.mem
1113        {
1114            match self.access(bus, op) {
1115                Ok(wb) => out.write_back = wb,
1116                // Stamp before the latch move so the abort travels with the
1117                // instruction that caused it -- see `abort_from`.
1118                Err(exc) => {
1119                    self.abort_with(Stage::Dc, exc, self.fault_vaddr);
1120                    out = self.ex_dc;
1121                }
1122            }
1123        }
1124        // The COP0 READ happens here, in DC (UM §4.6.9). The write does not --
1125        // it happens in WB, and keeping them in different stages is what makes
1126        // the CP0 bypass interlock expressible at all.
1127        self.apply_cop0_read(&mut out);
1128        self.dc_wb = out;
1129        self.ex_dc.occupied = false;
1130    }
1131
1132    /// The operand bypass network (UM §4.6).
1133    ///
1134    /// *"Bypassing ... allows data and conditions produced in the `EX`, `DC` and
1135    /// `WB` stages to be made available to the `EX` stage of the next cycle."*
1136    ///
1137    /// Without this, back-to-back dependent instructions read stale registers and
1138    /// essentially every real program computes wrong values — `LUI`+`ORI`, the
1139    /// standard way to build a 32-bit constant, breaks immediately. Its absence
1140    /// was invisible to every unit test in this crate and was caught only by
1141    /// `a_program_executes_through_the_whole_pipeline`.
1142    ///
1143    /// By the time `EX` runs, the reverse cascade has already committed one
1144    /// instruction (`WB` ran first, so the register file is current) and moved the
1145    /// next into `dc_wb`. Exactly **one** producer can therefore still be
1146    /// uncommitted, and `dc_wb` is it.
1147    ///
1148    /// Loads are the case this does *not* cover — a load's value is not ready in
1149    /// time, which is precisely why the hardware has a load-delay interlock. That
1150    /// lands with T-11-003 alongside the loads themselves.
1151    fn bypass(&self, reg: u8, regs: &Regs) -> u64 {
1152        if reg != 0
1153            && self.dc_wb.occupied
1154            && self.dc_wb.abort.is_none()
1155            && let WriteBack::Gpr { dest, value } = self.dc_wb.write_back
1156            && dest == reg
1157        {
1158            return value;
1159        }
1160        regs.read(reg)
1161    }
1162
1163    /// `HI`/`LO` as `EX` should see them, bypassing an uncommitted producer.
1164    fn bypass_hi_lo(&self, regs: &Regs) -> HiLo {
1165        if self.dc_wb.occupied && self.dc_wb.abort.is_none() {
1166            match self.dc_wb.write_back {
1167                WriteBack::HiLo(hl) => return hl,
1168                WriteBack::Hi(v) => return HiLo { hi: v, lo: regs.lo },
1169                WriteBack::Lo(v) => return HiLo { hi: regs.hi, lo: v },
1170                _ => {}
1171            }
1172        }
1173        HiLo {
1174            hi: regs.hi,
1175            lo: regs.lo,
1176        }
1177    }
1178
1179    /// Map a TLB fault to the exception it raises.
1180    ///
1181    /// The `store` flag selects `TLBL` vs `TLBS`; the *variant* selects the
1182    /// vector. Both matter and they are independent. `wide` is the addressing
1183    /// width of the faulting access, which picks the XTLB refill vector over the
1184    /// 32-bit one -- it is only meaningful for a refill, since every other TLB
1185    /// fault takes the general vector regardless.
1186    const fn tlb_exception(f: crate::tlb::TlbFault, store: bool, wide: bool) -> Exception {
1187        match f {
1188            crate::tlb::TlbFault::Refill => Exception::TlbRefill { store, wide },
1189            crate::tlb::TlbFault::Invalid => Exception::TlbInvalid { store },
1190            // Modified only ever arises on a store, so it carries no flag.
1191            crate::tlb::TlbFault::Modified => Exception::TlbModified,
1192        }
1193    }
1194
1195    /// Translate a data address through the TLB.
1196    ///
1197    /// Returns the whole [`crate::addr::Physical`], **cacheability included**.
1198    /// It used to return the address alone, which is exactly the information a
1199    /// D-cache needs and cannot recover: once KSEG0 and KSEG1 have both become
1200    /// the same physical number, nothing downstream can tell them apart.
1201    fn translate_data(
1202        &mut self,
1203        vaddr: u64,
1204        store: bool,
1205    ) -> Result<crate::addr::Physical, Exception> {
1206        let asid = (self.cop0.read(crate::cop0::reg::ENTRY_HI) & 0xFF) as u8;
1207        let access = self.access_mode();
1208        match crate::addr::translate_via(&mut self.tlb, vaddr, asid, store, access) {
1209            Ok(p) => Ok(p),
1210            Err(e) => {
1211                self.fault_vaddr = vaddr;
1212                self.note_shutdown();
1213                Err(match e {
1214                    crate::addr::TranslateError::Address => Exception::AddressError { store },
1215                    crate::addr::TranslateError::Tlb(f) => {
1216                        Self::tlb_exception(f, store, access.wide)
1217                    }
1218                })
1219            }
1220        }
1221    }
1222
1223    /// The unaligned `LWL`/`LWR`/`LDL`/`LDR`/`SWL`/`SWR`/`SDL`/`SDR` family.
1224    ///
1225    /// Split out of [`Pipeline::access`] purely for size; the merging rules live
1226    /// in [`crate::mem`] and the alignment exemption is by construction — being
1227    /// usable at any byte offset is the entire reason these instructions exist.
1228    ///
1229    /// # Errors
1230    ///
1231    /// A TLB fault on the container address.
1232    fn access_unaligned<B: Bus>(
1233        &mut self,
1234        bus: &mut B,
1235        op: crate::decode::Op,
1236        addr: u64,
1237        rt: u64,
1238        dest: u8,
1239    ) -> Result<WriteBack, Exception> {
1240        use crate::decode::Op;
1241        // The unaligned family splits into loads and stores, and the
1242        // TLB check differs: a store must find the `D` bit set.
1243        let is_store = matches!(op, Op::Swl | Op::Swr | Op::Sdl | Op::Sdr);
1244        // On a fault, `BadVAddr` reports the address the INSTRUCTION named, not
1245        // the container address translated on its behalf. `translate_data`
1246        // records whatever it was handed, so the aligned-down value has to be
1247        // corrected back -- otherwise a fault on `SWL 0x12345001` reports
1248        // `0x12345000`, and n64-systemtest checks that exact case.
1249        let translate = |p: &mut Self, a: u64| match p.translate_data(a, is_store) {
1250            Ok(v) => Ok(v),
1251            Err(e) => {
1252                p.fault_vaddr = addr;
1253                Err(e)
1254            }
1255        };
1256        // `Status.RE` applies the **byte** swap here, `addr ^ 7`, not the swap for
1257        // the container's width. These instructions address individual bytes, so
1258        // the byte lane is what moves — and one XOR relocates the container and
1259        // complements the byte index together, which is why the two do not need
1260        // separate treatment:
1261        //
1262        //   `LWL 0` becomes container 4 with byte index 3, because
1263        //   `0 ^ 7 == 7`, `7 & !3 == 4` and `7 & 3 == 3`.
1264        //
1265        // Derived from n64-systemtest's own expected tables rather than guessed:
1266        // `SWL` at offset 0 writes a single byte, `rt`'s most significant, into
1267        // the doubleword's LAST byte, which no width-based swap produces.
1268        let eaddr = if self.reverse_endian() {
1269            addr ^ 7
1270        } else {
1271            addr
1272        };
1273        let word_addr = translate(self, eaddr & !3)?;
1274        let dword_addr = translate(self, eaddr & !7)?;
1275        let byte4 = eaddr & 3;
1276        let byte8 = eaddr & 7;
1277        Ok(match op {
1278            Op::Lwl | Op::Lwr => {
1279                let w = self.read_width(bus, word_addr, 4) as u32;
1280                let v = if matches!(op, Op::Lwl) {
1281                    mem::lwl(rt, w, byte4)
1282                } else {
1283                    mem::lwr(rt, w, byte4)
1284                };
1285                WriteBack::Gpr { dest, value: v }
1286            }
1287            Op::Ldl | Op::Ldr => {
1288                let d = self.read_width(bus, dword_addr, 8);
1289                let v = if matches!(op, Op::Ldl) {
1290                    mem::ldl(rt, d, byte8)
1291                } else {
1292                    mem::ldr(rt, d, byte8)
1293                };
1294                WriteBack::Gpr { dest, value: v }
1295            }
1296            Op::Swl | Op::Swr => {
1297                let w = self.read_width(bus, word_addr, 4) as u32;
1298                let merged = if matches!(op, Op::Swl) {
1299                    mem::swl(rt, w, byte4)
1300                } else {
1301                    mem::swr(rt, w, byte4)
1302                };
1303                self.write_width(bus, word_addr, 4, u64::from(merged));
1304                WriteBack::None
1305            }
1306            Op::Sdl | Op::Sdr => {
1307                let d = self.read_width(bus, dword_addr, 8);
1308                let merged = if matches!(op, Op::Sdl) {
1309                    mem::sdl(rt, d, byte8)
1310                } else {
1311                    mem::sdr(rt, d, byte8)
1312                };
1313                self.write_width(bus, dword_addr, 8, merged);
1314                WriteBack::None
1315            }
1316            // `MemOp::Unaligned` is only ever constructed for the eight
1317            // forms above.
1318            _ => WriteBack::None,
1319        })
1320    }
1321
1322    /// Which coprocessor an instruction needs, if that unit is disabled.
1323    ///
1324    /// `Status.CU` (31:28) is one bit per unit. Two rules that are easy to miss:
1325    ///
1326    /// - **COP0 is usable from kernel mode regardless of `CU0`** — otherwise the
1327    ///   CPU could not run an exception handler before `Status` had been set up,
1328    ///   which is a chicken-and-egg the hardware does not have. Kernel mode is
1329    ///   `KSU == 0`, or `EXL`/`ERL` set.
1330    /// - A **valid but unimplemented** COP1 encoding still checks `CU1`. With
1331    ///   `CU1` set it must *not* raise here, so that Sprint 3's arithmetic is an
1332    ///   addition rather than a behavior change.
1333    fn unusable_coprocessor(&self, d: Decoded) -> Option<u8> {
1334        use crate::decode::Op;
1335        let unit = match d.op {
1336            Op::Mfc0
1337            | Op::Dmfc0
1338            | Op::Mtc0
1339            | Op::Dmtc0
1340            | Op::Tlbr
1341            | Op::Tlbwi
1342            | Op::Tlbwr
1343            | Op::Tlbp
1344            | Op::Eret => 0,
1345            Op::Cfc1
1346            | Op::Ctc1
1347            | Op::Mfc1
1348            | Op::Dmfc1
1349            | Op::Mtc1
1350            | Op::Dmtc1
1351            | Op::Lwc1
1352            | Op::Ldc1
1353            | Op::Swc1
1354            | Op::Sdc1
1355            | Op::Cop1Unimplemented
1356            | Op::Cop1ReservedControl
1357            // FP arithmetic is a COP1 instruction like any other and must raise
1358            // Coprocessor Unusable with `CU1` clear. It was omitted when
1359            // `FpArith` was introduced, which left the arithmetic executing
1360            // unconditionally -- a program that had not enabled COP1 would get
1361            // results instead of an exception.
1362            | Op::FpArith => 1,
1363            Op::Cop2 | Op::Cop2ReservedControl | Op::Mfc2 | Op::Dmfc2 | Op::Mtc2 => 2,
1364            _ => return None,
1365        };
1366        let status = self.cop0.read(crate::cop0::reg::STATUS);
1367        if unit == 0 {
1368            /// `Status.KSU` (4:3) — 0 is kernel, 1 supervisor, 2 user.
1369            const KSU: u64 = 0b11 << 3;
1370            /// `Status.EXL` (1) or `Status.ERL` (2): either forces kernel mode
1371            /// regardless of `KSU`, which is what makes an exception handler's
1372            /// first instructions safe.
1373            const EXL_OR_ERL: u64 = 0b110;
1374            let kernel = status & KSU == 0 || status & EXL_OR_ERL != 0;
1375            if kernel {
1376                return None;
1377            }
1378        }
1379        if status & (1 << (28 + u64::from(unit))) == 0 {
1380            return Some(unit);
1381        }
1382        None
1383    }
1384
1385    /// `Status.RE` — reverse endian, and **only in User mode** (UM §5.2).
1386    ///
1387    /// `EXL`/`ERL` force kernel mode regardless of `KSU`, so an exception
1388    /// handler reads memory the way it wrote it even with `RE` set. That is the
1389    /// same rule the coprocessor-usability check applies, for the same reason.
1390    fn reverse_endian(&self) -> bool {
1391        /// `Status.RE`, bit 25.
1392        const RE: u64 = 1 << 25;
1393        /// `Status.KSU` (4:3) — 2 is user.
1394        const KSU_USER: u64 = 0b10 << 3;
1395        /// `Status.KSU` (4:3).
1396        const KSU: u64 = 0b11 << 3;
1397        /// `Status.EXL` (1) or `Status.ERL` (2).
1398        const EXL_OR_ERL: u64 = 0b110;
1399        let status = self.cop0.read(crate::cop0::reg::STATUS);
1400        status & RE != 0 && status & KSU == KSU_USER && status & EXL_OR_ERL == 0
1401    }
1402
1403    /// The reverse-endian byte-lane swap for an access of `width` bytes.
1404    ///
1405    /// Reversing endianness on a 64-bit datapath is a permutation of byte lanes
1406    /// within the doubleword, expressed as an XOR of the low address bits: a
1407    /// doubleword access covers the whole lane set and does not move, a word
1408    /// moves by 4, a halfword by 6 and a byte by 7.
1409    const fn re_swap(width: u64) -> u32 {
1410        match width {
1411            1 => 7,
1412            2 => 6,
1413            4 => 4,
1414            _ => 0,
1415        }
1416    }
1417
1418    /// Translate a data address and apply the reverse-endian swap, if any.
1419    ///
1420    /// The swap is applied to the **physical** address, after translation. It
1421    /// touches only bits 2:0, which every translation maps identically, so this
1422    /// is exactly equivalent to swapping the virtual address first — and it
1423    /// keeps `BadVAddr` raw on a fault, which n64-systemtest asserts directly
1424    /// ("RE unmapped access keeps raw `BadVAddr`").
1425    ///
1426    /// # Errors
1427    ///
1428    /// Whatever [`Pipeline::translate_data`] raises on the untransformed
1429    /// address: a TLB fault, **or** [`Exception::AddressError`] when the address
1430    /// is not valid in the current privilege mode.
1431    fn translate_re(
1432        &mut self,
1433        vaddr: u64,
1434        store: bool,
1435        width: u64,
1436    ) -> Result<crate::addr::Physical, Exception> {
1437        let mut p = self.translate_data(vaddr, store)?;
1438        if self.reverse_endian() {
1439            p.addr ^= Self::re_swap(width);
1440        }
1441        Ok(p)
1442    }
1443
1444    /// The privilege mode and addressing width the next access runs under.
1445    ///
1446    /// `EXL`/`ERL` force Kernel regardless of `KSU` — the same rule the
1447    /// coprocessor-usability check applies, and what makes an exception
1448    /// handler's first instructions safe.
1449    ///
1450    /// The addressing-width bit is picked **for the resolved mode**, not for
1451    /// `KSU`: a User-mode program that takes an exception runs the handler in
1452    /// Kernel mode, and the handler's addresses are governed by `KX`.
1453    fn access_mode(&self) -> crate::addr::Access {
1454        /// `Status.KSU`, bits 4:3.
1455        const KSU: u64 = 0b11 << 3;
1456        /// `Status.EXL` (1) or `Status.ERL` (2).
1457        const EXL_OR_ERL: u64 = 0b110;
1458        let status = self.cop0.read(crate::cop0::reg::STATUS);
1459        let mode = if status & EXL_OR_ERL != 0 {
1460            crate::addr::Mode::Kernel
1461        } else {
1462            match (status & KSU) >> 3 {
1463                1 => crate::addr::Mode::Supervisor,
1464                2 => crate::addr::Mode::User,
1465                // 0 is Kernel. 3 is not a defined encoding and the manual
1466                // gives it no behavior; it falls in with Kernel because that is
1467                // what `KSU == 0` already does and it keeps the match total
1468                // without inventing a fourth mode. Note this is the most
1469                // PERMISSIVE choice, not the safest one -- if a test ever pins
1470                // `KSU == 3`, it belongs in the accuracy ledger, not here.
1471                _ => crate::addr::Mode::Kernel,
1472            }
1473        };
1474        // `Status.UX` (5), `SX` (6), `KX` (7).
1475        let wide = match mode {
1476            crate::addr::Mode::User => status & (1 << 5) != 0,
1477            crate::addr::Mode::Supervisor => status & (1 << 6) != 0,
1478            crate::addr::Mode::Kernel => status & (1 << 7) != 0,
1479        };
1480        crate::addr::Access {
1481            mode,
1482            wide,
1483            erl: self.erl(),
1484        }
1485    }
1486
1487    /// Is there an instruction in `DC` that an interrupt can legitimately be
1488    /// charged to (ledger **R-18**)?
1489    ///
1490    /// An interrupt is taken *"at instruction boundaries"* (UM §4.7.1), and
1491    /// [`Pipeline::abort_from`] records `EPC` from whatever sits in `ex_dc`. Two
1492    /// things can sit there that are **not** an instruction boundary, and
1493    /// charging either one corrupts the return address:
1494    ///
1495    /// - **A bubble.** A default [`Latch`] has `pc == 0`, so the interrupt is
1496    ///   charged to address 0 and the handler returns there.
1497    /// - **An `ERET` that has already redirected.** `ERET` resolves in `EX`
1498    ///   (see [`Pipeline::ex_stage`]), clearing `Status.EXL` and pointing
1499    ///   `next_pc` at `EPC`. One cycle later the `DC` check sees `EXL == 0` and
1500    ///   fires — onto the `ERET` itself, overwriting the very `EPC` it was about
1501    ///   to consume. The handler then returns to the `ERET`, which resumes at its
1502    ///   own address: an architectural **livelock**. This is what stopped
1503    ///   Banjo-Tooie dead, measured at 390,625 `ERET` retirements per frame with
1504    ///   no other instruction ever reaching `WB`.
1505    ///
1506    /// Both must be excluded together. Excluding only the `ERET` moves the
1507    /// corruption to the refill bubble that follows its redirect (`EPC == 0`);
1508    /// excluding only bubbles leaves the `ERET` livelock intact.
1509    ///
1510    /// **Deferring loses nothing, by two different mechanisms** — and the
1511    /// distinction matters, because `Cause.IP` is *not* uniformly a bus level:
1512    ///
1513    /// - **`IP2`** (the RCP line) is re-sampled from `bus.poll_irq()` every
1514    ///   cycle, so a still-asserted line is simply seen again next cycle.
1515    /// - **`IP7`** (the timer) is **latched** on the `Count`/`Compare` edge and
1516    ///   stays set until `Compare` is written (UM §6.4.18) — see the one-way
1517    ///   `if` in [`Pipeline::dc_stage`]. It survives deferral because it is
1518    ///   sticky, not because it is re-sampled.
1519    ///
1520    /// Either way the interrupt is taken on the next cycle that presents a real
1521    /// instruction — after an `ERET`, the instruction it returned to. It is
1522    /// re-attributed, never dropped.
1523    fn interrupt_has_an_instruction_to_charge(&self) -> bool {
1524        self.ex_dc.occupied && !matches!(self.ex_dc.cop0, Some(Cop0Access::Eret))
1525    }
1526
1527    /// Are the MIPS III 64-bit operations reserved right now?
1528    ///
1529    /// Yes in User or Supervisor mode with that mode's `UX`/`SX` bit clear; never
1530    /// in Kernel mode, whatever `KX` says. Both halves are load-bearing: gating
1531    /// on the width bit alone would reserve them for a 32-bit kernel, and gating
1532    /// on the mode alone would reserve them for a 64-bit user program.
1533    fn sixty_four_bit_is_reserved(&self) -> bool {
1534        let a = self.access_mode();
1535        !matches!(a.mode, crate::addr::Mode::Kernel) && !a.wide
1536    }
1537
1538    /// `Status.ERL` — the error level, which makes KUSEG unmapped (UM §5.2.2).
1539    fn erl(&self) -> bool {
1540        self.cop0.read(crate::cop0::reg::STATUS) & (1 << 2) != 0
1541    }
1542
1543    /// Mirror a TLB shutdown into `Status.TS`.
1544    ///
1545    /// `TS` is read-only to software (UM Fig. 6-6, p. 167), so it goes through
1546    /// `set_hardware`. Without this the shutdown flag would be recorded inside
1547    /// the TLB and never observed, which is worse than not tracking it: software
1548    /// polls `Status.TS` precisely to discover that the TLB has died.
1549    fn note_shutdown(&mut self) {
1550        if self.tlb.is_shutdown() {
1551            let status = self.cop0.read(crate::cop0::reg::STATUS);
1552            self.cop0
1553                .set_hardware(crate::cop0::reg::STATUS, status | (1 << 21));
1554        }
1555    }
1556
1557    /// Perform a memory access.
1558    ///
1559    /// # Errors
1560    ///
1561    /// [`Exception::AddressError`] when an *aligned* access is misaligned. The
1562    /// `LWL`/`LWR` family is exempt by construction — being usable at any byte
1563    /// offset is the entire reason it exists.
1564    fn access<B: Bus>(&mut self, bus: &mut B, op: MemOp) -> Result<WriteBack, Exception> {
1565        // TODO(T-11-003): charge the cache-miss cost (8..=9 + M PCycles for a
1566        // D-cache fill, UM Table 11-1) once `M` is measured -- accuracy-ledger C-1.
1567        match op {
1568            MemOp::Load { kind, addr, dest } => {
1569                if !kind.is_aligned(addr) {
1570                    self.fault_vaddr = addr;
1571                    return Err(Exception::AddressError { store: false });
1572                }
1573                let phys = self.translate_re(addr, false, kind.width())?;
1574                let raw = self.read_width(bus, phys, kind.width());
1575                Ok(WriteBack::Gpr {
1576                    dest,
1577                    value: kind.shape(raw),
1578                })
1579            }
1580            MemOp::Store { kind, addr, value } => {
1581                if !kind.is_aligned(addr) {
1582                    self.fault_vaddr = addr;
1583                    return Err(Exception::AddressError { store: true });
1584                }
1585                let phys = self.translate_re(addr, true, kind.width())?;
1586                self.write_width(bus, phys, kind.width(), value);
1587                Ok(WriteBack::None)
1588            }
1589            // Load linked: an ordinary aligned load that also arms the link.
1590            MemOp::LinkedLoad { kind, addr, dest } => {
1591                if !kind.is_aligned(addr) {
1592                    // "If either of the low-order two bits of the address are
1593                    // not zero, an address error exception takes place" (UM §16
1594                    // p. 453) -- and the link is NOT armed, because the
1595                    // instruction did not complete.
1596                    self.fault_vaddr = addr;
1597                    return Err(Exception::AddressError { store: false });
1598                }
1599                let phys = self.translate_re(addr, false, kind.width())?;
1600                let raw = self.read_width(bus, phys, kind.width());
1601                self.ll_bit = true;
1602                // "the value with the high-order four bits of the physical
1603                // address PA(31:4) ... zero-extended" (UM Figure 5-17). Written
1604                // via `set_hardware` because LLAddr is software-writable too:
1605                // this is the hardware side effect, not an MTC0.
1606                self.cop0
1607                    .set_hardware(crate::cop0::reg::LL_ADDR, u64::from(phys.addr >> 4));
1608                Ok(WriteBack::Gpr {
1609                    dest,
1610                    value: kind.shape(raw),
1611                })
1612            }
1613            // Store conditional: the store is conditional, the flag write is not.
1614            MemOp::ConditionalStore {
1615                kind,
1616                addr,
1617                value,
1618                dest,
1619            } => {
1620                if !kind.is_aligned(addr) {
1621                    // "If this instruction both fails and causes an exception,
1622                    // the exception takes precedence" (UM §16 p. 487) -- so the
1623                    // address check runs before the link bit is even consulted,
1624                    // and `dest` is left alone.
1625                    self.fault_vaddr = addr;
1626                    return Err(Exception::AddressError { store: true });
1627                }
1628                let stored = self.ll_bit;
1629                if stored {
1630                    let phys = self.translate_re(addr, true, kind.width())?;
1631                    self.write_width(bus, phys, kind.width(), value);
1632                }
1633                // Written whether or not the store happened. Note the link bit
1634                // is deliberately NOT cleared here -- see `Pipeline::ll_bit`.
1635                Ok(WriteBack::Gpr {
1636                    dest,
1637                    value: u64::from(stored),
1638                })
1639            }
1640            // FP loads and stores. Same alignment and translation rules as the
1641            // integer forms -- only the destination register file differs.
1642            MemOp::Fp { op, addr, ft } => {
1643                use crate::decode::Op;
1644                let double = matches!(op, Op::Ldc1 | Op::Sdc1);
1645                let store = matches!(op, Op::Swc1 | Op::Sdc1);
1646                let align = if double { 7 } else { 3 };
1647                if addr & align != 0 {
1648                    self.fault_vaddr = addr;
1649                    return Err(Exception::AddressError { store });
1650                }
1651                let phys = self.translate_re(addr, store, if double { 8 } else { 4 })?;
1652                // `Status.FR` selects the register-file view; a double under
1653                // FR = 0 occupies an FGR pair.
1654                let fr = fr_of(&self.cop0);
1655                match op {
1656                    Op::Lwc1 => {
1657                        let v = self.read_width(bus, phys, 4) as u32;
1658                        self.fpr.write_s(ft, fr_of(&self.cop0), v);
1659                    }
1660                    Op::Ldc1 => {
1661                        let v = self.read_width(bus, phys, 8);
1662                        self.fpr.write_d(ft, fr, v);
1663                    }
1664                    Op::Swc1 => {
1665                        let v = self.fpr.read_s(ft, fr_of(&self.cop0));
1666                        self.write_width(bus, phys, 4, u64::from(v));
1667                    }
1668                    _ => {
1669                        let v = self.fpr.read_d(ft, fr);
1670                        self.write_width(bus, phys, 8, v);
1671                    }
1672                }
1673                Ok(WriteBack::None)
1674            }
1675            // EMUX — n64-systemtest's emulator-extension protocol, executed
1676            // here because it needs the bus. See `docs/cpu.md` and ledger C-8:
1677            // hardware leaves COP0 CO `funct` 0x20-0x3F inert, which is exactly
1678            // why emux claimed the range.
1679            MemOp::Emux {
1680                funct,
1681                code,
1682                ptr,
1683                len,
1684                dest,
1685            } => self.emux(bus, funct, code, ptr, len, dest),
1686            MemOp::Cache { addr, op } => self.cache_op(bus, addr, op),
1687            // The unaligned family accesses the ALIGNED container holding `addr`
1688            // and merges, so it can never raise an address error -- but it CAN
1689            // still raise a TLB fault, which is why it is fallible.
1690            MemOp::Unaligned { op, addr, rt, dest } => {
1691                self.access_unaligned(bus, op, addr, rt, dest)
1692            }
1693        }
1694    }
1695
1696    /// **EMUX** — the emulator-extension protocol n64-systemtest uses to talk to
1697    /// its host (`ref-proj/n64-systemtest/src/emux.rs`).
1698    ///
1699    /// Three operations, all COP0 CO encodings in the `funct` 0x20-0x3F range
1700    /// that hardware retires inertly (ledger **C-8**):
1701    ///
1702    /// | `funct` | name | effect |
1703    /// |---|---|---|
1704    /// | `0x20` | `xdetect` | return a capability bitmask in `rd` |
1705    /// | `0x25` | `xlog` | print `GPR[rt]` bytes from `GPR[rd]` |
1706    /// | `0x2C` | `xioctl` | `code 1` exit, `code 2` "fast mode" |
1707    ///
1708    /// # Why implement it rather than keep the no-op
1709    ///
1710    /// `xlog` is a console that needs **no PI, SI or `ISViewer` emulation** — the
1711    /// suite's output reaches the host directly. `xioctl(EXIT)` turns "the run
1712    /// finished" from a tick-budget guess into a definite signal. Both are worth
1713    /// having long before the cartridge subsystem exists.
1714    ///
1715    /// Advertising a capability obliges us to implement it: the bitmask below
1716    /// claims exactly `xlog` and `xioctl`, so the suite routes text to us only
1717    /// because we can actually take it.
1718    fn emux<B: Bus>(
1719        &mut self,
1720        bus: &mut B,
1721        funct: u8,
1722        code: u16,
1723        ptr: u64,
1724        len: u64,
1725        dest: u8,
1726    ) -> Result<WriteBack, Exception> {
1727        /// `XDETECT` — capability probe.
1728        const XDETECT: u8 = 0o40;
1729        /// `XLOG` — write a string to the host.
1730        const XLOG: u8 = 0o45;
1731        /// `XIOCTL` — control operations.
1732        const XIOCTL: u8 = 0o54;
1733        /// `xdetect` code 1: "which of funct 0x20-0x3F do you support?"
1734        const CODE_EXTENSIONS: u16 = 1;
1735        /// `xioctl` code 1: terminate.
1736        const XIOCTL_EXIT: u16 = 1;
1737
1738        // Hardware has no EMUX. Unless the host opts in, every encoding in the
1739        // range stays inert -- which is both ledger C-8 and what a default ares
1740        // build does (`if(!system.homebrewMode) return;`). Getting this wrong is
1741        // observable: advertising capabilities makes n64-systemtest switch
1742        // console backends, which changes the retired-instruction stream and
1743        // diverges from a hardware-accurate reference trace.
1744        if !bus.emux_enabled() {
1745            return Ok(WriteBack::None);
1746        }
1747        match funct {
1748            XDETECT if code == CODE_EXTENSIONS => {
1749                // Bit N answers for `funct` 0x20 + N. We claim `xlog` and
1750                // `xioctl` and nothing else, because those are what is below.
1751                let mask = (1u64 << (XLOG - XDETECT)) | (1u64 << (XIOCTL - XDETECT));
1752                Ok(WriteBack::Gpr { dest, value: mask })
1753            }
1754            XLOG => {
1755                // Bounded deliberately: `len` is guest-controlled, and a corrupt
1756                // or hostile value must not turn a log call into an unbounded
1757                // read. 4 KiB is far above any line the suite emits.
1758                const MAX: u64 = 4096;
1759                let n = len.min(MAX);
1760                let mut buf = alloc::vec::Vec::with_capacity(n as usize);
1761                for i in 0..n {
1762                    // Translated per byte: the string may straddle a page, and a
1763                    // fault here is a real fault the guest must see.
1764                    let p = self.translate_data(ptr.wrapping_add(i), false)?;
1765                    // Read THROUGH the D-cache, exactly as a guest `LB` would.
1766                    // The string was just formatted by cached stores, so it is
1767                    // still sitting in dirty cache lines -- going straight to
1768                    // the bus reads stale RDRAM and prints blanks where the
1769                    // digits should be. That is precisely the staleness the
1770                    // cache model exists to reproduce, so the log channel has to
1771                    // respect it too.
1772                    buf.push(self.read_width(bus, p, 1) as u8);
1773                }
1774                bus.emux_log(&buf);
1775                Ok(WriteBack::None)
1776            }
1777            XIOCTL if code == XIOCTL_EXIT => {
1778                bus.emux_exit();
1779                Ok(WriteBack::None)
1780            }
1781            // Every other encoding in the range stays inert, which is the
1782            // hardware behavior C-8 records and the reason the range is usable
1783            // as extension space at all.
1784            _ => Ok(WriteBack::None),
1785        }
1786    }
1787
1788    /// `CACHE`: dispatch to the index- or address-addressed half.
1789    ///
1790    /// Only the ADDRESS-addressed operations translate. `op4..2`
1791    /// (UM Ch. 16, p. 404):
1792    ///
1793    /// - `0..=2` `Index_Invalidate` / `Index_Load_Tag` / `Index_Store_Tag` —
1794    ///   address the cache "at the index specified", so they never consult the
1795    ///   TLB and cannot fault.
1796    /// - `3` `Create_Dirty_Exclusive` — "set the cache block tag to the
1797    ///   specified physical address", so it does.
1798    /// - `4..=6` `Hit_*` — "if the cache block contains the specified address",
1799    ///   so they do.
1800    ///
1801    /// Translating unconditionally raises spurious TLB refills on `Index_*` ops
1802    /// against unmapped addresses, which is exactly what cache-init code does at
1803    /// boot: walk every index with an arbitrary base. An earlier revision of
1804    /// this comment described the distinction while the code ignored it.
1805    ///
1806    /// # Errors
1807    ///
1808    /// A TLB fault, on the address-addressed operations only.
1809    fn cache_op<B: Bus>(&mut self, bus: &mut B, addr: u64, op: u8) -> Result<WriteBack, Exception> {
1810        // Any cache operation can change what the I-cache would serve, so the
1811        // idle-loop recognition that borrows the I-cache's validity stops being
1812        // valid here. This hook is the ONLY thing that makes caching that
1813        // recognition sound (see `Pipeline::idle_pc`), so it sits at the single
1814        // entry point for every CACHE variant rather than at the invalidating
1815        // ones — a narrower hook would have to be re-argued every time a variant
1816        // is added, and being wrong about one would be silent.
1817        self.idle_pc = None;
1818        if (op >> 2) >= 3 {
1819            let p = self.translate_data(addr, false)?;
1820            self.cache_hit_op(bus, op, p.addr);
1821        } else {
1822            // An `Index_*` op indexes on the raw VIRTUAL address, which is what
1823            // the hardware does: both primary caches are VIRTUALLY indexed.
1824            //
1825            // This model indexes by physical address (ledger D-6), so on a
1826            // TLB-mapped page the two can select different lines -- translation
1827            // preserves only the low 12 bits, while the D-cache index reaches
1828            // bit 12 and the I-cache bit 13. That divergence is exactly what D-6
1829            // records. It is NOT the claim an earlier revision of this comment
1830            // made -- that translation leaves the index bits alone -- which is
1831            // false for every mapped segment.
1832            self.cache_index_op(bus, op, addr as u32);
1833        }
1834        Ok(WriteBack::None)
1835    }
1836
1837    /// The index-addressed half of `CACHE` (`op4..2` in `0..=2`).
1838    ///
1839    /// `addr` is the *virtual* address: these operations select a line by index
1840    /// and never translate. Bit 0 of `op` picks the cache.
1841    fn cache_index_op<B: Bus>(&mut self, bus: &mut B, op: u8, addr: u32) {
1842        let dcache = op & 1 == 1;
1843        match (op >> 2, dcache) {
1844            // Index_Invalidate (I) / Index_WriteBack_Invalidate (D).
1845            //
1846            // Both clear the valid bit and LEAVE THE TAG in place, which
1847            // `Index_Load_Tag` then reports. Clearing the tag as well would look
1848            // tidier and would be wrong: n64-systemtest asserts the PFN is
1849            // unchanged across an invalidate.
1850            (0, false) => self.icache.invalidate_index(addr),
1851            (0, true) => {
1852                if let Some(w) = self.dcache.flush_index(addr, true, false) {
1853                    Self::push_line(bus, w.addr, &w.data);
1854                }
1855            }
1856            // Index_Load_Tag -> COP0 TagLo. Written with `set_hardware` because
1857            // this is the hardware side effect of CACHE, not an MTC0, and the
1858            // register's software write mask must not apply to it.
1859            (1, false) => {
1860                let t = self.icache.load_tag(addr);
1861                self.cop0
1862                    .set_hardware(crate::cop0::reg::TAG_LO, u64::from(t));
1863            }
1864            (1, true) => {
1865                let t = self.dcache.load_tag(addr);
1866                self.cop0
1867                    .set_hardware(crate::cop0::reg::TAG_LO, u64::from(t));
1868            }
1869            // Index_Store_Tag <- COP0 TagLo.
1870            (2, false) => {
1871                let t = self.cop0.read(crate::cop0::reg::TAG_LO) as u32;
1872                self.icache.store_tag(addr, t);
1873            }
1874            (2, true) => {
1875                let t = self.cop0.read(crate::cop0::reg::TAG_LO) as u32;
1876                self.dcache.store_tag(addr, t);
1877            }
1878            _ => {}
1879        }
1880    }
1881
1882    /// The address-addressed half of `CACHE` (`op4..2` in `3..=6`).
1883    ///
1884    /// `addr` is PHYSICAL — the caller has already translated it, which is what
1885    /// makes a TLB fault on these operations possible.
1886    fn cache_hit_op<B: Bus>(&mut self, bus: &mut B, op: u8, addr: u32) {
1887        let dcache = op & 1 == 1;
1888        match (op >> 2, dcache) {
1889            // Create_Dirty_Exclusive (D only): claim the line without a fill.
1890            (3, true) => {
1891                if let Some(w) = self.dcache.create_dirty_exclusive(addr) {
1892                    Self::push_line(bus, w.addr, &w.data);
1893                }
1894            }
1895            // Hit_Invalidate: only if the line is actually resident.
1896            (4, false) => self.icache.hit_invalidate(addr),
1897            (4, true) => {
1898                if self.dcache.hits(addr) {
1899                    self.dcache.flush_index(addr, true, false);
1900                }
1901            }
1902            // Fill (I) — an unconditional line fill, no hit test.
1903            (5, false) => self.icache_fill(bus, addr),
1904            // Hit_WriteBack_Invalidate (D).
1905            (5, true) => {
1906                if self.dcache.hits(addr)
1907                    && let Some(w) = self.dcache.flush_index(addr, true, false)
1908                {
1909                    Self::push_line(bus, w.addr, &w.data);
1910                }
1911            }
1912            // Hit_WriteBack: the line stays resident, and stops being dirty.
1913            (6, false) => {
1914                if let Some(w) = self.icache.flush_hit(addr) {
1915                    Self::push_line(bus, w.addr, &w.data);
1916                }
1917            }
1918            (6, true) => {
1919                if self.dcache.hits(addr)
1920                    && let Some(w) = self.dcache.flush_index(addr, false, true)
1921                {
1922                    Self::push_line(bus, w.addr, &w.data);
1923                }
1924            }
1925            _ => {}
1926        }
1927    }
1928
1929    /// D-cache line fill cost in PClocks (`8..=9 + M(RDRAM)`, UM Table 11-1).
1930    ///
1931    /// `M(RDRAM)` is **not a scalar** (ledger C-1) — it is bank-state dependent
1932    /// (C-4): a 2 KiB RDRAM row spans 128 D-cache lines, so sequential access hits
1933    /// the open row (**fast**) and random access misses it (**slow**). This code
1934    /// charges this one value for **every** miss — there is **no row-state
1935    /// dispatch yet** — so 40 is a **provisional row-hit-typical estimate**, not a
1936    /// measured number: the UM gives the fill *formula* but not this warm value,
1937    /// and ares (40) / cen64 (44) are corroboration only. The documented
1938    /// **cold/row-miss** fill is ~60 PClocks (copetti's ~640 ns external estimate
1939    /// x 93.75 MHz PClock, `M ~= 52`); charging it, and the dirty-writeback case,
1940    /// needs the undocumented `RasInterval` cycles and is deferred to C-4.
1941    #[allow(clippy::doc_markdown)]
1942    const M_DCACHE_FILL: u32 = 40;
1943    /// I-cache line fill cost in PClocks (`14..=15 + M(RDRAM)`, UM Table 11-2).
1944    ///
1945    /// Defined as [`Self::M_DCACHE_FILL`] plus 6, not a bare literal, so the two
1946    /// stay in lockstep by construction: the fills share the same row-hit
1947    /// `M(RDRAM)` and differ only by the UM's line-transfer size — the I-line
1948    /// moves 8 words vs the D-line's critical doubleword of 2, and `14 - 8 = 6`
1949    /// (Table 11-2 minus 11-1). Currently `40 + 6 = 46` (cen64 uses 48); a future
1950    /// measured `M(RDRAM)` (ledger C-1/C-4) changes both together.
1951    #[cfg_attr(test, allow(dead_code))]
1952    #[allow(clippy::doc_markdown)]
1953    const M_ICACHE_FILL: u32 = Self::M_DCACHE_FILL + 6;
1954
1955    /// Make the I-cache line covering `addr` resident.
1956    fn icache_fill<B: Bus>(&mut self, bus: &mut B, addr: u32) {
1957        let base = addr & !(crate::cache::ICACHE_LINE - 1);
1958        let mut data = [0u8; 32];
1959        Self::pull_line(bus, base, &mut data);
1960        self.icache.install(base, data);
1961        // Called only on an I-cache miss (`ic_stage`), so the fill cost applies
1962        // here (fitted, ledger C-1). **Deliberate test seam:** an I-cache miss
1963        // fires on *every* cold fetch, and this crate's fine-grained pipeline unit
1964        // tests step fixed cycle counts for a free-fetch model AND assert on the
1965        // interlock/FPU stalls the fill would confound -- so the stall is charged
1966        // in real execution and in every INTEGRATION test (the i-cache microbench,
1967        // the systemtest, golden-log, residue -- where the pipeline runs as a
1968        // dependency with `cfg(test)` false), but not in this crate's own units.
1969        // The D-cache fill, by contrast, fires only on a rare cached load, so it
1970        // is charged unconditionally (two units absorbed it directly).
1971        #[cfg(not(test))]
1972        self.stall_for(Self::M_ICACHE_FILL, Interlock::Icb);
1973    }
1974
1975    /// Make the D-cache line covering `addr` resident, writing back whatever it
1976    /// evicts.
1977    ///
1978    /// A dirty-line writeback (the `push_line` below) is charged the **same**
1979    /// fixed `M_DCACHE_FILL` as a clean miss -- the fitted 40 does not separate
1980    /// clean- from dirty-eviction cost (a real hardware measurement would; the
1981    /// writeback is an extra RDRAM transaction). Honest given the value is a
1982    /// fitted anchor, not a measurement (ledger C-1).
1983    fn dcache_fill<B: Bus>(&mut self, bus: &mut B, addr: u32) {
1984        let Some(evicted) = self.dcache.miss_plan(addr) else {
1985            return; // hit -- no fill, no cost
1986        };
1987        if let Some(w) = evicted {
1988            Self::push_line(bus, w.addr, &w.data);
1989        }
1990        let base = addr & !(crate::cache::DCACHE_LINE - 1);
1991        let mut data = [0u8; 16];
1992        Self::pull_line(bus, base, &mut data);
1993        self.dcache.install(base, data);
1994        self.stall_for(Self::M_DCACHE_FILL, Interlock::Dcm);
1995    }
1996
1997    /// Write a whole cache line out to the bus, a word at a time.
1998    ///
1999    /// Word-wide rather than byte-wide because [`Bus::read_u32`]/[`Bus::write_u32`]
2000    /// are the calls `rustyn64-core` gives a fast RDRAM path; a byte loop would
2001    /// issue four times the bus traffic on every fill and eviction, and a line
2002    /// is always naturally aligned so the split is exact.
2003    fn push_line<B: Bus>(bus: &mut B, addr: u32, data: &[u8]) {
2004        for (k, w) in data.chunks_exact(4).enumerate() {
2005            let word = u32::from_be_bytes([w[0], w[1], w[2], w[3]]);
2006            bus.write_u32(addr.wrapping_add(k as u32 * 4), word);
2007        }
2008    }
2009
2010    /// Read a whole cache line from the bus into `data`.
2011    fn pull_line<B: Bus>(bus: &mut B, addr: u32, data: &mut [u8]) {
2012        for (k, w) in data.chunks_exact_mut(4).enumerate() {
2013            let word = bus.read_u32(addr.wrapping_add(k as u32 * 4));
2014            w.copy_from_slice(&word.to_be_bytes());
2015        }
2016    }
2017
2018    /// Uncached RCP-register access latency `M`, in PClocks — **measured, not
2019    /// tuned** (ledger C-1, T-11-003).
2020    ///
2021    /// Derived from the PeterLemon `CPUTIMINGNTSC` mult/div differential: the
2022    /// model `expected_i = W / (B + c_i)`, where `c_i` is each timed
2023    /// instruction's documented stall (UM Table 3-12: `mult` 5, `dmult` 8,
2024    /// `div` 37, `ddiv` 69), regresses to a window `W ~= 1.52e6` PClocks and a
2025    /// hardware base-loop `B ~= 25.5` PClocks (fit < 1.5% on the high-leverage
2026    /// mult/div points). Our base loop measured exactly 4.00 PClocks (four
2027    /// one-cycle instructions, no memory latency), so the missing `B - 4 ~= 21.5`
2028    /// PClocks is the uncached `lw VI_V_CURRENT` latency. Charging 22 drives the
2029    /// ROM's absolute iteration count from 304_180 to 56_330 vs the
2030    /// hardware-baked 56_092 (0.4%) -- the *independent* confirmation, since 22
2031    /// came from the differential, not from fitting the absolute count. Measured
2032    /// on VI; the sibling RCP registers share the bus and are charged the same
2033    /// pending their own vectors. RDRAM / cache-fill `M` stay unmeasured (0).
2034    #[allow(clippy::doc_markdown)]
2035    const M_RCP_REGISTER: u32 = 22;
2036    /// Low bound of the RCP MMIO block (SP..SI) charged [`Self::M_RCP_REGISTER`].
2037    const M_RCP_REGISTER_LO: u32 = 0x0400_0000;
2038    /// High bound of the RCP MMIO block.
2039    const M_RCP_REGISTER_HI: u32 = 0x04FF_FFFF;
2040
2041    /// Read `width` big-endian bytes, right-justified, through the D-cache when
2042    /// the access is cached.
2043    ///
2044    /// Dispatches on width so 4- and 8-byte *uncached* accesses go through
2045    /// [`Bus::read_u32`], which `rustyn64-core` overrides with a fast RDRAM path.
2046    /// A byte loop would issue 4-8x more bus calls on the *most common*
2047    /// operations, and memory access is the hot path for a core targeting full
2048    /// speed (`docs/performance.md`).
2049    ///
2050    /// Alignment is **not** rechecked here. `access` has already validated it
2051    /// against the specific [`crate::mem::LoadKind`]/[`crate::mem::StoreKind`],
2052    /// and the unaligned family passes an address it has aligned down itself.
2053    /// Duplicating the check would put the rule in two places, where it can drift.
2054    fn read_width<B: Bus>(&mut self, bus: &mut B, p: crate::addr::Physical, width: u64) -> u64 {
2055        let addr = p.addr;
2056        if p.cached == crate::addr::Cached::Yes {
2057            self.dcache_fill(bus, addr);
2058            return self.dcache.read(addr, width as usize);
2059        }
2060        let v = match width {
2061            1 => u64::from(bus.read_u8(addr)),
2062            2 => (u64::from(bus.read_u8(addr)) << 8) | u64::from(bus.read_u8(addr.wrapping_add(1))),
2063            4 => u64::from(bus.read_u32(addr)),
2064            // Big-endian: the high word is at the lower address.
2065            8 => {
2066                (u64::from(bus.read_u32(addr)) << 32)
2067                    | u64::from(bus.read_u32(addr.wrapping_add(4)))
2068            }
2069            _ => 0,
2070        };
2071        // Charge the uncached RCP-register access latency `M` (T-11-003, ledger
2072        // C-1). `addr` is physical (`p.addr`), so the KSEG1 `0xA4xx_xxxx` a game
2073        // uses arrives here as `0x04xx_xxxx`. Reuses the `Dcm` interlock (whose
2074        // doc already carries the `+ M` term) rather than adding a serialized enum
2075        // variant that would perturb the save-state format (ADR 0005). Only this
2076        // region is *measured*; other uncached reads (RDRAM) stay at 0.
2077        //
2078        // Charged once per access. `M` was measured for a 32-bit `lw`; the width
2079        // == 8 case issues two bus words, but a 64-bit uncached read of an RCP
2080        // register (an `ld` spanning two 32-bit registers) is not something the
2081        // oracle covers, so its two-transaction cost is left unmeasured rather
2082        // than assumed to be `2 * M`.
2083        if Self::is_rcp_mmio(addr) {
2084            self.stall_for(Self::M_RCP_REGISTER, Interlock::Dcm);
2085        }
2086        v
2087    }
2088
2089    /// Is the physical address `addr` in the RCP MMIO block charged
2090    /// [`Self::M_RCP_REGISTER`]?
2091    const fn is_rcp_mmio(addr: u32) -> bool {
2092        Self::M_RCP_REGISTER_LO <= addr && addr <= Self::M_RCP_REGISTER_HI
2093    }
2094
2095    /// Write the low `width` big-endian bytes of `value`, through the D-cache
2096    /// when the access is cached.
2097    ///
2098    /// A cached store is a **write-allocate**: the line is filled first, then
2099    /// modified and left dirty. The VR4300's D-cache has no write-around path,
2100    /// so a partial store to an absent line must read the rest of it from memory
2101    /// or the eventual write-back would push out fifteen bytes of nothing.
2102    ///
2103    /// Width-dispatched for the same reason as [`Pipeline::read_width`].
2104    fn write_width<B: Bus>(
2105        &mut self,
2106        bus: &mut B,
2107        p: crate::addr::Physical,
2108        width: u64,
2109        value: u64,
2110    ) {
2111        let addr = p.addr;
2112        if p.cached == crate::addr::Cached::Yes {
2113            self.dcache_fill(bus, addr);
2114            self.dcache.write(addr, width as usize, value);
2115            return;
2116        }
2117        // Hand the bus the width and the *untruncated* register. Narrowing here
2118        // would discard the upper bits before the target can decide whether it
2119        // wants them -- and every device on the RCP's internal bus does (see
2120        // `Bus::write_sized`).
2121        bus.write_sized(addr, width, value);
2122    }
2123
2124    /// The four checks an instruction must pass **before** it executes.
2125    ///
2126    /// Latch-independent, for the reason [`Self::fetch_word`] is: the caller
2127    /// decides what a refusal means. `ex_stage` turns an `Err` into an `Ex`-stage
2128    /// abort; the instruction-granular path (ADR 0013) needs the same four
2129    /// decisions with different consequences. Keeping one copy is what stops the
2130    /// two paths disagreeing about *which* exception an encoding raises — a
2131    /// disagreement n64-systemtest would report as a wrong cause code rather than
2132    /// as a missing check, which is much harder to read backwards.
2133    ///
2134    /// **The order is preserved exactly, and the first two refusals turn out to
2135    /// be disjoint.** The accurate path's comment says an unusable coprocessor is
2136    /// reported as such *"even when the encoding is also a 64-bit one"* — but
2137    /// mutation-checking that while extracting this changed nothing
2138    /// (n64-systemtest stayed at 0 failing in the Phase 1 categories, 90
2139    /// suite-wide), and the reason is stronger than a gap in the suite:
2140    /// [`Op::is_64_bit`](crate::decode::Op::is_64_bit) covers only CPU integer
2141    /// and load/store operations, while [`Self::unusable_coprocessor`] answers
2142    /// only for COP0/COP1/COP2 encodings. **No input is in both sets**, so no
2143    /// input can reach the second check by way of the first.
2144    ///
2145    /// So the ordering between those two is *unobservable*, not merely untested.
2146    /// It is kept because it is what the accurate path has always done, and
2147    /// `ex_gates_first_two_refusals_cannot_both_apply` sweeps the encoding space
2148    /// to keep the disjointness true — if a 64-bit coprocessor move is ever
2149    /// classified as 64-bit, that test fails and whoever changed it has to
2150    /// justify the precedence against the manual instead of inheriting it.
2151    ///
2152    /// Written out because "the order matters" is exactly the shape of claim this
2153    /// project keeps finding stale: nothing fails when it is wrong. A first draft
2154    /// of this comment asserted the order was load-bearing.
2155    ///
2156    /// # Errors
2157    ///
2158    /// One of [`Exception::CoprocessorUnusable`], [`Exception::ReservedInstruction`],
2159    /// [`Exception::CoprocessorReserved`], or [`Exception::FloatingPoint`].
2160    fn ex_gate(&mut self, decoded: Decoded) -> Result<(), Exception> {
2161        // Coprocessor usability is checked BEFORE execution, in EX (UM §4.7.5
2162        // lists CPU among the EX-stage exceptions). COP0 is exempt in kernel mode
2163        // regardless of `CU0`, which is why the CPU can run exception handlers
2164        // before any `Status` setup has happened.
2165        if let Some(unit) = self.unusable_coprocessor(decoded) {
2166            return Err(Exception::CoprocessorUnusable { unit });
2167        }
2168        // A 64-bit operation is RESERVED in 32-bit User or Supervisor mode.
2169        // Kernel may use them at any width, which is why this cannot be a
2170        // property of `Status.KX` alone.
2171        //
2172        // Checked after coprocessor usability, not before: an unusable
2173        // coprocessor is reported as such even when the encoding is also a
2174        // 64-bit one, and the suite distinguishes the two causes.
2175        if decoded.op.is_64_bit() && self.sixty_four_bit_is_reserved() {
2176            return Err(Exception::ReservedInstruction);
2177        }
2178        // `DCFC1`/`DCTC1` are usable-but-unimplemented: the `CU1` check above has
2179        // already passed, so this is the *other* outcome. The whole `Cause` field
2180        // is replaced, leaving only bit 17 -- the suite pre-loads unrelated cause
2181        // bits specifically to check they clear. COP2's equivalent declines
2182        // differently: Reserved Instruction, and `FCSR` is not involved at all.
2183        if decoded.op == crate::decode::Op::Cop2ReservedControl {
2184            return Err(Exception::CoprocessorReserved { unit: 2 });
2185        }
2186        if decoded.op == crate::decode::Op::Cop1ReservedControl {
2187            let fcsr = self.cop1.fcsr();
2188            self.cop1.ctc1(
2189                31,
2190                (fcsr & !crate::fpu::CAUSE_MASK) | crate::fpu::CAUSE_UNIMPLEMENTED,
2191            );
2192            return Err(Exception::FloatingPoint);
2193        }
2194        Ok(())
2195    }
2196
2197    /// `EX` — execute.
2198    fn ex_stage(&mut self, regs: &Regs, next_pc: &mut u64) {
2199        let mut out = self.rf_ex;
2200        if out.occupied && out.abort.is_none() {
2201            // Resolve operands through the bypass network rather than trusting
2202            // the values latched at RF, which may be one cycle stale.
2203            out.rs_val = self.bypass(out.decoded.rs, regs);
2204            out.rt_val = self.bypass(out.decoded.rt, regs);
2205            let hilo = self.bypass_hi_lo(regs);
2206            // The pre-execution refusals, all four of which are EX-stage
2207            // exceptions (UM §4.7.5). `ex_gate` holds the checks and the order
2208            // they must run in; here is only what an `Ex`-stage refusal *does*.
2209            if let Err(exc) = self.ex_gate(out.decoded) {
2210                self.abort_from(Stage::Ex, exc);
2211                out = self.rf_ex;
2212                self.rf_ex.occupied = false;
2213                self.ex_dc = out;
2214                return;
2215            }
2216            // `FCSR.C` — read here, in EX, where the branch resolves, THROUGH A
2217            // BYPASS. See `pending_fp_condition`.
2218            let fp_condition = self.pending_fp_condition().unwrap_or_else(|| {
2219                let p: &Self = self;
2220                p.cop1.fcsr() & FCSR_C != 0
2221            });
2222            match execute(
2223                out.decoded,
2224                out.rs_val,
2225                out.rt_val,
2226                hilo,
2227                out.pc,
2228                fp_condition,
2229            ) {
2230                Ok(e) => {
2231                    out.write_back = e.write_back;
2232                    out.mem = e.mem;
2233                    out.cop0 = e.cop0;
2234                    // Control flow. The delay slot has ALREADY been fetched -- it
2235                    // is in `ic_rf` right now, because IC ran a cycle ahead. That
2236                    // is the architectural delay slot, not a modeling artifact.
2237                    //
2238                    // Because the cascade runs backwards, `ic_stage` executes
2239                    // AFTER this in the same cycle, so writing `next_pc` here
2240                    // makes the very next fetch land on the target with exactly
2241                    // one delay slot in between. No wrong-path fetch needs
2242                    // squashing -- that falls out of the reverse order rather
2243                    // than being arranged.
2244                    // **The link is `next_pc`, read BEFORE this instruction's
2245                    // own redirect is applied.**
2246                    //
2247                    // At `EX` time `next_pc` already holds the address of the
2248                    // instruction that will run after this one's delay slot:
2249                    // `pc + 8` for an ordinary jump, and the OUTER target `+ 4`
2250                    // when this jump is itself in a delay slot, because the
2251                    // outer jump redirected a cycle earlier and `IC` has since
2252                    // advanced past it.
2253                    //
2254                    // Computing `pc + 8` in `execute` was right for the
2255                    // ordinary case and silently wrong for the nested one; the
2256                    // live `next_pc` is that address by construction rather
2257                    // than by a second formula that can disagree.
2258                    // **COP2 is one 64-bit latch, not a register file.** The
2259                    // register index is ignored entirely: n64-systemtest writes with
2260                    // one index and reads back with several others, including 30 and
2261                    // 31, and gets the same value every time. `MTC2` writes all 64
2262                    // bits despite being nominally a 32-bit move; `MFC2` returns the
2263                    // low half sign-extended and `DMFC2` the whole thing.
2264                    //
2265                    // The same shape as the reserved COP0 registers (ledger C-15) --
2266                    // this processor's answer to "a coprocessor that is not really
2267                    // there" is a single latch, twice over.
2268                    match out.decoded.op {
2269                        crate::decode::Op::Mtc2 => self.cop2_latch = out.rt_val,
2270                        crate::decode::Op::Mfc2 => {
2271                            out.write_back = WriteBack::Gpr {
2272                                dest: out.decoded.dest,
2273                                value: crate::alu::sext32(self.cop2_latch as u32),
2274                            };
2275                        }
2276                        crate::decode::Op::Dmfc2 => {
2277                            out.write_back = WriteBack::Gpr {
2278                                dest: out.decoded.dest,
2279                                value: self.cop2_latch,
2280                            };
2281                        }
2282                        _ => {}
2283                    }
2284                    self.resolve_branch_control(&mut out, e.link, e.redirect, next_pc);
2285                    // Multiply and divide stall the ENTIRE pipeline for the
2286                    // documented count (UM Table 3-12), so the request is raised
2287                    // here and honored from the next cycle onward.
2288                    if e.stall_cycles > 0 {
2289                        self.stall_for(e.stall_cycles, Interlock::Mci);
2290                    }
2291                    // ERET. Resolved here rather than in `execute` because its
2292                    // target comes out of COP0, not out of the instruction.
2293                    //
2294                    // It has NO delay slot (UM Ch. 16, p. 434) -- alone among
2295                    // the control transfers -- so the instruction IC already
2296                    // fetched must be squashed. Every branch reaches this point
2297                    // with its delay slot legitimately in flight, which is why
2298                    // the squash is spelled out here instead of falling out of
2299                    // the reverse cascade as a branch's does.
2300                    if matches!(e.cop0, Some(Cop0Access::Eret)) {
2301                        *next_pc = exception::eret(&mut self.cop0);
2302                        // "cleared by an ERET" (UM §3.1) -- the other half of
2303                        // the LL/SC contract, which had nothing clearing it
2304                        // until now.
2305                        self.ll_bit = false;
2306                        self.ic_rf = Latch::default();
2307                    }
2308                }
2309                // Stamp BEFORE the latch move, so the abort travels with the
2310                // instruction that caused it -- see `abort_from`.
2311                Err(exc) => {
2312                    self.abort_from(Stage::Ex, exc);
2313                    out = self.rf_ex;
2314                }
2315            }
2316        }
2317        self.ex_dc = out;
2318        self.rf_ex.occupied = false;
2319    }
2320
2321    /// Resolve a branch's link write and its taken-redirect, honoring the
2322    /// delay-slot-fault rule (ledger R-19).
2323    ///
2324    /// The delay slot IC fetched last cycle is in `ic_rf` right now; if it
2325    /// aborted, the exception has already pointed `next_pc` at the vector by the
2326    /// time this branch reaches EX. The branch must NOT take -- the exception PC
2327    /// wins -- yet it STILL retires and writes its link
2328    /// (`ExecuteTLBMappedMissInDelay` asserts `RA == fault_address + 4`). Letting
2329    /// the redirect fire would clobber the vector every cycle, so the handler
2330    /// never runs: a two-state hang between the faulting delay slot and the
2331    /// exception vector.
2332    fn resolve_branch_control(
2333        &mut self,
2334        out: &mut Latch,
2335        link: Option<u8>,
2336        redirect: Option<Redirect>,
2337        next_pc: &mut u64,
2338    ) {
2339        let delay_slot_faulted = out.decoded.op.has_delay_slot()
2340            && self.ic_rf.occupied
2341            && self.ic_rf.in_delay_slot
2342            && self.ic_rf.abort.is_some();
2343        if let Some(dest) = link {
2344            // The link is the address after the delay slot: normally the live
2345            // `next_pc`, but the architectural `pc + 8` when a fault has already
2346            // overwritten `next_pc` with the exception vector.
2347            let value = if delay_slot_faulted {
2348                out.pc.wrapping_add(8)
2349            } else {
2350                *next_pc
2351            };
2352            out.write_back = WriteBack::Gpr { dest, value };
2353        }
2354        if let Some(r) = redirect
2355            && !delay_slot_faulted
2356        {
2357            *next_pc = r.target;
2358            // A branch-LIKELY that was not taken squashes its already-fetched
2359            // delay slot. An ordinary branch never does.
2360            if r.nullify_delay_slot {
2361                self.ic_rf = Latch::default();
2362            }
2363        }
2364    }
2365
2366    /// `RF` — register fetch, and where the load interlock is detected.
2367    fn rf_stage(&mut self, regs: &Regs) {
2368        let mut out = self.ic_rf;
2369        if out.occupied {
2370            out.rs_val = regs.read(out.decoded.rs);
2371            out.rt_val = regs.read(out.decoded.rt);
2372        }
2373        // These reads are a first approximation: EX re-resolves them through the
2374        // bypass network, since a producer one instruction ahead has not
2375        // committed yet. RF still performs the read because that is where the
2376        // load interlock is detected (T-11-003).
2377        //
2378        // TODO(T-11-002): the MFHI/MFLO hazard window -- a MFHI followed within
2379        // two instructions by a HI write reads hardware's WRONG value, and that
2380        // is non-interlocked (alu::MFHI_MFLO_HAZARD_INSTRUCTIONS).
2381        // The load-delay interlock (UM §4.6.5). A load's result is not ready in
2382        // time to bypass, so if the NEXT instruction names the loaded register
2383        // the pipeline stalls one cycle. The detection is deliberately imprecise,
2384        // matching hardware -- see `load_interlocks`.
2385        //
2386        // Compare against `ex_dc`, not `rf_ex`. In the reverse cascade `EX` runs
2387        // before `RF`, so by now the instruction that was in `EX` this cycle has
2388        // already moved into `ex_dc` and `rf_ex` has been vacated. Checking
2389        // `rf_ex` here silently never fires -- which is exactly what it did
2390        // before `a_load_followed_by_its_use_interlocks...` caught it.
2391        if out.occupied
2392            && self.ex_dc.occupied
2393            && self.ex_dc.decoded.is_load()
2394            && load_interlocks(
2395                self.ex_dc.decoded.dest,
2396                out.decoded.rs,
2397                out.decoded.rt,
2398                self.ex_dc.decoded.targets_fpr() == out.decoded.targets_fpr(),
2399            )
2400        {
2401            self.stall_for(1, Interlock::Ldi);
2402        }
2403        self.rf_ex = out;
2404        self.ic_rf.occupied = false;
2405    }
2406
2407    /// The FP condition an in-flight `C.cond.fmt` is about to commit, if any.
2408    ///
2409    /// `BC1` resolves in `EX`; `C.cond.fmt` computes *and* commits `FCSR.C` in
2410    /// `WB` (ADR 0007's single commit-or-trap point). An adjacent pair therefore
2411    /// has the branch sampling the previous condition, and the ROM emits exactly
2412    /// that pair with no separating instruction.
2413    ///
2414    /// # Why this is a bypass and not a stall
2415    ///
2416    /// Stalling cannot work here. `stall_for` freezes the whole pipeline, so
2417    /// holding the branch delays the compare's `WB` by the same amount and the
2418    /// branch never catches up — an interlock on `ex_dc`/`dc_wb` was written,
2419    /// fired once, and changed nothing. This is the same shape as the **load**
2420    /// interlock, which works only because its consumer reads through the
2421    /// bypass network; the one-cycle stall buys `DC` time, and forwarding does
2422    /// the rest. The FP condition had no such path, so this is it.
2423    ///
2424    /// # Why recomputing is sound
2425    ///
2426    /// A compare reads two FP registers and writes only `FCSR.C`. Nothing
2427    /// between it and the branch can change those registers — the branch has no
2428    /// destination — so evaluating it early yields the value `WB` will commit.
2429    /// The flags are deliberately discarded: this is a forwarding path, and
2430    /// raising an exception from it would make the branch report the *compare's*
2431    /// trap.
2432    ///
2433    /// `ex_dc` is checked before `dc_wb` because it holds the **younger**
2434    /// instruction, and the most recent compare is the one whose value stands.
2435    fn pending_fp_condition(&self) -> Option<bool> {
2436        let fr = fr_of(&self.cop0);
2437        for latch in [&self.ex_dc, &self.dc_wb] {
2438            if !latch.occupied {
2439                continue;
2440            }
2441            if let Some(Cop0Access::Cop1(Cop1Access::Arith {
2442                fmt, funct, ft, fs, ..
2443            })) = latch.cop0
2444                && funct >= 0o60
2445            {
2446                if let (FpCommit::Condition(c), _, false) =
2447                    self.fp_compare(fmt, funct & 0xF, ft, fs, fr)
2448                {
2449                    return Some(c);
2450                }
2451                return None;
2452            }
2453        }
2454        None
2455    }
2456
2457    /// Fetch the instruction word at `pc`: alignment, the segment map, the
2458    /// micro-ITLB and JTLB, `Status.RE`, and the I-cache.
2459    ///
2460    /// **Latch-independent on purpose.** Everything here is a function of `pc`
2461    /// and the CPU's own state — it stamps no latch, raises no abort, and names
2462    /// no [`Stage`] — so the caller decides what a failure means. `ic_stage`
2463    /// turns an `Err` into an `Ic`-stage abort; the instruction-granular path
2464    /// (ADR 0013) needs the same fetch with different consequences, and
2465    /// duplicating 100 lines of segment/TLB/cache logic to get it is how the two
2466    /// paths would drift apart in exactly the place a test would not notice.
2467    ///
2468    /// The side effects it *does* keep are the ones that belong to fetching
2469    /// rather than to sequencing: an ITLB reload and its 3-`PCycle` stall, and an
2470    /// I-cache fill. Those happen on hardware whatever the caller does next.
2471    ///
2472    /// # Errors
2473    ///
2474    /// - [`Exception::AddressError`] — `pc` is not word-aligned, or the segment
2475    ///   is not valid in the current mode. The bus is **not** touched in either
2476    ///   case: the access itself is what is invalid.
2477    /// - A TLB exception from [`Self::tlb_exception`] on a JTLB miss or an
2478    ///   invalid entry. An instruction fetch is a load, so `TLBL`, never `TLBS`.
2479    fn fetch_word<B: Bus>(&mut self, bus: &mut B, pc: u64) -> Result<u32, Exception> {
2480        // An instruction fetch must be word-aligned. An unaligned PC raises an
2481        // address error (AdEL) rather than fetching.
2482        //
2483        // Not reachable from straight-line execution, which advances by 4 from an
2484        // aligned reset vector. It becomes reachable with the jump and branch
2485        // family (T-11-004), where a computed target can be unaligned, and it is
2486        // already reachable through the public `Cpu::set_pc` the golden-log
2487        // harness uses.
2488        if !pc.is_multiple_of(4) {
2489            return Err(Exception::AddressError { store: false });
2490        }
2491
2492        // The fetch itself goes through the I-cache (see below); what is still
2493        // outstanding is the COST.
2494        // TODO(T-11-003): charge the I-cache miss cost (14..=15 + M PCycles, UM
2495        // Table 11-2) once `M` is measured -- accuracy-ledger C-1.
2496        // Every address handed to the Bus is PHYSICAL (`docs/cpu.md`); the
2497        // segment map is applied here, in the CPU, not by the Bus.
2498        // Instruction fetch goes through the micro-ITLB in front of the JTLB
2499        // (UM §1.5.1). A micro-TLB miss is a STALL of 3 PCycles (UM §4.6.2); a
2500        // JTLB miss is an exception. Only the mapped segments involve either --
2501        // KSEG0/KSEG1 fetches, which is all of early boot, bypass both.
2502        let asid = (self.cop0.read(crate::cop0::reg::ENTRY_HI) & 0xFF) as u8;
2503        let fetch_access = self.access_mode();
2504        let (phys, cached) = match crate::addr::segment(pc, fetch_access) {
2505            crate::addr::Segment::Direct { addr, cached } => (addr, cached),
2506            // Not a valid address in this mode: an address error, raised without
2507            // consulting the TLB at all.
2508            crate::addr::Segment::Invalid => {
2509                return Err(Exception::AddressError { store: false });
2510            }
2511            crate::addr::Segment::Mapped => {
2512                // The 3-PCycle penalty is "incurred when the micro-TLB is
2513                // updated from the JTLB" (UM §4.6.2) -- so it is charged only
2514                // when a reload can actually happen. A fetch that misses BOTH
2515                // levels goes straight to its exception without paying for a
2516                // reload that never occurred.
2517                if !self.tlb.itlb_probe(pc, asid) && self.tlb.jtlb_has_match(pc, asid) {
2518                    self.tlb.itlb_fill(pc, asid);
2519                    self.stall_for(crate::tlb::ITLB_MISS_PCYCLES, Interlock::Itm);
2520                }
2521                match self.tlb.lookup(pc, asid, false) {
2522                    Ok(t) => (
2523                        t.addr,
2524                        if t.uncached {
2525                            crate::addr::Cached::No
2526                        } else {
2527                            crate::addr::Cached::Yes
2528                        },
2529                    ),
2530                    // An instruction fetch is a load, so TLBL never TLBS.
2531                    Err(f) => return Err(Self::tlb_exception(f, false, fetch_access.wide)),
2532                }
2533            }
2534        };
2535        // Instruction fetch is a 4-byte access, so `Status.RE` swaps it within
2536        // its doubleword exactly as it swaps a `LW` -- which is why the test ROM
2537        // has to emit its reverse-endian programs with each instruction PAIR
2538        // exchanged for them to execute in order.
2539        let phys = if self.reverse_endian() {
2540            phys ^ Self::re_swap(4)
2541        } else {
2542            phys
2543        };
2544        // Fetch through the I-cache when the segment is cached. This is what
2545        // makes an uncached patch to already-fetched code invisible until a
2546        // CACHE invalidate -- the behavior n64-systemtest's ICACHE group
2547        // asserts, and the reason the cache is modeled at all.
2548        Ok(if cached == crate::addr::Cached::Yes {
2549            if !self.icache.hits(phys) {
2550                self.icache_fill(bus, phys);
2551            }
2552            self.icache.read_word(phys)
2553        } else {
2554            bus.read_u32(phys)
2555        })
2556    }
2557
2558    /// `IC` — instruction-cache fetch, and where the delay-slot flag is set.
2559    fn ic_stage<B: Bus>(&mut self, bus: &mut B, next_pc: &mut u64) {
2560        // An abort raised earlier this cycle flushes younger instructions. The
2561        // fetch happening now is younger than all of them, so it must not become
2562        // a live instruction -- otherwise it escapes the flush entirely and
2563        // executes down the wrong path.
2564        //
2565        // TODO(T-11-002): redirect `next_pc` to the exception vector instead of
2566        // bubbling. Until the vector exists, a bubble is the honest behavior:
2567        // it declines to execute rather than executing the wrong thing.
2568        if self.flush_pending {
2569            self.ic_rf = Latch::default();
2570            return;
2571        }
2572        let pc = *next_pc;
2573
2574        // Computed BEFORE the alignment check, so a faulting fetch carries the
2575        // right delay-slot flag into its latch and therefore into `Cause.BD`.
2576        // Depends only on `rf_ex`, never on the fetch, so hoisting it is safe.
2577        //
2578        // Check `rf_ex`, not `ic_rf`: `rf_stage` runs immediately before this in
2579        // the reverse cascade and has already moved the previous instruction out
2580        // of `ic_rf`, so a branch fetched last cycle is in `rf_ex` by now.
2581        // Reading `ic_rf` here makes the flag silently always false.
2582        let in_delay_slot = self.rf_ex.occupied && self.rf_ex.decoded.op.has_delay_slot();
2583
2584        let word = match self.fetch_word(bus, pc) {
2585            Ok(word) => word,
2586            Err(exc) => {
2587                // Populate the latch BEFORE raising. `abort_with` captures the
2588                // faulting instruction's context out of the latch its stage reads,
2589                // which for `Stage::Ic` is `ic_rf` -- so raising first would capture
2590                // the PREVIOUS fetch's `pc` and delay-slot flag and write a wrong
2591                // `EPC`. Stamp before you move, and populate before you stamp.
2592                self.ic_rf = Latch {
2593                    cop0: None,
2594                    occupied: true,
2595                    pc,
2596                    in_delay_slot,
2597                    abort: Some(exc),
2598                    ..Latch::default()
2599                };
2600                self.abort_with(Stage::Ic, exc, pc);
2601                // `next_pc` is deliberately NOT realigned on an alignment fault.
2602                // Rounding it down would silently "fix" the faulting address and
2603                // let execution continue on a path hardware never takes -- turning
2604                // a raised exception into a wrong answer. The redirect to the
2605                // exception vector happens in `advance`, after the cascade
2606                // (T-12-002).
2607                return;
2608            }
2609        };
2610        // Decode here rather than at RF: a branch must be decoded before the
2611        // NEXT fetch, so that fetch can be marked as its delay slot.
2612        //
2613        // A branch decoded last cycle is in `rf_ex` by now, so the instruction
2614        // being fetched here is its delay slot -- see `in_delay_slot` above,
2615        // computed once at the top of the stage and used by both paths.
2616        self.ic_rf = Latch {
2617            occupied: true,
2618            pc,
2619            word,
2620            in_delay_slot,
2621            abort: None,
2622            decoded: decode(word),
2623            rs_val: 0,
2624            rt_val: 0,
2625            write_back: WriteBack::None,
2626            mem: None,
2627            cop0: None,
2628        };
2629        *next_pc = pc.wrapping_add(4);
2630    }
2631
2632    /// Perform a COP1 arithmetic operation against the FPR file.
2633    ///
2634    /// Lives here rather than in `exec::execute` because it reads two FPRs and
2635    /// writes a third, and `execute` has no access to the register file — the
2636    /// same reason the COP1 moves are split this way.
2637    ///
2638    /// # Register access goes through the `FR` view
2639    ///
2640    /// Operands are read with [`Fpr::read_s`]/[`Fpr::read_d`], **not**
2641    /// `read_raw`. Using the raw register was ledger U-7's bug: with `FR = 0` a
2642    /// double lives across an FGR *pair*, and a raw read returns half of it.
2643    ///
2644    /// # `FCSR`
2645    ///
2646    /// `Cause` is bits **17:12** and reports what *this* operation raised; it
2647    /// is replaced wholesale each time. `Flags` (6:2) is the sticky
2648    /// accumulation and is OR-ed in. `Flags::to_fcsr_bits` produces both, so
2649    /// clearing only `Cause` before OR-ing preserves the sticky half.
2650    ///
2651    /// **The field is 17:12, not 16:12.** Bit 17 is `Cause.E`, Unimplemented
2652    /// Operation — part of `Cause` despite having no `Enable` bit and no sticky
2653    /// `Flags` twin, which means the mask is the *only* thing that ever clears
2654    /// it. This comment said 16:12 while the mask covered 16:12 too, and the
2655    /// result was a bit that could never be cleared once raised. Only the five
2656    /// *maskable* conditions live in 16:12; that narrower range is what the
2657    /// enable comparison below uses, and it is a different statement.
2658    ///
2659    /// The mask is now [`fpu::CAUSE_MASK`](crate::fpu::CAUSE_MASK), defined once.
2660    /// It had reached **four** local copies of the same literal — in a file where
2661    /// getting this exact constant wrong has already shipped a bug.
2662    ///
2663    /// # Enabled traps
2664    ///
2665    /// A condition whose `FCSR.Enable` bit is set raises
2666    /// [`Exception::FloatingPoint`] instead of completing, and **three** things
2667    /// then differ from the untrapped path. All three are architectural, and
2668    /// each is separately observable by n64-systemtest:
2669    ///
2670    /// 1. **`fd` is not written.** The trap is precise, so the destination keeps
2671    ///    its old value — which is what the suite checks with its
2672    ///    `Result after operation (with exception)` assertion.
2673    /// 2. **The sticky `Flags` field is not updated.** Only `Cause` is. This is
2674    ///    easy to get wrong because the untrapped path sets both from the same
2675    ///    helper, and a trapped operation that also OR-ed into `Flags` looks
2676    ///    right in every test that does not read `FCSR` back.
2677    /// 3. **The instruction does not retire**, so it must not tick `Random`.
2678    ///
2679    /// Returns `true` when it trapped, so `wb_stage` can skip its retirement
2680    /// tail.
2681    ///
2682    /// # Still not handled
2683    ///
2684    /// The **unimplemented-operation** cause (bit 17) is unmaskable and is not
2685    /// produced by the arithmetic here — the VR4300 raises it for subnormal
2686    /// operands and results, which this FPU computes normally instead. That is
2687    /// a separate body of work from the maskable enables, and the suite's
2688    /// `expected_unimplemented` cases still fail.
2689    fn fp_arith(&mut self, fmt: u8, funct: u8, ft: u8, fs: u8, fd: u8) -> bool {
2690        use crate::fpu;
2691        let fr = fr_of(&self.cop0);
2692        // `fmt` is 16 (single) or 17 (double) -- decode admits no other value
2693        // into `FpArith`, so this is a two-way split, not a table.
2694        // `funct` 5/6/7 read only `fs`, so they are handled ahead of the
2695        // arithmetic split and `ft` is never read for an instruction whose
2696        // `ft` field is architecturally zero.
2697        //
2698        // **`MOV` (funct 6) alone is the pure bit move.** `ABS` (5) and `NEG`
2699        // (7) look like sign flips and are not: they classify their operand,
2700        // raising Invalid on a signaling NaN and unimplemented-operation on a
2701        // subnormal or an MSB-clear NaN, and they REPLACE the `Cause` field.
2702        // An earlier version of this comment described all three as raising
2703        // nothing, which was true of `MOV` and never of its neighbors.
2704        //
2705        // n64-systemtest settles which is which by construction rather than by
2706        // description: `MOV.S` is driven through
2707        // `test_floating_point_f32_which_preserves_cause_bits`, while `ABS.S`
2708        // and `NEG.S` go through the ordinary `test_floating_point_f32`, which
2709        // asserts `Cause` was cleared. Treating all three alike was worth 52
2710        // assertions.
2711        if matches!(funct, 5 | 7) {
2712            return self.fp_sign_op(fmt, funct, fs, fd, fr);
2713        }
2714        if funct == 6 {
2715            // **`MOV.S` moves all 64 bits, not just the formatted half.**
2716            // n64-systemtest's "Upper bits of 32 bit operation" reads the
2717            // destination back with `DMFC1` after a `MOV.S` and expects the
2718            // SOURCE's upper half there, not the destination's previous
2719            // contents -- so this is a whole-register transfer that happens to
2720            // be spelled `.S`.
2721            let v = self.fpr.read_d_fs(fs, fr);
2722            self.fpr.write_d_arith(fd, fr, v);
2723            // **`FCSR` is left completely alone**, `Cause` included.
2724            //
2725            // Clearing `Cause` here was written first, on no evidence, and was
2726            // measurably wrong: the compiler emits `MOV.fmt` to move an FP
2727            // return value, so a `MOV` sitting between an arithmetic operation
2728            // and the `CFC1` that reads its result wiped the very `Cause` bits
2729            // the program was about to inspect. n64-systemtest saw
2730            // `flags: inexact` with `causes: ""` — the sticky half surviving
2731            // and the per-operation half erased — which is the signature of a
2732            // later instruction overwriting it, not of a flag never set.
2733            //
2734            // The architectural rule is that `Cause` is written by operations
2735            // that *can* raise. These cannot, so they write nothing.
2736            return false;
2737        }
2738
2739        // `FCSR.RM` is read **here**, per operation, rather than being captured
2740        // anywhere earlier: software changes it between instructions, and
2741        // n64-systemtest sweeps all four modes over the same operand pair.
2742        let mode = fpu::Rounding::from_rm(self.cop1.rounding_mode());
2743
2744        // Computed but **not committed**. Whether the write happens depends on
2745        // the enables, and they cannot be consulted until the flags are known.
2746        // Writing inside a branch and undoing it afterwards would be wrong
2747        // under `FR = 0`, where a `.S` write can disturb a neighboring
2748        // register's half.
2749        let (commit, flags, unimplemented) = match funct {
2750            0o00..=0o03 => self.fp_binary(fmt, funct, ft, fs, fr, mode),
2751            0o04 => self.fp_sqrt(fmt, fs, fr, mode),
2752            0o10..=0o17 => self.fp_to_integer(fmt, funct, fs, fr),
2753            0o40 | 0o41 | 0o44 | 0o45 => self.fp_convert(fmt, funct, fs, fr, mode),
2754            // 0o60..=0o77 -- `C.cond.fmt`. The low four bits ARE the condition
2755            // (UM Table 7-11), so the sixteen mnemonics need no table.
2756            _ => self.fp_compare(fmt, funct & 0xF, ft, fs, fr),
2757        };
2758
2759        let raised = flags.to_fcsr_bits()
2760            | if unimplemented {
2761                fpu::CAUSE_UNIMPLEMENTED
2762            } else {
2763                0
2764            };
2765        let fcsr = self.cop1.fcsr();
2766
2767        // `Cause` bits 16:12 and the `Enable` field bits 11:7 hold the five
2768        // conditions in the SAME order, so shifting `Cause` down by 12 lines it
2769        // up with what `Cop1Control::enables` returns. Comparing them in
2770        // different orders is a silent mis-map that only shows up on whichever
2771        // condition happens to be tested first.
2772        //
2773        // **Unimplemented Operation (bit 17) is unmaskable** and sits above
2774        // that field, so it is checked separately rather than being folded into
2775        // the enable comparison — where it would have been silently ignored,
2776        // since no enable bit corresponds to it.
2777        if unimplemented || (raised >> 12) & self.cop1.enables() != 0 {
2778            // Cause only. The sticky `Flags` field is deliberately left
2779            // untouched — see the doc comment.
2780            self.cop1
2781                .ctc1(31, (fcsr & !fpu::CAUSE_MASK) | (raised & fpu::CAUSE_MASK));
2782            self.abort_from(Stage::Wb, Exception::FloatingPoint);
2783            return true;
2784        }
2785
2786        match commit {
2787            // CLEARS the upper half -- `write_s_arith`, not `write_s`. An
2788            // arithmetic `.S` result is not an `MTC1`: the suite reads the
2789            // destination back with `DMFC1` and expects zero above (C-10).
2790            // Going through the `Fpr` accessor rather than `write_raw` is what
2791            // keeps the `FR` view applied, which is the part ledger U-7 records.
2792            FpCommit::Single(v) => self.fpr.write_s_arith(fd, fr, v),
2793            FpCommit::Double(v) => self.fpr.write_d_arith(fd, fr, v),
2794            // `FCSR.C` is bit 23, and it is NOT part of the `Cause`/`Flags`
2795            // bookkeeping — a compare writes it and no other operation touches
2796            // it. Confirmed against n64-systemtest's own `FCSR` bitfield rather
2797            // than inferred.
2798            FpCommit::Condition(c) => {
2799                let base = (fcsr & !fpu::CAUSE_MASK & !FCSR_C) | raised;
2800                self.cop1.ctc1(31, base | if c { FCSR_C } else { 0 });
2801                return false;
2802            }
2803        }
2804        self.cop1.ctc1(31, (fcsr & !fpu::CAUSE_MASK) | raised);
2805        false
2806    }
2807
2808    /// `ABS` and `NEG` — sign manipulation, but **not** a pure bit flip.
2809    ///
2810    /// The VR4300 classifies the operand first: a subnormal or an MSB-clear NaN
2811    /// raises unimplemented-operation, and an MSB-set (signaling, ledger C-12)
2812    /// NaN raises Invalid and yields the default NaN rather than the operand
2813    /// with its sign changed. Only when the operand is ordinary does the sign
2814    /// bit move.
2815    ///
2816    /// Unlike `MOV`, these REPLACE the `Cause` field — clearing it on success.
2817    fn fp_sign_op(&mut self, fmt: u8, funct: u8, fs: u8, fd: u8, fr: bool) -> bool {
2818        use crate::fpu;
2819        let fcsr = self.cop1.fcsr();
2820        let (commit, flags, unimplemented) = if fmt == 0o20 {
2821            let a = f32::from_bits(self.fpr.read_s_fs(fs, fr));
2822            if fpu::is_subnormal_f32(a) || fpu::is_unimplemented_nan_f32(a) {
2823                (0u64, fpu::Flags::NONE, true)
2824            } else if fpu::is_snan_f32(a) {
2825                (
2826                    u64::from(crate::softfloat::F32.default_nan() as u32),
2827                    fpu::Flags::INVALID,
2828                    false,
2829                )
2830            } else {
2831                let v = if funct == 5 {
2832                    fpu::abs_s(a)
2833                } else {
2834                    fpu::neg_s(a)
2835                };
2836                (u64::from(v.to_bits()), fpu::Flags::NONE, false)
2837            }
2838        } else {
2839            let a = f64::from_bits(self.fpr.read_d_fs(fs, fr));
2840            if fpu::is_subnormal_f64(a) || fpu::is_unimplemented_nan_f64(a) {
2841                (0u64, fpu::Flags::NONE, true)
2842            } else if fpu::is_snan_f64(a) {
2843                (
2844                    crate::softfloat::F64.default_nan(),
2845                    fpu::Flags::INVALID,
2846                    false,
2847                )
2848            } else {
2849                let v = if funct == 5 {
2850                    fpu::abs_d(a)
2851                } else {
2852                    fpu::neg_d(a)
2853                };
2854                (v.to_bits(), fpu::Flags::NONE, false)
2855            }
2856        };
2857
2858        let raised = flags.to_fcsr_bits()
2859            | if unimplemented {
2860                fpu::CAUSE_UNIMPLEMENTED
2861            } else {
2862                0
2863            };
2864        if unimplemented {
2865            self.cop1
2866                .ctc1(31, (fcsr & !fpu::CAUSE_MASK) | (raised & fpu::CAUSE_MASK));
2867            self.abort_from(Stage::Wb, Exception::FloatingPoint);
2868            return true;
2869        }
2870        if flags.invalid && self.cop1.enables() & (1 << 4) != 0 {
2871            self.cop1
2872                .ctc1(31, (fcsr & !fpu::CAUSE_MASK) | (raised & fpu::CAUSE_MASK));
2873            self.abort_from(Stage::Wb, Exception::FloatingPoint);
2874            return true;
2875        }
2876        if fmt == 0o20 {
2877            self.fpr.write_s_arith(fd, fr, commit as u32);
2878        } else {
2879            self.fpr.write_d_arith(fd, fr, commit);
2880        }
2881        self.cop1.ctc1(31, (fcsr & !fpu::CAUSE_MASK) | raised);
2882        false
2883    }
2884
2885    /// `SQRT.fmt`.
2886    ///
2887    /// Unary, so it takes the operand policy but not the *result* policy: the
2888    /// square root of the smallest normal is about `1e-19`, nowhere near the
2889    /// subnormal range, so a normal operand cannot produce a subnormal result.
2890    ///
2891    /// A negative operand is Invalid rather than unimplemented — except `-0`,
2892    /// whose root is `-0` and raises nothing. [`softfloat::sqrt`] draws that
2893    /// distinction.
2894    fn fp_sqrt(
2895        &self,
2896        fmt: u8,
2897        fs: u8,
2898        fr: bool,
2899        mode: crate::fpu::Rounding,
2900    ) -> (FpCommit, crate::fpu::Flags, bool) {
2901        use crate::fpu;
2902        if fmt == 0o20 {
2903            let bits = u64::from(self.fpr.read_s_fs(fs, fr));
2904            let a = f32::from_bits(bits as u32);
2905            if fpu::is_subnormal_f32(a) || fpu::is_unimplemented_nan_f32(a) {
2906                return (FpCommit::Single(0), fpu::Flags::NONE, true);
2907            }
2908            let r = softfloat::sqrt(bits, softfloat::F32, mode);
2909            (FpCommit::Single(r.bits as u32), r.flags, false)
2910        } else {
2911            let bits = self.fpr.read_d_fs(fs, fr);
2912            let a = f64::from_bits(bits);
2913            if fpu::is_subnormal_f64(a) || fpu::is_unimplemented_nan_f64(a) {
2914                return (FpCommit::Double(0), fpu::Flags::NONE, true);
2915            }
2916            let r = softfloat::sqrt(bits, softfloat::F64, mode);
2917            (FpCommit::Double(r.bits), r.flags, false)
2918        }
2919    }
2920
2921    /// The VR4300's policy for a **subnormal result**, applied wherever one can
2922    /// be produced — arithmetic and the narrowing `CVT.S.D`.
2923    ///
2924    /// Three outcomes, in order:
2925    ///
2926    /// 1. `FCSR.FS` clear — the processor cannot represent the result at all,
2927    ///    so *unimplemented operation*.
2928    /// 2. `FS` set but underflow or inexact **enabled** — it cannot deliver a
2929    ///    trapped underflow's defined result either, so unimplemented again.
2930    ///    n64-systemtest's own comment on this case reads "(wow)".
2931    /// 3. `FS` set and both disabled — flush per
2932    ///    [`fpu::flush_subnormal_f32`], reporting underflow and inexact.
2933    fn subnormal_policy_s(
2934        &self,
2935        out: crate::fpu::Outcome<f32>,
2936        mode: crate::fpu::Rounding,
2937    ) -> (FpCommit, crate::fpu::Flags, bool) {
2938        use crate::fpu;
2939        // **Either** condition triggers, and both are needed.
2940        //
2941        // `is_subnormal` alone misses a result that underflows past the
2942        // subnormal grid all the way to zero (`f64::MIN_POSITIVE` narrowed to
2943        // `f32`; `MIN_POSITIVE` squared) — the VR4300 refuses those exactly as
2944        // it refuses a subnormal.
2945        //
2946        // `flags.underflow` alone misses an *exact* subnormal, because IEEE
2947        // signals underflow only when tiny **and inexact**. Replacing the
2948        // first test with the second was tried and regressed the oracle from
2949        // 89 to 131.
2950        if !fpu::is_subnormal_f32(out.value) && !out.flags.underflow {
2951            return (FpCommit::Single(out.value.to_bits()), out.flags, false);
2952        }
2953        if !self.cop1.flush_denorm_to_zero() || self.underflow_traps() {
2954            return (FpCommit::Single(0), fpu::Flags::NONE, true);
2955        }
2956        let mut flags = out.flags;
2957        flags.underflow = true;
2958        flags.inexact = true;
2959        let v = fpu::flush_subnormal_f32(out.value, mode);
2960        (FpCommit::Single(v.to_bits()), flags, false)
2961    }
2962
2963    /// See [`Pipeline::subnormal_policy_s`].
2964    fn subnormal_policy_d(
2965        &self,
2966        out: crate::fpu::Outcome<f64>,
2967        mode: crate::fpu::Rounding,
2968    ) -> (FpCommit, crate::fpu::Flags, bool) {
2969        use crate::fpu;
2970        // **Either** condition triggers, and both are needed.
2971        //
2972        // `is_subnormal` alone misses a result that underflows past the
2973        // subnormal grid all the way to zero (`f64::MIN_POSITIVE` narrowed to
2974        // `f32`; `MIN_POSITIVE` squared) — the VR4300 refuses those exactly as
2975        // it refuses a subnormal.
2976        //
2977        // `flags.underflow` alone misses an *exact* subnormal, because IEEE
2978        // signals underflow only when tiny **and inexact**. Replacing the
2979        // first test with the second was tried and regressed the oracle from
2980        // 89 to 131.
2981        if !fpu::is_subnormal_f64(out.value) && !out.flags.underflow {
2982            return (FpCommit::Double(out.value.to_bits()), out.flags, false);
2983        }
2984        if !self.cop1.flush_denorm_to_zero() || self.underflow_traps() {
2985            return (FpCommit::Double(0), fpu::Flags::NONE, true);
2986        }
2987        let mut flags = out.flags;
2988        flags.underflow = true;
2989        flags.inexact = true;
2990        let v = fpu::flush_subnormal_f64(out.value, mode);
2991        (FpCommit::Double(v.to_bits()), flags, false)
2992    }
2993
2994    /// Does the CURRENT `FCSR` already meet a trap condition?
2995    ///
2996    /// Used after a `CTC1` writes it: the register can be put into a trapping
2997    /// state directly, without any arithmetic having run.
2998    fn fcsr_traps_now(&self) -> bool {
2999        /// `FCSR.Cause` maskable bits 16:12, shifted to line up with `Enable`.
3000        const CAUSE_SHIFT: u32 = 12;
3001        let fcsr = self.cop1.fcsr();
3002        if fcsr & crate::fpu::CAUSE_UNIMPLEMENTED != 0 {
3003            return true;
3004        }
3005        (fcsr >> CAUSE_SHIFT) & self.cop1.enables() != 0
3006    }
3007
3008    /// Is underflow or inexact enabled? Either turns a flushed subnormal into
3009    /// an unimplemented operation.
3010    fn underflow_traps(&self) -> bool {
3011        /// `FCSR.Enable` underflow (bit 8) and inexact (bit 7), as
3012        /// `Cop1Control::enables` returns them — shifted down by 7.
3013        const ENABLE_UNDERFLOW_OR_INEXACT: u32 = 0b11;
3014        self.cop1.enables() & ENABLE_UNDERFLOW_OR_INEXACT != 0
3015    }
3016
3017    /// `ADD`/`SUB`/`MUL`/`DIV` in either format.
3018    ///
3019    /// # The VR4300 cannot compute with subnormals
3020    ///
3021    /// It raises the unmaskable *unimplemented operation* cause instead, and
3022    /// there are three distinct occasions (UM §7.5; pinned by n64-systemtest):
3023    ///
3024    /// 1. **A subnormal operand** — checked before the operation is attempted,
3025    ///    and it outranks everything, including a NaN that would otherwise
3026    ///    raise Invalid.
3027    /// 2. **A subnormal result with `FCSR.FS` clear.**
3028    /// 3. **A subnormal result with `FS` set but underflow or inexact
3029    ///    *enabled*.** The processor cannot deliver a trapped underflow's
3030    ///    defined result, so it declines instead — the suite's own comment on
3031    ///    this case reads "(wow)".
3032    ///
3033    /// With `FS` set and those enables clear it flushes, per
3034    /// [`fpu::flush_subnormal_f32`], and reports underflow + inexact.
3035    ///
3036    /// The returned flags are deliberately [`fpu::Flags::NONE`] on every
3037    /// unimplemented path: `FCSR` must end up with bit 17 and *nothing else*,
3038    /// which is what the suite asserts.
3039    fn fp_binary(
3040        &self,
3041        fmt: u8,
3042        funct: u8,
3043        ft: u8,
3044        fs: u8,
3045        fr: bool,
3046        mode: crate::fpu::Rounding,
3047    ) -> (FpCommit, crate::fpu::Flags, bool) {
3048        use crate::fpu;
3049        if fmt == 0o20 {
3050            let a = f32::from_bits(self.fpr.read_s_fs(fs, fr));
3051            let b = f32::from_bits(self.fpr.read_s_ft(ft, fr));
3052            if fpu::arith_unimplemented_s(a, b) {
3053                return (FpCommit::Single(0), fpu::Flags::NONE, true);
3054            }
3055            let out = match funct {
3056                0 => fpu::add_s(a, b, mode),
3057                1 => fpu::sub_s(a, b, mode),
3058                2 => fpu::mul_s(a, b, mode),
3059                _ => fpu::div_s(a, b, mode),
3060            };
3061            self.subnormal_policy_s(out, mode)
3062        } else {
3063            let a = f64::from_bits(self.fpr.read_d_fs(fs, fr));
3064            let b = f64::from_bits(self.fpr.read_d_ft(ft, fr));
3065            if fpu::arith_unimplemented_d(a, b) {
3066                return (FpCommit::Double(0), fpu::Flags::NONE, true);
3067            }
3068            let out = match funct {
3069                0 => fpu::add_d(a, b, mode),
3070                1 => fpu::sub_d(a, b, mode),
3071                2 => fpu::mul_d(a, b, mode),
3072                _ => fpu::div_d(a, b, mode),
3073            };
3074            self.subnormal_policy_d(out, mode)
3075        }
3076    }
3077
3078    /// `ROUND`/`TRUNC`/`CEIL`/`FLOOR` to `.W` or `.L` (funct 8..=15).
3079    ///
3080    /// These carry their rounding mode **in the opcode** and ignore `FCSR.RM`
3081    /// entirely — that is the whole reason they exist alongside `CVT.W`/`CVT.L`,
3082    /// which do consult it. Passing the live `RM` here would make all four
3083    /// behave identically whenever `RM` happened to match, and the difference
3084    /// would only show up under a non-default mode.
3085    fn fp_to_integer(
3086        &self,
3087        fmt: u8,
3088        funct: u8,
3089        fs: u8,
3090        fr: bool,
3091    ) -> (FpCommit, crate::fpu::Flags, bool) {
3092        use crate::fpu::{self, Rounding};
3093        let mode = match funct & 0o3 {
3094            0 => Rounding::Nearest,
3095            1 => Rounding::TowardZero,
3096            2 => Rounding::TowardPlusInf,
3097            _ => Rounding::TowardMinusInf,
3098        };
3099        // Refusal is decided BEFORE the source is read as a float: an integer
3100        // source format has no float to widen, and reading one anyway produces a
3101        // plausible number for an instruction that does not exist.
3102        if self.integer_conversion_unimplemented(fmt, fs, fr) {
3103            return (FpCommit::Single(0), fpu::Flags::NONE, true);
3104        }
3105        // The source is widened to `f64` first, which is EXACT for an `f32`, so
3106        // no rounding happens before the one the instruction asks for.
3107        let v = self.fp_source_as_f64(fmt, fs, fr);
3108        // funct 8..=11 target `.L`, 12..=15 target `.W`.
3109        let wide = funct < 0o14;
3110        if wide {
3111            let out = fpu::to_i64(v, mode);
3112            // `to_i64` reports NaN and out-of-range as Invalid, which is the
3113            // IEEE answer and NOT this processor's: the VR4300 declines with
3114            // *unimplemented operation* instead. n64-systemtest expects `Err`
3115            // for infinities, NaNs and anything past the target's range.
3116            if out.flags.invalid {
3117                return (FpCommit::Double(0), fpu::Flags::NONE, true);
3118            }
3119            #[allow(clippy::cast_sign_loss)] // a bit pattern, not a magnitude
3120            (FpCommit::Double(out.value as u64), out.flags, false)
3121        } else {
3122            let out = fpu::to_i32(v, mode);
3123            if out.flags.invalid {
3124                return (FpCommit::Single(0), fpu::Flags::NONE, true);
3125            }
3126            #[allow(clippy::cast_sign_loss)] // a bit pattern, not a magnitude
3127            (FpCommit::Single(out.value as u32), out.flags, false)
3128        }
3129    }
3130
3131    /// Is the source of a float-to-integer conversion one the VR4300 refuses?
3132    ///
3133    /// Subnormality, plus a magnitude of `2^53` or more. NaN and infinity are
3134    /// detected from the conversion's own result, because "out of range"
3135    /// depends on the target width.
3136    ///
3137    /// The limit applies to `.W` targets as well, where it is **unobservable**:
3138    /// `2^53` is far outside `i32`, so such a value is refused either way. It
3139    /// was originally guarded on the target width, and the guard was removed
3140    /// because no test could distinguish the two — an undistinguishable branch
3141    /// is one that rots.
3142    ///
3143    /// # The `2^53` limit is narrower than `i64`
3144    ///
3145    /// `9007198717870080` converts; `9007199254740992` (`2^53`) raises
3146    /// unimplemented — both far inside `i64`'s range. n64-systemtest brackets
3147    /// the threshold with adjacent values on either side, which is what
3148    /// identifies it as `2^53` rather than some larger bound. `2^53` is the
3149    /// last integer a `double` represents exactly, so the natural reading is
3150    /// that the conversion runs through double precision internally and
3151    /// declines whatever it cannot hold exactly.
3152    fn integer_conversion_unimplemented(&self, fmt: u8, fs: u8, fr: bool) -> bool {
3153        use crate::fpu;
3154        /// The last integer a `double` represents exactly.
3155        const TWO_POW_53: f64 = 9_007_199_254_740_992.0;
3156
3157        // An INTEGER source format is not a conversion this processor has:
3158        // `CVT.W.W`, `CVT.W.L`, `CVT.L.W`, `CVT.L.L` and the whole
3159        // `ROUND`/`TRUNC`/`CEIL`/`FLOOR` family from `.W`/`.L` do not exist, and
3160        // the VR4300 declines them with Unimplemented Operation rather than
3161        // reinterpreting the source. Checked first, because every branch below
3162        // would happily read the register as a float and return a number.
3163        if fmt != 0o20 && fmt != 0o21 {
3164            return true;
3165        }
3166        let v = if fmt == 0o20 {
3167            let a = f32::from_bits(self.fpr.read_s_fs(fs, fr));
3168            if fpu::is_subnormal_f32(a) {
3169                return true;
3170            }
3171            f64::from(a)
3172        } else {
3173            let a = f64::from_bits(self.fpr.read_d_fs(fs, fr));
3174            if fpu::is_subnormal_f64(a) {
3175                return true;
3176            }
3177            a
3178        };
3179        v.abs() >= TWO_POW_53
3180    }
3181
3182    /// `CVT.S`/`CVT.D`/`CVT.W`/`CVT.L`, from any source format.
3183    fn fp_convert(
3184        &self,
3185        fmt: u8,
3186        funct: u8,
3187        fs: u8,
3188        fr: bool,
3189        mode: crate::fpu::Rounding,
3190    ) -> (FpCommit, crate::fpu::Flags, bool) {
3191        use crate::fpu;
3192        match funct {
3193            // To single.
3194            0o40 => match fmt {
3195                0o21 => {
3196                    let bits = self.fpr.read_d_fs(fs, fr);
3197                    let a = f64::from_bits(bits);
3198                    if fpu::is_subnormal_f64(a) || fpu::is_unimplemented_nan_f64(a) {
3199                        return (FpCommit::Single(0), fpu::Flags::NONE, true);
3200                    }
3201                    // Narrowing is ARITHMETIC: it rounds 53 significand bits
3202                    // into 24, so it can be inexact, can overflow, and can land
3203                    // in the subnormal range -- each depending on `FCSR.RM`.
3204                    // `v as f32` reports none of that and rounds to nearest
3205                    // only, which is where ledger C-11 found this operation
3206                    // after the arithmetic had already been fixed.
3207                    let r = softfloat::convert(bits, softfloat::F64, softfloat::F32, mode);
3208                    let out = fpu::Outcome {
3209                        value: f32::from_bits(r.bits as u32),
3210                        flags: r.flags,
3211                    };
3212                    // Narrowing can produce a subnormal from a perfectly normal
3213                    // double, so the result policy applies exactly as it does
3214                    // to the arithmetic.
3215                    self.subnormal_policy_s(out, mode)
3216                }
3217                #[allow(clippy::cast_possible_wrap)] // reinterpreting a word as signed
3218                0o24 => {
3219                    // Through `softfloat`, not a Rust `as` cast: an i32 can need
3220                    // more than 24 significand bits, so this rounds -- and must
3221                    // round the way `FCSR.RM` says.
3222                    let v = i64::from(self.fpr.read_s_fs(fs, fr) as i32);
3223                    let r = softfloat::from_int(v, softfloat::F32, mode);
3224                    (FpCommit::Single(r.bits as u32), r.flags, false)
3225                }
3226                // From `.L`, which the VR4300 restricts: bits 63:55 must be all
3227                // zeroes or all ones (UM §7.5.2). Outside that it raises
3228                // Unimplemented rather than converting, and there is no defined
3229                // result -- so the commit value is a placeholder the trap path
3230                // discards.
3231                #[allow(clippy::cast_possible_wrap)]
3232                _ => {
3233                    let v = self.fpr.read_d_fs(fs, fr) as i64;
3234                    if fpu::long_convertible(v) {
3235                        let r = softfloat::from_int(v, softfloat::F32, mode);
3236                        (FpCommit::Single(r.bits as u32), r.flags, false)
3237                    } else {
3238                        // No defined result when the restriction is violated, so
3239                        // the value is a placeholder the trap path discards.
3240                        (FpCommit::Single(0), fpu::Flags::NONE, true)
3241                    }
3242                }
3243            },
3244            // To double.
3245            0o41 => match fmt {
3246                0o20 => {
3247                    let bits = u64::from(self.fpr.read_s_fs(fs, fr));
3248                    let a = f32::from_bits(bits as u32);
3249                    if fpu::is_subnormal_f32(a) || fpu::is_unimplemented_nan_f32(a) {
3250                        return (FpCommit::Double(0), fpu::Flags::NONE, true);
3251                    }
3252                    // Widening cannot lose anything, so no result policy is
3253                    // needed — but it goes through the same path so there is
3254                    // one conversion rather than two that can disagree.
3255                    let r = softfloat::convert(bits, softfloat::F32, softfloat::F64, mode);
3256                    (FpCommit::Double(r.bits), r.flags, false)
3257                }
3258                #[allow(clippy::cast_possible_wrap)]
3259                0o24 => {
3260                    // Exact for every i32 -- 53 bits hold 32 -- but routed the
3261                    // same way as the others so there is one conversion rather
3262                    // than two that can disagree.
3263                    let v = i64::from(self.fpr.read_s_fs(fs, fr) as i32);
3264                    let r = softfloat::from_int(v, softfloat::F64, mode);
3265                    (FpCommit::Double(r.bits), r.flags, false)
3266                }
3267                #[allow(clippy::cast_possible_wrap)]
3268                _ => {
3269                    let v = self.fpr.read_d_fs(fs, fr) as i64;
3270                    if fpu::long_convertible(v) {
3271                        let r = softfloat::from_int(v, softfloat::F64, mode);
3272                        (FpCommit::Double(r.bits), r.flags, false)
3273                    } else {
3274                        // No defined result when the restriction is violated, so
3275                        // the value is a placeholder the trap path discards.
3276                        (FpCommit::Double(0), fpu::Flags::NONE, true)
3277                    }
3278                }
3279            },
3280            // To word / to long, both honoring `FCSR.RM` -- which is what
3281            // separates them from the fixed-mode family above.
3282            0o44 => {
3283                if self.integer_conversion_unimplemented(fmt, fs, fr) {
3284                    return (FpCommit::Single(0), fpu::Flags::NONE, true);
3285                }
3286                let out = fpu::to_i32(self.fp_source_as_f64(fmt, fs, fr), mode);
3287                if out.flags.invalid {
3288                    return (FpCommit::Single(0), fpu::Flags::NONE, true);
3289                }
3290                #[allow(clippy::cast_sign_loss)]
3291                (FpCommit::Single(out.value as u32), out.flags, false)
3292            }
3293            _ => {
3294                if self.integer_conversion_unimplemented(fmt, fs, fr) {
3295                    return (FpCommit::Double(0), fpu::Flags::NONE, true);
3296                }
3297                let out = fpu::to_i64(self.fp_source_as_f64(fmt, fs, fr), mode);
3298                if out.flags.invalid {
3299                    return (FpCommit::Double(0), fpu::Flags::NONE, true);
3300                }
3301                #[allow(clippy::cast_sign_loss)]
3302                (FpCommit::Double(out.value as u64), out.flags, false)
3303            }
3304        }
3305    }
3306
3307    /// `C.cond.fmt` — writes `FCSR.C`, never an FPR.
3308    fn fp_compare(
3309        &self,
3310        fmt: u8,
3311        cond: u8,
3312        ft: u8,
3313        fs: u8,
3314        fr: bool,
3315    ) -> (FpCommit, crate::fpu::Flags, bool) {
3316        use crate::fpu;
3317        let out = if fmt == 0o20 {
3318            fpu::compare_s(
3319                f32::from_bits(self.fpr.read_s_fs(fs, fr)),
3320                f32::from_bits(self.fpr.read_s_ft(ft, fr)),
3321                cond,
3322            )
3323        } else {
3324            fpu::compare_d(
3325                f64::from_bits(self.fpr.read_d_fs(fs, fr)),
3326                f64::from_bits(self.fpr.read_d_ft(ft, fr)),
3327                cond,
3328            )
3329        };
3330        (FpCommit::Condition(out.value), out.flags, false)
3331    }
3332
3333    /// Read `fs` in `fmt` and widen to `f64`.
3334    ///
3335    /// `f32` to `f64` is exact, so a `.S` source loses nothing on the way in and
3336    /// the only rounding is the one the instruction performs.
3337    fn fp_source_as_f64(&self, fmt: u8, fs: u8, fr: bool) -> f64 {
3338        if fmt == 0o20 {
3339            f64::from(f32::from_bits(self.fpr.read_s_fs(fs, fr)))
3340        } else {
3341            f64::from_bits(self.fpr.read_d_fs(fs, fr))
3342        }
3343    }
3344}
3345
3346#[cfg(test)]
3347mod tests {
3348    use super::*;
3349
3350    struct NullBus {
3351        irq: bool,
3352    }
3353    impl Bus for NullBus {
3354        fn read_u8(&mut self, _addr: u32) -> u8 {
3355            0
3356        }
3357        fn write_u8(&mut self, _addr: u32, _val: u8) {}
3358        fn poll_irq(&mut self) -> bool {
3359            self.irq
3360        }
3361    }
3362    fn quiet() -> NullBus {
3363        NullBus { irq: false }
3364    }
3365
3366    /// **The reverse-order invariant.** A value must advance exactly one stage
3367    /// per cycle. If the cascade is ever reordered to run forwards, a value falls
3368    /// through several stages in one cycle and the pipeline is silently too fast.
3369    #[test]
3370    fn a_value_advances_exactly_one_stage_per_cycle() {
3371        let mut p = Pipeline::new();
3372        let mut pc = 0xFFFF_FFFF_8000_0000;
3373        let mut regs = Regs::new();
3374        let mut bus = quiet();
3375
3376        // Cycle 1 fetches a sentinel into ic_rf.
3377        p.advance(&mut bus, &mut regs, &mut pc);
3378        let sentinel = p.ic_rf.pc;
3379        assert!(p.ic_rf.occupied);
3380        assert_eq!(sentinel, 0xFFFF_FFFF_8000_0000);
3381
3382        // Each subsequent cycle moves it exactly one boundary along.
3383        p.advance(&mut bus, &mut regs, &mut pc);
3384        assert_eq!(p.rf_ex.pc, sentinel, "after 1 cycle it should be at RF->EX");
3385        assert_ne!(p.ic_rf.pc, sentinel, "and no longer at IC->RF");
3386
3387        p.advance(&mut bus, &mut regs, &mut pc);
3388        assert_eq!(p.ex_dc.pc, sentinel, "after 2 cycles, EX->DC");
3389
3390        p.advance(&mut bus, &mut regs, &mut pc);
3391        assert_eq!(p.dc_wb.pc, sentinel, "after 3 cycles, DC->WB");
3392
3393        // 5 stages => at least 5 PCycles per instruction (UM §4.1).
3394        assert_eq!(p.retired, 0, "nothing may retire before WB has run on it");
3395        p.advance(&mut bus, &mut regs, &mut pc);
3396        assert_eq!(p.retired, 1, "retires at WB on the 5th cycle, not sooner");
3397    }
3398
3399    /// **R-19: a branch whose delay slot faulted must NOT take, but STILL links.**
3400    ///
3401    /// The n64-systemtest `ExecuteTLBMappedMissInDelay` case hung the whole suite:
3402    /// a JALR whose target is its own address, with a delay slot in an unmapped
3403    /// page. When the delay-slot fetch faults, the exception has already pointed
3404    /// `next_pc` at the vector; if the branch then applies its redirect it clobbers
3405    /// the vector, re-fetches itself, and loops forever. The fix suppresses the
3406    /// redirect when the delay slot (in `ic_rf`) carries an abort, while still
3407    /// writing the link from `pc + 8`.
3408    ///
3409    /// Mutation guard: the control case (no delay-slot fault) asserts the redirect
3410    /// *is* applied and the link is the live `next_pc`, so removing the fault check
3411    /// turns exactly one of the two assertions red.
3412    #[test]
3413    fn a_branch_with_a_faulting_delay_slot_links_but_does_not_take() {
3414        // JALR $6, $3  (funct 9) -- the exact instruction from the hung test.
3415        let jalr = decode(0x0060_3009);
3416        assert!(jalr.op.has_delay_slot(), "JALR must have a delay slot");
3417        let redirect = Redirect {
3418            target: 0x1234_4FFC, // the JALR jumps to its own address
3419            nullify_delay_slot: false,
3420        };
3421        let branch_pc = 0x1234_4FFC;
3422        let vector = 0xFFFF_FFFF_8000_0180; // where the exception already sent us
3423
3424        // Faulting case: the delay slot in `ic_rf` carries an abort.
3425        {
3426            let mut p = Pipeline::new();
3427            p.ic_rf = Latch {
3428                occupied: true,
3429                in_delay_slot: true,
3430                abort: Some(Exception::AddressError { store: false }),
3431                ..Latch::default()
3432            };
3433            let mut out = Latch {
3434                occupied: true,
3435                pc: branch_pc,
3436                decoded: jalr,
3437                ..Latch::default()
3438            };
3439            let mut next_pc = vector;
3440            p.resolve_branch_control(&mut out, Some(6), Some(redirect), &mut next_pc);
3441            assert_eq!(
3442                next_pc, vector,
3443                "the exception vector must survive -- no take"
3444            );
3445            assert_eq!(
3446                out.write_back,
3447                WriteBack::Gpr {
3448                    dest: 6,
3449                    value: branch_pc + 8,
3450                },
3451                "the link is written from pc + 8, not the clobbered next_pc",
3452            );
3453        }
3454
3455        // Control case: the delay slot did NOT fault -> the branch takes normally
3456        // and links from the live next_pc.
3457        {
3458            let mut p = Pipeline::new();
3459            p.ic_rf = Latch {
3460                occupied: true,
3461                in_delay_slot: true,
3462                abort: None,
3463                ..Latch::default()
3464            };
3465            let mut out = Latch {
3466                occupied: true,
3467                pc: branch_pc,
3468                decoded: jalr,
3469                ..Latch::default()
3470            };
3471            let mut next_pc = branch_pc + 8;
3472            p.resolve_branch_control(&mut out, Some(6), Some(redirect), &mut next_pc);
3473            assert_eq!(
3474                next_pc, redirect.target,
3475                "an unfaulted branch takes its target"
3476            );
3477            assert_eq!(
3478                out.write_back,
3479                WriteBack::Gpr {
3480                    dest: 6,
3481                    value: branch_pc + 8,
3482                },
3483                "the link is the live next_pc (which was pc + 8 here)",
3484            );
3485        }
3486    }
3487
3488    /// **`Cause.BD` and `EPC` still come out right when a stall separates the
3489    /// branch from its delay slot.**
3490    ///
3491    /// The companion test below pins that the flag stays *attached* to its
3492    /// instruction across a stall. That is not the same claim: the flag could
3493    /// travel correctly and still be read from the wrong place at dispatch,
3494    /// which is the reverse-cascade hazard this project has hit twice. So this
3495    /// one drives the flagged instruction into a real exception and reads the
3496    /// architectural result — `EPC` must name the **branch**, four bytes back,
3497    /// not the faulting instruction.
3498    #[test]
3499    fn a_delay_slot_exception_after_a_stall_still_reports_the_branch() {
3500        use crate::cop0::reg;
3501        let mut p = Pipeline::new();
3502        let mut regs = Regs::new();
3503        // An unaligned `LW` — offset 0x101 — so the delay-slot instruction
3504        // faults with AdEL rather than needing a crafted overflow.
3505        let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1), ld_st(0o43, 0, 3, 0x101)]);
3506        p.cop0.set_hardware(reg::STATUS, 0);
3507        let mut pc = KSEG0_PROG;
3508
3509        // Advance until the FAULTING instruction (the second word) is the one
3510        // in `IC/RF`, then mark it as the delay slot. Flagging whatever happens
3511        // to be there after one step marks the first instruction instead, which
3512        // never faults -- and the test would then be asserting BD for an
3513        // exception that never came from a delay slot at all.
3514        let slot_pc = KSEG0_PROG.wrapping_add(4);
3515        let mut warm = 0;
3516        while p.ic_rf.pc != slot_pc || !p.ic_rf.occupied {
3517            p.advance(&mut bus, &mut regs, &mut pc);
3518            warm += 1;
3519            assert!(warm < 16, "the faulting instruction never reached IC/RF");
3520        }
3521        p.ic_rf.in_delay_slot = true;
3522
3523        // A `DDIV`-length interlock lands between the branch and its slot.
3524        p.stall_for(69, Interlock::Mci);
3525        let mut cycles = 0;
3526        while p.stalled_by() != Some(Interlock::Exception) {
3527            p.advance(&mut bus, &mut regs, &mut pc);
3528            cycles += 1;
3529            assert!(cycles < 128, "the delay-slot instruction never faulted");
3530        }
3531
3532        assert_ne!(
3533            p.cop0.read(reg::CAUSE) & (1 << 31),
3534            0,
3535            "BD must be set -- the faulting instruction was in a delay slot"
3536        );
3537        assert_eq!(
3538            p.cop0.read(reg::EPC),
3539            slot_pc.wrapping_sub(4),
3540            "EPC must name the branch, so the handler re-evaluates it"
3541        );
3542    }
3543
3544    /// **The delay-slot invariant** — the Phase 1 exit criterion.
3545    ///
3546    /// A multi-cycle stall between a branch and its delay slot must not
3547    /// desynchronize the flag. A global `in_delay_slot` bool passes the naive
3548    /// test and fails this one, which is why the flag rides in the latch.
3549    #[test]
3550    fn delay_slot_flag_survives_a_multi_cycle_stall() {
3551        let mut p = Pipeline::new();
3552        let mut pc = 0xFFFF_FFFF_8000_0000;
3553        let mut regs = Regs::new();
3554        let mut bus = quiet();
3555
3556        // Two instructions in flight; mark the younger as the delay slot.
3557        p.advance(&mut bus, &mut regs, &mut pc);
3558        p.ic_rf.in_delay_slot = true;
3559        let slot_pc = p.ic_rf.pc;
3560
3561        // A long interlock lands between them (e.g. DDIV = 69 PCycles).
3562        p.stall_for(69, Interlock::Mci);
3563        for _ in 0..69 {
3564            p.advance(&mut bus, &mut regs, &mut pc);
3565        }
3566        assert!(p.stalled_by().is_none(), "the stall should have expired");
3567
3568        // The flag must still be attached to the SAME instruction.
3569        assert!(
3570            p.ic_rf.in_delay_slot && p.ic_rf.pc == slot_pc,
3571            "the delay-slot flag detached from its instruction across the stall"
3572        );
3573
3574        // And it must travel with it, not stay behind at a fixed boundary.
3575        p.advance(&mut bus, &mut regs, &mut pc);
3576        assert!(
3577            p.rf_ex.in_delay_slot && p.rf_ex.pc == slot_pc,
3578            "the flag failed to travel with the instruction"
3579        );
3580        assert!(
3581            !p.ic_rf.in_delay_slot,
3582            "the flag was left behind on the boundary instead of moving"
3583        );
3584    }
3585
3586    /// A stall holds every latch in place — nothing advances, nothing retires.
3587    #[test]
3588    fn a_stall_freezes_the_pipeline() {
3589        let mut p = Pipeline::new();
3590        let mut pc = 0xFFFF_FFFF_8000_0000;
3591        let mut regs = Regs::new();
3592        let mut bus = quiet();
3593        for _ in 0..4 {
3594            p.advance(&mut bus, &mut regs, &mut pc);
3595        }
3596        let before = (p.ic_rf, p.rf_ex, p.ex_dc, p.dc_wb, p.retired);
3597
3598        p.stall_for(3, Interlock::Dcm);
3599        for _ in 0..3 {
3600            p.advance(&mut bus, &mut regs, &mut pc);
3601            assert_eq!(
3602                (p.ic_rf, p.rf_ex, p.ex_dc, p.dc_wb, p.retired),
3603                before,
3604                "a stalled cycle must not advance any latch"
3605            );
3606        }
3607        assert!(p.stalled_by().is_none());
3608        p.advance(&mut bus, &mut regs, &mut pc);
3609        assert_ne!(
3610            p.dc_wb, before.3,
3611            "the pipeline must resume after the stall"
3612        );
3613    }
3614
3615    /// **The interrupt gate** (UM §4.7.1): an interrupt is accepted only if the
3616    /// *previous* `PCycle` was a run cycle. The cycle right after a stall is not.
3617    #[test]
3618    fn interrupt_is_not_accepted_on_the_cycle_after_a_stall() {
3619        /// `IE` set, `IM2` (the RCP line) unmasked, `EXL`/`ERL` clear.
3620        ///
3621        /// Needed since T-12-003: an asserted line is no longer sufficient on
3622        /// its own, and cold reset leaves `ERL` **set**, which alone blocks
3623        /// every interrupt.
3624        const IRQ_READY: u64 = 1 | (1 << 10);
3625
3626        let mut p = Pipeline::new();
3627        p.cop0.set_hardware(crate::cop0::reg::STATUS, IRQ_READY);
3628        let mut pc = 0xFFFF_FFFF_8000_0000;
3629        let mut regs = Regs::new();
3630        let mut bus = NullBus { irq: true };
3631
3632        // Warm up so `prev_was_run` is true AND a real instruction has reached
3633        // DC, then confirm an IRQ IS taken.
3634        //
3635        // This used to advance exactly twice, which accepted the interrupt while
3636        // `ex_dc` was still an unoccupied fill bubble — so `EPC` was charged to
3637        // address 0. The test never noticed because it asserted only the abort
3638        // *flag* and never `EPC`: an assertion the broken machine satisfied just
3639        // as well as a correct one. That blind spot is half of ledger **R-18**
3640        // (see `an_interrupt_across_an_eret_is_charged_to_the_target_not_the_eret`),
3641        // so the loop now waits for a real instruction and `EPC` is asserted too.
3642        p.advance(&mut bus, &mut regs, &mut pc);
3643        assert!(p.prev_cycle_was_run());
3644        let mut accepted = false;
3645        for _ in 0..8 {
3646            p.advance(&mut bus, &mut regs, &mut pc);
3647            if p.ex_dc.abort == Some(Exception::Interrupt) {
3648                accepted = true;
3649                break;
3650            }
3651        }
3652        assert!(
3653            accepted,
3654            "an interrupt should be accepted after a run cycle once a real \
3655             instruction reaches DC"
3656        );
3657        assert_ne!(
3658            p.cop0.read(crate::cop0::reg::EPC),
3659            0,
3660            "EPC must name the interrupted instruction, not a fill bubble"
3661        );
3662
3663        // Now stall, and check the very next cycle refuses it.
3664        let mut p = Pipeline::new();
3665        p.cop0.set_hardware(crate::cop0::reg::STATUS, IRQ_READY);
3666        let mut pc = 0xFFFF_FFFF_8000_0000;
3667        let mut regs = Regs::new();
3668        p.advance(&mut bus, &mut regs, &mut pc);
3669        p.stall_for(1, Interlock::Dcb);
3670        p.advance(&mut bus, &mut regs, &mut pc); // the stalled cycle
3671        assert!(
3672            !p.prev_cycle_was_run(),
3673            "a stalled cycle must not count as a run cycle"
3674        );
3675        let ex_dc_before = p.ex_dc.abort;
3676        p.advance(&mut bus, &mut regs, &mut pc); // the cycle immediately after
3677        assert_eq!(
3678            p.ex_dc.abort, ex_dc_before,
3679            "an interrupt must NOT be accepted when the previous PCycle stalled"
3680        );
3681    }
3682
3683    /// **An abort must survive the cascade**, not merely be present the instant
3684    /// it is stamped.
3685    ///
3686    /// The shallow version of this test asserted latch state immediately after
3687    /// `abort_from` and never advanced — so it could not tell a real flush from
3688    /// one that gets overwritten by the reverse cascade in the same cycle. This
3689    /// version steps the pipeline and follows the consequences.
3690    #[test]
3691    fn an_abort_survives_the_cascade() {
3692        let mut p = Pipeline::new();
3693        let mut pc = 0xFFFF_FFFF_8000_0000;
3694        let mut regs = Regs::new();
3695        let mut bus = quiet();
3696        for _ in 0..4 {
3697            p.advance(&mut bus, &mut regs, &mut pc);
3698        }
3699
3700        // Abort at DC: the instruction in DC plus everything younger.
3701        p.abort_from(Stage::Dc, Exception::AddressError { store: false });
3702        let aborted_pc = p.ex_dc.pc;
3703        p.advance(&mut bus, &mut regs, &mut pc);
3704
3705        // The causing instruction carried its abort forward into WB's latch...
3706        assert_eq!(
3707            (p.dc_wb.abort, p.dc_wb.pc),
3708            (Some(Exception::AddressError { store: false }), aborted_pc),
3709            "the aborting instruction lost its flag while advancing"
3710        );
3711        // ...and the younger ones kept theirs rather than having them
3712        // overwritten by the latch moves.
3713        assert_eq!(
3714            p.ex_dc.abort,
3715            Some(Exception::AddressError { store: false }),
3716            "a younger instruction's abort was overwritten by the cascade"
3717        );
3718
3719        // The younger instruction that was already in flight kept its abort as it
3720        // advanced, rather than having it overwritten by the latch move.
3721        assert_eq!(
3722            p.rf_ex.abort,
3723            Some(Exception::AddressError { store: false }),
3724            "an in-flight younger instruction lost its abort while advancing"
3725        );
3726
3727        // And the fetch issued during the aborting cycle is a bubble, not a live
3728        // wrong-path instruction that would escape the flush entirely.
3729        assert!(
3730            !p.ic_rf.occupied,
3731            "the instruction fetched during the abort escaped the flush"
3732        );
3733    }
3734
3735    /// A zero-cycle stall request is ignored — recording it would burn a cycle
3736    /// and suppress interrupt acceptance on the next one, with no visible cause.
3737    #[test]
3738    fn a_zero_cycle_stall_is_not_a_stall() {
3739        let mut p = Pipeline::new();
3740        let mut pc = 0xFFFF_FFFF_8000_0000;
3741        let mut regs = Regs::new();
3742        let mut bus = quiet();
3743        p.advance(&mut bus, &mut regs, &mut pc);
3744
3745        p.stall_for(0, Interlock::Ldi);
3746        assert!(p.stalled_by().is_none(), "a 0-cycle request is not a stall");
3747
3748        let before = p.ic_rf.pc;
3749        p.advance(&mut bus, &mut regs, &mut pc);
3750        assert_ne!(before, p.ic_rf.pc, "the cycle was silently consumed");
3751        assert!(
3752            p.prev_cycle_was_run(),
3753            "a non-stall must still count as a run cycle, or the interrupt gate \
3754             is wrongly suppressed on the following cycle"
3755        );
3756    }
3757
3758    /// An abort kills its own stage and everything younger, never anything older.
3759    #[test]
3760    fn abort_kills_younger_instructions_only() {
3761        let mut p = Pipeline::new();
3762        let mut pc = 0xFFFF_FFFF_8000_0000;
3763        let mut regs = Regs::new();
3764        let mut bus = quiet();
3765        for _ in 0..4 {
3766            p.advance(&mut bus, &mut regs, &mut pc);
3767        }
3768        p.abort_from(Stage::Ex, Exception::Overflow);
3769        assert_eq!(p.rf_ex.abort, Some(Exception::Overflow), "EX's own latch");
3770        assert_eq!(p.ic_rf.abort, Some(Exception::Overflow), "younger: killed");
3771        assert_eq!(p.ex_dc.abort, None, "older instruction must survive");
3772        assert_eq!(p.dc_wb.abort, None, "older instruction must survive");
3773    }
3774
3775    /// An aborted instruction must not retire.
3776    #[test]
3777    fn an_aborted_instruction_does_not_retire() {
3778        let mut p = Pipeline::new();
3779        let mut pc = 0xFFFF_FFFF_8000_0000;
3780        let mut regs = Regs::new();
3781        let mut bus = quiet();
3782        for _ in 0..4 {
3783            p.advance(&mut bus, &mut regs, &mut pc);
3784        }
3785        p.dc_wb.abort = Some(Exception::AddressError { store: false });
3786        let retired = p.retired;
3787        p.advance(&mut bus, &mut regs, &mut pc);
3788        assert_eq!(p.retired, retired, "an aborted instruction retired anyway");
3789    }
3790
3791    /// **End to end**: a real program, fetched from a bus, decoded, executed and
3792    /// committed to the register file through all five stages.
3793    ///
3794    /// Until this passed, every other test in the crate exercised a piece in
3795    /// isolation. This is the one that says the CPU *runs*.
3796    #[test]
3797    fn a_program_executes_through_the_whole_pipeline() {
3798        /// A bus holding a program at 0, returning `NOP` past its end.
3799        struct Rom(alloc::vec::Vec<u32>);
3800        impl Bus for Rom {
3801            fn read_u8(&mut self, _addr: u32) -> u8 {
3802                0
3803            }
3804            fn write_u8(&mut self, _addr: u32, _val: u8) {}
3805            fn read_u32(&mut self, addr: u32) -> u32 {
3806                self.0.get((addr / 4) as usize).copied().unwrap_or(0)
3807            }
3808        }
3809        const fn r(funct: u32, rs: u32, rt: u32, rd: u32, sa: u32) -> u32 {
3810            (rs << 21) | (rt << 16) | (rd << 11) | (sa << 6) | funct
3811        }
3812        const fn i(opcode: u32, rs: u32, rt: u32, imm: u16) -> u32 {
3813            (opcode << 26) | (rs << 21) | (rt << 16) | imm as u32
3814        }
3815
3816        //   ADDIU $1, $0, 6      ; $1 = 6
3817        //   ADDIU $2, $0, 7      ; $2 = 7
3818        //   MULT  $1, $2         ; HI:LO = 42   (stalls 5 PCycles)
3819        //   MFLO  $3             ; $3 = 42
3820        //   ADDU  $4, $1, $2     ; $4 = 13
3821        //   SLL   $5, $2, 2      ; $5 = 28
3822        let program = alloc::vec![
3823            i(0o11, 0, 1, 6),
3824            i(0o11, 0, 2, 7),
3825            r(0o30, 1, 2, 0, 0),
3826            r(0o22, 0, 0, 3, 0),
3827            r(0o41, 1, 2, 4, 0),
3828            r(0o00, 0, 2, 5, 2),
3829        ];
3830        let mut bus = Rom(program);
3831        let mut regs = Regs::new();
3832        let mut p = Pipeline::new();
3833        let mut pc = 0xFFFF_FFFF_8000_0000u64;
3834
3835        // Generous budget: 6 instructions, 5 stages deep, plus the MULT stall.
3836        for _ in 0..64 {
3837            p.advance(&mut bus, &mut regs, &mut pc);
3838        }
3839
3840        assert_eq!(regs.read(1), 6, "ADDIU $1");
3841        assert_eq!(regs.read(2), 7, "ADDIU $2");
3842        assert_eq!(regs.lo, 42, "MULT wrote LO");
3843        assert_eq!(regs.hi, 0, "MULT wrote HI");
3844        assert_eq!(regs.read(3), 42, "MFLO read LO into $3");
3845        assert_eq!(regs.read(4), 13, "ADDU $1 + $2");
3846        assert_eq!(regs.read(5), 28, "SLL $2 << 2");
3847        assert_eq!(regs.read(0), 0, "$zero stayed zero");
3848        assert!(p.retired >= 6, "all six instructions retired");
3849    }
3850
3851    /// `$zero` must survive an instruction that nominally targets it — software
3852    /// depends on that, and `ADDIU $0, $0, 5` is a legal encoding.
3853    #[test]
3854    fn writes_to_zero_are_discarded_by_the_pipeline() {
3855        struct Ones;
3856        impl Bus for Ones {
3857            fn read_u8(&mut self, _addr: u32) -> u8 {
3858                0
3859            }
3860            fn write_u8(&mut self, _addr: u32, _val: u8) {}
3861            fn read_u32(&mut self, _addr: u32) -> u32 {
3862                // ADDIU $0, $0, 5 -- targets $zero, forever.
3863                (0o11 << 26) | 5
3864            }
3865        }
3866        let mut bus = Ones;
3867        let mut regs = Regs::new();
3868        let mut p = Pipeline::new();
3869        let mut pc = 0xFFFF_FFFF_8000_0000u64;
3870        for _ in 0..32 {
3871            p.advance(&mut bus, &mut regs, &mut pc);
3872        }
3873        assert_eq!(regs.read(0), 0, "$zero was written");
3874        assert_eq!(regs.gpr[0], 0, "$zero was written through the raw array");
3875    }
3876
3877    /// An overflowing `ADD` must abort rather than commit, and the destination
3878    /// register must be left untouched.
3879    #[test]
3880    fn an_overflow_trap_prevents_the_write_back() {
3881        struct Overflowing;
3882        impl Bus for Overflowing {
3883            fn read_u8(&mut self, _addr: u32) -> u8 {
3884                0
3885            }
3886            fn write_u8(&mut self, _addr: u32, _val: u8) {}
3887            fn read_u32(&mut self, addr: u32) -> u32 {
3888                match addr / 4 {
3889                    // LUI $1, 0x7FFF ; ORI $1, $1, 0xFFFF  => $1 = i32::MAX
3890                    0 => (0o17 << 26) | (1 << 16) | 0x7FFF,
3891                    1 => (0o15 << 26) | (1 << 21) | (1 << 16) | 0xFFFF,
3892                    // ADDIU $2, $0, 1
3893                    2 => (0o11 << 26) | (2 << 16) | 1,
3894                    // ADD $3, $1, $2  -> overflows
3895                    3 => (1 << 21) | (2 << 16) | (3 << 11) | 0o40,
3896                    _ => 0,
3897                }
3898            }
3899        }
3900        let mut bus = Overflowing;
3901        let mut regs = Regs::new();
3902        let mut p = Pipeline::new();
3903        let mut pc = 0xFFFF_FFFF_8000_0000u64;
3904        for _ in 0..48 {
3905            p.advance(&mut bus, &mut regs, &mut pc);
3906        }
3907        assert_eq!(regs.read(1), 0x7FFF_FFFF, "LUI+ORI built i32::MAX");
3908        assert_eq!(regs.read(3), 0, "the overflowing ADD must not commit");
3909    }
3910
3911    /// An unaligned instruction fetch raises an address error instead of
3912    /// fetching, and must not silently realign the PC — that would convert a
3913    /// raised exception into a wrong answer on a path hardware never takes.
3914    #[test]
3915    fn an_unaligned_fetch_raises_address_error_without_realigning() {
3916        struct Watch {
3917            fetched: bool,
3918        }
3919        impl Bus for Watch {
3920            fn read_u8(&mut self, _addr: u32) -> u8 {
3921                0
3922            }
3923            fn write_u8(&mut self, _addr: u32, _val: u8) {}
3924            fn read_u32(&mut self, _addr: u32) -> u32 {
3925                self.fetched = true;
3926                0
3927            }
3928        }
3929        let mut bus = Watch { fetched: false };
3930        let mut regs = Regs::new();
3931        let mut p = Pipeline::new();
3932        let mut pc = 0xFFFF_FFFF_8000_0002; // deliberately unaligned
3933
3934        p.advance(&mut bus, &mut regs, &mut pc);
3935
3936        assert_eq!(
3937            p.ic_rf.abort,
3938            Some(Exception::AddressError { store: false }),
3939            "an unaligned fetch must raise AdEL"
3940        );
3941        assert!(
3942            !bus.fetched,
3943            "the bus must not be accessed for a bad address"
3944        );
3945        // The PC is not silently realigned -- it is *vectored*, which is the
3946        // architectural response. Before T-12-002 nothing dispatched and this
3947        // asserted the PC stayed at the bad address; that was the absence of
3948        // dispatch, not a rule.
3949        assert_eq!(
3950            pc, 0xFFFF_FFFF_BFC0_0380,
3951            "the BEV=1 general vector -- cold reset leaves BEV set (UM §6.4.4), \
3952             so a fresh CPU vectors into the boot ROM, not into RDRAM"
3953        );
3954        assert_eq!(
3955            p.cop0.read(crate::cop0::reg::BAD_VADDR),
3956            0xFFFF_FFFF_8000_0002,
3957            "and BadVAddr holds the unaligned address, un-realigned"
3958        );
3959        assert_eq!(
3960            (p.cop0.read(crate::cop0::reg::CAUSE) >> 2) & 0x1F,
3961            crate::exception::exc_code::ADEL,
3962            "an instruction fetch is a load: AdEL, not AdES"
3963        );
3964        // EPC must name the FAULTING fetch. `abort_with` captures its context
3965        // out of `ic_rf`, so raising before populating that latch would silently
3966        // record the previous fetch's PC here -- which is exactly what this code
3967        // did until review caught it.
3968        assert_eq!(
3969            p.cop0.read(crate::cop0::reg::EPC),
3970            0xFFFF_FFFF_8000_0002,
3971            "EPC names the faulting fetch, not the previous one"
3972        );
3973
3974        // And the faulting instruction must never retire. Bounded to the
3975        // epilogue stall on purpose: past that the CPU is running the *handler*,
3976        // whose instructions retire legitimately, so a longer window would be
3977        // asserting that exception handling does not work.
3978        assert_eq!(p.retired, 0, "the faulting fetch retired");
3979        assert_eq!(
3980            p.stalled_by(),
3981            Some(Interlock::Exception),
3982            "the 2-PCycle epilogue stall is in progress (UM §4.7 p.114)"
3983        );
3984        for _ in 0..crate::exception::EPILOGUE_STALL {
3985            p.advance(&mut bus, &mut regs, &mut pc);
3986            assert_eq!(p.retired, 0, "nothing retires while the pipe drains");
3987        }
3988        assert_eq!(p.stalled_by(), None, "the stall was exactly 2 PCycles");
3989    }
3990
3991    /// A RAM-backed bus, so loads and stores can be exercised end to end.
3992    struct Ram {
3993        prog: alloc::vec::Vec<u32>,
3994        data: alloc::vec::Vec<u8>,
3995    }
3996    impl Ram {
3997        fn new(prog: alloc::vec::Vec<u32>) -> Self {
3998            Self {
3999                prog,
4000                data: alloc::vec![0; 0x1000],
4001            }
4002        }
4003    }
4004    impl Bus for Ram {
4005        fn read_u8(&mut self, addr: u32) -> u8 {
4006            self.data.get(addr as usize).copied().unwrap_or(0)
4007        }
4008        fn write_u8(&mut self, addr: u32, val: u8) {
4009            if let Some(b) = self.data.get_mut(addr as usize) {
4010                *b = val;
4011            }
4012        }
4013        fn read_u32(&mut self, addr: u32) -> u32 {
4014            // Instructions live above 0x800; data below it.
4015            if addr >= 0x800 {
4016                self.prog
4017                    .get(((addr - 0x800) / 4) as usize)
4018                    .copied()
4019                    .unwrap_or(0)
4020            } else {
4021                u32::from_be_bytes([
4022                    self.read_u8(addr),
4023                    self.read_u8(addr + 1),
4024                    self.read_u8(addr + 2),
4025                    self.read_u8(addr + 3),
4026                ])
4027            }
4028        }
4029    }
4030    const fn ld_st(opcode: u32, base: u32, rt: u32, off: u16) -> u32 {
4031        (opcode << 26) | (base << 21) | (rt << 16) | off as u32
4032    }
4033
4034    /// A store followed by a load round-trips through real memory, at every
4035    /// width, with the sign/zero-extension rules applied.
4036    #[test]
4037    fn stores_and_loads_round_trip_through_memory() {
4038        //   LUI   $1, 0x8000        ; KSEG0 base
4039        //   ADDIU $2, $0, -2        ; value = 0xFFFF_FFFF_FFFF_FFFE
4040        //   SW    $2, 0x100($1)
4041        //   LW    $3, 0x100($1)     ; sign-extended  -> 0xFFFF_FFFF_FFFF_FFFE
4042        //   LWU   $4, 0x100($1)     ; zero-extended  -> 0x0000_0000_FFFF_FFFE
4043        //   LBU   $5, 0x100($1)     ; big-endian MSB -> 0xFF
4044        let prog = alloc::vec![
4045            lui_kseg0(1),
4046            (0o11 << 26) | (2 << 16) | 0xFFFE,
4047            ld_st(0o53, 1, 2, 0x100),
4048            ld_st(0o43, 1, 3, 0x100),
4049            ld_st(0o47, 1, 4, 0x100),
4050            ld_st(0o44, 1, 5, 0x100),
4051        ];
4052        let mut bus = Ram::new(prog);
4053        let mut regs = Regs::new();
4054        let mut p = Pipeline::new();
4055        let mut pc = KSEG0_PROG;
4056        for _ in 0..80 {
4057            p.advance(&mut bus, &mut regs, &mut pc);
4058        }
4059        assert_eq!(regs.read(3), 0xFFFF_FFFF_FFFF_FFFE, "LW sign-extends");
4060        assert_eq!(regs.read(4), 0x0000_0000_FFFF_FFFE, "LWU zero-extends");
4061        assert_eq!(regs.read(5), 0xFF, "LBU reads the big-endian MSB");
4062    }
4063
4064    /// An unaligned `LW` raises an address error and does not commit — the whole
4065    /// reason the `LWL`/`LWR` family exists.
4066    #[test]
4067    fn an_unaligned_load_raises_address_error() {
4068        //   ADDIU $1, $0, 0x101   ; deliberately unaligned
4069        //   LW    $2, 0($1)
4070        let prog = alloc::vec![(0o11 << 26) | (1 << 16) | 0x101, ld_st(0o43, 1, 2, 0)];
4071        let mut bus = Ram::new(prog);
4072        let mut regs = Regs::new();
4073        let mut p = Pipeline::new();
4074        let mut pc = KSEG0_PROG;
4075        for _ in 0..40 {
4076            p.advance(&mut bus, &mut regs, &mut pc);
4077        }
4078        assert_eq!(regs.read(2), 0, "an unaligned LW must not commit");
4079    }
4080
4081    /// **The load-delay interlock** (UM §4.6.5) — it finally has something to
4082    /// interlock against. A load followed by an instruction naming the loaded
4083    /// register stalls one cycle, and the dependent instruction must still see
4084    /// the loaded value.
4085    #[test]
4086    fn a_load_followed_by_its_use_interlocks_and_still_reads_the_value() {
4087        //   LUI   $1, 0x8000     ; KSEG0 base -- unmapped, so no TLB entry needed
4088        //   ADDIU $2, $0, 0x55
4089        //   SW    $2, 0x100($1)
4090        //   LW    $3, 0x100($1)  ; load
4091        //   ADDIU $4, $3, 1      ; uses $3 immediately -> LDI stall
4092        let prog = alloc::vec![
4093            lui_kseg0(1),
4094            (0o11 << 26) | (2 << 16) | 0x55,
4095            ld_st(0o53, 1, 2, 0x100),
4096            ld_st(0o43, 1, 3, 0x100),
4097            (0o11 << 26) | (3 << 21) | (4 << 16) | 1,
4098        ];
4099        let mut bus = Ram::new(prog);
4100        let mut regs = Regs::new();
4101        let mut p = Pipeline::new();
4102        let mut pc = KSEG0_PROG;
4103        let mut saw_ldi = false;
4104        for _ in 0..80 {
4105            p.advance(&mut bus, &mut regs, &mut pc);
4106            if p.stalled_by() == Some(Interlock::Ldi) {
4107                saw_ldi = true;
4108            }
4109        }
4110        assert!(saw_ldi, "the load-delay interlock never fired");
4111        assert_eq!(regs.read(3), 0x55, "LW loaded the stored value");
4112        assert_eq!(regs.read(4), 0x56, "the dependent instruction saw it");
4113    }
4114
4115    /// The `LWL`/`LWR` pair assembles an unaligned word from memory — the
4116    /// end-to-end version of the unit tests in `mem`.
4117    #[test]
4118    fn the_unaligned_pair_assembles_a_word_from_real_memory() {
4119        //   ADDIU $1, $0, 0x100
4120        //   LWL   $3, 1($1)
4121        //   LWR   $3, 4($1)
4122        let prog = alloc::vec![
4123            lui_kseg0(1),
4124            ld_st(0o42, 1, 3, 0x101),
4125            ld_st(0o46, 1, 3, 0x104),
4126        ];
4127        let mut bus = Ram::new(prog);
4128        // Memory at 0x100: 00 11 22 33 44 55 66 77
4129        for (k, b) in [0x00u8, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77]
4130            .into_iter()
4131            .enumerate()
4132        {
4133            bus.write_u8(0x100 + k as u32, b);
4134        }
4135        let mut regs = Regs::new();
4136        let mut p = Pipeline::new();
4137        let mut pc = KSEG0_PROG;
4138        for _ in 0..60 {
4139            p.advance(&mut bus, &mut regs, &mut pc);
4140        }
4141        assert_eq!(
4142            regs.read(3),
4143            crate::alu::sext32(0x1122_3344),
4144            "LWL+LWR must assemble the unaligned word at 0x101"
4145        );
4146    }
4147
4148    /// **The branch delay slot executes before the target.** This is the single
4149    /// most load-bearing property of MIPS control flow: the instruction *after* a
4150    /// branch runs whether or not the branch is taken.
4151    #[test]
4152    fn the_delay_slot_executes_before_the_branch_target() {
4153        //   0x800: ADDIU $1, $0, 1
4154        //   0x804: BEQ   $0, $0, +2   ; taken, to 0x810
4155        //   0x808: ADDIU $2, $0, 2    ; DELAY SLOT -- must execute
4156        //   0x80C: ADDIU $3, $0, 3    ; skipped
4157        //   0x810: ADDIU $4, $0, 4    ; target
4158        let prog = alloc::vec![
4159            (0o11 << 26) | (1 << 16) | 1,
4160            (0o04 << 26) | 2,
4161            (0o11 << 26) | (2 << 16) | 2,
4162            (0o11 << 26) | (3 << 16) | 3,
4163            (0o11 << 26) | (4 << 16) | 4,
4164        ];
4165        let mut bus = Ram::new(prog);
4166        let mut regs = Regs::new();
4167        let mut p = Pipeline::new();
4168        let mut pc = KSEG0_PROG;
4169        for _ in 0..60 {
4170            p.advance(&mut bus, &mut regs, &mut pc);
4171        }
4172        assert_eq!(regs.read(1), 1, "before the branch");
4173        assert_eq!(regs.read(2), 2, "the DELAY SLOT must execute");
4174        assert_eq!(regs.read(3), 0, "the instruction after the slot is skipped");
4175        assert_eq!(regs.read(4), 4, "the target must execute");
4176    }
4177
4178    /// **Branch-likely nullifies its delay slot when NOT taken**; an ordinary
4179    /// branch does not. Confusing the two silently runs or skips one instruction
4180    /// per untaken branch, which is invisible until a loop's trip count is wrong.
4181    #[test]
4182    fn branch_likely_nullifies_its_delay_slot_but_an_ordinary_branch_does_not() {
4183        // BNEL $0, $0, +1   ; NOT taken (0 == 0), so the slot is nullified
4184        // ADDIU $2, $0, 2   ; DELAY SLOT -- must be squashed
4185        // ADDIU $3, $0, 3
4186        let likely = alloc::vec![
4187            (0o25 << 26) | 1,
4188            (0o11 << 26) | (2 << 16) | 2,
4189            (0o11 << 26) | (3 << 16) | 3,
4190        ];
4191        let mut bus = Ram::new(likely);
4192        let mut regs = Regs::new();
4193        let mut p = Pipeline::new();
4194        let mut pc = KSEG0_PROG;
4195        for _ in 0..50 {
4196            p.advance(&mut bus, &mut regs, &mut pc);
4197        }
4198        assert_eq!(
4199            regs.read(2),
4200            0,
4201            "BNEL not taken must NULLIFY its delay slot"
4202        );
4203        assert_eq!(regs.read(3), 3, "execution continues after the slot");
4204
4205        // The same shape with the ordinary BNE: the slot DOES execute.
4206        let ordinary = alloc::vec![
4207            (0o05 << 26) | 1,
4208            (0o11 << 26) | (2 << 16) | 2,
4209            (0o11 << 26) | (3 << 16) | 3,
4210        ];
4211        let mut bus = Ram::new(ordinary);
4212        let mut regs = Regs::new();
4213        let mut p = Pipeline::new();
4214        let mut pc = KSEG0_PROG;
4215        for _ in 0..50 {
4216            p.advance(&mut bus, &mut regs, &mut pc);
4217        }
4218        assert_eq!(
4219            regs.read(2),
4220            2,
4221            "BNE not taken must still RUN its delay slot"
4222        );
4223    }
4224
4225    /// `JAL` links the address *after* the delay slot, so a returning `JR $31`
4226    /// resumes past it rather than re-executing it.
4227    #[test]
4228    fn jal_links_past_the_delay_slot_and_jr_returns_there() {
4229        //   0x800: JAL 0x204        ; -> 0x810, links $31 = 0x808
4230        //   0x804: ADDIU $1, $0, 1  ; DELAY SLOT
4231        //   0x808: ADDIU $2, $0, 2  ; where JR $31 must return to
4232        //   0x80C: ADDIU $9, $0, 9  ; must NOT run before the return
4233        //   0x810: JR $31
4234        //   0x814: ADDIU $3, $0, 3  ; the callee's DELAY SLOT
4235        let prog = alloc::vec![
4236            (0o03 << 26) | 0x204,
4237            (0o11 << 26) | (1 << 16) | 1,
4238            (0o11 << 26) | (2 << 16) | 2,
4239            (0o11 << 26) | (9 << 16) | 9,
4240            (31 << 21) | 0o10,
4241            (0o11 << 26) | (3 << 16) | 3,
4242        ];
4243        let mut bus = Ram::new(prog);
4244        let mut regs = Regs::new();
4245        let mut p = Pipeline::new();
4246        let mut pc = KSEG0_PROG;
4247        for _ in 0..80 {
4248            p.advance(&mut bus, &mut regs, &mut pc);
4249        }
4250        assert_eq!(
4251            regs.read(31),
4252            KSEG0_PROG + 8,
4253            "JAL links PC+8, past the delay slot"
4254        );
4255        assert_eq!(regs.read(1), 1, "JAL's delay slot ran");
4256        assert_eq!(regs.read(3), 3, "JR's delay slot ran");
4257        assert_eq!(regs.read(2), 2, "JR $31 returned to the linked address");
4258    }
4259
4260    /// A trap whose condition holds raises an exception and does not commit.
4261    #[test]
4262    fn a_taken_trap_raises_and_an_untaken_one_does_not() {
4263        //   ADDIU $1, $0, 5
4264        //   TEQ   $1, $1      ; equal -> traps
4265        //   ADDIU $2, $0, 2   ; must not commit
4266        let prog = alloc::vec![
4267            (0o11 << 26) | (1 << 16) | 5,
4268            (1 << 21) | (1 << 16) | 0o64,
4269            (0o11 << 26) | (2 << 16) | 2,
4270        ];
4271        let mut bus = Ram::new(prog);
4272        let mut regs = Regs::new();
4273        let mut p = Pipeline::new();
4274        let mut pc = KSEG0_PROG;
4275        let mut trapped = false;
4276        for _ in 0..50 {
4277            p.advance(&mut bus, &mut regs, &mut pc);
4278            if p.ex_dc.abort == Some(Exception::Trap) || p.dc_wb.abort == Some(Exception::Trap) {
4279                trapped = true;
4280            }
4281        }
4282        assert!(trapped, "TEQ with equal operands must trap");
4283
4284        // TNE with equal operands does NOT trap.
4285        let prog = alloc::vec![
4286            (0o11 << 26) | (1 << 16) | 5,
4287            (1 << 21) | (1 << 16) | 0o66,
4288            (0o11 << 26) | (2 << 16) | 2,
4289        ];
4290        let mut bus = Ram::new(prog);
4291        let mut regs = Regs::new();
4292        let mut p = Pipeline::new();
4293        let mut pc = KSEG0_PROG;
4294        for _ in 0..50 {
4295            p.advance(&mut bus, &mut regs, &mut pc);
4296        }
4297        assert_eq!(
4298            regs.read(2),
4299            2,
4300            "an untaken trap must not disturb execution"
4301        );
4302    }
4303
4304    /// `SYSCALL` and `BREAK` raise their own exceptions.
4305    #[test]
4306    fn syscall_and_break_raise_their_exceptions() {
4307        for (funct, want) in [(0o14u32, Exception::Syscall), (0o15, Exception::Breakpoint)] {
4308            let mut bus = Ram::new(alloc::vec![funct]);
4309            let mut regs = Regs::new();
4310            let mut p = Pipeline::new();
4311            let mut pc = KSEG0_PROG;
4312            let mut seen = false;
4313            for _ in 0..40 {
4314                p.advance(&mut bus, &mut regs, &mut pc);
4315                if p.ex_dc.abort == Some(want) || p.dc_wb.abort == Some(want) {
4316                    seen = true;
4317                }
4318            }
4319            assert!(seen, "funct {funct:o} should raise {want:?}");
4320        }
4321    }
4322
4323    /// **`in_delay_slot` must actually be set**, and only on the instruction
4324    /// after a branch or jump.
4325    ///
4326    /// This test exists because mutation-testing found the flag was **not yet
4327    /// load-bearing**: forcing it to `false` broke nothing, since its only
4328    /// consumer is `Cause.BD`/`EPC` at exception time and COP0 arrives in
4329    /// Sprint 2. A field that is written and never read is the exact pattern
4330    /// this crate has twice deleted (`poll_irq_at_phase`, `Stall.resume`).
4331    ///
4332    /// Rather than delete it — it is genuinely needed, and it must ride in the
4333    /// latch rather than be recomputed later — it is pinned here so it is
4334    /// verified from the moment it is written.
4335    #[test]
4336    fn in_delay_slot_is_set_on_exactly_the_instruction_after_a_branch() {
4337        //   0x800: ADDIU $1, $0, 1   ; not a delay slot
4338        //   0x804: BEQ   $0, $0, +2  ; a branch, not itself a delay slot
4339        //   0x808: ADDIU $2, $0, 2   ; IS the delay slot
4340        //   0x810: ADDIU $4, $0, 4   ; target, not a delay slot
4341        let prog = alloc::vec![
4342            (0o11 << 26) | (1 << 16) | 1,
4343            (0o04 << 26) | 2,
4344            (0o11 << 26) | (2 << 16) | 2,
4345            (0o11 << 26) | (3 << 16) | 3,
4346            (0o11 << 26) | (4 << 16) | 4,
4347        ];
4348        let mut bus = Ram::new(prog);
4349        let mut regs = Regs::new();
4350        let mut p = Pipeline::new();
4351        let mut pc = KSEG0_PROG;
4352
4353        let mut flagged = alloc::vec::Vec::new();
4354        for _ in 0..8 {
4355            p.advance(&mut bus, &mut regs, &mut pc);
4356            if p.ic_rf.occupied && p.ic_rf.in_delay_slot {
4357                flagged.push(p.ic_rf.pc);
4358            }
4359        }
4360        assert_eq!(
4361            flagged,
4362            alloc::vec![KSEG0_PROG + 8],
4363            "exactly one instruction -- the one after the branch -- must be \
4364             flagged as a delay slot"
4365        );
4366    }
4367
4368    /// The load interlock reproduces the hardware's documented imprecision.
4369    /// Emulating *precise* behavior here is the bug.
4370    #[test]
4371    fn load_interlock_is_imprecise_exactly_as_hardware_is() {
4372        // Matches on rt: the ordinary true positive.
4373        assert!(load_interlocks(8, 0, 8, true));
4374        // Matches on rs.
4375        assert!(load_interlocks(8, 8, 0, true));
4376        // False positive hardware really has: LUI's unused rs field matching.
4377        assert!(
4378            load_interlocks(8, 8, 9, true),
4379            "hardware stalls even when the field is not used as a source"
4380        );
4381        // Two consecutive loads into the same register also stall.
4382        assert!(load_interlocks(8, 0, 8, true));
4383        // $zero is exempt -- a load into it can never be depended on.
4384        assert!(!load_interlocks(0, 0, 0, true));
4385        // GPR loads do not interlock with float instructions, or vice versa.
4386        assert!(!load_interlocks(8, 8, 8, false));
4387        // No overlap at all.
4388        assert!(!load_interlocks(8, 9, 10, true));
4389    }
4390
4391    // --- LL / SC (UM §16 pp. 453, 487; §3.1; §5.4.7) ------------------------
4392
4393    use crate::mem::{LoadKind, StoreKind};
4394
4395    /// Where [`Ram`] keeps test programs, addressed through **KSEG0**.
4396    ///
4397    /// KSEG0 is unmapped, so it reaches physical `0x800` without a TLB entry —
4398    /// which is how real code runs and, since T-12-004, the only way a test can
4399    /// fetch at all without installing a mapping. Fetching from a bare `0x800`
4400    /// is a KUSEG address and now correctly raises a TLB refill.
4401    const KSEG0_PROG: u64 = 0xFFFF_FFFF_8000_0800;
4402
4403    /// A bus that fetches `NOP`s and holds its interrupt line asserted.
4404    ///
4405    /// Reads return 0, which decodes to `SLL $0, $0, 0` — the canonical `NOP` —
4406    /// so the pipeline runs without any instruction interfering with what these
4407    /// tests observe.
4408    struct AlwaysIrq;
4409    impl Bus for AlwaysIrq {
4410        fn read_u8(&mut self, _a: u32) -> u8 {
4411            0
4412        }
4413        fn write_u8(&mut self, _a: u32, _v: u8) {}
4414        fn read_u32(&mut self, _a: u32) -> u32 {
4415            0
4416        }
4417        fn poll_irq(&mut self) -> bool {
4418            true
4419        }
4420    }
4421
4422    /// The synchronization tests need only the data half of [`Ram`].
4423    fn ram() -> Ram {
4424        Ram::new(alloc::vec![])
4425    }
4426
4427    /// Push the dirty D-cache line covering `vaddr` out to the bus.
4428    ///
4429    /// A store to a cached segment now lands in the write-back D-cache and RAM
4430    /// does not see it until something forces it out. That is the whole point of
4431    /// a write-back cache, and it is what real software does before handing a
4432    /// buffer to DMA -- so a test that asserts on RAM without this would be
4433    /// asserting the cache does not exist.
4434    fn writeback<B: Bus>(p: &mut Pipeline, bus: &mut B, vaddr: u64) {
4435        p.access(
4436            bus,
4437            MemOp::Cache {
4438                addr: vaddr,
4439                op: 25,
4440            },
4441        )
4442        .expect("Hit_WriteBack on a mapped address cannot fault");
4443    }
4444
4445    /// Without a preceding `LL` the store must not happen, and `rt` must still
4446    /// be written — with 0. A `Store`-shaped implementation writes memory
4447    /// unconditionally; a `Load`-shaped one never writes `rt` on failure.
4448    #[test]
4449    fn sc_without_ll_stores_nothing_and_reports_failure() {
4450        let mut p = Pipeline::new();
4451        let mut bus = ram();
4452        let wb = p
4453            .access(
4454                &mut bus,
4455                MemOp::ConditionalStore {
4456                    kind: StoreKind::Word,
4457                    addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4458                    value: 0xDEAD_BEEF,
4459                    dest: 9,
4460                },
4461            )
4462            .expect("aligned");
4463        assert_eq!(
4464            wb,
4465            WriteBack::Gpr { dest: 9, value: 0 },
4466            "failure is reported in rt as 0"
4467        );
4468        assert_eq!(bus.read_u32(0x40), 0, "memory untouched");
4469    }
4470
4471    /// The ordinary success path, and the `LLAddr` side effect.
4472    #[test]
4473    fn ll_arms_the_link_and_records_the_physical_address() {
4474        let mut p = Pipeline::new();
4475        let mut bus = ram();
4476        bus.data[0x40..0x44].copy_from_slice(&0x1234_5678u32.to_be_bytes());
4477
4478        // KSEG0, so `translate` has to strip the segment for LLAddr to be right.
4479        let wb = p
4480            .access(
4481                &mut bus,
4482                MemOp::LinkedLoad {
4483                    kind: LoadKind::SignedWord,
4484                    addr: 0xFFFF_FFFF_8000_0040,
4485                    dest: 8,
4486                },
4487            )
4488            .expect("aligned");
4489        assert_eq!(
4490            wb,
4491            WriteBack::Gpr {
4492                dest: 8,
4493                value: 0x1234_5678
4494            }
4495        );
4496        assert!(p.ll_bit(), "LL arms the link bit");
4497        assert_eq!(
4498            p.ll_addr(),
4499            0x40 >> 4,
4500            "LLAddr holds PA(31:4) of the PHYSICAL address, not the virtual one"
4501        );
4502
4503        let wb = p
4504            .access(
4505                &mut bus,
4506                MemOp::ConditionalStore {
4507                    kind: StoreKind::Word,
4508                    addr: 0xFFFF_FFFF_8000_0040,
4509                    value: 0xA5A5_A5A5,
4510                    dest: 9,
4511                },
4512            )
4513            .expect("aligned");
4514        assert_eq!(wb, WriteBack::Gpr { dest: 9, value: 1 });
4515        writeback(&mut p, &mut bus, 0xFFFF_FFFF_8000_0040);
4516        assert_eq!(bus.read_u32(0x40), 0xA5A5_A5A5, "the store happened");
4517    }
4518
4519    /// The manual lists exactly what clears `LLbit`: *"set by the LL
4520    /// instruction, cleared by an ERET, and tested by the SC instruction"*
4521    /// (UM §3.1). `SC` is a *tester*, not a clearer.
4522    ///
4523    /// This is the assertion that fails if someone "tidies up" by clearing the
4524    /// link in `SC` — which looks right, matches several other architectures,
4525    /// and would make a second `SC` spuriously fail.
4526    #[test]
4527    fn sc_does_not_clear_the_link_bit() {
4528        let mut p = Pipeline::new();
4529        let mut bus = ram();
4530        p.access(
4531            &mut bus,
4532            MemOp::LinkedLoad {
4533                kind: LoadKind::SignedWord,
4534                addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4535                dest: 8,
4536            },
4537        )
4538        .expect("aligned");
4539
4540        for round in 0..3 {
4541            let wb = p
4542                .access(
4543                    &mut bus,
4544                    MemOp::ConditionalStore {
4545                        kind: StoreKind::Word,
4546                        addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4547                        value: 1,
4548                        dest: 9,
4549                    },
4550                )
4551                .expect("aligned");
4552            assert_eq!(
4553                wb,
4554                WriteBack::Gpr { dest: 9, value: 1 },
4555                "SC #{round} must still succeed -- nothing has cleared LLbit"
4556            );
4557            assert!(p.ll_bit(), "and the bit is still armed after SC #{round}");
4558        }
4559    }
4560
4561    /// *"If this instruction both fails and causes an exception, the exception
4562    /// takes precedence"* (UM §16 p. 487) — so a misaligned `SC` must raise,
4563    /// not quietly report failure in `rt`.
4564    #[test]
4565    fn misaligned_sc_raises_rather_than_reporting_failure() {
4566        let mut p = Pipeline::new();
4567        let mut bus = ram();
4568        let err = p
4569            .access(
4570                &mut bus,
4571                MemOp::ConditionalStore {
4572                    kind: StoreKind::Word,
4573                    addr: 0xFFFF_FFFF_8000_0000 | 0x42,
4574                    value: 1,
4575                    dest: 9,
4576                },
4577            )
4578            .expect_err("misaligned");
4579        assert_eq!(
4580            err,
4581            Exception::AddressError { store: true },
4582            "SC is a store, so AdES not AdEL"
4583        );
4584    }
4585
4586    /// A misaligned `LL` must not arm the link — the instruction did not
4587    /// complete, so a following `SC` has nothing to succeed against.
4588    #[test]
4589    fn misaligned_ll_does_not_arm_the_link() {
4590        let mut p = Pipeline::new();
4591        let mut bus = ram();
4592        let err = p
4593            .access(
4594                &mut bus,
4595                MemOp::LinkedLoad {
4596                    kind: LoadKind::SignedWord,
4597                    addr: 0xFFFF_FFFF_8000_0000 | 0x42,
4598                    dest: 8,
4599                },
4600            )
4601            .expect_err("misaligned");
4602        assert_eq!(
4603            err,
4604            Exception::AddressError { store: false },
4605            "LL is a load, so AdEL not AdES"
4606        );
4607        assert!(!p.ll_bit(), "a faulted LL leaves the link disarmed");
4608    }
4609
4610    /// The doubleword forms share the path, but the width must actually differ.
4611    #[test]
4612    fn lld_and_scd_operate_on_eight_bytes() {
4613        let mut p = Pipeline::new();
4614        let mut bus = ram();
4615        bus.data[0x40..0x48].copy_from_slice(&0x0123_4567_89AB_CDEFu64.to_be_bytes());
4616
4617        let wb = p
4618            .access(
4619                &mut bus,
4620                MemOp::LinkedLoad {
4621                    kind: LoadKind::Double,
4622                    addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4623                    dest: 8,
4624                },
4625            )
4626            .expect("aligned");
4627        assert_eq!(
4628            wb,
4629            WriteBack::Gpr {
4630                dest: 8,
4631                value: 0x0123_4567_89AB_CDEF
4632            }
4633        );
4634
4635        p.access(
4636            &mut bus,
4637            MemOp::ConditionalStore {
4638                kind: StoreKind::Double,
4639                addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4640                value: u64::MAX,
4641                dest: 9,
4642            },
4643        )
4644        .expect("aligned");
4645        writeback(&mut p, &mut bus, 0xFFFF_FFFF_8000_0040);
4646        assert_eq!(bus.read_u32(0x40), u32::MAX);
4647        assert_eq!(bus.read_u32(0x44), u32::MAX, "all eight bytes, not four");
4648    }
4649
4650    // --- COP0 access through the pipeline (T-12-001) -----------------------
4651
4652    /// Build a `COP0` instruction word: opcode 0o20, `rs` = form, `rt` = GPR,
4653    /// `rd` = COP0 register.
4654    const fn cop0_word(rs: u32, rt: u32, rd: u32) -> u32 {
4655        (0o20 << 26) | (rs << 21) | (rt << 16) | (rd << 11)
4656    }
4657
4658    /// `LUI rt, 0x8000` — put a **KSEG0** base in `rt`.
4659    ///
4660    /// Data addresses need this for the same reason instruction fetches need
4661    /// [`KSEG0_PROG`]: a bare low address is KUSEG, which is TLB-mapped and now
4662    /// correctly raises a refill rather than being silently masked.
4663    const fn lui_kseg0(rt: u32) -> u32 {
4664        (0o17 << 26) | (rt << 16) | 0x8000
4665    }
4666
4667    /// `ADDIU rt, $0, imm` — the constant-loading prologue these tests share.
4668    /// Written as a helper rather than inline so the `rs = $0` term does not
4669    /// have to be spelled as a no-op shift that clippy objects to.
4670    const fn addiu_zero(rt: u32, imm: u16) -> u32 {
4671        (0o11 << 26) | (rt << 16) | imm as u32
4672    }
4673
4674    /// `MTC0` then `MFC0` must round-trip through the real register file,
4675    /// exercising the WB-write / DC-read split rather than a direct call.
4676    #[test]
4677    fn mtc0_then_mfc0_round_trips_through_the_pipeline() {
4678        //   ADDIU $1, $0, 0x18     ; a value to write
4679        //   MTC0  $1, Compare      ; COP0 write happens in WB
4680        //   MFC0  $2, Compare      ; COP0 read happens in DC
4681        let program = alloc::vec![
4682            addiu_zero(1, 0x18),
4683            cop0_word(0o04, 1, u32::from(crate::cop0::reg::COMPARE)),
4684            cop0_word(0o00, 2, u32::from(crate::cop0::reg::COMPARE)),
4685        ];
4686        let mut bus = Ram::new(program);
4687        let mut regs = Regs::new();
4688        let mut p = Pipeline::new();
4689        let mut pc = KSEG0_PROG;
4690
4691        for _ in 0..32 {
4692            p.advance(&mut bus, &mut regs, &mut pc);
4693        }
4694
4695        assert_eq!(
4696            p.cop0.read(crate::cop0::reg::COMPARE),
4697            0x18,
4698            "MTC0 reached the COP0 register file"
4699        );
4700        assert_eq!(regs.read(2), 0x18, "MFC0 brought it back into $2");
4701    }
4702
4703    /// `MTC0` must not be given a GPR destination by decode: it reads `rt` and
4704    /// writes COP0. If `dest` were set to `rt`, the instruction would clobber
4705    /// the very register it sourced its value from.
4706    #[test]
4707    fn mtc0_does_not_write_a_general_register() {
4708        let program = alloc::vec![
4709            addiu_zero(1, 0x55),
4710            cop0_word(0o04, 1, u32::from(crate::cop0::reg::COMPARE)),
4711        ];
4712        let mut bus = Ram::new(program);
4713        let mut regs = Regs::new();
4714        let mut p = Pipeline::new();
4715        let mut pc = KSEG0_PROG;
4716        for _ in 0..24 {
4717            p.advance(&mut bus, &mut regs, &mut pc);
4718        }
4719        assert_eq!(regs.read(1), 0x55, "$1 still holds the source value");
4720        assert_eq!(p.cop0.read(crate::cop0::reg::COMPARE), 0x55);
4721    }
4722
4723    /// The write-mask rules must survive the pipeline path, not just direct
4724    /// calls: `MTC0` to `Cause` may only touch IP1:IP0.
4725    #[test]
4726    fn a_pipelined_mtc0_still_respects_the_write_mask() {
4727        let program = alloc::vec![
4728            // ADDIU $1, $0, -1  => $1 = 0xFFFF_FFFF_FFFF_FFFF
4729            addiu_zero(1, 0xFFFF),
4730            cop0_word(0o04, 1, u32::from(crate::cop0::reg::CAUSE)),
4731        ];
4732        let mut bus = Ram::new(program);
4733        let mut regs = Regs::new();
4734        let mut p = Pipeline::new();
4735        // Move `Compare` off `Count`'s reset value first. Both reset undefined
4736        // (UM §6.4.4) and we choose a deterministic zero for each, so they match
4737        // at power-on and latch IP7 -- see accuracy-ledger D-3. Harmless, but it
4738        // would show up in `Cause` here and obscure what this test is about.
4739        p.cop0.mtc0(crate::cop0::reg::COMPARE, 0xFFFF);
4740        let mut pc = KSEG0_PROG;
4741        for _ in 0..24 {
4742            p.advance(&mut bus, &mut regs, &mut pc);
4743        }
4744        assert_eq!(
4745            p.cop0.read(crate::cop0::reg::CAUSE),
4746            0x0000_0300,
4747            "only the software interrupt bits took"
4748        );
4749    }
4750
4751    /// `LL` records `LLAddr`, which *is* COP0 register 17 — so a `MFC0` of it
4752    /// must see what `LL` wrote. Until COP0 existed, `LLAddr` lived on
4753    /// `Pipeline` as a second copy of the same architectural value; this test
4754    /// pins the fact that there is now only one.
4755    #[test]
4756    fn ll_writes_the_real_cop0_lladdr_register() {
4757        let mut p = Pipeline::new();
4758        let mut bus = Ram::new(alloc::vec![]);
4759        p.access(
4760            &mut bus,
4761            MemOp::LinkedLoad {
4762                kind: LoadKind::SignedWord,
4763                addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4764                dest: 8,
4765            },
4766        )
4767        .expect("aligned");
4768        assert_eq!(p.ll_addr(), 0x04, "PA(31:4) of 0x40");
4769        assert_eq!(
4770            p.cop0.read(crate::cop0::reg::LL_ADDR),
4771            0x04,
4772            "the accessor and COP0 reg 17 are the same storage, not two copies"
4773        );
4774        assert_eq!(
4775            p.cop0.mfc0(crate::cop0::reg::LL_ADDR),
4776            0x04,
4777            "and software can read it back with MFC0"
4778        );
4779    }
4780
4781    // --- exception dispatch and ERET through the pipeline (T-12-002) --------
4782
4783    /// The stale-capture regression, with a *plausible* wrong answer available.
4784    ///
4785    /// The pipeline runs valid instructions first, so `ic_rf` holds a real PC
4786    /// when the unaligned fetch arrives. A capture taken before the latch is
4787    /// populated therefore reports that earlier PC — a value that looks entirely
4788    /// reasonable in `EPC`, which is what makes the bug survive inspection.
4789    #[test]
4790    fn an_unaligned_fetch_after_valid_ones_still_reports_its_own_address() {
4791        let program = alloc::vec![addiu_zero(1, 1), addiu_zero(2, 2), addiu_zero(3, 3)];
4792        let mut bus = Ram::new(program);
4793        let mut regs = Regs::new();
4794        let mut p = Pipeline::new();
4795        p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
4796        let mut pc = KSEG0_PROG;
4797
4798        // Let the pipeline fill with real instructions.
4799        for _ in 0..3 {
4800            p.advance(&mut bus, &mut regs, &mut pc);
4801        }
4802        assert!(
4803            p.ic_rf.pc >= 0x800,
4804            "a real PC is latched to be stale about"
4805        );
4806
4807        // Now fetch an unaligned address.
4808        pc = 0xFFFF_FFFF_8000_0006;
4809        p.advance(&mut bus, &mut regs, &mut pc);
4810
4811        assert_eq!(
4812            p.cop0.read(crate::cop0::reg::EPC),
4813            0xFFFF_FFFF_8000_0006,
4814            "EPC must be the unaligned fetch, not the last good one"
4815        );
4816        assert_eq!(
4817            p.cop0.read(crate::cop0::reg::BAD_VADDR),
4818            0xFFFF_FFFF_8000_0006,
4819            "and BadVAddr likewise"
4820        );
4821    }
4822
4823    /// `ERET` clears `LLbit`, completing the `LL`/`SC` contract that Sprint 1
4824    /// left open: until now **nothing** cleared the link, so a `LL`; `ERET`;
4825    /// `SC` sequence wrongly succeeded.
4826    #[test]
4827    fn eret_clears_the_link_bit_and_makes_a_following_sc_fail() {
4828        let mut p = Pipeline::new();
4829        let mut bus = ram();
4830        p.access(
4831            &mut bus,
4832            MemOp::LinkedLoad {
4833                kind: LoadKind::SignedWord,
4834                addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4835                dest: 8,
4836            },
4837        )
4838        .expect("aligned");
4839        assert!(p.ll_bit(), "LL armed the link");
4840
4841        // ERET: opcode 0o20, rs = 0o20 (CO), funct = 0o30.
4842        let word = (0o20 << 26) | (0o20 << 21) | 0o30;
4843        assert_eq!(decode(word).op, crate::decode::Op::Eret);
4844
4845        p.cop0
4846            .set_hardware(crate::cop0::reg::STATUS, 1 << 1 /* EXL */);
4847        p.cop0
4848            .set_hardware(crate::cop0::reg::EPC, 0xFFFF_FFFF_8000_5000);
4849
4850        let mut regs = Regs::new();
4851        let mut prog = Ram::new(alloc::vec![word]);
4852        let mut pc = KSEG0_PROG;
4853        // The `LL` above filled the D-cache, whose stall (`M_DCACHE_FILL`, ledger
4854        // C-1) must drain before the ERET runs -- so more cycles than the bare
4855        // pipeline latency.
4856        for _ in 0..60 {
4857            p.advance(&mut prog, &mut regs, &mut pc);
4858        }
4859
4860        assert!(!p.ll_bit(), "ERET cleared the link (UM §3.1)");
4861        let wb = p
4862            .access(
4863                &mut bus,
4864                MemOp::ConditionalStore {
4865                    kind: StoreKind::Word,
4866                    addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4867                    value: 0xFFFF,
4868                    dest: 9,
4869                },
4870            )
4871            .expect("aligned");
4872        assert_eq!(
4873            wb,
4874            WriteBack::Gpr { dest: 9, value: 0 },
4875            "SC after ERET must fail"
4876        );
4877        assert_eq!(bus.data[0x40..0x44], [0, 0, 0, 0], "and store nothing");
4878    }
4879
4880    /// `ERET` resumes at `EPC` and clears `EXL`, and it has **no delay slot** —
4881    /// the instruction after it must not execute.
4882    #[test]
4883    fn eret_resumes_at_epc_and_has_no_delay_slot() {
4884        let eret = (0o20 << 26) | (0o20 << 21) | 0o30;
4885        // ERET, then an instruction that would be a delay slot for any branch.
4886        // If it runs, $5 becomes 0x1234 -- which is the whole assertion.
4887        let program = alloc::vec![eret, addiu_zero(5, 0x1234)];
4888        let mut bus = Ram::new(program);
4889        let mut regs = Regs::new();
4890        let mut p = Pipeline::new();
4891        p.cop0.set_hardware(crate::cop0::reg::STATUS, 1 << 1);
4892        p.cop0
4893            .set_hardware(crate::cop0::reg::EPC, 0xFFFF_FFFF_8000_5000);
4894        let mut pc = KSEG0_PROG;
4895
4896        for _ in 0..12 {
4897            p.advance(&mut bus, &mut regs, &mut pc);
4898        }
4899
4900        assert_eq!(
4901            regs.read(5),
4902            0,
4903            "ERET has no delay slot -- the following instruction must be squashed"
4904        );
4905        assert_eq!(
4906            p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1),
4907            0,
4908            "EXL cleared"
4909        );
4910    }
4911
4912    /// **An interrupt pending across an `ERET` is taken at the `ERET`'s TARGET,
4913    /// never at the `ERET` itself** — ledger R-18, and the defect that stopped
4914    /// Banjo-Tooie dead.
4915    ///
4916    /// `ERET` resolves in `EX`, where it clears `Status.EXL` and points
4917    /// `next_pc` at `EPC`. `DC` runs its interrupt check one cycle later, by
4918    /// which time `EXL` reads 0 and `interrupt_pending()` is true — so the
4919    /// interrupt was stamped onto the instruction sitting in `ex_dc`, **which is
4920    /// the `ERET`**. That overwrites `EPC` with the `ERET`'s own address, and
4921    /// the return address it was about to consume is gone. The handler then
4922    /// returns to the `ERET`, which resumes at itself: an architectural
4923    /// livelock, measured at 390,625 `ERET` retirements per frame with no other
4924    /// instruction ever reaching `WB`.
4925    ///
4926    /// The assertion is on `EPC`, not on "an interrupt happened". Both the
4927    /// broken and the fixed machine take the interrupt; they differ only in
4928    /// **which PC it is charged to**, so an assertion that merely observed the
4929    /// vector being entered would pass either way.
4930    #[test]
4931    fn an_interrupt_across_an_eret_is_charged_to_the_target_not_the_eret() {
4932        /// [`Ram`], but with the interrupt line held asserted.
4933        ///
4934        /// `dc_stage` re-samples `bus.poll_irq()` into `Cause.IP2` every cycle,
4935        /// so seeding `IP2` on the COP0 block directly is erased on the next
4936        /// cycle — the first version of this test did exactly that and passed
4937        /// having taken no interrupt at all.
4938        struct IrqRam(Ram);
4939        impl Bus for IrqRam {
4940            fn read_u8(&mut self, a: u32) -> u8 {
4941                self.0.read_u8(a)
4942            }
4943            fn write_u8(&mut self, a: u32, v: u8) {
4944                self.0.write_u8(a, v);
4945            }
4946            fn read_u32(&mut self, a: u32) -> u32 {
4947                self.0.read_u32(a)
4948            }
4949            fn poll_irq(&mut self) -> bool {
4950                true
4951            }
4952        }
4953
4954        const TARGET: u64 = 0xFFFF_FFFF_8000_5000;
4955        let eret = (0o20 << 26) | (0o20 << 21) | 0o30;
4956        let program = alloc::vec![eret, addiu_zero(5, 0x1234)];
4957        let mut bus = IrqRam(Ram::new(program));
4958        let mut regs = Regs::new();
4959        let mut p = Pipeline::new();
4960        // In a handler: EXL set, IE set, IM allowing IP2, and IP2 asserted. This
4961        // is the state libultra's interrupt epilogue actually reaches — the
4962        // handler unmasks and returns with an RCP interrupt still pending.
4963        //
4964        // `EXL` is **set**, which is what isolates this to the `ERET`. While a
4965        // handler runs, `interrupt_pending()` is false however hard the line is
4966        // asserted, so nothing can fire until the `ERET` itself clears `EXL` in
4967        // EX — which places the `ERET` in `ex_dc` on exactly the cycle the
4968        // interrupt becomes takeable. With `EXL` clear instead, the interrupt
4969        // fires during pipeline fill and is charged to a *bubble* (`EPC == 0`) —
4970        // a different defect that would mask this one.
4971        p.cop0
4972            .set_hardware(crate::cop0::reg::STATUS, (1 << 1) | (0xFF << 8) | 1);
4973        p.cop0.set_hardware(crate::cop0::reg::EPC, TARGET);
4974        p.cop0.set_ip(2, true);
4975        let mut pc = KSEG0_PROG;
4976
4977        // Witness BOTH events as they happen, because both end-state values this
4978        // test cares about are also their own starting values: `EXL` starts set
4979        // and `EPC` starts at `TARGET`. Checking either one at the end therefore
4980        // proves nothing on its own — a machine that executed nothing at all
4981        // satisfies both. (The first version of this test did exactly that, and
4982        // it took a review to notice.) So observe the transitions instead:
4983        // `EXL` going clear can only be the `ERET` retiring, and an `Interrupt`
4984        // abort appearing in a latch can only be the interrupt being taken.
4985        let mut eret_retired = false;
4986        let mut interrupt_taken = false;
4987        for _ in 0..12 {
4988            p.advance(&mut bus, &mut regs, &mut pc);
4989            if p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1) == 0 {
4990                eret_retired = true;
4991            }
4992            if [p.dc_wb.abort, p.ex_dc.abort, p.rf_ex.abort, p.ic_rf.abort]
4993                .contains(&Some(Exception::Interrupt))
4994            {
4995                interrupt_taken = true;
4996            }
4997        }
4998        assert!(
4999            eret_retired,
5000            "the ERET never cleared EXL, so it never executed -- nothing under \
5001             test happened"
5002        );
5003        assert!(
5004            interrupt_taken,
5005            "no interrupt was ever taken (Cause={:#010X}) -- this test would \
5006             prove nothing",
5007            p.cop0.read(crate::cop0::reg::CAUSE) as u32
5008        );
5009
5010        let epc = p.cop0.read(crate::cop0::reg::EPC);
5011        assert_ne!(
5012            epc, KSEG0_PROG,
5013            "the interrupt was charged to the ERET itself ({epc:#018X}); ERET would \
5014             then resume at its own address forever (R-18 livelock)"
5015        );
5016        assert_eq!(
5017            epc, TARGET,
5018            "an interrupt taken across an ERET belongs to the instruction the ERET \
5019             returned to, so EPC must still be the target"
5020        );
5021    }
5022
5023    /// A `SYSCALL` executed through the pipeline must vector, record `EPC`, and
5024    /// set the right `ExcCode` — the whole epilogue, end to end.
5025    #[test]
5026    fn a_syscall_vectors_and_records_its_cause() {
5027        // SYSCALL is SPECIAL funct 0o14.
5028        let program = alloc::vec![0o14];
5029        let mut bus = Ram::new(program);
5030        let mut regs = Regs::new();
5031        let mut p = Pipeline::new();
5032        // BEV=0 so the vector is the RDRAM one, which is what a running game
5033        // uses; cold reset would otherwise send us to the boot ROM.
5034        p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
5035        let mut pc = KSEG0_PROG;
5036
5037        // Stop AT the dispatch cycle. Running on would be fine architecturally
5038        // -- the handler starts fetching -- but then `pc` has moved past the
5039        // vector and asserting on it would be asserting that nothing executes.
5040        let mut cycles = 0;
5041        while p.stalled_by() != Some(Interlock::Exception) {
5042            p.advance(&mut bus, &mut regs, &mut pc);
5043            cycles += 1;
5044            assert!(cycles < 8, "SYSCALL never dispatched");
5045        }
5046
5047        assert_eq!(
5048            (p.cop0.read(crate::cop0::reg::CAUSE) >> 2) & 0x1F,
5049            crate::exception::exc_code::SYS
5050        );
5051        assert_eq!(p.cop0.read(crate::cop0::reg::EPC), KSEG0_PROG);
5052        assert_eq!(pc, 0xFFFF_FFFF_8000_0180);
5053        assert_ne!(
5054            p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1),
5055            0,
5056            "EXL set, so the handler runs in kernel mode with interrupts off"
5057        );
5058    }
5059
5060    /// The epilogue must not overwrite `EPC` when `EXL` is already set — tested
5061    /// **through the pipeline**, not just against `dispatch` directly, because
5062    /// this is the failure that only shows up when handlers nest.
5063    #[test]
5064    fn a_second_exception_in_a_handler_preserves_the_first_epc() {
5065        let program = alloc::vec![0o14, 0o14];
5066        let mut bus = Ram::new(program);
5067        let mut regs = Regs::new();
5068        let mut p = Pipeline::new();
5069        p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
5070        let mut pc = KSEG0_PROG;
5071
5072        for _ in 0..8 {
5073            p.advance(&mut bus, &mut regs, &mut pc);
5074        }
5075        let first_epc = p.cop0.read(crate::cop0::reg::EPC);
5076        assert_eq!(first_epc, KSEG0_PROG);
5077        assert_ne!(p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1), 0);
5078
5079        // Now run a second SYSCALL while EXL is still set. Point the fetch back
5080        // at the program so it hits the second word.
5081        pc = KSEG0_PROG + 4;
5082        for _ in 0..8 {
5083            p.advance(&mut bus, &mut regs, &mut pc);
5084        }
5085        assert_eq!(
5086            p.cop0.read(crate::cop0::reg::EPC),
5087            first_epc,
5088            "the first handler's return address must survive (UM §6.3.7)"
5089        );
5090    }
5091
5092    // --- interrupts, Count/Compare (T-12-003) -------------------------------
5093
5094    /// Every term of the recognition predicate is load-bearing. Dropping the
5095    /// `EXL`/`ERL` terms is the classic bug: it works until an interrupt arrives
5096    /// inside a handler, and then re-enters it forever.
5097    #[test]
5098    fn every_term_of_the_interrupt_predicate_is_required() {
5099        use crate::cop0::reg;
5100        let ready = 1u64 | (1 << 10); // IE | IM2
5101
5102        let mut p = Pipeline::new();
5103        p.cop0.set_hardware(reg::STATUS, ready);
5104        p.cop0.set_ip(2, true);
5105        assert!(p.cop0.interrupt_pending(), "all four conditions met");
5106
5107        for (name, status) in [
5108            ("IE clear", ready & !1),
5109            ("EXL set", ready | (1 << 1)),
5110            ("ERL set", ready | (1 << 2)),
5111            ("IM2 masked", ready & !(1 << 10)),
5112        ] {
5113            let mut p = Pipeline::new();
5114            p.cop0.set_hardware(reg::STATUS, status);
5115            p.cop0.set_ip(2, true);
5116            assert!(
5117                !p.cop0.interrupt_pending(),
5118                "{name}: the interrupt must NOT be recognized"
5119            );
5120        }
5121    }
5122
5123    /// A masked interrupt must still be *visible* in `Cause.IP`, because
5124    /// software polls it. Folding assertion and recognition into one step makes
5125    /// a masked line invisible to `MFC0 Cause`.
5126    #[test]
5127    fn a_masked_interrupt_is_still_visible_in_cause() {
5128        use crate::cop0::reg;
5129        let mut p = Pipeline::new();
5130        let mut regs = Regs::new();
5131        let mut pc = KSEG0_PROG;
5132        let mut bus = AlwaysIrq;
5133        // IE set but IM2 MASKED, so nothing is recognized.
5134        p.cop0.set_hardware(reg::STATUS, 1);
5135
5136        p.advance(&mut bus, &mut regs, &mut pc);
5137        assert_ne!(
5138            p.cop0.read(reg::CAUSE) & (1 << 10),
5139            0,
5140            "IP2 is asserted even though it is masked"
5141        );
5142        assert!(!p.cop0.interrupt_pending(), "but not recognized");
5143    }
5144
5145    /// **NMI ignores every mask, and saves to `ErrorEPC` rather than `EPC`.**
5146    ///
5147    /// The interesting part is what it is *not*: `Status` here has `IE` clear,
5148    /// `EXL` set and `ERL` set, which is the state in which an ordinary
5149    /// interrupt is unconditionally ignored. The companion assertion below
5150    /// checks exactly that, so the two differ only in which line was raised.
5151    #[test]
5152    fn an_nmi_is_taken_regardless_of_ie_exl_and_erl() {
5153        use crate::cop0::reg;
5154        let mut p = Pipeline::new();
5155        let mut regs = Regs::new();
5156        let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1); 8]);
5157        // IE clear, EXL set, ERL set -- every reason to refuse an interrupt.
5158        // EXL (bit 1) and ERL (bit 2).
5159        p.cop0.set_hardware(reg::STATUS, (1 << 1) | (1 << 2));
5160        p.cop0.set_hardware(reg::CAUSE, 0);
5161        let mut pc = KSEG0_PROG;
5162
5163        // Warm the pipeline first: the NMI is attributed to the instruction at
5164        // the boundary, and an empty `DC` latch has no PC to record.
5165        for _ in 0..5 {
5166            p.advance(&mut bus, &mut regs, &mut pc);
5167        }
5168        p.signal_nmi();
5169        // Stop at the redirect itself: past it, `pc` is the next fetch address
5170        // and has already walked on from the vector.
5171        let mut cycles = 0;
5172        while p.stalled_by() != Some(Interlock::Exception) {
5173            p.advance(&mut bus, &mut regs, &mut pc);
5174            cycles += 1;
5175            assert!(cycles < 16, "the NMI was never taken");
5176        }
5177
5178        assert_eq!(
5179            pc, 0xFFFF_FFFF_BFC0_0000,
5180            "NMI vectors to the Cold Reset location (UM §6.4.6)"
5181        );
5182        let status = p.cop0.read(reg::STATUS);
5183        assert_ne!(status & (1 << 20), 0, "SR set, to tell NMI from a reset");
5184        assert_ne!(status & (1 << 22), 0, "BEV set");
5185        assert_ne!(status & (1 << 2), 0, "ERL set");
5186        assert_eq!(status & (1 << 21), 0, "TS cleared");
5187        assert_eq!(
5188            p.cop0.read(reg::CAUSE),
5189            0,
5190            "NMI writes no Cause -- all registers are preserved but ErrorEPC \
5191             and those four Status bits"
5192        );
5193        assert_ne!(
5194            p.cop0.read(reg::ERROR_EPC),
5195            0,
5196            "the boundary PC is saved to ErrorEPC, not EPC"
5197        );
5198        assert_eq!(p.cop0.read(reg::EPC), 0, "and EPC is left alone");
5199    }
5200
5201    /// The control for the test above: in that same `Status`, an ordinary
5202    /// interrupt is refused. Without this, "NMI was taken" would be equally
5203    /// consistent with the masks simply not working.
5204    #[test]
5205    fn an_ordinary_interrupt_in_the_same_state_is_refused() {
5206        use crate::cop0::reg;
5207        let mut p = Pipeline::new();
5208        let mut regs = Regs::new();
5209        let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1); 8]);
5210        // EXL (bit 1) and ERL (bit 2).
5211        p.cop0.set_hardware(reg::STATUS, (1 << 1) | (1 << 2));
5212        // Raise the timer line, the strongest ordinary interrupt available.
5213        p.cop0.set_ip(7, true);
5214        let mut pc = KSEG0_PROG;
5215
5216        for _ in 0..16 {
5217            p.advance(&mut bus, &mut regs, &mut pc);
5218            assert_ne!(
5219                p.stalled_by(),
5220                Some(Interlock::Exception),
5221                "a maskable interrupt must NOT be taken with EXL and ERL set"
5222            );
5223        }
5224    }
5225
5226    /// **A stall must not swallow the timer interrupt.**
5227    ///
5228    /// `Count` keeps advancing while the pipeline is stalled — it is derived
5229    /// from the master clock, not from retired instructions — so `Compare` can
5230    /// be passed entirely inside a multi-cycle interlock. An `MCI` stall for a
5231    /// 64-bit multiply is 69 `PCycles` (UM Table 3-12), far longer than the
5232    /// single cycle for which the equality holds.
5233    ///
5234    /// The failure this pins is not a *late* interrupt but a **lost** one: an
5235    /// edge test that only asks "is `Count == Compare` right now?" and is polled
5236    /// only from `DC` never sees the match at all, and `IP7` stays clear
5237    /// forever. Software waiting on the timer then hangs.
5238    #[test]
5239    fn the_timer_interrupt_survives_a_multi_cycle_stall() {
5240        use crate::cop0::reg;
5241        let mut p = Pipeline::new();
5242        let mut regs = Regs::new();
5243        let mut pc = KSEG0_PROG;
5244        let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1)]);
5245        p.cop0.set_hardware(reg::STATUS, 0);
5246        // Compare sits in the middle of the stall window, never at its edges.
5247        p.cop0.mtc0(reg::COMPARE, 40);
5248
5249        p.advance_at(&mut bus, &mut regs, &mut pc, 0);
5250        // A 64-bit multiply's interlock, long enough to step over the match.
5251        p.stall_for(69, Interlock::Mci);
5252        for now in 1..=80 {
5253            p.advance_at(&mut bus, &mut regs, &mut pc, now);
5254        }
5255
5256        assert_ne!(
5257            p.cop0.read(reg::CAUSE) & (1 << 15),
5258            0,
5259            "Count passed Compare during the stall -- IP7 must still latch, \
5260             because the timer is tied to the clock and not to the pipeline"
5261        );
5262    }
5263
5264    /// `Count` reaching `Compare` raises `IP7`, and writing `Compare` clears it
5265    /// as a side effect (UM §6.3.4, p. 165).
5266    #[test]
5267    fn the_timer_interrupt_sets_ip7_and_a_compare_write_clears_it() {
5268        use crate::cop0::reg;
5269        let mut p = Pipeline::new();
5270        let mut regs = Regs::new();
5271        let mut pc = KSEG0_PROG;
5272        let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1)]);
5273        p.cop0.set_hardware(reg::STATUS, 0);
5274        p.cop0.mtc0(reg::COMPARE, 3);
5275
5276        // Walk the timeline to the match.
5277        for now in 0..=3 {
5278            p.advance_at(&mut bus, &mut regs, &mut pc, now);
5279        }
5280        assert_ne!(
5281            p.cop0.read(reg::CAUSE) & (1 << 15),
5282            0,
5283            "IP7 set at Count==Compare"
5284        );
5285
5286        // It must PERSIST as Count runs past Compare. A level implementation
5287        // clears here, and with it drops any timer interrupt raised while the
5288        // CPU could not accept one.
5289        for now in 4..20 {
5290            p.advance_at(&mut bus, &mut regs, &mut pc, now);
5291            assert_ne!(
5292                p.cop0.read(reg::CAUSE) & (1 << 15),
5293                0,
5294                "IP7 must stay latched past the match (UM §6.4.18)"
5295            );
5296        }
5297
5298        // Only writing Compare clears it.
5299        p.cop0.mtc0(reg::COMPARE, 999);
5300        assert_eq!(
5301            p.cop0.read(reg::CAUSE) & (1 << 15),
5302            0,
5303            "writing Compare clears the timer interrupt"
5304        );
5305        p.advance_at(&mut bus, &mut regs, &mut pc, 20);
5306        assert_eq!(p.cop0.read(reg::CAUSE) & (1 << 15), 0, "and it stays clear");
5307    }
5308
5309    /// The convenience [`Pipeline::advance`] **holds** the `Count` timeline
5310    /// rather than guessing a rate for it.
5311    ///
5312    /// `Count` runs at half `PClock`, so stepping it once per `advance` would run
5313    /// the timer at double rate — and halving it with a parity bit would be a
5314    /// second incremented counter, exactly what ADR 0006 forbids. Anything that
5315    /// exercises `Count` must use `advance_at`.
5316    #[test]
5317    fn the_convenience_advance_holds_the_count_timeline() {
5318        let mut p = Pipeline::new();
5319        let mut regs = Regs::new();
5320        let mut pc = KSEG0_PROG;
5321        let mut bus = Ram::new(alloc::vec![]);
5322
5323        p.advance_at(&mut bus, &mut regs, &mut pc, 7);
5324        assert_eq!(p.cop0.read(crate::cop0::reg::COUNT), 7);
5325        for _ in 0..10 {
5326            p.advance(&mut bus, &mut regs, &mut pc);
5327        }
5328        assert_eq!(
5329            p.cop0.read(crate::cop0::reg::COUNT),
5330            7,
5331            "held, not advanced at PClock rate"
5332        );
5333    }
5334
5335    /// **Why `IP7` must latch.** A timer interrupt that fires while the CPU
5336    /// cannot accept one — `EXL` set, i.e. inside a handler — must still be
5337    /// waiting when the handler returns.
5338    ///
5339    /// With `IP7` modeled as a level tied to `Count == Compare`, the equality
5340    /// holds for a single tick, so the interrupt is silently **lost**. That is
5341    /// the failure the latch prevents, and it is invisible to any test that only
5342    /// checks the match cycle itself.
5343    #[test]
5344    fn a_timer_interrupt_raised_while_exl_is_set_survives_until_eret() {
5345        use crate::cop0::reg;
5346        let mut p = Pipeline::new();
5347        let mut regs = Regs::new();
5348        let mut pc = KSEG0_PROG;
5349        let mut bus = Ram::new(alloc::vec![]);
5350        // IE and IM7 set, but EXL set too: a handler is running.
5351        p.cop0.set_hardware(reg::STATUS, 1 | (1 << 15) | (1 << 1));
5352        p.cop0.mtc0(reg::COMPARE, 3);
5353
5354        for now in 0..=10 {
5355            p.advance_at(&mut bus, &mut regs, &mut pc, now);
5356        }
5357        assert!(
5358            !p.cop0.interrupt_pending(),
5359            "EXL blocks it, correctly, for now"
5360        );
5361        assert_ne!(
5362            p.cop0.read(reg::CAUSE) & (1 << 15),
5363            0,
5364            "but IP7 is still asserted, waiting"
5365        );
5366
5367        // The handler returns.
5368        let status = p.cop0.read(reg::STATUS);
5369        p.cop0.set_hardware(reg::STATUS, status & !(1 << 1));
5370        assert!(
5371            p.cop0.interrupt_pending(),
5372            "and the timer interrupt is taken now, not dropped"
5373        );
5374    }
5375
5376    /// An interrupt taken through the pipeline runs the whole epilogue: `EXL`
5377    /// set, `ExcCode` = 0, vectored.
5378    #[test]
5379    fn an_accepted_interrupt_vectors_with_exccode_zero() {
5380        use crate::cop0::reg;
5381        let mut p = Pipeline::new();
5382        let mut regs = Regs::new();
5383        let mut pc = KSEG0_PROG;
5384        let mut bus = AlwaysIrq;
5385        p.cop0.set_hardware(reg::STATUS, 1 | (1 << 10));
5386
5387        let mut cycles = 0;
5388        while p.stalled_by() != Some(Interlock::Exception) {
5389            p.advance(&mut bus, &mut regs, &mut pc);
5390            cycles += 1;
5391            assert!(cycles < 12, "the interrupt was never taken");
5392        }
5393        assert_eq!(
5394            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5395            crate::exception::exc_code::INT
5396        );
5397        assert_ne!(p.cop0.read(reg::STATUS) & (1 << 1), 0, "EXL set");
5398        assert_eq!(pc, 0xFFFF_FFFF_8000_0180);
5399        // And now that EXL is set, the still-asserted line must NOT re-enter.
5400        assert!(
5401            !p.cop0.interrupt_pending(),
5402            "EXL blocks re-entry while the handler runs"
5403        );
5404    }
5405
5406    // --- the TLB through the pipeline (T-12-004) ---------------------------
5407
5408    /// Install a 4 KiB global mapping `vaddr` -> `pfn`, valid and writable.
5409    fn map(p: &mut Pipeline, index: u64, vaddr: u64, pfn: u64) {
5410        use crate::cop0::reg;
5411        p.cop0.set_hardware(reg::PAGE_MASK, 0);
5412        p.cop0.set_hardware(reg::ENTRY_HI, vaddr & 0xFFFF_E000);
5413        // V | D | C=3 | G, in both halves so the entry is global.
5414        p.cop0
5415            .set_hardware(reg::ENTRY_LO0, (pfn << 6) | (3 << 3) | 0b111);
5416        p.cop0
5417            .set_hardware(reg::ENTRY_LO1, ((pfn + 1) << 6) | (3 << 3) | 0b111);
5418        p.cop0.set_hardware(reg::INDEX, index);
5419        p.tlb.write_entry(index as usize, &p.cop0);
5420    }
5421
5422    /// A KUSEG access with no mapping raises a **refill**, which takes the
5423    /// refill vector — not the general one.
5424    #[test]
5425    fn an_unmapped_kuseg_access_takes_the_refill_vector() {
5426        use crate::cop0::reg;
5427        let mut p = Pipeline::new();
5428        let mut regs = Regs::new();
5429        let mut bus = Ram::new(alloc::vec![lui_kseg0(1), ld_st(0o43, 0, 3, 0x100)]);
5430        // BEV=0, EXL=0 so the refill vector is the RDRAM one.
5431        p.cop0.set_hardware(reg::STATUS, 0);
5432        let mut pc = KSEG0_PROG;
5433
5434        let mut cycles = 0;
5435        while p.stalled_by() != Some(Interlock::Exception) {
5436            p.advance(&mut bus, &mut regs, &mut pc);
5437            cycles += 1;
5438            assert!(cycles < 16, "no exception raised");
5439        }
5440        assert_eq!(
5441            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5442            crate::exception::exc_code::TLBL,
5443            "a load miss is TLBL"
5444        );
5445        assert_eq!(
5446            pc, 0xFFFF_FFFF_8000_0000,
5447            "the REFILL vector (0x000), not the general one (0x180)"
5448        );
5449        assert_eq!(p.cop0.read(reg::BAD_VADDR), 0x100);
5450    }
5451
5452    /// The same miss, but with **64-bit addressing enabled**, must reach the
5453    /// **XTLB** refill vector at `0x080`.
5454    ///
5455    /// This is the end-to-end companion to
5456    /// `a_refill_in_64_bit_addressing_takes_the_xtlb_vector`, which only pins the
5457    /// exception-to-vector mapping. That mapping is satisfied just as well by a
5458    /// `wide` flag that is *never true in practice* — the plumbing from
5459    /// `Status.KX` through the faulting access to the exception is exactly what
5460    /// a mapping test cannot see. The two differ only in `Status`, so the
5461    /// vectors landing on different addresses is attributable to nothing else.
5462    #[test]
5463    fn an_unmapped_access_under_64_bit_addressing_takes_the_xtlb_vector() {
5464        use crate::cop0::reg;
5465        let mut p = Pipeline::new();
5466        let mut regs = Regs::new();
5467        let mut bus = Ram::new(alloc::vec![lui_kseg0(1), ld_st(0o43, 0, 3, 0x100)]);
5468        // Identical to the 32-bit case except for KX (bit 7), which turns on
5469        // 64-bit addressing for Kernel mode.
5470        p.cop0.set_hardware(reg::STATUS, 1 << 7);
5471        let mut pc = KSEG0_PROG;
5472
5473        let mut cycles = 0;
5474        while p.stalled_by() != Some(Interlock::Exception) {
5475            p.advance(&mut bus, &mut regs, &mut pc);
5476            cycles += 1;
5477            assert!(cycles < 16, "no exception raised");
5478        }
5479        assert_eq!(
5480            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5481            crate::exception::exc_code::TLBL,
5482            "still TLBL -- the ExcCode does not change with the vector"
5483        );
5484        assert_eq!(
5485            pc, 0xFFFF_FFFF_8000_0080,
5486            "the XTLB refill vector (0x080), not the 32-bit one (0x000)"
5487        );
5488    }
5489
5490    /// A mapped, valid page translates end to end through a real load.
5491    #[test]
5492    fn a_mapped_page_translates_a_real_load() {
5493        let mut p = Pipeline::new();
5494        let mut regs = Regs::new();
5495        // Map KUSEG page-pair 0 with pfn 0 (even) / 1 (odd), so the even page is
5496        // an identity mapping onto the small test RAM. Note 0x1000 and 0x0000
5497        // are the SAME pair -- VPN2 tags at 8 KiB granularity, not 4 KiB.
5498        let mut bus = Ram::new(alloc::vec![ld_st(0o43, 0, 3, 0x100)]);
5499        bus.write_u8(0x100, 0xAB);
5500        bus.write_u8(0x101, 0xCD);
5501        bus.write_u8(0x102, 0xEF);
5502        bus.write_u8(0x103, 0x01);
5503        map(&mut p, 0, 0x1000, 0);
5504        p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
5505        let mut pc = KSEG0_PROG;
5506
5507        // Enough cycles to cover the D-cache fill stall the cached load now pays
5508        // (`M_DCACHE_FILL`, ledger C-1) on top of the pipeline latency.
5509        for _ in 0..100 {
5510            p.advance(&mut bus, &mut regs, &mut pc);
5511        }
5512        assert_eq!(
5513            regs.read(3),
5514            crate::alu::sext32(0xABCD_EF01),
5515            "the even page of pair 0 maps to physical 0x100 via the TLB"
5516        );
5517    }
5518
5519    /// A store to a page whose `D` bit is clear raises **Modified**, which takes
5520    /// the general vector — an entry was found, so there is nothing to refill.
5521    #[test]
5522    fn a_store_to_a_clean_page_raises_modified_at_the_general_vector() {
5523        use crate::cop0::reg;
5524        let mut p = Pipeline::new();
5525        let mut regs = Regs::new();
5526        let mut bus = Ram::new(alloc::vec![ld_st(0o53, 0, 0, 0x1100)]);
5527        // V | C=3 | G but NOT D -- readable, not writable.
5528        p.cop0.set_hardware(reg::PAGE_MASK, 0);
5529        p.cop0.set_hardware(reg::ENTRY_HI, 0x1000);
5530        p.cop0.set_hardware(reg::ENTRY_LO0, (3 << 3) | 0b011);
5531        p.cop0
5532            .set_hardware(reg::ENTRY_LO1, (1 << 6) | (3 << 3) | 0b011);
5533        p.tlb.write_entry(0, &p.cop0);
5534        p.cop0.set_hardware(reg::STATUS, 0);
5535        let mut pc = KSEG0_PROG;
5536
5537        let mut cycles = 0;
5538        while p.stalled_by() != Some(Interlock::Exception) {
5539            p.advance(&mut bus, &mut regs, &mut pc);
5540            cycles += 1;
5541            assert!(cycles < 16, "no exception raised");
5542        }
5543        assert_eq!(
5544            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5545            crate::exception::exc_code::MOD
5546        );
5547        assert_eq!(
5548            pc, 0xFFFF_FFFF_8000_0180,
5549            "Modified takes the GENERAL vector"
5550        );
5551    }
5552
5553    /// A TLB exception fills `EntryHi`, `Context` and `XContext` — the refill
5554    /// handler reads `Context` as a ready-made page-table pointer, which is why
5555    /// hardware assembles it rather than leaving it to software.
5556    #[test]
5557    fn a_tlb_exception_assembles_entryhi_and_context() {
5558        use crate::cop0::reg;
5559        let mut p = Pipeline::new();
5560        let mut regs = Regs::new();
5561        let mut bus = Ram::new(alloc::vec![ld_st(0o43, 0, 3, 0x4000)]);
5562        p.cop0.set_hardware(reg::STATUS, 0);
5563        // A page-table base the handler would have set up.
5564        p.cop0.set_hardware(reg::CONTEXT, 0xFFFF_FFFF_8080_0000);
5565        let mut pc = KSEG0_PROG;
5566
5567        let mut cycles = 0;
5568        while p.stalled_by() != Some(Interlock::Exception) {
5569            p.advance(&mut bus, &mut regs, &mut pc);
5570            cycles += 1;
5571            assert!(cycles < 16, "no exception raised");
5572        }
5573        assert_eq!(
5574            p.cop0.read(reg::ENTRY_HI) & 0xFFFF_E000,
5575            0x4000,
5576            "EntryHi holds the faulting VPN2"
5577        );
5578        assert_eq!(
5579            p.cop0.read(reg::CONTEXT) & 0xFFFF_FFFF_FF80_0000,
5580            0xFFFF_FFFF_8080_0000,
5581            "PTEBase is preserved"
5582        );
5583        assert_eq!(
5584            (p.cop0.read(reg::CONTEXT) >> 4) & 0x7_FFFF,
5585            0x4000 >> 13,
5586            "BadVPN2 is filled in"
5587        );
5588    }
5589
5590    /// `TLBWI` writes the entry `Index` names; `TLBWR` writes the one `Random`
5591    /// names. Using the wrong register is a silent, hard-to-see swap.
5592    #[test]
5593    fn tlbwi_uses_index_and_tlbwr_uses_random() {
5594        use crate::cop0::reg;
5595        const TLBWI: u32 = (0o20 << 26) | (0o20 << 21) | 0o02;
5596        const TLBWR: u32 = (0o20 << 26) | (0o20 << 21) | 0o06;
5597
5598        let mut p = Pipeline::new();
5599        let mut regs = Regs::new();
5600        let mut bus = Ram::new(alloc::vec![TLBWI]);
5601        p.cop0.set_hardware(reg::STATUS, 0);
5602        p.cop0.set_hardware(reg::ENTRY_HI, 0x2000);
5603        p.cop0
5604            .set_hardware(reg::ENTRY_LO0, (7 << 6) | (3 << 3) | 0b111);
5605        p.cop0
5606            .set_hardware(reg::ENTRY_LO1, (8 << 6) | (3 << 3) | 0b111);
5607        p.cop0.set_hardware(reg::INDEX, 5);
5608        let mut pc = KSEG0_PROG;
5609        for _ in 0..16 {
5610            p.advance(&mut bus, &mut regs, &mut pc);
5611        }
5612        assert_eq!(p.tlb.entry(5).lo0.pfn, 7, "TLBWI wrote entry Index = 5");
5613
5614        // TLBWR with Random forced to a different index.
5615        let mut p = Pipeline::new();
5616        let mut regs = Regs::new();
5617        let mut bus = Ram::new(alloc::vec![TLBWR]);
5618        p.cop0.set_hardware(reg::STATUS, 0);
5619        p.cop0.set_hardware(reg::ENTRY_HI, 0x2000);
5620        p.cop0
5621            .set_hardware(reg::ENTRY_LO0, (9 << 6) | (3 << 3) | 0b111);
5622        p.cop0
5623            .set_hardware(reg::ENTRY_LO1, (10 << 6) | (3 << 3) | 0b111);
5624        p.cop0.set_hardware(reg::INDEX, 5);
5625        p.cop0.set_hardware(reg::RANDOM, 20);
5626        let mut pc = KSEG0_PROG;
5627        for _ in 0..16 {
5628            p.advance(&mut bus, &mut regs, &mut pc);
5629        }
5630        assert_eq!(p.tlb.entry(20).lo0.pfn, 9, "TLBWR wrote entry Random = 20");
5631        assert_ne!(p.tlb.entry(5).lo0.pfn, 9, "and NOT entry Index");
5632    }
5633
5634    /// `TLBWR` cannot reach a wired entry, because `Random` never goes below
5635    /// `Wired` — but **`TLBWI` can** (UM §5.4.4, p. 150). Guarding both is a
5636    /// natural-looking mistake that makes wired entries unwritable at all.
5637    #[test]
5638    fn tlbwi_can_overwrite_a_wired_entry_even_though_tlbwr_cannot() {
5639        use crate::cop0::reg;
5640        const TLBWI: u32 = (0o20 << 26) | (0o20 << 21) | 0o02;
5641        let mut p = Pipeline::new();
5642        let mut regs = Regs::new();
5643        let mut bus = Ram::new(alloc::vec![TLBWI]);
5644        p.cop0.set_hardware(reg::STATUS, 0);
5645        p.cop0.mtc0(reg::WIRED, 8);
5646        p.cop0.set_hardware(reg::ENTRY_HI, 0x2000);
5647        p.cop0
5648            .set_hardware(reg::ENTRY_LO0, (11 << 6) | (3 << 3) | 0b111);
5649        p.cop0
5650            .set_hardware(reg::ENTRY_LO1, (12 << 6) | (3 << 3) | 0b111);
5651        p.cop0.set_hardware(reg::INDEX, 3); // inside the wired range
5652        let mut pc = KSEG0_PROG;
5653        for _ in 0..16 {
5654            p.advance(&mut bus, &mut regs, &mut pc);
5655        }
5656        assert_eq!(
5657            p.tlb.entry(3).lo0.pfn,
5658            11,
5659            "TLBWI must be able to write a wired entry"
5660        );
5661
5662        // And Random's range protects those entries from TLBWR structurally,
5663        // rather than by a check.
5664        for _ in 0..200 {
5665            p.cop0.tick_random();
5666            assert!(
5667                p.cop0.read(reg::RANDOM) >= 8,
5668                "Random must never select a wired entry"
5669            );
5670        }
5671    }
5672
5673    /// A TLB fault on a sign-extended kernel address must record the **`R`**
5674    /// region in `EntryHi`, not just `VPN2`.
5675    ///
5676    /// Leaving `R` zero puts every such fault in region 0, so the handler's
5677    /// `TLBWR` installs an entry that can never match the address that faulted —
5678    /// an infinite refill loop, not a visibly wrong value.
5679    #[test]
5680    fn a_fault_on_a_sign_extended_address_records_its_region_in_entryhi() {
5681        use crate::cop0::reg;
5682        let mut p = Pipeline::new();
5683        let mut regs = Regs::new();
5684        // LW $3, 0($1) with $1 = 0xFFFF_FFFF_E000_0000 (KSEG3, mapped).
5685        let mut bus = Ram::new(alloc::vec![
5686            (0o17 << 26) | (1 << 16) | 0xE000, // LUI $1, 0xE000
5687            ld_st(0o43, 1, 3, 0),
5688        ]);
5689        p.cop0.set_hardware(reg::STATUS, 0);
5690        let mut pc = KSEG0_PROG;
5691
5692        let mut cycles = 0;
5693        while p.stalled_by() != Some(Interlock::Exception) {
5694            p.advance(&mut bus, &mut regs, &mut pc);
5695            cycles += 1;
5696            assert!(cycles < 20, "no TLB exception raised");
5697        }
5698        assert_eq!(
5699            p.cop0.read(reg::BAD_VADDR),
5700            0xFFFF_FFFF_E000_0000,
5701            "the full sign-extended address faulted"
5702        );
5703        assert_eq!(
5704            (p.cop0.read(reg::ENTRY_HI) >> 62) & 0b11,
5705            0b11,
5706            "EntryHi.R must carry the faulting region, not 0"
5707        );
5708        assert_eq!(
5709            p.cop0.read(reg::ENTRY_HI) & crate::tlb::VPN2_MASK,
5710            0xFFFF_FFFF_E000_0000 & crate::tlb::VPN2_MASK,
5711            "and VPN2 alongside it"
5712        );
5713    }
5714
5715    /// **`Random` advances as instructions retire** (UM §5.4.2, p. 147).
5716    ///
5717    /// It was implemented and never called from the pipeline, so it sat at 31
5718    /// forever and every `TLBWR` overwrote the same entry. A stuck counter is
5719    /// invisible to any test that calls `tick_random` itself — which is what the
5720    /// COP0 unit tests do — so this asserts it through `advance`.
5721    #[test]
5722    fn random_advances_as_instructions_retire() {
5723        use crate::cop0::reg;
5724        let mut p = Pipeline::new();
5725        let mut regs = Regs::new();
5726        let mut bus = Ram::new(alloc::vec![
5727            addiu_zero(1, 1),
5728            addiu_zero(2, 2),
5729            addiu_zero(3, 3),
5730            addiu_zero(4, 4),
5731        ]);
5732        p.cop0.set_hardware(reg::STATUS, 0);
5733        p.cop0.mtc0(reg::WIRED, 0);
5734        let start = p.cop0.read(reg::RANDOM);
5735        let mut pc = KSEG0_PROG;
5736        for _ in 0..24 {
5737            p.advance(&mut bus, &mut regs, &mut pc);
5738        }
5739        assert!(p.retired >= 4, "instructions retired");
5740        assert_ne!(
5741            p.cop0.read(reg::RANDOM),
5742            start,
5743            "Random must move as instructions retire -- a stuck Random makes \
5744             every TLBWR overwrite the same entry"
5745        );
5746    }
5747
5748    /// TLB shutdown must reach **`Status.TS`**, not just an internal flag —
5749    /// software polls `TS` precisely to discover that the TLB has died.
5750    #[test]
5751    fn tlb_shutdown_sets_status_ts() {
5752        use crate::cop0::reg;
5753        let mut p = Pipeline::new();
5754        let mut regs = Regs::new();
5755        let mut bus = Ram::new(alloc::vec![ld_st(0o43, 0, 3, 0x100)]);
5756        p.cop0.set_hardware(reg::STATUS, 0);
5757        // Two coinciding entries.
5758        map(&mut p, 0, 0x0000, 0);
5759        map(&mut p, 7, 0x0000, 4);
5760        assert_eq!(p.cop0.read(reg::STATUS) & (1 << 21), 0, "TS clear so far");
5761
5762        let mut pc = KSEG0_PROG;
5763        for _ in 0..24 {
5764            p.advance(&mut bus, &mut regs, &mut pc);
5765        }
5766        assert!(p.tlb.is_shutdown(), "the duplicate was noticed");
5767        assert_ne!(
5768            p.cop0.read(reg::STATUS) & (1 << 21),
5769            0,
5770            "Status.TS must be set (UM Fig. 6-6) -- an internal flag nobody can \
5771             read is worse than not tracking it"
5772        );
5773    }
5774
5775    /// The 3-PCycle micro-ITLB reload is charged **only when a reload happens**
5776    /// (UM §4.6.2). A fetch that misses both levels goes straight to its
5777    /// exception rather than paying for a reload that never occurred.
5778    ///
5779    /// **This test cannot currently observe the charge itself.** `stall_for`
5780    /// replaces any pending stall, so the exception's 2-PCycle stall supersedes
5781    /// a wrongly-charged 3-PCycle reload in the same cycle — mutating the guard
5782    /// away produces no behavioral difference today, which mutation testing
5783    /// duly reported. The guard is kept because it is what the manual says and
5784    /// because it becomes observable the moment stalls compose rather than
5785    /// replace; what is asserted here is the *decision*
5786    /// ([`Tlb::jtlb_has_match`]) and the absence of a reload, not the timing.
5787    #[test]
5788    fn a_fetch_missing_both_tlb_levels_is_not_charged_for_a_reload() {
5789        use crate::cop0::reg;
5790        let mut p = Pipeline::new();
5791        let mut regs = Regs::new();
5792        let mut bus = Ram::new(alloc::vec![]);
5793        p.cop0.set_hardware(reg::STATUS, 0);
5794        // Fetch from an unmapped KUSEG address: misses ITLB and JTLB alike.
5795        let mut pc = 0x0000_4000u64;
5796
5797        assert!(
5798            !p.tlb.jtlb_has_match(pc, 0),
5799            "the JTLB has nothing to reload the micro-TLB from"
5800        );
5801        p.advance(&mut bus, &mut regs, &mut pc);
5802        assert_eq!(
5803            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5804            crate::exception::exc_code::TLBL,
5805            "it went straight to the refill exception"
5806        );
5807    }
5808
5809    // --- COP1 control and coprocessor usability (T-12-006) -----------------
5810
5811    /// The exact instruction n64-systemtest dies on: `CTC1 $rt, $31`, its fourth
5812    /// statement. If this does not work the suite reports nothing at all, and
5813    /// every COP0/TLB test in Sprint 2 is unreachable behind it.
5814    #[test]
5815    fn ctc1_to_fcr31_works_which_is_what_unblocks_the_oracle() {
5816        use crate::cop0::reg;
5817        // CTC1: opcode 0o21, rs = 0o06, rt = GPR, rd = fs.
5818        const fn ctc1(rt: u32, fs: u32) -> u32 {
5819            (0o21 << 26) | (0o06 << 21) | (rt << 16) | (fs << 11)
5820        }
5821        const fn cfc1(rt: u32, fs: u32) -> u32 {
5822            (0o21 << 26) | (0o02 << 21) | (rt << 16) | (fs << 11)
5823        }
5824        //   LUI   $1, 0x0100    ; bit 24 -- flush_denorm_to_zero
5825        //   ORI   $1, $1, 0x800 ; bit 11 -- enable_invalid_operation
5826        //   CTC1  $1, $31
5827        //   CFC1  $2, $31
5828        let prog = alloc::vec![
5829            (0o17 << 26) | (1 << 16) | 0x0100,
5830            (0o15 << 26) | (1 << 21) | (1 << 16) | 0x0800,
5831            ctc1(1, 31),
5832            cfc1(2, 31),
5833        ];
5834        let mut bus = Ram::new(prog);
5835        let mut regs = Regs::new();
5836        let mut p = Pipeline::new();
5837        // CU1 enabled, as IPL3 leaves it (Status = 0x3400_0000).
5838        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
5839        let mut pc = KSEG0_PROG;
5840        for _ in 0..40 {
5841            p.advance(&mut bus, &mut regs, &mut pc);
5842        }
5843
5844        let want = (1u64 << 24) | (1 << 11);
5845        assert_eq!(u64::from(p.cop1.fcsr()), want, "CTC1 reached FCR31");
5846        assert_eq!(regs.read(2), want, "and CFC1 read it back");
5847        assert!(p.cop1.flush_denorm_to_zero());
5848    }
5849
5850    /// With `CU1` clear, a COP1 instruction raises **Coprocessor Unusable** with
5851    /// `Cause.CE = 1` — not Reserved Instruction, which is the natural mistake
5852    /// for an unimplemented encoding.
5853    #[test]
5854    fn a_cop1_instruction_with_cu1_clear_raises_coprocessor_unusable() {
5855        use crate::cop0::reg;
5856        const CTC1: u32 = (0o21 << 26) | (0o06 << 21) | (1 << 16) | (31 << 11);
5857        let mut bus = Ram::new(alloc::vec![CTC1]);
5858        let mut regs = Regs::new();
5859        let mut p = Pipeline::new();
5860        // Kernel mode, but CU1 CLEAR.
5861        p.cop0.set_hardware(reg::STATUS, 0);
5862        let mut pc = KSEG0_PROG;
5863
5864        let mut cycles = 0;
5865        while p.stalled_by() != Some(Interlock::Exception) {
5866            p.advance(&mut bus, &mut regs, &mut pc);
5867            cycles += 1;
5868            assert!(cycles < 16, "no exception raised");
5869        }
5870        assert_eq!(
5871            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5872            crate::exception::exc_code::CPU,
5873            "Coprocessor Unusable, not Reserved Instruction"
5874        );
5875        assert_eq!(
5876            (p.cop0.read(reg::CAUSE) >> 28) & 0b11,
5877            1,
5878            "Cause.CE names the offending unit"
5879        );
5880        assert_eq!(p.cop1.fcsr(), 0, "and the write did not take effect");
5881    }
5882
5883    /// **COP0 is usable from kernel mode regardless of `CU0`.** Otherwise the CPU
5884    /// could not run an exception handler before `Status` had been set up — a
5885    /// chicken-and-egg the hardware does not have.
5886    #[test]
5887    fn cop0_is_usable_in_kernel_mode_even_with_cu0_clear() {
5888        use crate::cop0::reg;
5889        const MTC0: u32 = (0o20 << 26) | (0o04 << 21) | (1 << 16) | ((reg::COMPARE as u32) << 11);
5890        let mut bus = Ram::new(alloc::vec![addiu_zero(1, 0x77), MTC0]);
5891        let mut regs = Regs::new();
5892        let mut p = Pipeline::new();
5893        // KSU = 0 (kernel), CU0 clear, EXL/ERL clear.
5894        p.cop0.set_hardware(reg::STATUS, 0);
5895        let mut pc = KSEG0_PROG;
5896        for _ in 0..24 {
5897            p.advance(&mut bus, &mut regs, &mut pc);
5898        }
5899        assert_eq!(
5900            p.cop0.read(reg::COMPARE),
5901            0x77,
5902            "MTC0 must work in kernel mode without CU0"
5903        );
5904        assert_eq!(
5905            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5906            0,
5907            "and must not have raised"
5908        );
5909    }
5910
5911    /// In **user** mode with `CU0` clear, COP0 *is* unusable — otherwise the
5912    /// kernel-mode exemption would be a blanket bypass rather than a rule.
5913    #[test]
5914    fn cop0_is_unusable_in_user_mode_without_cu0() {
5915        use crate::cop0::reg;
5916        const MTC0: u32 = (0o20 << 26) | (0o04 << 21) | (1 << 16) | ((reg::COMPARE as u32) << 11);
5917        let mut bus = Ram::new(alloc::vec![MTC0]);
5918        let mut regs = Regs::new();
5919        let mut p = Pipeline::new();
5920        // KSU = 2 (user), CU0 clear, EXL/ERL clear.
5921        p.cop0.set_hardware(reg::STATUS, 0b10 << 3);
5922        // The program cannot live in KSEG0 here: that segment does not exist in
5923        // User mode, so the FETCH would raise AdEL and the test would pass on
5924        // the wrong exception. Map page-pair 0 identically instead and run from
5925        // USEG, which is the only place a user program can be.
5926        map(&mut p, 0, 0, 0);
5927        let mut pc = KSEG0_PROG & 0x1FFF_FFFF;
5928
5929        let mut cycles = 0;
5930        while p.stalled_by() != Some(Interlock::Exception) {
5931            p.advance(&mut bus, &mut regs, &mut pc);
5932            cycles += 1;
5933            assert!(cycles < 16, "no exception raised");
5934        }
5935        assert_eq!(
5936            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5937            crate::exception::exc_code::CPU
5938        );
5939        assert_eq!((p.cop0.read(reg::CAUSE) >> 28) & 0b11, 0, "unit 0");
5940    }
5941
5942    /// A 64-bit operation is Reserved in 32-bit User or Supervisor mode, and
5943    /// usable everywhere else.
5944    ///
5945    /// All four rows are needed. Gating on the width bit alone reserves them for
5946    /// a 32-bit *kernel* — which is the mode every N64 boots into, so that
5947    /// mistake breaks everything and would be caught. Gating on the mode alone
5948    /// reserves them for a 64-bit *user* program, which nothing common does, so
5949    /// that mistake would sit unnoticed behind the rows that do pass.
5950    #[test]
5951    fn a_64_bit_operation_is_reserved_only_in_32_bit_non_kernel_mode() {
5952        use crate::cop0::reg;
5953        /// `DADD $2, $1, $1` — SPECIAL funct 0o54.
5954        const DADD: u32 = (1 << 21) | (1 << 16) | (2 << 11) | 0o54;
5955        /// `Status.KSU` = user, and `Status.UX`.
5956        const USER: u64 = 0b10 << 3;
5957        const UX: u64 = 1 << 5;
5958
5959        for (status, want_reserved, why) in [
5960            (0, false, "32-bit kernel: usable"),
5961            (1 << 7, false, "64-bit kernel: usable"),
5962            (USER, true, "32-bit user: reserved"),
5963            (USER | UX, false, "64-bit user: usable"),
5964        ] {
5965            let mut bus = Ram::new(alloc::vec![DADD]);
5966            let mut regs = Regs::new();
5967            let mut p = Pipeline::new();
5968            p.cop0.set_hardware(reg::STATUS, status);
5969            // User mode cannot fetch from KSEG0, so the program runs from USEG
5970            // through an identity mapping in both cases -- keeping the only
5971            // difference between the rows the one under test.
5972            map(&mut p, 0, 0, 0);
5973            let mut pc = KSEG0_PROG & 0x1FFF_FFFF;
5974
5975            let mut raised = false;
5976            for _ in 0..16 {
5977                p.advance(&mut bus, &mut regs, &mut pc);
5978                if p.stalled_by() == Some(Interlock::Exception) {
5979                    raised = true;
5980                    break;
5981                }
5982            }
5983            assert_eq!(raised, want_reserved, "{why}");
5984            if raised {
5985                assert_eq!(
5986                    (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5987                    crate::exception::exc_code::RI,
5988                    "{why}: and it is Reserved Instruction, not something else"
5989                );
5990            }
5991        }
5992    }
5993
5994    /// A `BC1` immediately after a `C.cond.fmt` sees that compare's result.
5995    ///
5996    /// `BC1` resolves in `EX`; the compare commits `FCSR.C` in `WB`. Without a
5997    /// forwarding path the branch samples the **previous** condition, falls
5998    /// through, and nothing anywhere reports an error — the ROM emits exactly
5999    /// this pair with no separating instruction.
6000    ///
6001    /// Both directions are asserted. Testing only the taken case would pass
6002    /// against a bypass that returned `true` unconditionally.
6003    #[test]
6004    fn a_bc1_immediately_after_a_compare_sees_its_result() {
6005        use crate::cop0::reg;
6006        /// `C.EQ.S $f0, $f2` — fmt S, ft = 2, fs = 0, cond = EQ.
6007        const C_EQ: u32 = (0o21 << 26) | (0o20 << 21) | (2 << 16) | 0o62;
6008        /// `BC1T +2` — skips the `ORI` two instructions ahead.
6009        const BC1T: u32 = (0o21 << 26) | (0o10 << 21) | (1 << 16) | 2;
6010        /// `ORI $8, $0, 1` — runs only if the branch was NOT taken.
6011        const ORI: u32 = (0o15 << 26) | (8 << 16) | 1;
6012
6013        for (lo0, equal) in [(0x2222_3333u64, true), (0x4444_5555, false)] {
6014            let mut bus = Ram::new(alloc::vec![C_EQ, BC1T, 0, ORI, 0, 0, 0, 0, 0, 0]);
6015            let mut regs = Regs::new();
6016            let mut p = Pipeline::new();
6017            p.cop0.set_hardware(reg::STATUS, 1 << 29); // CU1
6018            // Upper halves differ deliberately: a single-precision compare must
6019            // ignore them, so a 64-bit comparison would report "not equal" for
6020            // the row that is supposed to match.
6021            p.fpr.write_d(0, true, lo0);
6022            p.fpr.write_d(2, true, 0x1111_1111_2222_3333);
6023            let mut pc = KSEG0_PROG;
6024            for _ in 0..14 {
6025                p.advance(&mut bus, &mut regs, &mut pc);
6026            }
6027            assert_eq!(
6028                regs.read(8) == 0,
6029                equal,
6030                "operands equal = {equal}: the branch must {} the ORI",
6031                if equal { "skip" } else { "run" }
6032            );
6033        }
6034    }
6035
6036    /// An unimplemented COP1 encoding with `CU1` **set** must not raise. Sprint 3
6037    /// then *adds* behavior rather than changing it — and an emulator that
6038    /// raised here would look correct until the FPU landed.
6039    #[test]
6040    fn an_unimplemented_cop1_encoding_does_not_raise_when_cu1_is_set() {
6041        use crate::cop0::reg;
6042        // COP1 `rs = 0o11` -- an unassigned encoding in the coprocessor's own
6043        // opcode space, so it is valid to *fetch* and does nothing.
6044        //
6045        // This test has now outlived three subjects: `ADD.S` until the S/D
6046        // arithmetic landed, `SQRT.S` until T-13-005, then `BC1F` until the
6047        // branch was wired. Each move is the test doing its job -- the point is
6048        // the *unimplemented* path, so the subject follows whatever is still on
6049        // it. When `rs = 0o11` acquires a meaning, move it again rather than
6050        // deleting it.
6051        const UNASSIGNED: u32 = (0o21 << 26) | (0o11 << 21);
6052        assert_eq!(
6053            decode(UNASSIGNED).op,
6054            crate::decode::Op::Cop1Unimplemented,
6055            "valid encoding, not Reserved"
6056        );
6057        let mut bus = Ram::new(alloc::vec![UNASSIGNED]);
6058        let mut regs = Regs::new();
6059        let mut p = Pipeline::new();
6060        p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 set
6061        let mut pc = KSEG0_PROG;
6062        for _ in 0..16 {
6063            p.advance(&mut bus, &mut regs, &mut pc);
6064        }
6065        assert_eq!(
6066            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6067            0,
6068            "no exception with CU1 set"
6069        );
6070    }
6071
6072    /// **`MOV.S` must actually move.** Decoding it is not enough — the failure
6073    /// this pins is a *silent no-op*, which is invisible to every test that
6074    /// only checks for an absent exception.
6075    ///
6076    /// The encoding is the one the correlated capture found in the delay slot
6077    /// of the failing n64-systemtest thunk's `jr $ra` (ledger C-10): with the
6078    /// move doing nothing, the callee's result never reached its caller and
6079    /// ~250 FP results were reported against a register the instruction under
6080    /// test never wrote.
6081    ///
6082    /// The destination is seeded with a value that differs from the source in
6083    /// **both halves**, so neither a no-op nor a half-width copy passes.
6084    #[test]
6085    fn mov_s_copies_the_low_word_and_a_no_op_would_fail_here() {
6086        use crate::cop0::reg;
6087        /// `MOV.S $f0, $f4` — fmt 16, fs 4, fd 0, funct 6.
6088        const MOV_S: u32 = 0x4600_2006;
6089
6090        let mut bus = Ram::new(alloc::vec![MOV_S]);
6091        let mut regs = Regs::new();
6092        let mut p = Pipeline::new();
6093        p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 | FR
6094        p.fpr.write_raw(4, 0x0011_0011_4000_0000); // source: 2.0f
6095        p.fpr.write_raw(0, 0xDEAD_BEEF_1122_3344); // destination: junk
6096
6097        let mut pc = KSEG0_PROG;
6098        for _ in 0..16 {
6099            p.advance(&mut bus, &mut regs, &mut pc);
6100        }
6101
6102        // The WHOLE register, not just the low word. `MOV.S` is a 64-bit bit
6103        // move (C-10), so checking only the low half passes against a formatted
6104        // half-copy -- which is what the destination's junk upper word is here
6105        // to catch.
6106        assert_eq!(
6107            p.fpr.read_raw(0),
6108            0x0011_0011_4000_0000,
6109            "MOV.S copies fs entire, upper half included"
6110        );
6111        assert_eq!(
6112            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6113            0,
6114            "MOV.S raises nothing"
6115        );
6116    }
6117
6118    /// **The documented FPU rates are actually charged.**
6119    ///
6120    /// `fpu::delay_cycles` transcribes UM Table 7-14 and its own test asserts
6121    /// the numbers, but a table nothing consults is inert — the failure mode
6122    /// here is a correct table that is simply never called. So this drives real
6123    /// instructions and reads the interlock back.
6124    ///
6125    /// The two cases are chosen to differ: `MUL.D` is 8 cycles and `MOV.S` is 1,
6126    /// i.e. no stall at all. A blanket stall on every COP1 op would pass a
6127    /// `MUL.D`-only test and fail this one.
6128    #[test]
6129    fn the_documented_fpu_rates_are_charged_as_stalls() {
6130        use crate::cop0::reg;
6131        /// fmt in bits 25:21, fs 4, ft 2, fd 0.
6132        const fn fp(fmt: u32, funct: u32) -> u32 {
6133            (0o21 << 26) | (fmt << 21) | (2 << 16) | (4 << 11) | funct
6134        }
6135        // MUL.D (fmt 17, funct 2) = 8 PCycles; MOV.S (fmt 16, funct 6) = 1, so
6136        // it must add nothing.
6137        for (word, want) in [(fp(0o21, 2), Some(Interlock::Mci)), (fp(0o20, 6), None)] {
6138            let mut bus = Ram::new(alloc::vec![word]);
6139            let mut regs = Regs::new();
6140            let mut p = Pipeline::new();
6141            p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6142            p.fpr.write_d(4, true, 2.0f64.to_bits());
6143            p.fpr.write_d(2, true, 3.0f64.to_bits());
6144            let mut pc = KSEG0_PROG;
6145
6146            // Run until the instruction reaches WB, where COP1 arithmetic
6147            // executes and the delay is charged.
6148            let mut seen = None;
6149            for _ in 0..8 {
6150                p.advance(&mut bus, &mut regs, &mut pc);
6151                if p.stalled_by() == Some(Interlock::Mci) {
6152                    seen = Some(Interlock::Mci);
6153                    break;
6154                }
6155            }
6156            assert_eq!(
6157                seen, want,
6158                "word {word:#010x}: a multi-cycle FPU op must stall and a \
6159                 single-cycle one must not"
6160            );
6161        }
6162    }
6163
6164    /// `NEG.S` and `ABS.S` share the arm `MOV.S` was missing from, and are just
6165    /// as silent when absent: both leave a plausible-looking value behind.
6166    #[test]
6167    fn neg_s_and_abs_s_execute_rather_than_no_op() {
6168        use crate::cop0::reg;
6169        /// fmt 16, fs 4, fd 0.
6170        const fn fp(funct: u32) -> u32 {
6171            (0o21 << 26) | (0o20 << 21) | (4 << 11) | funct
6172        }
6173
6174        for (funct, input, want) in [
6175            (7u32, 0x4000_0000u32, 0xC000_0000u32), // NEG.S: 2.0 -> -2.0
6176            (5, 0xC000_0000, 0x4000_0000),          // ABS.S: -2.0 -> 2.0
6177        ] {
6178            let mut bus = Ram::new(alloc::vec![fp(funct)]);
6179            let mut regs = Regs::new();
6180            let mut p = Pipeline::new();
6181            p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6182            p.fpr.write_s(4, true, input);
6183            p.fpr.write_s(0, true, 0x1122_3344);
6184            let mut pc = KSEG0_PROG;
6185            for _ in 0..16 {
6186                p.advance(&mut bus, &mut regs, &mut pc);
6187            }
6188            assert_eq!(p.fpr.read_s(0, true), want, "funct {funct} did not execute");
6189        }
6190    }
6191
6192    // --- Enabled FP traps (T-13-002) ----------------------------------------
6193
6194    /// `ADD.S $f4, $f0, $f2` — the encoding the FP-trap tests drive.
6195    const ADD_S_F4_F0_F2: u32 = 0x4602_0100;
6196    /// `FCSR.Enable` for Invalid Operation (bit 11).
6197    const ENABLE_INVALID: u32 = 1 << 11;
6198    /// `FCSR.Cause` for Invalid Operation (bit 16).
6199    const CAUSE_INVALID: u32 = 1 << 16;
6200    /// `FCSR.Flags` (sticky) for Invalid Operation (bit 6).
6201    const FLAG_INVALID: u32 = 1 << 6;
6202
6203    /// Run `ADD.S $f4, $f0, $f2` on `inf + (-inf)` — an Invalid Operation —
6204    /// with `FCSR` preloaded, and report what the machine ended up in.
6205    fn run_invalid_add_s(fcsr: u32) -> (Pipeline, u64) {
6206        use crate::cop0::reg;
6207        let mut bus = Ram::new(alloc::vec![ADD_S_F4_F0_F2]);
6208        let mut regs = Regs::new();
6209        let mut p = Pipeline::new();
6210        p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 | FR
6211        p.cop1.ctc1(31, fcsr);
6212        p.fpr.write_s(0, true, 0x7F80_0000); // +inf
6213        p.fpr.write_s(2, true, 0xFF80_0000); // -inf
6214        p.fpr.write_raw(4, 0x1122_3344_5566_7788); // untouched-if-trapped marker
6215        let mut pc = KSEG0_PROG;
6216        for _ in 0..24 {
6217            p.advance(&mut bus, &mut regs, &mut pc);
6218        }
6219        let code = (p.cop0.read(reg::CAUSE) >> 2) & 0x1F;
6220        (p, code)
6221    }
6222
6223    /// With the Invalid enable **clear**, the operation completes: `fd` is
6224    /// written, both `Cause` and the sticky `Flags` record it, and nothing
6225    /// raises. This is the control for the trap test below — without it, a
6226    /// pipeline that raised on *every* invalid operation would pass that test.
6227    #[test]
6228    fn a_masked_fp_condition_completes_and_sets_both_cause_and_flags() {
6229        let (p, code) = run_invalid_add_s(0);
6230        assert_eq!(code, 0, "masked: no exception");
6231        assert_ne!(
6232            p.fpr.read_s(4, true),
6233            0x5566_7788,
6234            "fd must be written when no trap is taken"
6235        );
6236        let fcsr = p.cop1.fcsr();
6237        assert_ne!(fcsr & CAUSE_INVALID, 0, "Cause.V set");
6238        assert_ne!(fcsr & FLAG_INVALID, 0, "sticky Flags.V set");
6239    }
6240
6241    /// With the enable **set**, the same operation traps: `ExcCode` is 15, `fd`
6242    /// keeps its old value, and `Cause` records the condition.
6243    #[test]
6244    fn an_enabled_fp_condition_raises_and_leaves_the_destination_alone() {
6245        use crate::cop0::reg;
6246        let (p, code) = run_invalid_add_s(ENABLE_INVALID);
6247        assert_eq!(code, crate::exception::exc_code::FPE, "ExcCode 15 (FPE)");
6248        assert_eq!(
6249            p.fpr.read_raw(4),
6250            0x1122_3344_5566_7788,
6251            "a trapped operation must not write fd"
6252        );
6253        assert_ne!(p.cop1.fcsr() & CAUSE_INVALID, 0, "Cause.V set");
6254        assert_eq!(
6255            (p.cop0.read(reg::CAUSE) >> 28) & 0b11,
6256            0,
6257            "Cause.CE is 0 for an FP exception, not the coprocessor number"
6258        );
6259    }
6260
6261    /// **The sticky `Flags` field is NOT updated on a trap** — only `Cause` is.
6262    ///
6263    /// Split from the test above deliberately. Both come from the same
6264    /// `Flags::to_fcsr_bits` value, so writing the whole thing on the trap path
6265    /// is the natural implementation and is wrong; it passes every assertion
6266    /// about the exception itself and only shows up when `FCSR` is read back,
6267    /// which is exactly what n64-systemtest does.
6268    #[test]
6269    fn a_trapped_operation_does_not_accumulate_into_the_sticky_flags() {
6270        let (p, _) = run_invalid_add_s(ENABLE_INVALID);
6271        assert_eq!(
6272            p.cop1.fcsr() & FLAG_INVALID,
6273            0,
6274            "Flags must be left alone when the trap is taken"
6275        );
6276    }
6277
6278    /// A trapped FP operation does not retire, so it must not tick `Random`.
6279    ///
6280    /// `Random` is decremented in the retirement tail of `WB`, which is also
6281    /// where the FP write-back happens — so an implementation that raises the
6282    /// exception but falls through keeps counting an instruction that never
6283    /// completed.
6284    ///
6285    /// **Asserted on the trap cycle specifically.** Comparing total `retired`
6286    /// between a trapping and a non-trapping run was tried first and proved
6287    /// nothing: the trap flushes the pipe and redirects `PC`, so the totals
6288    /// differ over a fixed cycle budget whether or not the trapping instruction
6289    /// itself retired. That version passed with the fix removed.
6290    #[test]
6291    fn a_trapped_fp_operation_does_not_retire() {
6292        use crate::cop0::reg;
6293        let mut bus = Ram::new(alloc::vec![ADD_S_F4_F0_F2]);
6294        let mut regs = Regs::new();
6295        let mut p = Pipeline::new();
6296        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6297        p.cop1.ctc1(31, ENABLE_INVALID);
6298        p.fpr.write_s(0, true, 0x7F80_0000);
6299        p.fpr.write_s(2, true, 0xFF80_0000);
6300
6301        let mut pc = KSEG0_PROG;
6302        let mut saw_trap = false;
6303        for _ in 0..24 {
6304            let retired_before = p.retired;
6305            let random_before = p.cop0.read(reg::RANDOM);
6306            p.advance(&mut bus, &mut regs, &mut pc);
6307            let code = (p.cop0.read(reg::CAUSE) >> 2) & 0x1F;
6308            if !saw_trap && code == crate::exception::exc_code::FPE {
6309                saw_trap = true;
6310                assert_eq!(
6311                    p.retired, retired_before,
6312                    "the trapping instruction must not retire on the trap cycle"
6313                );
6314                assert_eq!(
6315                    p.cop0.read(reg::RANDOM),
6316                    random_before,
6317                    "and must not tick Random"
6318                );
6319            }
6320        }
6321        assert!(saw_trap, "no FP trap was taken -- the test proved nothing");
6322    }
6323
6324    /// **A `MOV.S` must not disturb `FCSR.Cause`.**
6325    ///
6326    /// This is a regression test for a real defect. `MOV`/`ABS`/`NEG` were
6327    /// first written to clear the `Cause` field, on no evidence — and because
6328    /// the compiler emits `MOV.fmt` to move an FP return value, a `MOV` almost
6329    /// always sits between an arithmetic operation and the `CFC1` that reads
6330    /// its result. It therefore erased the very bits the program was about to
6331    /// inspect, costing 112 n64-systemtest assertions.
6332    ///
6333    /// The signature was distinctive and worth recording: the suite reported
6334    /// `flags: inexact` with `causes: ""` — the sticky half surviving and the
6335    /// per-operation half gone. That shape means *a later instruction
6336    /// overwrote it*, not that the flag was never raised.
6337    ///
6338    /// The sequence below is exactly the shape a compiled FP call has:
6339    /// arithmetic, then a move of the result.
6340    #[test]
6341    fn a_following_mov_s_leaves_the_cause_field_of_a_previous_operation_intact() {
6342        use crate::cop0::reg;
6343        /// `ADD.S $f4, $f0, $f2` then `MOV.S $f6, $f4`.
6344        const MOV_S_F6_F4: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 11) | (6 << 6) | 6;
6345        /// `FCSR.Cause` for Inexact (bit 12).
6346        const CAUSE_INEXACT: u32 = 1 << 12;
6347
6348        let mut bus = Ram::new(alloc::vec![ADD_S_F4_F0_F2, MOV_S_F6_F4]);
6349        let mut regs = Regs::new();
6350        let mut p = Pipeline::new();
6351        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6352        // f32::MAX + 1.0 -- the value is unchanged and the operation is
6353        // inexact, which is the n64-systemtest case from ledger C-11.
6354        p.fpr.write_s(0, true, f32::MAX.to_bits());
6355        p.fpr.write_s(2, true, 1.0f32.to_bits());
6356
6357        let mut pc = KSEG0_PROG;
6358        for _ in 0..8 {
6359            p.advance(&mut bus, &mut regs, &mut pc);
6360        }
6361        assert_ne!(
6362            p.cop1.fcsr() & CAUSE_INEXACT,
6363            0,
6364            "ADD.S must raise Cause.Inexact in the first place"
6365        );
6366        for _ in 0..16 {
6367            p.advance(&mut bus, &mut regs, &mut pc);
6368        }
6369        assert_eq!(
6370            p.fpr.read_s(6, true),
6371            p.fpr.read_s(4, true),
6372            "the MOV.S did run"
6373        );
6374        assert_ne!(
6375            p.cop1.fcsr() & CAUSE_INEXACT,
6376            0,
6377            "and must not have cleared the ADD.S's Cause"
6378        );
6379    }
6380
6381    /// **A later COP1 operation clears a stale `Cause.E` (bit 17).**
6382    ///
6383    /// `Cause` is bits **17:12** and is replaced wholesale by each operation.
6384    /// The mask here originally covered only 16:12, so the unimplemented-
6385    /// operation bit — which has no `Enable` and no sticky `Flags` twin, and so
6386    /// is only ever cleared by that mask — stayed set forever once raised.
6387    /// Software reading `FCSR` after a perfectly good conversion would still
6388    /// see the previous failure.
6389    ///
6390    /// Found by a review bot, not by this suite, which had no case that raised
6391    /// bit 17 and then ran another COP1 instruction.
6392    #[test]
6393    fn a_later_operation_clears_a_stale_unimplemented_cause() {
6394        use crate::cop0::reg;
6395        /// `ADD.S $f4, $f0, $f2` — an ordinary, entirely successful operation.
6396        const ADD_S: u32 = 0x4602_0100;
6397        /// `FCSR.Cause.E`, bit 17.
6398        const CAUSE_E: u32 = 1 << 17;
6399
6400        let mut bus = Ram::new(alloc::vec![ADD_S]);
6401        let mut regs = Regs::new();
6402        let mut p = Pipeline::new();
6403        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6404        // Pre-set the bit, as a previous unimplemented operation would have.
6405        p.cop1.ctc1(31, CAUSE_E);
6406        p.fpr.write_s(0, true, 1.0f32.to_bits());
6407        p.fpr.write_s(2, true, 2.0f32.to_bits());
6408
6409        let mut pc = KSEG0_PROG;
6410        for _ in 0..16 {
6411            p.advance(&mut bus, &mut regs, &mut pc);
6412        }
6413        assert_eq!(p.fpr.read_s(4, true), 3.0f32.to_bits(), "the ADD.S ran");
6414        assert_eq!(
6415            p.cop1.fcsr() & CAUSE_E,
6416            0,
6417            "a successful operation must clear the whole Cause field"
6418        );
6419    }
6420
6421    /// `C.cond.fmt` writes `FCSR.C` and **no FPR at all**.
6422    ///
6423    /// Both halves matter. A compare that also wrote `fd` would corrupt a
6424    /// register the program never named, and one that computed the right
6425    /// condition without storing it leaves every dependent branch wrong.
6426    #[test]
6427    fn a_compare_writes_the_fcsr_condition_and_leaves_the_registers_alone() {
6428        use crate::cop0::reg;
6429        /// `FCSR.C`, bit 23.
6430        const FCSR_C: u32 = 1 << 23;
6431        /// `C.EQ.S $f0, $f2` — fmt 16, funct 0o62 (cond 2 = EQ).
6432        const C_EQ_S: u32 = (0o21 << 26) | (0o20 << 21) | (2 << 16) | 0o62;
6433
6434        for (a, b, want) in [(1.0f32, 1.0f32, true), (1.0, 2.0, false)] {
6435            let mut bus = Ram::new(alloc::vec![C_EQ_S]);
6436            let mut regs = Regs::new();
6437            let mut p = Pipeline::new();
6438            p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6439            // Start with the condition at the OPPOSITE of the expected result,
6440            // so "wrote the right value" is distinguishable from "left it".
6441            p.cop1.ctc1(31, if want { 0 } else { FCSR_C });
6442            p.fpr.write_s(0, true, a.to_bits());
6443            p.fpr.write_s(2, true, b.to_bits());
6444            p.fpr.write_raw(4, 0xDEAD_BEEF_1122_3344);
6445
6446            let mut pc = KSEG0_PROG;
6447            for _ in 0..16 {
6448                p.advance(&mut bus, &mut regs, &mut pc);
6449            }
6450            assert_eq!(p.cop1.fcsr() & FCSR_C != 0, want, "{a} == {b}");
6451            assert_eq!(
6452                p.fpr.read_raw(4),
6453                0xDEAD_BEEF_1122_3344,
6454                "a compare must not write an FPR"
6455            );
6456        }
6457    }
6458
6459    /// **`TRUNC.W.S` takes its rounding mode from the OPCODE, not `FCSR.RM`.**
6460    ///
6461    /// This is the entire difference between the `ROUND`/`TRUNC`/`CEIL`/`FLOOR`
6462    /// family and `CVT.W`/`CVT.L`, and it is invisible whenever `RM` happens to
6463    /// agree with the opcode. So `FCSR.RM` is set to round-to-nearest and the
6464    /// input chosen where nearest and truncate disagree: `-1.5` truncates to
6465    /// `-1` and rounds to `-2`.
6466    ///
6467    /// `CVT.W.S` on the same input under the same `FCSR` must give `-2`,
6468    /// proving the two families really are wired differently rather than both
6469    /// happening to truncate.
6470    #[test]
6471    fn the_fixed_mode_conversions_ignore_fcsr_rm_and_cvt_w_honors_it() {
6472        use crate::cop0::reg;
6473        /// `TRUNC.W.S $f4, $f0` — fmt 16, `fs` 0 (the zero shift is elided),
6474        /// `fd` 4, funct 0o15.
6475        const TRUNC_W_S: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 6) | 0o15;
6476        /// `CVT.W.S $f4, $f0` — fmt 16, `fs` 0, `fd` 4, funct 0o44.
6477        const CVT_W_S: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 6) | 0o44;
6478
6479        for (word, want) in [(TRUNC_W_S, -1i32), (CVT_W_S, -2)] {
6480            let mut bus = Ram::new(alloc::vec![word]);
6481            let mut regs = Regs::new();
6482            let mut p = Pipeline::new();
6483            p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6484            p.cop1.ctc1(31, 0); // RM = 0, round to nearest even
6485            p.fpr.write_s(0, true, (-1.5f32).to_bits());
6486
6487            let mut pc = KSEG0_PROG;
6488            for _ in 0..16 {
6489                p.advance(&mut bus, &mut regs, &mut pc);
6490            }
6491            #[allow(clippy::cast_possible_wrap)] // reading the word back as signed
6492            let got = p.fpr.read_s(4, true) as i32;
6493            assert_eq!(got, want, "instruction {word:#010X} on -1.5");
6494        }
6495    }
6496
6497    /// `CVT.S.W` reads its source as a **32-bit integer**, which is a different
6498    /// format carried in the same `fmt` field.
6499    ///
6500    /// A decoder that admits only formats 16/17 leaves every integer-to-float
6501    /// conversion a silent no-op, and `fd` keeps whatever it had — which looks
6502    /// exactly like a plausible float.
6503    #[test]
6504    fn cvt_s_w_converts_an_integer_source() {
6505        use crate::cop0::reg;
6506        /// `CVT.S.W $f4, $f0` — fmt 20 (`.W`), `fs` 0, `fd` 4, funct 0o40.
6507        const CVT_S_W: u32 = (0o21 << 26) | (0o24 << 21) | (4 << 6) | 0o40;
6508
6509        let mut bus = Ram::new(alloc::vec![CVT_S_W]);
6510        let mut regs = Regs::new();
6511        let mut p = Pipeline::new();
6512        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6513        p.fpr.write_s(0, true, 12345u32);
6514        p.fpr.write_s(4, true, 0x1122_3344);
6515
6516        let mut pc = KSEG0_PROG;
6517        for _ in 0..16 {
6518            p.advance(&mut bus, &mut regs, &mut pc);
6519        }
6520        // Compared as BITS, not as a float: 12345.0 is exactly representable,
6521        // so this is the stricter check and it also catches a wrong-signed
6522        // zero or a NaN payload that float equality would accept.
6523        assert_eq!(
6524            p.fpr.read_s(4, true),
6525            12345.0f32.to_bits(),
6526            "the integer source must be converted, not reinterpreted"
6527        );
6528    }
6529
6530    /// **`CTC1` can raise an FP exception by itself.** Writing `FCSR` with a
6531    /// Cause bit whose Enable is also set meets the trap condition
6532    /// immediately — no arithmetic has to run.
6533    ///
6534    /// The instruction must also not retire, exactly as a trapping arithmetic
6535    /// operation does not.
6536    #[test]
6537    fn ctc1_raises_when_it_writes_a_cause_bit_that_is_enabled() {
6538        use crate::cop0::reg;
6539        /// `CTC1 $1, $f31` — rs 0o06, rt 1, fs 31.
6540        const CTC1_F31: u32 = (0o21 << 26) | (0o06 << 21) | (1 << 16) | (31 << 11);
6541        /// `Cause.overflow` (bit 14) with `Enable.overflow` (bit 9).
6542        const OVERFLOW_ARMED: u64 = (1 << 14) | (1 << 9);
6543
6544        let mut bus = Ram::new(alloc::vec![CTC1_F31]);
6545        let mut regs = Regs::new();
6546        let mut p = Pipeline::new();
6547        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6548        regs.write(1, OVERFLOW_ARMED);
6549        let mut pc = KSEG0_PROG;
6550        for _ in 0..24 {
6551            p.advance(&mut bus, &mut regs, &mut pc);
6552        }
6553        assert_eq!(
6554            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6555            crate::exception::exc_code::FPE,
6556            "the CTC1 itself must raise"
6557        );
6558        assert_eq!(
6559            u64::from(p.cop1.fcsr()),
6560            OVERFLOW_ARMED,
6561            "and the written value stands"
6562        );
6563    }
6564
6565    /// The *enable* half is load-bearing: the same Cause bit with its Enable
6566    /// clear must NOT raise. Without this, an implementation that trapped on
6567    /// any non-zero Cause would pass the test above.
6568    #[test]
6569    fn ctc1_does_not_raise_when_the_matching_enable_is_clear() {
6570        use crate::cop0::reg;
6571        const CTC1_F31: u32 = (0o21 << 26) | (0o06 << 21) | (1 << 16) | (31 << 11);
6572
6573        let mut bus = Ram::new(alloc::vec![CTC1_F31]);
6574        let mut regs = Regs::new();
6575        let mut p = Pipeline::new();
6576        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6577        // Cause.overflow set, Enable.overflow clear; an unrelated enable set.
6578        regs.write(1, (1 << 14) | (1 << 11));
6579        let mut pc = KSEG0_PROG;
6580        for _ in 0..24 {
6581            p.advance(&mut bus, &mut regs, &mut pc);
6582        }
6583        assert_eq!(
6584            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6585            0,
6586            "a Cause bit whose Enable is clear must not trap"
6587        );
6588    }
6589
6590    /// **`DCFC1`/`DCTC1` and `DCFC2`/`DCTC2` decline in DIFFERENT ways.**
6591    ///
6592    /// The encodings are structurally identical — the doubleword control moves
6593    /// of their respective coprocessors — and it is tempting to give them one
6594    /// behavior. COP1's raise a *floating-point* exception with `FCSR.Cause`
6595    /// set to unimplemented-operation; COP2's raise *Reserved Instruction* with
6596    /// `Cause.CE = 2` and do not touch `FCSR` at all.
6597    ///
6598    /// Both are tested here together precisely because treating them uniformly
6599    /// is the natural mistake.
6600    #[test]
6601    fn the_doubleword_control_moves_decline_differently_per_coprocessor() {
6602        use crate::cop0::reg;
6603        /// `DCFC1 $1, $f0` — COP1, rs 0o03.
6604        const DCFC1: u32 = (0o21 << 26) | (0o03 << 21) | (1 << 16);
6605        /// `DCFC2 $1, $0` — COP2, rs 0o03.
6606        const DCFC2: u32 = (0o22 << 26) | (0o03 << 21) | (1 << 16);
6607
6608        // COP1: floating-point exception, FCSR.Cause = unimplemented ONLY.
6609        let mut bus = Ram::new(alloc::vec![DCFC1]);
6610        let mut regs = Regs::new();
6611        let mut p = Pipeline::new();
6612        p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 set
6613        // Pre-load unrelated cause bits: they must be cleared, not merged.
6614        p.cop1.ctc1(31, 0x0001_F000);
6615        let mut pc = KSEG0_PROG;
6616        for _ in 0..24 {
6617            p.advance(&mut bus, &mut regs, &mut pc);
6618        }
6619        assert_eq!(
6620            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6621            crate::exception::exc_code::FPE,
6622            "DCFC1 raises FPE"
6623        );
6624        assert_eq!(
6625            p.cop1.fcsr() & (0x3F << 12),
6626            crate::fpu::CAUSE_UNIMPLEMENTED,
6627            "and Cause is ONLY the unimplemented bit"
6628        );
6629
6630        // COP2: Reserved Instruction, and Cause.CE names the coprocessor.
6631        let mut bus = Ram::new(alloc::vec![DCFC2]);
6632        let mut regs = Regs::new();
6633        let mut p = Pipeline::new();
6634        // CU2 set (bit 30) as well as CU1, so this is not an unusable fault.
6635        p.cop0.set_hardware(reg::STATUS, 0x7400_0000);
6636        let mut pc = KSEG0_PROG;
6637        for _ in 0..24 {
6638            p.advance(&mut bus, &mut regs, &mut pc);
6639        }
6640        assert_eq!(
6641            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6642            crate::exception::exc_code::RI,
6643            "DCFC2 raises Reserved Instruction, not FPE"
6644        );
6645        assert_eq!(
6646            (p.cop0.read(reg::CAUSE) >> 28) & 0b11,
6647            2,
6648            "and Cause.CE names COP2 -- a plain RI would leave it zero"
6649        );
6650    }
6651
6652    /// **A `JAL` inside another jump's delay slot links to the OUTER target
6653    /// + 4**, not to its own `pc + 8`.
6654    ///
6655    /// Its own delay slot never runs — the outer jump already redirected — so
6656    /// the instruction after it is the outer target. n64-systemtest states the
6657    /// rule in its assertion text: *"JAL in delay slot writes target address+4
6658    /// of original jump into delay slot"*.
6659    ///
6660    /// The ordinary case is asserted alongside it, because the two share one
6661    /// implementation and a test of only the nested case would pass with the
6662    /// link hard-wired to `next_pc` in a way that broke normal jumps.
6663    #[test]
6664    fn a_jal_in_a_delay_slot_links_past_the_outer_target() {
6665        /// `JAL <target>` — the target is a word index within the 256 MB region.
6666        const fn jal(target_word: u32) -> u32 {
6667            (0o03 << 26) | target_word
6668        }
6669        /// `J <target>`.
6670        const fn j(target_word: u32) -> u32 {
6671            (0o02 << 26) | target_word
6672        }
6673
6674        // Ordinary: JAL at KSEG0_PROG, delay slot after it, link = pc + 8.
6675        {
6676            let outer = u32::try_from(KSEG0_PROG & 0x0FFF_FFFF).unwrap() >> 2;
6677            let mut bus = Ram::new(alloc::vec![jal(outer + 8), 0, 0, 0, 0, 0, 0, 0, 0, 0]);
6678            let mut regs = Regs::new();
6679            let mut p = Pipeline::new();
6680            let mut pc = KSEG0_PROG;
6681            for _ in 0..24 {
6682                p.advance(&mut bus, &mut regs, &mut pc);
6683            }
6684            assert_eq!(regs.read(31), KSEG0_PROG + 8, "an ordinary JAL links pc+8");
6685        }
6686
6687        // Nested: J to T, with a JAL in its delay slot. The JAL must link to
6688        // T + 4, NOT to its own address + 8.
6689        {
6690            let base = u32::try_from(KSEG0_PROG & 0x0FFF_FFFF).unwrap() >> 2;
6691            // T is eight words along; the JAL sits in the J's delay slot.
6692            let t_word = base + 8;
6693            let prog = alloc::vec![j(t_word), jal(base + 4), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
6694            let mut bus = Ram::new(prog);
6695            let mut regs = Regs::new();
6696            let mut p = Pipeline::new();
6697            let mut pc = KSEG0_PROG;
6698            for _ in 0..24 {
6699                p.advance(&mut bus, &mut regs, &mut pc);
6700            }
6701            let outer_target = KSEG0_PROG + 8 * 4;
6702            assert_eq!(
6703                regs.read(31),
6704                outer_target + 4,
6705                "the nested JAL links to the OUTER target + 4"
6706            );
6707            assert_ne!(
6708                regs.read(31),
6709                KSEG0_PROG + 4 + 8,
6710                "and specifically NOT to its own pc + 8"
6711            );
6712        }
6713    }
6714
6715    /// **COP2 is one 64-bit latch, not a register file.**
6716    ///
6717    /// The register index is ignored entirely: n64-systemtest writes with one
6718    /// index and reads back with several others — including 30 and 31 — and
6719    /// gets the same value every time. `MTC2` writes all 64 bits despite being
6720    /// nominally a 32-bit move; `MFC2` returns the low half sign-extended and
6721    /// `DMFC2` the whole thing.
6722    ///
6723    /// The index-independence is the assertion that matters: a real 32-entry
6724    /// register file passes a write-then-read-same-index test perfectly.
6725    #[test]
6726    fn cop2_is_a_single_latch_whose_register_index_is_ignored() {
6727        use crate::cop0::reg;
6728        /// `MTC2 $1, $5` — COP2 rs 0o04, rt 1, rd 5.
6729        const MTC2_R1_TO_5: u32 = (0o22 << 26) | (0o04 << 21) | (1 << 16) | (5 << 11);
6730        /// `DMFC2 $2, $30` — rs 0o01, rt 2, rd 30. A DIFFERENT index.
6731        const DMFC2_30_TO_R2: u32 = (0o22 << 26) | (0o01 << 21) | (2 << 16) | (30 << 11);
6732        /// `MFC2 $3, $31` — rs 0o00 (so no `rs` term), rt 3, rd 31. Another
6733        /// different index again.
6734        const MFC2_31_TO_R3: u32 = (0o22 << 26) | (3 << 16) | (31 << 11);
6735
6736        let mut bus = Ram::new(alloc::vec![MTC2_R1_TO_5, DMFC2_30_TO_R2, MFC2_31_TO_R3]);
6737        let mut regs = Regs::new();
6738        let mut p = Pipeline::new();
6739        // CU2 usable, or these fault before reaching the latch.
6740        p.cop0.set_hardware(reg::STATUS, 0x5000_0000);
6741        regs.write(1, 0x0123_4567_89AB_CDEF);
6742
6743        let mut pc = KSEG0_PROG;
6744        for _ in 0..32 {
6745            p.advance(&mut bus, &mut regs, &mut pc);
6746        }
6747        assert_eq!(
6748            regs.read(2),
6749            0x0123_4567_89AB_CDEF,
6750            "DMFC2 from a DIFFERENT index reads all 64 bits MTC2 wrote"
6751        );
6752        assert_eq!(
6753            regs.read(3),
6754            0xFFFF_FFFF_89AB_CDEF,
6755            "MFC2 from another index reads the low half, sign-extended"
6756        );
6757    }
6758
6759    // --- Unimplemented operation on subnormals (T-13-004) -------------------
6760
6761    /// `ADD.S $f4, $f0, $f2`.
6762    const ADD_S_SUB: u32 = 0x4602_0100;
6763    /// `FCSR.Cause.E`, bit 17 — unmaskable unimplemented operation.
6764    const CAUSE_E: u32 = 1 << 17;
6765    /// `FCSR.FS`, bit 24 — flush denormals to zero.
6766    const FCSR_FS: u32 = 1 << 24;
6767
6768    /// Run `ADD.S $f4, $f0, $f2` with the given operands and `FCSR`.
6769    fn run_add_s(fcsr: u32, a: u32, b: u32) -> Pipeline {
6770        use crate::cop0::reg;
6771        let mut bus = Ram::new(alloc::vec![ADD_S_SUB]);
6772        let mut regs = Regs::new();
6773        let mut p = Pipeline::new();
6774        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6775        p.cop1.ctc1(31, fcsr);
6776        p.fpr.write_s(0, true, a);
6777        p.fpr.write_s(2, true, b);
6778        p.fpr.write_raw(4, 0x1122_3344_5566_7788);
6779        let mut pc = KSEG0_PROG;
6780        for _ in 0..24 {
6781            p.advance(&mut bus, &mut regs, &mut pc);
6782        }
6783        p
6784    }
6785
6786    /// A **subnormal operand** raises unimplemented operation before the
6787    /// arithmetic is attempted — even with `FS` set, which does not rescue it.
6788    ///
6789    /// `FCSR` must end up with bit 17 and *nothing else*: no Invalid, no
6790    /// Inexact, and the sticky `Flags` untouched.
6791    #[test]
6792    fn a_subnormal_operand_raises_unimplemented_operation() {
6793        use crate::cop0::reg;
6794        let subnormal = 1u32; // the smallest positive subnormal
6795        for fcsr in [0, FCSR_FS] {
6796            let p = run_add_s(fcsr, subnormal, 2.0f32.to_bits());
6797            assert_eq!(
6798                (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6799                crate::exception::exc_code::FPE,
6800                "fcsr={fcsr:#X}"
6801            );
6802            assert_ne!(p.cop1.fcsr() & CAUSE_E, 0, "Cause.E set");
6803            assert_eq!(
6804                p.cop1.fcsr() & !(CAUSE_E | FCSR_FS),
6805                0,
6806                "bit 17 and nothing else -- no flags, no other causes"
6807            );
6808            assert_eq!(
6809                p.fpr.read_raw(4),
6810                0x1122_3344_5566_7788,
6811                "fd must be untouched"
6812            );
6813        }
6814    }
6815
6816    /// A **subnormal result** with `FS` clear is equally refused. The operands
6817    /// here are both normal, so this is the result path and not the operand
6818    /// one — the two are separate checks and a test using a subnormal input
6819    /// would pass with the result check deleted.
6820    #[test]
6821    fn a_subnormal_result_raises_unimplemented_when_fs_is_clear() {
6822        use crate::cop0::reg;
6823        let a = 1.528_510_4e-37f32;
6824        let b = -1.539_154_3e-37f32;
6825        assert!(!crate::fpu::is_subnormal_f32(a), "operands are normal");
6826        assert!(!crate::fpu::is_subnormal_f32(b));
6827
6828        let p = run_add_s(0, a.to_bits(), b.to_bits());
6829        assert_eq!(
6830            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6831            crate::exception::exc_code::FPE
6832        );
6833        assert_ne!(p.cop1.fcsr() & CAUSE_E, 0);
6834    }
6835
6836    /// **An underflow that rounds past the subnormal grid to ZERO is refused
6837    /// too**, not just one that lands on a subnormal.
6838    ///
6839    /// `MIN_POSITIVE * MIN_POSITIVE` is about `1.4e-76`, far below `f32`'s
6840    /// smallest subnormal, so the rounded result is plain zero and
6841    /// `is_subnormal` is false for it. Testing only that condition let every
6842    /// such case through silently — worth 22 oracle assertions.
6843    ///
6844    /// The converse matters as much and is covered by
6845    /// `a_subnormal_result_raises_unimplemented_when_fs_is_clear`: IEEE signals
6846    /// underflow only when tiny **and inexact**, so an *exact* subnormal has
6847    /// `underflow` clear. Neither condition implies the other; the
6848    /// implementation needs both.
6849    #[test]
6850    fn an_underflow_that_reaches_zero_is_refused_as_well() {
6851        use crate::cop0::reg;
6852        /// `MUL.S $f4, $f0, $f2` — fmt 16, funct 2.
6853        const MUL_S: u32 = (0o21 << 26) | (0o20 << 21) | (2 << 16) | (4 << 6) | 2;
6854
6855        let mut bus = Ram::new(alloc::vec![MUL_S]);
6856        let mut regs = Regs::new();
6857        let mut p = Pipeline::new();
6858        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6859        p.fpr.write_s(0, true, f32::MIN_POSITIVE.to_bits());
6860        p.fpr.write_s(2, true, f32::MIN_POSITIVE.to_bits());
6861
6862        let mut pc = KSEG0_PROG;
6863        for _ in 0..24 {
6864            p.advance(&mut bus, &mut regs, &mut pc);
6865        }
6866        assert_ne!(
6867            p.cop1.fcsr() & CAUSE_E,
6868            0,
6869            "an underflow to zero must raise unimplemented with FS clear"
6870        );
6871    }
6872
6873    /// With `FS` set, the same operation **flushes** — and where it flushes to
6874    /// depends on the rounding mode. These are n64-systemtest's own vectors.
6875    ///
6876    /// Round-to-nearest and toward-zero give a signed zero; a mode that rounds
6877    /// *away* from zero must give the smallest **normal** instead, because zero
6878    /// is on the wrong side of the true result. Getting that wrong yields `-0`
6879    /// in all four cases, which looks entirely reasonable.
6880    #[test]
6881    fn with_fs_set_a_subnormal_result_flushes_per_rounding_mode() {
6882        let a = 1.528_510_4e-37f32.to_bits();
6883        let b = (-1.539_154_3e-37f32).to_bits();
6884        // (RM, expected) -- the true result is a tiny NEGATIVE subnormal.
6885        for (rm, want) in [
6886            (0u32, (-0.0f32).to_bits()),         // nearest
6887            (1, (-0.0f32).to_bits()),            // toward zero
6888            (2, (-0.0f32).to_bits()),            // toward +inf
6889            (3, (-f32::MIN_POSITIVE).to_bits()), // toward -inf: away from zero
6890        ] {
6891            let p = run_add_s(FCSR_FS | rm, a, b);
6892            assert_eq!(p.fpr.read_s(4, true), want, "RM={rm}");
6893            let fcsr = p.cop1.fcsr();
6894            assert_ne!(fcsr & (1 << 13), 0, "Cause.underflow, RM={rm}");
6895            assert_ne!(fcsr & (1 << 12), 0, "Cause.inexact, RM={rm}");
6896            assert_eq!(fcsr & CAUSE_E, 0, "not unimplemented, RM={rm}");
6897        }
6898        // ...and the mirrored operands flush the other way.
6899        let p = run_add_s(
6900            FCSR_FS | 2,
6901            (-1.528_510_4e-37f32).to_bits(),
6902            1.539_154_3e-37f32.to_bits(),
6903        );
6904        assert_eq!(
6905            p.fpr.read_s(4, true),
6906            f32::MIN_POSITIVE.to_bits(),
6907            "a positive tiny result under toward-+inf"
6908        );
6909    }
6910
6911    /// **`FS` set but underflow enabled is unimplemented, not a trap.** The
6912    /// processor cannot deliver a trapped underflow's defined result either.
6913    ///
6914    /// Easy to miss because it is the interaction of two features that each
6915    /// work: flushing works, and enabled traps work, but together they do not.
6916    #[test]
6917    fn fs_plus_an_enabled_underflow_is_unimplemented_rather_than_a_trap() {
6918        let a = 1.528_510_4e-37f32.to_bits();
6919        let b = (-1.539_154_3e-37f32).to_bits();
6920        // bit 8 = enable underflow, bit 7 = enable inexact.
6921        for enable in [1u32 << 8, 1 << 7] {
6922            let p = run_add_s(FCSR_FS | enable, a, b);
6923            assert_ne!(
6924                p.cop1.fcsr() & CAUSE_E,
6925                0,
6926                "enable={enable:#X} must give unimplemented"
6927            );
6928        }
6929    }
6930
6931    /// **The two NaN classes trap differently.** MSB clear (quiet by the
6932    /// VR4300's convention) is unimplemented; MSB set (signaling) is Invalid.
6933    ///
6934    /// Swapping them is invisible until `FCSR` is read back, and both are
6935    /// "the operation trapped", so a test asserting only that would pass either
6936    /// way. See ledger C-12.
6937    #[test]
6938    fn the_two_nan_classes_raise_different_causes() {
6939        let msb_clear = 0x7F80_0001u32; // unimplemented here
6940        let msb_set = 0x7FC0_0001u32; // signaling here -> Invalid
6941
6942        let p = run_add_s(0, msb_clear, 2.0f32.to_bits());
6943        assert_ne!(p.cop1.fcsr() & CAUSE_E, 0, "MSB clear -> unimplemented");
6944        assert_eq!(p.cop1.fcsr() & (1 << 16), 0, "and NOT invalid");
6945
6946        let p = run_add_s(0, msb_set, 2.0f32.to_bits());
6947        assert_ne!(p.cop1.fcsr() & (1 << 16), 0, "MSB set -> Cause.invalid");
6948        assert_eq!(p.cop1.fcsr() & CAUSE_E, 0, "and NOT unimplemented");
6949    }
6950
6951    /// **`ABS`/`NEG` classify their operand; `MOV` does not.** All three look
6952    /// like sign/bit manipulation, and only `MOV` actually is.
6953    ///
6954    /// n64-systemtest settles it by construction rather than description:
6955    /// `MOV.S` is driven through the cause-*preserving* harness while `ABS.S`
6956    /// and `NEG.S` go through the ordinary one. Treating all three alike was
6957    /// worth 52 assertions.
6958    #[test]
6959    fn abs_and_neg_refuse_a_subnormal_but_mov_moves_it() {
6960        use crate::cop0::reg;
6961        /// fmt 16, `fs` 0, `fd` 4.
6962        const fn unary(funct: u32) -> u32 {
6963            (0o21 << 26) | (0o20 << 21) | (4 << 6) | funct
6964        }
6965        let subnormal = 1u32;
6966
6967        for funct in [5u32, 7] {
6968            let mut bus = Ram::new(alloc::vec![unary(funct)]);
6969            let mut regs = Regs::new();
6970            let mut p = Pipeline::new();
6971            p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6972            p.fpr.write_s(0, true, subnormal);
6973            let mut pc = KSEG0_PROG;
6974            for _ in 0..24 {
6975                p.advance(&mut bus, &mut regs, &mut pc);
6976            }
6977            assert_ne!(
6978                p.cop1.fcsr() & CAUSE_E,
6979                0,
6980                "funct {funct} (ABS/NEG) must refuse a subnormal"
6981            );
6982        }
6983
6984        // MOV.S is a pure move: it transports the subnormal untouched and
6985        // raises nothing at all.
6986        let mut bus = Ram::new(alloc::vec![unary(6)]);
6987        let mut regs = Regs::new();
6988        let mut p = Pipeline::new();
6989        p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6990        p.fpr.write_s(0, true, subnormal);
6991        let mut pc = KSEG0_PROG;
6992        for _ in 0..24 {
6993            p.advance(&mut bus, &mut regs, &mut pc);
6994        }
6995        assert_eq!(
6996            p.fpr.read_s(4, true),
6997            subnormal,
6998            "MOV.S moves the subnormal"
6999        );
7000        assert_eq!(p.cop1.fcsr(), 0, "and raises nothing");
7001    }
7002
7003    /// **A `.L` conversion refuses a magnitude of `2^53` or more**, which is
7004    /// far narrower than `i64`.
7005    ///
7006    /// The threshold is bracketed rather than assumed: `9007198717870080`
7007    /// converts and `9007199254740992` (`2^53`) does not, and both are
7008    /// comfortably inside `i64`. A test using only a huge value would pass with
7009    /// the limit set anywhere between `2^53` and `i64::MAX`.
7010    ///
7011    /// The same limit applies to `.W`, where it is **unobservable**: `2^53` is
7012    /// far outside `i32`, so both paths end in unimplemented. Guarding it on
7013    /// the target width was tried and removed — a branch no test can
7014    /// distinguish is a branch that will rot.
7015    #[test]
7016    fn a_long_conversion_refuses_two_to_the_fifty_three_but_not_just_below() {
7017        use crate::cop0::reg;
7018        /// `CVT.L.D $f4, $f0` — fmt 17 (`.D`), `fs` 0, `fd` 4, funct 0o45.
7019        const CVT_L_D: u32 = (0o21 << 26) | (0o21 << 21) | (4 << 6) | 0o45;
7020
7021        for (src, want_unimplemented) in [
7022            (9_007_198_717_870_080.0f64, false),
7023            (9_007_199_254_740_992.0f64, true), // 2^53
7024            (-9_007_199_254_740_992.0f64, true),
7025            (-9_007_198_717_870_080.0f64, false),
7026        ] {
7027            let mut bus = Ram::new(alloc::vec![CVT_L_D]);
7028            let mut regs = Regs::new();
7029            let mut p = Pipeline::new();
7030            p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
7031            p.fpr.write_d(0, true, src.to_bits());
7032            let mut pc = KSEG0_PROG;
7033            for _ in 0..24 {
7034                p.advance(&mut bus, &mut regs, &mut pc);
7035            }
7036            let raised = p.cop1.fcsr() & CAUSE_E != 0;
7037            assert_eq!(raised, want_unimplemented, "{src:e}");
7038            if !want_unimplemented {
7039                #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
7040                let want = src as i64 as u64;
7041                assert_eq!(p.fpr.read_d(4, true), want, "{src:e} must convert");
7042            }
7043        }
7044    }
7045
7046    /// An out-of-range float-to-integer conversion is **unimplemented**, not
7047    /// Invalid. IEEE says Invalid; this processor declines instead, and
7048    /// `fpu::to_i32` reports the IEEE answer that must be translated.
7049    #[test]
7050    fn an_out_of_range_integer_conversion_is_unimplemented_not_invalid() {
7051        use crate::cop0::reg;
7052        /// `CVT.W.S $f4, $f0` — fmt 16, `fs` 0, `fd` 4, funct 0o44.
7053        const CVT_W_S: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 6) | 0o44;
7054
7055        for src in [1e30f32, f32::INFINITY, f32::from_bits(0x7FC0_0000)] {
7056            let mut bus = Ram::new(alloc::vec![CVT_W_S]);
7057            let mut regs = Regs::new();
7058            let mut p = Pipeline::new();
7059            p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
7060            p.fpr.write_s(0, true, src.to_bits());
7061            let mut pc = KSEG0_PROG;
7062            for _ in 0..24 {
7063                p.advance(&mut bus, &mut regs, &mut pc);
7064            }
7065            assert_ne!(p.cop1.fcsr() & CAUSE_E, 0, "{src:e} -> unimplemented");
7066            assert_eq!(p.cop1.fcsr() & (1 << 16), 0, "{src:e} -> NOT invalid");
7067            assert_eq!(
7068                (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7069                crate::exception::exc_code::FPE
7070            );
7071        }
7072    }
7073
7074    // --- CACHE (T-12-005) ---------------------------------------------------
7075
7076    /// `CACHE` must **not** raise. IPL3 and libdragon both issue it, so a
7077    /// reserved-instruction exception here blocks every real ROM — which is why
7078    /// this was called out as a hard blocker before it was implemented.
7079    #[test]
7080    fn cache_executes_instead_of_raising() {
7081        use crate::cop0::reg;
7082        // CACHE op=0, 0($1) with $1 = KSEG0 base.
7083        let prog = alloc::vec![lui_kseg0(1), ld_st(0o57, 1, 0, 0x100)];
7084        assert_eq!(decode(prog[1]).op, crate::decode::Op::Cache);
7085        let mut bus = Ram::new(prog);
7086        let mut regs = Regs::new();
7087        let mut p = Pipeline::new();
7088        p.cop0.set_hardware(reg::STATUS, 0);
7089        let mut pc = KSEG0_PROG;
7090        for _ in 0..24 {
7091            p.advance(&mut bus, &mut regs, &mut pc);
7092        }
7093        assert_eq!(
7094            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7095            0,
7096            "CACHE must not raise"
7097        );
7098        assert_eq!(p.stalled_by(), None, "and must not be stuck in an epilogue");
7099    }
7100
7101    /// `CACHE`'s `rt` slot is the **operation selector**, not a destination.
7102    /// Decoding it as a load would clobber whichever GPR the cache-op encoding
7103    /// happens to name — a spectacularly confusing bug, since the register
7104    /// destroyed depends on which cache operation was requested.
7105    #[test]
7106    fn cache_writes_no_general_register() {
7107        // op = 0b10101 = 21, which as a destination would be $21.
7108        let word = ld_st(0o57, 1, 21, 0);
7109        let d = decode(word);
7110        assert_eq!(d.op, crate::decode::Op::Cache);
7111        assert_eq!(d.dest, 0, "rt is the cache operation, not a destination");
7112
7113        let prog = alloc::vec![lui_kseg0(1), addiu_zero(21, 0x33), word];
7114        let mut bus = Ram::new(prog);
7115        let mut regs = Regs::new();
7116        let mut p = Pipeline::new();
7117        p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
7118        let mut pc = KSEG0_PROG;
7119        for _ in 0..32 {
7120            p.advance(&mut bus, &mut regs, &mut pc);
7121        }
7122        assert_eq!(regs.read(21), 0x33, "$21 must survive CACHE op 21");
7123    }
7124
7125    /// An **`Index_*`** `CACHE` op addresses the cache by index and **must not
7126    /// translate**, so it cannot fault however unmapped the address is.
7127    ///
7128    /// This matters at boot: cache-initialization code walks every index with an
7129    /// arbitrary base address, and translating would raise a TLB refill on the
7130    /// first one — before any mapping exists to satisfy it.
7131    #[test]
7132    fn an_index_cache_op_never_faults_however_unmapped_the_address() {
7133        use crate::cop0::reg;
7134        for op in [0u16, 4, 8] {
7135            // op4..2 = 0, 1, 2 -> Index_Invalidate / Load_Tag / Store_Tag.
7136            assert!((op >> 2) < 3, "op {op} must be an Index form");
7137            let mut bus = Ram::new(alloc::vec![ld_st(0o57, 0, u32::from(op), 0x4000)]);
7138            let mut regs = Regs::new();
7139            let mut p = Pipeline::new();
7140            p.cop0.set_hardware(reg::STATUS, 0);
7141            let mut pc = KSEG0_PROG;
7142            for _ in 0..24 {
7143                p.advance(&mut bus, &mut regs, &mut pc);
7144            }
7145            assert_eq!(
7146                (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7147                0,
7148                "Index op {op} must not raise -- it never consults the TLB"
7149            );
7150        }
7151    }
7152
7153    /// A **`Hit_*`** `CACHE` op translates, so it raises a TLB fault on an
7154    /// unmapped address — it is defined in terms of *"the specified address"*.
7155    #[test]
7156    fn a_hit_cache_op_on_an_unmapped_address_faults() {
7157        use crate::cop0::reg;
7158        // op = 16 -> op4..2 = 4 = Hit_Invalidate.
7159        let mut bus = Ram::new(alloc::vec![ld_st(0o57, 0, 16, 0x4000)]);
7160        let mut regs = Regs::new();
7161        let mut p = Pipeline::new();
7162        p.cop0.set_hardware(reg::STATUS, 0);
7163        let mut pc = KSEG0_PROG;
7164
7165        let mut cycles = 0;
7166        while p.stalled_by() != Some(Interlock::Exception) {
7167            p.advance(&mut bus, &mut regs, &mut pc);
7168            cycles += 1;
7169            assert!(cycles < 16, "CACHE did not translate");
7170        }
7171        assert_eq!(
7172            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7173            crate::exception::exc_code::TLBL
7174        );
7175    }
7176
7177    // --- FP register file, moves and loads/stores (T-13-001) ---------------
7178
7179    /// `MTC1` then `MFC1` round-trips through the real FP register file, and
7180    /// `MTC1` must **not** write a general register — `rt` is its source.
7181    #[test]
7182    fn mtc1_then_mfc1_round_trips_and_writes_no_gpr() {
7183        use crate::cop0::reg;
7184        const fn cop1(rs: u32, rt: u32, fs: u32) -> u32 {
7185            (0o21 << 26) | (rs << 21) | (rt << 16) | (fs << 11)
7186        }
7187        //   ADDIU $1, $0, 0x55
7188        //   MTC1  $1, $f4
7189        //   MFC1  $2, $f4
7190        let prog = alloc::vec![addiu_zero(1, 0x55), cop1(0o04, 1, 4), cop1(0o00, 2, 4)];
7191        let mut bus = Ram::new(prog);
7192        let mut regs = Regs::new();
7193        let mut p = Pipeline::new();
7194        p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 | FR
7195        let mut pc = KSEG0_PROG;
7196        for _ in 0..40 {
7197            p.advance(&mut bus, &mut regs, &mut pc);
7198        }
7199        assert_eq!(
7200            p.fpr.read_s(4, true),
7201            0x55,
7202            "MTC1 reached the FP register file"
7203        );
7204        assert_eq!(regs.read(2), 0x55, "and MFC1 read it back");
7205        assert_eq!(regs.read(1), 0x55, "$1 -- MTC1's source -- must survive");
7206    }
7207
7208    /// **`DMFC1`/`DMTC1` apply the `FR` view**, they do not move the physical
7209    /// register. UM Ch. 17's pseudocode: with `FR = 0` and an even `fs`,
7210    /// In half mode a 64-bit access addresses **FGR `fs & !1` in its
7211    /// entirety**, and the odd FGR is not part of it.
7212    ///
7213    /// This test previously asserted the *pair* model — low word in the even
7214    /// FGR, high word in the odd one — which round-trips through
7215    /// `DMTC1`/`DMFC1` perfectly and is still wrong: hardware never touches
7216    /// the odd register. See `fpr.rs` for the n64-systemtest vector that
7217    /// settles it.
7218    #[test]
7219    fn dmtc1_and_dmfc1_apply_the_fr_view_rather_than_moving_the_raw_fgr() {
7220        use crate::cop0::reg;
7221        const fn cop1(rs: u32, rt: u32, fs: u32) -> u32 {
7222            (0o21 << 26) | (rs << 21) | (rt << 16) | (fs << 11)
7223        }
7224        let mut p = Pipeline::new();
7225        // CU1 set, FR CLEAR -- the paired view.
7226        p.cop0.set_hardware(reg::STATUS, 1 << 29);
7227        // DMTC1 $1, $f2 with $1 = 0x1122_3344_5566_7788.
7228        let prog = alloc::vec![cop1(0o05, 1, 2), cop1(0o01, 2, 2)];
7229        let mut bus = Ram::new(prog);
7230        let mut regs = Regs::new();
7231        regs.write(1, 0x1122_3344_5566_7788);
7232        let mut pc = KSEG0_PROG;
7233        for _ in 0..32 {
7234            p.advance(&mut bus, &mut regs, &mut pc);
7235        }
7236
7237        // All 64 bits in FGR 2; FGR 3 is not involved.
7238        assert_eq!(
7239            p.fpr.read_raw(2),
7240            0x1122_3344_5566_7788,
7241            "the whole value lives in the even FGR"
7242        );
7243        assert_eq!(p.fpr.read_raw(3), 0, "the odd FGR is untouched");
7244        assert_eq!(regs.read(2), 0x1122_3344_5566_7788, "DMFC1 reassembles it");
7245        // And it agrees with the LDC1/SDC1 view of the same register.
7246        assert_eq!(p.fpr.read_d(2, false), 0x1122_3344_5566_7788);
7247    }
7248
7249    /// `SDC1` then `LDC1` round-trips a double through memory, and with
7250    /// `FR = 0` the value lives in an **FGR pair** — so this exercises the view
7251    /// that a direct-index register file gets wrong.
7252    #[test]
7253    fn ldc1_and_sdc1_round_trip_a_double_with_fr_clear() {
7254        use crate::cop0::reg;
7255        //   LUI  $1, 0x8000
7256        //   SDC1 $f2, 0x100($1)
7257        //   LDC1 $f4, 0x100($1)
7258        let prog = alloc::vec![
7259            lui_kseg0(1),
7260            ld_st(0o75, 1, 2, 0x100),
7261            ld_st(0o65, 1, 4, 0x100),
7262        ];
7263        let mut bus = Ram::new(prog);
7264        let mut regs = Regs::new();
7265        let mut p = Pipeline::new();
7266        // CU1 set, FR CLEAR -- the paired view.
7267        p.cop0.set_hardware(reg::STATUS, 1 << 29);
7268        p.fpr.write_d(2, false, 0x0123_4567_89AB_CDEF);
7269        let mut pc = KSEG0_PROG;
7270        for _ in 0..48 {
7271            p.advance(&mut bus, &mut regs, &mut pc);
7272        }
7273        assert_eq!(
7274            p.fpr.read_d(4, false),
7275            0x0123_4567_89AB_CDEF,
7276            "the double survived memory in the FR = 0 paired view"
7277        );
7278        writeback(&mut p, &mut bus, 0xFFFF_FFFF_8000_0100);
7279        assert_eq!(
7280            bus.read_u32(0x100),
7281            0x0123_4567,
7282            "big-endian high word first"
7283        );
7284    }
7285
7286    /// FP loads and stores obey the same alignment rules as the integer forms —
7287    /// `LDC1` needs 8-byte alignment, and a misaligned one raises `AdEL`.
7288    #[test]
7289    fn a_misaligned_ldc1_raises_an_address_error() {
7290        use crate::cop0::reg;
7291        let prog = alloc::vec![lui_kseg0(1), ld_st(0o65, 1, 4, 0x104)];
7292        let mut bus = Ram::new(prog);
7293        let mut regs = Regs::new();
7294        let mut p = Pipeline::new();
7295        p.cop0.set_hardware(reg::STATUS, 1 << 29);
7296        let mut pc = KSEG0_PROG;
7297
7298        let mut cycles = 0;
7299        while p.stalled_by() != Some(Interlock::Exception) {
7300            p.advance(&mut bus, &mut regs, &mut pc);
7301            cycles += 1;
7302            assert!(cycles < 24, "no address error raised");
7303        }
7304        assert_eq!(
7305            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7306            crate::exception::exc_code::ADEL,
7307            "a misaligned LDC1 is a load address error"
7308        );
7309    }
7310
7311    /// The FP moves and loads need `CU1` like every other COP1 instruction.
7312    #[test]
7313    fn fp_loads_need_cu1() {
7314        use crate::cop0::reg;
7315        let prog = alloc::vec![lui_kseg0(1), ld_st(0o61, 1, 4, 0x100)];
7316        let mut bus = Ram::new(prog);
7317        let mut regs = Regs::new();
7318        let mut p = Pipeline::new();
7319        p.cop0.set_hardware(reg::STATUS, 0); // CU1 CLEAR
7320        let mut pc = KSEG0_PROG;
7321
7322        let mut cycles = 0;
7323        while p.stalled_by() != Some(Interlock::Exception) {
7324            p.advance(&mut bus, &mut regs, &mut pc);
7325            cycles += 1;
7326            assert!(cycles < 24, "no exception raised");
7327        }
7328        assert_eq!(
7329            (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7330            crate::exception::exc_code::CPU
7331        );
7332        assert_eq!((p.cop0.read(reg::CAUSE) >> 28) & 0b11, 1, "unit 1");
7333    }
7334
7335    // ---------------------------------------------------------------------
7336    // The latch-independent primitives, tested directly (ADR 0013's seam)
7337    // ---------------------------------------------------------------------
7338    //
7339    // The stage tests above reach these through the cascade, which is the right
7340    // way to test the accurate path and the wrong way to pin a *contract*. These
7341    // call the primitives directly, because the instruction-granular path will
7342    // call them with different consequences and needs the decisions to hold on
7343    // their own.
7344
7345    /// A faulting fetch must report the fault **and leave the bus untouched** —
7346    /// the access itself is what is invalid.
7347    ///
7348    /// Distinct from `an_unaligned_fetch_raises_address_error_without_realigning`
7349    /// above, which checks what `ic_stage` *does* with the fault (stamps `ic_rf`,
7350    /// declines to realign). This checks the primitive's own contract, which is
7351    /// the half a second caller depends on.
7352    #[test]
7353    fn fetch_word_reports_a_fault_without_touching_the_bus() {
7354        struct Watch {
7355            touched: bool,
7356        }
7357        impl Bus for Watch {
7358            fn read_u8(&mut self, _addr: u32) -> u8 {
7359                self.touched = true;
7360                0
7361            }
7362            fn write_u8(&mut self, _addr: u32, _val: u8) {
7363                self.touched = true;
7364            }
7365            fn read_u32(&mut self, _addr: u32) -> u32 {
7366                self.touched = true;
7367                0
7368            }
7369        }
7370        let mut bus = Watch { touched: false };
7371        let mut p = Pipeline::new();
7372
7373        assert_eq!(
7374            p.fetch_word(&mut bus, 0xFFFF_FFFF_8000_0002),
7375            Err(Exception::AddressError { store: false }),
7376            "an unaligned fetch is an address error"
7377        );
7378        assert!(
7379            !bus.touched,
7380            "the bus was accessed for a fetch that is invalid before it happens"
7381        );
7382    }
7383
7384    /// The positive case, so the test above cannot pass against a `fetch_word`
7385    /// that simply always fails.
7386    ///
7387    /// The word is a distinctive sentinel rather than zero: zero is `NOP` and is
7388    /// also what a bus returning nothing produces, so it cannot tell a real fetch
7389    /// from a missing one.
7390    #[test]
7391    fn fetch_word_returns_the_word_at_an_aligned_pc() {
7392        struct Fixed(u32);
7393        impl Bus for Fixed {
7394            fn read_u8(&mut self, _addr: u32) -> u8 {
7395                0
7396            }
7397            fn write_u8(&mut self, _addr: u32, _val: u8) {}
7398            fn read_u32(&mut self, _addr: u32) -> u32 {
7399                self.0
7400            }
7401        }
7402        const SENTINEL: u32 = 0xDEAD_BEEF;
7403        let mut bus = Fixed(SENTINEL);
7404        let mut p = Pipeline::new();
7405
7406        // KSEG1 — uncached and unmapped, so this reaches the bus directly and
7407        // tests the fetch rather than the I-cache.
7408        assert_eq!(p.fetch_word(&mut bus, 0xFFFF_FFFF_A000_0000), Ok(SENTINEL));
7409    }
7410
7411    /// `ex_gate` refuses `DCFC1`/`DCTC1` by **replacing** `FCSR.Cause`, not by
7412    /// OR-ing into it.
7413    ///
7414    /// Asserted as an *effect*: `FCSR` is seeded with unrelated cause bits first,
7415    /// and they must be gone afterwards. A test that only checked bit 17 had
7416    /// arrived would pass against an implementation that OR-ed — which is the
7417    /// behavior n64-systemtest specifically pre-loads unrelated bits to catch.
7418    #[test]
7419    fn ex_gate_replaces_the_whole_fcsr_cause_field() {
7420        /// `FCSR.Cause`, bits 17:12 — the field under test.
7421        const CAUSE: u32 = 0x3F << 12;
7422        /// Unrelated cause bits (Inexact and Overflow), seeded to be cleared.
7423        const SEEDED: u32 = 0b0001_0100 << 12;
7424
7425        use crate::cop0::reg;
7426
7427        let mut p = Pipeline::new();
7428        p.cop1.ctc1(31, SEEDED);
7429        assert_eq!(p.cop1.fcsr() & CAUSE, SEEDED, "the seed took");
7430
7431        // COP1 opcode with rs = 3 is `DCFC1`, the usable-but-unimplemented
7432        // control move. Decoded rather than hand-built so the encoding stays
7433        // owned by `decode`.
7434        let d = decode((0o21 << 26) | (3 << 21));
7435        assert_eq!(
7436            d.op,
7437            crate::decode::Op::Cop1ReservedControl,
7438            "the encoding under test must actually be the reserved control move"
7439        );
7440
7441        // `CU1` must be SET, or the coprocessor-usability check refuses first and
7442        // this would test the wrong branch.
7443        p.cop0.set_hardware(reg::STATUS, 1 << 29);
7444        assert_eq!(p.ex_gate(d), Err(Exception::FloatingPoint));
7445        assert_eq!(
7446            p.cop1.fcsr() & CAUSE,
7447            crate::fpu::CAUSE_UNIMPLEMENTED,
7448            "the seeded cause bits survived: `Cause` was OR-ed, not replaced"
7449        );
7450    }
7451
7452    /// **`ex_gate`'s first two refusals are DISJOINT, so their order cannot be
7453    /// observed — and this test is what keeps that true.**
7454    ///
7455    /// The extraction started from the accurate path's comment, which says an
7456    /// unusable coprocessor is reported as such *"even when the encoding is also
7457    /// a 64-bit one"*. Mutation-checking that against n64-systemtest changed
7458    /// nothing, and the reason is stronger than "the suite misses it":
7459    ///
7460    /// - `Op::is_64_bit` covers only CPU integer and load/store operations
7461    ///   (`Dadd` … `Sdr`);
7462    /// - `unusable_coprocessor` returns `Some` only for COP0/COP1/COP2 encodings.
7463    ///
7464    /// No encoding is in both sets, so **no input can reach the second check by
7465    /// way of the first**. The ordering is unobservable rather than merely
7466    /// untested, which is why the oracle was silent.
7467    ///
7468    /// Swept over a structured slice of the encoding space rather than asserted
7469    /// against a copy of either list: a duplicated list stops covering the thing
7470    /// it duplicates the moment one side grows, which is the failure this test
7471    /// exists to catch. If `Dmfc1`/`Dmtc1`/`Dmfc0`/`Dmtc0` are ever added to
7472    /// `is_64_bit` — they *are* 64-bit operations, and whether their omission is
7473    /// correct is an open question, not a claim made here — this fails, and
7474    /// whoever adds them has to decide what the precedence should be with the
7475    /// case in front of them.
7476    #[test]
7477    fn ex_gates_first_two_refusals_cannot_both_apply() {
7478        use crate::cop0::reg;
7479
7480        let mut p = Pipeline::new();
7481        // 32-bit USER mode with every `CU` bit clear, so *every* coprocessor
7482        // encoding is unusable and `sixty_four_bit_is_reserved` holds. Kernel
7483        // mode would exempt COP0 and hide half the question.
7484        p.cop0.set_hardware(reg::STATUS, 0b10 << 3);
7485        assert!(
7486            p.sixty_four_bit_is_reserved(),
7487            "the sweep is meaningless unless 64-bit really is reserved here"
7488        );
7489
7490        let mut coprocessor = 0u32;
7491        let mut sixty_four = 0u32;
7492        for opcode in 0..64u32 {
7493            for rs in 0..32u32 {
7494                for funct in 0..64u32 {
7495                    let d = decode((opcode << 26) | (rs << 21) | funct);
7496                    let cop = p.unusable_coprocessor(d).is_some();
7497                    let wide = d.op.is_64_bit();
7498                    coprocessor += u32::from(cop);
7499                    sixty_four += u32::from(wide);
7500                    assert!(
7501                        !(cop && wide),
7502                        "opcode {opcode:#o} rs {rs:#o} funct {funct:#o} decodes to \
7503                         {:?}, which is BOTH 64-bit and on an unusable coprocessor \
7504                         -- `ex_gate`'s first two checks are no longer disjoint, so \
7505                         their order is now observable and has to be justified \
7506                         against the manual rather than inherited",
7507                        d.op
7508                    );
7509                }
7510            }
7511        }
7512        // A sweep that decoded nothing interesting would pass just as
7513        // convincingly. Both populations must be non-empty for the disjointness
7514        // above to be a claim about anything.
7515        assert!(
7516            coprocessor > 0 && sixty_four > 0,
7517            "the sweep found {coprocessor} coprocessor and {sixty_four} 64-bit \
7518             encodings; with either at zero it proves nothing"
7519        );
7520    }
7521}
7522
7523/// The commit census must classify what it claims to classify.
7524///
7525/// A census that silently called every instruction SIMPLE would report a
7526/// wonderful 100% and send a fast commit path off to serve loads it cannot
7527/// serve. The measured share on Super Mario 64 (96.81% SIMPLE, 3.11% MEM) is
7528/// low enough on MEM to be worth disbelieving until the classifier is witnessed
7529/// doing its job.
7530#[cfg(all(test, feature = "work-counters"))]
7531mod commit_census_tests {
7532    use super::{Pipeline, commit_class as cc};
7533    use crate::exec::{Cop0Access, MemOp, WriteBack};
7534    use crate::mem::{LoadKind, StoreKind};
7535
7536    #[test]
7537    fn reachable_commit_classes_are_distinct() {
7538        let mut p = Pipeline::new();
7539
7540        // A plain ALU result.
7541        p.count_commit(None, None, WriteBack::Gpr { dest: 3, value: 1 });
7542        // A branch: commits nothing.
7543        p.count_commit(None, None, WriteBack::None);
7544        // A load. `mem` is what makes it not-simple, whatever the write-back says.
7545        p.count_commit(
7546            None,
7547            Some(MemOp::Load {
7548                kind: LoadKind::SignedWord,
7549                addr: 0x8000_0000,
7550                dest: 4,
7551            }),
7552            WriteBack::None,
7553        );
7554        // A store.
7555        p.count_commit(
7556            None,
7557            Some(MemOp::Store {
7558                kind: StoreKind::Word,
7559                addr: 0x8000_0000,
7560                value: 0,
7561            }),
7562            WriteBack::None,
7563        );
7564        // A multiply.
7565        p.count_commit(
7566            None,
7567            None,
7568            WriteBack::HiLo(crate::alu::HiLo { hi: 0, lo: 0 }),
7569        );
7570
7571        let c = p.commit_census();
7572        assert_eq!(
7573            c[cc::SIMPLE],
7574            2,
7575            "GPR write and no-write-back are both simple"
7576        );
7577        assert_eq!(c[cc::MEM], 2, "a load and a store both have a DC access");
7578        assert_eq!(c[cc::HILO], 1, "a multiply writes HI/LO");
7579        assert_eq!(c[cc::COP], 0);
7580        assert_eq!(c[cc::OTHER], 0);
7581    }
7582
7583    /// A COP0 access outranks a memory op: an instruction with both is NOT
7584    /// something a direct commit path may serve, and counting it as MEM would
7585    /// understate the class that must keep using `wb_stage`.
7586    #[test]
7587    fn a_cop_access_outranks_a_memory_op() {
7588        let mut p = Pipeline::new();
7589        p.count_commit(
7590            Some(Cop0Access::Read {
7591                src: 12,
7592                dest: 3,
7593                wide: false,
7594            }),
7595            Some(MemOp::Store {
7596                kind: StoreKind::Word,
7597                addr: 0x8000_0000,
7598                value: 0,
7599            }),
7600            WriteBack::None,
7601        );
7602        let c = p.commit_census();
7603        assert_eq!(c[cc::COP], 1, "the COP class must win");
7604        assert_eq!(c[cc::MEM], 0);
7605        assert_eq!(c[cc::SIMPLE], 0);
7606    }
7607
7608    #[test]
7609    fn unretired_or_aborted_instructions_do_not_increment_census() {
7610        let mut p = Pipeline::new();
7611        let mut regs = crate::regs::Regs::new();
7612        // A latch with abort set: does not retire
7613        p.dc_wb = super::Latch {
7614            occupied: true,
7615            abort: Some(super::Exception::Breakpoint),
7616            ..Default::default()
7617        };
7618        p.wb_stage(&mut regs);
7619        let c = p.commit_census();
7620        assert_eq!(p.retired, 0);
7621        assert_eq!(c[cc::SIMPLE], 0);
7622        assert_eq!(c[cc::MEM], 0);
7623        assert_eq!(c[cc::COP], 0);
7624        assert_eq!(c[cc::HILO], 0);
7625        assert_eq!(c[cc::OTHER], 0);
7626    }
7627}