rustyn64_cpu/pipeline.rs
1//! The VR4300's five-stage pipeline (ADR 0007).
2//!
3//! `IC` → `RF` → `EX` → `DC` → `WB` (VR4300 User's Manual §4.1, Figure 4-1):
4//! Instruction Cache fetch, Register Fetch, Execution, Data Cache fetch, Write
5//! Back. In-order, single-issue, with one architectural delay slot. At least 5
6//! `PCycle`s are required to execute an instruction, and up to five are in flight
7//! at once when the pipe flows.
8//!
9//! # Latches, not stages
10//!
11//! Five stages have four boundaries, and the state lives on the **boundaries**.
12//! [`Latch`] is what travels with an instruction as it advances.
13//!
14//! `in_delay_slot` riding in the latch rather than in a global CPU flag is
15//! load-bearing: a multi-cycle stall between a branch and its delay slot
16//! desynchronizes a global flag, and that is the classic bug in this area. With
17//! the flag attached to the instruction, `Cause.BD` and `EPC` come out right for
18//! free. `delay_slot_flag_survives_a_multi_cycle_stall` pins it.
19//!
20//! # Reverse step order is the latching
21//!
22//! [`Pipeline::advance`] runs **WB → DC → EX → RF → IC**. Each stage reads its
23//! input latch and writes its output latch, so running downstream-first means a
24//! stage's input still holds the *previous* cycle's value when it is read. No
25//! value can therefore propagate two stages in one cycle, and **no double
26//! buffering is needed** — the reverse order *is* the latching.
27//!
28//! This is a load-bearing invariant, not a style choice. Reversing it silently
29//! makes the pipeline one-cycle-too-fast; `a_value_advances_exactly_one_stage_per_cycle`
30//! is the guard.
31//!
32//! # Status
33//!
34//! **Structure only.** The stages move latches and account for time; they do not
35//! decode or execute yet (T-11-002 onward). What is real here is the shape, the
36//! stall/interlock mechanism, the delay-slot carriage, and the interrupt gate —
37//! the parts that cannot be retrofitted later without rewriting every consumer.
38
39use crate::Bus;
40use crate::alu::HiLo;
41use crate::cop0::Cop0;
42use crate::cop1::Cop1Control;
43use crate::decode::{Decoded, decode};
44use crate::exception;
45use crate::exec::{Cop0Access, Cop1Access, MemOp, Redirect, TlbOp, WriteBack, execute};
46use crate::fpr::Fpr;
47use crate::mem;
48use crate::regs::Regs;
49use crate::softfloat;
50use crate::tlb::Tlb;
51use serde::{Deserialize, Serialize};
52
53/// The five pipeline stages, in hardware order (UM §4.1, Figure 4-1).
54///
55/// Note the names: **IC** and **DC**, not `IF`/`DF`. The manual's whole interlock
56/// and exception taxonomy is stated stage-relative, so these spellings are what
57/// make a citation resolvable.
58#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
59pub enum Stage {
60 /// Instruction Cache fetch.
61 Ic,
62 /// Register Fetch.
63 Rf,
64 /// Execution.
65 Ex,
66 /// Data Cache fetch — the bus access, and where interrupts are sampled.
67 Dc,
68 /// Write Back.
69 Wb,
70}
71
72/// An aborting condition travelling down the pipe.
73///
74/// Deliberately **not** called `Fault`. UM §4.5 defines a *fault* as the union of
75/// interlocks and exceptions (Figure 4-11: Faults = Interlocks ∪ Exceptions,
76/// split into Stalls vs Abort), and CEN64 follows that wider usage. What rides in
77/// a latch here is only the aborting subset, so it carries the narrower name.
78#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
79pub enum Exception {
80 /// An interrupt was accepted (`Cause.IP` unmasked, `IE` set, `EXL`/`ERL` clear).
81 Interrupt,
82 /// Address error on an instruction fetch or data access.
83 ///
84 /// Carries the direction because the architecture does: `AdEL` (4) and
85 /// `AdES` (5) are **different** `ExcCode` values (UM Table 6-2, p. 172), and
86 /// a handler distinguishes them. An instruction fetch is a load.
87 AddressError {
88 /// The faulting access was a store.
89 store: bool,
90 },
91 /// Integer overflow (`ADD`, `ADDI`, `SUB`, `DADD`, …).
92 Overflow,
93 /// `SYSCALL`.
94 Syscall,
95 /// `BREAK`.
96 Breakpoint,
97 /// A conditional trap (`TGE`, `TEQ`, `TNEI`, …) whose condition held.
98 Trap,
99 /// A reserved / unimplemented opcode.
100 ReservedInstruction,
101 /// A reserved encoding **within a coprocessor's** instruction space.
102 ///
103 /// Reports the same `ExcCode` as [`Exception::ReservedInstruction`] but
104 /// also sets `Cause.CE` to the coprocessor number. A plain
105 /// `ReservedInstruction` leaves `CE` at zero, so the two are not
106 /// interchangeable — n64-systemtest checks the whole `Cause` register, and
107 /// `DCFC2` with `CU2` set expects `CE = 2`.
108 CoprocessorReserved {
109 /// Which coprocessor, for `Cause.CE`.
110 unit: u8,
111 },
112 /// A coprocessor instruction with that unit disabled in `Status.CU`.
113 CoprocessorUnusable {
114 /// Which coprocessor, for `Cause.CE`.
115 unit: u8,
116 },
117 /// **Non-maskable interrupt** (UM §6.4.6).
118 ///
119 /// Not one of the `Cause.ExcCode` exceptions: it writes no `Cause`, saves to
120 /// `ErrorEPC` rather than `EPC`, sets `ERL` rather than `EXL`, and is taken
121 /// *"regardless of the settings of the EXL, ERL, and the IE bits"*. It rides
122 /// in this enum anyway so it can reuse the pipeline-flush machinery every
123 /// other exception uses; [`crate::exception::dispatch`] intercepts it before
124 /// any of the general path runs.
125 Nmi,
126 /// A TLB refill — no entry matched. Takes the **refill** vector.
127 TlbRefill {
128 /// The faulting access was a store (`TLBS` rather than `TLBL`).
129 store: bool,
130 /// The access was made with **64-bit addressing** enabled for its mode
131 /// (`Status.KX`/`SX`/`UX`), which selects the **XTLB** refill vector at
132 /// offset `0x080` instead of the 32-bit one at `0x000`.
133 ///
134 /// Carried on the exception rather than re-derived at dispatch because
135 /// by then `EXL` is set, which forces `Pipeline::access_mode` to
136 /// report Kernel and would answer for the handler's mode instead of the
137 /// faulting access's.
138 wide: bool,
139 },
140 /// A TLB entry matched but was invalid. Takes the **general** vector, with
141 /// the same `ExcCode` as a refill — the vector is the only difference, which
142 /// is why they are separate variants rather than one with a flag nobody
143 /// reads.
144 TlbInvalid {
145 /// The faulting access was a store.
146 store: bool,
147 },
148 /// A store to a valid but non-writable page.
149 TlbModified,
150 /// A floating-point operation raised a condition whose `FCSR.Enable` bit is
151 /// set.
152 ///
153 /// Carries nothing: which condition fired is reported in `FCSR.Cause`, not
154 /// in COP0 `Cause`, and the handler reads it from there. Adding a field
155 /// here would duplicate — and could contradict — the architectural record.
156 FloatingPoint,
157}
158
159/// What a COP1 operation writes when it does not trap.
160///
161/// Named rather than a `(u64, bool)` pair because the destinations are of
162/// genuinely different kinds — two FPR widths and a single `FCSR` bit — and a
163/// flag pair makes "write the condition to `fd`" representable.
164#[derive(Clone, Copy, Debug, Eq, PartialEq)]
165enum FpCommit {
166 /// A 32-bit result into `fd`'s low half: `.S` values and `.W` integers.
167 Single(u32),
168 /// A 64-bit result into `fd` through the `FR` view: `.D` values and `.L`
169 /// integers.
170 Double(u64),
171 /// `FCSR.C`. Only `C.cond.fmt` produces this, and it writes no FPR at all.
172 Condition(bool),
173}
174
175/// The documented interlocks (UM Table 4-3).
176///
177/// Held as a named enum rather than a bare cycle count so a stall is always
178/// attributable — "why did this stall" is answerable from a trace.
179#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
180pub enum Interlock {
181 /// Load interlock — 1 cycle (UM §4.6.5).
182 ///
183 /// Deliberately **imprecise**, matching hardware: it fires when the next
184 /// instruction's `rs` *or* `rt` field equals the load's `rt`, whether or not
185 /// that field is actually used as a source. See [`load_interlocks`].
186 Ldi,
187 /// Data cache busy — a cached store keeps the cache busy for its `DC` *and*
188 /// `WB` stages, so a following cache access stalls 1 cycle (UM §4.6.7).
189 Dcb,
190 /// Data cache miss — the fill cost is `8..=9 + M` `PCycle`s (UM Table 11-1).
191 Dcm,
192 /// Instruction cache busy (UM §4.6.3).
193 Icb,
194 /// Instruction micro-TLB miss — 3 `PCycle`s (UM §4.6.2).
195 Itm,
196 /// Multi-cycle interlock: `MULT`/`DIV`/FPU stall the whole pipeline for the
197 /// documented count (UM Tables 3-12, 7-14).
198 Mci,
199 /// Cache operation (UM Table 4-3).
200 Cop,
201 /// CP0 bypass interlock — **1 `PCycle`** (UM §4.6.9, p. 113).
202 ///
203 /// **Named but never raised.** On hardware it fires when an instruction that
204 /// caused an exception reaches `WB` while the next instruction in `DC` reads
205 /// any CP0 register. Here, taking an exception already flushes the pipeline
206 /// and stalls for the 2-`PCycle` epilogue, so the two triggers overlap and
207 /// no case distinguishes them — charging this on top would invent a cycle
208 /// the hardware may not spend. Separating them needs the timing set
209 /// n64-systemtest ships default-off, the same instrument ledger C-1 waits on.
210 ///
211 /// The variant stays because [`Interlock`] enumerates the documented
212 /// taxonomy, and a gap in a taxonomy is worth more than a silently missing
213 /// row. This doc previously said it *fires*, in the present tense, which is
214 /// the comment-is-not-an-implementation hazard `docs/engineering-lessons.md`
215 /// §3.3c describes -- the cost was itself recorded as undocumented in three
216 /// files while sitting in the very paragraph they cited (§3.3b).
217 Cp0i,
218 /// Taking an exception — **2 `PCycle`s** (UM §4.7, p. 114).
219 ///
220 /// Not strictly one of Table 4-3's eight interlocks: the pipeline stalls
221 /// while the epilogue runs and the aborted instructions drain. Named here so
222 /// a trace can attribute the cycles rather than showing an unexplained gap.
223 Exception,
224}
225
226/// A stall request: how long, and what caused it.
227///
228/// ADR 0007 describes an interlock as `(cycles, resume_stage)`. The `resume`
229/// half is **deliberately absent** until it can be load-bearing. Today
230/// [`Pipeline::advance`] always runs the full cascade when not stalled, so a
231/// stored `resume` would be read by nothing — and a field that looks like it
232/// carries information while carrying none is the exact hazard
233/// `docs/engineering-lessons.md` §3.2 is about. `Bus::poll_irq_at_phase` was
234/// removed for the same reason rather than left in place looking wired.
235///
236/// It lands with T-11-002, when stages can stall independently and a partial
237/// resume becomes meaningful.
238#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
239pub struct Stall {
240 /// `PCycle`s remaining.
241 pub cycles: u32,
242 /// Which documented interlock caused it.
243 pub cause: Interlock,
244}
245
246/// State carried across one inter-stage boundary — what travels *with* an
247/// instruction as it advances.
248#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
249pub struct Latch {
250 /// Is an instruction present at this boundary? A bubble is `false`.
251 pub occupied: bool,
252 /// The PC of the instruction in flight.
253 pub pc: u64,
254 /// The raw instruction word (decode is T-11-002).
255 pub word: u32,
256 /// Is this instruction in a branch delay slot?
257 ///
258 /// Travels with the instruction, never as global CPU state — that is the
259 /// whole point (see the module docs).
260 pub in_delay_slot: bool,
261 /// An aborting condition stamped into this latch and every latch upstream.
262 pub abort: Option<Exception>,
263 /// The decoded instruction, filled at `IC`.
264 pub decoded: Decoded,
265 /// `rs` value, read at `RF`.
266 pub rs_val: u64,
267 /// `rt` value, read at `RF`.
268 pub rt_val: u64,
269 /// What `EX` computed, committed at `WB`.
270 pub write_back: WriteBack,
271 /// A memory access `EX` resolved for `DC` to perform.
272 pub mem: Option<MemOp>,
273 /// A COP0 access `EX` resolved: read performed in `DC`, write in `WB`.
274 ///
275 /// Split across those two stages because UM §4.6.9 defines the CP0 bypass
276 /// interlock in terms of a write reaching `WB` while the next instruction
277 /// reads in `DC` — a rule that cannot be expressed if both happen in `EX`.
278 pub cop0: Option<Cop0Access>,
279}
280
281// The inter-stage latch is **copied four times per emulated CPU cycle**, so its size is
282// a performance fact and not merely a layout detail: at ~1.56 M steps a frame those
283// copies are ~15% of the frame (`docs/performance.md` §"The latch copies, anatomized").
284//
285// This pins it. `repr(Rust)` layout is not stable across compiler versions, and the
286// measured breakdown in that document — 120 bytes, with the six scalars occupying
287// exactly their naive sum, so no padding is wasted — would otherwise decay silently
288// into a claim about a toolchain nobody is using any more.
289//
290// **If this fires, do not just change the number.** Either a field was added, in which
291// case re-measure and update the breakdown, or the layout algorithm moved, in which
292// case the "no padding is wasted" conclusion needs re-deriving before it is re-quoted.
293//
294// **120 is not a universal fact**, and the distinction was learned in review. Every
295// field is fixed-width — no `usize`, no references, no pointers — but that does *not*
296// make the layout width-independent, because it is `u64` **alignment** that varies, not
297// pointer size:
298//
299// | target | `size_of::<Latch>()` |
300// | --- | --- |
301// | `x86_64`, `thumbv7em-none-eabihf`, `wasm32-unknown-unknown`, `armv7-*` | 120 |
302// | `i686-*` (32-bit x86, `u64` aligns to 4) | 108 |
303//
304// `armv7` is in that first row by measurement, not by assumption: it is 32-bit with an
305// 8-byte-aligned `u64`, which is the combination the implication below is keyed on, so
306// it is the case that would fail first if the keying were wrong.
307//
308// So there are two assertions rather than one, and neither breaks a cross-compile:
309//
310// 1. **No padding**, universally. This is the property the breakdown actually rests on
311// — that no field ordering would make the struct smaller — and it holds on every
312// target above, `i686` included.
313// 2. **120 where a `u64` aligns to 8**, written as an implication rather than a
314// `#[cfg]`. It pins the documented figure on every ABI the figure describes, and
315// makes no claim on the ones it does not.
316//
317// `#[repr(C)]` is not the alternative either, and this was measured rather than
318// reasoned: `repr(C)` lays fields out in declaration order, which costs **128 bytes**
319// instead of 120 because the two `bool`s can no longer sit in alignment gaps. On a
320// struct copied four times per emulated cycle that adds ~1.2 ms a frame, to exactly
321// the copies `docs/performance.md` is trying to shrink.
322const _: () = {
323 // The scalars: `occupied` + `pc` + `word` + `in_delay_slot` + `rs_val` + `rt_val`.
324 // Written as `size_of` of each field's type rather than as literals, so a field
325 // whose type changes updates this term instead of silently desynchronizing it —
326 // which would mask exactly the padding this is meant to detect.
327 let scalars = core::mem::size_of::<bool>()
328 + core::mem::size_of::<u64>()
329 + core::mem::size_of::<u32>()
330 + core::mem::size_of::<bool>()
331 + core::mem::size_of::<u64>()
332 + core::mem::size_of::<u64>();
333 let parts = core::mem::size_of::<Decoded>()
334 + core::mem::size_of::<Option<Exception>>()
335 + core::mem::size_of::<WriteBack>()
336 + core::mem::size_of::<Option<MemOp>>()
337 + core::mem::size_of::<Option<Cop0Access>>()
338 + scalars;
339 assert!(
340 core::mem::size_of::<Latch>() == parts,
341 "Latch has acquired padding; docs/performance.md's copy-cost breakdown assumes none"
342 );
343 // And the documented figure itself, conditioned on the property that actually
344 // determines it. Writing it as an implication rather than a `#[cfg]` keeps it
345 // compilable everywhere while still pinning 120 on every ABI the number describes:
346 // on 32-bit x86 a `u64` aligns to 4 and `Latch` is 108, which is not a defect and
347 // not something the document claims.
348 assert!(
349 core::mem::align_of::<u64>() != 8 || core::mem::size_of::<Latch>() == 120,
350 "Latch is no longer 120 bytes where a u64 aligns to 8; re-measure docs/performance.md"
351 );
352};
353
354/// Does a load into `load_rt` interlock with the following instruction?
355///
356/// `rs` / `rt` are the *raw encoded fields* of the next instruction,
357/// deliberately named for the encoding rather than for operands — the hardware
358/// checks the fields whether or not they are used as sources, and naming them
359/// `next_rs`/`next_operand` would imply a semantics the check does not have.
360///
361/// Reproduces the hardware's **imprecision**, which is the specification here —
362/// emulating precise behavior is the bug. From
363/// `n64brew_wiki/markdown/VR4300.md` § Microarchitecture → Load Delay Interlock:
364///
365/// - Matches the load's `rt` against the next instruction's `rs` **or** `rt`
366/// field, "whether or not they are actually used as a source". So a load
367/// followed by `LUI` into the same register stalls, and two consecutive loads
368/// into the same register stall.
369/// - A load into `$zero` never interlocks.
370/// - GPR loads interlock only with non-float instructions, and FPR loads only
371/// with float instructions.
372#[must_use]
373pub const fn load_interlocks(load_rt: u8, rs: u8, rt: u8, same_reg_file: bool) -> bool {
374 // The zero register is exempt: a load into $zero is discarded, so nothing
375 // downstream can depend on it.
376 if load_rt == 0 {
377 return false;
378 }
379 if !same_reg_file {
380 return false;
381 }
382 load_rt == rs || load_rt == rt
383}
384
385/// `Status.FR` — whether the FP register file presents 32 independent 64-bit
386/// registers (set) or 16 built from FGR pairs (clear).
387fn fr_of(cop0: &Cop0) -> bool {
388 cop0.read(crate::cop0::reg::STATUS) & (1 << 26) != 0
389}
390
391/// An exception captured at its raising site, with the context the epilogue
392/// needs.
393#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
394struct Pending {
395 /// What happened.
396 exc: Exception,
397 /// The faulting instruction's address.
398 pc: u64,
399 /// Was the faulting instruction in a branch delay slot?
400 in_delay_slot: bool,
401 /// The offending address, for the exceptions that write `BadVAddr`.
402 bad_vaddr: u64,
403}
404
405/// The instruction-granular execution path (ADR 0013), behind `fast-exec`.
406///
407/// A child module so it can reach `Pipeline`'s private fields — privacy extends to
408/// descendants — without widening the surface for anyone else. Absent entirely
409/// with the feature off, so the default build gains nothing (ADR 0011 §1).
410#[cfg(feature = "fast-exec")]
411mod fastexec;
412
413/// How a retired instruction's commit was classified, for
414/// [`Pipeline::commit_census`]. The order is the reporting order.
415#[cfg(feature = "work-counters")]
416pub mod commit_class {
417 /// Commit is a GPR write or nothing: no COP0/COP1 access, no memory op, no
418 /// abort. **This is the set a direct commit path could serve** without a
419 /// second implementation of the subtle cases.
420 pub const SIMPLE: usize = 0;
421 /// Has a memory operation (`DC` work).
422 pub const MEM: usize = 1;
423 /// Touches COP0 or COP1 — the cases whose semantics live in `wb_stage` and
424 /// must not be duplicated.
425 pub const COP: usize = 2;
426 /// Writes `HI`/`LO` (every multiply and divide).
427 pub const HILO: usize = 3;
428 /// Reserved for future instruction classes (aborted instructions never reach retirement).
429 pub const OTHER: usize = 4;
430 /// Number of classes.
431 pub const COUNT: usize = 5;
432}
433
434/// The four inter-stage latches plus the pipeline control state.
435#[derive(Clone, Debug, Default, Serialize, Deserialize)]
436// The bools are independent hardware lines and latches -- `prev_was_run`,
437// `flush_pending`, `nmi_pending`, `ll_bit`. Clippy suggests folding them into a
438// state machine; they do not form one, because any combination is reachable and
439// each is set and cleared by a different part of the machine. Naming them is
440// what makes the exception and interrupt rules readable.
441#[allow(clippy::struct_excessive_bools)]
442pub struct Pipeline {
443 /// Retired-instruction census by commit class, indexed by [`commit_class`].
444 ///
445 /// A **counter**, not a schedule input (ADR 0006). It exists to answer one
446 /// question before any code is written against it: what fraction of executed
447 /// instructions could a direct commit path actually serve? Sizing a change on
448 /// an assumed fraction is how the last program produced four reverts.
449 ///
450 /// `#[serde(skip)]` because it is diagnostic and must not enter the
451 /// save-state layout (ADR 0011 §4).
452 #[cfg(feature = "work-counters")]
453 #[serde(skip)]
454 commit_census: [u64; commit_class::COUNT],
455 /// The **whole** of COP2: one 64-bit latch.
456 ///
457 /// COP2 is not populated on the VR4300, and what remains is a single
458 /// value that every `MTC2`/`DMTC2` writes and every `MFC2`/`DMFC2` reads,
459 /// with the register index ignored. See the `EX` stage and ledger C-20.
460 pub cop2_latch: u64,
461 /// `IC` → `RF`.
462 pub ic_rf: Latch,
463 /// `RF` → `EX`.
464 pub rf_ex: Latch,
465 /// `EX` → `DC`.
466 pub ex_dc: Latch,
467 /// `DC` → `WB`.
468 pub dc_wb: Latch,
469 /// Remaining stall cycles; while non-zero the pipeline does not advance.
470 stall: Option<Stall>,
471 /// Was the *previous* `PCycle` a run cycle (not a stall)?
472 ///
473 /// UM §4.7.1: *"NMI and interrupt exception requests are accepted only if the
474 /// previous `PCycle` was a run cycle."* This is the gate, and it is the
475 /// reason the flag exists at all.
476 prev_was_run: bool,
477 /// The NMI line, latched until an instruction boundary lets it be taken.
478 ///
479 /// A level rather than an edge here: the hardware signal is the falling edge
480 /// of the NMI pin (or bit 6 of the internal interrupt register written over
481 /// `SysAD`), and the edge is what `signal_nmi` represents. Holding it means
482 /// an NMI raised during a stall is taken when the stall drains rather than
483 /// being dropped -- the same reason the timer latches `IP7`.
484 nmi_pending: bool,
485 /// An abort was raised this cycle, so `IC` must fetch a bubble rather than a
486 /// live instruction — the wrong-path fetch would otherwise escape the flush.
487 ///
488 /// Cleared at the end of each [`Pipeline::advance`].
489 flush_pending: bool,
490 /// Instructions retired at `WB` — a work tally, not a time position.
491 pub retired: u64,
492 /// An exception raised this cycle, awaiting dispatch at the end of it.
493 ///
494 /// Captured where it is *raised* rather than reconstructed afterwards,
495 /// because the epilogue needs the faulting instruction's PC and delay-slot
496 /// flag — and by the end of the cycle the reverse cascade has moved every
497 /// latch, so the faulting instruction is no longer where it was.
498 pending: Option<Pending>,
499 /// The virtual address of the access that faulted this cycle.
500 ///
501 /// Recorded where the fault is *detected*, because `BadVAddr` needs the
502 /// address and `Self::access` reports only which exception. Reconstructing
503 /// it at dispatch time is impossible — the `MemOp` has been consumed.
504 fault_vaddr: u64,
505 /// COP1 **control** registers (T-12-006).
506 pub cop1: Cop1Control,
507 /// The floating-point register file (T-13-001).
508 pub fpr: Fpr,
509 /// The joint TLB and its instruction micro-TLB (T-12-004).
510 pub tlb: Tlb,
511 /// The 16 KiB primary instruction cache (T-11-003).
512 pub icache: crate::cache::Icache,
513 /// The PC of a recognized self-branch idle loop, or `None` (ADR 0013 mode only).
514 ///
515 /// `beq $0, $0, -1` followed by a `nop` is a branch to itself: after both
516 /// instructions the PC is back where it started and no register, no memory
517 /// word and no COP0 field other than `Count` and `Random` has changed. Super
518 /// Mario 64 and Mario Kart 64 spend roughly **90% of every instruction they
519 /// retire** in exactly that loop — it is the N64 idle thread — so recognizing
520 /// it and charging its cost without fetching, decoding or executing it is
521 /// worth 1.65x and 1.93x respectively (`docs/performance.md`).
522 ///
523 /// **Why caching this is as correct as the hardware.** The N64 does not snoop
524 /// DMA against the I-cache: software that overwrites code must issue a `CACHE`
525 /// instruction itself, and until it does the CPU keeps executing the stale
526 /// cached line. So a recognition keyed to the I-cache's own lifetime cannot
527 /// diverge from the machine — [`Pipeline::cache_op`] clears this, which covers
528 /// every invalidate, fill and tag write, and so does taking any exception.
529 ///
530 /// Kept on `Pipeline` rather than in `fastexec` so it lives with the `icache`
531 /// whose validity it borrows; it is simply never set on the accurate path.
532 pub(crate) idle_pc: Option<u64>,
533 /// The 8 KiB primary write-back data cache (T-11-003).
534 pub dcache: crate::cache::Dcache,
535 /// The COP0 register file (T-12-001).
536 ///
537 /// Public because exception dispatch, the TLB and the interrupt path all
538 /// read it, and they land in separate tickets.
539 pub cop0: Cop0,
540 /// The link bit, `LLbit`.
541 ///
542 /// *"set by the LL instruction, cleared by an ERET, and tested by the SC
543 /// instruction"* (UM §3.1). Note what is **absent** from that list: `SC`
544 /// itself does not clear it, and neither does an intervening load or store.
545 /// Clearing it in `SC` is the natural-looking mistake, and it makes a
546 /// retried `LL`/`SC` loop fail forever on the second iteration.
547 ///
548 /// `ERET` is the other half and lands with the exception model (Sprint 2);
549 /// until then nothing clears this, which is correct-so-far rather than
550 /// finished — recorded as such in `docs/cpu.md`.
551 ll_bit: bool,
552}
553
554/// `FCSR.C`, the compare condition — bit 23, above the `Cause` field.
555///
556/// Written only by `C.cond.fmt` and read only by the `BC1` family, so it is
557/// deliberately outside the `Cause`/`Flags` bookkeeping that every other FP
558/// operation touches.
559const FCSR_C: u32 = 1 << 23;
560
561impl Pipeline {
562 /// A fresh, empty pipeline.
563 #[must_use]
564 pub const fn new() -> Self {
565 const EMPTY: Latch = Latch {
566 occupied: false,
567 pc: 0,
568 word: 0,
569 in_delay_slot: false,
570 abort: None,
571 cop0: None,
572 decoded: Decoded {
573 op: crate::decode::Op::Reserved,
574 rs: 0,
575 rt: 0,
576 rd: 0,
577 sa: 0,
578 imm: 0,
579 dest: 0,
580 target: 0,
581 },
582 rs_val: 0,
583 rt_val: 0,
584 write_back: WriteBack::None,
585 mem: None,
586 };
587 Self {
588 #[cfg(feature = "work-counters")]
589 commit_census: [0; commit_class::COUNT],
590 // Power-on value: a documented zero (ADR 0004).
591 cop2_latch: 0,
592 ic_rf: EMPTY,
593 rf_ex: EMPTY,
594 ex_dc: EMPTY,
595 dc_wb: EMPTY,
596 stall: None,
597 prev_was_run: false,
598 nmi_pending: false,
599 flush_pending: false,
600 retired: 0,
601 pending: None,
602 fault_vaddr: 0,
603 cop1: Cop1Control::new(),
604 fpr: Fpr::new(),
605 tlb: Tlb::new(),
606 icache: crate::cache::Icache::new(),
607 idle_pc: None,
608 dcache: crate::cache::Dcache::new(),
609 cop0: Cop0::new(),
610 ll_bit: false,
611 }
612 }
613
614 /// The interlock currently stalling the pipeline, if any.
615 #[must_use]
616 pub const fn stalled_by(&self) -> Option<Interlock> {
617 match self.stall {
618 Some(s) => Some(s.cause),
619 None => None,
620 }
621 }
622
623 /// Was the previous `PCycle` a run cycle? Gates interrupt acceptance.
624 #[must_use]
625 pub const fn prev_cycle_was_run(&self) -> bool {
626 self.prev_was_run
627 }
628
629 /// The link bit, as `SC` would test it.
630 ///
631 /// Exposed for the COP0 / `ERET` work in Sprint 2, which must clear it.
632 #[must_use]
633 pub const fn ll_bit(&self) -> bool {
634 self.ll_bit
635 }
636
637 /// `LLAddr` (COP0 register 17): `PA(31:4)` of the most recent `LL`.
638 ///
639 /// Reads straight out of the COP0 file. `LL` writes it there and nowhere
640 /// else — there is deliberately no second copy, because two stores of one
641 /// architectural value drift, and `MFC0 $rt, $17` would then disagree with
642 /// the CPU's own idea of the link address.
643 #[must_use]
644 pub const fn ll_addr(&self) -> u64 {
645 self.cop0.read(crate::cop0::reg::LL_ADDR)
646 }
647
648 /// Request a stall of `cycles` `PCycle`s.
649 ///
650 /// A zero-cycle request is **not** a stall and is ignored. Recording it would
651 /// still consume a cycle in [`Pipeline::advance`] and mark it as not-a-run
652 /// cycle, which silently inserts a bubble *and* suppresses interrupt
653 /// acceptance on the following cycle (UM §4.7.1) — a one-cycle timing error
654 /// with no visible cause.
655 pub const fn stall_for(&mut self, cycles: u32, cause: Interlock) {
656 if cycles == 0 {
657 return;
658 }
659 self.stall = Some(Stall { cycles, cause });
660 }
661
662 /// Assert the **NMI** line.
663 ///
664 /// Latched, then taken at the next instruction boundary. Nothing in the core
665 /// calls this yet: on hardware the source is the console's reset button,
666 /// which reaches the CPU as PRENMI followed by NMI, and the frontend owns
667 /// that button (Phase 6). It exists now because the exception itself is
668 /// Phase 1 work and belongs with the rest of the exception model — the wire
669 /// from the button is a separate, later concern.
670 pub const fn signal_nmi(&mut self) {
671 self.nmi_pending = true;
672 }
673
674 /// Stamp an abort into `at` **and every latch upstream of it** — the
675 /// kill-younger-instructions step. Instructions older than `at` have already
676 /// passed and are unaffected.
677 ///
678 /// # Ordering contract
679 ///
680 /// **A stage must call this BEFORE it moves its latch.** The instruction
681 /// executing in stage S this cycle sits in S's *input* latch until the move,
682 /// so stamping first is what makes the abort travel with the instruction that
683 /// caused it. Calling it after the move stamps the abort onto the *younger*
684 /// instruction instead, and the causing one escapes — a misalignment that no
685 /// single-cycle assertion catches. `an_abort_survives_the_cascade` advances
686 /// the pipeline to verify it, rather than checking latch state in place.
687 ///
688 /// The abort also raises an internal pending-flush flag, so the instruction
689 /// fetched later in the same cycle is a bubble rather than a live
690 /// wrong-path fetch.
691 pub const fn abort_from(&mut self, at: Stage, exc: Exception) {
692 self.abort_with(at, exc, 0);
693 }
694
695 /// [`Pipeline::abort_from`], additionally recording the offending address
696 /// for the exceptions that write `BadVAddr`.
697 pub const fn abort_with(&mut self, at: Stage, exc: Exception, bad_vaddr: u64) {
698 // Capture the faulting instruction's context NOW. The latch holding it
699 // is the one this stage is reading -- `ex_dc` for DC, `rf_ex` for EX --
700 // and by the end of the cycle the cascade will have moved it on.
701 //
702 // Priority: an exception already pending this cycle came from a LATER
703 // stage (the cascade runs WB first), and UM §4.7.2 gives a later stage
704 // precedence over an earlier one. So the first capture wins.
705 if self.pending.is_none() {
706 let src = match at {
707 Stage::Wb => &self.dc_wb,
708 Stage::Dc => &self.ex_dc,
709 Stage::Ex => &self.rf_ex,
710 Stage::Rf | Stage::Ic => &self.ic_rf,
711 };
712 self.pending = Some(Pending {
713 exc,
714 pc: src.pc,
715 in_delay_slot: src.in_delay_slot,
716 bad_vaddr,
717 });
718 }
719 self.flush_pending = true;
720 match at {
721 Stage::Wb => {
722 self.dc_wb.abort = Some(exc);
723 self.ex_dc.abort = Some(exc);
724 self.rf_ex.abort = Some(exc);
725 self.ic_rf.abort = Some(exc);
726 }
727 Stage::Dc => {
728 self.ex_dc.abort = Some(exc);
729 self.rf_ex.abort = Some(exc);
730 self.ic_rf.abort = Some(exc);
731 }
732 Stage::Ex => {
733 self.rf_ex.abort = Some(exc);
734 self.ic_rf.abort = Some(exc);
735 }
736 Stage::Rf | Stage::Ic => self.ic_rf.abort = Some(exc),
737 }
738 }
739
740 /// Advance the pipeline by exactly one `PCycle`.
741 ///
742 /// Stages run **WB → DC → EX → RF → IC**. Because each stage reads its input
743 /// latch before any upstream stage writes it, no value moves two stages in
744 /// one cycle and no double buffering is required. Do not reorder this.
745 ///
746 /// Hot path: allocation-free.
747 pub fn advance<B: Bus>(&mut self, bus: &mut B, regs: &mut Regs, next_pc: &mut u64) {
748 // The timeline is HELD, not advanced. `Count` runs at half PClock, so
749 // bumping it once per PClock here would run the timer at double rate --
750 // and inventing a parity bit to halve it would be a second incremented
751 // counter, which is what ADR 0006 exists to forbid.
752 //
753 // Holding is the honest option: with no scheduler attached there is no
754 // timeline, so `Count` does not move. Anything exercising `Count` or
755 // `Compare` must call `advance_at` and supply the position.
756 self.advance_at(bus, regs, next_pc, self.cop0.count_now());
757 }
758
759 /// [`Pipeline::advance`], with the scheduler's `Count` timeline supplied.
760 ///
761 /// `Count` is **derived** from the master clock (ADR 0006), so the position
762 /// is passed in rather than incremented here. This is the path the scheduler
763 /// uses ([`crate::Cpu::tick_at`]); [`Pipeline::advance`] is a convenience
764 /// for callers with no scheduler, and **holds** the timeline rather than
765 /// guessing at it.
766 pub fn advance_at<B: Bus>(
767 &mut self,
768 bus: &mut B,
769 regs: &mut Regs,
770 next_pc: &mut u64,
771 count_now: u64,
772 ) {
773 self.cop0.set_now(count_now);
774 // A stall consumes the cycle. The pipeline holds its state, and the cycle
775 // is recorded as NOT a run cycle so an interrupt cannot be accepted on the
776 // cycle following it (UM §4.7.1).
777 if let Some(mut s) = self.stall {
778 s.cycles = s.cycles.saturating_sub(1);
779 self.stall = if s.cycles == 0 { None } else { Some(s) };
780 self.prev_was_run = false;
781 // The timer belongs to the clock, not to the pipeline. `Count` keeps
782 // advancing through the stall, so `Compare` can be reached -- and
783 // passed -- entirely inside one. Latching `IP7` here does not accept
784 // the interrupt (that still needs a run cycle, guarded by
785 // `prev_was_run` below); it only records in `Cause` what the hardware
786 // records, so the handler can run once the pipeline resumes.
787 return;
788 }
789
790 self.wb_stage(regs);
791 self.dc_stage(bus);
792 self.ex_stage(regs, next_pc);
793 self.rf_stage(regs);
794 self.ic_stage(bus, next_pc);
795
796 self.prev_was_run = true;
797 self.flush_pending = false;
798
799 // Dispatch AFTER the cascade, so every stage has seen the abort and the
800 // pipeline is drained, and exactly once per cycle regardless of how many
801 // stages raised.
802 if let Some(p) = self.pending.take() {
803 let d = exception::dispatch(&mut self.cop0, p.exc, p.pc, p.in_delay_slot, p.bad_vaddr);
804 *next_pc = d.vector;
805 self.stall_for(d.stall_cycles, Interlock::Exception);
806 }
807 }
808
809 /// The retired-instruction census by commit class ([`commit_class`]).
810 #[cfg(feature = "work-counters")]
811 #[must_use]
812 pub const fn commit_census(&self) -> &[u64; commit_class::COUNT] {
813 &self.commit_census
814 }
815
816 /// Classify one retired instruction for the census.
817 ///
818 /// Takes the pieces rather than the assembled `Latch`, so it can be called
819 /// from the fast path *before* the latch is built — the whole point being to
820 /// find out how often building it is avoidable.
821 #[cfg(feature = "work-counters")]
822 pub(crate) fn count_commit(
823 &mut self,
824 cop0: Option<crate::exec::Cop0Access>,
825 mem: Option<crate::exec::MemOp>,
826 write_back: crate::exec::WriteBack,
827 ) {
828 use crate::exec::WriteBack as W;
829 // Order matters: an instruction with BOTH a memory op and a COP0 access
830 // is not simple, and must not be counted as if the cheaper class applied.
831 let class = if cop0.is_some() {
832 commit_class::COP
833 } else if mem.is_some() {
834 commit_class::MEM
835 } else {
836 match write_back {
837 W::None | W::Gpr { .. } => commit_class::SIMPLE,
838 W::HiLo(_) | W::Hi(_) | W::Lo(_) => commit_class::HILO,
839 #[allow(unreachable_patterns, reason = "future WriteBack variants land here")]
840 _ => commit_class::OTHER,
841 }
842 };
843 self.commit_census[class] = self.commit_census[class].saturating_add(1);
844 }
845
846 /// `WB` — commit the result and retire the instruction.
847 fn wb_stage(&mut self, regs: &mut Regs) {
848 if self.dc_wb.occupied && self.dc_wb.abort.is_none() {
849 // The COP0 WRITE lands here (UM §4.6.9). A `Read` in this latch was
850 // already performed in DC and left its value in `write_back`.
851 // The TLB instructions land in WB, with the COP0 write: they are
852 // CP0 operations and `TLBR`/`TLBP` write COP0 registers, so doing
853 // them earlier would let a following `MFC0` read the result a cycle
854 // before hardware produces it.
855 if let Some(Cop0Access::Cop1(Cop1Access::WriteFpr { dest, value, wide })) =
856 self.dc_wb.cop0
857 {
858 // DMTC1 mirrors DMFC1: the FR view, not the physical register.
859 if wide {
860 let fr = fr_of(&self.cop0);
861 self.fpr.write_d(dest, fr, value);
862 } else {
863 self.fpr.write_s(dest, fr_of(&self.cop0), value as u32);
864 }
865 }
866 if let Some(Cop0Access::Cop1(Cop1Access::WriteControl { dest, value })) =
867 self.dc_wb.cop0
868 {
869 self.cop1.ctc1(dest, value);
870 // **`CTC1` can raise on its own.** Writing `FCSR` with a Cause
871 // bit whose Enable is also set meets the trap condition
872 // immediately -- no arithmetic required. n64-systemtest writes
873 // `enable_overflow | cause_overflow` in one go and expects the
874 // FP exception to report the `CTC1` itself as `ExceptPC`.
875 //
876 // Bit 17 (Unimplemented) is unmaskable and traps regardless,
877 // which is why it is tested outside the enable comparison.
878 if self.fcsr_traps_now() {
879 self.abort_from(Stage::Wb, Exception::FloatingPoint);
880 self.dc_wb.occupied = false;
881 return;
882 }
883 }
884 if let Some(Cop0Access::Cop1(Cop1Access::Arith {
885 fmt,
886 funct,
887 ft,
888 fs,
889 fd,
890 })) = self.dc_wb.cop0
891 {
892 if self.fp_arith(fmt, funct, ft, fs, fd) {
893 // Trapped. The instruction does **not** complete, so it must
894 // not reach the retirement tail below: `Random` "decrements
895 // as each instruction executes" (UM §5.4.2), and one that
896 // took an exception did not execute.
897 //
898 // No delay is charged either. A trapped operation abandons
899 // its result, and the exception's own 2-PCycle epilogue is
900 // the cost that applies.
901 self.dc_wb.occupied = false;
902 return;
903 }
904 // The multi-cycle FPU operations stall the pipeline for their
905 // documented rate (UM Table 7-14). The manual's "latency = rate
906 // + 1 for a dependent consumer" falls out of this rather than
907 // being added: the stall holds every stage, so a consumer spends
908 // its own cycle once the stall drains.
909 self.stall_for(crate::fpu::stall_cycles(funct, fmt), Interlock::Mci);
910 }
911 if let Some(Cop0Access::Tlb(op)) = self.dc_wb.cop0 {
912 match op {
913 TlbOp::Read => {
914 let i = (self.cop0.read(crate::cop0::reg::INDEX) & 0x3F) as usize;
915 self.tlb.read_entry(i, &mut self.cop0);
916 }
917 TlbOp::WriteIndexed => {
918 let i = (self.cop0.read(crate::cop0::reg::INDEX) & 0x3F) as usize;
919 // TLBWI CAN overwrite a wired entry; only TLBWR cannot
920 // (UM §5.4.4, p. 150). Guarding both is a natural-looking
921 // mistake that makes wired entries unwritable at all.
922 self.tlb.write_entry(i, &self.cop0);
923 }
924 TlbOp::WriteRandom => {
925 // `Random` never goes below `Wired`, so the wired entries
926 // are protected by the counter's range rather than by a
927 // check here -- which is how hardware does it.
928 let i = (self.cop0.read(crate::cop0::reg::RANDOM) & 0x3F) as usize;
929 self.tlb.write_entry(i, &self.cop0);
930 }
931 TlbOp::Probe => self.tlb.probe(&mut self.cop0),
932 }
933 }
934 if let Some(Cop0Access::Write { dest, value, wide }) = self.dc_wb.cop0 {
935 if wide {
936 self.cop0.dmtc0(dest, value);
937 } else {
938 self.cop0.mtc0(dest, value);
939 }
940 }
941 match self.dc_wb.write_back {
942 WriteBack::None => {}
943 // `Regs::write` discards `$zero`, so no guard is needed here --
944 // and must not be added, or the rule lives in two places.
945 WriteBack::Gpr { dest, value } => regs.write(dest, value),
946 WriteBack::HiLo(hl) => {
947 regs.hi = hl.hi;
948 regs.lo = hl.lo;
949 }
950 WriteBack::Hi(v) => regs.hi = v,
951 WriteBack::Lo(v) => regs.lo = v,
952 }
953 self.retired = self.retired.wrapping_add(1);
954 #[cfg(feature = "work-counters")]
955 self.count_commit(self.dc_wb.cop0, self.dc_wb.mem, self.dc_wb.write_back);
956 // "Random decrements as each instruction executes" (UM §5.4.2,
957 // p. 147) -- advanced HERE, at retirement, so it counts executed
958 // instructions rather than cycles.
959 //
960 // This was implemented and then never called from the pipeline, so
961 // `Random` sat at 31 forever and **every `TLBWR` overwrote the same
962 // entry**. A refill handler that needs more than one mapping live at
963 // once therefore destroys its previous entry on each miss and faults
964 // again immediately -- an infinite refill loop, which is exactly what
965 // n64-systemtest hit. A stuck counter is invisible to any test that
966 // calls `tick_random` itself.
967 self.cop0.tick_random();
968 }
969 self.dc_wb.occupied = false;
970 }
971
972 /// Perform the COP0/COP1 **reads** an instruction resolved, leaving the value
973 /// in the latch's `write_back` for the commit to store.
974 ///
975 /// The fourth latch-independent primitive (`docs/cpu.md`), and the one PR #231
976 /// deliberately left alone: at that point it had a single caller, and the shape
977 /// of a seam guessed without a second caller is a guess. `fast-exec`'s
978 /// sequential path is that caller, so the extraction is now driven rather than
979 /// speculative.
980 ///
981 /// It takes the latch **by reference** rather than the fields by value because
982 /// the three reads each replace `write_back` wholesale, and threading four
983 /// values in and one out would be a worse signature than the one the accurate
984 /// path already has.
985 ///
986 /// A **read** happens here, in `DC`; a **write** happens at `WB`. They are
987 /// split across the two stages because UM §4.6.9 defines the CP0 bypass
988 /// interlock in terms of a write reaching `WB` while the next instruction reads
989 /// in `DC` — a rule that cannot be expressed if both happen in one stage.
990 fn apply_cop0_read(&self, out: &mut Latch) {
991 if out.occupied
992 && out.abort.is_none()
993 && let Some(Cop0Access::Cop1(Cop1Access::ReadFpr { src, dest, wide })) = out.cop0
994 {
995 out.write_back = WriteBack::Gpr {
996 dest,
997 // DMFC1 applies the FR view -- it does NOT move the physical
998 // register. UM Ch. 17's pseudocode is explicit:
999 //
1000 // if FR = 1 then data <- FGR[fs]
1001 // else if fs0 = 0 then data <- FGR[fs+1] || FGR[fs]
1002 // else data <- undefined
1003 //
1004 // So with FR = 0 and an even `fs` it reads the PAIR, exactly like
1005 // LDC1. Only an odd `fs` with FR = 0 is undefined -- and it is
1006 // *undefined*, not a Reserved Instruction exception.
1007 //
1008 // MFC1 moves the low word, sign-extended, in both modes.
1009 value: if wide {
1010 self.fpr.read_d(src, fr_of(&self.cop0))
1011 } else {
1012 crate::alu::sext32(self.fpr.read_s(src, fr_of(&self.cop0)))
1013 },
1014 };
1015 }
1016 if out.occupied
1017 && out.abort.is_none()
1018 && let Some(Cop0Access::Cop1(Cop1Access::ReadControl { src, dest })) = out.cop0
1019 {
1020 out.write_back = WriteBack::Gpr {
1021 dest,
1022 // CFC1 is a 32-bit move, so the result is sign-extended into the
1023 // 64-bit GPR exactly as MFC0's is.
1024 value: crate::alu::sext32(self.cop1.cfc1(src)),
1025 };
1026 }
1027 if out.occupied
1028 && out.abort.is_none()
1029 && let Some(Cop0Access::Read { src, dest, wide }) = out.cop0
1030 {
1031 let value = if wide {
1032 self.cop0.dmfc0(src)
1033 } else {
1034 self.cop0.mfc0(src)
1035 };
1036 out.write_back = WriteBack::Gpr { dest, value };
1037 }
1038 }
1039
1040 /// Update the `Cause.IP` lines from the hardware: the RCP's aggregate
1041 /// interrupt and the `Count == Compare` timer edge.
1042 ///
1043 /// **Asserting a line is not recognizing an interrupt**, and separating the
1044 /// two is the point. The `IP` bits track what the hardware is *asserting*
1045 /// regardless of masks; recognition then applies `IE`/`EXL`/`ERL`/`IM` and is
1046 /// the caller's business. Folding them together would make a masked interrupt
1047 /// invisible to `MFC0 Cause`, which software polls.
1048 ///
1049 /// Latch-independent, like [`Self::fetch_word`] and [`Self::ex_gate`]: it
1050 /// stamps nothing and names no [`Stage`]. Asserting a line happens on
1051 /// hardware whether or not the CPU is in a position to take the interrupt, so
1052 /// the instruction-granular path (ADR 0013) needs exactly this and nothing
1053 /// more.
1054 ///
1055 /// `IP2` is the RCP's aggregate line from the MI (libdragon `cop0.h`:
1056 /// `C0_INTERRUPT_RCP = C0_INTERRUPT_2`; ledger U-4). `IP3` is CART, `IP4`
1057 /// PRENMI, `IP7` the timer; the rest are unused on this board.
1058 fn sample_interrupt_lines<B: Bus>(&mut self, bus: &mut B) {
1059 self.cop0.set_ip(2, bus.poll_irq());
1060 // IP7 is LATCHED on the match and stays set until `Compare` is written
1061 // (UM §6.4.18, p. 200) -- note the one-way `if`, with no `else` clearing
1062 // it. Modeling it as a level tied to `Count == Compare` looks tidier
1063 // and silently DROPS any timer interrupt that fires while `EXL` is set,
1064 // because the equality holds for one tick and the handler never sees it.
1065 //
1066 // The trigger is the rising EDGE of the match, not the standing
1067 // equality: both `Count` and `Compare` reset to zero, so an equality
1068 // test latches `IP7` before a single instruction retires. See
1069 // `Cop0::timer_edge`.
1070 if self.cop0.timer_edge() {
1071 self.cop0.set_ip(7, true);
1072 }
1073 }
1074
1075 /// `DC` — the data-cache access, and the interrupt sampling point.
1076 ///
1077 /// The stage placement is documented, not inherited from a reference
1078 /// implementation: UM Figure 4-12 puts `INTR` in the `DC` column and §4.7.6
1079 /// "DC-Stage Interlock and Exception Priorities" lists the interrupt
1080 /// exception among them.
1081 fn dc_stage<B: Bus>(&mut self, bus: &mut B) {
1082 // Sample the lines once per PCycle here. `sample_interrupt_lines` carries
1083 // why asserting and recognizing are two different steps; what follows is
1084 // the recognition half.
1085 self.sample_interrupt_lines(bus);
1086
1087 // Accepted only if the previous PCycle was a run cycle (UM §4.7.1). This
1088 // is the ONLY interrupt recognition predicate in the tree -- carrying two
1089 // subtly different ones is a known source of one-cycle discrepancies in
1090 // other emulators.
1091
1092 // NMI is checked FIRST and without consulting `interrupt_pending`.
1093 // *"Unlike all other interrupts, this interrupt is not maskable; it
1094 // occurs regardless of the settings of the EXL, ERL, and the IE bits"*
1095 // (UM SS6.4.6). Only the run-cycle gate applies, because NMI is still
1096 // *"taken only at instruction boundaries"* -- which is exactly what
1097 // `prev_was_run` expresses (UM SS4.7.1).
1098 if self.prev_was_run && self.nmi_pending {
1099 self.nmi_pending = false;
1100 self.abort_from(Stage::Dc, Exception::Nmi);
1101 } else if self.prev_was_run
1102 && self.cop0.interrupt_pending()
1103 && self.interrupt_has_an_instruction_to_charge()
1104 {
1105 self.abort_from(Stage::Dc, Exception::Interrupt);
1106 }
1107 // The memory access. This is the point the scheduler interleaves the RCP
1108 // around -- the whole reason the pipeline is modeled at all (ADR 0007).
1109 let mut out = self.ex_dc;
1110 if out.occupied
1111 && out.abort.is_none()
1112 && let Some(op) = out.mem
1113 {
1114 match self.access(bus, op) {
1115 Ok(wb) => out.write_back = wb,
1116 // Stamp before the latch move so the abort travels with the
1117 // instruction that caused it -- see `abort_from`.
1118 Err(exc) => {
1119 self.abort_with(Stage::Dc, exc, self.fault_vaddr);
1120 out = self.ex_dc;
1121 }
1122 }
1123 }
1124 // The COP0 READ happens here, in DC (UM §4.6.9). The write does not --
1125 // it happens in WB, and keeping them in different stages is what makes
1126 // the CP0 bypass interlock expressible at all.
1127 self.apply_cop0_read(&mut out);
1128 self.dc_wb = out;
1129 self.ex_dc.occupied = false;
1130 }
1131
1132 /// The operand bypass network (UM §4.6).
1133 ///
1134 /// *"Bypassing ... allows data and conditions produced in the `EX`, `DC` and
1135 /// `WB` stages to be made available to the `EX` stage of the next cycle."*
1136 ///
1137 /// Without this, back-to-back dependent instructions read stale registers and
1138 /// essentially every real program computes wrong values — `LUI`+`ORI`, the
1139 /// standard way to build a 32-bit constant, breaks immediately. Its absence
1140 /// was invisible to every unit test in this crate and was caught only by
1141 /// `a_program_executes_through_the_whole_pipeline`.
1142 ///
1143 /// By the time `EX` runs, the reverse cascade has already committed one
1144 /// instruction (`WB` ran first, so the register file is current) and moved the
1145 /// next into `dc_wb`. Exactly **one** producer can therefore still be
1146 /// uncommitted, and `dc_wb` is it.
1147 ///
1148 /// Loads are the case this does *not* cover — a load's value is not ready in
1149 /// time, which is precisely why the hardware has a load-delay interlock. That
1150 /// lands with T-11-003 alongside the loads themselves.
1151 fn bypass(&self, reg: u8, regs: &Regs) -> u64 {
1152 if reg != 0
1153 && self.dc_wb.occupied
1154 && self.dc_wb.abort.is_none()
1155 && let WriteBack::Gpr { dest, value } = self.dc_wb.write_back
1156 && dest == reg
1157 {
1158 return value;
1159 }
1160 regs.read(reg)
1161 }
1162
1163 /// `HI`/`LO` as `EX` should see them, bypassing an uncommitted producer.
1164 fn bypass_hi_lo(&self, regs: &Regs) -> HiLo {
1165 if self.dc_wb.occupied && self.dc_wb.abort.is_none() {
1166 match self.dc_wb.write_back {
1167 WriteBack::HiLo(hl) => return hl,
1168 WriteBack::Hi(v) => return HiLo { hi: v, lo: regs.lo },
1169 WriteBack::Lo(v) => return HiLo { hi: regs.hi, lo: v },
1170 _ => {}
1171 }
1172 }
1173 HiLo {
1174 hi: regs.hi,
1175 lo: regs.lo,
1176 }
1177 }
1178
1179 /// Map a TLB fault to the exception it raises.
1180 ///
1181 /// The `store` flag selects `TLBL` vs `TLBS`; the *variant* selects the
1182 /// vector. Both matter and they are independent. `wide` is the addressing
1183 /// width of the faulting access, which picks the XTLB refill vector over the
1184 /// 32-bit one -- it is only meaningful for a refill, since every other TLB
1185 /// fault takes the general vector regardless.
1186 const fn tlb_exception(f: crate::tlb::TlbFault, store: bool, wide: bool) -> Exception {
1187 match f {
1188 crate::tlb::TlbFault::Refill => Exception::TlbRefill { store, wide },
1189 crate::tlb::TlbFault::Invalid => Exception::TlbInvalid { store },
1190 // Modified only ever arises on a store, so it carries no flag.
1191 crate::tlb::TlbFault::Modified => Exception::TlbModified,
1192 }
1193 }
1194
1195 /// Translate a data address through the TLB.
1196 ///
1197 /// Returns the whole [`crate::addr::Physical`], **cacheability included**.
1198 /// It used to return the address alone, which is exactly the information a
1199 /// D-cache needs and cannot recover: once KSEG0 and KSEG1 have both become
1200 /// the same physical number, nothing downstream can tell them apart.
1201 fn translate_data(
1202 &mut self,
1203 vaddr: u64,
1204 store: bool,
1205 ) -> Result<crate::addr::Physical, Exception> {
1206 let asid = (self.cop0.read(crate::cop0::reg::ENTRY_HI) & 0xFF) as u8;
1207 let access = self.access_mode();
1208 match crate::addr::translate_via(&mut self.tlb, vaddr, asid, store, access) {
1209 Ok(p) => Ok(p),
1210 Err(e) => {
1211 self.fault_vaddr = vaddr;
1212 self.note_shutdown();
1213 Err(match e {
1214 crate::addr::TranslateError::Address => Exception::AddressError { store },
1215 crate::addr::TranslateError::Tlb(f) => {
1216 Self::tlb_exception(f, store, access.wide)
1217 }
1218 })
1219 }
1220 }
1221 }
1222
1223 /// The unaligned `LWL`/`LWR`/`LDL`/`LDR`/`SWL`/`SWR`/`SDL`/`SDR` family.
1224 ///
1225 /// Split out of [`Pipeline::access`] purely for size; the merging rules live
1226 /// in [`crate::mem`] and the alignment exemption is by construction — being
1227 /// usable at any byte offset is the entire reason these instructions exist.
1228 ///
1229 /// # Errors
1230 ///
1231 /// A TLB fault on the container address.
1232 fn access_unaligned<B: Bus>(
1233 &mut self,
1234 bus: &mut B,
1235 op: crate::decode::Op,
1236 addr: u64,
1237 rt: u64,
1238 dest: u8,
1239 ) -> Result<WriteBack, Exception> {
1240 use crate::decode::Op;
1241 // The unaligned family splits into loads and stores, and the
1242 // TLB check differs: a store must find the `D` bit set.
1243 let is_store = matches!(op, Op::Swl | Op::Swr | Op::Sdl | Op::Sdr);
1244 // On a fault, `BadVAddr` reports the address the INSTRUCTION named, not
1245 // the container address translated on its behalf. `translate_data`
1246 // records whatever it was handed, so the aligned-down value has to be
1247 // corrected back -- otherwise a fault on `SWL 0x12345001` reports
1248 // `0x12345000`, and n64-systemtest checks that exact case.
1249 let translate = |p: &mut Self, a: u64| match p.translate_data(a, is_store) {
1250 Ok(v) => Ok(v),
1251 Err(e) => {
1252 p.fault_vaddr = addr;
1253 Err(e)
1254 }
1255 };
1256 // `Status.RE` applies the **byte** swap here, `addr ^ 7`, not the swap for
1257 // the container's width. These instructions address individual bytes, so
1258 // the byte lane is what moves — and one XOR relocates the container and
1259 // complements the byte index together, which is why the two do not need
1260 // separate treatment:
1261 //
1262 // `LWL 0` becomes container 4 with byte index 3, because
1263 // `0 ^ 7 == 7`, `7 & !3 == 4` and `7 & 3 == 3`.
1264 //
1265 // Derived from n64-systemtest's own expected tables rather than guessed:
1266 // `SWL` at offset 0 writes a single byte, `rt`'s most significant, into
1267 // the doubleword's LAST byte, which no width-based swap produces.
1268 let eaddr = if self.reverse_endian() {
1269 addr ^ 7
1270 } else {
1271 addr
1272 };
1273 let word_addr = translate(self, eaddr & !3)?;
1274 let dword_addr = translate(self, eaddr & !7)?;
1275 let byte4 = eaddr & 3;
1276 let byte8 = eaddr & 7;
1277 Ok(match op {
1278 Op::Lwl | Op::Lwr => {
1279 let w = self.read_width(bus, word_addr, 4) as u32;
1280 let v = if matches!(op, Op::Lwl) {
1281 mem::lwl(rt, w, byte4)
1282 } else {
1283 mem::lwr(rt, w, byte4)
1284 };
1285 WriteBack::Gpr { dest, value: v }
1286 }
1287 Op::Ldl | Op::Ldr => {
1288 let d = self.read_width(bus, dword_addr, 8);
1289 let v = if matches!(op, Op::Ldl) {
1290 mem::ldl(rt, d, byte8)
1291 } else {
1292 mem::ldr(rt, d, byte8)
1293 };
1294 WriteBack::Gpr { dest, value: v }
1295 }
1296 Op::Swl | Op::Swr => {
1297 let w = self.read_width(bus, word_addr, 4) as u32;
1298 let merged = if matches!(op, Op::Swl) {
1299 mem::swl(rt, w, byte4)
1300 } else {
1301 mem::swr(rt, w, byte4)
1302 };
1303 self.write_width(bus, word_addr, 4, u64::from(merged));
1304 WriteBack::None
1305 }
1306 Op::Sdl | Op::Sdr => {
1307 let d = self.read_width(bus, dword_addr, 8);
1308 let merged = if matches!(op, Op::Sdl) {
1309 mem::sdl(rt, d, byte8)
1310 } else {
1311 mem::sdr(rt, d, byte8)
1312 };
1313 self.write_width(bus, dword_addr, 8, merged);
1314 WriteBack::None
1315 }
1316 // `MemOp::Unaligned` is only ever constructed for the eight
1317 // forms above.
1318 _ => WriteBack::None,
1319 })
1320 }
1321
1322 /// Which coprocessor an instruction needs, if that unit is disabled.
1323 ///
1324 /// `Status.CU` (31:28) is one bit per unit. Two rules that are easy to miss:
1325 ///
1326 /// - **COP0 is usable from kernel mode regardless of `CU0`** — otherwise the
1327 /// CPU could not run an exception handler before `Status` had been set up,
1328 /// which is a chicken-and-egg the hardware does not have. Kernel mode is
1329 /// `KSU == 0`, or `EXL`/`ERL` set.
1330 /// - A **valid but unimplemented** COP1 encoding still checks `CU1`. With
1331 /// `CU1` set it must *not* raise here, so that Sprint 3's arithmetic is an
1332 /// addition rather than a behavior change.
1333 fn unusable_coprocessor(&self, d: Decoded) -> Option<u8> {
1334 use crate::decode::Op;
1335 let unit = match d.op {
1336 Op::Mfc0
1337 | Op::Dmfc0
1338 | Op::Mtc0
1339 | Op::Dmtc0
1340 | Op::Tlbr
1341 | Op::Tlbwi
1342 | Op::Tlbwr
1343 | Op::Tlbp
1344 | Op::Eret => 0,
1345 Op::Cfc1
1346 | Op::Ctc1
1347 | Op::Mfc1
1348 | Op::Dmfc1
1349 | Op::Mtc1
1350 | Op::Dmtc1
1351 | Op::Lwc1
1352 | Op::Ldc1
1353 | Op::Swc1
1354 | Op::Sdc1
1355 | Op::Cop1Unimplemented
1356 | Op::Cop1ReservedControl
1357 // FP arithmetic is a COP1 instruction like any other and must raise
1358 // Coprocessor Unusable with `CU1` clear. It was omitted when
1359 // `FpArith` was introduced, which left the arithmetic executing
1360 // unconditionally -- a program that had not enabled COP1 would get
1361 // results instead of an exception.
1362 | Op::FpArith => 1,
1363 Op::Cop2 | Op::Cop2ReservedControl | Op::Mfc2 | Op::Dmfc2 | Op::Mtc2 => 2,
1364 _ => return None,
1365 };
1366 let status = self.cop0.read(crate::cop0::reg::STATUS);
1367 if unit == 0 {
1368 /// `Status.KSU` (4:3) — 0 is kernel, 1 supervisor, 2 user.
1369 const KSU: u64 = 0b11 << 3;
1370 /// `Status.EXL` (1) or `Status.ERL` (2): either forces kernel mode
1371 /// regardless of `KSU`, which is what makes an exception handler's
1372 /// first instructions safe.
1373 const EXL_OR_ERL: u64 = 0b110;
1374 let kernel = status & KSU == 0 || status & EXL_OR_ERL != 0;
1375 if kernel {
1376 return None;
1377 }
1378 }
1379 if status & (1 << (28 + u64::from(unit))) == 0 {
1380 return Some(unit);
1381 }
1382 None
1383 }
1384
1385 /// `Status.RE` — reverse endian, and **only in User mode** (UM §5.2).
1386 ///
1387 /// `EXL`/`ERL` force kernel mode regardless of `KSU`, so an exception
1388 /// handler reads memory the way it wrote it even with `RE` set. That is the
1389 /// same rule the coprocessor-usability check applies, for the same reason.
1390 fn reverse_endian(&self) -> bool {
1391 /// `Status.RE`, bit 25.
1392 const RE: u64 = 1 << 25;
1393 /// `Status.KSU` (4:3) — 2 is user.
1394 const KSU_USER: u64 = 0b10 << 3;
1395 /// `Status.KSU` (4:3).
1396 const KSU: u64 = 0b11 << 3;
1397 /// `Status.EXL` (1) or `Status.ERL` (2).
1398 const EXL_OR_ERL: u64 = 0b110;
1399 let status = self.cop0.read(crate::cop0::reg::STATUS);
1400 status & RE != 0 && status & KSU == KSU_USER && status & EXL_OR_ERL == 0
1401 }
1402
1403 /// The reverse-endian byte-lane swap for an access of `width` bytes.
1404 ///
1405 /// Reversing endianness on a 64-bit datapath is a permutation of byte lanes
1406 /// within the doubleword, expressed as an XOR of the low address bits: a
1407 /// doubleword access covers the whole lane set and does not move, a word
1408 /// moves by 4, a halfword by 6 and a byte by 7.
1409 const fn re_swap(width: u64) -> u32 {
1410 match width {
1411 1 => 7,
1412 2 => 6,
1413 4 => 4,
1414 _ => 0,
1415 }
1416 }
1417
1418 /// Translate a data address and apply the reverse-endian swap, if any.
1419 ///
1420 /// The swap is applied to the **physical** address, after translation. It
1421 /// touches only bits 2:0, which every translation maps identically, so this
1422 /// is exactly equivalent to swapping the virtual address first — and it
1423 /// keeps `BadVAddr` raw on a fault, which n64-systemtest asserts directly
1424 /// ("RE unmapped access keeps raw `BadVAddr`").
1425 ///
1426 /// # Errors
1427 ///
1428 /// Whatever [`Pipeline::translate_data`] raises on the untransformed
1429 /// address: a TLB fault, **or** [`Exception::AddressError`] when the address
1430 /// is not valid in the current privilege mode.
1431 fn translate_re(
1432 &mut self,
1433 vaddr: u64,
1434 store: bool,
1435 width: u64,
1436 ) -> Result<crate::addr::Physical, Exception> {
1437 let mut p = self.translate_data(vaddr, store)?;
1438 if self.reverse_endian() {
1439 p.addr ^= Self::re_swap(width);
1440 }
1441 Ok(p)
1442 }
1443
1444 /// The privilege mode and addressing width the next access runs under.
1445 ///
1446 /// `EXL`/`ERL` force Kernel regardless of `KSU` — the same rule the
1447 /// coprocessor-usability check applies, and what makes an exception
1448 /// handler's first instructions safe.
1449 ///
1450 /// The addressing-width bit is picked **for the resolved mode**, not for
1451 /// `KSU`: a User-mode program that takes an exception runs the handler in
1452 /// Kernel mode, and the handler's addresses are governed by `KX`.
1453 fn access_mode(&self) -> crate::addr::Access {
1454 /// `Status.KSU`, bits 4:3.
1455 const KSU: u64 = 0b11 << 3;
1456 /// `Status.EXL` (1) or `Status.ERL` (2).
1457 const EXL_OR_ERL: u64 = 0b110;
1458 let status = self.cop0.read(crate::cop0::reg::STATUS);
1459 let mode = if status & EXL_OR_ERL != 0 {
1460 crate::addr::Mode::Kernel
1461 } else {
1462 match (status & KSU) >> 3 {
1463 1 => crate::addr::Mode::Supervisor,
1464 2 => crate::addr::Mode::User,
1465 // 0 is Kernel. 3 is not a defined encoding and the manual
1466 // gives it no behavior; it falls in with Kernel because that is
1467 // what `KSU == 0` already does and it keeps the match total
1468 // without inventing a fourth mode. Note this is the most
1469 // PERMISSIVE choice, not the safest one -- if a test ever pins
1470 // `KSU == 3`, it belongs in the accuracy ledger, not here.
1471 _ => crate::addr::Mode::Kernel,
1472 }
1473 };
1474 // `Status.UX` (5), `SX` (6), `KX` (7).
1475 let wide = match mode {
1476 crate::addr::Mode::User => status & (1 << 5) != 0,
1477 crate::addr::Mode::Supervisor => status & (1 << 6) != 0,
1478 crate::addr::Mode::Kernel => status & (1 << 7) != 0,
1479 };
1480 crate::addr::Access {
1481 mode,
1482 wide,
1483 erl: self.erl(),
1484 }
1485 }
1486
1487 /// Is there an instruction in `DC` that an interrupt can legitimately be
1488 /// charged to (ledger **R-18**)?
1489 ///
1490 /// An interrupt is taken *"at instruction boundaries"* (UM §4.7.1), and
1491 /// [`Pipeline::abort_from`] records `EPC` from whatever sits in `ex_dc`. Two
1492 /// things can sit there that are **not** an instruction boundary, and
1493 /// charging either one corrupts the return address:
1494 ///
1495 /// - **A bubble.** A default [`Latch`] has `pc == 0`, so the interrupt is
1496 /// charged to address 0 and the handler returns there.
1497 /// - **An `ERET` that has already redirected.** `ERET` resolves in `EX`
1498 /// (see [`Pipeline::ex_stage`]), clearing `Status.EXL` and pointing
1499 /// `next_pc` at `EPC`. One cycle later the `DC` check sees `EXL == 0` and
1500 /// fires — onto the `ERET` itself, overwriting the very `EPC` it was about
1501 /// to consume. The handler then returns to the `ERET`, which resumes at its
1502 /// own address: an architectural **livelock**. This is what stopped
1503 /// Banjo-Tooie dead, measured at 390,625 `ERET` retirements per frame with
1504 /// no other instruction ever reaching `WB`.
1505 ///
1506 /// Both must be excluded together. Excluding only the `ERET` moves the
1507 /// corruption to the refill bubble that follows its redirect (`EPC == 0`);
1508 /// excluding only bubbles leaves the `ERET` livelock intact.
1509 ///
1510 /// **Deferring loses nothing, by two different mechanisms** — and the
1511 /// distinction matters, because `Cause.IP` is *not* uniformly a bus level:
1512 ///
1513 /// - **`IP2`** (the RCP line) is re-sampled from `bus.poll_irq()` every
1514 /// cycle, so a still-asserted line is simply seen again next cycle.
1515 /// - **`IP7`** (the timer) is **latched** on the `Count`/`Compare` edge and
1516 /// stays set until `Compare` is written (UM §6.4.18) — see the one-way
1517 /// `if` in [`Pipeline::dc_stage`]. It survives deferral because it is
1518 /// sticky, not because it is re-sampled.
1519 ///
1520 /// Either way the interrupt is taken on the next cycle that presents a real
1521 /// instruction — after an `ERET`, the instruction it returned to. It is
1522 /// re-attributed, never dropped.
1523 fn interrupt_has_an_instruction_to_charge(&self) -> bool {
1524 self.ex_dc.occupied && !matches!(self.ex_dc.cop0, Some(Cop0Access::Eret))
1525 }
1526
1527 /// Are the MIPS III 64-bit operations reserved right now?
1528 ///
1529 /// Yes in User or Supervisor mode with that mode's `UX`/`SX` bit clear; never
1530 /// in Kernel mode, whatever `KX` says. Both halves are load-bearing: gating
1531 /// on the width bit alone would reserve them for a 32-bit kernel, and gating
1532 /// on the mode alone would reserve them for a 64-bit user program.
1533 fn sixty_four_bit_is_reserved(&self) -> bool {
1534 let a = self.access_mode();
1535 !matches!(a.mode, crate::addr::Mode::Kernel) && !a.wide
1536 }
1537
1538 /// `Status.ERL` — the error level, which makes KUSEG unmapped (UM §5.2.2).
1539 fn erl(&self) -> bool {
1540 self.cop0.read(crate::cop0::reg::STATUS) & (1 << 2) != 0
1541 }
1542
1543 /// Mirror a TLB shutdown into `Status.TS`.
1544 ///
1545 /// `TS` is read-only to software (UM Fig. 6-6, p. 167), so it goes through
1546 /// `set_hardware`. Without this the shutdown flag would be recorded inside
1547 /// the TLB and never observed, which is worse than not tracking it: software
1548 /// polls `Status.TS` precisely to discover that the TLB has died.
1549 fn note_shutdown(&mut self) {
1550 if self.tlb.is_shutdown() {
1551 let status = self.cop0.read(crate::cop0::reg::STATUS);
1552 self.cop0
1553 .set_hardware(crate::cop0::reg::STATUS, status | (1 << 21));
1554 }
1555 }
1556
1557 /// Perform a memory access.
1558 ///
1559 /// # Errors
1560 ///
1561 /// [`Exception::AddressError`] when an *aligned* access is misaligned. The
1562 /// `LWL`/`LWR` family is exempt by construction — being usable at any byte
1563 /// offset is the entire reason it exists.
1564 fn access<B: Bus>(&mut self, bus: &mut B, op: MemOp) -> Result<WriteBack, Exception> {
1565 // TODO(T-11-003): charge the cache-miss cost (8..=9 + M PCycles for a
1566 // D-cache fill, UM Table 11-1) once `M` is measured -- accuracy-ledger C-1.
1567 match op {
1568 MemOp::Load { kind, addr, dest } => {
1569 if !kind.is_aligned(addr) {
1570 self.fault_vaddr = addr;
1571 return Err(Exception::AddressError { store: false });
1572 }
1573 let phys = self.translate_re(addr, false, kind.width())?;
1574 let raw = self.read_width(bus, phys, kind.width());
1575 Ok(WriteBack::Gpr {
1576 dest,
1577 value: kind.shape(raw),
1578 })
1579 }
1580 MemOp::Store { kind, addr, value } => {
1581 if !kind.is_aligned(addr) {
1582 self.fault_vaddr = addr;
1583 return Err(Exception::AddressError { store: true });
1584 }
1585 let phys = self.translate_re(addr, true, kind.width())?;
1586 self.write_width(bus, phys, kind.width(), value);
1587 Ok(WriteBack::None)
1588 }
1589 // Load linked: an ordinary aligned load that also arms the link.
1590 MemOp::LinkedLoad { kind, addr, dest } => {
1591 if !kind.is_aligned(addr) {
1592 // "If either of the low-order two bits of the address are
1593 // not zero, an address error exception takes place" (UM §16
1594 // p. 453) -- and the link is NOT armed, because the
1595 // instruction did not complete.
1596 self.fault_vaddr = addr;
1597 return Err(Exception::AddressError { store: false });
1598 }
1599 let phys = self.translate_re(addr, false, kind.width())?;
1600 let raw = self.read_width(bus, phys, kind.width());
1601 self.ll_bit = true;
1602 // "the value with the high-order four bits of the physical
1603 // address PA(31:4) ... zero-extended" (UM Figure 5-17). Written
1604 // via `set_hardware` because LLAddr is software-writable too:
1605 // this is the hardware side effect, not an MTC0.
1606 self.cop0
1607 .set_hardware(crate::cop0::reg::LL_ADDR, u64::from(phys.addr >> 4));
1608 Ok(WriteBack::Gpr {
1609 dest,
1610 value: kind.shape(raw),
1611 })
1612 }
1613 // Store conditional: the store is conditional, the flag write is not.
1614 MemOp::ConditionalStore {
1615 kind,
1616 addr,
1617 value,
1618 dest,
1619 } => {
1620 if !kind.is_aligned(addr) {
1621 // "If this instruction both fails and causes an exception,
1622 // the exception takes precedence" (UM §16 p. 487) -- so the
1623 // address check runs before the link bit is even consulted,
1624 // and `dest` is left alone.
1625 self.fault_vaddr = addr;
1626 return Err(Exception::AddressError { store: true });
1627 }
1628 let stored = self.ll_bit;
1629 if stored {
1630 let phys = self.translate_re(addr, true, kind.width())?;
1631 self.write_width(bus, phys, kind.width(), value);
1632 }
1633 // Written whether or not the store happened. Note the link bit
1634 // is deliberately NOT cleared here -- see `Pipeline::ll_bit`.
1635 Ok(WriteBack::Gpr {
1636 dest,
1637 value: u64::from(stored),
1638 })
1639 }
1640 // FP loads and stores. Same alignment and translation rules as the
1641 // integer forms -- only the destination register file differs.
1642 MemOp::Fp { op, addr, ft } => {
1643 use crate::decode::Op;
1644 let double = matches!(op, Op::Ldc1 | Op::Sdc1);
1645 let store = matches!(op, Op::Swc1 | Op::Sdc1);
1646 let align = if double { 7 } else { 3 };
1647 if addr & align != 0 {
1648 self.fault_vaddr = addr;
1649 return Err(Exception::AddressError { store });
1650 }
1651 let phys = self.translate_re(addr, store, if double { 8 } else { 4 })?;
1652 // `Status.FR` selects the register-file view; a double under
1653 // FR = 0 occupies an FGR pair.
1654 let fr = fr_of(&self.cop0);
1655 match op {
1656 Op::Lwc1 => {
1657 let v = self.read_width(bus, phys, 4) as u32;
1658 self.fpr.write_s(ft, fr_of(&self.cop0), v);
1659 }
1660 Op::Ldc1 => {
1661 let v = self.read_width(bus, phys, 8);
1662 self.fpr.write_d(ft, fr, v);
1663 }
1664 Op::Swc1 => {
1665 let v = self.fpr.read_s(ft, fr_of(&self.cop0));
1666 self.write_width(bus, phys, 4, u64::from(v));
1667 }
1668 _ => {
1669 let v = self.fpr.read_d(ft, fr);
1670 self.write_width(bus, phys, 8, v);
1671 }
1672 }
1673 Ok(WriteBack::None)
1674 }
1675 // EMUX — n64-systemtest's emulator-extension protocol, executed
1676 // here because it needs the bus. See `docs/cpu.md` and ledger C-8:
1677 // hardware leaves COP0 CO `funct` 0x20-0x3F inert, which is exactly
1678 // why emux claimed the range.
1679 MemOp::Emux {
1680 funct,
1681 code,
1682 ptr,
1683 len,
1684 dest,
1685 } => self.emux(bus, funct, code, ptr, len, dest),
1686 MemOp::Cache { addr, op } => self.cache_op(bus, addr, op),
1687 // The unaligned family accesses the ALIGNED container holding `addr`
1688 // and merges, so it can never raise an address error -- but it CAN
1689 // still raise a TLB fault, which is why it is fallible.
1690 MemOp::Unaligned { op, addr, rt, dest } => {
1691 self.access_unaligned(bus, op, addr, rt, dest)
1692 }
1693 }
1694 }
1695
1696 /// **EMUX** — the emulator-extension protocol n64-systemtest uses to talk to
1697 /// its host (`ref-proj/n64-systemtest/src/emux.rs`).
1698 ///
1699 /// Three operations, all COP0 CO encodings in the `funct` 0x20-0x3F range
1700 /// that hardware retires inertly (ledger **C-8**):
1701 ///
1702 /// | `funct` | name | effect |
1703 /// |---|---|---|
1704 /// | `0x20` | `xdetect` | return a capability bitmask in `rd` |
1705 /// | `0x25` | `xlog` | print `GPR[rt]` bytes from `GPR[rd]` |
1706 /// | `0x2C` | `xioctl` | `code 1` exit, `code 2` "fast mode" |
1707 ///
1708 /// # Why implement it rather than keep the no-op
1709 ///
1710 /// `xlog` is a console that needs **no PI, SI or `ISViewer` emulation** — the
1711 /// suite's output reaches the host directly. `xioctl(EXIT)` turns "the run
1712 /// finished" from a tick-budget guess into a definite signal. Both are worth
1713 /// having long before the cartridge subsystem exists.
1714 ///
1715 /// Advertising a capability obliges us to implement it: the bitmask below
1716 /// claims exactly `xlog` and `xioctl`, so the suite routes text to us only
1717 /// because we can actually take it.
1718 fn emux<B: Bus>(
1719 &mut self,
1720 bus: &mut B,
1721 funct: u8,
1722 code: u16,
1723 ptr: u64,
1724 len: u64,
1725 dest: u8,
1726 ) -> Result<WriteBack, Exception> {
1727 /// `XDETECT` — capability probe.
1728 const XDETECT: u8 = 0o40;
1729 /// `XLOG` — write a string to the host.
1730 const XLOG: u8 = 0o45;
1731 /// `XIOCTL` — control operations.
1732 const XIOCTL: u8 = 0o54;
1733 /// `xdetect` code 1: "which of funct 0x20-0x3F do you support?"
1734 const CODE_EXTENSIONS: u16 = 1;
1735 /// `xioctl` code 1: terminate.
1736 const XIOCTL_EXIT: u16 = 1;
1737
1738 // Hardware has no EMUX. Unless the host opts in, every encoding in the
1739 // range stays inert -- which is both ledger C-8 and what a default ares
1740 // build does (`if(!system.homebrewMode) return;`). Getting this wrong is
1741 // observable: advertising capabilities makes n64-systemtest switch
1742 // console backends, which changes the retired-instruction stream and
1743 // diverges from a hardware-accurate reference trace.
1744 if !bus.emux_enabled() {
1745 return Ok(WriteBack::None);
1746 }
1747 match funct {
1748 XDETECT if code == CODE_EXTENSIONS => {
1749 // Bit N answers for `funct` 0x20 + N. We claim `xlog` and
1750 // `xioctl` and nothing else, because those are what is below.
1751 let mask = (1u64 << (XLOG - XDETECT)) | (1u64 << (XIOCTL - XDETECT));
1752 Ok(WriteBack::Gpr { dest, value: mask })
1753 }
1754 XLOG => {
1755 // Bounded deliberately: `len` is guest-controlled, and a corrupt
1756 // or hostile value must not turn a log call into an unbounded
1757 // read. 4 KiB is far above any line the suite emits.
1758 const MAX: u64 = 4096;
1759 let n = len.min(MAX);
1760 let mut buf = alloc::vec::Vec::with_capacity(n as usize);
1761 for i in 0..n {
1762 // Translated per byte: the string may straddle a page, and a
1763 // fault here is a real fault the guest must see.
1764 let p = self.translate_data(ptr.wrapping_add(i), false)?;
1765 // Read THROUGH the D-cache, exactly as a guest `LB` would.
1766 // The string was just formatted by cached stores, so it is
1767 // still sitting in dirty cache lines -- going straight to
1768 // the bus reads stale RDRAM and prints blanks where the
1769 // digits should be. That is precisely the staleness the
1770 // cache model exists to reproduce, so the log channel has to
1771 // respect it too.
1772 buf.push(self.read_width(bus, p, 1) as u8);
1773 }
1774 bus.emux_log(&buf);
1775 Ok(WriteBack::None)
1776 }
1777 XIOCTL if code == XIOCTL_EXIT => {
1778 bus.emux_exit();
1779 Ok(WriteBack::None)
1780 }
1781 // Every other encoding in the range stays inert, which is the
1782 // hardware behavior C-8 records and the reason the range is usable
1783 // as extension space at all.
1784 _ => Ok(WriteBack::None),
1785 }
1786 }
1787
1788 /// `CACHE`: dispatch to the index- or address-addressed half.
1789 ///
1790 /// Only the ADDRESS-addressed operations translate. `op4..2`
1791 /// (UM Ch. 16, p. 404):
1792 ///
1793 /// - `0..=2` `Index_Invalidate` / `Index_Load_Tag` / `Index_Store_Tag` —
1794 /// address the cache "at the index specified", so they never consult the
1795 /// TLB and cannot fault.
1796 /// - `3` `Create_Dirty_Exclusive` — "set the cache block tag to the
1797 /// specified physical address", so it does.
1798 /// - `4..=6` `Hit_*` — "if the cache block contains the specified address",
1799 /// so they do.
1800 ///
1801 /// Translating unconditionally raises spurious TLB refills on `Index_*` ops
1802 /// against unmapped addresses, which is exactly what cache-init code does at
1803 /// boot: walk every index with an arbitrary base. An earlier revision of
1804 /// this comment described the distinction while the code ignored it.
1805 ///
1806 /// # Errors
1807 ///
1808 /// A TLB fault, on the address-addressed operations only.
1809 fn cache_op<B: Bus>(&mut self, bus: &mut B, addr: u64, op: u8) -> Result<WriteBack, Exception> {
1810 // Any cache operation can change what the I-cache would serve, so the
1811 // idle-loop recognition that borrows the I-cache's validity stops being
1812 // valid here. This hook is the ONLY thing that makes caching that
1813 // recognition sound (see `Pipeline::idle_pc`), so it sits at the single
1814 // entry point for every CACHE variant rather than at the invalidating
1815 // ones — a narrower hook would have to be re-argued every time a variant
1816 // is added, and being wrong about one would be silent.
1817 self.idle_pc = None;
1818 if (op >> 2) >= 3 {
1819 let p = self.translate_data(addr, false)?;
1820 self.cache_hit_op(bus, op, p.addr);
1821 } else {
1822 // An `Index_*` op indexes on the raw VIRTUAL address, which is what
1823 // the hardware does: both primary caches are VIRTUALLY indexed.
1824 //
1825 // This model indexes by physical address (ledger D-6), so on a
1826 // TLB-mapped page the two can select different lines -- translation
1827 // preserves only the low 12 bits, while the D-cache index reaches
1828 // bit 12 and the I-cache bit 13. That divergence is exactly what D-6
1829 // records. It is NOT the claim an earlier revision of this comment
1830 // made -- that translation leaves the index bits alone -- which is
1831 // false for every mapped segment.
1832 self.cache_index_op(bus, op, addr as u32);
1833 }
1834 Ok(WriteBack::None)
1835 }
1836
1837 /// The index-addressed half of `CACHE` (`op4..2` in `0..=2`).
1838 ///
1839 /// `addr` is the *virtual* address: these operations select a line by index
1840 /// and never translate. Bit 0 of `op` picks the cache.
1841 fn cache_index_op<B: Bus>(&mut self, bus: &mut B, op: u8, addr: u32) {
1842 let dcache = op & 1 == 1;
1843 match (op >> 2, dcache) {
1844 // Index_Invalidate (I) / Index_WriteBack_Invalidate (D).
1845 //
1846 // Both clear the valid bit and LEAVE THE TAG in place, which
1847 // `Index_Load_Tag` then reports. Clearing the tag as well would look
1848 // tidier and would be wrong: n64-systemtest asserts the PFN is
1849 // unchanged across an invalidate.
1850 (0, false) => self.icache.invalidate_index(addr),
1851 (0, true) => {
1852 if let Some(w) = self.dcache.flush_index(addr, true, false) {
1853 Self::push_line(bus, w.addr, &w.data);
1854 }
1855 }
1856 // Index_Load_Tag -> COP0 TagLo. Written with `set_hardware` because
1857 // this is the hardware side effect of CACHE, not an MTC0, and the
1858 // register's software write mask must not apply to it.
1859 (1, false) => {
1860 let t = self.icache.load_tag(addr);
1861 self.cop0
1862 .set_hardware(crate::cop0::reg::TAG_LO, u64::from(t));
1863 }
1864 (1, true) => {
1865 let t = self.dcache.load_tag(addr);
1866 self.cop0
1867 .set_hardware(crate::cop0::reg::TAG_LO, u64::from(t));
1868 }
1869 // Index_Store_Tag <- COP0 TagLo.
1870 (2, false) => {
1871 let t = self.cop0.read(crate::cop0::reg::TAG_LO) as u32;
1872 self.icache.store_tag(addr, t);
1873 }
1874 (2, true) => {
1875 let t = self.cop0.read(crate::cop0::reg::TAG_LO) as u32;
1876 self.dcache.store_tag(addr, t);
1877 }
1878 _ => {}
1879 }
1880 }
1881
1882 /// The address-addressed half of `CACHE` (`op4..2` in `3..=6`).
1883 ///
1884 /// `addr` is PHYSICAL — the caller has already translated it, which is what
1885 /// makes a TLB fault on these operations possible.
1886 fn cache_hit_op<B: Bus>(&mut self, bus: &mut B, op: u8, addr: u32) {
1887 let dcache = op & 1 == 1;
1888 match (op >> 2, dcache) {
1889 // Create_Dirty_Exclusive (D only): claim the line without a fill.
1890 (3, true) => {
1891 if let Some(w) = self.dcache.create_dirty_exclusive(addr) {
1892 Self::push_line(bus, w.addr, &w.data);
1893 }
1894 }
1895 // Hit_Invalidate: only if the line is actually resident.
1896 (4, false) => self.icache.hit_invalidate(addr),
1897 (4, true) => {
1898 if self.dcache.hits(addr) {
1899 self.dcache.flush_index(addr, true, false);
1900 }
1901 }
1902 // Fill (I) — an unconditional line fill, no hit test.
1903 (5, false) => self.icache_fill(bus, addr),
1904 // Hit_WriteBack_Invalidate (D).
1905 (5, true) => {
1906 if self.dcache.hits(addr)
1907 && let Some(w) = self.dcache.flush_index(addr, true, false)
1908 {
1909 Self::push_line(bus, w.addr, &w.data);
1910 }
1911 }
1912 // Hit_WriteBack: the line stays resident, and stops being dirty.
1913 (6, false) => {
1914 if let Some(w) = self.icache.flush_hit(addr) {
1915 Self::push_line(bus, w.addr, &w.data);
1916 }
1917 }
1918 (6, true) => {
1919 if self.dcache.hits(addr)
1920 && let Some(w) = self.dcache.flush_index(addr, false, true)
1921 {
1922 Self::push_line(bus, w.addr, &w.data);
1923 }
1924 }
1925 _ => {}
1926 }
1927 }
1928
1929 /// D-cache line fill cost in PClocks (`8..=9 + M(RDRAM)`, UM Table 11-1).
1930 ///
1931 /// `M(RDRAM)` is **not a scalar** (ledger C-1) — it is bank-state dependent
1932 /// (C-4): a 2 KiB RDRAM row spans 128 D-cache lines, so sequential access hits
1933 /// the open row (**fast**) and random access misses it (**slow**). This code
1934 /// charges this one value for **every** miss — there is **no row-state
1935 /// dispatch yet** — so 40 is a **provisional row-hit-typical estimate**, not a
1936 /// measured number: the UM gives the fill *formula* but not this warm value,
1937 /// and ares (40) / cen64 (44) are corroboration only. The documented
1938 /// **cold/row-miss** fill is ~60 PClocks (copetti's ~640 ns external estimate
1939 /// x 93.75 MHz PClock, `M ~= 52`); charging it, and the dirty-writeback case,
1940 /// needs the undocumented `RasInterval` cycles and is deferred to C-4.
1941 #[allow(clippy::doc_markdown)]
1942 const M_DCACHE_FILL: u32 = 40;
1943 /// I-cache line fill cost in PClocks (`14..=15 + M(RDRAM)`, UM Table 11-2).
1944 ///
1945 /// Defined as [`Self::M_DCACHE_FILL`] plus 6, not a bare literal, so the two
1946 /// stay in lockstep by construction: the fills share the same row-hit
1947 /// `M(RDRAM)` and differ only by the UM's line-transfer size — the I-line
1948 /// moves 8 words vs the D-line's critical doubleword of 2, and `14 - 8 = 6`
1949 /// (Table 11-2 minus 11-1). Currently `40 + 6 = 46` (cen64 uses 48); a future
1950 /// measured `M(RDRAM)` (ledger C-1/C-4) changes both together.
1951 #[cfg_attr(test, allow(dead_code))]
1952 #[allow(clippy::doc_markdown)]
1953 const M_ICACHE_FILL: u32 = Self::M_DCACHE_FILL + 6;
1954
1955 /// Make the I-cache line covering `addr` resident.
1956 fn icache_fill<B: Bus>(&mut self, bus: &mut B, addr: u32) {
1957 let base = addr & !(crate::cache::ICACHE_LINE - 1);
1958 let mut data = [0u8; 32];
1959 Self::pull_line(bus, base, &mut data);
1960 self.icache.install(base, data);
1961 // Called only on an I-cache miss (`ic_stage`), so the fill cost applies
1962 // here (fitted, ledger C-1). **Deliberate test seam:** an I-cache miss
1963 // fires on *every* cold fetch, and this crate's fine-grained pipeline unit
1964 // tests step fixed cycle counts for a free-fetch model AND assert on the
1965 // interlock/FPU stalls the fill would confound -- so the stall is charged
1966 // in real execution and in every INTEGRATION test (the i-cache microbench,
1967 // the systemtest, golden-log, residue -- where the pipeline runs as a
1968 // dependency with `cfg(test)` false), but not in this crate's own units.
1969 // The D-cache fill, by contrast, fires only on a rare cached load, so it
1970 // is charged unconditionally (two units absorbed it directly).
1971 #[cfg(not(test))]
1972 self.stall_for(Self::M_ICACHE_FILL, Interlock::Icb);
1973 }
1974
1975 /// Make the D-cache line covering `addr` resident, writing back whatever it
1976 /// evicts.
1977 ///
1978 /// A dirty-line writeback (the `push_line` below) is charged the **same**
1979 /// fixed `M_DCACHE_FILL` as a clean miss -- the fitted 40 does not separate
1980 /// clean- from dirty-eviction cost (a real hardware measurement would; the
1981 /// writeback is an extra RDRAM transaction). Honest given the value is a
1982 /// fitted anchor, not a measurement (ledger C-1).
1983 fn dcache_fill<B: Bus>(&mut self, bus: &mut B, addr: u32) {
1984 let Some(evicted) = self.dcache.miss_plan(addr) else {
1985 return; // hit -- no fill, no cost
1986 };
1987 if let Some(w) = evicted {
1988 Self::push_line(bus, w.addr, &w.data);
1989 }
1990 let base = addr & !(crate::cache::DCACHE_LINE - 1);
1991 let mut data = [0u8; 16];
1992 Self::pull_line(bus, base, &mut data);
1993 self.dcache.install(base, data);
1994 self.stall_for(Self::M_DCACHE_FILL, Interlock::Dcm);
1995 }
1996
1997 /// Write a whole cache line out to the bus, a word at a time.
1998 ///
1999 /// Word-wide rather than byte-wide because [`Bus::read_u32`]/[`Bus::write_u32`]
2000 /// are the calls `rustyn64-core` gives a fast RDRAM path; a byte loop would
2001 /// issue four times the bus traffic on every fill and eviction, and a line
2002 /// is always naturally aligned so the split is exact.
2003 fn push_line<B: Bus>(bus: &mut B, addr: u32, data: &[u8]) {
2004 for (k, w) in data.chunks_exact(4).enumerate() {
2005 let word = u32::from_be_bytes([w[0], w[1], w[2], w[3]]);
2006 bus.write_u32(addr.wrapping_add(k as u32 * 4), word);
2007 }
2008 }
2009
2010 /// Read a whole cache line from the bus into `data`.
2011 fn pull_line<B: Bus>(bus: &mut B, addr: u32, data: &mut [u8]) {
2012 for (k, w) in data.chunks_exact_mut(4).enumerate() {
2013 let word = bus.read_u32(addr.wrapping_add(k as u32 * 4));
2014 w.copy_from_slice(&word.to_be_bytes());
2015 }
2016 }
2017
2018 /// Uncached RCP-register access latency `M`, in PClocks — **measured, not
2019 /// tuned** (ledger C-1, T-11-003).
2020 ///
2021 /// Derived from the PeterLemon `CPUTIMINGNTSC` mult/div differential: the
2022 /// model `expected_i = W / (B + c_i)`, where `c_i` is each timed
2023 /// instruction's documented stall (UM Table 3-12: `mult` 5, `dmult` 8,
2024 /// `div` 37, `ddiv` 69), regresses to a window `W ~= 1.52e6` PClocks and a
2025 /// hardware base-loop `B ~= 25.5` PClocks (fit < 1.5% on the high-leverage
2026 /// mult/div points). Our base loop measured exactly 4.00 PClocks (four
2027 /// one-cycle instructions, no memory latency), so the missing `B - 4 ~= 21.5`
2028 /// PClocks is the uncached `lw VI_V_CURRENT` latency. Charging 22 drives the
2029 /// ROM's absolute iteration count from 304_180 to 56_330 vs the
2030 /// hardware-baked 56_092 (0.4%) -- the *independent* confirmation, since 22
2031 /// came from the differential, not from fitting the absolute count. Measured
2032 /// on VI; the sibling RCP registers share the bus and are charged the same
2033 /// pending their own vectors. RDRAM / cache-fill `M` stay unmeasured (0).
2034 #[allow(clippy::doc_markdown)]
2035 const M_RCP_REGISTER: u32 = 22;
2036 /// Low bound of the RCP MMIO block (SP..SI) charged [`Self::M_RCP_REGISTER`].
2037 const M_RCP_REGISTER_LO: u32 = 0x0400_0000;
2038 /// High bound of the RCP MMIO block.
2039 const M_RCP_REGISTER_HI: u32 = 0x04FF_FFFF;
2040
2041 /// Read `width` big-endian bytes, right-justified, through the D-cache when
2042 /// the access is cached.
2043 ///
2044 /// Dispatches on width so 4- and 8-byte *uncached* accesses go through
2045 /// [`Bus::read_u32`], which `rustyn64-core` overrides with a fast RDRAM path.
2046 /// A byte loop would issue 4-8x more bus calls on the *most common*
2047 /// operations, and memory access is the hot path for a core targeting full
2048 /// speed (`docs/performance.md`).
2049 ///
2050 /// Alignment is **not** rechecked here. `access` has already validated it
2051 /// against the specific [`crate::mem::LoadKind`]/[`crate::mem::StoreKind`],
2052 /// and the unaligned family passes an address it has aligned down itself.
2053 /// Duplicating the check would put the rule in two places, where it can drift.
2054 fn read_width<B: Bus>(&mut self, bus: &mut B, p: crate::addr::Physical, width: u64) -> u64 {
2055 let addr = p.addr;
2056 if p.cached == crate::addr::Cached::Yes {
2057 self.dcache_fill(bus, addr);
2058 return self.dcache.read(addr, width as usize);
2059 }
2060 let v = match width {
2061 1 => u64::from(bus.read_u8(addr)),
2062 2 => (u64::from(bus.read_u8(addr)) << 8) | u64::from(bus.read_u8(addr.wrapping_add(1))),
2063 4 => u64::from(bus.read_u32(addr)),
2064 // Big-endian: the high word is at the lower address.
2065 8 => {
2066 (u64::from(bus.read_u32(addr)) << 32)
2067 | u64::from(bus.read_u32(addr.wrapping_add(4)))
2068 }
2069 _ => 0,
2070 };
2071 // Charge the uncached RCP-register access latency `M` (T-11-003, ledger
2072 // C-1). `addr` is physical (`p.addr`), so the KSEG1 `0xA4xx_xxxx` a game
2073 // uses arrives here as `0x04xx_xxxx`. Reuses the `Dcm` interlock (whose
2074 // doc already carries the `+ M` term) rather than adding a serialized enum
2075 // variant that would perturb the save-state format (ADR 0005). Only this
2076 // region is *measured*; other uncached reads (RDRAM) stay at 0.
2077 //
2078 // Charged once per access. `M` was measured for a 32-bit `lw`; the width
2079 // == 8 case issues two bus words, but a 64-bit uncached read of an RCP
2080 // register (an `ld` spanning two 32-bit registers) is not something the
2081 // oracle covers, so its two-transaction cost is left unmeasured rather
2082 // than assumed to be `2 * M`.
2083 if Self::is_rcp_mmio(addr) {
2084 self.stall_for(Self::M_RCP_REGISTER, Interlock::Dcm);
2085 }
2086 v
2087 }
2088
2089 /// Is the physical address `addr` in the RCP MMIO block charged
2090 /// [`Self::M_RCP_REGISTER`]?
2091 const fn is_rcp_mmio(addr: u32) -> bool {
2092 Self::M_RCP_REGISTER_LO <= addr && addr <= Self::M_RCP_REGISTER_HI
2093 }
2094
2095 /// Write the low `width` big-endian bytes of `value`, through the D-cache
2096 /// when the access is cached.
2097 ///
2098 /// A cached store is a **write-allocate**: the line is filled first, then
2099 /// modified and left dirty. The VR4300's D-cache has no write-around path,
2100 /// so a partial store to an absent line must read the rest of it from memory
2101 /// or the eventual write-back would push out fifteen bytes of nothing.
2102 ///
2103 /// Width-dispatched for the same reason as [`Pipeline::read_width`].
2104 fn write_width<B: Bus>(
2105 &mut self,
2106 bus: &mut B,
2107 p: crate::addr::Physical,
2108 width: u64,
2109 value: u64,
2110 ) {
2111 let addr = p.addr;
2112 if p.cached == crate::addr::Cached::Yes {
2113 self.dcache_fill(bus, addr);
2114 self.dcache.write(addr, width as usize, value);
2115 return;
2116 }
2117 // Hand the bus the width and the *untruncated* register. Narrowing here
2118 // would discard the upper bits before the target can decide whether it
2119 // wants them -- and every device on the RCP's internal bus does (see
2120 // `Bus::write_sized`).
2121 bus.write_sized(addr, width, value);
2122 }
2123
2124 /// The four checks an instruction must pass **before** it executes.
2125 ///
2126 /// Latch-independent, for the reason [`Self::fetch_word`] is: the caller
2127 /// decides what a refusal means. `ex_stage` turns an `Err` into an `Ex`-stage
2128 /// abort; the instruction-granular path (ADR 0013) needs the same four
2129 /// decisions with different consequences. Keeping one copy is what stops the
2130 /// two paths disagreeing about *which* exception an encoding raises — a
2131 /// disagreement n64-systemtest would report as a wrong cause code rather than
2132 /// as a missing check, which is much harder to read backwards.
2133 ///
2134 /// **The order is preserved exactly, and the first two refusals turn out to
2135 /// be disjoint.** The accurate path's comment says an unusable coprocessor is
2136 /// reported as such *"even when the encoding is also a 64-bit one"* — but
2137 /// mutation-checking that while extracting this changed nothing
2138 /// (n64-systemtest stayed at 0 failing in the Phase 1 categories, 90
2139 /// suite-wide), and the reason is stronger than a gap in the suite:
2140 /// [`Op::is_64_bit`](crate::decode::Op::is_64_bit) covers only CPU integer
2141 /// and load/store operations, while [`Self::unusable_coprocessor`] answers
2142 /// only for COP0/COP1/COP2 encodings. **No input is in both sets**, so no
2143 /// input can reach the second check by way of the first.
2144 ///
2145 /// So the ordering between those two is *unobservable*, not merely untested.
2146 /// It is kept because it is what the accurate path has always done, and
2147 /// `ex_gates_first_two_refusals_cannot_both_apply` sweeps the encoding space
2148 /// to keep the disjointness true — if a 64-bit coprocessor move is ever
2149 /// classified as 64-bit, that test fails and whoever changed it has to
2150 /// justify the precedence against the manual instead of inheriting it.
2151 ///
2152 /// Written out because "the order matters" is exactly the shape of claim this
2153 /// project keeps finding stale: nothing fails when it is wrong. A first draft
2154 /// of this comment asserted the order was load-bearing.
2155 ///
2156 /// # Errors
2157 ///
2158 /// One of [`Exception::CoprocessorUnusable`], [`Exception::ReservedInstruction`],
2159 /// [`Exception::CoprocessorReserved`], or [`Exception::FloatingPoint`].
2160 fn ex_gate(&mut self, decoded: Decoded) -> Result<(), Exception> {
2161 // Coprocessor usability is checked BEFORE execution, in EX (UM §4.7.5
2162 // lists CPU among the EX-stage exceptions). COP0 is exempt in kernel mode
2163 // regardless of `CU0`, which is why the CPU can run exception handlers
2164 // before any `Status` setup has happened.
2165 if let Some(unit) = self.unusable_coprocessor(decoded) {
2166 return Err(Exception::CoprocessorUnusable { unit });
2167 }
2168 // A 64-bit operation is RESERVED in 32-bit User or Supervisor mode.
2169 // Kernel may use them at any width, which is why this cannot be a
2170 // property of `Status.KX` alone.
2171 //
2172 // Checked after coprocessor usability, not before: an unusable
2173 // coprocessor is reported as such even when the encoding is also a
2174 // 64-bit one, and the suite distinguishes the two causes.
2175 if decoded.op.is_64_bit() && self.sixty_four_bit_is_reserved() {
2176 return Err(Exception::ReservedInstruction);
2177 }
2178 // `DCFC1`/`DCTC1` are usable-but-unimplemented: the `CU1` check above has
2179 // already passed, so this is the *other* outcome. The whole `Cause` field
2180 // is replaced, leaving only bit 17 -- the suite pre-loads unrelated cause
2181 // bits specifically to check they clear. COP2's equivalent declines
2182 // differently: Reserved Instruction, and `FCSR` is not involved at all.
2183 if decoded.op == crate::decode::Op::Cop2ReservedControl {
2184 return Err(Exception::CoprocessorReserved { unit: 2 });
2185 }
2186 if decoded.op == crate::decode::Op::Cop1ReservedControl {
2187 let fcsr = self.cop1.fcsr();
2188 self.cop1.ctc1(
2189 31,
2190 (fcsr & !crate::fpu::CAUSE_MASK) | crate::fpu::CAUSE_UNIMPLEMENTED,
2191 );
2192 return Err(Exception::FloatingPoint);
2193 }
2194 Ok(())
2195 }
2196
2197 /// `EX` — execute.
2198 fn ex_stage(&mut self, regs: &Regs, next_pc: &mut u64) {
2199 let mut out = self.rf_ex;
2200 if out.occupied && out.abort.is_none() {
2201 // Resolve operands through the bypass network rather than trusting
2202 // the values latched at RF, which may be one cycle stale.
2203 out.rs_val = self.bypass(out.decoded.rs, regs);
2204 out.rt_val = self.bypass(out.decoded.rt, regs);
2205 let hilo = self.bypass_hi_lo(regs);
2206 // The pre-execution refusals, all four of which are EX-stage
2207 // exceptions (UM §4.7.5). `ex_gate` holds the checks and the order
2208 // they must run in; here is only what an `Ex`-stage refusal *does*.
2209 if let Err(exc) = self.ex_gate(out.decoded) {
2210 self.abort_from(Stage::Ex, exc);
2211 out = self.rf_ex;
2212 self.rf_ex.occupied = false;
2213 self.ex_dc = out;
2214 return;
2215 }
2216 // `FCSR.C` — read here, in EX, where the branch resolves, THROUGH A
2217 // BYPASS. See `pending_fp_condition`.
2218 let fp_condition = self.pending_fp_condition().unwrap_or_else(|| {
2219 let p: &Self = self;
2220 p.cop1.fcsr() & FCSR_C != 0
2221 });
2222 match execute(
2223 out.decoded,
2224 out.rs_val,
2225 out.rt_val,
2226 hilo,
2227 out.pc,
2228 fp_condition,
2229 ) {
2230 Ok(e) => {
2231 out.write_back = e.write_back;
2232 out.mem = e.mem;
2233 out.cop0 = e.cop0;
2234 // Control flow. The delay slot has ALREADY been fetched -- it
2235 // is in `ic_rf` right now, because IC ran a cycle ahead. That
2236 // is the architectural delay slot, not a modeling artifact.
2237 //
2238 // Because the cascade runs backwards, `ic_stage` executes
2239 // AFTER this in the same cycle, so writing `next_pc` here
2240 // makes the very next fetch land on the target with exactly
2241 // one delay slot in between. No wrong-path fetch needs
2242 // squashing -- that falls out of the reverse order rather
2243 // than being arranged.
2244 // **The link is `next_pc`, read BEFORE this instruction's
2245 // own redirect is applied.**
2246 //
2247 // At `EX` time `next_pc` already holds the address of the
2248 // instruction that will run after this one's delay slot:
2249 // `pc + 8` for an ordinary jump, and the OUTER target `+ 4`
2250 // when this jump is itself in a delay slot, because the
2251 // outer jump redirected a cycle earlier and `IC` has since
2252 // advanced past it.
2253 //
2254 // Computing `pc + 8` in `execute` was right for the
2255 // ordinary case and silently wrong for the nested one; the
2256 // live `next_pc` is that address by construction rather
2257 // than by a second formula that can disagree.
2258 // **COP2 is one 64-bit latch, not a register file.** The
2259 // register index is ignored entirely: n64-systemtest writes with
2260 // one index and reads back with several others, including 30 and
2261 // 31, and gets the same value every time. `MTC2` writes all 64
2262 // bits despite being nominally a 32-bit move; `MFC2` returns the
2263 // low half sign-extended and `DMFC2` the whole thing.
2264 //
2265 // The same shape as the reserved COP0 registers (ledger C-15) --
2266 // this processor's answer to "a coprocessor that is not really
2267 // there" is a single latch, twice over.
2268 match out.decoded.op {
2269 crate::decode::Op::Mtc2 => self.cop2_latch = out.rt_val,
2270 crate::decode::Op::Mfc2 => {
2271 out.write_back = WriteBack::Gpr {
2272 dest: out.decoded.dest,
2273 value: crate::alu::sext32(self.cop2_latch as u32),
2274 };
2275 }
2276 crate::decode::Op::Dmfc2 => {
2277 out.write_back = WriteBack::Gpr {
2278 dest: out.decoded.dest,
2279 value: self.cop2_latch,
2280 };
2281 }
2282 _ => {}
2283 }
2284 self.resolve_branch_control(&mut out, e.link, e.redirect, next_pc);
2285 // Multiply and divide stall the ENTIRE pipeline for the
2286 // documented count (UM Table 3-12), so the request is raised
2287 // here and honored from the next cycle onward.
2288 if e.stall_cycles > 0 {
2289 self.stall_for(e.stall_cycles, Interlock::Mci);
2290 }
2291 // ERET. Resolved here rather than in `execute` because its
2292 // target comes out of COP0, not out of the instruction.
2293 //
2294 // It has NO delay slot (UM Ch. 16, p. 434) -- alone among
2295 // the control transfers -- so the instruction IC already
2296 // fetched must be squashed. Every branch reaches this point
2297 // with its delay slot legitimately in flight, which is why
2298 // the squash is spelled out here instead of falling out of
2299 // the reverse cascade as a branch's does.
2300 if matches!(e.cop0, Some(Cop0Access::Eret)) {
2301 *next_pc = exception::eret(&mut self.cop0);
2302 // "cleared by an ERET" (UM §3.1) -- the other half of
2303 // the LL/SC contract, which had nothing clearing it
2304 // until now.
2305 self.ll_bit = false;
2306 self.ic_rf = Latch::default();
2307 }
2308 }
2309 // Stamp BEFORE the latch move, so the abort travels with the
2310 // instruction that caused it -- see `abort_from`.
2311 Err(exc) => {
2312 self.abort_from(Stage::Ex, exc);
2313 out = self.rf_ex;
2314 }
2315 }
2316 }
2317 self.ex_dc = out;
2318 self.rf_ex.occupied = false;
2319 }
2320
2321 /// Resolve a branch's link write and its taken-redirect, honoring the
2322 /// delay-slot-fault rule (ledger R-19).
2323 ///
2324 /// The delay slot IC fetched last cycle is in `ic_rf` right now; if it
2325 /// aborted, the exception has already pointed `next_pc` at the vector by the
2326 /// time this branch reaches EX. The branch must NOT take -- the exception PC
2327 /// wins -- yet it STILL retires and writes its link
2328 /// (`ExecuteTLBMappedMissInDelay` asserts `RA == fault_address + 4`). Letting
2329 /// the redirect fire would clobber the vector every cycle, so the handler
2330 /// never runs: a two-state hang between the faulting delay slot and the
2331 /// exception vector.
2332 fn resolve_branch_control(
2333 &mut self,
2334 out: &mut Latch,
2335 link: Option<u8>,
2336 redirect: Option<Redirect>,
2337 next_pc: &mut u64,
2338 ) {
2339 let delay_slot_faulted = out.decoded.op.has_delay_slot()
2340 && self.ic_rf.occupied
2341 && self.ic_rf.in_delay_slot
2342 && self.ic_rf.abort.is_some();
2343 if let Some(dest) = link {
2344 // The link is the address after the delay slot: normally the live
2345 // `next_pc`, but the architectural `pc + 8` when a fault has already
2346 // overwritten `next_pc` with the exception vector.
2347 let value = if delay_slot_faulted {
2348 out.pc.wrapping_add(8)
2349 } else {
2350 *next_pc
2351 };
2352 out.write_back = WriteBack::Gpr { dest, value };
2353 }
2354 if let Some(r) = redirect
2355 && !delay_slot_faulted
2356 {
2357 *next_pc = r.target;
2358 // A branch-LIKELY that was not taken squashes its already-fetched
2359 // delay slot. An ordinary branch never does.
2360 if r.nullify_delay_slot {
2361 self.ic_rf = Latch::default();
2362 }
2363 }
2364 }
2365
2366 /// `RF` — register fetch, and where the load interlock is detected.
2367 fn rf_stage(&mut self, regs: &Regs) {
2368 let mut out = self.ic_rf;
2369 if out.occupied {
2370 out.rs_val = regs.read(out.decoded.rs);
2371 out.rt_val = regs.read(out.decoded.rt);
2372 }
2373 // These reads are a first approximation: EX re-resolves them through the
2374 // bypass network, since a producer one instruction ahead has not
2375 // committed yet. RF still performs the read because that is where the
2376 // load interlock is detected (T-11-003).
2377 //
2378 // TODO(T-11-002): the MFHI/MFLO hazard window -- a MFHI followed within
2379 // two instructions by a HI write reads hardware's WRONG value, and that
2380 // is non-interlocked (alu::MFHI_MFLO_HAZARD_INSTRUCTIONS).
2381 // The load-delay interlock (UM §4.6.5). A load's result is not ready in
2382 // time to bypass, so if the NEXT instruction names the loaded register
2383 // the pipeline stalls one cycle. The detection is deliberately imprecise,
2384 // matching hardware -- see `load_interlocks`.
2385 //
2386 // Compare against `ex_dc`, not `rf_ex`. In the reverse cascade `EX` runs
2387 // before `RF`, so by now the instruction that was in `EX` this cycle has
2388 // already moved into `ex_dc` and `rf_ex` has been vacated. Checking
2389 // `rf_ex` here silently never fires -- which is exactly what it did
2390 // before `a_load_followed_by_its_use_interlocks...` caught it.
2391 if out.occupied
2392 && self.ex_dc.occupied
2393 && self.ex_dc.decoded.is_load()
2394 && load_interlocks(
2395 self.ex_dc.decoded.dest,
2396 out.decoded.rs,
2397 out.decoded.rt,
2398 self.ex_dc.decoded.targets_fpr() == out.decoded.targets_fpr(),
2399 )
2400 {
2401 self.stall_for(1, Interlock::Ldi);
2402 }
2403 self.rf_ex = out;
2404 self.ic_rf.occupied = false;
2405 }
2406
2407 /// The FP condition an in-flight `C.cond.fmt` is about to commit, if any.
2408 ///
2409 /// `BC1` resolves in `EX`; `C.cond.fmt` computes *and* commits `FCSR.C` in
2410 /// `WB` (ADR 0007's single commit-or-trap point). An adjacent pair therefore
2411 /// has the branch sampling the previous condition, and the ROM emits exactly
2412 /// that pair with no separating instruction.
2413 ///
2414 /// # Why this is a bypass and not a stall
2415 ///
2416 /// Stalling cannot work here. `stall_for` freezes the whole pipeline, so
2417 /// holding the branch delays the compare's `WB` by the same amount and the
2418 /// branch never catches up — an interlock on `ex_dc`/`dc_wb` was written,
2419 /// fired once, and changed nothing. This is the same shape as the **load**
2420 /// interlock, which works only because its consumer reads through the
2421 /// bypass network; the one-cycle stall buys `DC` time, and forwarding does
2422 /// the rest. The FP condition had no such path, so this is it.
2423 ///
2424 /// # Why recomputing is sound
2425 ///
2426 /// A compare reads two FP registers and writes only `FCSR.C`. Nothing
2427 /// between it and the branch can change those registers — the branch has no
2428 /// destination — so evaluating it early yields the value `WB` will commit.
2429 /// The flags are deliberately discarded: this is a forwarding path, and
2430 /// raising an exception from it would make the branch report the *compare's*
2431 /// trap.
2432 ///
2433 /// `ex_dc` is checked before `dc_wb` because it holds the **younger**
2434 /// instruction, and the most recent compare is the one whose value stands.
2435 fn pending_fp_condition(&self) -> Option<bool> {
2436 let fr = fr_of(&self.cop0);
2437 for latch in [&self.ex_dc, &self.dc_wb] {
2438 if !latch.occupied {
2439 continue;
2440 }
2441 if let Some(Cop0Access::Cop1(Cop1Access::Arith {
2442 fmt, funct, ft, fs, ..
2443 })) = latch.cop0
2444 && funct >= 0o60
2445 {
2446 if let (FpCommit::Condition(c), _, false) =
2447 self.fp_compare(fmt, funct & 0xF, ft, fs, fr)
2448 {
2449 return Some(c);
2450 }
2451 return None;
2452 }
2453 }
2454 None
2455 }
2456
2457 /// Fetch the instruction word at `pc`: alignment, the segment map, the
2458 /// micro-ITLB and JTLB, `Status.RE`, and the I-cache.
2459 ///
2460 /// **Latch-independent on purpose.** Everything here is a function of `pc`
2461 /// and the CPU's own state — it stamps no latch, raises no abort, and names
2462 /// no [`Stage`] — so the caller decides what a failure means. `ic_stage`
2463 /// turns an `Err` into an `Ic`-stage abort; the instruction-granular path
2464 /// (ADR 0013) needs the same fetch with different consequences, and
2465 /// duplicating 100 lines of segment/TLB/cache logic to get it is how the two
2466 /// paths would drift apart in exactly the place a test would not notice.
2467 ///
2468 /// The side effects it *does* keep are the ones that belong to fetching
2469 /// rather than to sequencing: an ITLB reload and its 3-`PCycle` stall, and an
2470 /// I-cache fill. Those happen on hardware whatever the caller does next.
2471 ///
2472 /// # Errors
2473 ///
2474 /// - [`Exception::AddressError`] — `pc` is not word-aligned, or the segment
2475 /// is not valid in the current mode. The bus is **not** touched in either
2476 /// case: the access itself is what is invalid.
2477 /// - A TLB exception from [`Self::tlb_exception`] on a JTLB miss or an
2478 /// invalid entry. An instruction fetch is a load, so `TLBL`, never `TLBS`.
2479 fn fetch_word<B: Bus>(&mut self, bus: &mut B, pc: u64) -> Result<u32, Exception> {
2480 // An instruction fetch must be word-aligned. An unaligned PC raises an
2481 // address error (AdEL) rather than fetching.
2482 //
2483 // Not reachable from straight-line execution, which advances by 4 from an
2484 // aligned reset vector. It becomes reachable with the jump and branch
2485 // family (T-11-004), where a computed target can be unaligned, and it is
2486 // already reachable through the public `Cpu::set_pc` the golden-log
2487 // harness uses.
2488 if !pc.is_multiple_of(4) {
2489 return Err(Exception::AddressError { store: false });
2490 }
2491
2492 // The fetch itself goes through the I-cache (see below); what is still
2493 // outstanding is the COST.
2494 // TODO(T-11-003): charge the I-cache miss cost (14..=15 + M PCycles, UM
2495 // Table 11-2) once `M` is measured -- accuracy-ledger C-1.
2496 // Every address handed to the Bus is PHYSICAL (`docs/cpu.md`); the
2497 // segment map is applied here, in the CPU, not by the Bus.
2498 // Instruction fetch goes through the micro-ITLB in front of the JTLB
2499 // (UM §1.5.1). A micro-TLB miss is a STALL of 3 PCycles (UM §4.6.2); a
2500 // JTLB miss is an exception. Only the mapped segments involve either --
2501 // KSEG0/KSEG1 fetches, which is all of early boot, bypass both.
2502 let asid = (self.cop0.read(crate::cop0::reg::ENTRY_HI) & 0xFF) as u8;
2503 let fetch_access = self.access_mode();
2504 let (phys, cached) = match crate::addr::segment(pc, fetch_access) {
2505 crate::addr::Segment::Direct { addr, cached } => (addr, cached),
2506 // Not a valid address in this mode: an address error, raised without
2507 // consulting the TLB at all.
2508 crate::addr::Segment::Invalid => {
2509 return Err(Exception::AddressError { store: false });
2510 }
2511 crate::addr::Segment::Mapped => {
2512 // The 3-PCycle penalty is "incurred when the micro-TLB is
2513 // updated from the JTLB" (UM §4.6.2) -- so it is charged only
2514 // when a reload can actually happen. A fetch that misses BOTH
2515 // levels goes straight to its exception without paying for a
2516 // reload that never occurred.
2517 if !self.tlb.itlb_probe(pc, asid) && self.tlb.jtlb_has_match(pc, asid) {
2518 self.tlb.itlb_fill(pc, asid);
2519 self.stall_for(crate::tlb::ITLB_MISS_PCYCLES, Interlock::Itm);
2520 }
2521 match self.tlb.lookup(pc, asid, false) {
2522 Ok(t) => (
2523 t.addr,
2524 if t.uncached {
2525 crate::addr::Cached::No
2526 } else {
2527 crate::addr::Cached::Yes
2528 },
2529 ),
2530 // An instruction fetch is a load, so TLBL never TLBS.
2531 Err(f) => return Err(Self::tlb_exception(f, false, fetch_access.wide)),
2532 }
2533 }
2534 };
2535 // Instruction fetch is a 4-byte access, so `Status.RE` swaps it within
2536 // its doubleword exactly as it swaps a `LW` -- which is why the test ROM
2537 // has to emit its reverse-endian programs with each instruction PAIR
2538 // exchanged for them to execute in order.
2539 let phys = if self.reverse_endian() {
2540 phys ^ Self::re_swap(4)
2541 } else {
2542 phys
2543 };
2544 // Fetch through the I-cache when the segment is cached. This is what
2545 // makes an uncached patch to already-fetched code invisible until a
2546 // CACHE invalidate -- the behavior n64-systemtest's ICACHE group
2547 // asserts, and the reason the cache is modeled at all.
2548 Ok(if cached == crate::addr::Cached::Yes {
2549 if !self.icache.hits(phys) {
2550 self.icache_fill(bus, phys);
2551 }
2552 self.icache.read_word(phys)
2553 } else {
2554 bus.read_u32(phys)
2555 })
2556 }
2557
2558 /// `IC` — instruction-cache fetch, and where the delay-slot flag is set.
2559 fn ic_stage<B: Bus>(&mut self, bus: &mut B, next_pc: &mut u64) {
2560 // An abort raised earlier this cycle flushes younger instructions. The
2561 // fetch happening now is younger than all of them, so it must not become
2562 // a live instruction -- otherwise it escapes the flush entirely and
2563 // executes down the wrong path.
2564 //
2565 // TODO(T-11-002): redirect `next_pc` to the exception vector instead of
2566 // bubbling. Until the vector exists, a bubble is the honest behavior:
2567 // it declines to execute rather than executing the wrong thing.
2568 if self.flush_pending {
2569 self.ic_rf = Latch::default();
2570 return;
2571 }
2572 let pc = *next_pc;
2573
2574 // Computed BEFORE the alignment check, so a faulting fetch carries the
2575 // right delay-slot flag into its latch and therefore into `Cause.BD`.
2576 // Depends only on `rf_ex`, never on the fetch, so hoisting it is safe.
2577 //
2578 // Check `rf_ex`, not `ic_rf`: `rf_stage` runs immediately before this in
2579 // the reverse cascade and has already moved the previous instruction out
2580 // of `ic_rf`, so a branch fetched last cycle is in `rf_ex` by now.
2581 // Reading `ic_rf` here makes the flag silently always false.
2582 let in_delay_slot = self.rf_ex.occupied && self.rf_ex.decoded.op.has_delay_slot();
2583
2584 let word = match self.fetch_word(bus, pc) {
2585 Ok(word) => word,
2586 Err(exc) => {
2587 // Populate the latch BEFORE raising. `abort_with` captures the
2588 // faulting instruction's context out of the latch its stage reads,
2589 // which for `Stage::Ic` is `ic_rf` -- so raising first would capture
2590 // the PREVIOUS fetch's `pc` and delay-slot flag and write a wrong
2591 // `EPC`. Stamp before you move, and populate before you stamp.
2592 self.ic_rf = Latch {
2593 cop0: None,
2594 occupied: true,
2595 pc,
2596 in_delay_slot,
2597 abort: Some(exc),
2598 ..Latch::default()
2599 };
2600 self.abort_with(Stage::Ic, exc, pc);
2601 // `next_pc` is deliberately NOT realigned on an alignment fault.
2602 // Rounding it down would silently "fix" the faulting address and
2603 // let execution continue on a path hardware never takes -- turning
2604 // a raised exception into a wrong answer. The redirect to the
2605 // exception vector happens in `advance`, after the cascade
2606 // (T-12-002).
2607 return;
2608 }
2609 };
2610 // Decode here rather than at RF: a branch must be decoded before the
2611 // NEXT fetch, so that fetch can be marked as its delay slot.
2612 //
2613 // A branch decoded last cycle is in `rf_ex` by now, so the instruction
2614 // being fetched here is its delay slot -- see `in_delay_slot` above,
2615 // computed once at the top of the stage and used by both paths.
2616 self.ic_rf = Latch {
2617 occupied: true,
2618 pc,
2619 word,
2620 in_delay_slot,
2621 abort: None,
2622 decoded: decode(word),
2623 rs_val: 0,
2624 rt_val: 0,
2625 write_back: WriteBack::None,
2626 mem: None,
2627 cop0: None,
2628 };
2629 *next_pc = pc.wrapping_add(4);
2630 }
2631
2632 /// Perform a COP1 arithmetic operation against the FPR file.
2633 ///
2634 /// Lives here rather than in `exec::execute` because it reads two FPRs and
2635 /// writes a third, and `execute` has no access to the register file — the
2636 /// same reason the COP1 moves are split this way.
2637 ///
2638 /// # Register access goes through the `FR` view
2639 ///
2640 /// Operands are read with [`Fpr::read_s`]/[`Fpr::read_d`], **not**
2641 /// `read_raw`. Using the raw register was ledger U-7's bug: with `FR = 0` a
2642 /// double lives across an FGR *pair*, and a raw read returns half of it.
2643 ///
2644 /// # `FCSR`
2645 ///
2646 /// `Cause` is bits **17:12** and reports what *this* operation raised; it
2647 /// is replaced wholesale each time. `Flags` (6:2) is the sticky
2648 /// accumulation and is OR-ed in. `Flags::to_fcsr_bits` produces both, so
2649 /// clearing only `Cause` before OR-ing preserves the sticky half.
2650 ///
2651 /// **The field is 17:12, not 16:12.** Bit 17 is `Cause.E`, Unimplemented
2652 /// Operation — part of `Cause` despite having no `Enable` bit and no sticky
2653 /// `Flags` twin, which means the mask is the *only* thing that ever clears
2654 /// it. This comment said 16:12 while the mask covered 16:12 too, and the
2655 /// result was a bit that could never be cleared once raised. Only the five
2656 /// *maskable* conditions live in 16:12; that narrower range is what the
2657 /// enable comparison below uses, and it is a different statement.
2658 ///
2659 /// The mask is now [`fpu::CAUSE_MASK`](crate::fpu::CAUSE_MASK), defined once.
2660 /// It had reached **four** local copies of the same literal — in a file where
2661 /// getting this exact constant wrong has already shipped a bug.
2662 ///
2663 /// # Enabled traps
2664 ///
2665 /// A condition whose `FCSR.Enable` bit is set raises
2666 /// [`Exception::FloatingPoint`] instead of completing, and **three** things
2667 /// then differ from the untrapped path. All three are architectural, and
2668 /// each is separately observable by n64-systemtest:
2669 ///
2670 /// 1. **`fd` is not written.** The trap is precise, so the destination keeps
2671 /// its old value — which is what the suite checks with its
2672 /// `Result after operation (with exception)` assertion.
2673 /// 2. **The sticky `Flags` field is not updated.** Only `Cause` is. This is
2674 /// easy to get wrong because the untrapped path sets both from the same
2675 /// helper, and a trapped operation that also OR-ed into `Flags` looks
2676 /// right in every test that does not read `FCSR` back.
2677 /// 3. **The instruction does not retire**, so it must not tick `Random`.
2678 ///
2679 /// Returns `true` when it trapped, so `wb_stage` can skip its retirement
2680 /// tail.
2681 ///
2682 /// # Still not handled
2683 ///
2684 /// The **unimplemented-operation** cause (bit 17) is unmaskable and is not
2685 /// produced by the arithmetic here — the VR4300 raises it for subnormal
2686 /// operands and results, which this FPU computes normally instead. That is
2687 /// a separate body of work from the maskable enables, and the suite's
2688 /// `expected_unimplemented` cases still fail.
2689 fn fp_arith(&mut self, fmt: u8, funct: u8, ft: u8, fs: u8, fd: u8) -> bool {
2690 use crate::fpu;
2691 let fr = fr_of(&self.cop0);
2692 // `fmt` is 16 (single) or 17 (double) -- decode admits no other value
2693 // into `FpArith`, so this is a two-way split, not a table.
2694 // `funct` 5/6/7 read only `fs`, so they are handled ahead of the
2695 // arithmetic split and `ft` is never read for an instruction whose
2696 // `ft` field is architecturally zero.
2697 //
2698 // **`MOV` (funct 6) alone is the pure bit move.** `ABS` (5) and `NEG`
2699 // (7) look like sign flips and are not: they classify their operand,
2700 // raising Invalid on a signaling NaN and unimplemented-operation on a
2701 // subnormal or an MSB-clear NaN, and they REPLACE the `Cause` field.
2702 // An earlier version of this comment described all three as raising
2703 // nothing, which was true of `MOV` and never of its neighbors.
2704 //
2705 // n64-systemtest settles which is which by construction rather than by
2706 // description: `MOV.S` is driven through
2707 // `test_floating_point_f32_which_preserves_cause_bits`, while `ABS.S`
2708 // and `NEG.S` go through the ordinary `test_floating_point_f32`, which
2709 // asserts `Cause` was cleared. Treating all three alike was worth 52
2710 // assertions.
2711 if matches!(funct, 5 | 7) {
2712 return self.fp_sign_op(fmt, funct, fs, fd, fr);
2713 }
2714 if funct == 6 {
2715 // **`MOV.S` moves all 64 bits, not just the formatted half.**
2716 // n64-systemtest's "Upper bits of 32 bit operation" reads the
2717 // destination back with `DMFC1` after a `MOV.S` and expects the
2718 // SOURCE's upper half there, not the destination's previous
2719 // contents -- so this is a whole-register transfer that happens to
2720 // be spelled `.S`.
2721 let v = self.fpr.read_d_fs(fs, fr);
2722 self.fpr.write_d_arith(fd, fr, v);
2723 // **`FCSR` is left completely alone**, `Cause` included.
2724 //
2725 // Clearing `Cause` here was written first, on no evidence, and was
2726 // measurably wrong: the compiler emits `MOV.fmt` to move an FP
2727 // return value, so a `MOV` sitting between an arithmetic operation
2728 // and the `CFC1` that reads its result wiped the very `Cause` bits
2729 // the program was about to inspect. n64-systemtest saw
2730 // `flags: inexact` with `causes: ""` — the sticky half surviving
2731 // and the per-operation half erased — which is the signature of a
2732 // later instruction overwriting it, not of a flag never set.
2733 //
2734 // The architectural rule is that `Cause` is written by operations
2735 // that *can* raise. These cannot, so they write nothing.
2736 return false;
2737 }
2738
2739 // `FCSR.RM` is read **here**, per operation, rather than being captured
2740 // anywhere earlier: software changes it between instructions, and
2741 // n64-systemtest sweeps all four modes over the same operand pair.
2742 let mode = fpu::Rounding::from_rm(self.cop1.rounding_mode());
2743
2744 // Computed but **not committed**. Whether the write happens depends on
2745 // the enables, and they cannot be consulted until the flags are known.
2746 // Writing inside a branch and undoing it afterwards would be wrong
2747 // under `FR = 0`, where a `.S` write can disturb a neighboring
2748 // register's half.
2749 let (commit, flags, unimplemented) = match funct {
2750 0o00..=0o03 => self.fp_binary(fmt, funct, ft, fs, fr, mode),
2751 0o04 => self.fp_sqrt(fmt, fs, fr, mode),
2752 0o10..=0o17 => self.fp_to_integer(fmt, funct, fs, fr),
2753 0o40 | 0o41 | 0o44 | 0o45 => self.fp_convert(fmt, funct, fs, fr, mode),
2754 // 0o60..=0o77 -- `C.cond.fmt`. The low four bits ARE the condition
2755 // (UM Table 7-11), so the sixteen mnemonics need no table.
2756 _ => self.fp_compare(fmt, funct & 0xF, ft, fs, fr),
2757 };
2758
2759 let raised = flags.to_fcsr_bits()
2760 | if unimplemented {
2761 fpu::CAUSE_UNIMPLEMENTED
2762 } else {
2763 0
2764 };
2765 let fcsr = self.cop1.fcsr();
2766
2767 // `Cause` bits 16:12 and the `Enable` field bits 11:7 hold the five
2768 // conditions in the SAME order, so shifting `Cause` down by 12 lines it
2769 // up with what `Cop1Control::enables` returns. Comparing them in
2770 // different orders is a silent mis-map that only shows up on whichever
2771 // condition happens to be tested first.
2772 //
2773 // **Unimplemented Operation (bit 17) is unmaskable** and sits above
2774 // that field, so it is checked separately rather than being folded into
2775 // the enable comparison — where it would have been silently ignored,
2776 // since no enable bit corresponds to it.
2777 if unimplemented || (raised >> 12) & self.cop1.enables() != 0 {
2778 // Cause only. The sticky `Flags` field is deliberately left
2779 // untouched — see the doc comment.
2780 self.cop1
2781 .ctc1(31, (fcsr & !fpu::CAUSE_MASK) | (raised & fpu::CAUSE_MASK));
2782 self.abort_from(Stage::Wb, Exception::FloatingPoint);
2783 return true;
2784 }
2785
2786 match commit {
2787 // CLEARS the upper half -- `write_s_arith`, not `write_s`. An
2788 // arithmetic `.S` result is not an `MTC1`: the suite reads the
2789 // destination back with `DMFC1` and expects zero above (C-10).
2790 // Going through the `Fpr` accessor rather than `write_raw` is what
2791 // keeps the `FR` view applied, which is the part ledger U-7 records.
2792 FpCommit::Single(v) => self.fpr.write_s_arith(fd, fr, v),
2793 FpCommit::Double(v) => self.fpr.write_d_arith(fd, fr, v),
2794 // `FCSR.C` is bit 23, and it is NOT part of the `Cause`/`Flags`
2795 // bookkeeping — a compare writes it and no other operation touches
2796 // it. Confirmed against n64-systemtest's own `FCSR` bitfield rather
2797 // than inferred.
2798 FpCommit::Condition(c) => {
2799 let base = (fcsr & !fpu::CAUSE_MASK & !FCSR_C) | raised;
2800 self.cop1.ctc1(31, base | if c { FCSR_C } else { 0 });
2801 return false;
2802 }
2803 }
2804 self.cop1.ctc1(31, (fcsr & !fpu::CAUSE_MASK) | raised);
2805 false
2806 }
2807
2808 /// `ABS` and `NEG` — sign manipulation, but **not** a pure bit flip.
2809 ///
2810 /// The VR4300 classifies the operand first: a subnormal or an MSB-clear NaN
2811 /// raises unimplemented-operation, and an MSB-set (signaling, ledger C-12)
2812 /// NaN raises Invalid and yields the default NaN rather than the operand
2813 /// with its sign changed. Only when the operand is ordinary does the sign
2814 /// bit move.
2815 ///
2816 /// Unlike `MOV`, these REPLACE the `Cause` field — clearing it on success.
2817 fn fp_sign_op(&mut self, fmt: u8, funct: u8, fs: u8, fd: u8, fr: bool) -> bool {
2818 use crate::fpu;
2819 let fcsr = self.cop1.fcsr();
2820 let (commit, flags, unimplemented) = if fmt == 0o20 {
2821 let a = f32::from_bits(self.fpr.read_s_fs(fs, fr));
2822 if fpu::is_subnormal_f32(a) || fpu::is_unimplemented_nan_f32(a) {
2823 (0u64, fpu::Flags::NONE, true)
2824 } else if fpu::is_snan_f32(a) {
2825 (
2826 u64::from(crate::softfloat::F32.default_nan() as u32),
2827 fpu::Flags::INVALID,
2828 false,
2829 )
2830 } else {
2831 let v = if funct == 5 {
2832 fpu::abs_s(a)
2833 } else {
2834 fpu::neg_s(a)
2835 };
2836 (u64::from(v.to_bits()), fpu::Flags::NONE, false)
2837 }
2838 } else {
2839 let a = f64::from_bits(self.fpr.read_d_fs(fs, fr));
2840 if fpu::is_subnormal_f64(a) || fpu::is_unimplemented_nan_f64(a) {
2841 (0u64, fpu::Flags::NONE, true)
2842 } else if fpu::is_snan_f64(a) {
2843 (
2844 crate::softfloat::F64.default_nan(),
2845 fpu::Flags::INVALID,
2846 false,
2847 )
2848 } else {
2849 let v = if funct == 5 {
2850 fpu::abs_d(a)
2851 } else {
2852 fpu::neg_d(a)
2853 };
2854 (v.to_bits(), fpu::Flags::NONE, false)
2855 }
2856 };
2857
2858 let raised = flags.to_fcsr_bits()
2859 | if unimplemented {
2860 fpu::CAUSE_UNIMPLEMENTED
2861 } else {
2862 0
2863 };
2864 if unimplemented {
2865 self.cop1
2866 .ctc1(31, (fcsr & !fpu::CAUSE_MASK) | (raised & fpu::CAUSE_MASK));
2867 self.abort_from(Stage::Wb, Exception::FloatingPoint);
2868 return true;
2869 }
2870 if flags.invalid && self.cop1.enables() & (1 << 4) != 0 {
2871 self.cop1
2872 .ctc1(31, (fcsr & !fpu::CAUSE_MASK) | (raised & fpu::CAUSE_MASK));
2873 self.abort_from(Stage::Wb, Exception::FloatingPoint);
2874 return true;
2875 }
2876 if fmt == 0o20 {
2877 self.fpr.write_s_arith(fd, fr, commit as u32);
2878 } else {
2879 self.fpr.write_d_arith(fd, fr, commit);
2880 }
2881 self.cop1.ctc1(31, (fcsr & !fpu::CAUSE_MASK) | raised);
2882 false
2883 }
2884
2885 /// `SQRT.fmt`.
2886 ///
2887 /// Unary, so it takes the operand policy but not the *result* policy: the
2888 /// square root of the smallest normal is about `1e-19`, nowhere near the
2889 /// subnormal range, so a normal operand cannot produce a subnormal result.
2890 ///
2891 /// A negative operand is Invalid rather than unimplemented — except `-0`,
2892 /// whose root is `-0` and raises nothing. [`softfloat::sqrt`] draws that
2893 /// distinction.
2894 fn fp_sqrt(
2895 &self,
2896 fmt: u8,
2897 fs: u8,
2898 fr: bool,
2899 mode: crate::fpu::Rounding,
2900 ) -> (FpCommit, crate::fpu::Flags, bool) {
2901 use crate::fpu;
2902 if fmt == 0o20 {
2903 let bits = u64::from(self.fpr.read_s_fs(fs, fr));
2904 let a = f32::from_bits(bits as u32);
2905 if fpu::is_subnormal_f32(a) || fpu::is_unimplemented_nan_f32(a) {
2906 return (FpCommit::Single(0), fpu::Flags::NONE, true);
2907 }
2908 let r = softfloat::sqrt(bits, softfloat::F32, mode);
2909 (FpCommit::Single(r.bits as u32), r.flags, false)
2910 } else {
2911 let bits = self.fpr.read_d_fs(fs, fr);
2912 let a = f64::from_bits(bits);
2913 if fpu::is_subnormal_f64(a) || fpu::is_unimplemented_nan_f64(a) {
2914 return (FpCommit::Double(0), fpu::Flags::NONE, true);
2915 }
2916 let r = softfloat::sqrt(bits, softfloat::F64, mode);
2917 (FpCommit::Double(r.bits), r.flags, false)
2918 }
2919 }
2920
2921 /// The VR4300's policy for a **subnormal result**, applied wherever one can
2922 /// be produced — arithmetic and the narrowing `CVT.S.D`.
2923 ///
2924 /// Three outcomes, in order:
2925 ///
2926 /// 1. `FCSR.FS` clear — the processor cannot represent the result at all,
2927 /// so *unimplemented operation*.
2928 /// 2. `FS` set but underflow or inexact **enabled** — it cannot deliver a
2929 /// trapped underflow's defined result either, so unimplemented again.
2930 /// n64-systemtest's own comment on this case reads "(wow)".
2931 /// 3. `FS` set and both disabled — flush per
2932 /// [`fpu::flush_subnormal_f32`], reporting underflow and inexact.
2933 fn subnormal_policy_s(
2934 &self,
2935 out: crate::fpu::Outcome<f32>,
2936 mode: crate::fpu::Rounding,
2937 ) -> (FpCommit, crate::fpu::Flags, bool) {
2938 use crate::fpu;
2939 // **Either** condition triggers, and both are needed.
2940 //
2941 // `is_subnormal` alone misses a result that underflows past the
2942 // subnormal grid all the way to zero (`f64::MIN_POSITIVE` narrowed to
2943 // `f32`; `MIN_POSITIVE` squared) — the VR4300 refuses those exactly as
2944 // it refuses a subnormal.
2945 //
2946 // `flags.underflow` alone misses an *exact* subnormal, because IEEE
2947 // signals underflow only when tiny **and inexact**. Replacing the
2948 // first test with the second was tried and regressed the oracle from
2949 // 89 to 131.
2950 if !fpu::is_subnormal_f32(out.value) && !out.flags.underflow {
2951 return (FpCommit::Single(out.value.to_bits()), out.flags, false);
2952 }
2953 if !self.cop1.flush_denorm_to_zero() || self.underflow_traps() {
2954 return (FpCommit::Single(0), fpu::Flags::NONE, true);
2955 }
2956 let mut flags = out.flags;
2957 flags.underflow = true;
2958 flags.inexact = true;
2959 let v = fpu::flush_subnormal_f32(out.value, mode);
2960 (FpCommit::Single(v.to_bits()), flags, false)
2961 }
2962
2963 /// See [`Pipeline::subnormal_policy_s`].
2964 fn subnormal_policy_d(
2965 &self,
2966 out: crate::fpu::Outcome<f64>,
2967 mode: crate::fpu::Rounding,
2968 ) -> (FpCommit, crate::fpu::Flags, bool) {
2969 use crate::fpu;
2970 // **Either** condition triggers, and both are needed.
2971 //
2972 // `is_subnormal` alone misses a result that underflows past the
2973 // subnormal grid all the way to zero (`f64::MIN_POSITIVE` narrowed to
2974 // `f32`; `MIN_POSITIVE` squared) — the VR4300 refuses those exactly as
2975 // it refuses a subnormal.
2976 //
2977 // `flags.underflow` alone misses an *exact* subnormal, because IEEE
2978 // signals underflow only when tiny **and inexact**. Replacing the
2979 // first test with the second was tried and regressed the oracle from
2980 // 89 to 131.
2981 if !fpu::is_subnormal_f64(out.value) && !out.flags.underflow {
2982 return (FpCommit::Double(out.value.to_bits()), out.flags, false);
2983 }
2984 if !self.cop1.flush_denorm_to_zero() || self.underflow_traps() {
2985 return (FpCommit::Double(0), fpu::Flags::NONE, true);
2986 }
2987 let mut flags = out.flags;
2988 flags.underflow = true;
2989 flags.inexact = true;
2990 let v = fpu::flush_subnormal_f64(out.value, mode);
2991 (FpCommit::Double(v.to_bits()), flags, false)
2992 }
2993
2994 /// Does the CURRENT `FCSR` already meet a trap condition?
2995 ///
2996 /// Used after a `CTC1` writes it: the register can be put into a trapping
2997 /// state directly, without any arithmetic having run.
2998 fn fcsr_traps_now(&self) -> bool {
2999 /// `FCSR.Cause` maskable bits 16:12, shifted to line up with `Enable`.
3000 const CAUSE_SHIFT: u32 = 12;
3001 let fcsr = self.cop1.fcsr();
3002 if fcsr & crate::fpu::CAUSE_UNIMPLEMENTED != 0 {
3003 return true;
3004 }
3005 (fcsr >> CAUSE_SHIFT) & self.cop1.enables() != 0
3006 }
3007
3008 /// Is underflow or inexact enabled? Either turns a flushed subnormal into
3009 /// an unimplemented operation.
3010 fn underflow_traps(&self) -> bool {
3011 /// `FCSR.Enable` underflow (bit 8) and inexact (bit 7), as
3012 /// `Cop1Control::enables` returns them — shifted down by 7.
3013 const ENABLE_UNDERFLOW_OR_INEXACT: u32 = 0b11;
3014 self.cop1.enables() & ENABLE_UNDERFLOW_OR_INEXACT != 0
3015 }
3016
3017 /// `ADD`/`SUB`/`MUL`/`DIV` in either format.
3018 ///
3019 /// # The VR4300 cannot compute with subnormals
3020 ///
3021 /// It raises the unmaskable *unimplemented operation* cause instead, and
3022 /// there are three distinct occasions (UM §7.5; pinned by n64-systemtest):
3023 ///
3024 /// 1. **A subnormal operand** — checked before the operation is attempted,
3025 /// and it outranks everything, including a NaN that would otherwise
3026 /// raise Invalid.
3027 /// 2. **A subnormal result with `FCSR.FS` clear.**
3028 /// 3. **A subnormal result with `FS` set but underflow or inexact
3029 /// *enabled*.** The processor cannot deliver a trapped underflow's
3030 /// defined result, so it declines instead — the suite's own comment on
3031 /// this case reads "(wow)".
3032 ///
3033 /// With `FS` set and those enables clear it flushes, per
3034 /// [`fpu::flush_subnormal_f32`], and reports underflow + inexact.
3035 ///
3036 /// The returned flags are deliberately [`fpu::Flags::NONE`] on every
3037 /// unimplemented path: `FCSR` must end up with bit 17 and *nothing else*,
3038 /// which is what the suite asserts.
3039 fn fp_binary(
3040 &self,
3041 fmt: u8,
3042 funct: u8,
3043 ft: u8,
3044 fs: u8,
3045 fr: bool,
3046 mode: crate::fpu::Rounding,
3047 ) -> (FpCommit, crate::fpu::Flags, bool) {
3048 use crate::fpu;
3049 if fmt == 0o20 {
3050 let a = f32::from_bits(self.fpr.read_s_fs(fs, fr));
3051 let b = f32::from_bits(self.fpr.read_s_ft(ft, fr));
3052 if fpu::arith_unimplemented_s(a, b) {
3053 return (FpCommit::Single(0), fpu::Flags::NONE, true);
3054 }
3055 let out = match funct {
3056 0 => fpu::add_s(a, b, mode),
3057 1 => fpu::sub_s(a, b, mode),
3058 2 => fpu::mul_s(a, b, mode),
3059 _ => fpu::div_s(a, b, mode),
3060 };
3061 self.subnormal_policy_s(out, mode)
3062 } else {
3063 let a = f64::from_bits(self.fpr.read_d_fs(fs, fr));
3064 let b = f64::from_bits(self.fpr.read_d_ft(ft, fr));
3065 if fpu::arith_unimplemented_d(a, b) {
3066 return (FpCommit::Double(0), fpu::Flags::NONE, true);
3067 }
3068 let out = match funct {
3069 0 => fpu::add_d(a, b, mode),
3070 1 => fpu::sub_d(a, b, mode),
3071 2 => fpu::mul_d(a, b, mode),
3072 _ => fpu::div_d(a, b, mode),
3073 };
3074 self.subnormal_policy_d(out, mode)
3075 }
3076 }
3077
3078 /// `ROUND`/`TRUNC`/`CEIL`/`FLOOR` to `.W` or `.L` (funct 8..=15).
3079 ///
3080 /// These carry their rounding mode **in the opcode** and ignore `FCSR.RM`
3081 /// entirely — that is the whole reason they exist alongside `CVT.W`/`CVT.L`,
3082 /// which do consult it. Passing the live `RM` here would make all four
3083 /// behave identically whenever `RM` happened to match, and the difference
3084 /// would only show up under a non-default mode.
3085 fn fp_to_integer(
3086 &self,
3087 fmt: u8,
3088 funct: u8,
3089 fs: u8,
3090 fr: bool,
3091 ) -> (FpCommit, crate::fpu::Flags, bool) {
3092 use crate::fpu::{self, Rounding};
3093 let mode = match funct & 0o3 {
3094 0 => Rounding::Nearest,
3095 1 => Rounding::TowardZero,
3096 2 => Rounding::TowardPlusInf,
3097 _ => Rounding::TowardMinusInf,
3098 };
3099 // Refusal is decided BEFORE the source is read as a float: an integer
3100 // source format has no float to widen, and reading one anyway produces a
3101 // plausible number for an instruction that does not exist.
3102 if self.integer_conversion_unimplemented(fmt, fs, fr) {
3103 return (FpCommit::Single(0), fpu::Flags::NONE, true);
3104 }
3105 // The source is widened to `f64` first, which is EXACT for an `f32`, so
3106 // no rounding happens before the one the instruction asks for.
3107 let v = self.fp_source_as_f64(fmt, fs, fr);
3108 // funct 8..=11 target `.L`, 12..=15 target `.W`.
3109 let wide = funct < 0o14;
3110 if wide {
3111 let out = fpu::to_i64(v, mode);
3112 // `to_i64` reports NaN and out-of-range as Invalid, which is the
3113 // IEEE answer and NOT this processor's: the VR4300 declines with
3114 // *unimplemented operation* instead. n64-systemtest expects `Err`
3115 // for infinities, NaNs and anything past the target's range.
3116 if out.flags.invalid {
3117 return (FpCommit::Double(0), fpu::Flags::NONE, true);
3118 }
3119 #[allow(clippy::cast_sign_loss)] // a bit pattern, not a magnitude
3120 (FpCommit::Double(out.value as u64), out.flags, false)
3121 } else {
3122 let out = fpu::to_i32(v, mode);
3123 if out.flags.invalid {
3124 return (FpCommit::Single(0), fpu::Flags::NONE, true);
3125 }
3126 #[allow(clippy::cast_sign_loss)] // a bit pattern, not a magnitude
3127 (FpCommit::Single(out.value as u32), out.flags, false)
3128 }
3129 }
3130
3131 /// Is the source of a float-to-integer conversion one the VR4300 refuses?
3132 ///
3133 /// Subnormality, plus a magnitude of `2^53` or more. NaN and infinity are
3134 /// detected from the conversion's own result, because "out of range"
3135 /// depends on the target width.
3136 ///
3137 /// The limit applies to `.W` targets as well, where it is **unobservable**:
3138 /// `2^53` is far outside `i32`, so such a value is refused either way. It
3139 /// was originally guarded on the target width, and the guard was removed
3140 /// because no test could distinguish the two — an undistinguishable branch
3141 /// is one that rots.
3142 ///
3143 /// # The `2^53` limit is narrower than `i64`
3144 ///
3145 /// `9007198717870080` converts; `9007199254740992` (`2^53`) raises
3146 /// unimplemented — both far inside `i64`'s range. n64-systemtest brackets
3147 /// the threshold with adjacent values on either side, which is what
3148 /// identifies it as `2^53` rather than some larger bound. `2^53` is the
3149 /// last integer a `double` represents exactly, so the natural reading is
3150 /// that the conversion runs through double precision internally and
3151 /// declines whatever it cannot hold exactly.
3152 fn integer_conversion_unimplemented(&self, fmt: u8, fs: u8, fr: bool) -> bool {
3153 use crate::fpu;
3154 /// The last integer a `double` represents exactly.
3155 const TWO_POW_53: f64 = 9_007_199_254_740_992.0;
3156
3157 // An INTEGER source format is not a conversion this processor has:
3158 // `CVT.W.W`, `CVT.W.L`, `CVT.L.W`, `CVT.L.L` and the whole
3159 // `ROUND`/`TRUNC`/`CEIL`/`FLOOR` family from `.W`/`.L` do not exist, and
3160 // the VR4300 declines them with Unimplemented Operation rather than
3161 // reinterpreting the source. Checked first, because every branch below
3162 // would happily read the register as a float and return a number.
3163 if fmt != 0o20 && fmt != 0o21 {
3164 return true;
3165 }
3166 let v = if fmt == 0o20 {
3167 let a = f32::from_bits(self.fpr.read_s_fs(fs, fr));
3168 if fpu::is_subnormal_f32(a) {
3169 return true;
3170 }
3171 f64::from(a)
3172 } else {
3173 let a = f64::from_bits(self.fpr.read_d_fs(fs, fr));
3174 if fpu::is_subnormal_f64(a) {
3175 return true;
3176 }
3177 a
3178 };
3179 v.abs() >= TWO_POW_53
3180 }
3181
3182 /// `CVT.S`/`CVT.D`/`CVT.W`/`CVT.L`, from any source format.
3183 fn fp_convert(
3184 &self,
3185 fmt: u8,
3186 funct: u8,
3187 fs: u8,
3188 fr: bool,
3189 mode: crate::fpu::Rounding,
3190 ) -> (FpCommit, crate::fpu::Flags, bool) {
3191 use crate::fpu;
3192 match funct {
3193 // To single.
3194 0o40 => match fmt {
3195 0o21 => {
3196 let bits = self.fpr.read_d_fs(fs, fr);
3197 let a = f64::from_bits(bits);
3198 if fpu::is_subnormal_f64(a) || fpu::is_unimplemented_nan_f64(a) {
3199 return (FpCommit::Single(0), fpu::Flags::NONE, true);
3200 }
3201 // Narrowing is ARITHMETIC: it rounds 53 significand bits
3202 // into 24, so it can be inexact, can overflow, and can land
3203 // in the subnormal range -- each depending on `FCSR.RM`.
3204 // `v as f32` reports none of that and rounds to nearest
3205 // only, which is where ledger C-11 found this operation
3206 // after the arithmetic had already been fixed.
3207 let r = softfloat::convert(bits, softfloat::F64, softfloat::F32, mode);
3208 let out = fpu::Outcome {
3209 value: f32::from_bits(r.bits as u32),
3210 flags: r.flags,
3211 };
3212 // Narrowing can produce a subnormal from a perfectly normal
3213 // double, so the result policy applies exactly as it does
3214 // to the arithmetic.
3215 self.subnormal_policy_s(out, mode)
3216 }
3217 #[allow(clippy::cast_possible_wrap)] // reinterpreting a word as signed
3218 0o24 => {
3219 // Through `softfloat`, not a Rust `as` cast: an i32 can need
3220 // more than 24 significand bits, so this rounds -- and must
3221 // round the way `FCSR.RM` says.
3222 let v = i64::from(self.fpr.read_s_fs(fs, fr) as i32);
3223 let r = softfloat::from_int(v, softfloat::F32, mode);
3224 (FpCommit::Single(r.bits as u32), r.flags, false)
3225 }
3226 // From `.L`, which the VR4300 restricts: bits 63:55 must be all
3227 // zeroes or all ones (UM §7.5.2). Outside that it raises
3228 // Unimplemented rather than converting, and there is no defined
3229 // result -- so the commit value is a placeholder the trap path
3230 // discards.
3231 #[allow(clippy::cast_possible_wrap)]
3232 _ => {
3233 let v = self.fpr.read_d_fs(fs, fr) as i64;
3234 if fpu::long_convertible(v) {
3235 let r = softfloat::from_int(v, softfloat::F32, mode);
3236 (FpCommit::Single(r.bits as u32), r.flags, false)
3237 } else {
3238 // No defined result when the restriction is violated, so
3239 // the value is a placeholder the trap path discards.
3240 (FpCommit::Single(0), fpu::Flags::NONE, true)
3241 }
3242 }
3243 },
3244 // To double.
3245 0o41 => match fmt {
3246 0o20 => {
3247 let bits = u64::from(self.fpr.read_s_fs(fs, fr));
3248 let a = f32::from_bits(bits as u32);
3249 if fpu::is_subnormal_f32(a) || fpu::is_unimplemented_nan_f32(a) {
3250 return (FpCommit::Double(0), fpu::Flags::NONE, true);
3251 }
3252 // Widening cannot lose anything, so no result policy is
3253 // needed — but it goes through the same path so there is
3254 // one conversion rather than two that can disagree.
3255 let r = softfloat::convert(bits, softfloat::F32, softfloat::F64, mode);
3256 (FpCommit::Double(r.bits), r.flags, false)
3257 }
3258 #[allow(clippy::cast_possible_wrap)]
3259 0o24 => {
3260 // Exact for every i32 -- 53 bits hold 32 -- but routed the
3261 // same way as the others so there is one conversion rather
3262 // than two that can disagree.
3263 let v = i64::from(self.fpr.read_s_fs(fs, fr) as i32);
3264 let r = softfloat::from_int(v, softfloat::F64, mode);
3265 (FpCommit::Double(r.bits), r.flags, false)
3266 }
3267 #[allow(clippy::cast_possible_wrap)]
3268 _ => {
3269 let v = self.fpr.read_d_fs(fs, fr) as i64;
3270 if fpu::long_convertible(v) {
3271 let r = softfloat::from_int(v, softfloat::F64, mode);
3272 (FpCommit::Double(r.bits), r.flags, false)
3273 } else {
3274 // No defined result when the restriction is violated, so
3275 // the value is a placeholder the trap path discards.
3276 (FpCommit::Double(0), fpu::Flags::NONE, true)
3277 }
3278 }
3279 },
3280 // To word / to long, both honoring `FCSR.RM` -- which is what
3281 // separates them from the fixed-mode family above.
3282 0o44 => {
3283 if self.integer_conversion_unimplemented(fmt, fs, fr) {
3284 return (FpCommit::Single(0), fpu::Flags::NONE, true);
3285 }
3286 let out = fpu::to_i32(self.fp_source_as_f64(fmt, fs, fr), mode);
3287 if out.flags.invalid {
3288 return (FpCommit::Single(0), fpu::Flags::NONE, true);
3289 }
3290 #[allow(clippy::cast_sign_loss)]
3291 (FpCommit::Single(out.value as u32), out.flags, false)
3292 }
3293 _ => {
3294 if self.integer_conversion_unimplemented(fmt, fs, fr) {
3295 return (FpCommit::Double(0), fpu::Flags::NONE, true);
3296 }
3297 let out = fpu::to_i64(self.fp_source_as_f64(fmt, fs, fr), mode);
3298 if out.flags.invalid {
3299 return (FpCommit::Double(0), fpu::Flags::NONE, true);
3300 }
3301 #[allow(clippy::cast_sign_loss)]
3302 (FpCommit::Double(out.value as u64), out.flags, false)
3303 }
3304 }
3305 }
3306
3307 /// `C.cond.fmt` — writes `FCSR.C`, never an FPR.
3308 fn fp_compare(
3309 &self,
3310 fmt: u8,
3311 cond: u8,
3312 ft: u8,
3313 fs: u8,
3314 fr: bool,
3315 ) -> (FpCommit, crate::fpu::Flags, bool) {
3316 use crate::fpu;
3317 let out = if fmt == 0o20 {
3318 fpu::compare_s(
3319 f32::from_bits(self.fpr.read_s_fs(fs, fr)),
3320 f32::from_bits(self.fpr.read_s_ft(ft, fr)),
3321 cond,
3322 )
3323 } else {
3324 fpu::compare_d(
3325 f64::from_bits(self.fpr.read_d_fs(fs, fr)),
3326 f64::from_bits(self.fpr.read_d_ft(ft, fr)),
3327 cond,
3328 )
3329 };
3330 (FpCommit::Condition(out.value), out.flags, false)
3331 }
3332
3333 /// Read `fs` in `fmt` and widen to `f64`.
3334 ///
3335 /// `f32` to `f64` is exact, so a `.S` source loses nothing on the way in and
3336 /// the only rounding is the one the instruction performs.
3337 fn fp_source_as_f64(&self, fmt: u8, fs: u8, fr: bool) -> f64 {
3338 if fmt == 0o20 {
3339 f64::from(f32::from_bits(self.fpr.read_s_fs(fs, fr)))
3340 } else {
3341 f64::from_bits(self.fpr.read_d_fs(fs, fr))
3342 }
3343 }
3344}
3345
3346#[cfg(test)]
3347mod tests {
3348 use super::*;
3349
3350 struct NullBus {
3351 irq: bool,
3352 }
3353 impl Bus for NullBus {
3354 fn read_u8(&mut self, _addr: u32) -> u8 {
3355 0
3356 }
3357 fn write_u8(&mut self, _addr: u32, _val: u8) {}
3358 fn poll_irq(&mut self) -> bool {
3359 self.irq
3360 }
3361 }
3362 fn quiet() -> NullBus {
3363 NullBus { irq: false }
3364 }
3365
3366 /// **The reverse-order invariant.** A value must advance exactly one stage
3367 /// per cycle. If the cascade is ever reordered to run forwards, a value falls
3368 /// through several stages in one cycle and the pipeline is silently too fast.
3369 #[test]
3370 fn a_value_advances_exactly_one_stage_per_cycle() {
3371 let mut p = Pipeline::new();
3372 let mut pc = 0xFFFF_FFFF_8000_0000;
3373 let mut regs = Regs::new();
3374 let mut bus = quiet();
3375
3376 // Cycle 1 fetches a sentinel into ic_rf.
3377 p.advance(&mut bus, &mut regs, &mut pc);
3378 let sentinel = p.ic_rf.pc;
3379 assert!(p.ic_rf.occupied);
3380 assert_eq!(sentinel, 0xFFFF_FFFF_8000_0000);
3381
3382 // Each subsequent cycle moves it exactly one boundary along.
3383 p.advance(&mut bus, &mut regs, &mut pc);
3384 assert_eq!(p.rf_ex.pc, sentinel, "after 1 cycle it should be at RF->EX");
3385 assert_ne!(p.ic_rf.pc, sentinel, "and no longer at IC->RF");
3386
3387 p.advance(&mut bus, &mut regs, &mut pc);
3388 assert_eq!(p.ex_dc.pc, sentinel, "after 2 cycles, EX->DC");
3389
3390 p.advance(&mut bus, &mut regs, &mut pc);
3391 assert_eq!(p.dc_wb.pc, sentinel, "after 3 cycles, DC->WB");
3392
3393 // 5 stages => at least 5 PCycles per instruction (UM §4.1).
3394 assert_eq!(p.retired, 0, "nothing may retire before WB has run on it");
3395 p.advance(&mut bus, &mut regs, &mut pc);
3396 assert_eq!(p.retired, 1, "retires at WB on the 5th cycle, not sooner");
3397 }
3398
3399 /// **R-19: a branch whose delay slot faulted must NOT take, but STILL links.**
3400 ///
3401 /// The n64-systemtest `ExecuteTLBMappedMissInDelay` case hung the whole suite:
3402 /// a JALR whose target is its own address, with a delay slot in an unmapped
3403 /// page. When the delay-slot fetch faults, the exception has already pointed
3404 /// `next_pc` at the vector; if the branch then applies its redirect it clobbers
3405 /// the vector, re-fetches itself, and loops forever. The fix suppresses the
3406 /// redirect when the delay slot (in `ic_rf`) carries an abort, while still
3407 /// writing the link from `pc + 8`.
3408 ///
3409 /// Mutation guard: the control case (no delay-slot fault) asserts the redirect
3410 /// *is* applied and the link is the live `next_pc`, so removing the fault check
3411 /// turns exactly one of the two assertions red.
3412 #[test]
3413 fn a_branch_with_a_faulting_delay_slot_links_but_does_not_take() {
3414 // JALR $6, $3 (funct 9) -- the exact instruction from the hung test.
3415 let jalr = decode(0x0060_3009);
3416 assert!(jalr.op.has_delay_slot(), "JALR must have a delay slot");
3417 let redirect = Redirect {
3418 target: 0x1234_4FFC, // the JALR jumps to its own address
3419 nullify_delay_slot: false,
3420 };
3421 let branch_pc = 0x1234_4FFC;
3422 let vector = 0xFFFF_FFFF_8000_0180; // where the exception already sent us
3423
3424 // Faulting case: the delay slot in `ic_rf` carries an abort.
3425 {
3426 let mut p = Pipeline::new();
3427 p.ic_rf = Latch {
3428 occupied: true,
3429 in_delay_slot: true,
3430 abort: Some(Exception::AddressError { store: false }),
3431 ..Latch::default()
3432 };
3433 let mut out = Latch {
3434 occupied: true,
3435 pc: branch_pc,
3436 decoded: jalr,
3437 ..Latch::default()
3438 };
3439 let mut next_pc = vector;
3440 p.resolve_branch_control(&mut out, Some(6), Some(redirect), &mut next_pc);
3441 assert_eq!(
3442 next_pc, vector,
3443 "the exception vector must survive -- no take"
3444 );
3445 assert_eq!(
3446 out.write_back,
3447 WriteBack::Gpr {
3448 dest: 6,
3449 value: branch_pc + 8,
3450 },
3451 "the link is written from pc + 8, not the clobbered next_pc",
3452 );
3453 }
3454
3455 // Control case: the delay slot did NOT fault -> the branch takes normally
3456 // and links from the live next_pc.
3457 {
3458 let mut p = Pipeline::new();
3459 p.ic_rf = Latch {
3460 occupied: true,
3461 in_delay_slot: true,
3462 abort: None,
3463 ..Latch::default()
3464 };
3465 let mut out = Latch {
3466 occupied: true,
3467 pc: branch_pc,
3468 decoded: jalr,
3469 ..Latch::default()
3470 };
3471 let mut next_pc = branch_pc + 8;
3472 p.resolve_branch_control(&mut out, Some(6), Some(redirect), &mut next_pc);
3473 assert_eq!(
3474 next_pc, redirect.target,
3475 "an unfaulted branch takes its target"
3476 );
3477 assert_eq!(
3478 out.write_back,
3479 WriteBack::Gpr {
3480 dest: 6,
3481 value: branch_pc + 8,
3482 },
3483 "the link is the live next_pc (which was pc + 8 here)",
3484 );
3485 }
3486 }
3487
3488 /// **`Cause.BD` and `EPC` still come out right when a stall separates the
3489 /// branch from its delay slot.**
3490 ///
3491 /// The companion test below pins that the flag stays *attached* to its
3492 /// instruction across a stall. That is not the same claim: the flag could
3493 /// travel correctly and still be read from the wrong place at dispatch,
3494 /// which is the reverse-cascade hazard this project has hit twice. So this
3495 /// one drives the flagged instruction into a real exception and reads the
3496 /// architectural result — `EPC` must name the **branch**, four bytes back,
3497 /// not the faulting instruction.
3498 #[test]
3499 fn a_delay_slot_exception_after_a_stall_still_reports_the_branch() {
3500 use crate::cop0::reg;
3501 let mut p = Pipeline::new();
3502 let mut regs = Regs::new();
3503 // An unaligned `LW` — offset 0x101 — so the delay-slot instruction
3504 // faults with AdEL rather than needing a crafted overflow.
3505 let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1), ld_st(0o43, 0, 3, 0x101)]);
3506 p.cop0.set_hardware(reg::STATUS, 0);
3507 let mut pc = KSEG0_PROG;
3508
3509 // Advance until the FAULTING instruction (the second word) is the one
3510 // in `IC/RF`, then mark it as the delay slot. Flagging whatever happens
3511 // to be there after one step marks the first instruction instead, which
3512 // never faults -- and the test would then be asserting BD for an
3513 // exception that never came from a delay slot at all.
3514 let slot_pc = KSEG0_PROG.wrapping_add(4);
3515 let mut warm = 0;
3516 while p.ic_rf.pc != slot_pc || !p.ic_rf.occupied {
3517 p.advance(&mut bus, &mut regs, &mut pc);
3518 warm += 1;
3519 assert!(warm < 16, "the faulting instruction never reached IC/RF");
3520 }
3521 p.ic_rf.in_delay_slot = true;
3522
3523 // A `DDIV`-length interlock lands between the branch and its slot.
3524 p.stall_for(69, Interlock::Mci);
3525 let mut cycles = 0;
3526 while p.stalled_by() != Some(Interlock::Exception) {
3527 p.advance(&mut bus, &mut regs, &mut pc);
3528 cycles += 1;
3529 assert!(cycles < 128, "the delay-slot instruction never faulted");
3530 }
3531
3532 assert_ne!(
3533 p.cop0.read(reg::CAUSE) & (1 << 31),
3534 0,
3535 "BD must be set -- the faulting instruction was in a delay slot"
3536 );
3537 assert_eq!(
3538 p.cop0.read(reg::EPC),
3539 slot_pc.wrapping_sub(4),
3540 "EPC must name the branch, so the handler re-evaluates it"
3541 );
3542 }
3543
3544 /// **The delay-slot invariant** — the Phase 1 exit criterion.
3545 ///
3546 /// A multi-cycle stall between a branch and its delay slot must not
3547 /// desynchronize the flag. A global `in_delay_slot` bool passes the naive
3548 /// test and fails this one, which is why the flag rides in the latch.
3549 #[test]
3550 fn delay_slot_flag_survives_a_multi_cycle_stall() {
3551 let mut p = Pipeline::new();
3552 let mut pc = 0xFFFF_FFFF_8000_0000;
3553 let mut regs = Regs::new();
3554 let mut bus = quiet();
3555
3556 // Two instructions in flight; mark the younger as the delay slot.
3557 p.advance(&mut bus, &mut regs, &mut pc);
3558 p.ic_rf.in_delay_slot = true;
3559 let slot_pc = p.ic_rf.pc;
3560
3561 // A long interlock lands between them (e.g. DDIV = 69 PCycles).
3562 p.stall_for(69, Interlock::Mci);
3563 for _ in 0..69 {
3564 p.advance(&mut bus, &mut regs, &mut pc);
3565 }
3566 assert!(p.stalled_by().is_none(), "the stall should have expired");
3567
3568 // The flag must still be attached to the SAME instruction.
3569 assert!(
3570 p.ic_rf.in_delay_slot && p.ic_rf.pc == slot_pc,
3571 "the delay-slot flag detached from its instruction across the stall"
3572 );
3573
3574 // And it must travel with it, not stay behind at a fixed boundary.
3575 p.advance(&mut bus, &mut regs, &mut pc);
3576 assert!(
3577 p.rf_ex.in_delay_slot && p.rf_ex.pc == slot_pc,
3578 "the flag failed to travel with the instruction"
3579 );
3580 assert!(
3581 !p.ic_rf.in_delay_slot,
3582 "the flag was left behind on the boundary instead of moving"
3583 );
3584 }
3585
3586 /// A stall holds every latch in place — nothing advances, nothing retires.
3587 #[test]
3588 fn a_stall_freezes_the_pipeline() {
3589 let mut p = Pipeline::new();
3590 let mut pc = 0xFFFF_FFFF_8000_0000;
3591 let mut regs = Regs::new();
3592 let mut bus = quiet();
3593 for _ in 0..4 {
3594 p.advance(&mut bus, &mut regs, &mut pc);
3595 }
3596 let before = (p.ic_rf, p.rf_ex, p.ex_dc, p.dc_wb, p.retired);
3597
3598 p.stall_for(3, Interlock::Dcm);
3599 for _ in 0..3 {
3600 p.advance(&mut bus, &mut regs, &mut pc);
3601 assert_eq!(
3602 (p.ic_rf, p.rf_ex, p.ex_dc, p.dc_wb, p.retired),
3603 before,
3604 "a stalled cycle must not advance any latch"
3605 );
3606 }
3607 assert!(p.stalled_by().is_none());
3608 p.advance(&mut bus, &mut regs, &mut pc);
3609 assert_ne!(
3610 p.dc_wb, before.3,
3611 "the pipeline must resume after the stall"
3612 );
3613 }
3614
3615 /// **The interrupt gate** (UM §4.7.1): an interrupt is accepted only if the
3616 /// *previous* `PCycle` was a run cycle. The cycle right after a stall is not.
3617 #[test]
3618 fn interrupt_is_not_accepted_on_the_cycle_after_a_stall() {
3619 /// `IE` set, `IM2` (the RCP line) unmasked, `EXL`/`ERL` clear.
3620 ///
3621 /// Needed since T-12-003: an asserted line is no longer sufficient on
3622 /// its own, and cold reset leaves `ERL` **set**, which alone blocks
3623 /// every interrupt.
3624 const IRQ_READY: u64 = 1 | (1 << 10);
3625
3626 let mut p = Pipeline::new();
3627 p.cop0.set_hardware(crate::cop0::reg::STATUS, IRQ_READY);
3628 let mut pc = 0xFFFF_FFFF_8000_0000;
3629 let mut regs = Regs::new();
3630 let mut bus = NullBus { irq: true };
3631
3632 // Warm up so `prev_was_run` is true AND a real instruction has reached
3633 // DC, then confirm an IRQ IS taken.
3634 //
3635 // This used to advance exactly twice, which accepted the interrupt while
3636 // `ex_dc` was still an unoccupied fill bubble — so `EPC` was charged to
3637 // address 0. The test never noticed because it asserted only the abort
3638 // *flag* and never `EPC`: an assertion the broken machine satisfied just
3639 // as well as a correct one. That blind spot is half of ledger **R-18**
3640 // (see `an_interrupt_across_an_eret_is_charged_to_the_target_not_the_eret`),
3641 // so the loop now waits for a real instruction and `EPC` is asserted too.
3642 p.advance(&mut bus, &mut regs, &mut pc);
3643 assert!(p.prev_cycle_was_run());
3644 let mut accepted = false;
3645 for _ in 0..8 {
3646 p.advance(&mut bus, &mut regs, &mut pc);
3647 if p.ex_dc.abort == Some(Exception::Interrupt) {
3648 accepted = true;
3649 break;
3650 }
3651 }
3652 assert!(
3653 accepted,
3654 "an interrupt should be accepted after a run cycle once a real \
3655 instruction reaches DC"
3656 );
3657 assert_ne!(
3658 p.cop0.read(crate::cop0::reg::EPC),
3659 0,
3660 "EPC must name the interrupted instruction, not a fill bubble"
3661 );
3662
3663 // Now stall, and check the very next cycle refuses it.
3664 let mut p = Pipeline::new();
3665 p.cop0.set_hardware(crate::cop0::reg::STATUS, IRQ_READY);
3666 let mut pc = 0xFFFF_FFFF_8000_0000;
3667 let mut regs = Regs::new();
3668 p.advance(&mut bus, &mut regs, &mut pc);
3669 p.stall_for(1, Interlock::Dcb);
3670 p.advance(&mut bus, &mut regs, &mut pc); // the stalled cycle
3671 assert!(
3672 !p.prev_cycle_was_run(),
3673 "a stalled cycle must not count as a run cycle"
3674 );
3675 let ex_dc_before = p.ex_dc.abort;
3676 p.advance(&mut bus, &mut regs, &mut pc); // the cycle immediately after
3677 assert_eq!(
3678 p.ex_dc.abort, ex_dc_before,
3679 "an interrupt must NOT be accepted when the previous PCycle stalled"
3680 );
3681 }
3682
3683 /// **An abort must survive the cascade**, not merely be present the instant
3684 /// it is stamped.
3685 ///
3686 /// The shallow version of this test asserted latch state immediately after
3687 /// `abort_from` and never advanced — so it could not tell a real flush from
3688 /// one that gets overwritten by the reverse cascade in the same cycle. This
3689 /// version steps the pipeline and follows the consequences.
3690 #[test]
3691 fn an_abort_survives_the_cascade() {
3692 let mut p = Pipeline::new();
3693 let mut pc = 0xFFFF_FFFF_8000_0000;
3694 let mut regs = Regs::new();
3695 let mut bus = quiet();
3696 for _ in 0..4 {
3697 p.advance(&mut bus, &mut regs, &mut pc);
3698 }
3699
3700 // Abort at DC: the instruction in DC plus everything younger.
3701 p.abort_from(Stage::Dc, Exception::AddressError { store: false });
3702 let aborted_pc = p.ex_dc.pc;
3703 p.advance(&mut bus, &mut regs, &mut pc);
3704
3705 // The causing instruction carried its abort forward into WB's latch...
3706 assert_eq!(
3707 (p.dc_wb.abort, p.dc_wb.pc),
3708 (Some(Exception::AddressError { store: false }), aborted_pc),
3709 "the aborting instruction lost its flag while advancing"
3710 );
3711 // ...and the younger ones kept theirs rather than having them
3712 // overwritten by the latch moves.
3713 assert_eq!(
3714 p.ex_dc.abort,
3715 Some(Exception::AddressError { store: false }),
3716 "a younger instruction's abort was overwritten by the cascade"
3717 );
3718
3719 // The younger instruction that was already in flight kept its abort as it
3720 // advanced, rather than having it overwritten by the latch move.
3721 assert_eq!(
3722 p.rf_ex.abort,
3723 Some(Exception::AddressError { store: false }),
3724 "an in-flight younger instruction lost its abort while advancing"
3725 );
3726
3727 // And the fetch issued during the aborting cycle is a bubble, not a live
3728 // wrong-path instruction that would escape the flush entirely.
3729 assert!(
3730 !p.ic_rf.occupied,
3731 "the instruction fetched during the abort escaped the flush"
3732 );
3733 }
3734
3735 /// A zero-cycle stall request is ignored — recording it would burn a cycle
3736 /// and suppress interrupt acceptance on the next one, with no visible cause.
3737 #[test]
3738 fn a_zero_cycle_stall_is_not_a_stall() {
3739 let mut p = Pipeline::new();
3740 let mut pc = 0xFFFF_FFFF_8000_0000;
3741 let mut regs = Regs::new();
3742 let mut bus = quiet();
3743 p.advance(&mut bus, &mut regs, &mut pc);
3744
3745 p.stall_for(0, Interlock::Ldi);
3746 assert!(p.stalled_by().is_none(), "a 0-cycle request is not a stall");
3747
3748 let before = p.ic_rf.pc;
3749 p.advance(&mut bus, &mut regs, &mut pc);
3750 assert_ne!(before, p.ic_rf.pc, "the cycle was silently consumed");
3751 assert!(
3752 p.prev_cycle_was_run(),
3753 "a non-stall must still count as a run cycle, or the interrupt gate \
3754 is wrongly suppressed on the following cycle"
3755 );
3756 }
3757
3758 /// An abort kills its own stage and everything younger, never anything older.
3759 #[test]
3760 fn abort_kills_younger_instructions_only() {
3761 let mut p = Pipeline::new();
3762 let mut pc = 0xFFFF_FFFF_8000_0000;
3763 let mut regs = Regs::new();
3764 let mut bus = quiet();
3765 for _ in 0..4 {
3766 p.advance(&mut bus, &mut regs, &mut pc);
3767 }
3768 p.abort_from(Stage::Ex, Exception::Overflow);
3769 assert_eq!(p.rf_ex.abort, Some(Exception::Overflow), "EX's own latch");
3770 assert_eq!(p.ic_rf.abort, Some(Exception::Overflow), "younger: killed");
3771 assert_eq!(p.ex_dc.abort, None, "older instruction must survive");
3772 assert_eq!(p.dc_wb.abort, None, "older instruction must survive");
3773 }
3774
3775 /// An aborted instruction must not retire.
3776 #[test]
3777 fn an_aborted_instruction_does_not_retire() {
3778 let mut p = Pipeline::new();
3779 let mut pc = 0xFFFF_FFFF_8000_0000;
3780 let mut regs = Regs::new();
3781 let mut bus = quiet();
3782 for _ in 0..4 {
3783 p.advance(&mut bus, &mut regs, &mut pc);
3784 }
3785 p.dc_wb.abort = Some(Exception::AddressError { store: false });
3786 let retired = p.retired;
3787 p.advance(&mut bus, &mut regs, &mut pc);
3788 assert_eq!(p.retired, retired, "an aborted instruction retired anyway");
3789 }
3790
3791 /// **End to end**: a real program, fetched from a bus, decoded, executed and
3792 /// committed to the register file through all five stages.
3793 ///
3794 /// Until this passed, every other test in the crate exercised a piece in
3795 /// isolation. This is the one that says the CPU *runs*.
3796 #[test]
3797 fn a_program_executes_through_the_whole_pipeline() {
3798 /// A bus holding a program at 0, returning `NOP` past its end.
3799 struct Rom(alloc::vec::Vec<u32>);
3800 impl Bus for Rom {
3801 fn read_u8(&mut self, _addr: u32) -> u8 {
3802 0
3803 }
3804 fn write_u8(&mut self, _addr: u32, _val: u8) {}
3805 fn read_u32(&mut self, addr: u32) -> u32 {
3806 self.0.get((addr / 4) as usize).copied().unwrap_or(0)
3807 }
3808 }
3809 const fn r(funct: u32, rs: u32, rt: u32, rd: u32, sa: u32) -> u32 {
3810 (rs << 21) | (rt << 16) | (rd << 11) | (sa << 6) | funct
3811 }
3812 const fn i(opcode: u32, rs: u32, rt: u32, imm: u16) -> u32 {
3813 (opcode << 26) | (rs << 21) | (rt << 16) | imm as u32
3814 }
3815
3816 // ADDIU $1, $0, 6 ; $1 = 6
3817 // ADDIU $2, $0, 7 ; $2 = 7
3818 // MULT $1, $2 ; HI:LO = 42 (stalls 5 PCycles)
3819 // MFLO $3 ; $3 = 42
3820 // ADDU $4, $1, $2 ; $4 = 13
3821 // SLL $5, $2, 2 ; $5 = 28
3822 let program = alloc::vec![
3823 i(0o11, 0, 1, 6),
3824 i(0o11, 0, 2, 7),
3825 r(0o30, 1, 2, 0, 0),
3826 r(0o22, 0, 0, 3, 0),
3827 r(0o41, 1, 2, 4, 0),
3828 r(0o00, 0, 2, 5, 2),
3829 ];
3830 let mut bus = Rom(program);
3831 let mut regs = Regs::new();
3832 let mut p = Pipeline::new();
3833 let mut pc = 0xFFFF_FFFF_8000_0000u64;
3834
3835 // Generous budget: 6 instructions, 5 stages deep, plus the MULT stall.
3836 for _ in 0..64 {
3837 p.advance(&mut bus, &mut regs, &mut pc);
3838 }
3839
3840 assert_eq!(regs.read(1), 6, "ADDIU $1");
3841 assert_eq!(regs.read(2), 7, "ADDIU $2");
3842 assert_eq!(regs.lo, 42, "MULT wrote LO");
3843 assert_eq!(regs.hi, 0, "MULT wrote HI");
3844 assert_eq!(regs.read(3), 42, "MFLO read LO into $3");
3845 assert_eq!(regs.read(4), 13, "ADDU $1 + $2");
3846 assert_eq!(regs.read(5), 28, "SLL $2 << 2");
3847 assert_eq!(regs.read(0), 0, "$zero stayed zero");
3848 assert!(p.retired >= 6, "all six instructions retired");
3849 }
3850
3851 /// `$zero` must survive an instruction that nominally targets it — software
3852 /// depends on that, and `ADDIU $0, $0, 5` is a legal encoding.
3853 #[test]
3854 fn writes_to_zero_are_discarded_by_the_pipeline() {
3855 struct Ones;
3856 impl Bus for Ones {
3857 fn read_u8(&mut self, _addr: u32) -> u8 {
3858 0
3859 }
3860 fn write_u8(&mut self, _addr: u32, _val: u8) {}
3861 fn read_u32(&mut self, _addr: u32) -> u32 {
3862 // ADDIU $0, $0, 5 -- targets $zero, forever.
3863 (0o11 << 26) | 5
3864 }
3865 }
3866 let mut bus = Ones;
3867 let mut regs = Regs::new();
3868 let mut p = Pipeline::new();
3869 let mut pc = 0xFFFF_FFFF_8000_0000u64;
3870 for _ in 0..32 {
3871 p.advance(&mut bus, &mut regs, &mut pc);
3872 }
3873 assert_eq!(regs.read(0), 0, "$zero was written");
3874 assert_eq!(regs.gpr[0], 0, "$zero was written through the raw array");
3875 }
3876
3877 /// An overflowing `ADD` must abort rather than commit, and the destination
3878 /// register must be left untouched.
3879 #[test]
3880 fn an_overflow_trap_prevents_the_write_back() {
3881 struct Overflowing;
3882 impl Bus for Overflowing {
3883 fn read_u8(&mut self, _addr: u32) -> u8 {
3884 0
3885 }
3886 fn write_u8(&mut self, _addr: u32, _val: u8) {}
3887 fn read_u32(&mut self, addr: u32) -> u32 {
3888 match addr / 4 {
3889 // LUI $1, 0x7FFF ; ORI $1, $1, 0xFFFF => $1 = i32::MAX
3890 0 => (0o17 << 26) | (1 << 16) | 0x7FFF,
3891 1 => (0o15 << 26) | (1 << 21) | (1 << 16) | 0xFFFF,
3892 // ADDIU $2, $0, 1
3893 2 => (0o11 << 26) | (2 << 16) | 1,
3894 // ADD $3, $1, $2 -> overflows
3895 3 => (1 << 21) | (2 << 16) | (3 << 11) | 0o40,
3896 _ => 0,
3897 }
3898 }
3899 }
3900 let mut bus = Overflowing;
3901 let mut regs = Regs::new();
3902 let mut p = Pipeline::new();
3903 let mut pc = 0xFFFF_FFFF_8000_0000u64;
3904 for _ in 0..48 {
3905 p.advance(&mut bus, &mut regs, &mut pc);
3906 }
3907 assert_eq!(regs.read(1), 0x7FFF_FFFF, "LUI+ORI built i32::MAX");
3908 assert_eq!(regs.read(3), 0, "the overflowing ADD must not commit");
3909 }
3910
3911 /// An unaligned instruction fetch raises an address error instead of
3912 /// fetching, and must not silently realign the PC — that would convert a
3913 /// raised exception into a wrong answer on a path hardware never takes.
3914 #[test]
3915 fn an_unaligned_fetch_raises_address_error_without_realigning() {
3916 struct Watch {
3917 fetched: bool,
3918 }
3919 impl Bus for Watch {
3920 fn read_u8(&mut self, _addr: u32) -> u8 {
3921 0
3922 }
3923 fn write_u8(&mut self, _addr: u32, _val: u8) {}
3924 fn read_u32(&mut self, _addr: u32) -> u32 {
3925 self.fetched = true;
3926 0
3927 }
3928 }
3929 let mut bus = Watch { fetched: false };
3930 let mut regs = Regs::new();
3931 let mut p = Pipeline::new();
3932 let mut pc = 0xFFFF_FFFF_8000_0002; // deliberately unaligned
3933
3934 p.advance(&mut bus, &mut regs, &mut pc);
3935
3936 assert_eq!(
3937 p.ic_rf.abort,
3938 Some(Exception::AddressError { store: false }),
3939 "an unaligned fetch must raise AdEL"
3940 );
3941 assert!(
3942 !bus.fetched,
3943 "the bus must not be accessed for a bad address"
3944 );
3945 // The PC is not silently realigned -- it is *vectored*, which is the
3946 // architectural response. Before T-12-002 nothing dispatched and this
3947 // asserted the PC stayed at the bad address; that was the absence of
3948 // dispatch, not a rule.
3949 assert_eq!(
3950 pc, 0xFFFF_FFFF_BFC0_0380,
3951 "the BEV=1 general vector -- cold reset leaves BEV set (UM §6.4.4), \
3952 so a fresh CPU vectors into the boot ROM, not into RDRAM"
3953 );
3954 assert_eq!(
3955 p.cop0.read(crate::cop0::reg::BAD_VADDR),
3956 0xFFFF_FFFF_8000_0002,
3957 "and BadVAddr holds the unaligned address, un-realigned"
3958 );
3959 assert_eq!(
3960 (p.cop0.read(crate::cop0::reg::CAUSE) >> 2) & 0x1F,
3961 crate::exception::exc_code::ADEL,
3962 "an instruction fetch is a load: AdEL, not AdES"
3963 );
3964 // EPC must name the FAULTING fetch. `abort_with` captures its context
3965 // out of `ic_rf`, so raising before populating that latch would silently
3966 // record the previous fetch's PC here -- which is exactly what this code
3967 // did until review caught it.
3968 assert_eq!(
3969 p.cop0.read(crate::cop0::reg::EPC),
3970 0xFFFF_FFFF_8000_0002,
3971 "EPC names the faulting fetch, not the previous one"
3972 );
3973
3974 // And the faulting instruction must never retire. Bounded to the
3975 // epilogue stall on purpose: past that the CPU is running the *handler*,
3976 // whose instructions retire legitimately, so a longer window would be
3977 // asserting that exception handling does not work.
3978 assert_eq!(p.retired, 0, "the faulting fetch retired");
3979 assert_eq!(
3980 p.stalled_by(),
3981 Some(Interlock::Exception),
3982 "the 2-PCycle epilogue stall is in progress (UM §4.7 p.114)"
3983 );
3984 for _ in 0..crate::exception::EPILOGUE_STALL {
3985 p.advance(&mut bus, &mut regs, &mut pc);
3986 assert_eq!(p.retired, 0, "nothing retires while the pipe drains");
3987 }
3988 assert_eq!(p.stalled_by(), None, "the stall was exactly 2 PCycles");
3989 }
3990
3991 /// A RAM-backed bus, so loads and stores can be exercised end to end.
3992 struct Ram {
3993 prog: alloc::vec::Vec<u32>,
3994 data: alloc::vec::Vec<u8>,
3995 }
3996 impl Ram {
3997 fn new(prog: alloc::vec::Vec<u32>) -> Self {
3998 Self {
3999 prog,
4000 data: alloc::vec![0; 0x1000],
4001 }
4002 }
4003 }
4004 impl Bus for Ram {
4005 fn read_u8(&mut self, addr: u32) -> u8 {
4006 self.data.get(addr as usize).copied().unwrap_or(0)
4007 }
4008 fn write_u8(&mut self, addr: u32, val: u8) {
4009 if let Some(b) = self.data.get_mut(addr as usize) {
4010 *b = val;
4011 }
4012 }
4013 fn read_u32(&mut self, addr: u32) -> u32 {
4014 // Instructions live above 0x800; data below it.
4015 if addr >= 0x800 {
4016 self.prog
4017 .get(((addr - 0x800) / 4) as usize)
4018 .copied()
4019 .unwrap_or(0)
4020 } else {
4021 u32::from_be_bytes([
4022 self.read_u8(addr),
4023 self.read_u8(addr + 1),
4024 self.read_u8(addr + 2),
4025 self.read_u8(addr + 3),
4026 ])
4027 }
4028 }
4029 }
4030 const fn ld_st(opcode: u32, base: u32, rt: u32, off: u16) -> u32 {
4031 (opcode << 26) | (base << 21) | (rt << 16) | off as u32
4032 }
4033
4034 /// A store followed by a load round-trips through real memory, at every
4035 /// width, with the sign/zero-extension rules applied.
4036 #[test]
4037 fn stores_and_loads_round_trip_through_memory() {
4038 // LUI $1, 0x8000 ; KSEG0 base
4039 // ADDIU $2, $0, -2 ; value = 0xFFFF_FFFF_FFFF_FFFE
4040 // SW $2, 0x100($1)
4041 // LW $3, 0x100($1) ; sign-extended -> 0xFFFF_FFFF_FFFF_FFFE
4042 // LWU $4, 0x100($1) ; zero-extended -> 0x0000_0000_FFFF_FFFE
4043 // LBU $5, 0x100($1) ; big-endian MSB -> 0xFF
4044 let prog = alloc::vec![
4045 lui_kseg0(1),
4046 (0o11 << 26) | (2 << 16) | 0xFFFE,
4047 ld_st(0o53, 1, 2, 0x100),
4048 ld_st(0o43, 1, 3, 0x100),
4049 ld_st(0o47, 1, 4, 0x100),
4050 ld_st(0o44, 1, 5, 0x100),
4051 ];
4052 let mut bus = Ram::new(prog);
4053 let mut regs = Regs::new();
4054 let mut p = Pipeline::new();
4055 let mut pc = KSEG0_PROG;
4056 for _ in 0..80 {
4057 p.advance(&mut bus, &mut regs, &mut pc);
4058 }
4059 assert_eq!(regs.read(3), 0xFFFF_FFFF_FFFF_FFFE, "LW sign-extends");
4060 assert_eq!(regs.read(4), 0x0000_0000_FFFF_FFFE, "LWU zero-extends");
4061 assert_eq!(regs.read(5), 0xFF, "LBU reads the big-endian MSB");
4062 }
4063
4064 /// An unaligned `LW` raises an address error and does not commit — the whole
4065 /// reason the `LWL`/`LWR` family exists.
4066 #[test]
4067 fn an_unaligned_load_raises_address_error() {
4068 // ADDIU $1, $0, 0x101 ; deliberately unaligned
4069 // LW $2, 0($1)
4070 let prog = alloc::vec![(0o11 << 26) | (1 << 16) | 0x101, ld_st(0o43, 1, 2, 0)];
4071 let mut bus = Ram::new(prog);
4072 let mut regs = Regs::new();
4073 let mut p = Pipeline::new();
4074 let mut pc = KSEG0_PROG;
4075 for _ in 0..40 {
4076 p.advance(&mut bus, &mut regs, &mut pc);
4077 }
4078 assert_eq!(regs.read(2), 0, "an unaligned LW must not commit");
4079 }
4080
4081 /// **The load-delay interlock** (UM §4.6.5) — it finally has something to
4082 /// interlock against. A load followed by an instruction naming the loaded
4083 /// register stalls one cycle, and the dependent instruction must still see
4084 /// the loaded value.
4085 #[test]
4086 fn a_load_followed_by_its_use_interlocks_and_still_reads_the_value() {
4087 // LUI $1, 0x8000 ; KSEG0 base -- unmapped, so no TLB entry needed
4088 // ADDIU $2, $0, 0x55
4089 // SW $2, 0x100($1)
4090 // LW $3, 0x100($1) ; load
4091 // ADDIU $4, $3, 1 ; uses $3 immediately -> LDI stall
4092 let prog = alloc::vec![
4093 lui_kseg0(1),
4094 (0o11 << 26) | (2 << 16) | 0x55,
4095 ld_st(0o53, 1, 2, 0x100),
4096 ld_st(0o43, 1, 3, 0x100),
4097 (0o11 << 26) | (3 << 21) | (4 << 16) | 1,
4098 ];
4099 let mut bus = Ram::new(prog);
4100 let mut regs = Regs::new();
4101 let mut p = Pipeline::new();
4102 let mut pc = KSEG0_PROG;
4103 let mut saw_ldi = false;
4104 for _ in 0..80 {
4105 p.advance(&mut bus, &mut regs, &mut pc);
4106 if p.stalled_by() == Some(Interlock::Ldi) {
4107 saw_ldi = true;
4108 }
4109 }
4110 assert!(saw_ldi, "the load-delay interlock never fired");
4111 assert_eq!(regs.read(3), 0x55, "LW loaded the stored value");
4112 assert_eq!(regs.read(4), 0x56, "the dependent instruction saw it");
4113 }
4114
4115 /// The `LWL`/`LWR` pair assembles an unaligned word from memory — the
4116 /// end-to-end version of the unit tests in `mem`.
4117 #[test]
4118 fn the_unaligned_pair_assembles_a_word_from_real_memory() {
4119 // ADDIU $1, $0, 0x100
4120 // LWL $3, 1($1)
4121 // LWR $3, 4($1)
4122 let prog = alloc::vec![
4123 lui_kseg0(1),
4124 ld_st(0o42, 1, 3, 0x101),
4125 ld_st(0o46, 1, 3, 0x104),
4126 ];
4127 let mut bus = Ram::new(prog);
4128 // Memory at 0x100: 00 11 22 33 44 55 66 77
4129 for (k, b) in [0x00u8, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77]
4130 .into_iter()
4131 .enumerate()
4132 {
4133 bus.write_u8(0x100 + k as u32, b);
4134 }
4135 let mut regs = Regs::new();
4136 let mut p = Pipeline::new();
4137 let mut pc = KSEG0_PROG;
4138 for _ in 0..60 {
4139 p.advance(&mut bus, &mut regs, &mut pc);
4140 }
4141 assert_eq!(
4142 regs.read(3),
4143 crate::alu::sext32(0x1122_3344),
4144 "LWL+LWR must assemble the unaligned word at 0x101"
4145 );
4146 }
4147
4148 /// **The branch delay slot executes before the target.** This is the single
4149 /// most load-bearing property of MIPS control flow: the instruction *after* a
4150 /// branch runs whether or not the branch is taken.
4151 #[test]
4152 fn the_delay_slot_executes_before_the_branch_target() {
4153 // 0x800: ADDIU $1, $0, 1
4154 // 0x804: BEQ $0, $0, +2 ; taken, to 0x810
4155 // 0x808: ADDIU $2, $0, 2 ; DELAY SLOT -- must execute
4156 // 0x80C: ADDIU $3, $0, 3 ; skipped
4157 // 0x810: ADDIU $4, $0, 4 ; target
4158 let prog = alloc::vec![
4159 (0o11 << 26) | (1 << 16) | 1,
4160 (0o04 << 26) | 2,
4161 (0o11 << 26) | (2 << 16) | 2,
4162 (0o11 << 26) | (3 << 16) | 3,
4163 (0o11 << 26) | (4 << 16) | 4,
4164 ];
4165 let mut bus = Ram::new(prog);
4166 let mut regs = Regs::new();
4167 let mut p = Pipeline::new();
4168 let mut pc = KSEG0_PROG;
4169 for _ in 0..60 {
4170 p.advance(&mut bus, &mut regs, &mut pc);
4171 }
4172 assert_eq!(regs.read(1), 1, "before the branch");
4173 assert_eq!(regs.read(2), 2, "the DELAY SLOT must execute");
4174 assert_eq!(regs.read(3), 0, "the instruction after the slot is skipped");
4175 assert_eq!(regs.read(4), 4, "the target must execute");
4176 }
4177
4178 /// **Branch-likely nullifies its delay slot when NOT taken**; an ordinary
4179 /// branch does not. Confusing the two silently runs or skips one instruction
4180 /// per untaken branch, which is invisible until a loop's trip count is wrong.
4181 #[test]
4182 fn branch_likely_nullifies_its_delay_slot_but_an_ordinary_branch_does_not() {
4183 // BNEL $0, $0, +1 ; NOT taken (0 == 0), so the slot is nullified
4184 // ADDIU $2, $0, 2 ; DELAY SLOT -- must be squashed
4185 // ADDIU $3, $0, 3
4186 let likely = alloc::vec![
4187 (0o25 << 26) | 1,
4188 (0o11 << 26) | (2 << 16) | 2,
4189 (0o11 << 26) | (3 << 16) | 3,
4190 ];
4191 let mut bus = Ram::new(likely);
4192 let mut regs = Regs::new();
4193 let mut p = Pipeline::new();
4194 let mut pc = KSEG0_PROG;
4195 for _ in 0..50 {
4196 p.advance(&mut bus, &mut regs, &mut pc);
4197 }
4198 assert_eq!(
4199 regs.read(2),
4200 0,
4201 "BNEL not taken must NULLIFY its delay slot"
4202 );
4203 assert_eq!(regs.read(3), 3, "execution continues after the slot");
4204
4205 // The same shape with the ordinary BNE: the slot DOES execute.
4206 let ordinary = alloc::vec![
4207 (0o05 << 26) | 1,
4208 (0o11 << 26) | (2 << 16) | 2,
4209 (0o11 << 26) | (3 << 16) | 3,
4210 ];
4211 let mut bus = Ram::new(ordinary);
4212 let mut regs = Regs::new();
4213 let mut p = Pipeline::new();
4214 let mut pc = KSEG0_PROG;
4215 for _ in 0..50 {
4216 p.advance(&mut bus, &mut regs, &mut pc);
4217 }
4218 assert_eq!(
4219 regs.read(2),
4220 2,
4221 "BNE not taken must still RUN its delay slot"
4222 );
4223 }
4224
4225 /// `JAL` links the address *after* the delay slot, so a returning `JR $31`
4226 /// resumes past it rather than re-executing it.
4227 #[test]
4228 fn jal_links_past_the_delay_slot_and_jr_returns_there() {
4229 // 0x800: JAL 0x204 ; -> 0x810, links $31 = 0x808
4230 // 0x804: ADDIU $1, $0, 1 ; DELAY SLOT
4231 // 0x808: ADDIU $2, $0, 2 ; where JR $31 must return to
4232 // 0x80C: ADDIU $9, $0, 9 ; must NOT run before the return
4233 // 0x810: JR $31
4234 // 0x814: ADDIU $3, $0, 3 ; the callee's DELAY SLOT
4235 let prog = alloc::vec![
4236 (0o03 << 26) | 0x204,
4237 (0o11 << 26) | (1 << 16) | 1,
4238 (0o11 << 26) | (2 << 16) | 2,
4239 (0o11 << 26) | (9 << 16) | 9,
4240 (31 << 21) | 0o10,
4241 (0o11 << 26) | (3 << 16) | 3,
4242 ];
4243 let mut bus = Ram::new(prog);
4244 let mut regs = Regs::new();
4245 let mut p = Pipeline::new();
4246 let mut pc = KSEG0_PROG;
4247 for _ in 0..80 {
4248 p.advance(&mut bus, &mut regs, &mut pc);
4249 }
4250 assert_eq!(
4251 regs.read(31),
4252 KSEG0_PROG + 8,
4253 "JAL links PC+8, past the delay slot"
4254 );
4255 assert_eq!(regs.read(1), 1, "JAL's delay slot ran");
4256 assert_eq!(regs.read(3), 3, "JR's delay slot ran");
4257 assert_eq!(regs.read(2), 2, "JR $31 returned to the linked address");
4258 }
4259
4260 /// A trap whose condition holds raises an exception and does not commit.
4261 #[test]
4262 fn a_taken_trap_raises_and_an_untaken_one_does_not() {
4263 // ADDIU $1, $0, 5
4264 // TEQ $1, $1 ; equal -> traps
4265 // ADDIU $2, $0, 2 ; must not commit
4266 let prog = alloc::vec![
4267 (0o11 << 26) | (1 << 16) | 5,
4268 (1 << 21) | (1 << 16) | 0o64,
4269 (0o11 << 26) | (2 << 16) | 2,
4270 ];
4271 let mut bus = Ram::new(prog);
4272 let mut regs = Regs::new();
4273 let mut p = Pipeline::new();
4274 let mut pc = KSEG0_PROG;
4275 let mut trapped = false;
4276 for _ in 0..50 {
4277 p.advance(&mut bus, &mut regs, &mut pc);
4278 if p.ex_dc.abort == Some(Exception::Trap) || p.dc_wb.abort == Some(Exception::Trap) {
4279 trapped = true;
4280 }
4281 }
4282 assert!(trapped, "TEQ with equal operands must trap");
4283
4284 // TNE with equal operands does NOT trap.
4285 let prog = alloc::vec![
4286 (0o11 << 26) | (1 << 16) | 5,
4287 (1 << 21) | (1 << 16) | 0o66,
4288 (0o11 << 26) | (2 << 16) | 2,
4289 ];
4290 let mut bus = Ram::new(prog);
4291 let mut regs = Regs::new();
4292 let mut p = Pipeline::new();
4293 let mut pc = KSEG0_PROG;
4294 for _ in 0..50 {
4295 p.advance(&mut bus, &mut regs, &mut pc);
4296 }
4297 assert_eq!(
4298 regs.read(2),
4299 2,
4300 "an untaken trap must not disturb execution"
4301 );
4302 }
4303
4304 /// `SYSCALL` and `BREAK` raise their own exceptions.
4305 #[test]
4306 fn syscall_and_break_raise_their_exceptions() {
4307 for (funct, want) in [(0o14u32, Exception::Syscall), (0o15, Exception::Breakpoint)] {
4308 let mut bus = Ram::new(alloc::vec![funct]);
4309 let mut regs = Regs::new();
4310 let mut p = Pipeline::new();
4311 let mut pc = KSEG0_PROG;
4312 let mut seen = false;
4313 for _ in 0..40 {
4314 p.advance(&mut bus, &mut regs, &mut pc);
4315 if p.ex_dc.abort == Some(want) || p.dc_wb.abort == Some(want) {
4316 seen = true;
4317 }
4318 }
4319 assert!(seen, "funct {funct:o} should raise {want:?}");
4320 }
4321 }
4322
4323 /// **`in_delay_slot` must actually be set**, and only on the instruction
4324 /// after a branch or jump.
4325 ///
4326 /// This test exists because mutation-testing found the flag was **not yet
4327 /// load-bearing**: forcing it to `false` broke nothing, since its only
4328 /// consumer is `Cause.BD`/`EPC` at exception time and COP0 arrives in
4329 /// Sprint 2. A field that is written and never read is the exact pattern
4330 /// this crate has twice deleted (`poll_irq_at_phase`, `Stall.resume`).
4331 ///
4332 /// Rather than delete it — it is genuinely needed, and it must ride in the
4333 /// latch rather than be recomputed later — it is pinned here so it is
4334 /// verified from the moment it is written.
4335 #[test]
4336 fn in_delay_slot_is_set_on_exactly_the_instruction_after_a_branch() {
4337 // 0x800: ADDIU $1, $0, 1 ; not a delay slot
4338 // 0x804: BEQ $0, $0, +2 ; a branch, not itself a delay slot
4339 // 0x808: ADDIU $2, $0, 2 ; IS the delay slot
4340 // 0x810: ADDIU $4, $0, 4 ; target, not a delay slot
4341 let prog = alloc::vec![
4342 (0o11 << 26) | (1 << 16) | 1,
4343 (0o04 << 26) | 2,
4344 (0o11 << 26) | (2 << 16) | 2,
4345 (0o11 << 26) | (3 << 16) | 3,
4346 (0o11 << 26) | (4 << 16) | 4,
4347 ];
4348 let mut bus = Ram::new(prog);
4349 let mut regs = Regs::new();
4350 let mut p = Pipeline::new();
4351 let mut pc = KSEG0_PROG;
4352
4353 let mut flagged = alloc::vec::Vec::new();
4354 for _ in 0..8 {
4355 p.advance(&mut bus, &mut regs, &mut pc);
4356 if p.ic_rf.occupied && p.ic_rf.in_delay_slot {
4357 flagged.push(p.ic_rf.pc);
4358 }
4359 }
4360 assert_eq!(
4361 flagged,
4362 alloc::vec![KSEG0_PROG + 8],
4363 "exactly one instruction -- the one after the branch -- must be \
4364 flagged as a delay slot"
4365 );
4366 }
4367
4368 /// The load interlock reproduces the hardware's documented imprecision.
4369 /// Emulating *precise* behavior here is the bug.
4370 #[test]
4371 fn load_interlock_is_imprecise_exactly_as_hardware_is() {
4372 // Matches on rt: the ordinary true positive.
4373 assert!(load_interlocks(8, 0, 8, true));
4374 // Matches on rs.
4375 assert!(load_interlocks(8, 8, 0, true));
4376 // False positive hardware really has: LUI's unused rs field matching.
4377 assert!(
4378 load_interlocks(8, 8, 9, true),
4379 "hardware stalls even when the field is not used as a source"
4380 );
4381 // Two consecutive loads into the same register also stall.
4382 assert!(load_interlocks(8, 0, 8, true));
4383 // $zero is exempt -- a load into it can never be depended on.
4384 assert!(!load_interlocks(0, 0, 0, true));
4385 // GPR loads do not interlock with float instructions, or vice versa.
4386 assert!(!load_interlocks(8, 8, 8, false));
4387 // No overlap at all.
4388 assert!(!load_interlocks(8, 9, 10, true));
4389 }
4390
4391 // --- LL / SC (UM §16 pp. 453, 487; §3.1; §5.4.7) ------------------------
4392
4393 use crate::mem::{LoadKind, StoreKind};
4394
4395 /// Where [`Ram`] keeps test programs, addressed through **KSEG0**.
4396 ///
4397 /// KSEG0 is unmapped, so it reaches physical `0x800` without a TLB entry —
4398 /// which is how real code runs and, since T-12-004, the only way a test can
4399 /// fetch at all without installing a mapping. Fetching from a bare `0x800`
4400 /// is a KUSEG address and now correctly raises a TLB refill.
4401 const KSEG0_PROG: u64 = 0xFFFF_FFFF_8000_0800;
4402
4403 /// A bus that fetches `NOP`s and holds its interrupt line asserted.
4404 ///
4405 /// Reads return 0, which decodes to `SLL $0, $0, 0` — the canonical `NOP` —
4406 /// so the pipeline runs without any instruction interfering with what these
4407 /// tests observe.
4408 struct AlwaysIrq;
4409 impl Bus for AlwaysIrq {
4410 fn read_u8(&mut self, _a: u32) -> u8 {
4411 0
4412 }
4413 fn write_u8(&mut self, _a: u32, _v: u8) {}
4414 fn read_u32(&mut self, _a: u32) -> u32 {
4415 0
4416 }
4417 fn poll_irq(&mut self) -> bool {
4418 true
4419 }
4420 }
4421
4422 /// The synchronization tests need only the data half of [`Ram`].
4423 fn ram() -> Ram {
4424 Ram::new(alloc::vec![])
4425 }
4426
4427 /// Push the dirty D-cache line covering `vaddr` out to the bus.
4428 ///
4429 /// A store to a cached segment now lands in the write-back D-cache and RAM
4430 /// does not see it until something forces it out. That is the whole point of
4431 /// a write-back cache, and it is what real software does before handing a
4432 /// buffer to DMA -- so a test that asserts on RAM without this would be
4433 /// asserting the cache does not exist.
4434 fn writeback<B: Bus>(p: &mut Pipeline, bus: &mut B, vaddr: u64) {
4435 p.access(
4436 bus,
4437 MemOp::Cache {
4438 addr: vaddr,
4439 op: 25,
4440 },
4441 )
4442 .expect("Hit_WriteBack on a mapped address cannot fault");
4443 }
4444
4445 /// Without a preceding `LL` the store must not happen, and `rt` must still
4446 /// be written — with 0. A `Store`-shaped implementation writes memory
4447 /// unconditionally; a `Load`-shaped one never writes `rt` on failure.
4448 #[test]
4449 fn sc_without_ll_stores_nothing_and_reports_failure() {
4450 let mut p = Pipeline::new();
4451 let mut bus = ram();
4452 let wb = p
4453 .access(
4454 &mut bus,
4455 MemOp::ConditionalStore {
4456 kind: StoreKind::Word,
4457 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4458 value: 0xDEAD_BEEF,
4459 dest: 9,
4460 },
4461 )
4462 .expect("aligned");
4463 assert_eq!(
4464 wb,
4465 WriteBack::Gpr { dest: 9, value: 0 },
4466 "failure is reported in rt as 0"
4467 );
4468 assert_eq!(bus.read_u32(0x40), 0, "memory untouched");
4469 }
4470
4471 /// The ordinary success path, and the `LLAddr` side effect.
4472 #[test]
4473 fn ll_arms_the_link_and_records_the_physical_address() {
4474 let mut p = Pipeline::new();
4475 let mut bus = ram();
4476 bus.data[0x40..0x44].copy_from_slice(&0x1234_5678u32.to_be_bytes());
4477
4478 // KSEG0, so `translate` has to strip the segment for LLAddr to be right.
4479 let wb = p
4480 .access(
4481 &mut bus,
4482 MemOp::LinkedLoad {
4483 kind: LoadKind::SignedWord,
4484 addr: 0xFFFF_FFFF_8000_0040,
4485 dest: 8,
4486 },
4487 )
4488 .expect("aligned");
4489 assert_eq!(
4490 wb,
4491 WriteBack::Gpr {
4492 dest: 8,
4493 value: 0x1234_5678
4494 }
4495 );
4496 assert!(p.ll_bit(), "LL arms the link bit");
4497 assert_eq!(
4498 p.ll_addr(),
4499 0x40 >> 4,
4500 "LLAddr holds PA(31:4) of the PHYSICAL address, not the virtual one"
4501 );
4502
4503 let wb = p
4504 .access(
4505 &mut bus,
4506 MemOp::ConditionalStore {
4507 kind: StoreKind::Word,
4508 addr: 0xFFFF_FFFF_8000_0040,
4509 value: 0xA5A5_A5A5,
4510 dest: 9,
4511 },
4512 )
4513 .expect("aligned");
4514 assert_eq!(wb, WriteBack::Gpr { dest: 9, value: 1 });
4515 writeback(&mut p, &mut bus, 0xFFFF_FFFF_8000_0040);
4516 assert_eq!(bus.read_u32(0x40), 0xA5A5_A5A5, "the store happened");
4517 }
4518
4519 /// The manual lists exactly what clears `LLbit`: *"set by the LL
4520 /// instruction, cleared by an ERET, and tested by the SC instruction"*
4521 /// (UM §3.1). `SC` is a *tester*, not a clearer.
4522 ///
4523 /// This is the assertion that fails if someone "tidies up" by clearing the
4524 /// link in `SC` — which looks right, matches several other architectures,
4525 /// and would make a second `SC` spuriously fail.
4526 #[test]
4527 fn sc_does_not_clear_the_link_bit() {
4528 let mut p = Pipeline::new();
4529 let mut bus = ram();
4530 p.access(
4531 &mut bus,
4532 MemOp::LinkedLoad {
4533 kind: LoadKind::SignedWord,
4534 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4535 dest: 8,
4536 },
4537 )
4538 .expect("aligned");
4539
4540 for round in 0..3 {
4541 let wb = p
4542 .access(
4543 &mut bus,
4544 MemOp::ConditionalStore {
4545 kind: StoreKind::Word,
4546 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4547 value: 1,
4548 dest: 9,
4549 },
4550 )
4551 .expect("aligned");
4552 assert_eq!(
4553 wb,
4554 WriteBack::Gpr { dest: 9, value: 1 },
4555 "SC #{round} must still succeed -- nothing has cleared LLbit"
4556 );
4557 assert!(p.ll_bit(), "and the bit is still armed after SC #{round}");
4558 }
4559 }
4560
4561 /// *"If this instruction both fails and causes an exception, the exception
4562 /// takes precedence"* (UM §16 p. 487) — so a misaligned `SC` must raise,
4563 /// not quietly report failure in `rt`.
4564 #[test]
4565 fn misaligned_sc_raises_rather_than_reporting_failure() {
4566 let mut p = Pipeline::new();
4567 let mut bus = ram();
4568 let err = p
4569 .access(
4570 &mut bus,
4571 MemOp::ConditionalStore {
4572 kind: StoreKind::Word,
4573 addr: 0xFFFF_FFFF_8000_0000 | 0x42,
4574 value: 1,
4575 dest: 9,
4576 },
4577 )
4578 .expect_err("misaligned");
4579 assert_eq!(
4580 err,
4581 Exception::AddressError { store: true },
4582 "SC is a store, so AdES not AdEL"
4583 );
4584 }
4585
4586 /// A misaligned `LL` must not arm the link — the instruction did not
4587 /// complete, so a following `SC` has nothing to succeed against.
4588 #[test]
4589 fn misaligned_ll_does_not_arm_the_link() {
4590 let mut p = Pipeline::new();
4591 let mut bus = ram();
4592 let err = p
4593 .access(
4594 &mut bus,
4595 MemOp::LinkedLoad {
4596 kind: LoadKind::SignedWord,
4597 addr: 0xFFFF_FFFF_8000_0000 | 0x42,
4598 dest: 8,
4599 },
4600 )
4601 .expect_err("misaligned");
4602 assert_eq!(
4603 err,
4604 Exception::AddressError { store: false },
4605 "LL is a load, so AdEL not AdES"
4606 );
4607 assert!(!p.ll_bit(), "a faulted LL leaves the link disarmed");
4608 }
4609
4610 /// The doubleword forms share the path, but the width must actually differ.
4611 #[test]
4612 fn lld_and_scd_operate_on_eight_bytes() {
4613 let mut p = Pipeline::new();
4614 let mut bus = ram();
4615 bus.data[0x40..0x48].copy_from_slice(&0x0123_4567_89AB_CDEFu64.to_be_bytes());
4616
4617 let wb = p
4618 .access(
4619 &mut bus,
4620 MemOp::LinkedLoad {
4621 kind: LoadKind::Double,
4622 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4623 dest: 8,
4624 },
4625 )
4626 .expect("aligned");
4627 assert_eq!(
4628 wb,
4629 WriteBack::Gpr {
4630 dest: 8,
4631 value: 0x0123_4567_89AB_CDEF
4632 }
4633 );
4634
4635 p.access(
4636 &mut bus,
4637 MemOp::ConditionalStore {
4638 kind: StoreKind::Double,
4639 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4640 value: u64::MAX,
4641 dest: 9,
4642 },
4643 )
4644 .expect("aligned");
4645 writeback(&mut p, &mut bus, 0xFFFF_FFFF_8000_0040);
4646 assert_eq!(bus.read_u32(0x40), u32::MAX);
4647 assert_eq!(bus.read_u32(0x44), u32::MAX, "all eight bytes, not four");
4648 }
4649
4650 // --- COP0 access through the pipeline (T-12-001) -----------------------
4651
4652 /// Build a `COP0` instruction word: opcode 0o20, `rs` = form, `rt` = GPR,
4653 /// `rd` = COP0 register.
4654 const fn cop0_word(rs: u32, rt: u32, rd: u32) -> u32 {
4655 (0o20 << 26) | (rs << 21) | (rt << 16) | (rd << 11)
4656 }
4657
4658 /// `LUI rt, 0x8000` — put a **KSEG0** base in `rt`.
4659 ///
4660 /// Data addresses need this for the same reason instruction fetches need
4661 /// [`KSEG0_PROG`]: a bare low address is KUSEG, which is TLB-mapped and now
4662 /// correctly raises a refill rather than being silently masked.
4663 const fn lui_kseg0(rt: u32) -> u32 {
4664 (0o17 << 26) | (rt << 16) | 0x8000
4665 }
4666
4667 /// `ADDIU rt, $0, imm` — the constant-loading prologue these tests share.
4668 /// Written as a helper rather than inline so the `rs = $0` term does not
4669 /// have to be spelled as a no-op shift that clippy objects to.
4670 const fn addiu_zero(rt: u32, imm: u16) -> u32 {
4671 (0o11 << 26) | (rt << 16) | imm as u32
4672 }
4673
4674 /// `MTC0` then `MFC0` must round-trip through the real register file,
4675 /// exercising the WB-write / DC-read split rather than a direct call.
4676 #[test]
4677 fn mtc0_then_mfc0_round_trips_through_the_pipeline() {
4678 // ADDIU $1, $0, 0x18 ; a value to write
4679 // MTC0 $1, Compare ; COP0 write happens in WB
4680 // MFC0 $2, Compare ; COP0 read happens in DC
4681 let program = alloc::vec![
4682 addiu_zero(1, 0x18),
4683 cop0_word(0o04, 1, u32::from(crate::cop0::reg::COMPARE)),
4684 cop0_word(0o00, 2, u32::from(crate::cop0::reg::COMPARE)),
4685 ];
4686 let mut bus = Ram::new(program);
4687 let mut regs = Regs::new();
4688 let mut p = Pipeline::new();
4689 let mut pc = KSEG0_PROG;
4690
4691 for _ in 0..32 {
4692 p.advance(&mut bus, &mut regs, &mut pc);
4693 }
4694
4695 assert_eq!(
4696 p.cop0.read(crate::cop0::reg::COMPARE),
4697 0x18,
4698 "MTC0 reached the COP0 register file"
4699 );
4700 assert_eq!(regs.read(2), 0x18, "MFC0 brought it back into $2");
4701 }
4702
4703 /// `MTC0` must not be given a GPR destination by decode: it reads `rt` and
4704 /// writes COP0. If `dest` were set to `rt`, the instruction would clobber
4705 /// the very register it sourced its value from.
4706 #[test]
4707 fn mtc0_does_not_write_a_general_register() {
4708 let program = alloc::vec![
4709 addiu_zero(1, 0x55),
4710 cop0_word(0o04, 1, u32::from(crate::cop0::reg::COMPARE)),
4711 ];
4712 let mut bus = Ram::new(program);
4713 let mut regs = Regs::new();
4714 let mut p = Pipeline::new();
4715 let mut pc = KSEG0_PROG;
4716 for _ in 0..24 {
4717 p.advance(&mut bus, &mut regs, &mut pc);
4718 }
4719 assert_eq!(regs.read(1), 0x55, "$1 still holds the source value");
4720 assert_eq!(p.cop0.read(crate::cop0::reg::COMPARE), 0x55);
4721 }
4722
4723 /// The write-mask rules must survive the pipeline path, not just direct
4724 /// calls: `MTC0` to `Cause` may only touch IP1:IP0.
4725 #[test]
4726 fn a_pipelined_mtc0_still_respects_the_write_mask() {
4727 let program = alloc::vec![
4728 // ADDIU $1, $0, -1 => $1 = 0xFFFF_FFFF_FFFF_FFFF
4729 addiu_zero(1, 0xFFFF),
4730 cop0_word(0o04, 1, u32::from(crate::cop0::reg::CAUSE)),
4731 ];
4732 let mut bus = Ram::new(program);
4733 let mut regs = Regs::new();
4734 let mut p = Pipeline::new();
4735 // Move `Compare` off `Count`'s reset value first. Both reset undefined
4736 // (UM §6.4.4) and we choose a deterministic zero for each, so they match
4737 // at power-on and latch IP7 -- see accuracy-ledger D-3. Harmless, but it
4738 // would show up in `Cause` here and obscure what this test is about.
4739 p.cop0.mtc0(crate::cop0::reg::COMPARE, 0xFFFF);
4740 let mut pc = KSEG0_PROG;
4741 for _ in 0..24 {
4742 p.advance(&mut bus, &mut regs, &mut pc);
4743 }
4744 assert_eq!(
4745 p.cop0.read(crate::cop0::reg::CAUSE),
4746 0x0000_0300,
4747 "only the software interrupt bits took"
4748 );
4749 }
4750
4751 /// `LL` records `LLAddr`, which *is* COP0 register 17 — so a `MFC0` of it
4752 /// must see what `LL` wrote. Until COP0 existed, `LLAddr` lived on
4753 /// `Pipeline` as a second copy of the same architectural value; this test
4754 /// pins the fact that there is now only one.
4755 #[test]
4756 fn ll_writes_the_real_cop0_lladdr_register() {
4757 let mut p = Pipeline::new();
4758 let mut bus = Ram::new(alloc::vec![]);
4759 p.access(
4760 &mut bus,
4761 MemOp::LinkedLoad {
4762 kind: LoadKind::SignedWord,
4763 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4764 dest: 8,
4765 },
4766 )
4767 .expect("aligned");
4768 assert_eq!(p.ll_addr(), 0x04, "PA(31:4) of 0x40");
4769 assert_eq!(
4770 p.cop0.read(crate::cop0::reg::LL_ADDR),
4771 0x04,
4772 "the accessor and COP0 reg 17 are the same storage, not two copies"
4773 );
4774 assert_eq!(
4775 p.cop0.mfc0(crate::cop0::reg::LL_ADDR),
4776 0x04,
4777 "and software can read it back with MFC0"
4778 );
4779 }
4780
4781 // --- exception dispatch and ERET through the pipeline (T-12-002) --------
4782
4783 /// The stale-capture regression, with a *plausible* wrong answer available.
4784 ///
4785 /// The pipeline runs valid instructions first, so `ic_rf` holds a real PC
4786 /// when the unaligned fetch arrives. A capture taken before the latch is
4787 /// populated therefore reports that earlier PC — a value that looks entirely
4788 /// reasonable in `EPC`, which is what makes the bug survive inspection.
4789 #[test]
4790 fn an_unaligned_fetch_after_valid_ones_still_reports_its_own_address() {
4791 let program = alloc::vec![addiu_zero(1, 1), addiu_zero(2, 2), addiu_zero(3, 3)];
4792 let mut bus = Ram::new(program);
4793 let mut regs = Regs::new();
4794 let mut p = Pipeline::new();
4795 p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
4796 let mut pc = KSEG0_PROG;
4797
4798 // Let the pipeline fill with real instructions.
4799 for _ in 0..3 {
4800 p.advance(&mut bus, &mut regs, &mut pc);
4801 }
4802 assert!(
4803 p.ic_rf.pc >= 0x800,
4804 "a real PC is latched to be stale about"
4805 );
4806
4807 // Now fetch an unaligned address.
4808 pc = 0xFFFF_FFFF_8000_0006;
4809 p.advance(&mut bus, &mut regs, &mut pc);
4810
4811 assert_eq!(
4812 p.cop0.read(crate::cop0::reg::EPC),
4813 0xFFFF_FFFF_8000_0006,
4814 "EPC must be the unaligned fetch, not the last good one"
4815 );
4816 assert_eq!(
4817 p.cop0.read(crate::cop0::reg::BAD_VADDR),
4818 0xFFFF_FFFF_8000_0006,
4819 "and BadVAddr likewise"
4820 );
4821 }
4822
4823 /// `ERET` clears `LLbit`, completing the `LL`/`SC` contract that Sprint 1
4824 /// left open: until now **nothing** cleared the link, so a `LL`; `ERET`;
4825 /// `SC` sequence wrongly succeeded.
4826 #[test]
4827 fn eret_clears_the_link_bit_and_makes_a_following_sc_fail() {
4828 let mut p = Pipeline::new();
4829 let mut bus = ram();
4830 p.access(
4831 &mut bus,
4832 MemOp::LinkedLoad {
4833 kind: LoadKind::SignedWord,
4834 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4835 dest: 8,
4836 },
4837 )
4838 .expect("aligned");
4839 assert!(p.ll_bit(), "LL armed the link");
4840
4841 // ERET: opcode 0o20, rs = 0o20 (CO), funct = 0o30.
4842 let word = (0o20 << 26) | (0o20 << 21) | 0o30;
4843 assert_eq!(decode(word).op, crate::decode::Op::Eret);
4844
4845 p.cop0
4846 .set_hardware(crate::cop0::reg::STATUS, 1 << 1 /* EXL */);
4847 p.cop0
4848 .set_hardware(crate::cop0::reg::EPC, 0xFFFF_FFFF_8000_5000);
4849
4850 let mut regs = Regs::new();
4851 let mut prog = Ram::new(alloc::vec![word]);
4852 let mut pc = KSEG0_PROG;
4853 // The `LL` above filled the D-cache, whose stall (`M_DCACHE_FILL`, ledger
4854 // C-1) must drain before the ERET runs -- so more cycles than the bare
4855 // pipeline latency.
4856 for _ in 0..60 {
4857 p.advance(&mut prog, &mut regs, &mut pc);
4858 }
4859
4860 assert!(!p.ll_bit(), "ERET cleared the link (UM §3.1)");
4861 let wb = p
4862 .access(
4863 &mut bus,
4864 MemOp::ConditionalStore {
4865 kind: StoreKind::Word,
4866 addr: 0xFFFF_FFFF_8000_0000 | 0x40,
4867 value: 0xFFFF,
4868 dest: 9,
4869 },
4870 )
4871 .expect("aligned");
4872 assert_eq!(
4873 wb,
4874 WriteBack::Gpr { dest: 9, value: 0 },
4875 "SC after ERET must fail"
4876 );
4877 assert_eq!(bus.data[0x40..0x44], [0, 0, 0, 0], "and store nothing");
4878 }
4879
4880 /// `ERET` resumes at `EPC` and clears `EXL`, and it has **no delay slot** —
4881 /// the instruction after it must not execute.
4882 #[test]
4883 fn eret_resumes_at_epc_and_has_no_delay_slot() {
4884 let eret = (0o20 << 26) | (0o20 << 21) | 0o30;
4885 // ERET, then an instruction that would be a delay slot for any branch.
4886 // If it runs, $5 becomes 0x1234 -- which is the whole assertion.
4887 let program = alloc::vec![eret, addiu_zero(5, 0x1234)];
4888 let mut bus = Ram::new(program);
4889 let mut regs = Regs::new();
4890 let mut p = Pipeline::new();
4891 p.cop0.set_hardware(crate::cop0::reg::STATUS, 1 << 1);
4892 p.cop0
4893 .set_hardware(crate::cop0::reg::EPC, 0xFFFF_FFFF_8000_5000);
4894 let mut pc = KSEG0_PROG;
4895
4896 for _ in 0..12 {
4897 p.advance(&mut bus, &mut regs, &mut pc);
4898 }
4899
4900 assert_eq!(
4901 regs.read(5),
4902 0,
4903 "ERET has no delay slot -- the following instruction must be squashed"
4904 );
4905 assert_eq!(
4906 p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1),
4907 0,
4908 "EXL cleared"
4909 );
4910 }
4911
4912 /// **An interrupt pending across an `ERET` is taken at the `ERET`'s TARGET,
4913 /// never at the `ERET` itself** — ledger R-18, and the defect that stopped
4914 /// Banjo-Tooie dead.
4915 ///
4916 /// `ERET` resolves in `EX`, where it clears `Status.EXL` and points
4917 /// `next_pc` at `EPC`. `DC` runs its interrupt check one cycle later, by
4918 /// which time `EXL` reads 0 and `interrupt_pending()` is true — so the
4919 /// interrupt was stamped onto the instruction sitting in `ex_dc`, **which is
4920 /// the `ERET`**. That overwrites `EPC` with the `ERET`'s own address, and
4921 /// the return address it was about to consume is gone. The handler then
4922 /// returns to the `ERET`, which resumes at itself: an architectural
4923 /// livelock, measured at 390,625 `ERET` retirements per frame with no other
4924 /// instruction ever reaching `WB`.
4925 ///
4926 /// The assertion is on `EPC`, not on "an interrupt happened". Both the
4927 /// broken and the fixed machine take the interrupt; they differ only in
4928 /// **which PC it is charged to**, so an assertion that merely observed the
4929 /// vector being entered would pass either way.
4930 #[test]
4931 fn an_interrupt_across_an_eret_is_charged_to_the_target_not_the_eret() {
4932 /// [`Ram`], but with the interrupt line held asserted.
4933 ///
4934 /// `dc_stage` re-samples `bus.poll_irq()` into `Cause.IP2` every cycle,
4935 /// so seeding `IP2` on the COP0 block directly is erased on the next
4936 /// cycle — the first version of this test did exactly that and passed
4937 /// having taken no interrupt at all.
4938 struct IrqRam(Ram);
4939 impl Bus for IrqRam {
4940 fn read_u8(&mut self, a: u32) -> u8 {
4941 self.0.read_u8(a)
4942 }
4943 fn write_u8(&mut self, a: u32, v: u8) {
4944 self.0.write_u8(a, v);
4945 }
4946 fn read_u32(&mut self, a: u32) -> u32 {
4947 self.0.read_u32(a)
4948 }
4949 fn poll_irq(&mut self) -> bool {
4950 true
4951 }
4952 }
4953
4954 const TARGET: u64 = 0xFFFF_FFFF_8000_5000;
4955 let eret = (0o20 << 26) | (0o20 << 21) | 0o30;
4956 let program = alloc::vec![eret, addiu_zero(5, 0x1234)];
4957 let mut bus = IrqRam(Ram::new(program));
4958 let mut regs = Regs::new();
4959 let mut p = Pipeline::new();
4960 // In a handler: EXL set, IE set, IM allowing IP2, and IP2 asserted. This
4961 // is the state libultra's interrupt epilogue actually reaches — the
4962 // handler unmasks and returns with an RCP interrupt still pending.
4963 //
4964 // `EXL` is **set**, which is what isolates this to the `ERET`. While a
4965 // handler runs, `interrupt_pending()` is false however hard the line is
4966 // asserted, so nothing can fire until the `ERET` itself clears `EXL` in
4967 // EX — which places the `ERET` in `ex_dc` on exactly the cycle the
4968 // interrupt becomes takeable. With `EXL` clear instead, the interrupt
4969 // fires during pipeline fill and is charged to a *bubble* (`EPC == 0`) —
4970 // a different defect that would mask this one.
4971 p.cop0
4972 .set_hardware(crate::cop0::reg::STATUS, (1 << 1) | (0xFF << 8) | 1);
4973 p.cop0.set_hardware(crate::cop0::reg::EPC, TARGET);
4974 p.cop0.set_ip(2, true);
4975 let mut pc = KSEG0_PROG;
4976
4977 // Witness BOTH events as they happen, because both end-state values this
4978 // test cares about are also their own starting values: `EXL` starts set
4979 // and `EPC` starts at `TARGET`. Checking either one at the end therefore
4980 // proves nothing on its own — a machine that executed nothing at all
4981 // satisfies both. (The first version of this test did exactly that, and
4982 // it took a review to notice.) So observe the transitions instead:
4983 // `EXL` going clear can only be the `ERET` retiring, and an `Interrupt`
4984 // abort appearing in a latch can only be the interrupt being taken.
4985 let mut eret_retired = false;
4986 let mut interrupt_taken = false;
4987 for _ in 0..12 {
4988 p.advance(&mut bus, &mut regs, &mut pc);
4989 if p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1) == 0 {
4990 eret_retired = true;
4991 }
4992 if [p.dc_wb.abort, p.ex_dc.abort, p.rf_ex.abort, p.ic_rf.abort]
4993 .contains(&Some(Exception::Interrupt))
4994 {
4995 interrupt_taken = true;
4996 }
4997 }
4998 assert!(
4999 eret_retired,
5000 "the ERET never cleared EXL, so it never executed -- nothing under \
5001 test happened"
5002 );
5003 assert!(
5004 interrupt_taken,
5005 "no interrupt was ever taken (Cause={:#010X}) -- this test would \
5006 prove nothing",
5007 p.cop0.read(crate::cop0::reg::CAUSE) as u32
5008 );
5009
5010 let epc = p.cop0.read(crate::cop0::reg::EPC);
5011 assert_ne!(
5012 epc, KSEG0_PROG,
5013 "the interrupt was charged to the ERET itself ({epc:#018X}); ERET would \
5014 then resume at its own address forever (R-18 livelock)"
5015 );
5016 assert_eq!(
5017 epc, TARGET,
5018 "an interrupt taken across an ERET belongs to the instruction the ERET \
5019 returned to, so EPC must still be the target"
5020 );
5021 }
5022
5023 /// A `SYSCALL` executed through the pipeline must vector, record `EPC`, and
5024 /// set the right `ExcCode` — the whole epilogue, end to end.
5025 #[test]
5026 fn a_syscall_vectors_and_records_its_cause() {
5027 // SYSCALL is SPECIAL funct 0o14.
5028 let program = alloc::vec![0o14];
5029 let mut bus = Ram::new(program);
5030 let mut regs = Regs::new();
5031 let mut p = Pipeline::new();
5032 // BEV=0 so the vector is the RDRAM one, which is what a running game
5033 // uses; cold reset would otherwise send us to the boot ROM.
5034 p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
5035 let mut pc = KSEG0_PROG;
5036
5037 // Stop AT the dispatch cycle. Running on would be fine architecturally
5038 // -- the handler starts fetching -- but then `pc` has moved past the
5039 // vector and asserting on it would be asserting that nothing executes.
5040 let mut cycles = 0;
5041 while p.stalled_by() != Some(Interlock::Exception) {
5042 p.advance(&mut bus, &mut regs, &mut pc);
5043 cycles += 1;
5044 assert!(cycles < 8, "SYSCALL never dispatched");
5045 }
5046
5047 assert_eq!(
5048 (p.cop0.read(crate::cop0::reg::CAUSE) >> 2) & 0x1F,
5049 crate::exception::exc_code::SYS
5050 );
5051 assert_eq!(p.cop0.read(crate::cop0::reg::EPC), KSEG0_PROG);
5052 assert_eq!(pc, 0xFFFF_FFFF_8000_0180);
5053 assert_ne!(
5054 p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1),
5055 0,
5056 "EXL set, so the handler runs in kernel mode with interrupts off"
5057 );
5058 }
5059
5060 /// The epilogue must not overwrite `EPC` when `EXL` is already set — tested
5061 /// **through the pipeline**, not just against `dispatch` directly, because
5062 /// this is the failure that only shows up when handlers nest.
5063 #[test]
5064 fn a_second_exception_in_a_handler_preserves_the_first_epc() {
5065 let program = alloc::vec![0o14, 0o14];
5066 let mut bus = Ram::new(program);
5067 let mut regs = Regs::new();
5068 let mut p = Pipeline::new();
5069 p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
5070 let mut pc = KSEG0_PROG;
5071
5072 for _ in 0..8 {
5073 p.advance(&mut bus, &mut regs, &mut pc);
5074 }
5075 let first_epc = p.cop0.read(crate::cop0::reg::EPC);
5076 assert_eq!(first_epc, KSEG0_PROG);
5077 assert_ne!(p.cop0.read(crate::cop0::reg::STATUS) & (1 << 1), 0);
5078
5079 // Now run a second SYSCALL while EXL is still set. Point the fetch back
5080 // at the program so it hits the second word.
5081 pc = KSEG0_PROG + 4;
5082 for _ in 0..8 {
5083 p.advance(&mut bus, &mut regs, &mut pc);
5084 }
5085 assert_eq!(
5086 p.cop0.read(crate::cop0::reg::EPC),
5087 first_epc,
5088 "the first handler's return address must survive (UM §6.3.7)"
5089 );
5090 }
5091
5092 // --- interrupts, Count/Compare (T-12-003) -------------------------------
5093
5094 /// Every term of the recognition predicate is load-bearing. Dropping the
5095 /// `EXL`/`ERL` terms is the classic bug: it works until an interrupt arrives
5096 /// inside a handler, and then re-enters it forever.
5097 #[test]
5098 fn every_term_of_the_interrupt_predicate_is_required() {
5099 use crate::cop0::reg;
5100 let ready = 1u64 | (1 << 10); // IE | IM2
5101
5102 let mut p = Pipeline::new();
5103 p.cop0.set_hardware(reg::STATUS, ready);
5104 p.cop0.set_ip(2, true);
5105 assert!(p.cop0.interrupt_pending(), "all four conditions met");
5106
5107 for (name, status) in [
5108 ("IE clear", ready & !1),
5109 ("EXL set", ready | (1 << 1)),
5110 ("ERL set", ready | (1 << 2)),
5111 ("IM2 masked", ready & !(1 << 10)),
5112 ] {
5113 let mut p = Pipeline::new();
5114 p.cop0.set_hardware(reg::STATUS, status);
5115 p.cop0.set_ip(2, true);
5116 assert!(
5117 !p.cop0.interrupt_pending(),
5118 "{name}: the interrupt must NOT be recognized"
5119 );
5120 }
5121 }
5122
5123 /// A masked interrupt must still be *visible* in `Cause.IP`, because
5124 /// software polls it. Folding assertion and recognition into one step makes
5125 /// a masked line invisible to `MFC0 Cause`.
5126 #[test]
5127 fn a_masked_interrupt_is_still_visible_in_cause() {
5128 use crate::cop0::reg;
5129 let mut p = Pipeline::new();
5130 let mut regs = Regs::new();
5131 let mut pc = KSEG0_PROG;
5132 let mut bus = AlwaysIrq;
5133 // IE set but IM2 MASKED, so nothing is recognized.
5134 p.cop0.set_hardware(reg::STATUS, 1);
5135
5136 p.advance(&mut bus, &mut regs, &mut pc);
5137 assert_ne!(
5138 p.cop0.read(reg::CAUSE) & (1 << 10),
5139 0,
5140 "IP2 is asserted even though it is masked"
5141 );
5142 assert!(!p.cop0.interrupt_pending(), "but not recognized");
5143 }
5144
5145 /// **NMI ignores every mask, and saves to `ErrorEPC` rather than `EPC`.**
5146 ///
5147 /// The interesting part is what it is *not*: `Status` here has `IE` clear,
5148 /// `EXL` set and `ERL` set, which is the state in which an ordinary
5149 /// interrupt is unconditionally ignored. The companion assertion below
5150 /// checks exactly that, so the two differ only in which line was raised.
5151 #[test]
5152 fn an_nmi_is_taken_regardless_of_ie_exl_and_erl() {
5153 use crate::cop0::reg;
5154 let mut p = Pipeline::new();
5155 let mut regs = Regs::new();
5156 let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1); 8]);
5157 // IE clear, EXL set, ERL set -- every reason to refuse an interrupt.
5158 // EXL (bit 1) and ERL (bit 2).
5159 p.cop0.set_hardware(reg::STATUS, (1 << 1) | (1 << 2));
5160 p.cop0.set_hardware(reg::CAUSE, 0);
5161 let mut pc = KSEG0_PROG;
5162
5163 // Warm the pipeline first: the NMI is attributed to the instruction at
5164 // the boundary, and an empty `DC` latch has no PC to record.
5165 for _ in 0..5 {
5166 p.advance(&mut bus, &mut regs, &mut pc);
5167 }
5168 p.signal_nmi();
5169 // Stop at the redirect itself: past it, `pc` is the next fetch address
5170 // and has already walked on from the vector.
5171 let mut cycles = 0;
5172 while p.stalled_by() != Some(Interlock::Exception) {
5173 p.advance(&mut bus, &mut regs, &mut pc);
5174 cycles += 1;
5175 assert!(cycles < 16, "the NMI was never taken");
5176 }
5177
5178 assert_eq!(
5179 pc, 0xFFFF_FFFF_BFC0_0000,
5180 "NMI vectors to the Cold Reset location (UM §6.4.6)"
5181 );
5182 let status = p.cop0.read(reg::STATUS);
5183 assert_ne!(status & (1 << 20), 0, "SR set, to tell NMI from a reset");
5184 assert_ne!(status & (1 << 22), 0, "BEV set");
5185 assert_ne!(status & (1 << 2), 0, "ERL set");
5186 assert_eq!(status & (1 << 21), 0, "TS cleared");
5187 assert_eq!(
5188 p.cop0.read(reg::CAUSE),
5189 0,
5190 "NMI writes no Cause -- all registers are preserved but ErrorEPC \
5191 and those four Status bits"
5192 );
5193 assert_ne!(
5194 p.cop0.read(reg::ERROR_EPC),
5195 0,
5196 "the boundary PC is saved to ErrorEPC, not EPC"
5197 );
5198 assert_eq!(p.cop0.read(reg::EPC), 0, "and EPC is left alone");
5199 }
5200
5201 /// The control for the test above: in that same `Status`, an ordinary
5202 /// interrupt is refused. Without this, "NMI was taken" would be equally
5203 /// consistent with the masks simply not working.
5204 #[test]
5205 fn an_ordinary_interrupt_in_the_same_state_is_refused() {
5206 use crate::cop0::reg;
5207 let mut p = Pipeline::new();
5208 let mut regs = Regs::new();
5209 let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1); 8]);
5210 // EXL (bit 1) and ERL (bit 2).
5211 p.cop0.set_hardware(reg::STATUS, (1 << 1) | (1 << 2));
5212 // Raise the timer line, the strongest ordinary interrupt available.
5213 p.cop0.set_ip(7, true);
5214 let mut pc = KSEG0_PROG;
5215
5216 for _ in 0..16 {
5217 p.advance(&mut bus, &mut regs, &mut pc);
5218 assert_ne!(
5219 p.stalled_by(),
5220 Some(Interlock::Exception),
5221 "a maskable interrupt must NOT be taken with EXL and ERL set"
5222 );
5223 }
5224 }
5225
5226 /// **A stall must not swallow the timer interrupt.**
5227 ///
5228 /// `Count` keeps advancing while the pipeline is stalled — it is derived
5229 /// from the master clock, not from retired instructions — so `Compare` can
5230 /// be passed entirely inside a multi-cycle interlock. An `MCI` stall for a
5231 /// 64-bit multiply is 69 `PCycles` (UM Table 3-12), far longer than the
5232 /// single cycle for which the equality holds.
5233 ///
5234 /// The failure this pins is not a *late* interrupt but a **lost** one: an
5235 /// edge test that only asks "is `Count == Compare` right now?" and is polled
5236 /// only from `DC` never sees the match at all, and `IP7` stays clear
5237 /// forever. Software waiting on the timer then hangs.
5238 #[test]
5239 fn the_timer_interrupt_survives_a_multi_cycle_stall() {
5240 use crate::cop0::reg;
5241 let mut p = Pipeline::new();
5242 let mut regs = Regs::new();
5243 let mut pc = KSEG0_PROG;
5244 let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1)]);
5245 p.cop0.set_hardware(reg::STATUS, 0);
5246 // Compare sits in the middle of the stall window, never at its edges.
5247 p.cop0.mtc0(reg::COMPARE, 40);
5248
5249 p.advance_at(&mut bus, &mut regs, &mut pc, 0);
5250 // A 64-bit multiply's interlock, long enough to step over the match.
5251 p.stall_for(69, Interlock::Mci);
5252 for now in 1..=80 {
5253 p.advance_at(&mut bus, &mut regs, &mut pc, now);
5254 }
5255
5256 assert_ne!(
5257 p.cop0.read(reg::CAUSE) & (1 << 15),
5258 0,
5259 "Count passed Compare during the stall -- IP7 must still latch, \
5260 because the timer is tied to the clock and not to the pipeline"
5261 );
5262 }
5263
5264 /// `Count` reaching `Compare` raises `IP7`, and writing `Compare` clears it
5265 /// as a side effect (UM §6.3.4, p. 165).
5266 #[test]
5267 fn the_timer_interrupt_sets_ip7_and_a_compare_write_clears_it() {
5268 use crate::cop0::reg;
5269 let mut p = Pipeline::new();
5270 let mut regs = Regs::new();
5271 let mut pc = KSEG0_PROG;
5272 let mut bus = Ram::new(alloc::vec![addiu_zero(1, 1)]);
5273 p.cop0.set_hardware(reg::STATUS, 0);
5274 p.cop0.mtc0(reg::COMPARE, 3);
5275
5276 // Walk the timeline to the match.
5277 for now in 0..=3 {
5278 p.advance_at(&mut bus, &mut regs, &mut pc, now);
5279 }
5280 assert_ne!(
5281 p.cop0.read(reg::CAUSE) & (1 << 15),
5282 0,
5283 "IP7 set at Count==Compare"
5284 );
5285
5286 // It must PERSIST as Count runs past Compare. A level implementation
5287 // clears here, and with it drops any timer interrupt raised while the
5288 // CPU could not accept one.
5289 for now in 4..20 {
5290 p.advance_at(&mut bus, &mut regs, &mut pc, now);
5291 assert_ne!(
5292 p.cop0.read(reg::CAUSE) & (1 << 15),
5293 0,
5294 "IP7 must stay latched past the match (UM §6.4.18)"
5295 );
5296 }
5297
5298 // Only writing Compare clears it.
5299 p.cop0.mtc0(reg::COMPARE, 999);
5300 assert_eq!(
5301 p.cop0.read(reg::CAUSE) & (1 << 15),
5302 0,
5303 "writing Compare clears the timer interrupt"
5304 );
5305 p.advance_at(&mut bus, &mut regs, &mut pc, 20);
5306 assert_eq!(p.cop0.read(reg::CAUSE) & (1 << 15), 0, "and it stays clear");
5307 }
5308
5309 /// The convenience [`Pipeline::advance`] **holds** the `Count` timeline
5310 /// rather than guessing a rate for it.
5311 ///
5312 /// `Count` runs at half `PClock`, so stepping it once per `advance` would run
5313 /// the timer at double rate — and halving it with a parity bit would be a
5314 /// second incremented counter, exactly what ADR 0006 forbids. Anything that
5315 /// exercises `Count` must use `advance_at`.
5316 #[test]
5317 fn the_convenience_advance_holds_the_count_timeline() {
5318 let mut p = Pipeline::new();
5319 let mut regs = Regs::new();
5320 let mut pc = KSEG0_PROG;
5321 let mut bus = Ram::new(alloc::vec![]);
5322
5323 p.advance_at(&mut bus, &mut regs, &mut pc, 7);
5324 assert_eq!(p.cop0.read(crate::cop0::reg::COUNT), 7);
5325 for _ in 0..10 {
5326 p.advance(&mut bus, &mut regs, &mut pc);
5327 }
5328 assert_eq!(
5329 p.cop0.read(crate::cop0::reg::COUNT),
5330 7,
5331 "held, not advanced at PClock rate"
5332 );
5333 }
5334
5335 /// **Why `IP7` must latch.** A timer interrupt that fires while the CPU
5336 /// cannot accept one — `EXL` set, i.e. inside a handler — must still be
5337 /// waiting when the handler returns.
5338 ///
5339 /// With `IP7` modeled as a level tied to `Count == Compare`, the equality
5340 /// holds for a single tick, so the interrupt is silently **lost**. That is
5341 /// the failure the latch prevents, and it is invisible to any test that only
5342 /// checks the match cycle itself.
5343 #[test]
5344 fn a_timer_interrupt_raised_while_exl_is_set_survives_until_eret() {
5345 use crate::cop0::reg;
5346 let mut p = Pipeline::new();
5347 let mut regs = Regs::new();
5348 let mut pc = KSEG0_PROG;
5349 let mut bus = Ram::new(alloc::vec![]);
5350 // IE and IM7 set, but EXL set too: a handler is running.
5351 p.cop0.set_hardware(reg::STATUS, 1 | (1 << 15) | (1 << 1));
5352 p.cop0.mtc0(reg::COMPARE, 3);
5353
5354 for now in 0..=10 {
5355 p.advance_at(&mut bus, &mut regs, &mut pc, now);
5356 }
5357 assert!(
5358 !p.cop0.interrupt_pending(),
5359 "EXL blocks it, correctly, for now"
5360 );
5361 assert_ne!(
5362 p.cop0.read(reg::CAUSE) & (1 << 15),
5363 0,
5364 "but IP7 is still asserted, waiting"
5365 );
5366
5367 // The handler returns.
5368 let status = p.cop0.read(reg::STATUS);
5369 p.cop0.set_hardware(reg::STATUS, status & !(1 << 1));
5370 assert!(
5371 p.cop0.interrupt_pending(),
5372 "and the timer interrupt is taken now, not dropped"
5373 );
5374 }
5375
5376 /// An interrupt taken through the pipeline runs the whole epilogue: `EXL`
5377 /// set, `ExcCode` = 0, vectored.
5378 #[test]
5379 fn an_accepted_interrupt_vectors_with_exccode_zero() {
5380 use crate::cop0::reg;
5381 let mut p = Pipeline::new();
5382 let mut regs = Regs::new();
5383 let mut pc = KSEG0_PROG;
5384 let mut bus = AlwaysIrq;
5385 p.cop0.set_hardware(reg::STATUS, 1 | (1 << 10));
5386
5387 let mut cycles = 0;
5388 while p.stalled_by() != Some(Interlock::Exception) {
5389 p.advance(&mut bus, &mut regs, &mut pc);
5390 cycles += 1;
5391 assert!(cycles < 12, "the interrupt was never taken");
5392 }
5393 assert_eq!(
5394 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5395 crate::exception::exc_code::INT
5396 );
5397 assert_ne!(p.cop0.read(reg::STATUS) & (1 << 1), 0, "EXL set");
5398 assert_eq!(pc, 0xFFFF_FFFF_8000_0180);
5399 // And now that EXL is set, the still-asserted line must NOT re-enter.
5400 assert!(
5401 !p.cop0.interrupt_pending(),
5402 "EXL blocks re-entry while the handler runs"
5403 );
5404 }
5405
5406 // --- the TLB through the pipeline (T-12-004) ---------------------------
5407
5408 /// Install a 4 KiB global mapping `vaddr` -> `pfn`, valid and writable.
5409 fn map(p: &mut Pipeline, index: u64, vaddr: u64, pfn: u64) {
5410 use crate::cop0::reg;
5411 p.cop0.set_hardware(reg::PAGE_MASK, 0);
5412 p.cop0.set_hardware(reg::ENTRY_HI, vaddr & 0xFFFF_E000);
5413 // V | D | C=3 | G, in both halves so the entry is global.
5414 p.cop0
5415 .set_hardware(reg::ENTRY_LO0, (pfn << 6) | (3 << 3) | 0b111);
5416 p.cop0
5417 .set_hardware(reg::ENTRY_LO1, ((pfn + 1) << 6) | (3 << 3) | 0b111);
5418 p.cop0.set_hardware(reg::INDEX, index);
5419 p.tlb.write_entry(index as usize, &p.cop0);
5420 }
5421
5422 /// A KUSEG access with no mapping raises a **refill**, which takes the
5423 /// refill vector — not the general one.
5424 #[test]
5425 fn an_unmapped_kuseg_access_takes_the_refill_vector() {
5426 use crate::cop0::reg;
5427 let mut p = Pipeline::new();
5428 let mut regs = Regs::new();
5429 let mut bus = Ram::new(alloc::vec![lui_kseg0(1), ld_st(0o43, 0, 3, 0x100)]);
5430 // BEV=0, EXL=0 so the refill vector is the RDRAM one.
5431 p.cop0.set_hardware(reg::STATUS, 0);
5432 let mut pc = KSEG0_PROG;
5433
5434 let mut cycles = 0;
5435 while p.stalled_by() != Some(Interlock::Exception) {
5436 p.advance(&mut bus, &mut regs, &mut pc);
5437 cycles += 1;
5438 assert!(cycles < 16, "no exception raised");
5439 }
5440 assert_eq!(
5441 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5442 crate::exception::exc_code::TLBL,
5443 "a load miss is TLBL"
5444 );
5445 assert_eq!(
5446 pc, 0xFFFF_FFFF_8000_0000,
5447 "the REFILL vector (0x000), not the general one (0x180)"
5448 );
5449 assert_eq!(p.cop0.read(reg::BAD_VADDR), 0x100);
5450 }
5451
5452 /// The same miss, but with **64-bit addressing enabled**, must reach the
5453 /// **XTLB** refill vector at `0x080`.
5454 ///
5455 /// This is the end-to-end companion to
5456 /// `a_refill_in_64_bit_addressing_takes_the_xtlb_vector`, which only pins the
5457 /// exception-to-vector mapping. That mapping is satisfied just as well by a
5458 /// `wide` flag that is *never true in practice* — the plumbing from
5459 /// `Status.KX` through the faulting access to the exception is exactly what
5460 /// a mapping test cannot see. The two differ only in `Status`, so the
5461 /// vectors landing on different addresses is attributable to nothing else.
5462 #[test]
5463 fn an_unmapped_access_under_64_bit_addressing_takes_the_xtlb_vector() {
5464 use crate::cop0::reg;
5465 let mut p = Pipeline::new();
5466 let mut regs = Regs::new();
5467 let mut bus = Ram::new(alloc::vec![lui_kseg0(1), ld_st(0o43, 0, 3, 0x100)]);
5468 // Identical to the 32-bit case except for KX (bit 7), which turns on
5469 // 64-bit addressing for Kernel mode.
5470 p.cop0.set_hardware(reg::STATUS, 1 << 7);
5471 let mut pc = KSEG0_PROG;
5472
5473 let mut cycles = 0;
5474 while p.stalled_by() != Some(Interlock::Exception) {
5475 p.advance(&mut bus, &mut regs, &mut pc);
5476 cycles += 1;
5477 assert!(cycles < 16, "no exception raised");
5478 }
5479 assert_eq!(
5480 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5481 crate::exception::exc_code::TLBL,
5482 "still TLBL -- the ExcCode does not change with the vector"
5483 );
5484 assert_eq!(
5485 pc, 0xFFFF_FFFF_8000_0080,
5486 "the XTLB refill vector (0x080), not the 32-bit one (0x000)"
5487 );
5488 }
5489
5490 /// A mapped, valid page translates end to end through a real load.
5491 #[test]
5492 fn a_mapped_page_translates_a_real_load() {
5493 let mut p = Pipeline::new();
5494 let mut regs = Regs::new();
5495 // Map KUSEG page-pair 0 with pfn 0 (even) / 1 (odd), so the even page is
5496 // an identity mapping onto the small test RAM. Note 0x1000 and 0x0000
5497 // are the SAME pair -- VPN2 tags at 8 KiB granularity, not 4 KiB.
5498 let mut bus = Ram::new(alloc::vec![ld_st(0o43, 0, 3, 0x100)]);
5499 bus.write_u8(0x100, 0xAB);
5500 bus.write_u8(0x101, 0xCD);
5501 bus.write_u8(0x102, 0xEF);
5502 bus.write_u8(0x103, 0x01);
5503 map(&mut p, 0, 0x1000, 0);
5504 p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
5505 let mut pc = KSEG0_PROG;
5506
5507 // Enough cycles to cover the D-cache fill stall the cached load now pays
5508 // (`M_DCACHE_FILL`, ledger C-1) on top of the pipeline latency.
5509 for _ in 0..100 {
5510 p.advance(&mut bus, &mut regs, &mut pc);
5511 }
5512 assert_eq!(
5513 regs.read(3),
5514 crate::alu::sext32(0xABCD_EF01),
5515 "the even page of pair 0 maps to physical 0x100 via the TLB"
5516 );
5517 }
5518
5519 /// A store to a page whose `D` bit is clear raises **Modified**, which takes
5520 /// the general vector — an entry was found, so there is nothing to refill.
5521 #[test]
5522 fn a_store_to_a_clean_page_raises_modified_at_the_general_vector() {
5523 use crate::cop0::reg;
5524 let mut p = Pipeline::new();
5525 let mut regs = Regs::new();
5526 let mut bus = Ram::new(alloc::vec![ld_st(0o53, 0, 0, 0x1100)]);
5527 // V | C=3 | G but NOT D -- readable, not writable.
5528 p.cop0.set_hardware(reg::PAGE_MASK, 0);
5529 p.cop0.set_hardware(reg::ENTRY_HI, 0x1000);
5530 p.cop0.set_hardware(reg::ENTRY_LO0, (3 << 3) | 0b011);
5531 p.cop0
5532 .set_hardware(reg::ENTRY_LO1, (1 << 6) | (3 << 3) | 0b011);
5533 p.tlb.write_entry(0, &p.cop0);
5534 p.cop0.set_hardware(reg::STATUS, 0);
5535 let mut pc = KSEG0_PROG;
5536
5537 let mut cycles = 0;
5538 while p.stalled_by() != Some(Interlock::Exception) {
5539 p.advance(&mut bus, &mut regs, &mut pc);
5540 cycles += 1;
5541 assert!(cycles < 16, "no exception raised");
5542 }
5543 assert_eq!(
5544 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5545 crate::exception::exc_code::MOD
5546 );
5547 assert_eq!(
5548 pc, 0xFFFF_FFFF_8000_0180,
5549 "Modified takes the GENERAL vector"
5550 );
5551 }
5552
5553 /// A TLB exception fills `EntryHi`, `Context` and `XContext` — the refill
5554 /// handler reads `Context` as a ready-made page-table pointer, which is why
5555 /// hardware assembles it rather than leaving it to software.
5556 #[test]
5557 fn a_tlb_exception_assembles_entryhi_and_context() {
5558 use crate::cop0::reg;
5559 let mut p = Pipeline::new();
5560 let mut regs = Regs::new();
5561 let mut bus = Ram::new(alloc::vec![ld_st(0o43, 0, 3, 0x4000)]);
5562 p.cop0.set_hardware(reg::STATUS, 0);
5563 // A page-table base the handler would have set up.
5564 p.cop0.set_hardware(reg::CONTEXT, 0xFFFF_FFFF_8080_0000);
5565 let mut pc = KSEG0_PROG;
5566
5567 let mut cycles = 0;
5568 while p.stalled_by() != Some(Interlock::Exception) {
5569 p.advance(&mut bus, &mut regs, &mut pc);
5570 cycles += 1;
5571 assert!(cycles < 16, "no exception raised");
5572 }
5573 assert_eq!(
5574 p.cop0.read(reg::ENTRY_HI) & 0xFFFF_E000,
5575 0x4000,
5576 "EntryHi holds the faulting VPN2"
5577 );
5578 assert_eq!(
5579 p.cop0.read(reg::CONTEXT) & 0xFFFF_FFFF_FF80_0000,
5580 0xFFFF_FFFF_8080_0000,
5581 "PTEBase is preserved"
5582 );
5583 assert_eq!(
5584 (p.cop0.read(reg::CONTEXT) >> 4) & 0x7_FFFF,
5585 0x4000 >> 13,
5586 "BadVPN2 is filled in"
5587 );
5588 }
5589
5590 /// `TLBWI` writes the entry `Index` names; `TLBWR` writes the one `Random`
5591 /// names. Using the wrong register is a silent, hard-to-see swap.
5592 #[test]
5593 fn tlbwi_uses_index_and_tlbwr_uses_random() {
5594 use crate::cop0::reg;
5595 const TLBWI: u32 = (0o20 << 26) | (0o20 << 21) | 0o02;
5596 const TLBWR: u32 = (0o20 << 26) | (0o20 << 21) | 0o06;
5597
5598 let mut p = Pipeline::new();
5599 let mut regs = Regs::new();
5600 let mut bus = Ram::new(alloc::vec![TLBWI]);
5601 p.cop0.set_hardware(reg::STATUS, 0);
5602 p.cop0.set_hardware(reg::ENTRY_HI, 0x2000);
5603 p.cop0
5604 .set_hardware(reg::ENTRY_LO0, (7 << 6) | (3 << 3) | 0b111);
5605 p.cop0
5606 .set_hardware(reg::ENTRY_LO1, (8 << 6) | (3 << 3) | 0b111);
5607 p.cop0.set_hardware(reg::INDEX, 5);
5608 let mut pc = KSEG0_PROG;
5609 for _ in 0..16 {
5610 p.advance(&mut bus, &mut regs, &mut pc);
5611 }
5612 assert_eq!(p.tlb.entry(5).lo0.pfn, 7, "TLBWI wrote entry Index = 5");
5613
5614 // TLBWR with Random forced to a different index.
5615 let mut p = Pipeline::new();
5616 let mut regs = Regs::new();
5617 let mut bus = Ram::new(alloc::vec![TLBWR]);
5618 p.cop0.set_hardware(reg::STATUS, 0);
5619 p.cop0.set_hardware(reg::ENTRY_HI, 0x2000);
5620 p.cop0
5621 .set_hardware(reg::ENTRY_LO0, (9 << 6) | (3 << 3) | 0b111);
5622 p.cop0
5623 .set_hardware(reg::ENTRY_LO1, (10 << 6) | (3 << 3) | 0b111);
5624 p.cop0.set_hardware(reg::INDEX, 5);
5625 p.cop0.set_hardware(reg::RANDOM, 20);
5626 let mut pc = KSEG0_PROG;
5627 for _ in 0..16 {
5628 p.advance(&mut bus, &mut regs, &mut pc);
5629 }
5630 assert_eq!(p.tlb.entry(20).lo0.pfn, 9, "TLBWR wrote entry Random = 20");
5631 assert_ne!(p.tlb.entry(5).lo0.pfn, 9, "and NOT entry Index");
5632 }
5633
5634 /// `TLBWR` cannot reach a wired entry, because `Random` never goes below
5635 /// `Wired` — but **`TLBWI` can** (UM §5.4.4, p. 150). Guarding both is a
5636 /// natural-looking mistake that makes wired entries unwritable at all.
5637 #[test]
5638 fn tlbwi_can_overwrite_a_wired_entry_even_though_tlbwr_cannot() {
5639 use crate::cop0::reg;
5640 const TLBWI: u32 = (0o20 << 26) | (0o20 << 21) | 0o02;
5641 let mut p = Pipeline::new();
5642 let mut regs = Regs::new();
5643 let mut bus = Ram::new(alloc::vec![TLBWI]);
5644 p.cop0.set_hardware(reg::STATUS, 0);
5645 p.cop0.mtc0(reg::WIRED, 8);
5646 p.cop0.set_hardware(reg::ENTRY_HI, 0x2000);
5647 p.cop0
5648 .set_hardware(reg::ENTRY_LO0, (11 << 6) | (3 << 3) | 0b111);
5649 p.cop0
5650 .set_hardware(reg::ENTRY_LO1, (12 << 6) | (3 << 3) | 0b111);
5651 p.cop0.set_hardware(reg::INDEX, 3); // inside the wired range
5652 let mut pc = KSEG0_PROG;
5653 for _ in 0..16 {
5654 p.advance(&mut bus, &mut regs, &mut pc);
5655 }
5656 assert_eq!(
5657 p.tlb.entry(3).lo0.pfn,
5658 11,
5659 "TLBWI must be able to write a wired entry"
5660 );
5661
5662 // And Random's range protects those entries from TLBWR structurally,
5663 // rather than by a check.
5664 for _ in 0..200 {
5665 p.cop0.tick_random();
5666 assert!(
5667 p.cop0.read(reg::RANDOM) >= 8,
5668 "Random must never select a wired entry"
5669 );
5670 }
5671 }
5672
5673 /// A TLB fault on a sign-extended kernel address must record the **`R`**
5674 /// region in `EntryHi`, not just `VPN2`.
5675 ///
5676 /// Leaving `R` zero puts every such fault in region 0, so the handler's
5677 /// `TLBWR` installs an entry that can never match the address that faulted —
5678 /// an infinite refill loop, not a visibly wrong value.
5679 #[test]
5680 fn a_fault_on_a_sign_extended_address_records_its_region_in_entryhi() {
5681 use crate::cop0::reg;
5682 let mut p = Pipeline::new();
5683 let mut regs = Regs::new();
5684 // LW $3, 0($1) with $1 = 0xFFFF_FFFF_E000_0000 (KSEG3, mapped).
5685 let mut bus = Ram::new(alloc::vec![
5686 (0o17 << 26) | (1 << 16) | 0xE000, // LUI $1, 0xE000
5687 ld_st(0o43, 1, 3, 0),
5688 ]);
5689 p.cop0.set_hardware(reg::STATUS, 0);
5690 let mut pc = KSEG0_PROG;
5691
5692 let mut cycles = 0;
5693 while p.stalled_by() != Some(Interlock::Exception) {
5694 p.advance(&mut bus, &mut regs, &mut pc);
5695 cycles += 1;
5696 assert!(cycles < 20, "no TLB exception raised");
5697 }
5698 assert_eq!(
5699 p.cop0.read(reg::BAD_VADDR),
5700 0xFFFF_FFFF_E000_0000,
5701 "the full sign-extended address faulted"
5702 );
5703 assert_eq!(
5704 (p.cop0.read(reg::ENTRY_HI) >> 62) & 0b11,
5705 0b11,
5706 "EntryHi.R must carry the faulting region, not 0"
5707 );
5708 assert_eq!(
5709 p.cop0.read(reg::ENTRY_HI) & crate::tlb::VPN2_MASK,
5710 0xFFFF_FFFF_E000_0000 & crate::tlb::VPN2_MASK,
5711 "and VPN2 alongside it"
5712 );
5713 }
5714
5715 /// **`Random` advances as instructions retire** (UM §5.4.2, p. 147).
5716 ///
5717 /// It was implemented and never called from the pipeline, so it sat at 31
5718 /// forever and every `TLBWR` overwrote the same entry. A stuck counter is
5719 /// invisible to any test that calls `tick_random` itself — which is what the
5720 /// COP0 unit tests do — so this asserts it through `advance`.
5721 #[test]
5722 fn random_advances_as_instructions_retire() {
5723 use crate::cop0::reg;
5724 let mut p = Pipeline::new();
5725 let mut regs = Regs::new();
5726 let mut bus = Ram::new(alloc::vec![
5727 addiu_zero(1, 1),
5728 addiu_zero(2, 2),
5729 addiu_zero(3, 3),
5730 addiu_zero(4, 4),
5731 ]);
5732 p.cop0.set_hardware(reg::STATUS, 0);
5733 p.cop0.mtc0(reg::WIRED, 0);
5734 let start = p.cop0.read(reg::RANDOM);
5735 let mut pc = KSEG0_PROG;
5736 for _ in 0..24 {
5737 p.advance(&mut bus, &mut regs, &mut pc);
5738 }
5739 assert!(p.retired >= 4, "instructions retired");
5740 assert_ne!(
5741 p.cop0.read(reg::RANDOM),
5742 start,
5743 "Random must move as instructions retire -- a stuck Random makes \
5744 every TLBWR overwrite the same entry"
5745 );
5746 }
5747
5748 /// TLB shutdown must reach **`Status.TS`**, not just an internal flag —
5749 /// software polls `TS` precisely to discover that the TLB has died.
5750 #[test]
5751 fn tlb_shutdown_sets_status_ts() {
5752 use crate::cop0::reg;
5753 let mut p = Pipeline::new();
5754 let mut regs = Regs::new();
5755 let mut bus = Ram::new(alloc::vec![ld_st(0o43, 0, 3, 0x100)]);
5756 p.cop0.set_hardware(reg::STATUS, 0);
5757 // Two coinciding entries.
5758 map(&mut p, 0, 0x0000, 0);
5759 map(&mut p, 7, 0x0000, 4);
5760 assert_eq!(p.cop0.read(reg::STATUS) & (1 << 21), 0, "TS clear so far");
5761
5762 let mut pc = KSEG0_PROG;
5763 for _ in 0..24 {
5764 p.advance(&mut bus, &mut regs, &mut pc);
5765 }
5766 assert!(p.tlb.is_shutdown(), "the duplicate was noticed");
5767 assert_ne!(
5768 p.cop0.read(reg::STATUS) & (1 << 21),
5769 0,
5770 "Status.TS must be set (UM Fig. 6-6) -- an internal flag nobody can \
5771 read is worse than not tracking it"
5772 );
5773 }
5774
5775 /// The 3-PCycle micro-ITLB reload is charged **only when a reload happens**
5776 /// (UM §4.6.2). A fetch that misses both levels goes straight to its
5777 /// exception rather than paying for a reload that never occurred.
5778 ///
5779 /// **This test cannot currently observe the charge itself.** `stall_for`
5780 /// replaces any pending stall, so the exception's 2-PCycle stall supersedes
5781 /// a wrongly-charged 3-PCycle reload in the same cycle — mutating the guard
5782 /// away produces no behavioral difference today, which mutation testing
5783 /// duly reported. The guard is kept because it is what the manual says and
5784 /// because it becomes observable the moment stalls compose rather than
5785 /// replace; what is asserted here is the *decision*
5786 /// ([`Tlb::jtlb_has_match`]) and the absence of a reload, not the timing.
5787 #[test]
5788 fn a_fetch_missing_both_tlb_levels_is_not_charged_for_a_reload() {
5789 use crate::cop0::reg;
5790 let mut p = Pipeline::new();
5791 let mut regs = Regs::new();
5792 let mut bus = Ram::new(alloc::vec![]);
5793 p.cop0.set_hardware(reg::STATUS, 0);
5794 // Fetch from an unmapped KUSEG address: misses ITLB and JTLB alike.
5795 let mut pc = 0x0000_4000u64;
5796
5797 assert!(
5798 !p.tlb.jtlb_has_match(pc, 0),
5799 "the JTLB has nothing to reload the micro-TLB from"
5800 );
5801 p.advance(&mut bus, &mut regs, &mut pc);
5802 assert_eq!(
5803 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5804 crate::exception::exc_code::TLBL,
5805 "it went straight to the refill exception"
5806 );
5807 }
5808
5809 // --- COP1 control and coprocessor usability (T-12-006) -----------------
5810
5811 /// The exact instruction n64-systemtest dies on: `CTC1 $rt, $31`, its fourth
5812 /// statement. If this does not work the suite reports nothing at all, and
5813 /// every COP0/TLB test in Sprint 2 is unreachable behind it.
5814 #[test]
5815 fn ctc1_to_fcr31_works_which_is_what_unblocks_the_oracle() {
5816 use crate::cop0::reg;
5817 // CTC1: opcode 0o21, rs = 0o06, rt = GPR, rd = fs.
5818 const fn ctc1(rt: u32, fs: u32) -> u32 {
5819 (0o21 << 26) | (0o06 << 21) | (rt << 16) | (fs << 11)
5820 }
5821 const fn cfc1(rt: u32, fs: u32) -> u32 {
5822 (0o21 << 26) | (0o02 << 21) | (rt << 16) | (fs << 11)
5823 }
5824 // LUI $1, 0x0100 ; bit 24 -- flush_denorm_to_zero
5825 // ORI $1, $1, 0x800 ; bit 11 -- enable_invalid_operation
5826 // CTC1 $1, $31
5827 // CFC1 $2, $31
5828 let prog = alloc::vec![
5829 (0o17 << 26) | (1 << 16) | 0x0100,
5830 (0o15 << 26) | (1 << 21) | (1 << 16) | 0x0800,
5831 ctc1(1, 31),
5832 cfc1(2, 31),
5833 ];
5834 let mut bus = Ram::new(prog);
5835 let mut regs = Regs::new();
5836 let mut p = Pipeline::new();
5837 // CU1 enabled, as IPL3 leaves it (Status = 0x3400_0000).
5838 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
5839 let mut pc = KSEG0_PROG;
5840 for _ in 0..40 {
5841 p.advance(&mut bus, &mut regs, &mut pc);
5842 }
5843
5844 let want = (1u64 << 24) | (1 << 11);
5845 assert_eq!(u64::from(p.cop1.fcsr()), want, "CTC1 reached FCR31");
5846 assert_eq!(regs.read(2), want, "and CFC1 read it back");
5847 assert!(p.cop1.flush_denorm_to_zero());
5848 }
5849
5850 /// With `CU1` clear, a COP1 instruction raises **Coprocessor Unusable** with
5851 /// `Cause.CE = 1` — not Reserved Instruction, which is the natural mistake
5852 /// for an unimplemented encoding.
5853 #[test]
5854 fn a_cop1_instruction_with_cu1_clear_raises_coprocessor_unusable() {
5855 use crate::cop0::reg;
5856 const CTC1: u32 = (0o21 << 26) | (0o06 << 21) | (1 << 16) | (31 << 11);
5857 let mut bus = Ram::new(alloc::vec![CTC1]);
5858 let mut regs = Regs::new();
5859 let mut p = Pipeline::new();
5860 // Kernel mode, but CU1 CLEAR.
5861 p.cop0.set_hardware(reg::STATUS, 0);
5862 let mut pc = KSEG0_PROG;
5863
5864 let mut cycles = 0;
5865 while p.stalled_by() != Some(Interlock::Exception) {
5866 p.advance(&mut bus, &mut regs, &mut pc);
5867 cycles += 1;
5868 assert!(cycles < 16, "no exception raised");
5869 }
5870 assert_eq!(
5871 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5872 crate::exception::exc_code::CPU,
5873 "Coprocessor Unusable, not Reserved Instruction"
5874 );
5875 assert_eq!(
5876 (p.cop0.read(reg::CAUSE) >> 28) & 0b11,
5877 1,
5878 "Cause.CE names the offending unit"
5879 );
5880 assert_eq!(p.cop1.fcsr(), 0, "and the write did not take effect");
5881 }
5882
5883 /// **COP0 is usable from kernel mode regardless of `CU0`.** Otherwise the CPU
5884 /// could not run an exception handler before `Status` had been set up — a
5885 /// chicken-and-egg the hardware does not have.
5886 #[test]
5887 fn cop0_is_usable_in_kernel_mode_even_with_cu0_clear() {
5888 use crate::cop0::reg;
5889 const MTC0: u32 = (0o20 << 26) | (0o04 << 21) | (1 << 16) | ((reg::COMPARE as u32) << 11);
5890 let mut bus = Ram::new(alloc::vec![addiu_zero(1, 0x77), MTC0]);
5891 let mut regs = Regs::new();
5892 let mut p = Pipeline::new();
5893 // KSU = 0 (kernel), CU0 clear, EXL/ERL clear.
5894 p.cop0.set_hardware(reg::STATUS, 0);
5895 let mut pc = KSEG0_PROG;
5896 for _ in 0..24 {
5897 p.advance(&mut bus, &mut regs, &mut pc);
5898 }
5899 assert_eq!(
5900 p.cop0.read(reg::COMPARE),
5901 0x77,
5902 "MTC0 must work in kernel mode without CU0"
5903 );
5904 assert_eq!(
5905 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5906 0,
5907 "and must not have raised"
5908 );
5909 }
5910
5911 /// In **user** mode with `CU0` clear, COP0 *is* unusable — otherwise the
5912 /// kernel-mode exemption would be a blanket bypass rather than a rule.
5913 #[test]
5914 fn cop0_is_unusable_in_user_mode_without_cu0() {
5915 use crate::cop0::reg;
5916 const MTC0: u32 = (0o20 << 26) | (0o04 << 21) | (1 << 16) | ((reg::COMPARE as u32) << 11);
5917 let mut bus = Ram::new(alloc::vec![MTC0]);
5918 let mut regs = Regs::new();
5919 let mut p = Pipeline::new();
5920 // KSU = 2 (user), CU0 clear, EXL/ERL clear.
5921 p.cop0.set_hardware(reg::STATUS, 0b10 << 3);
5922 // The program cannot live in KSEG0 here: that segment does not exist in
5923 // User mode, so the FETCH would raise AdEL and the test would pass on
5924 // the wrong exception. Map page-pair 0 identically instead and run from
5925 // USEG, which is the only place a user program can be.
5926 map(&mut p, 0, 0, 0);
5927 let mut pc = KSEG0_PROG & 0x1FFF_FFFF;
5928
5929 let mut cycles = 0;
5930 while p.stalled_by() != Some(Interlock::Exception) {
5931 p.advance(&mut bus, &mut regs, &mut pc);
5932 cycles += 1;
5933 assert!(cycles < 16, "no exception raised");
5934 }
5935 assert_eq!(
5936 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5937 crate::exception::exc_code::CPU
5938 );
5939 assert_eq!((p.cop0.read(reg::CAUSE) >> 28) & 0b11, 0, "unit 0");
5940 }
5941
5942 /// A 64-bit operation is Reserved in 32-bit User or Supervisor mode, and
5943 /// usable everywhere else.
5944 ///
5945 /// All four rows are needed. Gating on the width bit alone reserves them for
5946 /// a 32-bit *kernel* — which is the mode every N64 boots into, so that
5947 /// mistake breaks everything and would be caught. Gating on the mode alone
5948 /// reserves them for a 64-bit *user* program, which nothing common does, so
5949 /// that mistake would sit unnoticed behind the rows that do pass.
5950 #[test]
5951 fn a_64_bit_operation_is_reserved_only_in_32_bit_non_kernel_mode() {
5952 use crate::cop0::reg;
5953 /// `DADD $2, $1, $1` — SPECIAL funct 0o54.
5954 const DADD: u32 = (1 << 21) | (1 << 16) | (2 << 11) | 0o54;
5955 /// `Status.KSU` = user, and `Status.UX`.
5956 const USER: u64 = 0b10 << 3;
5957 const UX: u64 = 1 << 5;
5958
5959 for (status, want_reserved, why) in [
5960 (0, false, "32-bit kernel: usable"),
5961 (1 << 7, false, "64-bit kernel: usable"),
5962 (USER, true, "32-bit user: reserved"),
5963 (USER | UX, false, "64-bit user: usable"),
5964 ] {
5965 let mut bus = Ram::new(alloc::vec![DADD]);
5966 let mut regs = Regs::new();
5967 let mut p = Pipeline::new();
5968 p.cop0.set_hardware(reg::STATUS, status);
5969 // User mode cannot fetch from KSEG0, so the program runs from USEG
5970 // through an identity mapping in both cases -- keeping the only
5971 // difference between the rows the one under test.
5972 map(&mut p, 0, 0, 0);
5973 let mut pc = KSEG0_PROG & 0x1FFF_FFFF;
5974
5975 let mut raised = false;
5976 for _ in 0..16 {
5977 p.advance(&mut bus, &mut regs, &mut pc);
5978 if p.stalled_by() == Some(Interlock::Exception) {
5979 raised = true;
5980 break;
5981 }
5982 }
5983 assert_eq!(raised, want_reserved, "{why}");
5984 if raised {
5985 assert_eq!(
5986 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
5987 crate::exception::exc_code::RI,
5988 "{why}: and it is Reserved Instruction, not something else"
5989 );
5990 }
5991 }
5992 }
5993
5994 /// A `BC1` immediately after a `C.cond.fmt` sees that compare's result.
5995 ///
5996 /// `BC1` resolves in `EX`; the compare commits `FCSR.C` in `WB`. Without a
5997 /// forwarding path the branch samples the **previous** condition, falls
5998 /// through, and nothing anywhere reports an error — the ROM emits exactly
5999 /// this pair with no separating instruction.
6000 ///
6001 /// Both directions are asserted. Testing only the taken case would pass
6002 /// against a bypass that returned `true` unconditionally.
6003 #[test]
6004 fn a_bc1_immediately_after_a_compare_sees_its_result() {
6005 use crate::cop0::reg;
6006 /// `C.EQ.S $f0, $f2` — fmt S, ft = 2, fs = 0, cond = EQ.
6007 const C_EQ: u32 = (0o21 << 26) | (0o20 << 21) | (2 << 16) | 0o62;
6008 /// `BC1T +2` — skips the `ORI` two instructions ahead.
6009 const BC1T: u32 = (0o21 << 26) | (0o10 << 21) | (1 << 16) | 2;
6010 /// `ORI $8, $0, 1` — runs only if the branch was NOT taken.
6011 const ORI: u32 = (0o15 << 26) | (8 << 16) | 1;
6012
6013 for (lo0, equal) in [(0x2222_3333u64, true), (0x4444_5555, false)] {
6014 let mut bus = Ram::new(alloc::vec![C_EQ, BC1T, 0, ORI, 0, 0, 0, 0, 0, 0]);
6015 let mut regs = Regs::new();
6016 let mut p = Pipeline::new();
6017 p.cop0.set_hardware(reg::STATUS, 1 << 29); // CU1
6018 // Upper halves differ deliberately: a single-precision compare must
6019 // ignore them, so a 64-bit comparison would report "not equal" for
6020 // the row that is supposed to match.
6021 p.fpr.write_d(0, true, lo0);
6022 p.fpr.write_d(2, true, 0x1111_1111_2222_3333);
6023 let mut pc = KSEG0_PROG;
6024 for _ in 0..14 {
6025 p.advance(&mut bus, &mut regs, &mut pc);
6026 }
6027 assert_eq!(
6028 regs.read(8) == 0,
6029 equal,
6030 "operands equal = {equal}: the branch must {} the ORI",
6031 if equal { "skip" } else { "run" }
6032 );
6033 }
6034 }
6035
6036 /// An unimplemented COP1 encoding with `CU1` **set** must not raise. Sprint 3
6037 /// then *adds* behavior rather than changing it — and an emulator that
6038 /// raised here would look correct until the FPU landed.
6039 #[test]
6040 fn an_unimplemented_cop1_encoding_does_not_raise_when_cu1_is_set() {
6041 use crate::cop0::reg;
6042 // COP1 `rs = 0o11` -- an unassigned encoding in the coprocessor's own
6043 // opcode space, so it is valid to *fetch* and does nothing.
6044 //
6045 // This test has now outlived three subjects: `ADD.S` until the S/D
6046 // arithmetic landed, `SQRT.S` until T-13-005, then `BC1F` until the
6047 // branch was wired. Each move is the test doing its job -- the point is
6048 // the *unimplemented* path, so the subject follows whatever is still on
6049 // it. When `rs = 0o11` acquires a meaning, move it again rather than
6050 // deleting it.
6051 const UNASSIGNED: u32 = (0o21 << 26) | (0o11 << 21);
6052 assert_eq!(
6053 decode(UNASSIGNED).op,
6054 crate::decode::Op::Cop1Unimplemented,
6055 "valid encoding, not Reserved"
6056 );
6057 let mut bus = Ram::new(alloc::vec![UNASSIGNED]);
6058 let mut regs = Regs::new();
6059 let mut p = Pipeline::new();
6060 p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 set
6061 let mut pc = KSEG0_PROG;
6062 for _ in 0..16 {
6063 p.advance(&mut bus, &mut regs, &mut pc);
6064 }
6065 assert_eq!(
6066 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6067 0,
6068 "no exception with CU1 set"
6069 );
6070 }
6071
6072 /// **`MOV.S` must actually move.** Decoding it is not enough — the failure
6073 /// this pins is a *silent no-op*, which is invisible to every test that
6074 /// only checks for an absent exception.
6075 ///
6076 /// The encoding is the one the correlated capture found in the delay slot
6077 /// of the failing n64-systemtest thunk's `jr $ra` (ledger C-10): with the
6078 /// move doing nothing, the callee's result never reached its caller and
6079 /// ~250 FP results were reported against a register the instruction under
6080 /// test never wrote.
6081 ///
6082 /// The destination is seeded with a value that differs from the source in
6083 /// **both halves**, so neither a no-op nor a half-width copy passes.
6084 #[test]
6085 fn mov_s_copies_the_low_word_and_a_no_op_would_fail_here() {
6086 use crate::cop0::reg;
6087 /// `MOV.S $f0, $f4` — fmt 16, fs 4, fd 0, funct 6.
6088 const MOV_S: u32 = 0x4600_2006;
6089
6090 let mut bus = Ram::new(alloc::vec![MOV_S]);
6091 let mut regs = Regs::new();
6092 let mut p = Pipeline::new();
6093 p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 | FR
6094 p.fpr.write_raw(4, 0x0011_0011_4000_0000); // source: 2.0f
6095 p.fpr.write_raw(0, 0xDEAD_BEEF_1122_3344); // destination: junk
6096
6097 let mut pc = KSEG0_PROG;
6098 for _ in 0..16 {
6099 p.advance(&mut bus, &mut regs, &mut pc);
6100 }
6101
6102 // The WHOLE register, not just the low word. `MOV.S` is a 64-bit bit
6103 // move (C-10), so checking only the low half passes against a formatted
6104 // half-copy -- which is what the destination's junk upper word is here
6105 // to catch.
6106 assert_eq!(
6107 p.fpr.read_raw(0),
6108 0x0011_0011_4000_0000,
6109 "MOV.S copies fs entire, upper half included"
6110 );
6111 assert_eq!(
6112 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6113 0,
6114 "MOV.S raises nothing"
6115 );
6116 }
6117
6118 /// **The documented FPU rates are actually charged.**
6119 ///
6120 /// `fpu::delay_cycles` transcribes UM Table 7-14 and its own test asserts
6121 /// the numbers, but a table nothing consults is inert — the failure mode
6122 /// here is a correct table that is simply never called. So this drives real
6123 /// instructions and reads the interlock back.
6124 ///
6125 /// The two cases are chosen to differ: `MUL.D` is 8 cycles and `MOV.S` is 1,
6126 /// i.e. no stall at all. A blanket stall on every COP1 op would pass a
6127 /// `MUL.D`-only test and fail this one.
6128 #[test]
6129 fn the_documented_fpu_rates_are_charged_as_stalls() {
6130 use crate::cop0::reg;
6131 /// fmt in bits 25:21, fs 4, ft 2, fd 0.
6132 const fn fp(fmt: u32, funct: u32) -> u32 {
6133 (0o21 << 26) | (fmt << 21) | (2 << 16) | (4 << 11) | funct
6134 }
6135 // MUL.D (fmt 17, funct 2) = 8 PCycles; MOV.S (fmt 16, funct 6) = 1, so
6136 // it must add nothing.
6137 for (word, want) in [(fp(0o21, 2), Some(Interlock::Mci)), (fp(0o20, 6), None)] {
6138 let mut bus = Ram::new(alloc::vec![word]);
6139 let mut regs = Regs::new();
6140 let mut p = Pipeline::new();
6141 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6142 p.fpr.write_d(4, true, 2.0f64.to_bits());
6143 p.fpr.write_d(2, true, 3.0f64.to_bits());
6144 let mut pc = KSEG0_PROG;
6145
6146 // Run until the instruction reaches WB, where COP1 arithmetic
6147 // executes and the delay is charged.
6148 let mut seen = None;
6149 for _ in 0..8 {
6150 p.advance(&mut bus, &mut regs, &mut pc);
6151 if p.stalled_by() == Some(Interlock::Mci) {
6152 seen = Some(Interlock::Mci);
6153 break;
6154 }
6155 }
6156 assert_eq!(
6157 seen, want,
6158 "word {word:#010x}: a multi-cycle FPU op must stall and a \
6159 single-cycle one must not"
6160 );
6161 }
6162 }
6163
6164 /// `NEG.S` and `ABS.S` share the arm `MOV.S` was missing from, and are just
6165 /// as silent when absent: both leave a plausible-looking value behind.
6166 #[test]
6167 fn neg_s_and_abs_s_execute_rather_than_no_op() {
6168 use crate::cop0::reg;
6169 /// fmt 16, fs 4, fd 0.
6170 const fn fp(funct: u32) -> u32 {
6171 (0o21 << 26) | (0o20 << 21) | (4 << 11) | funct
6172 }
6173
6174 for (funct, input, want) in [
6175 (7u32, 0x4000_0000u32, 0xC000_0000u32), // NEG.S: 2.0 -> -2.0
6176 (5, 0xC000_0000, 0x4000_0000), // ABS.S: -2.0 -> 2.0
6177 ] {
6178 let mut bus = Ram::new(alloc::vec![fp(funct)]);
6179 let mut regs = Regs::new();
6180 let mut p = Pipeline::new();
6181 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6182 p.fpr.write_s(4, true, input);
6183 p.fpr.write_s(0, true, 0x1122_3344);
6184 let mut pc = KSEG0_PROG;
6185 for _ in 0..16 {
6186 p.advance(&mut bus, &mut regs, &mut pc);
6187 }
6188 assert_eq!(p.fpr.read_s(0, true), want, "funct {funct} did not execute");
6189 }
6190 }
6191
6192 // --- Enabled FP traps (T-13-002) ----------------------------------------
6193
6194 /// `ADD.S $f4, $f0, $f2` — the encoding the FP-trap tests drive.
6195 const ADD_S_F4_F0_F2: u32 = 0x4602_0100;
6196 /// `FCSR.Enable` for Invalid Operation (bit 11).
6197 const ENABLE_INVALID: u32 = 1 << 11;
6198 /// `FCSR.Cause` for Invalid Operation (bit 16).
6199 const CAUSE_INVALID: u32 = 1 << 16;
6200 /// `FCSR.Flags` (sticky) for Invalid Operation (bit 6).
6201 const FLAG_INVALID: u32 = 1 << 6;
6202
6203 /// Run `ADD.S $f4, $f0, $f2` on `inf + (-inf)` — an Invalid Operation —
6204 /// with `FCSR` preloaded, and report what the machine ended up in.
6205 fn run_invalid_add_s(fcsr: u32) -> (Pipeline, u64) {
6206 use crate::cop0::reg;
6207 let mut bus = Ram::new(alloc::vec![ADD_S_F4_F0_F2]);
6208 let mut regs = Regs::new();
6209 let mut p = Pipeline::new();
6210 p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 | FR
6211 p.cop1.ctc1(31, fcsr);
6212 p.fpr.write_s(0, true, 0x7F80_0000); // +inf
6213 p.fpr.write_s(2, true, 0xFF80_0000); // -inf
6214 p.fpr.write_raw(4, 0x1122_3344_5566_7788); // untouched-if-trapped marker
6215 let mut pc = KSEG0_PROG;
6216 for _ in 0..24 {
6217 p.advance(&mut bus, &mut regs, &mut pc);
6218 }
6219 let code = (p.cop0.read(reg::CAUSE) >> 2) & 0x1F;
6220 (p, code)
6221 }
6222
6223 /// With the Invalid enable **clear**, the operation completes: `fd` is
6224 /// written, both `Cause` and the sticky `Flags` record it, and nothing
6225 /// raises. This is the control for the trap test below — without it, a
6226 /// pipeline that raised on *every* invalid operation would pass that test.
6227 #[test]
6228 fn a_masked_fp_condition_completes_and_sets_both_cause_and_flags() {
6229 let (p, code) = run_invalid_add_s(0);
6230 assert_eq!(code, 0, "masked: no exception");
6231 assert_ne!(
6232 p.fpr.read_s(4, true),
6233 0x5566_7788,
6234 "fd must be written when no trap is taken"
6235 );
6236 let fcsr = p.cop1.fcsr();
6237 assert_ne!(fcsr & CAUSE_INVALID, 0, "Cause.V set");
6238 assert_ne!(fcsr & FLAG_INVALID, 0, "sticky Flags.V set");
6239 }
6240
6241 /// With the enable **set**, the same operation traps: `ExcCode` is 15, `fd`
6242 /// keeps its old value, and `Cause` records the condition.
6243 #[test]
6244 fn an_enabled_fp_condition_raises_and_leaves_the_destination_alone() {
6245 use crate::cop0::reg;
6246 let (p, code) = run_invalid_add_s(ENABLE_INVALID);
6247 assert_eq!(code, crate::exception::exc_code::FPE, "ExcCode 15 (FPE)");
6248 assert_eq!(
6249 p.fpr.read_raw(4),
6250 0x1122_3344_5566_7788,
6251 "a trapped operation must not write fd"
6252 );
6253 assert_ne!(p.cop1.fcsr() & CAUSE_INVALID, 0, "Cause.V set");
6254 assert_eq!(
6255 (p.cop0.read(reg::CAUSE) >> 28) & 0b11,
6256 0,
6257 "Cause.CE is 0 for an FP exception, not the coprocessor number"
6258 );
6259 }
6260
6261 /// **The sticky `Flags` field is NOT updated on a trap** — only `Cause` is.
6262 ///
6263 /// Split from the test above deliberately. Both come from the same
6264 /// `Flags::to_fcsr_bits` value, so writing the whole thing on the trap path
6265 /// is the natural implementation and is wrong; it passes every assertion
6266 /// about the exception itself and only shows up when `FCSR` is read back,
6267 /// which is exactly what n64-systemtest does.
6268 #[test]
6269 fn a_trapped_operation_does_not_accumulate_into_the_sticky_flags() {
6270 let (p, _) = run_invalid_add_s(ENABLE_INVALID);
6271 assert_eq!(
6272 p.cop1.fcsr() & FLAG_INVALID,
6273 0,
6274 "Flags must be left alone when the trap is taken"
6275 );
6276 }
6277
6278 /// A trapped FP operation does not retire, so it must not tick `Random`.
6279 ///
6280 /// `Random` is decremented in the retirement tail of `WB`, which is also
6281 /// where the FP write-back happens — so an implementation that raises the
6282 /// exception but falls through keeps counting an instruction that never
6283 /// completed.
6284 ///
6285 /// **Asserted on the trap cycle specifically.** Comparing total `retired`
6286 /// between a trapping and a non-trapping run was tried first and proved
6287 /// nothing: the trap flushes the pipe and redirects `PC`, so the totals
6288 /// differ over a fixed cycle budget whether or not the trapping instruction
6289 /// itself retired. That version passed with the fix removed.
6290 #[test]
6291 fn a_trapped_fp_operation_does_not_retire() {
6292 use crate::cop0::reg;
6293 let mut bus = Ram::new(alloc::vec![ADD_S_F4_F0_F2]);
6294 let mut regs = Regs::new();
6295 let mut p = Pipeline::new();
6296 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6297 p.cop1.ctc1(31, ENABLE_INVALID);
6298 p.fpr.write_s(0, true, 0x7F80_0000);
6299 p.fpr.write_s(2, true, 0xFF80_0000);
6300
6301 let mut pc = KSEG0_PROG;
6302 let mut saw_trap = false;
6303 for _ in 0..24 {
6304 let retired_before = p.retired;
6305 let random_before = p.cop0.read(reg::RANDOM);
6306 p.advance(&mut bus, &mut regs, &mut pc);
6307 let code = (p.cop0.read(reg::CAUSE) >> 2) & 0x1F;
6308 if !saw_trap && code == crate::exception::exc_code::FPE {
6309 saw_trap = true;
6310 assert_eq!(
6311 p.retired, retired_before,
6312 "the trapping instruction must not retire on the trap cycle"
6313 );
6314 assert_eq!(
6315 p.cop0.read(reg::RANDOM),
6316 random_before,
6317 "and must not tick Random"
6318 );
6319 }
6320 }
6321 assert!(saw_trap, "no FP trap was taken -- the test proved nothing");
6322 }
6323
6324 /// **A `MOV.S` must not disturb `FCSR.Cause`.**
6325 ///
6326 /// This is a regression test for a real defect. `MOV`/`ABS`/`NEG` were
6327 /// first written to clear the `Cause` field, on no evidence — and because
6328 /// the compiler emits `MOV.fmt` to move an FP return value, a `MOV` almost
6329 /// always sits between an arithmetic operation and the `CFC1` that reads
6330 /// its result. It therefore erased the very bits the program was about to
6331 /// inspect, costing 112 n64-systemtest assertions.
6332 ///
6333 /// The signature was distinctive and worth recording: the suite reported
6334 /// `flags: inexact` with `causes: ""` — the sticky half surviving and the
6335 /// per-operation half gone. That shape means *a later instruction
6336 /// overwrote it*, not that the flag was never raised.
6337 ///
6338 /// The sequence below is exactly the shape a compiled FP call has:
6339 /// arithmetic, then a move of the result.
6340 #[test]
6341 fn a_following_mov_s_leaves_the_cause_field_of_a_previous_operation_intact() {
6342 use crate::cop0::reg;
6343 /// `ADD.S $f4, $f0, $f2` then `MOV.S $f6, $f4`.
6344 const MOV_S_F6_F4: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 11) | (6 << 6) | 6;
6345 /// `FCSR.Cause` for Inexact (bit 12).
6346 const CAUSE_INEXACT: u32 = 1 << 12;
6347
6348 let mut bus = Ram::new(alloc::vec![ADD_S_F4_F0_F2, MOV_S_F6_F4]);
6349 let mut regs = Regs::new();
6350 let mut p = Pipeline::new();
6351 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6352 // f32::MAX + 1.0 -- the value is unchanged and the operation is
6353 // inexact, which is the n64-systemtest case from ledger C-11.
6354 p.fpr.write_s(0, true, f32::MAX.to_bits());
6355 p.fpr.write_s(2, true, 1.0f32.to_bits());
6356
6357 let mut pc = KSEG0_PROG;
6358 for _ in 0..8 {
6359 p.advance(&mut bus, &mut regs, &mut pc);
6360 }
6361 assert_ne!(
6362 p.cop1.fcsr() & CAUSE_INEXACT,
6363 0,
6364 "ADD.S must raise Cause.Inexact in the first place"
6365 );
6366 for _ in 0..16 {
6367 p.advance(&mut bus, &mut regs, &mut pc);
6368 }
6369 assert_eq!(
6370 p.fpr.read_s(6, true),
6371 p.fpr.read_s(4, true),
6372 "the MOV.S did run"
6373 );
6374 assert_ne!(
6375 p.cop1.fcsr() & CAUSE_INEXACT,
6376 0,
6377 "and must not have cleared the ADD.S's Cause"
6378 );
6379 }
6380
6381 /// **A later COP1 operation clears a stale `Cause.E` (bit 17).**
6382 ///
6383 /// `Cause` is bits **17:12** and is replaced wholesale by each operation.
6384 /// The mask here originally covered only 16:12, so the unimplemented-
6385 /// operation bit — which has no `Enable` and no sticky `Flags` twin, and so
6386 /// is only ever cleared by that mask — stayed set forever once raised.
6387 /// Software reading `FCSR` after a perfectly good conversion would still
6388 /// see the previous failure.
6389 ///
6390 /// Found by a review bot, not by this suite, which had no case that raised
6391 /// bit 17 and then ran another COP1 instruction.
6392 #[test]
6393 fn a_later_operation_clears_a_stale_unimplemented_cause() {
6394 use crate::cop0::reg;
6395 /// `ADD.S $f4, $f0, $f2` — an ordinary, entirely successful operation.
6396 const ADD_S: u32 = 0x4602_0100;
6397 /// `FCSR.Cause.E`, bit 17.
6398 const CAUSE_E: u32 = 1 << 17;
6399
6400 let mut bus = Ram::new(alloc::vec![ADD_S]);
6401 let mut regs = Regs::new();
6402 let mut p = Pipeline::new();
6403 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6404 // Pre-set the bit, as a previous unimplemented operation would have.
6405 p.cop1.ctc1(31, CAUSE_E);
6406 p.fpr.write_s(0, true, 1.0f32.to_bits());
6407 p.fpr.write_s(2, true, 2.0f32.to_bits());
6408
6409 let mut pc = KSEG0_PROG;
6410 for _ in 0..16 {
6411 p.advance(&mut bus, &mut regs, &mut pc);
6412 }
6413 assert_eq!(p.fpr.read_s(4, true), 3.0f32.to_bits(), "the ADD.S ran");
6414 assert_eq!(
6415 p.cop1.fcsr() & CAUSE_E,
6416 0,
6417 "a successful operation must clear the whole Cause field"
6418 );
6419 }
6420
6421 /// `C.cond.fmt` writes `FCSR.C` and **no FPR at all**.
6422 ///
6423 /// Both halves matter. A compare that also wrote `fd` would corrupt a
6424 /// register the program never named, and one that computed the right
6425 /// condition without storing it leaves every dependent branch wrong.
6426 #[test]
6427 fn a_compare_writes_the_fcsr_condition_and_leaves_the_registers_alone() {
6428 use crate::cop0::reg;
6429 /// `FCSR.C`, bit 23.
6430 const FCSR_C: u32 = 1 << 23;
6431 /// `C.EQ.S $f0, $f2` — fmt 16, funct 0o62 (cond 2 = EQ).
6432 const C_EQ_S: u32 = (0o21 << 26) | (0o20 << 21) | (2 << 16) | 0o62;
6433
6434 for (a, b, want) in [(1.0f32, 1.0f32, true), (1.0, 2.0, false)] {
6435 let mut bus = Ram::new(alloc::vec![C_EQ_S]);
6436 let mut regs = Regs::new();
6437 let mut p = Pipeline::new();
6438 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6439 // Start with the condition at the OPPOSITE of the expected result,
6440 // so "wrote the right value" is distinguishable from "left it".
6441 p.cop1.ctc1(31, if want { 0 } else { FCSR_C });
6442 p.fpr.write_s(0, true, a.to_bits());
6443 p.fpr.write_s(2, true, b.to_bits());
6444 p.fpr.write_raw(4, 0xDEAD_BEEF_1122_3344);
6445
6446 let mut pc = KSEG0_PROG;
6447 for _ in 0..16 {
6448 p.advance(&mut bus, &mut regs, &mut pc);
6449 }
6450 assert_eq!(p.cop1.fcsr() & FCSR_C != 0, want, "{a} == {b}");
6451 assert_eq!(
6452 p.fpr.read_raw(4),
6453 0xDEAD_BEEF_1122_3344,
6454 "a compare must not write an FPR"
6455 );
6456 }
6457 }
6458
6459 /// **`TRUNC.W.S` takes its rounding mode from the OPCODE, not `FCSR.RM`.**
6460 ///
6461 /// This is the entire difference between the `ROUND`/`TRUNC`/`CEIL`/`FLOOR`
6462 /// family and `CVT.W`/`CVT.L`, and it is invisible whenever `RM` happens to
6463 /// agree with the opcode. So `FCSR.RM` is set to round-to-nearest and the
6464 /// input chosen where nearest and truncate disagree: `-1.5` truncates to
6465 /// `-1` and rounds to `-2`.
6466 ///
6467 /// `CVT.W.S` on the same input under the same `FCSR` must give `-2`,
6468 /// proving the two families really are wired differently rather than both
6469 /// happening to truncate.
6470 #[test]
6471 fn the_fixed_mode_conversions_ignore_fcsr_rm_and_cvt_w_honors_it() {
6472 use crate::cop0::reg;
6473 /// `TRUNC.W.S $f4, $f0` — fmt 16, `fs` 0 (the zero shift is elided),
6474 /// `fd` 4, funct 0o15.
6475 const TRUNC_W_S: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 6) | 0o15;
6476 /// `CVT.W.S $f4, $f0` — fmt 16, `fs` 0, `fd` 4, funct 0o44.
6477 const CVT_W_S: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 6) | 0o44;
6478
6479 for (word, want) in [(TRUNC_W_S, -1i32), (CVT_W_S, -2)] {
6480 let mut bus = Ram::new(alloc::vec![word]);
6481 let mut regs = Regs::new();
6482 let mut p = Pipeline::new();
6483 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6484 p.cop1.ctc1(31, 0); // RM = 0, round to nearest even
6485 p.fpr.write_s(0, true, (-1.5f32).to_bits());
6486
6487 let mut pc = KSEG0_PROG;
6488 for _ in 0..16 {
6489 p.advance(&mut bus, &mut regs, &mut pc);
6490 }
6491 #[allow(clippy::cast_possible_wrap)] // reading the word back as signed
6492 let got = p.fpr.read_s(4, true) as i32;
6493 assert_eq!(got, want, "instruction {word:#010X} on -1.5");
6494 }
6495 }
6496
6497 /// `CVT.S.W` reads its source as a **32-bit integer**, which is a different
6498 /// format carried in the same `fmt` field.
6499 ///
6500 /// A decoder that admits only formats 16/17 leaves every integer-to-float
6501 /// conversion a silent no-op, and `fd` keeps whatever it had — which looks
6502 /// exactly like a plausible float.
6503 #[test]
6504 fn cvt_s_w_converts_an_integer_source() {
6505 use crate::cop0::reg;
6506 /// `CVT.S.W $f4, $f0` — fmt 20 (`.W`), `fs` 0, `fd` 4, funct 0o40.
6507 const CVT_S_W: u32 = (0o21 << 26) | (0o24 << 21) | (4 << 6) | 0o40;
6508
6509 let mut bus = Ram::new(alloc::vec![CVT_S_W]);
6510 let mut regs = Regs::new();
6511 let mut p = Pipeline::new();
6512 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6513 p.fpr.write_s(0, true, 12345u32);
6514 p.fpr.write_s(4, true, 0x1122_3344);
6515
6516 let mut pc = KSEG0_PROG;
6517 for _ in 0..16 {
6518 p.advance(&mut bus, &mut regs, &mut pc);
6519 }
6520 // Compared as BITS, not as a float: 12345.0 is exactly representable,
6521 // so this is the stricter check and it also catches a wrong-signed
6522 // zero or a NaN payload that float equality would accept.
6523 assert_eq!(
6524 p.fpr.read_s(4, true),
6525 12345.0f32.to_bits(),
6526 "the integer source must be converted, not reinterpreted"
6527 );
6528 }
6529
6530 /// **`CTC1` can raise an FP exception by itself.** Writing `FCSR` with a
6531 /// Cause bit whose Enable is also set meets the trap condition
6532 /// immediately — no arithmetic has to run.
6533 ///
6534 /// The instruction must also not retire, exactly as a trapping arithmetic
6535 /// operation does not.
6536 #[test]
6537 fn ctc1_raises_when_it_writes_a_cause_bit_that_is_enabled() {
6538 use crate::cop0::reg;
6539 /// `CTC1 $1, $f31` — rs 0o06, rt 1, fs 31.
6540 const CTC1_F31: u32 = (0o21 << 26) | (0o06 << 21) | (1 << 16) | (31 << 11);
6541 /// `Cause.overflow` (bit 14) with `Enable.overflow` (bit 9).
6542 const OVERFLOW_ARMED: u64 = (1 << 14) | (1 << 9);
6543
6544 let mut bus = Ram::new(alloc::vec![CTC1_F31]);
6545 let mut regs = Regs::new();
6546 let mut p = Pipeline::new();
6547 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6548 regs.write(1, OVERFLOW_ARMED);
6549 let mut pc = KSEG0_PROG;
6550 for _ in 0..24 {
6551 p.advance(&mut bus, &mut regs, &mut pc);
6552 }
6553 assert_eq!(
6554 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6555 crate::exception::exc_code::FPE,
6556 "the CTC1 itself must raise"
6557 );
6558 assert_eq!(
6559 u64::from(p.cop1.fcsr()),
6560 OVERFLOW_ARMED,
6561 "and the written value stands"
6562 );
6563 }
6564
6565 /// The *enable* half is load-bearing: the same Cause bit with its Enable
6566 /// clear must NOT raise. Without this, an implementation that trapped on
6567 /// any non-zero Cause would pass the test above.
6568 #[test]
6569 fn ctc1_does_not_raise_when_the_matching_enable_is_clear() {
6570 use crate::cop0::reg;
6571 const CTC1_F31: u32 = (0o21 << 26) | (0o06 << 21) | (1 << 16) | (31 << 11);
6572
6573 let mut bus = Ram::new(alloc::vec![CTC1_F31]);
6574 let mut regs = Regs::new();
6575 let mut p = Pipeline::new();
6576 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6577 // Cause.overflow set, Enable.overflow clear; an unrelated enable set.
6578 regs.write(1, (1 << 14) | (1 << 11));
6579 let mut pc = KSEG0_PROG;
6580 for _ in 0..24 {
6581 p.advance(&mut bus, &mut regs, &mut pc);
6582 }
6583 assert_eq!(
6584 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6585 0,
6586 "a Cause bit whose Enable is clear must not trap"
6587 );
6588 }
6589
6590 /// **`DCFC1`/`DCTC1` and `DCFC2`/`DCTC2` decline in DIFFERENT ways.**
6591 ///
6592 /// The encodings are structurally identical — the doubleword control moves
6593 /// of their respective coprocessors — and it is tempting to give them one
6594 /// behavior. COP1's raise a *floating-point* exception with `FCSR.Cause`
6595 /// set to unimplemented-operation; COP2's raise *Reserved Instruction* with
6596 /// `Cause.CE = 2` and do not touch `FCSR` at all.
6597 ///
6598 /// Both are tested here together precisely because treating them uniformly
6599 /// is the natural mistake.
6600 #[test]
6601 fn the_doubleword_control_moves_decline_differently_per_coprocessor() {
6602 use crate::cop0::reg;
6603 /// `DCFC1 $1, $f0` — COP1, rs 0o03.
6604 const DCFC1: u32 = (0o21 << 26) | (0o03 << 21) | (1 << 16);
6605 /// `DCFC2 $1, $0` — COP2, rs 0o03.
6606 const DCFC2: u32 = (0o22 << 26) | (0o03 << 21) | (1 << 16);
6607
6608 // COP1: floating-point exception, FCSR.Cause = unimplemented ONLY.
6609 let mut bus = Ram::new(alloc::vec![DCFC1]);
6610 let mut regs = Regs::new();
6611 let mut p = Pipeline::new();
6612 p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 set
6613 // Pre-load unrelated cause bits: they must be cleared, not merged.
6614 p.cop1.ctc1(31, 0x0001_F000);
6615 let mut pc = KSEG0_PROG;
6616 for _ in 0..24 {
6617 p.advance(&mut bus, &mut regs, &mut pc);
6618 }
6619 assert_eq!(
6620 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6621 crate::exception::exc_code::FPE,
6622 "DCFC1 raises FPE"
6623 );
6624 assert_eq!(
6625 p.cop1.fcsr() & (0x3F << 12),
6626 crate::fpu::CAUSE_UNIMPLEMENTED,
6627 "and Cause is ONLY the unimplemented bit"
6628 );
6629
6630 // COP2: Reserved Instruction, and Cause.CE names the coprocessor.
6631 let mut bus = Ram::new(alloc::vec![DCFC2]);
6632 let mut regs = Regs::new();
6633 let mut p = Pipeline::new();
6634 // CU2 set (bit 30) as well as CU1, so this is not an unusable fault.
6635 p.cop0.set_hardware(reg::STATUS, 0x7400_0000);
6636 let mut pc = KSEG0_PROG;
6637 for _ in 0..24 {
6638 p.advance(&mut bus, &mut regs, &mut pc);
6639 }
6640 assert_eq!(
6641 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6642 crate::exception::exc_code::RI,
6643 "DCFC2 raises Reserved Instruction, not FPE"
6644 );
6645 assert_eq!(
6646 (p.cop0.read(reg::CAUSE) >> 28) & 0b11,
6647 2,
6648 "and Cause.CE names COP2 -- a plain RI would leave it zero"
6649 );
6650 }
6651
6652 /// **A `JAL` inside another jump's delay slot links to the OUTER target
6653 /// + 4**, not to its own `pc + 8`.
6654 ///
6655 /// Its own delay slot never runs — the outer jump already redirected — so
6656 /// the instruction after it is the outer target. n64-systemtest states the
6657 /// rule in its assertion text: *"JAL in delay slot writes target address+4
6658 /// of original jump into delay slot"*.
6659 ///
6660 /// The ordinary case is asserted alongside it, because the two share one
6661 /// implementation and a test of only the nested case would pass with the
6662 /// link hard-wired to `next_pc` in a way that broke normal jumps.
6663 #[test]
6664 fn a_jal_in_a_delay_slot_links_past_the_outer_target() {
6665 /// `JAL <target>` — the target is a word index within the 256 MB region.
6666 const fn jal(target_word: u32) -> u32 {
6667 (0o03 << 26) | target_word
6668 }
6669 /// `J <target>`.
6670 const fn j(target_word: u32) -> u32 {
6671 (0o02 << 26) | target_word
6672 }
6673
6674 // Ordinary: JAL at KSEG0_PROG, delay slot after it, link = pc + 8.
6675 {
6676 let outer = u32::try_from(KSEG0_PROG & 0x0FFF_FFFF).unwrap() >> 2;
6677 let mut bus = Ram::new(alloc::vec![jal(outer + 8), 0, 0, 0, 0, 0, 0, 0, 0, 0]);
6678 let mut regs = Regs::new();
6679 let mut p = Pipeline::new();
6680 let mut pc = KSEG0_PROG;
6681 for _ in 0..24 {
6682 p.advance(&mut bus, &mut regs, &mut pc);
6683 }
6684 assert_eq!(regs.read(31), KSEG0_PROG + 8, "an ordinary JAL links pc+8");
6685 }
6686
6687 // Nested: J to T, with a JAL in its delay slot. The JAL must link to
6688 // T + 4, NOT to its own address + 8.
6689 {
6690 let base = u32::try_from(KSEG0_PROG & 0x0FFF_FFFF).unwrap() >> 2;
6691 // T is eight words along; the JAL sits in the J's delay slot.
6692 let t_word = base + 8;
6693 let prog = alloc::vec![j(t_word), jal(base + 4), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
6694 let mut bus = Ram::new(prog);
6695 let mut regs = Regs::new();
6696 let mut p = Pipeline::new();
6697 let mut pc = KSEG0_PROG;
6698 for _ in 0..24 {
6699 p.advance(&mut bus, &mut regs, &mut pc);
6700 }
6701 let outer_target = KSEG0_PROG + 8 * 4;
6702 assert_eq!(
6703 regs.read(31),
6704 outer_target + 4,
6705 "the nested JAL links to the OUTER target + 4"
6706 );
6707 assert_ne!(
6708 regs.read(31),
6709 KSEG0_PROG + 4 + 8,
6710 "and specifically NOT to its own pc + 8"
6711 );
6712 }
6713 }
6714
6715 /// **COP2 is one 64-bit latch, not a register file.**
6716 ///
6717 /// The register index is ignored entirely: n64-systemtest writes with one
6718 /// index and reads back with several others — including 30 and 31 — and
6719 /// gets the same value every time. `MTC2` writes all 64 bits despite being
6720 /// nominally a 32-bit move; `MFC2` returns the low half sign-extended and
6721 /// `DMFC2` the whole thing.
6722 ///
6723 /// The index-independence is the assertion that matters: a real 32-entry
6724 /// register file passes a write-then-read-same-index test perfectly.
6725 #[test]
6726 fn cop2_is_a_single_latch_whose_register_index_is_ignored() {
6727 use crate::cop0::reg;
6728 /// `MTC2 $1, $5` — COP2 rs 0o04, rt 1, rd 5.
6729 const MTC2_R1_TO_5: u32 = (0o22 << 26) | (0o04 << 21) | (1 << 16) | (5 << 11);
6730 /// `DMFC2 $2, $30` — rs 0o01, rt 2, rd 30. A DIFFERENT index.
6731 const DMFC2_30_TO_R2: u32 = (0o22 << 26) | (0o01 << 21) | (2 << 16) | (30 << 11);
6732 /// `MFC2 $3, $31` — rs 0o00 (so no `rs` term), rt 3, rd 31. Another
6733 /// different index again.
6734 const MFC2_31_TO_R3: u32 = (0o22 << 26) | (3 << 16) | (31 << 11);
6735
6736 let mut bus = Ram::new(alloc::vec![MTC2_R1_TO_5, DMFC2_30_TO_R2, MFC2_31_TO_R3]);
6737 let mut regs = Regs::new();
6738 let mut p = Pipeline::new();
6739 // CU2 usable, or these fault before reaching the latch.
6740 p.cop0.set_hardware(reg::STATUS, 0x5000_0000);
6741 regs.write(1, 0x0123_4567_89AB_CDEF);
6742
6743 let mut pc = KSEG0_PROG;
6744 for _ in 0..32 {
6745 p.advance(&mut bus, &mut regs, &mut pc);
6746 }
6747 assert_eq!(
6748 regs.read(2),
6749 0x0123_4567_89AB_CDEF,
6750 "DMFC2 from a DIFFERENT index reads all 64 bits MTC2 wrote"
6751 );
6752 assert_eq!(
6753 regs.read(3),
6754 0xFFFF_FFFF_89AB_CDEF,
6755 "MFC2 from another index reads the low half, sign-extended"
6756 );
6757 }
6758
6759 // --- Unimplemented operation on subnormals (T-13-004) -------------------
6760
6761 /// `ADD.S $f4, $f0, $f2`.
6762 const ADD_S_SUB: u32 = 0x4602_0100;
6763 /// `FCSR.Cause.E`, bit 17 — unmaskable unimplemented operation.
6764 const CAUSE_E: u32 = 1 << 17;
6765 /// `FCSR.FS`, bit 24 — flush denormals to zero.
6766 const FCSR_FS: u32 = 1 << 24;
6767
6768 /// Run `ADD.S $f4, $f0, $f2` with the given operands and `FCSR`.
6769 fn run_add_s(fcsr: u32, a: u32, b: u32) -> Pipeline {
6770 use crate::cop0::reg;
6771 let mut bus = Ram::new(alloc::vec![ADD_S_SUB]);
6772 let mut regs = Regs::new();
6773 let mut p = Pipeline::new();
6774 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6775 p.cop1.ctc1(31, fcsr);
6776 p.fpr.write_s(0, true, a);
6777 p.fpr.write_s(2, true, b);
6778 p.fpr.write_raw(4, 0x1122_3344_5566_7788);
6779 let mut pc = KSEG0_PROG;
6780 for _ in 0..24 {
6781 p.advance(&mut bus, &mut regs, &mut pc);
6782 }
6783 p
6784 }
6785
6786 /// A **subnormal operand** raises unimplemented operation before the
6787 /// arithmetic is attempted — even with `FS` set, which does not rescue it.
6788 ///
6789 /// `FCSR` must end up with bit 17 and *nothing else*: no Invalid, no
6790 /// Inexact, and the sticky `Flags` untouched.
6791 #[test]
6792 fn a_subnormal_operand_raises_unimplemented_operation() {
6793 use crate::cop0::reg;
6794 let subnormal = 1u32; // the smallest positive subnormal
6795 for fcsr in [0, FCSR_FS] {
6796 let p = run_add_s(fcsr, subnormal, 2.0f32.to_bits());
6797 assert_eq!(
6798 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6799 crate::exception::exc_code::FPE,
6800 "fcsr={fcsr:#X}"
6801 );
6802 assert_ne!(p.cop1.fcsr() & CAUSE_E, 0, "Cause.E set");
6803 assert_eq!(
6804 p.cop1.fcsr() & !(CAUSE_E | FCSR_FS),
6805 0,
6806 "bit 17 and nothing else -- no flags, no other causes"
6807 );
6808 assert_eq!(
6809 p.fpr.read_raw(4),
6810 0x1122_3344_5566_7788,
6811 "fd must be untouched"
6812 );
6813 }
6814 }
6815
6816 /// A **subnormal result** with `FS` clear is equally refused. The operands
6817 /// here are both normal, so this is the result path and not the operand
6818 /// one — the two are separate checks and a test using a subnormal input
6819 /// would pass with the result check deleted.
6820 #[test]
6821 fn a_subnormal_result_raises_unimplemented_when_fs_is_clear() {
6822 use crate::cop0::reg;
6823 let a = 1.528_510_4e-37f32;
6824 let b = -1.539_154_3e-37f32;
6825 assert!(!crate::fpu::is_subnormal_f32(a), "operands are normal");
6826 assert!(!crate::fpu::is_subnormal_f32(b));
6827
6828 let p = run_add_s(0, a.to_bits(), b.to_bits());
6829 assert_eq!(
6830 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
6831 crate::exception::exc_code::FPE
6832 );
6833 assert_ne!(p.cop1.fcsr() & CAUSE_E, 0);
6834 }
6835
6836 /// **An underflow that rounds past the subnormal grid to ZERO is refused
6837 /// too**, not just one that lands on a subnormal.
6838 ///
6839 /// `MIN_POSITIVE * MIN_POSITIVE` is about `1.4e-76`, far below `f32`'s
6840 /// smallest subnormal, so the rounded result is plain zero and
6841 /// `is_subnormal` is false for it. Testing only that condition let every
6842 /// such case through silently — worth 22 oracle assertions.
6843 ///
6844 /// The converse matters as much and is covered by
6845 /// `a_subnormal_result_raises_unimplemented_when_fs_is_clear`: IEEE signals
6846 /// underflow only when tiny **and inexact**, so an *exact* subnormal has
6847 /// `underflow` clear. Neither condition implies the other; the
6848 /// implementation needs both.
6849 #[test]
6850 fn an_underflow_that_reaches_zero_is_refused_as_well() {
6851 use crate::cop0::reg;
6852 /// `MUL.S $f4, $f0, $f2` — fmt 16, funct 2.
6853 const MUL_S: u32 = (0o21 << 26) | (0o20 << 21) | (2 << 16) | (4 << 6) | 2;
6854
6855 let mut bus = Ram::new(alloc::vec![MUL_S]);
6856 let mut regs = Regs::new();
6857 let mut p = Pipeline::new();
6858 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6859 p.fpr.write_s(0, true, f32::MIN_POSITIVE.to_bits());
6860 p.fpr.write_s(2, true, f32::MIN_POSITIVE.to_bits());
6861
6862 let mut pc = KSEG0_PROG;
6863 for _ in 0..24 {
6864 p.advance(&mut bus, &mut regs, &mut pc);
6865 }
6866 assert_ne!(
6867 p.cop1.fcsr() & CAUSE_E,
6868 0,
6869 "an underflow to zero must raise unimplemented with FS clear"
6870 );
6871 }
6872
6873 /// With `FS` set, the same operation **flushes** — and where it flushes to
6874 /// depends on the rounding mode. These are n64-systemtest's own vectors.
6875 ///
6876 /// Round-to-nearest and toward-zero give a signed zero; a mode that rounds
6877 /// *away* from zero must give the smallest **normal** instead, because zero
6878 /// is on the wrong side of the true result. Getting that wrong yields `-0`
6879 /// in all four cases, which looks entirely reasonable.
6880 #[test]
6881 fn with_fs_set_a_subnormal_result_flushes_per_rounding_mode() {
6882 let a = 1.528_510_4e-37f32.to_bits();
6883 let b = (-1.539_154_3e-37f32).to_bits();
6884 // (RM, expected) -- the true result is a tiny NEGATIVE subnormal.
6885 for (rm, want) in [
6886 (0u32, (-0.0f32).to_bits()), // nearest
6887 (1, (-0.0f32).to_bits()), // toward zero
6888 (2, (-0.0f32).to_bits()), // toward +inf
6889 (3, (-f32::MIN_POSITIVE).to_bits()), // toward -inf: away from zero
6890 ] {
6891 let p = run_add_s(FCSR_FS | rm, a, b);
6892 assert_eq!(p.fpr.read_s(4, true), want, "RM={rm}");
6893 let fcsr = p.cop1.fcsr();
6894 assert_ne!(fcsr & (1 << 13), 0, "Cause.underflow, RM={rm}");
6895 assert_ne!(fcsr & (1 << 12), 0, "Cause.inexact, RM={rm}");
6896 assert_eq!(fcsr & CAUSE_E, 0, "not unimplemented, RM={rm}");
6897 }
6898 // ...and the mirrored operands flush the other way.
6899 let p = run_add_s(
6900 FCSR_FS | 2,
6901 (-1.528_510_4e-37f32).to_bits(),
6902 1.539_154_3e-37f32.to_bits(),
6903 );
6904 assert_eq!(
6905 p.fpr.read_s(4, true),
6906 f32::MIN_POSITIVE.to_bits(),
6907 "a positive tiny result under toward-+inf"
6908 );
6909 }
6910
6911 /// **`FS` set but underflow enabled is unimplemented, not a trap.** The
6912 /// processor cannot deliver a trapped underflow's defined result either.
6913 ///
6914 /// Easy to miss because it is the interaction of two features that each
6915 /// work: flushing works, and enabled traps work, but together they do not.
6916 #[test]
6917 fn fs_plus_an_enabled_underflow_is_unimplemented_rather_than_a_trap() {
6918 let a = 1.528_510_4e-37f32.to_bits();
6919 let b = (-1.539_154_3e-37f32).to_bits();
6920 // bit 8 = enable underflow, bit 7 = enable inexact.
6921 for enable in [1u32 << 8, 1 << 7] {
6922 let p = run_add_s(FCSR_FS | enable, a, b);
6923 assert_ne!(
6924 p.cop1.fcsr() & CAUSE_E,
6925 0,
6926 "enable={enable:#X} must give unimplemented"
6927 );
6928 }
6929 }
6930
6931 /// **The two NaN classes trap differently.** MSB clear (quiet by the
6932 /// VR4300's convention) is unimplemented; MSB set (signaling) is Invalid.
6933 ///
6934 /// Swapping them is invisible until `FCSR` is read back, and both are
6935 /// "the operation trapped", so a test asserting only that would pass either
6936 /// way. See ledger C-12.
6937 #[test]
6938 fn the_two_nan_classes_raise_different_causes() {
6939 let msb_clear = 0x7F80_0001u32; // unimplemented here
6940 let msb_set = 0x7FC0_0001u32; // signaling here -> Invalid
6941
6942 let p = run_add_s(0, msb_clear, 2.0f32.to_bits());
6943 assert_ne!(p.cop1.fcsr() & CAUSE_E, 0, "MSB clear -> unimplemented");
6944 assert_eq!(p.cop1.fcsr() & (1 << 16), 0, "and NOT invalid");
6945
6946 let p = run_add_s(0, msb_set, 2.0f32.to_bits());
6947 assert_ne!(p.cop1.fcsr() & (1 << 16), 0, "MSB set -> Cause.invalid");
6948 assert_eq!(p.cop1.fcsr() & CAUSE_E, 0, "and NOT unimplemented");
6949 }
6950
6951 /// **`ABS`/`NEG` classify their operand; `MOV` does not.** All three look
6952 /// like sign/bit manipulation, and only `MOV` actually is.
6953 ///
6954 /// n64-systemtest settles it by construction rather than description:
6955 /// `MOV.S` is driven through the cause-*preserving* harness while `ABS.S`
6956 /// and `NEG.S` go through the ordinary one. Treating all three alike was
6957 /// worth 52 assertions.
6958 #[test]
6959 fn abs_and_neg_refuse_a_subnormal_but_mov_moves_it() {
6960 use crate::cop0::reg;
6961 /// fmt 16, `fs` 0, `fd` 4.
6962 const fn unary(funct: u32) -> u32 {
6963 (0o21 << 26) | (0o20 << 21) | (4 << 6) | funct
6964 }
6965 let subnormal = 1u32;
6966
6967 for funct in [5u32, 7] {
6968 let mut bus = Ram::new(alloc::vec![unary(funct)]);
6969 let mut regs = Regs::new();
6970 let mut p = Pipeline::new();
6971 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6972 p.fpr.write_s(0, true, subnormal);
6973 let mut pc = KSEG0_PROG;
6974 for _ in 0..24 {
6975 p.advance(&mut bus, &mut regs, &mut pc);
6976 }
6977 assert_ne!(
6978 p.cop1.fcsr() & CAUSE_E,
6979 0,
6980 "funct {funct} (ABS/NEG) must refuse a subnormal"
6981 );
6982 }
6983
6984 // MOV.S is a pure move: it transports the subnormal untouched and
6985 // raises nothing at all.
6986 let mut bus = Ram::new(alloc::vec![unary(6)]);
6987 let mut regs = Regs::new();
6988 let mut p = Pipeline::new();
6989 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
6990 p.fpr.write_s(0, true, subnormal);
6991 let mut pc = KSEG0_PROG;
6992 for _ in 0..24 {
6993 p.advance(&mut bus, &mut regs, &mut pc);
6994 }
6995 assert_eq!(
6996 p.fpr.read_s(4, true),
6997 subnormal,
6998 "MOV.S moves the subnormal"
6999 );
7000 assert_eq!(p.cop1.fcsr(), 0, "and raises nothing");
7001 }
7002
7003 /// **A `.L` conversion refuses a magnitude of `2^53` or more**, which is
7004 /// far narrower than `i64`.
7005 ///
7006 /// The threshold is bracketed rather than assumed: `9007198717870080`
7007 /// converts and `9007199254740992` (`2^53`) does not, and both are
7008 /// comfortably inside `i64`. A test using only a huge value would pass with
7009 /// the limit set anywhere between `2^53` and `i64::MAX`.
7010 ///
7011 /// The same limit applies to `.W`, where it is **unobservable**: `2^53` is
7012 /// far outside `i32`, so both paths end in unimplemented. Guarding it on
7013 /// the target width was tried and removed — a branch no test can
7014 /// distinguish is a branch that will rot.
7015 #[test]
7016 fn a_long_conversion_refuses_two_to_the_fifty_three_but_not_just_below() {
7017 use crate::cop0::reg;
7018 /// `CVT.L.D $f4, $f0` — fmt 17 (`.D`), `fs` 0, `fd` 4, funct 0o45.
7019 const CVT_L_D: u32 = (0o21 << 26) | (0o21 << 21) | (4 << 6) | 0o45;
7020
7021 for (src, want_unimplemented) in [
7022 (9_007_198_717_870_080.0f64, false),
7023 (9_007_199_254_740_992.0f64, true), // 2^53
7024 (-9_007_199_254_740_992.0f64, true),
7025 (-9_007_198_717_870_080.0f64, false),
7026 ] {
7027 let mut bus = Ram::new(alloc::vec![CVT_L_D]);
7028 let mut regs = Regs::new();
7029 let mut p = Pipeline::new();
7030 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
7031 p.fpr.write_d(0, true, src.to_bits());
7032 let mut pc = KSEG0_PROG;
7033 for _ in 0..24 {
7034 p.advance(&mut bus, &mut regs, &mut pc);
7035 }
7036 let raised = p.cop1.fcsr() & CAUSE_E != 0;
7037 assert_eq!(raised, want_unimplemented, "{src:e}");
7038 if !want_unimplemented {
7039 #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
7040 let want = src as i64 as u64;
7041 assert_eq!(p.fpr.read_d(4, true), want, "{src:e} must convert");
7042 }
7043 }
7044 }
7045
7046 /// An out-of-range float-to-integer conversion is **unimplemented**, not
7047 /// Invalid. IEEE says Invalid; this processor declines instead, and
7048 /// `fpu::to_i32` reports the IEEE answer that must be translated.
7049 #[test]
7050 fn an_out_of_range_integer_conversion_is_unimplemented_not_invalid() {
7051 use crate::cop0::reg;
7052 /// `CVT.W.S $f4, $f0` — fmt 16, `fs` 0, `fd` 4, funct 0o44.
7053 const CVT_W_S: u32 = (0o21 << 26) | (0o20 << 21) | (4 << 6) | 0o44;
7054
7055 for src in [1e30f32, f32::INFINITY, f32::from_bits(0x7FC0_0000)] {
7056 let mut bus = Ram::new(alloc::vec![CVT_W_S]);
7057 let mut regs = Regs::new();
7058 let mut p = Pipeline::new();
7059 p.cop0.set_hardware(reg::STATUS, 0x3400_0000);
7060 p.fpr.write_s(0, true, src.to_bits());
7061 let mut pc = KSEG0_PROG;
7062 for _ in 0..24 {
7063 p.advance(&mut bus, &mut regs, &mut pc);
7064 }
7065 assert_ne!(p.cop1.fcsr() & CAUSE_E, 0, "{src:e} -> unimplemented");
7066 assert_eq!(p.cop1.fcsr() & (1 << 16), 0, "{src:e} -> NOT invalid");
7067 assert_eq!(
7068 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7069 crate::exception::exc_code::FPE
7070 );
7071 }
7072 }
7073
7074 // --- CACHE (T-12-005) ---------------------------------------------------
7075
7076 /// `CACHE` must **not** raise. IPL3 and libdragon both issue it, so a
7077 /// reserved-instruction exception here blocks every real ROM — which is why
7078 /// this was called out as a hard blocker before it was implemented.
7079 #[test]
7080 fn cache_executes_instead_of_raising() {
7081 use crate::cop0::reg;
7082 // CACHE op=0, 0($1) with $1 = KSEG0 base.
7083 let prog = alloc::vec![lui_kseg0(1), ld_st(0o57, 1, 0, 0x100)];
7084 assert_eq!(decode(prog[1]).op, crate::decode::Op::Cache);
7085 let mut bus = Ram::new(prog);
7086 let mut regs = Regs::new();
7087 let mut p = Pipeline::new();
7088 p.cop0.set_hardware(reg::STATUS, 0);
7089 let mut pc = KSEG0_PROG;
7090 for _ in 0..24 {
7091 p.advance(&mut bus, &mut regs, &mut pc);
7092 }
7093 assert_eq!(
7094 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7095 0,
7096 "CACHE must not raise"
7097 );
7098 assert_eq!(p.stalled_by(), None, "and must not be stuck in an epilogue");
7099 }
7100
7101 /// `CACHE`'s `rt` slot is the **operation selector**, not a destination.
7102 /// Decoding it as a load would clobber whichever GPR the cache-op encoding
7103 /// happens to name — a spectacularly confusing bug, since the register
7104 /// destroyed depends on which cache operation was requested.
7105 #[test]
7106 fn cache_writes_no_general_register() {
7107 // op = 0b10101 = 21, which as a destination would be $21.
7108 let word = ld_st(0o57, 1, 21, 0);
7109 let d = decode(word);
7110 assert_eq!(d.op, crate::decode::Op::Cache);
7111 assert_eq!(d.dest, 0, "rt is the cache operation, not a destination");
7112
7113 let prog = alloc::vec![lui_kseg0(1), addiu_zero(21, 0x33), word];
7114 let mut bus = Ram::new(prog);
7115 let mut regs = Regs::new();
7116 let mut p = Pipeline::new();
7117 p.cop0.set_hardware(crate::cop0::reg::STATUS, 0);
7118 let mut pc = KSEG0_PROG;
7119 for _ in 0..32 {
7120 p.advance(&mut bus, &mut regs, &mut pc);
7121 }
7122 assert_eq!(regs.read(21), 0x33, "$21 must survive CACHE op 21");
7123 }
7124
7125 /// An **`Index_*`** `CACHE` op addresses the cache by index and **must not
7126 /// translate**, so it cannot fault however unmapped the address is.
7127 ///
7128 /// This matters at boot: cache-initialization code walks every index with an
7129 /// arbitrary base address, and translating would raise a TLB refill on the
7130 /// first one — before any mapping exists to satisfy it.
7131 #[test]
7132 fn an_index_cache_op_never_faults_however_unmapped_the_address() {
7133 use crate::cop0::reg;
7134 for op in [0u16, 4, 8] {
7135 // op4..2 = 0, 1, 2 -> Index_Invalidate / Load_Tag / Store_Tag.
7136 assert!((op >> 2) < 3, "op {op} must be an Index form");
7137 let mut bus = Ram::new(alloc::vec![ld_st(0o57, 0, u32::from(op), 0x4000)]);
7138 let mut regs = Regs::new();
7139 let mut p = Pipeline::new();
7140 p.cop0.set_hardware(reg::STATUS, 0);
7141 let mut pc = KSEG0_PROG;
7142 for _ in 0..24 {
7143 p.advance(&mut bus, &mut regs, &mut pc);
7144 }
7145 assert_eq!(
7146 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7147 0,
7148 "Index op {op} must not raise -- it never consults the TLB"
7149 );
7150 }
7151 }
7152
7153 /// A **`Hit_*`** `CACHE` op translates, so it raises a TLB fault on an
7154 /// unmapped address — it is defined in terms of *"the specified address"*.
7155 #[test]
7156 fn a_hit_cache_op_on_an_unmapped_address_faults() {
7157 use crate::cop0::reg;
7158 // op = 16 -> op4..2 = 4 = Hit_Invalidate.
7159 let mut bus = Ram::new(alloc::vec![ld_st(0o57, 0, 16, 0x4000)]);
7160 let mut regs = Regs::new();
7161 let mut p = Pipeline::new();
7162 p.cop0.set_hardware(reg::STATUS, 0);
7163 let mut pc = KSEG0_PROG;
7164
7165 let mut cycles = 0;
7166 while p.stalled_by() != Some(Interlock::Exception) {
7167 p.advance(&mut bus, &mut regs, &mut pc);
7168 cycles += 1;
7169 assert!(cycles < 16, "CACHE did not translate");
7170 }
7171 assert_eq!(
7172 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7173 crate::exception::exc_code::TLBL
7174 );
7175 }
7176
7177 // --- FP register file, moves and loads/stores (T-13-001) ---------------
7178
7179 /// `MTC1` then `MFC1` round-trips through the real FP register file, and
7180 /// `MTC1` must **not** write a general register — `rt` is its source.
7181 #[test]
7182 fn mtc1_then_mfc1_round_trips_and_writes_no_gpr() {
7183 use crate::cop0::reg;
7184 const fn cop1(rs: u32, rt: u32, fs: u32) -> u32 {
7185 (0o21 << 26) | (rs << 21) | (rt << 16) | (fs << 11)
7186 }
7187 // ADDIU $1, $0, 0x55
7188 // MTC1 $1, $f4
7189 // MFC1 $2, $f4
7190 let prog = alloc::vec![addiu_zero(1, 0x55), cop1(0o04, 1, 4), cop1(0o00, 2, 4)];
7191 let mut bus = Ram::new(prog);
7192 let mut regs = Regs::new();
7193 let mut p = Pipeline::new();
7194 p.cop0.set_hardware(reg::STATUS, 0x3400_0000); // CU1 | FR
7195 let mut pc = KSEG0_PROG;
7196 for _ in 0..40 {
7197 p.advance(&mut bus, &mut regs, &mut pc);
7198 }
7199 assert_eq!(
7200 p.fpr.read_s(4, true),
7201 0x55,
7202 "MTC1 reached the FP register file"
7203 );
7204 assert_eq!(regs.read(2), 0x55, "and MFC1 read it back");
7205 assert_eq!(regs.read(1), 0x55, "$1 -- MTC1's source -- must survive");
7206 }
7207
7208 /// **`DMFC1`/`DMTC1` apply the `FR` view**, they do not move the physical
7209 /// register. UM Ch. 17's pseudocode: with `FR = 0` and an even `fs`,
7210 /// In half mode a 64-bit access addresses **FGR `fs & !1` in its
7211 /// entirety**, and the odd FGR is not part of it.
7212 ///
7213 /// This test previously asserted the *pair* model — low word in the even
7214 /// FGR, high word in the odd one — which round-trips through
7215 /// `DMTC1`/`DMFC1` perfectly and is still wrong: hardware never touches
7216 /// the odd register. See `fpr.rs` for the n64-systemtest vector that
7217 /// settles it.
7218 #[test]
7219 fn dmtc1_and_dmfc1_apply_the_fr_view_rather_than_moving_the_raw_fgr() {
7220 use crate::cop0::reg;
7221 const fn cop1(rs: u32, rt: u32, fs: u32) -> u32 {
7222 (0o21 << 26) | (rs << 21) | (rt << 16) | (fs << 11)
7223 }
7224 let mut p = Pipeline::new();
7225 // CU1 set, FR CLEAR -- the paired view.
7226 p.cop0.set_hardware(reg::STATUS, 1 << 29);
7227 // DMTC1 $1, $f2 with $1 = 0x1122_3344_5566_7788.
7228 let prog = alloc::vec![cop1(0o05, 1, 2), cop1(0o01, 2, 2)];
7229 let mut bus = Ram::new(prog);
7230 let mut regs = Regs::new();
7231 regs.write(1, 0x1122_3344_5566_7788);
7232 let mut pc = KSEG0_PROG;
7233 for _ in 0..32 {
7234 p.advance(&mut bus, &mut regs, &mut pc);
7235 }
7236
7237 // All 64 bits in FGR 2; FGR 3 is not involved.
7238 assert_eq!(
7239 p.fpr.read_raw(2),
7240 0x1122_3344_5566_7788,
7241 "the whole value lives in the even FGR"
7242 );
7243 assert_eq!(p.fpr.read_raw(3), 0, "the odd FGR is untouched");
7244 assert_eq!(regs.read(2), 0x1122_3344_5566_7788, "DMFC1 reassembles it");
7245 // And it agrees with the LDC1/SDC1 view of the same register.
7246 assert_eq!(p.fpr.read_d(2, false), 0x1122_3344_5566_7788);
7247 }
7248
7249 /// `SDC1` then `LDC1` round-trips a double through memory, and with
7250 /// `FR = 0` the value lives in an **FGR pair** — so this exercises the view
7251 /// that a direct-index register file gets wrong.
7252 #[test]
7253 fn ldc1_and_sdc1_round_trip_a_double_with_fr_clear() {
7254 use crate::cop0::reg;
7255 // LUI $1, 0x8000
7256 // SDC1 $f2, 0x100($1)
7257 // LDC1 $f4, 0x100($1)
7258 let prog = alloc::vec![
7259 lui_kseg0(1),
7260 ld_st(0o75, 1, 2, 0x100),
7261 ld_st(0o65, 1, 4, 0x100),
7262 ];
7263 let mut bus = Ram::new(prog);
7264 let mut regs = Regs::new();
7265 let mut p = Pipeline::new();
7266 // CU1 set, FR CLEAR -- the paired view.
7267 p.cop0.set_hardware(reg::STATUS, 1 << 29);
7268 p.fpr.write_d(2, false, 0x0123_4567_89AB_CDEF);
7269 let mut pc = KSEG0_PROG;
7270 for _ in 0..48 {
7271 p.advance(&mut bus, &mut regs, &mut pc);
7272 }
7273 assert_eq!(
7274 p.fpr.read_d(4, false),
7275 0x0123_4567_89AB_CDEF,
7276 "the double survived memory in the FR = 0 paired view"
7277 );
7278 writeback(&mut p, &mut bus, 0xFFFF_FFFF_8000_0100);
7279 assert_eq!(
7280 bus.read_u32(0x100),
7281 0x0123_4567,
7282 "big-endian high word first"
7283 );
7284 }
7285
7286 /// FP loads and stores obey the same alignment rules as the integer forms —
7287 /// `LDC1` needs 8-byte alignment, and a misaligned one raises `AdEL`.
7288 #[test]
7289 fn a_misaligned_ldc1_raises_an_address_error() {
7290 use crate::cop0::reg;
7291 let prog = alloc::vec![lui_kseg0(1), ld_st(0o65, 1, 4, 0x104)];
7292 let mut bus = Ram::new(prog);
7293 let mut regs = Regs::new();
7294 let mut p = Pipeline::new();
7295 p.cop0.set_hardware(reg::STATUS, 1 << 29);
7296 let mut pc = KSEG0_PROG;
7297
7298 let mut cycles = 0;
7299 while p.stalled_by() != Some(Interlock::Exception) {
7300 p.advance(&mut bus, &mut regs, &mut pc);
7301 cycles += 1;
7302 assert!(cycles < 24, "no address error raised");
7303 }
7304 assert_eq!(
7305 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7306 crate::exception::exc_code::ADEL,
7307 "a misaligned LDC1 is a load address error"
7308 );
7309 }
7310
7311 /// The FP moves and loads need `CU1` like every other COP1 instruction.
7312 #[test]
7313 fn fp_loads_need_cu1() {
7314 use crate::cop0::reg;
7315 let prog = alloc::vec![lui_kseg0(1), ld_st(0o61, 1, 4, 0x100)];
7316 let mut bus = Ram::new(prog);
7317 let mut regs = Regs::new();
7318 let mut p = Pipeline::new();
7319 p.cop0.set_hardware(reg::STATUS, 0); // CU1 CLEAR
7320 let mut pc = KSEG0_PROG;
7321
7322 let mut cycles = 0;
7323 while p.stalled_by() != Some(Interlock::Exception) {
7324 p.advance(&mut bus, &mut regs, &mut pc);
7325 cycles += 1;
7326 assert!(cycles < 24, "no exception raised");
7327 }
7328 assert_eq!(
7329 (p.cop0.read(reg::CAUSE) >> 2) & 0x1F,
7330 crate::exception::exc_code::CPU
7331 );
7332 assert_eq!((p.cop0.read(reg::CAUSE) >> 28) & 0b11, 1, "unit 1");
7333 }
7334
7335 // ---------------------------------------------------------------------
7336 // The latch-independent primitives, tested directly (ADR 0013's seam)
7337 // ---------------------------------------------------------------------
7338 //
7339 // The stage tests above reach these through the cascade, which is the right
7340 // way to test the accurate path and the wrong way to pin a *contract*. These
7341 // call the primitives directly, because the instruction-granular path will
7342 // call them with different consequences and needs the decisions to hold on
7343 // their own.
7344
7345 /// A faulting fetch must report the fault **and leave the bus untouched** —
7346 /// the access itself is what is invalid.
7347 ///
7348 /// Distinct from `an_unaligned_fetch_raises_address_error_without_realigning`
7349 /// above, which checks what `ic_stage` *does* with the fault (stamps `ic_rf`,
7350 /// declines to realign). This checks the primitive's own contract, which is
7351 /// the half a second caller depends on.
7352 #[test]
7353 fn fetch_word_reports_a_fault_without_touching_the_bus() {
7354 struct Watch {
7355 touched: bool,
7356 }
7357 impl Bus for Watch {
7358 fn read_u8(&mut self, _addr: u32) -> u8 {
7359 self.touched = true;
7360 0
7361 }
7362 fn write_u8(&mut self, _addr: u32, _val: u8) {
7363 self.touched = true;
7364 }
7365 fn read_u32(&mut self, _addr: u32) -> u32 {
7366 self.touched = true;
7367 0
7368 }
7369 }
7370 let mut bus = Watch { touched: false };
7371 let mut p = Pipeline::new();
7372
7373 assert_eq!(
7374 p.fetch_word(&mut bus, 0xFFFF_FFFF_8000_0002),
7375 Err(Exception::AddressError { store: false }),
7376 "an unaligned fetch is an address error"
7377 );
7378 assert!(
7379 !bus.touched,
7380 "the bus was accessed for a fetch that is invalid before it happens"
7381 );
7382 }
7383
7384 /// The positive case, so the test above cannot pass against a `fetch_word`
7385 /// that simply always fails.
7386 ///
7387 /// The word is a distinctive sentinel rather than zero: zero is `NOP` and is
7388 /// also what a bus returning nothing produces, so it cannot tell a real fetch
7389 /// from a missing one.
7390 #[test]
7391 fn fetch_word_returns_the_word_at_an_aligned_pc() {
7392 struct Fixed(u32);
7393 impl Bus for Fixed {
7394 fn read_u8(&mut self, _addr: u32) -> u8 {
7395 0
7396 }
7397 fn write_u8(&mut self, _addr: u32, _val: u8) {}
7398 fn read_u32(&mut self, _addr: u32) -> u32 {
7399 self.0
7400 }
7401 }
7402 const SENTINEL: u32 = 0xDEAD_BEEF;
7403 let mut bus = Fixed(SENTINEL);
7404 let mut p = Pipeline::new();
7405
7406 // KSEG1 — uncached and unmapped, so this reaches the bus directly and
7407 // tests the fetch rather than the I-cache.
7408 assert_eq!(p.fetch_word(&mut bus, 0xFFFF_FFFF_A000_0000), Ok(SENTINEL));
7409 }
7410
7411 /// `ex_gate` refuses `DCFC1`/`DCTC1` by **replacing** `FCSR.Cause`, not by
7412 /// OR-ing into it.
7413 ///
7414 /// Asserted as an *effect*: `FCSR` is seeded with unrelated cause bits first,
7415 /// and they must be gone afterwards. A test that only checked bit 17 had
7416 /// arrived would pass against an implementation that OR-ed — which is the
7417 /// behavior n64-systemtest specifically pre-loads unrelated bits to catch.
7418 #[test]
7419 fn ex_gate_replaces_the_whole_fcsr_cause_field() {
7420 /// `FCSR.Cause`, bits 17:12 — the field under test.
7421 const CAUSE: u32 = 0x3F << 12;
7422 /// Unrelated cause bits (Inexact and Overflow), seeded to be cleared.
7423 const SEEDED: u32 = 0b0001_0100 << 12;
7424
7425 use crate::cop0::reg;
7426
7427 let mut p = Pipeline::new();
7428 p.cop1.ctc1(31, SEEDED);
7429 assert_eq!(p.cop1.fcsr() & CAUSE, SEEDED, "the seed took");
7430
7431 // COP1 opcode with rs = 3 is `DCFC1`, the usable-but-unimplemented
7432 // control move. Decoded rather than hand-built so the encoding stays
7433 // owned by `decode`.
7434 let d = decode((0o21 << 26) | (3 << 21));
7435 assert_eq!(
7436 d.op,
7437 crate::decode::Op::Cop1ReservedControl,
7438 "the encoding under test must actually be the reserved control move"
7439 );
7440
7441 // `CU1` must be SET, or the coprocessor-usability check refuses first and
7442 // this would test the wrong branch.
7443 p.cop0.set_hardware(reg::STATUS, 1 << 29);
7444 assert_eq!(p.ex_gate(d), Err(Exception::FloatingPoint));
7445 assert_eq!(
7446 p.cop1.fcsr() & CAUSE,
7447 crate::fpu::CAUSE_UNIMPLEMENTED,
7448 "the seeded cause bits survived: `Cause` was OR-ed, not replaced"
7449 );
7450 }
7451
7452 /// **`ex_gate`'s first two refusals are DISJOINT, so their order cannot be
7453 /// observed — and this test is what keeps that true.**
7454 ///
7455 /// The extraction started from the accurate path's comment, which says an
7456 /// unusable coprocessor is reported as such *"even when the encoding is also
7457 /// a 64-bit one"*. Mutation-checking that against n64-systemtest changed
7458 /// nothing, and the reason is stronger than "the suite misses it":
7459 ///
7460 /// - `Op::is_64_bit` covers only CPU integer and load/store operations
7461 /// (`Dadd` … `Sdr`);
7462 /// - `unusable_coprocessor` returns `Some` only for COP0/COP1/COP2 encodings.
7463 ///
7464 /// No encoding is in both sets, so **no input can reach the second check by
7465 /// way of the first**. The ordering is unobservable rather than merely
7466 /// untested, which is why the oracle was silent.
7467 ///
7468 /// Swept over a structured slice of the encoding space rather than asserted
7469 /// against a copy of either list: a duplicated list stops covering the thing
7470 /// it duplicates the moment one side grows, which is the failure this test
7471 /// exists to catch. If `Dmfc1`/`Dmtc1`/`Dmfc0`/`Dmtc0` are ever added to
7472 /// `is_64_bit` — they *are* 64-bit operations, and whether their omission is
7473 /// correct is an open question, not a claim made here — this fails, and
7474 /// whoever adds them has to decide what the precedence should be with the
7475 /// case in front of them.
7476 #[test]
7477 fn ex_gates_first_two_refusals_cannot_both_apply() {
7478 use crate::cop0::reg;
7479
7480 let mut p = Pipeline::new();
7481 // 32-bit USER mode with every `CU` bit clear, so *every* coprocessor
7482 // encoding is unusable and `sixty_four_bit_is_reserved` holds. Kernel
7483 // mode would exempt COP0 and hide half the question.
7484 p.cop0.set_hardware(reg::STATUS, 0b10 << 3);
7485 assert!(
7486 p.sixty_four_bit_is_reserved(),
7487 "the sweep is meaningless unless 64-bit really is reserved here"
7488 );
7489
7490 let mut coprocessor = 0u32;
7491 let mut sixty_four = 0u32;
7492 for opcode in 0..64u32 {
7493 for rs in 0..32u32 {
7494 for funct in 0..64u32 {
7495 let d = decode((opcode << 26) | (rs << 21) | funct);
7496 let cop = p.unusable_coprocessor(d).is_some();
7497 let wide = d.op.is_64_bit();
7498 coprocessor += u32::from(cop);
7499 sixty_four += u32::from(wide);
7500 assert!(
7501 !(cop && wide),
7502 "opcode {opcode:#o} rs {rs:#o} funct {funct:#o} decodes to \
7503 {:?}, which is BOTH 64-bit and on an unusable coprocessor \
7504 -- `ex_gate`'s first two checks are no longer disjoint, so \
7505 their order is now observable and has to be justified \
7506 against the manual rather than inherited",
7507 d.op
7508 );
7509 }
7510 }
7511 }
7512 // A sweep that decoded nothing interesting would pass just as
7513 // convincingly. Both populations must be non-empty for the disjointness
7514 // above to be a claim about anything.
7515 assert!(
7516 coprocessor > 0 && sixty_four > 0,
7517 "the sweep found {coprocessor} coprocessor and {sixty_four} 64-bit \
7518 encodings; with either at zero it proves nothing"
7519 );
7520 }
7521}
7522
7523/// The commit census must classify what it claims to classify.
7524///
7525/// A census that silently called every instruction SIMPLE would report a
7526/// wonderful 100% and send a fast commit path off to serve loads it cannot
7527/// serve. The measured share on Super Mario 64 (96.81% SIMPLE, 3.11% MEM) is
7528/// low enough on MEM to be worth disbelieving until the classifier is witnessed
7529/// doing its job.
7530#[cfg(all(test, feature = "work-counters"))]
7531mod commit_census_tests {
7532 use super::{Pipeline, commit_class as cc};
7533 use crate::exec::{Cop0Access, MemOp, WriteBack};
7534 use crate::mem::{LoadKind, StoreKind};
7535
7536 #[test]
7537 fn reachable_commit_classes_are_distinct() {
7538 let mut p = Pipeline::new();
7539
7540 // A plain ALU result.
7541 p.count_commit(None, None, WriteBack::Gpr { dest: 3, value: 1 });
7542 // A branch: commits nothing.
7543 p.count_commit(None, None, WriteBack::None);
7544 // A load. `mem` is what makes it not-simple, whatever the write-back says.
7545 p.count_commit(
7546 None,
7547 Some(MemOp::Load {
7548 kind: LoadKind::SignedWord,
7549 addr: 0x8000_0000,
7550 dest: 4,
7551 }),
7552 WriteBack::None,
7553 );
7554 // A store.
7555 p.count_commit(
7556 None,
7557 Some(MemOp::Store {
7558 kind: StoreKind::Word,
7559 addr: 0x8000_0000,
7560 value: 0,
7561 }),
7562 WriteBack::None,
7563 );
7564 // A multiply.
7565 p.count_commit(
7566 None,
7567 None,
7568 WriteBack::HiLo(crate::alu::HiLo { hi: 0, lo: 0 }),
7569 );
7570
7571 let c = p.commit_census();
7572 assert_eq!(
7573 c[cc::SIMPLE],
7574 2,
7575 "GPR write and no-write-back are both simple"
7576 );
7577 assert_eq!(c[cc::MEM], 2, "a load and a store both have a DC access");
7578 assert_eq!(c[cc::HILO], 1, "a multiply writes HI/LO");
7579 assert_eq!(c[cc::COP], 0);
7580 assert_eq!(c[cc::OTHER], 0);
7581 }
7582
7583 /// A COP0 access outranks a memory op: an instruction with both is NOT
7584 /// something a direct commit path may serve, and counting it as MEM would
7585 /// understate the class that must keep using `wb_stage`.
7586 #[test]
7587 fn a_cop_access_outranks_a_memory_op() {
7588 let mut p = Pipeline::new();
7589 p.count_commit(
7590 Some(Cop0Access::Read {
7591 src: 12,
7592 dest: 3,
7593 wide: false,
7594 }),
7595 Some(MemOp::Store {
7596 kind: StoreKind::Word,
7597 addr: 0x8000_0000,
7598 value: 0,
7599 }),
7600 WriteBack::None,
7601 );
7602 let c = p.commit_census();
7603 assert_eq!(c[cc::COP], 1, "the COP class must win");
7604 assert_eq!(c[cc::MEM], 0);
7605 assert_eq!(c[cc::SIMPLE], 0);
7606 }
7607
7608 #[test]
7609 fn unretired_or_aborted_instructions_do_not_increment_census() {
7610 let mut p = Pipeline::new();
7611 let mut regs = crate::regs::Regs::new();
7612 // A latch with abort set: does not retire
7613 p.dc_wb = super::Latch {
7614 occupied: true,
7615 abort: Some(super::Exception::Breakpoint),
7616 ..Default::default()
7617 };
7618 p.wb_stage(&mut regs);
7619 let c = p.commit_census();
7620 assert_eq!(p.retired, 0);
7621 assert_eq!(c[cc::SIMPLE], 0);
7622 assert_eq!(c[cc::MEM], 0);
7623 assert_eq!(c[cc::COP], 0);
7624 assert_eq!(c[cc::HILO], 0);
7625 assert_eq!(c[cc::OTHER], 0);
7626 }
7627}