veridian_kernel/browser/
js_lexer.rs

1//! JavaScript Lexer
2//!
3//! Tokenizes JavaScript source code into a stream of tokens. Handles
4//! string escapes, number parsing (integer and decimal to 32.32 fixed-point),
5//! and automatic semicolon insertion (ASI).
6
7#![allow(dead_code)]
8
9use alloc::{
10    string::{String, ToString},
11    vec::Vec,
12};
13
14/// JavaScript number type: 32.32 fixed-point (i64)
15pub type JsNumber = i64;
16
17/// Fractional bits for JsNumber
18pub const JS_FRAC_BITS: u32 = 32;
19
20/// Convert integer to JsNumber
21#[inline]
22pub const fn js_int(v: i64) -> JsNumber {
23    v << JS_FRAC_BITS
24}
25
26/// Convert JsNumber to integer (truncate)
27#[inline]
28pub const fn js_to_int(v: JsNumber) -> i64 {
29    v >> JS_FRAC_BITS
30}
31
32/// Check if a JsNumber has no fractional part
33#[inline]
34pub const fn js_is_integer(v: JsNumber) -> bool {
35    (v & ((1i64 << JS_FRAC_BITS) - 1)) == 0
36}
37
38/// JsNumber zero
39pub const JS_ZERO: JsNumber = 0;
40
41/// JsNumber one
42pub const JS_ONE: JsNumber = 1i64 << JS_FRAC_BITS;
43
44/// JsNumber NaN sentinel (max i64 -- not a real number)
45pub const JS_NAN: JsNumber = i64::MAX;
46
47// ---------------------------------------------------------------------------
48// Token type
49// ---------------------------------------------------------------------------
50
51/// JavaScript token
52#[derive(Debug, Clone, PartialEq)]
53pub enum JsToken {
54    // Literals
55    Identifier(String),
56    Number(JsNumber),
57    StringLiteral(String),
58    Bool(bool),
59    Null,
60    Undefined,
61
62    // Arithmetic operators
63    Plus,
64    Minus,
65    Star,
66    Slash,
67    Percent,
68
69    // Comparison / equality
70    Eq,
71    EqEq,
72    EqEqEq,
73    NotEq,
74    NotEqEq,
75    Lt,
76    Gt,
77    LtEq,
78    GtEq,
79
80    // Logical
81    And,
82    Or,
83    Not,
84
85    // Bitwise
86    BitAnd,
87    BitOr,
88    BitXor,
89    ShiftLeft,
90    ShiftRight,
91
92    // Assignment
93    Assign,
94    PlusAssign,
95    MinusAssign,
96    StarAssign,
97    SlashAssign,
98
99    // Delimiters
100    OpenParen,
101    CloseParen,
102    OpenBrace,
103    CloseBrace,
104    OpenBracket,
105    CloseBracket,
106
107    // Punctuation
108    Dot,
109    Comma,
110    Semicolon,
111    Colon,
112    Question,
113    Arrow,
114    Spread,
115
116    // Keywords
117    Let,
118    Const,
119    Var,
120    Function,
121    Return,
122    If,
123    Else,
124    While,
125    For,
126    Break,
127    Continue,
128    New,
129    This,
130    Typeof,
131    Instanceof,
132    In,
133    Of,
134    Delete,
135    Void,
136    Throw,
137    Try,
138    Catch,
139    Finally,
140    Switch,
141    Case,
142    Default,
143    Class,
144    Extends,
145    Super,
146    Import,
147    Export,
148    True,
149    False,
150
151    // Special
152    Eof,
153}
154
155impl JsToken {
156    /// Whether this token can end a statement (for ASI purposes)
157    pub fn can_end_statement(&self) -> bool {
158        matches!(
159            self,
160            Self::Identifier(_)
161                | Self::Number(_)
162                | Self::StringLiteral(_)
163                | Self::Bool(_)
164                | Self::Null
165                | Self::Undefined
166                | Self::True
167                | Self::False
168                | Self::This
169                | Self::CloseParen
170                | Self::CloseBracket
171                | Self::Return
172                | Self::Break
173                | Self::Continue
174        )
175    }
176
177    /// Whether this token is a keyword
178    pub fn is_keyword(&self) -> bool {
179        matches!(
180            self,
181            Self::Let
182                | Self::Const
183                | Self::Var
184                | Self::Function
185                | Self::Return
186                | Self::If
187                | Self::Else
188                | Self::While
189                | Self::For
190                | Self::Break
191                | Self::Continue
192                | Self::New
193                | Self::This
194                | Self::Typeof
195                | Self::Instanceof
196                | Self::In
197                | Self::Of
198                | Self::Delete
199                | Self::Void
200                | Self::Throw
201                | Self::Try
202                | Self::Catch
203                | Self::Finally
204                | Self::Switch
205                | Self::Case
206                | Self::Default
207                | Self::Class
208                | Self::Extends
209                | Self::Super
210                | Self::Import
211                | Self::Export
212                | Self::True
213                | Self::False
214        )
215    }
216}
217
218// ---------------------------------------------------------------------------
219// Lexer
220// ---------------------------------------------------------------------------
221
222/// JavaScript lexer
223pub struct JsLexer {
224    /// Input source as bytes
225    input: Vec<u8>,
226    /// Current position
227    pos: usize,
228    /// Current line number (1-based)
229    pub line: u32,
230    /// Current column (1-based)
231    pub col: u32,
232    /// Whether the previous token could end a statement (for ASI)
233    prev_can_end: bool,
234    /// Whether we crossed a newline since the last token
235    newline_before: bool,
236}
237
238impl JsLexer {
239    pub fn new(source: &str) -> Self {
240        Self {
241            input: source.as_bytes().to_vec(),
242            pos: 0,
243            line: 1,
244            col: 1,
245            prev_can_end: false,
246            newline_before: false,
247        }
248    }
249
250    /// Tokenize the entire input into a Vec
251    pub fn tokenize_all(&mut self) -> Vec<JsToken> {
252        let mut tokens = Vec::new();
253        loop {
254            let tok = self.next_token();
255            if tok == JsToken::Eof {
256                tokens.push(tok);
257                break;
258            }
259            tokens.push(tok);
260        }
261        tokens
262    }
263
264    /// Get the next token
265    pub fn next_token(&mut self) -> JsToken {
266        self.newline_before = false;
267        self.skip_whitespace_and_comments();
268
269        // ASI: if we crossed a newline and the previous token can end
270        // a statement, insert a semicolon
271        if self.newline_before && self.prev_can_end {
272            self.prev_can_end = false;
273            return JsToken::Semicolon;
274        }
275
276        if self.pos >= self.input.len() {
277            return JsToken::Eof;
278        }
279
280        let ch = self.input[self.pos];
281        let tok = match ch {
282            b'(' => {
283                self.advance();
284                JsToken::OpenParen
285            }
286            b')' => {
287                self.advance();
288                JsToken::CloseParen
289            }
290            b'{' => {
291                self.advance();
292                JsToken::OpenBrace
293            }
294            b'}' => {
295                self.advance();
296                JsToken::CloseBrace
297            }
298            b'[' => {
299                self.advance();
300                JsToken::OpenBracket
301            }
302            b']' => {
303                self.advance();
304                JsToken::CloseBracket
305            }
306            b',' => {
307                self.advance();
308                JsToken::Comma
309            }
310            b';' => {
311                self.advance();
312                JsToken::Semicolon
313            }
314            b':' => {
315                self.advance();
316                JsToken::Colon
317            }
318            b'?' => {
319                self.advance();
320                JsToken::Question
321            }
322            b'~' => {
323                self.advance();
324                JsToken::BitXor
325            } // simplified
326            b'.' => {
327                if self.peek_at(1) == Some(b'.') && self.peek_at(2) == Some(b'.') {
328                    self.advance();
329                    self.advance();
330                    self.advance();
331                    JsToken::Spread
332                } else {
333                    self.advance();
334                    JsToken::Dot
335                }
336            }
337
338            b'+' => {
339                self.advance();
340                if self.peek() == Some(b'=') {
341                    self.advance();
342                    JsToken::PlusAssign
343                } else {
344                    JsToken::Plus
345                }
346            }
347            b'-' => {
348                self.advance();
349                if self.peek() == Some(b'=') {
350                    self.advance();
351                    JsToken::MinusAssign
352                } else {
353                    JsToken::Minus
354                }
355            }
356            b'*' => {
357                self.advance();
358                if self.peek() == Some(b'=') {
359                    self.advance();
360                    JsToken::StarAssign
361                } else {
362                    JsToken::Star
363                }
364            }
365            b'/' => {
366                self.advance();
367                if self.peek() == Some(b'=') {
368                    self.advance();
369                    JsToken::SlashAssign
370                } else {
371                    JsToken::Slash
372                }
373            }
374            b'%' => {
375                self.advance();
376                JsToken::Percent
377            }
378
379            b'=' => {
380                self.advance();
381                if self.peek() == Some(b'=') {
382                    self.advance();
383                    if self.peek() == Some(b'=') {
384                        self.advance();
385                        JsToken::EqEqEq
386                    } else {
387                        JsToken::EqEq
388                    }
389                } else if self.peek() == Some(b'>') {
390                    self.advance();
391                    JsToken::Arrow
392                } else {
393                    JsToken::Assign
394                }
395            }
396
397            b'!' => {
398                self.advance();
399                if self.peek() == Some(b'=') {
400                    self.advance();
401                    if self.peek() == Some(b'=') {
402                        self.advance();
403                        JsToken::NotEqEq
404                    } else {
405                        JsToken::NotEq
406                    }
407                } else {
408                    JsToken::Not
409                }
410            }
411
412            b'<' => {
413                self.advance();
414                if self.peek() == Some(b'=') {
415                    self.advance();
416                    JsToken::LtEq
417                } else if self.peek() == Some(b'<') {
418                    self.advance();
419                    JsToken::ShiftLeft
420                } else {
421                    JsToken::Lt
422                }
423            }
424            b'>' => {
425                self.advance();
426                if self.peek() == Some(b'=') {
427                    self.advance();
428                    JsToken::GtEq
429                } else if self.peek() == Some(b'>') {
430                    self.advance();
431                    JsToken::ShiftRight
432                } else {
433                    JsToken::Gt
434                }
435            }
436
437            b'&' => {
438                self.advance();
439                if self.peek() == Some(b'&') {
440                    self.advance();
441                    JsToken::And
442                } else {
443                    JsToken::BitAnd
444                }
445            }
446            b'|' => {
447                self.advance();
448                if self.peek() == Some(b'|') {
449                    self.advance();
450                    JsToken::Or
451                } else {
452                    JsToken::BitOr
453                }
454            }
455            b'^' => {
456                self.advance();
457                JsToken::BitXor
458            }
459
460            b'"' | b'\'' => self.read_string(ch),
461
462            b'0'..=b'9' => self.read_number(),
463
464            _ if is_ident_start(ch) => self.read_identifier(),
465
466            _ => {
467                self.advance();
468                // Skip unknown characters
469                return self.next_token();
470            }
471        };
472
473        self.prev_can_end = tok.can_end_statement();
474        tok
475    }
476
477    // -- Internal methods --
478
479    fn advance(&mut self) {
480        if self.pos < self.input.len() {
481            if self.input[self.pos] == b'\n' {
482                self.line += 1;
483                self.col = 1;
484            } else {
485                self.col += 1;
486            }
487            self.pos += 1;
488        }
489    }
490
491    fn peek(&self) -> Option<u8> {
492        self.input.get(self.pos).copied()
493    }
494
495    fn peek_at(&self, offset: usize) -> Option<u8> {
496        self.input.get(self.pos + offset).copied()
497    }
498
499    fn skip_whitespace_and_comments(&mut self) {
500        while self.pos < self.input.len() {
501            let ch = self.input[self.pos];
502            match ch {
503                b' ' | b'\t' | b'\r' => {
504                    self.advance();
505                }
506                b'\n' => {
507                    self.newline_before = true;
508                    self.advance();
509                }
510                b'/' if self.peek_at(1) == Some(b'/') => {
511                    // Single-line comment
512                    while self.pos < self.input.len() && self.input[self.pos] != b'\n' {
513                        self.advance();
514                    }
515                }
516                b'/' if self.peek_at(1) == Some(b'*') => {
517                    // Multi-line comment
518                    self.advance(); // /
519                    self.advance(); // *
520                    while self.pos + 1 < self.input.len() {
521                        if self.input[self.pos] == b'*' && self.input[self.pos + 1] == b'/' {
522                            self.advance(); // *
523                            self.advance(); // /
524                            break;
525                        }
526                        if self.input[self.pos] == b'\n' {
527                            self.newline_before = true;
528                        }
529                        self.advance();
530                    }
531                }
532                _ => break,
533            }
534        }
535    }
536
537    fn read_string(&mut self, quote: u8) -> JsToken {
538        self.advance(); // opening quote
539        let mut s = String::new();
540        while self.pos < self.input.len() {
541            let ch = self.input[self.pos];
542            if ch == quote {
543                self.advance();
544                return JsToken::StringLiteral(s);
545            }
546            if ch == b'\\' {
547                self.advance();
548                if self.pos >= self.input.len() {
549                    break;
550                }
551                let esc = self.input[self.pos];
552                match esc {
553                    b'n' => s.push('\n'),
554                    b't' => s.push('\t'),
555                    b'r' => s.push('\r'),
556                    b'\\' => s.push('\\'),
557                    b'\'' => s.push('\''),
558                    b'"' => s.push('"'),
559                    b'0' => s.push('\0'),
560                    b'x' => {
561                        // \xHH
562                        self.advance();
563                        let hi = self.hex_digit().unwrap_or(0);
564                        self.advance();
565                        let lo = self.hex_digit().unwrap_or(0);
566                        self.advance();
567                        s.push((hi * 16 + lo) as char);
568                        continue;
569                    }
570                    b'u' => {
571                        // \uHHHH
572                        self.advance();
573                        let mut code: u32 = 0;
574                        for _ in 0..4 {
575                            code = code * 16 + self.hex_digit().unwrap_or(0) as u32;
576                            self.advance();
577                        }
578                        if let Some(ch) = char::from_u32(code) {
579                            s.push(ch);
580                        }
581                        continue;
582                    }
583                    _ => {
584                        s.push(esc as char);
585                    }
586                }
587                self.advance();
588            } else {
589                s.push(ch as char);
590                self.advance();
591            }
592        }
593        JsToken::StringLiteral(s) // unterminated string
594    }
595
596    fn read_number(&mut self) -> JsToken {
597        let start = self.pos;
598        let mut has_dot = false;
599
600        // Check for 0x hex prefix
601        if self.input[self.pos] == b'0' && self.peek_at(1) == Some(b'x') {
602            self.advance(); // 0
603            self.advance(); // x
604            let mut val: i64 = 0;
605            while self.pos < self.input.len() {
606                if let Some(d) = self.hex_digit() {
607                    val = val * 16 + d as i64;
608                    self.advance();
609                } else {
610                    break;
611                }
612            }
613            return JsToken::Number(js_int(val));
614        }
615
616        // Integer part
617        while self.pos < self.input.len() && self.input[self.pos].is_ascii_digit() {
618            self.advance();
619        }
620
621        // Fractional part
622        if self.pos < self.input.len() && self.input[self.pos] == b'.' {
623            // Check it's not a method call (e.g., 1.toString())
624            if self.peek_at(1).is_none_or(|c| c.is_ascii_digit()) {
625                has_dot = true;
626                self.advance(); // .
627                while self.pos < self.input.len() && self.input[self.pos].is_ascii_digit() {
628                    self.advance();
629                }
630            }
631        }
632
633        let text = core::str::from_utf8(&self.input[start..self.pos]).unwrap_or("0");
634
635        if has_dot {
636            // Parse decimal to 32.32 fixed-point
637            JsToken::Number(parse_decimal_to_fixed(text))
638        } else {
639            // Integer
640            let val: i64 = parse_int_simple(text);
641            JsToken::Number(js_int(val))
642        }
643    }
644
645    fn read_identifier(&mut self) -> JsToken {
646        let start = self.pos;
647        while self.pos < self.input.len() && is_ident_continue(self.input[self.pos]) {
648            self.advance();
649        }
650        let ident = core::str::from_utf8(&self.input[start..self.pos]).unwrap_or("");
651
652        match ident {
653            "let" => JsToken::Let,
654            "const" => JsToken::Const,
655            "var" => JsToken::Var,
656            "function" => JsToken::Function,
657            "return" => JsToken::Return,
658            "if" => JsToken::If,
659            "else" => JsToken::Else,
660            "while" => JsToken::While,
661            "for" => JsToken::For,
662            "break" => JsToken::Break,
663            "continue" => JsToken::Continue,
664            "new" => JsToken::New,
665            "this" => JsToken::This,
666            "typeof" => JsToken::Typeof,
667            "instanceof" => JsToken::Instanceof,
668            "in" => JsToken::In,
669            "of" => JsToken::Of,
670            "delete" => JsToken::Delete,
671            "void" => JsToken::Void,
672            "throw" => JsToken::Throw,
673            "try" => JsToken::Try,
674            "catch" => JsToken::Catch,
675            "finally" => JsToken::Finally,
676            "switch" => JsToken::Switch,
677            "case" => JsToken::Case,
678            "default" => JsToken::Default,
679            "class" => JsToken::Class,
680            "extends" => JsToken::Extends,
681            "super" => JsToken::Super,
682            "import" => JsToken::Import,
683            "export" => JsToken::Export,
684            "true" => JsToken::True,
685            "false" => JsToken::False,
686            "null" => JsToken::Null,
687            "undefined" => JsToken::Undefined,
688            _ => JsToken::Identifier(ident.to_string()),
689        }
690    }
691
692    fn hex_digit(&self) -> Option<u8> {
693        let ch = *self.input.get(self.pos)?;
694        match ch {
695            b'0'..=b'9' => Some(ch - b'0'),
696            b'a'..=b'f' => Some(ch - b'a' + 10),
697            b'A'..=b'F' => Some(ch - b'A' + 10),
698            _ => None,
699        }
700    }
701}
702
703// ---------------------------------------------------------------------------
704// Helpers
705// ---------------------------------------------------------------------------
706
707fn is_ident_start(ch: u8) -> bool {
708    ch.is_ascii_alphabetic() || ch == b'_' || ch == b'$'
709}
710
711fn is_ident_continue(ch: u8) -> bool {
712    ch.is_ascii_alphanumeric() || ch == b'_' || ch == b'$'
713}
714
715/// Parse a simple integer string (no sign prefix needed -- lexer handles
716/// negation)
717fn parse_int_simple(s: &str) -> i64 {
718    let mut result: i64 = 0;
719    for &b in s.as_bytes() {
720        if b.is_ascii_digit() {
721            result = result.wrapping_mul(10).wrapping_add((b - b'0') as i64);
722        }
723    }
724    result
725}
726
727/// Parse a decimal number string to 32.32 fixed-point
728fn parse_decimal_to_fixed(s: &str) -> JsNumber {
729    let mut int_part: i64 = 0;
730    let mut frac_part: i64 = 0;
731    let mut frac_divisor: i64 = 1;
732    let mut after_dot = false;
733
734    for &b in s.as_bytes() {
735        if b == b'.' {
736            after_dot = true;
737            continue;
738        }
739        if !b.is_ascii_digit() {
740            continue;
741        }
742        if after_dot {
743            frac_part = frac_part.wrapping_mul(10).wrapping_add((b - b'0') as i64);
744            frac_divisor = frac_divisor.wrapping_mul(10);
745        } else {
746            int_part = int_part.wrapping_mul(10).wrapping_add((b - b'0') as i64);
747        }
748    }
749
750    let int_fixed = int_part << JS_FRAC_BITS;
751    let frac_fixed = if frac_divisor > 0 {
752        (frac_part << JS_FRAC_BITS) / frac_divisor
753    } else {
754        0
755    };
756    int_fixed + frac_fixed
757}
758
759// ---------------------------------------------------------------------------
760// Tests
761// ---------------------------------------------------------------------------
762
763#[cfg(test)]
764mod tests {
765    #[allow(unused_imports)]
766    use alloc::vec;
767
768    use super::*;
769
770    fn lex(src: &str) -> Vec<JsToken> {
771        JsLexer::new(src).tokenize_all()
772    }
773
774    #[test]
775    fn test_empty() {
776        let tokens = lex("");
777        assert_eq!(tokens, vec![JsToken::Eof]);
778    }
779
780    #[test]
781    fn test_number_integer() {
782        let tokens = lex("42");
783        assert_eq!(tokens[0], JsToken::Number(js_int(42)));
784    }
785
786    #[test]
787    fn test_number_decimal() {
788        let tokens = lex("3.5");
789        let n = match &tokens[0] {
790            JsToken::Number(n) => *n,
791            _ => panic!("expected number"),
792        };
793        // 3.5 in 32.32: 3 << 32 + 0.5 << 32 = 3 << 32 + 1 << 31
794        assert_eq!(js_to_int(n), 3);
795        assert!(!js_is_integer(n)); // has fractional part
796    }
797
798    #[test]
799    fn test_number_hex() {
800        let tokens = lex("0xFF");
801        assert_eq!(tokens[0], JsToken::Number(js_int(255)));
802    }
803
804    #[test]
805    fn test_string_double() {
806        let tokens = lex("\"hello\"");
807        assert_eq!(tokens[0], JsToken::StringLiteral("hello".to_string()));
808    }
809
810    #[test]
811    fn test_string_single() {
812        let tokens = lex("'world'");
813        assert_eq!(tokens[0], JsToken::StringLiteral("world".to_string()));
814    }
815
816    #[test]
817    fn test_string_escapes() {
818        let tokens = lex("\"a\\nb\\tc\"");
819        assert_eq!(tokens[0], JsToken::StringLiteral("a\nb\tc".to_string()));
820    }
821
822    #[test]
823    fn test_string_hex_escape() {
824        let tokens = lex("\"\\x41\"");
825        assert_eq!(tokens[0], JsToken::StringLiteral("A".to_string()));
826    }
827
828    #[test]
829    fn test_string_unicode_escape() {
830        let tokens = lex("\"\\u0041\"");
831        assert_eq!(tokens[0], JsToken::StringLiteral("A".to_string()));
832    }
833
834    #[test]
835    fn test_keywords() {
836        let tokens = lex("let const var function if else while for return");
837        assert_eq!(tokens[0], JsToken::Let);
838        assert_eq!(tokens[1], JsToken::Const);
839        assert_eq!(tokens[2], JsToken::Var);
840        assert_eq!(tokens[3], JsToken::Function);
841        assert_eq!(tokens[4], JsToken::If);
842        assert_eq!(tokens[5], JsToken::Else);
843        assert_eq!(tokens[6], JsToken::While);
844        assert_eq!(tokens[7], JsToken::For);
845        assert_eq!(tokens[8], JsToken::Return);
846    }
847
848    #[test]
849    fn test_identifiers() {
850        let tokens = lex("foo bar_1 $var _private");
851        assert_eq!(tokens[0], JsToken::Identifier("foo".to_string()));
852        assert_eq!(tokens[1], JsToken::Identifier("bar_1".to_string()));
853        assert_eq!(tokens[2], JsToken::Identifier("$var".to_string()));
854        assert_eq!(tokens[3], JsToken::Identifier("_private".to_string()));
855    }
856
857    #[test]
858    fn test_operators() {
859        let tokens = lex("+ - * / % === !== <= >=");
860        assert_eq!(tokens[0], JsToken::Plus);
861        assert_eq!(tokens[1], JsToken::Minus);
862        assert_eq!(tokens[2], JsToken::Star);
863        assert_eq!(tokens[3], JsToken::Slash);
864        assert_eq!(tokens[4], JsToken::Percent);
865        assert_eq!(tokens[5], JsToken::EqEqEq);
866        assert_eq!(tokens[6], JsToken::NotEqEq);
867        assert_eq!(tokens[7], JsToken::LtEq);
868        assert_eq!(tokens[8], JsToken::GtEq);
869    }
870
871    #[test]
872    fn test_assignment_ops() {
873        let tokens = lex("+= -= *= /=");
874        assert_eq!(tokens[0], JsToken::PlusAssign);
875        assert_eq!(tokens[1], JsToken::MinusAssign);
876        assert_eq!(tokens[2], JsToken::StarAssign);
877        assert_eq!(tokens[3], JsToken::SlashAssign);
878    }
879
880    #[test]
881    fn test_delimiters() {
882        let tokens = lex("(){}[]");
883        assert_eq!(tokens[0], JsToken::OpenParen);
884        assert_eq!(tokens[1], JsToken::CloseParen);
885        assert_eq!(tokens[2], JsToken::OpenBrace);
886        assert_eq!(tokens[3], JsToken::CloseBrace);
887        assert_eq!(tokens[4], JsToken::OpenBracket);
888        assert_eq!(tokens[5], JsToken::CloseBracket);
889    }
890
891    #[test]
892    fn test_logical_ops() {
893        let tokens = lex("&& || !");
894        assert_eq!(tokens[0], JsToken::And);
895        assert_eq!(tokens[1], JsToken::Or);
896        assert_eq!(tokens[2], JsToken::Not);
897    }
898
899    #[test]
900    fn test_bitwise_ops() {
901        let tokens = lex("& | ^ << >>");
902        assert_eq!(tokens[0], JsToken::BitAnd);
903        assert_eq!(tokens[1], JsToken::BitOr);
904        assert_eq!(tokens[2], JsToken::BitXor);
905        assert_eq!(tokens[3], JsToken::ShiftLeft);
906        assert_eq!(tokens[4], JsToken::ShiftRight);
907    }
908
909    #[test]
910    fn test_arrow() {
911        let tokens = lex("=>");
912        assert_eq!(tokens[0], JsToken::Arrow);
913    }
914
915    #[test]
916    fn test_spread() {
917        let tokens = lex("...");
918        assert_eq!(tokens[0], JsToken::Spread);
919    }
920
921    #[test]
922    fn test_single_line_comment() {
923        let tokens = lex("42 // comment\n43");
924        assert_eq!(tokens[0], JsToken::Number(js_int(42)));
925        // ASI may insert semicolon
926        let last_num = tokens
927            .iter()
928            .filter(|t| matches!(t, JsToken::Number(_)))
929            .count();
930        assert_eq!(last_num, 2);
931    }
932
933    #[test]
934    fn test_multi_line_comment() {
935        let tokens = lex("1 /* block\ncomment */ 2");
936        let nums: Vec<_> = tokens
937            .iter()
938            .filter(|t| matches!(t, JsToken::Number(_)))
939            .collect();
940        assert_eq!(nums.len(), 2);
941    }
942
943    #[test]
944    fn test_boolean_literals() {
945        let tokens = lex("true false");
946        assert_eq!(tokens[0], JsToken::True);
947        assert_eq!(tokens[1], JsToken::False);
948    }
949
950    #[test]
951    fn test_null_undefined() {
952        let tokens = lex("null undefined");
953        assert_eq!(tokens[0], JsToken::Null);
954        assert_eq!(tokens[1], JsToken::Undefined);
955    }
956
957    #[test]
958    fn test_var_declaration() {
959        let tokens = lex("let x = 10;");
960        assert_eq!(tokens[0], JsToken::Let);
961        assert_eq!(tokens[1], JsToken::Identifier("x".to_string()));
962        assert_eq!(tokens[2], JsToken::Assign);
963        assert_eq!(tokens[3], JsToken::Number(js_int(10)));
964        assert_eq!(tokens[4], JsToken::Semicolon);
965    }
966
967    #[test]
968    fn test_function_call() {
969        let tokens = lex("foo(1, 2)");
970        assert_eq!(tokens[0], JsToken::Identifier("foo".to_string()));
971        assert_eq!(tokens[1], JsToken::OpenParen);
972        assert_eq!(tokens[2], JsToken::Number(js_int(1)));
973        assert_eq!(tokens[3], JsToken::Comma);
974        assert_eq!(tokens[4], JsToken::Number(js_int(2)));
975        assert_eq!(tokens[5], JsToken::CloseParen);
976    }
977
978    #[test]
979    fn test_more_keywords() {
980        let tokens = lex("try catch finally throw switch case default");
981        assert_eq!(tokens[0], JsToken::Try);
982        assert_eq!(tokens[1], JsToken::Catch);
983        assert_eq!(tokens[2], JsToken::Finally);
984        assert_eq!(tokens[3], JsToken::Throw);
985        assert_eq!(tokens[4], JsToken::Switch);
986        assert_eq!(tokens[5], JsToken::Case);
987        assert_eq!(tokens[6], JsToken::Default);
988    }
989
990    #[test]
991    fn test_class_keywords() {
992        let tokens = lex("class extends super new");
993        assert_eq!(tokens[0], JsToken::Class);
994        assert_eq!(tokens[1], JsToken::Extends);
995        assert_eq!(tokens[2], JsToken::Super);
996        assert_eq!(tokens[3], JsToken::New);
997    }
998
999    #[test]
1000    fn test_typeof_instanceof() {
1001        let tokens = lex("typeof x instanceof Y");
1002        assert_eq!(tokens[0], JsToken::Typeof);
1003        assert_eq!(tokens[1], JsToken::Identifier("x".to_string()));
1004        assert_eq!(tokens[2], JsToken::Instanceof);
1005        assert_eq!(tokens[3], JsToken::Identifier("Y".to_string()));
1006    }
1007
1008    #[test]
1009    fn test_js_int_helpers() {
1010        assert_eq!(js_to_int(js_int(42)), 42);
1011        assert!(js_is_integer(js_int(10)));
1012        assert!(js_is_integer(JS_ZERO));
1013    }
1014
1015    #[test]
1016    fn test_token_can_end_statement() {
1017        assert!(JsToken::Identifier("x".to_string()).can_end_statement());
1018        assert!(JsToken::Number(JS_ZERO).can_end_statement());
1019        assert!(JsToken::CloseParen.can_end_statement());
1020        assert!(!JsToken::Plus.can_end_statement());
1021        assert!(!JsToken::OpenBrace.can_end_statement());
1022    }
1023
1024    #[test]
1025    fn test_comparison_eq() {
1026        let tokens = lex("== ===");
1027        assert_eq!(tokens[0], JsToken::EqEq);
1028        assert_eq!(tokens[1], JsToken::EqEqEq);
1029    }
1030
1031    #[test]
1032    fn test_dot_vs_spread() {
1033        let tokens = lex(". ...");
1034        assert_eq!(tokens[0], JsToken::Dot);
1035        assert_eq!(tokens[1], JsToken::Spread);
1036    }
1037
1038    #[test]
1039    fn test_asi_basic() {
1040        // After "x" (an identifier that can_end_statement), a newline triggers ASI
1041        let tokens = lex("x\ny");
1042        // Should produce: Identifier("x"), Semicolon (ASI), Identifier("y"), Eof
1043        assert!(tokens.contains(&JsToken::Semicolon));
1044    }
1045
1046    #[test]
1047    fn test_number_zero() {
1048        let tokens = lex("0");
1049        assert_eq!(tokens[0], JsToken::Number(js_int(0)));
1050    }
1051
1052    #[test]
1053    fn test_line_tracking() {
1054        let mut lexer = JsLexer::new("a\nb\nc");
1055        lexer.next_token(); // a
1056        assert_eq!(lexer.line, 1);
1057        lexer.next_token(); // ASI semicolon
1058        lexer.next_token(); // b (after newline)
1059                            // line should have advanced
1060    }
1061
1062    #[test]
1063    fn test_parse_decimal_to_fixed() {
1064        let fp = parse_decimal_to_fixed("1.5");
1065        assert_eq!(js_to_int(fp), 1);
1066        assert!(!js_is_integer(fp));
1067    }
1068}
veridian_kernel/browser/js_lexer.rs

veridian_kernel/browser/
js_lexer.rs
