cl_token/
lib.rs

1//! # Token
2//!
3//! Stores a component of a file as a [Lexeme], [TKind], and [struct@Span]
4#![warn(clippy::all)]
5//! The Token defines an interface between lexer and parser
6
7use cl_structures::span::Span;
8
9/// A unit of lexical information produced by a Lexer
10#[derive(Clone, Debug)]
11pub struct Token {
12    pub lexeme: Lexeme,
13    pub kind: TKind,
14    pub span: Span,
15}
16
17impl Token {
18    /// Extracts the [`kind`](Token::kind) field of this Token
19    pub const fn kind(&self) -> TKind {
20        self.kind
21    }
22}
23
24/// The (possibly pre-processed) lexical information, in the form of a [String], [u128], or [char]
25#[derive(Clone, Debug)]
26pub enum Lexeme {
27    String(String),
28    Integer(u128, u32),
29    Char(char),
30}
31
32impl Lexeme {
33    pub fn string(self) -> Option<String> {
34        match self {
35            Self::String(s) => Some(s),
36            _ => None,
37        }
38    }
39    pub fn str(&self) -> Option<&str> {
40        match self {
41            Self::String(s) => Some(s),
42            _ => None,
43        }
44    }
45    pub const fn int(&self) -> Option<u128> {
46        match self {
47            Self::Integer(i, _) => Some(*i),
48            _ => None,
49        }
50    }
51    pub const fn char(&self) -> Option<char> {
52        match self {
53            Self::Char(c) => Some(*c),
54            _ => None,
55        }
56    }
57}
58
59impl std::fmt::Display for Lexeme {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        match self {
62            Self::String(v) => v.fmt(f),
63            Self::Integer(v, _) => v.fmt(f),
64            Self::Char(v) => v.fmt(f),
65        }
66    }
67}
68
69/// The lexical classification of a [Token].
70#[derive(Clone, Copy, Debug, PartialEq, Eq)]
71pub enum TKind {
72    /// Line or block comment
73    Comment,
74    /// Inner doc comment: //!.*
75    InDoc,
76    /// Outer doc comment ///.*
77    OutDoc,
78
79    As,
80    Break,
81    Const,
82    Continue,
83    Defer,
84    Do,
85    Else,
86    Enum,
87    False,
88    Fn,
89    For,
90    If,
91    Impl,
92    In,
93    Let,
94    Loop,
95    Macro,
96    Match,
97    Mod,
98    Mut,
99    Pub,
100    Return,
101    Static,
102    Struct,
103    True,
104    Type,
105    Use,
106    While,
107
108    Identifier, // or Keyword
109    Character,
110    String,
111    /// `0(x[0-9A-Fa-f]* | d[0-9]* | o[0-7]* | b[0-1]*) | [1-9][0-9]*`
112    Integer,
113    /// {
114    LCurly,
115    /// }
116    RCurly,
117    /// [
118    LBrack,
119    /// ]
120    RBrack,
121    /// (
122    LParen,
123    /// )
124    RParen,
125    /// &
126    Amp,
127    /// &&
128    AmpAmp,
129    /// &=
130    AmpEq,
131    /// ->
132    Arrow,
133    /// @
134    At,
135    /// \
136    Backslash,
137    /// !
138    Bang,
139    /// !!
140    BangBang,
141    /// !=
142    BangEq,
143    /// |
144    Bar,
145    /// ||
146    BarBar,
147    /// |=
148    BarEq,
149    /// :
150    Colon,
151    /// ::
152    ColonColon,
153    /// ,
154    Comma,
155    /// $
156    Dollar,
157    /// .
158    Dot,
159    /// ..
160    DotDot,
161    /// ...
162    DotDotDot,
163    /// ..=
164    DotDotEq,
165    /// =
166    Eq,
167    /// ==
168    EqEq,
169    /// =>
170    FatArrow,
171    /// \`
172    Grave,
173    /// >
174    Gt,
175    /// >=
176    GtEq,
177    /// >>
178    GtGt,
179    /// >>=
180    GtGtEq,
181    /// #
182    Hash,
183    /// #!
184    HashBang,
185    /// <
186    Lt,
187    /// <=
188    LtEq,
189    /// <<
190    LtLt,
191    /// <<=
192    LtLtEq,
193    /// -
194    Minus,
195    /// -=
196    MinusEq,
197    /// +
198    Plus,
199    /// +=
200    PlusEq,
201    /// ?
202    Question,
203    /// %
204    Rem,
205    /// %=
206    RemEq,
207    /// ;
208    Semi,
209    /// /
210    Slash,
211    /// /=
212    SlashEq,
213    /// *
214    Star,
215    /// *=
216    StarEq,
217    /// ~
218    Tilde,
219    /// ^
220    Xor,
221    /// ^=
222    XorEq,
223    /// ^^
224    XorXor,
225}
226
227impl TKind {
228    pub const fn flip(self) -> Self {
229        match self {
230            Self::LCurly => Self::RCurly,
231            Self::RCurly => Self::LCurly,
232            Self::LBrack => Self::RBrack,
233            Self::RBrack => Self::LBrack,
234            Self::LParen => Self::RParen,
235            Self::RParen => Self::LParen,
236            Self::Gt => Self::Lt,
237            Self::GtGt => Self::LtLt,
238            Self::LtLt => Self::GtGt,
239            Self::Lt => Self::Gt,
240            _ => self,
241        }
242    }
243
244    /// Splits a single [TKind] into two, if possible, or returns the original.
245    pub const fn split(self) -> Result<(Self, Self), Self> {
246        Ok(match self {
247            Self::AmpAmp => (Self::Amp, Self::Amp),
248            Self::AmpEq => (Self::Amp, Self::Eq),
249            Self::Arrow => (Self::Minus, Self::Gt),
250            Self::BangBang => (Self::Bang, Self::Bang),
251            Self::BangEq => (Self::Bang, Self::Eq),
252            Self::BarBar => (Self::Bar, Self::Bar),
253            Self::BarEq => (Self::Bar, Self::Eq),
254            Self::ColonColon => (Self::Colon, Self::Colon),
255            Self::DotDot => (Self::Dot, Self::Dot),
256            Self::DotDotEq => (Self::DotDot, Self::Eq),
257            Self::EqEq => (Self::Eq, Self::Eq),
258            Self::FatArrow => (Self::Eq, Self::Gt),
259            Self::GtEq => (Self::Gt, Self::Eq),
260            Self::GtGt => (Self::Gt, Self::Gt),
261            Self::GtGtEq => (Self::Gt, Self::GtEq),
262            Self::HashBang => (Self::Hash, Self::Bang),
263            Self::LtEq => (Self::Lt, Self::Eq),
264            Self::LtLt => (Self::Lt, Self::Lt),
265            Self::LtLtEq => (Self::Lt, Self::LtEq),
266            Self::MinusEq => (Self::Minus, Self::Eq),
267            Self::PlusEq => (Self::Plus, Self::Eq),
268            Self::RemEq => (Self::Rem, Self::Eq),
269            Self::SlashEq => (Self::Slash, Self::Eq),
270            Self::StarEq => (Self::Star, Self::Eq),
271            Self::XorEq => (Self::Xor, Self::Eq),
272            Self::XorXor => (Self::Xor, Self::Xor),
273            _ => return Err(self),
274        })
275    }
276}
cl_token/lib.rs

cl_token/
lib.rs
