Struct Lexer

pub(in front::wgsl) struct Lexer<'a> {
    input: &'a str,
    pub(in front::wgsl) source: &'a str,
    last_end_offset: usize,
    ignore_doc_comments: bool,
    pub(in front::wgsl) enable_extensions: EnableExtensions,
}

Fields

input: &'a str

The remaining unconsumed input.

source: &'a str

The full original source code.

We compare input against this to compute the lexer’s current offset in the source.

last_end_offset: usize

The byte offset of the end of the most recently returned non-trivia token.

This is consulted by the span_from function, for finding the end of the span for larger structures like expressions or statements.

ignore_doc_comments: bool

Whether or not to ignore doc comments. If true, doc comments are treated as Token::Trivia.

enable_extensions: EnableExtensions

Implementations

impl<'a> Lexer<'a>

pub(in front::wgsl) const fn new( input: &'a str, ignore_doc_comments: bool, ) -> Self

pub fn capture_span<T, E>( &mut self, inner: impl FnOnce(&mut Self) -> Result<T, E>, ) -> Result<(T, Span), E>

Calls the function with a lexer and returns the result of the function as well as the span for everything the function parsed

Examples

let lexer = Lexer::new("5");
let (value, span) = lexer.capture_span(Lexer::next_uint_literal);
assert_eq!(value, 5);

pub(in front::wgsl) fn start_byte_offset( &mut self, ) -> usize

fn peek_token_and_rest(&mut self) -> ((Token<'a>, Span), &'a str)

pub(in front::wgsl) fn accumulate_module_doc_comments( &mut self, ) -> Vec<&'a str>

Collect all module doc comments until a non doc token is found.

pub(in front::wgsl) fn accumulate_doc_comments( &mut self, ) -> Vec<&'a str>

Collect all doc comments until a non doc token is found.

const fn current_byte_offset(&self) -> usize

pub(in front::wgsl) fn span_from( &self, offset: usize, ) -> Span

pub(in front::wgsl) fn next( &mut self, ) -> (Token<'a>, Span)

Return the next non-whitespace token from self.

Assume we are a parse state where bit shift operators may occur, but not angle brackets.

pub(in front::wgsl) fn next_generic( &mut self, ) -> (Token<'a>, Span)

Return the next non-whitespace token from self.

Assume we are in a parse state where angle brackets may occur, but not bit shift operators.

fn next_impl( &mut self, generic: bool, ignore_doc_comments: bool, ) -> (Token<'a>, Span)

Return the next non-whitespace token from self, with a span.

See consume_token for the meaning of generic.

pub(in front::wgsl) fn peek( &mut self, ) -> (Token<'a>, Span)

pub(in front::wgsl) fn expect_span( &mut self, expected: Token<'a>, ) -> Result<Span, Box<Error<'a>>>

pub(in front::wgsl) fn expect( &mut self, expected: Token<'a>, ) -> Result<(), Box<Error<'a>>>

pub(in front::wgsl) fn expect_generic_paren( &mut self, expected: char, ) -> Result<(), Box<Error<'a>>>

pub(in front::wgsl) fn end_of_generic_arguments( &mut self, ) -> bool

pub(in front::wgsl) fn skip( &mut self, what: Token<'_>, ) -> bool

If the next token matches it is skipped and true is returned

pub(in front::wgsl) fn next_ident_with_span( &mut self, ) -> Result<(&'a str, Span), Box<Error<'a>>>

pub(in front::wgsl) fn peek_ident_with_span( &mut self, ) -> Result<(&'a str, Span), Box<Error<'a>>>

fn word_as_ident_with_span( word: &'a str, span: Span, ) -> Result<(&'a str, Span), Box<Error<'a>>>

pub(in front::wgsl) fn next_ident( &mut self, ) -> Result<Ident<'a>, Box<Error<'a>>>

fn word_as_ident( word: &'a str, span: Span, ) -> Result<(&'a str, Span), Box<Error<'a>>>

pub(in front::wgsl) fn next_scalar_generic( &mut self, ) -> Result<Scalar, Box<Error<'a>>>

Parses a generic scalar type, for example <f32>.

pub(in front::wgsl) fn next_scalar_generic_with_span( &mut self, ) -> Result<(Scalar, Span), Box<Error<'a>>>

Parses a generic scalar type, for example <f32>.

Returns the span covering the inner type, excluding the brackets.

pub(in front::wgsl) fn next_storage_access( &mut self, ) -> Result<StorageAccess, Box<Error<'a>>>

pub(in front::wgsl) fn next_format_generic( &mut self, ) -> Result<(StorageFormat, StorageAccess), Box<Error<'a>>>

pub(in front::wgsl) fn next_acceleration_structure_flags( &mut self, ) -> Result<bool, Box<Error<'a>>>

pub(in front::wgsl) fn open_arguments( &mut self, ) -> Result<(), Box<Error<'a>>>

pub(in front::wgsl) fn close_arguments( &mut self, ) -> Result<(), Box<Error<'a>>>

pub(in front::wgsl) fn next_argument( &mut self, ) -> Result<bool, Box<Error<'a>>>

Trait Implementations

impl<'a> Clone for Lexer<'a>

fn clone(&self) -> Lexer<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.