erick-alcachofa
ffd66f1f86
Signed-off-by: erick-alcachofa <erick@artichoke.dev> Introduced `peekExpect(std::size_t, TokenV)` to the Tokenizer class, enabling token lookahead with explicit token type checks. This method returns an `Unexpected` error with diagnostic info if the expected token type does not match the peeked token. Includes a special case handling (workaround) for distinguishing between `>` and `>>` tokens when parsing the token stream.
60 lines
1.4 KiB
C++
60 lines
1.4 KiB
C++
#pragma once
|
|
|
|
#include <deque>
|
|
|
|
#include <artichoke/Util/Expected.hpp>
|
|
#include <artichoke/Coroutine/Generator.hpp>
|
|
|
|
#include <artichoke/Tokenizer/Token.hpp>
|
|
#include <artichoke/Tokenizer/TokenizerRange.hpp>
|
|
|
|
namespace arti::lang {
|
|
|
|
struct [[nodiscard]] Tokenizer {
|
|
Tokenizer() noexcept = delete;
|
|
|
|
Tokenizer(std::string source) noexcept;
|
|
|
|
~Tokenizer() noexcept = default;
|
|
|
|
Tokenizer(Tokenizer &&rhs) noexcept;
|
|
Tokenizer &operator=(Tokenizer &&rhs) noexcept;
|
|
|
|
Tokenizer(const Tokenizer &rhs) = delete;
|
|
Tokenizer &operator=(const Tokenizer &rhs) = delete;
|
|
|
|
Expected<void> consume(std::size_t n = 1) noexcept;
|
|
Expected<Token> peek(std::size_t n = 0) noexcept;
|
|
Expected<Token> peekExpect(std::size_t n, TokenV tokenType) noexcept;
|
|
|
|
bool finished() const noexcept;
|
|
|
|
void swap(Tokenizer &other) noexcept;
|
|
|
|
TokenizerRange range() noexcept;
|
|
|
|
private:
|
|
Generator<Expected<Token>> tokenize();
|
|
|
|
void skip_whitespace();
|
|
Expected<void> skip_comment();
|
|
|
|
Expected<Token> readNumber();
|
|
Expected<Token> readString();
|
|
Expected<Token> readCharacter();
|
|
Expected<Token> readIdentifier();
|
|
Expected<Token> readOperator();
|
|
|
|
size_t line;
|
|
size_t column;
|
|
std::string::iterator iter;
|
|
|
|
Generator<Expected<Token>> tokensGenerator;
|
|
|
|
std::deque<Token> tokensBuffer;
|
|
|
|
std::string source;
|
|
};
|
|
|
|
} // namespace arti::lang
|