feat: Added peekExpect method for token type validation in tokenizer

Signed-off-by: erick-alcachofa <erick@artichoke.dev>

Introduced `peekExpect(std::size_t, TokenV)` to the Tokenizer class, enabling
token lookahead with explicit token type checks. This method returns an
`Unexpected` error with diagnostic info if the expected token type does not
match the peeked token.

Includes a special case handling (workaround) for distinguishing between
`>` and `>>` tokens when parsing the token stream.
This commit is contained in:
erick-alcachofa 2025-10-05 22:51:16 -06:00
parent f5be339f43
commit ffd66f1f86
Signed by: me
GPG Key ID: 6FA5F8643444BAFA
2 changed files with 57 additions and 0 deletions

View File

@ -25,6 +25,7 @@ namespace arti::lang {
Expected<void> consume(std::size_t n = 1) noexcept;
Expected<Token> peek(std::size_t n = 0) noexcept;
Expected<Token> peekExpect(std::size_t n, TokenV tokenType) noexcept;
bool finished() const noexcept;

View File

@ -106,6 +106,62 @@ namespace arti::lang {
return tokensBuffer.at(n);
}
Expected<Token>
Tokenizer::peekExpect(std::size_t n, TokenV tokenType) noexcept {
if (tokensBuffer.size() > (n + 1)) {
auto tokenAt = tokensBuffer.at(n);
if (tokenAt.value != tokenType) {
return Unexpected<> {
Exception{
.line = tokenAt.line,
.column = tokenAt.column,
.message = std::format(
"OExpected token of type {}, got {}",
toString(tokenType), toString(tokenAt)
)
}
};
}
return tokenAt;
}
auto token = peek(n);
if (!token) {
return token;
}
/* TODO: Look for a nicer fix for this corner case */
if (tokenType == TokenV::opGt && token->value == TokenV::opRShift) {
token->strValue.remove_suffix(1);
token->value = TokenV::opGt;
auto peekTok = *token;
tokensBuffer.pop_back();
tokensBuffer.push_back(*token);
token->column += 1;
token->strValue = std::string_view{token->strValue.begin() + 1, 1};
tokensBuffer.push_back(*token);
token = peekTok;
}
if (token->value != tokenType) {
return Unexpected<> {
Exception{
.line = token->line,
.column = token->column,
.message = std::format(
"OExpected token of type {}, got {}",
toString(tokenType), toString(*token)
)
}
};
}
return *token;
}
bool Tokenizer::finished() const noexcept {
if (tokensGenerator.finished()) {
if (!tokensBuffer.empty()) {