feat: Added peekExpect method for token type validation in tokenizer
Signed-off-by: erick-alcachofa <erick@artichoke.dev> Introduced `peekExpect(std::size_t, TokenV)` to the Tokenizer class, enabling token lookahead with explicit token type checks. This method returns an `Unexpected` error with diagnostic info if the expected token type does not match the peeked token. Includes a special case handling (workaround) for distinguishing between `>` and `>>` tokens when parsing the token stream.
This commit is contained in:
parent
f5be339f43
commit
ffd66f1f86
@ -25,6 +25,7 @@ namespace arti::lang {
|
||||
|
||||
Expected<void> consume(std::size_t n = 1) noexcept;
|
||||
Expected<Token> peek(std::size_t n = 0) noexcept;
|
||||
Expected<Token> peekExpect(std::size_t n, TokenV tokenType) noexcept;
|
||||
|
||||
bool finished() const noexcept;
|
||||
|
||||
|
||||
@ -106,6 +106,62 @@ namespace arti::lang {
|
||||
return tokensBuffer.at(n);
|
||||
}
|
||||
|
||||
Expected<Token>
|
||||
Tokenizer::peekExpect(std::size_t n, TokenV tokenType) noexcept {
|
||||
if (tokensBuffer.size() > (n + 1)) {
|
||||
auto tokenAt = tokensBuffer.at(n);
|
||||
|
||||
if (tokenAt.value != tokenType) {
|
||||
return Unexpected<> {
|
||||
Exception{
|
||||
.line = tokenAt.line,
|
||||
.column = tokenAt.column,
|
||||
.message = std::format(
|
||||
"OExpected token of type {}, got {}",
|
||||
toString(tokenType), toString(tokenAt)
|
||||
)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return tokenAt;
|
||||
}
|
||||
|
||||
auto token = peek(n);
|
||||
|
||||
if (!token) {
|
||||
return token;
|
||||
}
|
||||
|
||||
/* TODO: Look for a nicer fix for this corner case */
|
||||
if (tokenType == TokenV::opGt && token->value == TokenV::opRShift) {
|
||||
token->strValue.remove_suffix(1);
|
||||
token->value = TokenV::opGt;
|
||||
auto peekTok = *token;
|
||||
tokensBuffer.pop_back();
|
||||
tokensBuffer.push_back(*token);
|
||||
token->column += 1;
|
||||
token->strValue = std::string_view{token->strValue.begin() + 1, 1};
|
||||
tokensBuffer.push_back(*token);
|
||||
token = peekTok;
|
||||
}
|
||||
|
||||
if (token->value != tokenType) {
|
||||
return Unexpected<> {
|
||||
Exception{
|
||||
.line = token->line,
|
||||
.column = token->column,
|
||||
.message = std::format(
|
||||
"OExpected token of type {}, got {}",
|
||||
toString(tokenType), toString(*token)
|
||||
)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return *token;
|
||||
}
|
||||
|
||||
bool Tokenizer::finished() const noexcept {
|
||||
if (tokensGenerator.finished()) {
|
||||
if (!tokensBuffer.empty()) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user