erick-alcachofa
d0599d374f
Signed-off-by: erick-alcachofa <erick@artichoke.dev> This commit lays the foundational groundwork for the artichoke language parser by introducing the formal language grammar specification. The tokenizer was updated to include new operators and keywords, also added the posibility to handle comments. Key Additions: - Implemented support for C-style block comments (`/* ... */`), including error handling for unclosed comments. - Added all necessary tokens for missing keywords (e.g., `module`, `export`, `using`, `match`, `loop`) and operators (e.g., `+=`, `:=`, `.#`, `.*`, `.@`). - The `Token` enum has been expanded to reflect the full language feature set. Documentation: - Added `docs/grammar.ebnf` which contains the official, well-structured EBNF grammar for the language. - Added `docs/readme.md` providing a detailed technical overview of the language's features, syntax, and semantics. BREAKING CHANGE: The `kwVariant` and `kwMut` tokens have been removed to align with the updated language design defined in the new grammar.
57 lines
1.2 KiB
C++
57 lines
1.2 KiB
C++
#pragma once
|
|
|
|
#include <deque>
|
|
#include <vector>
|
|
|
|
#include <artichoke/Util/Expected.hpp>
|
|
#include <artichoke/Coroutine/Generator.hpp>
|
|
|
|
#include <artichoke/Tokenizer/Token.hpp>
|
|
|
|
namespace arti::lang {
|
|
|
|
struct [[nodiscard]] Tokenizer {
|
|
Tokenizer() noexcept = delete;
|
|
|
|
Tokenizer(std::string source) noexcept;
|
|
|
|
~Tokenizer() noexcept = default;
|
|
|
|
Tokenizer(Tokenizer &&rhs) noexcept;
|
|
Tokenizer &operator=(Tokenizer &&rhs) noexcept;
|
|
|
|
Tokenizer(const Tokenizer &rhs) = delete;
|
|
Tokenizer &operator=(const Tokenizer &rhs) = delete;
|
|
|
|
Expected<void> consume(std::size_t n = 1) noexcept;
|
|
Expected<Token> peek(std::size_t n = 0) noexcept;
|
|
|
|
bool finished() const noexcept;
|
|
|
|
void swap(Tokenizer &other) noexcept;
|
|
|
|
private:
|
|
Generator<Expected<Token>> tokenize();
|
|
|
|
void skip_whitespace();
|
|
Expected<void> skip_comment();
|
|
|
|
Expected<Token> readNumber();
|
|
Expected<Token> readString();
|
|
Expected<Token> readCharacter();
|
|
Expected<Token> readIdentifier();
|
|
Expected<Token> readOperator();
|
|
|
|
size_t line;
|
|
size_t column;
|
|
std::string::iterator iter;
|
|
|
|
Generator<Expected<Token>> tokensGenerator;
|
|
|
|
std::deque<Token> tokensBuffer;
|
|
|
|
std::string source;
|
|
};
|
|
|
|
} // namespace arti::lang
|