erick-alcachofa
85a34bdd65
Initial version of Tokenizer and Token Generator template for coroutines (used in tokenizer) Utilities like string related functions, TrieMap, and error handling TODO: Add tests for Tokenizer TODO: Add tests for Generator
56 lines
1.2 KiB
C++
56 lines
1.2 KiB
C++
#pragma once
|
|
|
|
#include <deque>
|
|
#include <vector>
|
|
|
|
#include <artichoke/Util/Expected.hpp>
|
|
#include <artichoke/Coroutine/Generator.hpp>
|
|
|
|
#include <artichoke/Tokenizer/Token.hpp>
|
|
|
|
namespace arti::lang {
|
|
|
|
struct [[nodiscard]] Tokenizer {
|
|
Tokenizer() noexcept = delete;
|
|
|
|
Tokenizer(std::string source) noexcept;
|
|
|
|
~Tokenizer() noexcept = default;
|
|
|
|
Tokenizer(Tokenizer &&rhs) noexcept;
|
|
Tokenizer &operator=(Tokenizer &&rhs) noexcept;
|
|
|
|
Tokenizer(const Tokenizer &rhs) = delete;
|
|
Tokenizer &operator=(const Tokenizer &rhs) = delete;
|
|
|
|
Expected<void> consume(std::size_t n = 1) noexcept;
|
|
Expected<Token> peek(std::size_t n = 0) noexcept;
|
|
|
|
bool finished() const noexcept;
|
|
|
|
void swap(Tokenizer &other) noexcept;
|
|
|
|
private:
|
|
Generator<Expected<Token>> tokenize();
|
|
|
|
void skip_whitespace();
|
|
|
|
Expected<Token> readNumber();
|
|
Expected<Token> readString();
|
|
Expected<Token> readCharacter();
|
|
Expected<Token> readIdentifier();
|
|
Expected<Token> readOperator();
|
|
|
|
size_t line;
|
|
size_t column;
|
|
std::string::iterator iter;
|
|
|
|
Generator<Expected<Token>> tokensGenerator;
|
|
|
|
std::deque<Token> tokensBuffer;
|
|
|
|
std::string source;
|
|
};
|
|
|
|
} // namespace arti::lang
|