#pragma once #include #include #include #include #include namespace arti::lang { struct [[nodiscard]] Tokenizer { Tokenizer() noexcept = delete; Tokenizer(std::string source) noexcept; ~Tokenizer() noexcept = default; Tokenizer(Tokenizer &&rhs) noexcept; Tokenizer &operator=(Tokenizer &&rhs) noexcept; Tokenizer(const Tokenizer &rhs) = delete; Tokenizer &operator=(const Tokenizer &rhs) = delete; Expected consume(std::size_t n = 1) noexcept; Expected peek(std::size_t n = 0) noexcept; bool finished() const noexcept; void swap(Tokenizer &other) noexcept; private: Generator> tokenize(); void skip_whitespace(); Expected skip_comment(); Expected readNumber(); Expected readString(); Expected readCharacter(); Expected readIdentifier(); Expected readOperator(); size_t line; size_t column; std::string::iterator iter; Generator> tokensGenerator; std::deque tokensBuffer; std::string source; }; } // namespace arti::lang