//============================================================================// // // // artichoke programming language // // // // Copyright (C) 2025 Erick Saul Guzman Ramos, whoami.artichoke.dev // // // // // // This program is free software: you can redistribute it and/or modify // // it under the terms of the GNU Affero General Public License as published // // by the Free Software Foundation, either version 3 of the License, or // // (at your option) any later version. // // // // This program is distributed in the hope that it will be useful, // // but WITHOUT ANY WARRANTY; without even the implied warranty of // // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // // GNU Affero General Public License for more details. // // // // You should have received a copy of the GNU Affero General Public License // // along with this program. If not, see . // // // //============================================================================// #pragma once #include #include #include #include #include namespace arti::lang { struct [[nodiscard]] Tokenizer { Tokenizer() noexcept = delete; Tokenizer(std::string source) noexcept; ~Tokenizer() noexcept = default; Tokenizer(Tokenizer &&rhs) noexcept; Tokenizer &operator=(Tokenizer &&rhs) noexcept; Tokenizer(const Tokenizer &rhs) = delete; Tokenizer &operator=(const Tokenizer &rhs) = delete; Expected consume(std::size_t n = 1) noexcept; Expected peek(std::size_t n = 0) noexcept; Expected peekExpect( TokenV tokenType, std::string_view message = "", std::size_t n = 0 ) noexcept; bool finished() const noexcept; void swap(Tokenizer &other) noexcept; TokenizerRange range() noexcept; private: Generator> tokenize(); void skip_whitespace(); Expected skip_comment(); Expected readNumber(); Expected readString(); Expected readCharacter(); Expected readIdentifier(); Expected readOperator(); size_t line; size_t column; std::string::iterator iter; Generator> tokensGenerator; std::deque tokensBuffer; std::string source; }; } // namespace arti::lang