//============================================================================//
// //
// artichoke programming language //
// //
// Copyright (C) 2025 Erick Saul Guzman Ramos, whoami.artichoke.dev //
// //
// //
// This program is free software: you can redistribute it and/or modify //
// it under the terms of the GNU Affero General Public License as published //
// by the Free Software Foundation, either version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU Affero General Public License for more details. //
// //
// You should have received a copy of the GNU Affero General Public License //
// along with this program. If not, see . //
// //
//============================================================================//
#pragma once
#include
#include
#include
#include
#include
namespace arti::lang {
struct [[nodiscard]] Tokenizer {
Tokenizer() noexcept = delete;
Tokenizer(std::string source) noexcept;
~Tokenizer() noexcept = default;
Tokenizer(Tokenizer &&rhs) noexcept;
Tokenizer &operator=(Tokenizer &&rhs) noexcept;
Tokenizer(const Tokenizer &rhs) = delete;
Tokenizer &operator=(const Tokenizer &rhs) = delete;
Expected consume(std::size_t n = 1) noexcept;
Expected peek(std::size_t n = 0) noexcept;
Expected peekExpect(
TokenV tokenType,
std::string_view message = "",
std::size_t n = 0
) noexcept;
bool finished() const noexcept;
void swap(Tokenizer &other) noexcept;
TokenizerRange range() noexcept;
private:
Generator> tokenize();
void skip_whitespace();
Expected skip_comment();
Expected readNumber();
Expected readString();
Expected readCharacter();
Expected readIdentifier();
Expected readOperator();
size_t line;
size_t column;
std::string::iterator iter;
Generator> tokensGenerator;
std::deque tokensBuffer;
std::string source;
};
} // namespace arti::lang