86 lines
3.1 KiB
C++
86 lines
3.1 KiB
C++
//============================================================================//
|
|
// //
|
|
// artichoke programming language //
|
|
// //
|
|
// Copyright (C) 2025 Erick Saul Guzman Ramos, whoami.artichoke.dev //
|
|
// //
|
|
// //
|
|
// This program is free software: you can redistribute it and/or modify //
|
|
// it under the terms of the GNU Affero General Public License as published //
|
|
// by the Free Software Foundation, either version 3 of the License, or //
|
|
// (at your option) any later version. //
|
|
// //
|
|
// This program is distributed in the hope that it will be useful, //
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
|
|
// GNU Affero General Public License for more details. //
|
|
// //
|
|
// You should have received a copy of the GNU Affero General Public License //
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>. //
|
|
// //
|
|
//============================================================================//
|
|
|
|
#pragma once
|
|
|
|
#include <deque>
|
|
|
|
#include <artichoke/Util/Expected.hpp>
|
|
#include <artichoke/Coroutine/Generator.hpp>
|
|
|
|
#include <artichoke/Tokenizer/Token.hpp>
|
|
#include <artichoke/Tokenizer/TokenizerRange.hpp>
|
|
|
|
namespace arti::lang {
|
|
|
|
struct [[nodiscard]] Tokenizer {
|
|
Tokenizer() noexcept = delete;
|
|
|
|
Tokenizer(std::string source) noexcept;
|
|
|
|
~Tokenizer() noexcept = default;
|
|
|
|
Tokenizer(Tokenizer &&rhs) noexcept;
|
|
Tokenizer &operator=(Tokenizer &&rhs) noexcept;
|
|
|
|
Tokenizer(const Tokenizer &rhs) = delete;
|
|
Tokenizer &operator=(const Tokenizer &rhs) = delete;
|
|
|
|
Expected<void> consume(std::size_t n = 1) noexcept;
|
|
Expected<Token> peek(std::size_t n = 0) noexcept;
|
|
Expected<Token> peekExpect(
|
|
TokenV tokenType,
|
|
std::string_view message = "",
|
|
std::size_t n = 0
|
|
) noexcept;
|
|
|
|
bool finished() const noexcept;
|
|
|
|
void swap(Tokenizer &other) noexcept;
|
|
|
|
TokenizerRange range() noexcept;
|
|
|
|
private:
|
|
Generator<Expected<Token>> tokenize();
|
|
|
|
void skip_whitespace();
|
|
Expected<void> skip_comment();
|
|
|
|
Expected<Token> readNumber();
|
|
Expected<Token> readString();
|
|
Expected<Token> readCharacter();
|
|
Expected<Token> readIdentifier();
|
|
Expected<Token> readOperator();
|
|
|
|
size_t line;
|
|
size_t column;
|
|
std::string::iterator iter;
|
|
|
|
Generator<Expected<Token>> tokensGenerator;
|
|
|
|
std::deque<Token> tokensBuffer;
|
|
|
|
std::string source;
|
|
};
|
|
|
|
} // namespace arti::lang
|