From 85a34bdd65bb7cd2afb5a0bf6263a4452404e6f8 Mon Sep 17 00:00:00 2001 From: erick-alcachofa Date: Mon, 10 Mar 2025 01:20:23 -0600 Subject: [PATCH] feat: Added Token, Tokenizer, Generator, and some utilities Initial version of Tokenizer and Token Generator template for coroutines (used in tokenizer) Utilities like string related functions, TrieMap, and error handling TODO: Add tests for Tokenizer TODO: Add tests for Generator --- lib/include/artichoke/Coroutine/Generator.hpp | 272 +++++++ lib/include/artichoke/Coroutine/Utils.hpp | 5 + lib/include/artichoke/Tokenizer/Token.hpp | 108 +++ lib/include/artichoke/Tokenizer/Tokenizer.hpp | 55 ++ lib/include/artichoke/Util/Demangle.hpp | 16 + lib/include/artichoke/Util/Expected.hpp | 84 +++ lib/include/artichoke/Util/OverloadSet.hpp | 13 + lib/include/artichoke/Util/Strings.hpp | 49 ++ lib/include/artichoke/Util/TrieMap.hpp | 67 ++ lib/src/Tokenizer/Token.cpp | 192 +++++ lib/src/Tokenizer/Tokenizer.cpp | 661 ++++++++++++++++++ lib/src/Util/Demangle.cpp | 52 ++ lib/src/Util/Util.cpp | 0 13 files changed, 1574 insertions(+) create mode 100644 lib/include/artichoke/Coroutine/Generator.hpp create mode 100644 lib/include/artichoke/Coroutine/Utils.hpp create mode 100644 lib/include/artichoke/Tokenizer/Token.hpp create mode 100644 lib/include/artichoke/Util/Demangle.hpp create mode 100644 lib/include/artichoke/Util/Expected.hpp create mode 100644 lib/include/artichoke/Util/Strings.hpp create mode 100644 lib/include/artichoke/Util/TrieMap.hpp create mode 100644 lib/src/Tokenizer/Token.cpp create mode 100644 lib/src/Util/Demangle.cpp delete mode 100644 lib/src/Util/Util.cpp diff --git a/lib/include/artichoke/Coroutine/Generator.hpp b/lib/include/artichoke/Coroutine/Generator.hpp new file mode 100644 index 0000000..53b811d --- /dev/null +++ b/lib/include/artichoke/Coroutine/Generator.hpp @@ -0,0 +1,272 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace arti::lang { + + template + requires (not std::is_void_v) + struct Generator; + + template + requires (not std::is_void_v) + struct GeneratorPromise { + using value_type = std::remove_cvref_t; + using reference_type = value_type &; + using pointer_type = value_type *; + + using ValueType = value_type; + using ReferenceType = reference_type; + using PointerType = pointer_type; + + GeneratorPromise() = default; + + Generator get_return_object(); + + std::suspend_always initial_suspend() const noexcept { + return {}; + } + + std::suspend_always final_suspend() const noexcept { + return {}; + } + + void unhandled_exception() { + exception = std::current_exception(); + } + + void rethrow_if_exception() { + if (exception) { + std::rethrow_exception(exception); + } + } + + void return_value(const ValueType &val) noexcept { + value = val; + } + + void return_value(ValueType &&val) noexcept { + value = std::forward(val); + } + + std::suspend_always yield_value(const ValueType &val) noexcept { + value = val; + return {}; + } + + std::suspend_always yield_value(ValueType &&val) noexcept { + value = std::forward(val); + return {}; + } + + decltype(auto) await_transform(auto &&awaitable) = delete; + + std::exception_ptr exception; + std::variant value; + }; + + struct GeneratorSentinel { }; + + template + requires (not std::is_void_v) + struct GeneratorIterator { + using iterator_category = std::input_iterator_tag; + using difference_type = std::ptrdiff_t; + + using promise_type = GeneratorPromise; + + using value_type = typename promise_type::value_type; + using reference_type = value_type &; + using pointer_type = value_type *; + + using handle_type = std::coroutine_handle; + + using IteratorCategory = iterator_category; + using DifferenceType = difference_type; + using PromiseType = promise_type; + using ValueType = value_type; + using ReferenceType = reference_type; + using PointerType = pointer_type; + using HandleType = handle_type; + + GeneratorIterator() noexcept = default; + ~GeneratorIterator() noexcept = default; + + GeneratorIterator(HandleType handle) noexcept + : handle(handle) { } + + GeneratorIterator(GeneratorIterator &&rhs) noexcept + : handle(std::exchange(rhs.handle, nullptr)) { } + + GeneratorIterator &operator=(GeneratorIterator &&rhs) noexcept { + handle = std::exchange(rhs.handle, nullptr); + return *this; + } + + GeneratorIterator(const GeneratorIterator &) noexcept = delete; + GeneratorIterator &operator=(const GeneratorIterator &) noexcept = delete; + + GeneratorIterator &operator++() { + handle.resume(); + + if (handle.done()) { + handle.promise().rethrow_if_exception(); + } + + return *this; + } + + void operator++(int) { + (void) this->operator++(); + } + + ReferenceType operator*() const noexcept { + auto &ref = handle.promise().value; + + if (std::holds_alternative(ref)) { + return *std::get(ref); + } + else { + return std::get(ref); + } + } + + PointerType operator->() const noexcept { + auto &ref = handle.promise().value; + + if (std::holds_alternative(ref)) { + return std::get(ref); + } + else { + return &std::get(ref); + } + } + + friend bool operator==(const GeneratorIterator &it, GeneratorSentinel) { + return (! it.handle || it.handle.done()); + } + + friend bool operator!=(const GeneratorIterator &it, GeneratorSentinel s) { + return ! (it == s); + } + + friend bool operator==(GeneratorSentinel s, const GeneratorIterator &it) { + return it == s; + } + + friend bool operator!=(GeneratorSentinel s, const GeneratorIterator &it) { + return it != s; + } + + HandleType handle; + }; + + template + requires (not std::is_void_v) + struct [[nodiscard]] Generator { + using promise_type = GeneratorPromise; + using handle_type = std::coroutine_handle; + + using iterator_type = GeneratorIterator; + using sentinel_type = GeneratorSentinel; + + using PromiseType = promise_type; + using HandleType = handle_type; + using IteratorType = iterator_type; + using SentinelType = sentinel_type; + + Generator() noexcept = default; + + ~Generator() noexcept { + if (handle) { + handle.destroy(); + } + } + + explicit Generator(HandleType handle) noexcept + : handle(handle) { } + + Generator(Generator &&rhs) noexcept + : handle(std::exchange(rhs.handle, nullptr)) { } + + Generator &operator=(Generator &&rhs) noexcept { + this->~Generator(); + handle = std::exchange(rhs.handle, nullptr); + return *this; + } + + Generator(const Generator &) noexcept = delete; + Generator &operator=(const Generator &) noexcept = delete; + + bool finished() const noexcept { + return (not handle || (handle.done() and !handle.promise().exception)); + } + + void next() { + if (not finished()) { + handle.resume(); + + if (handle.done()) { + handle.promise().rethrow_if_exception(); + } + } + } + + PromiseType::ReferenceType get() { + using PointerType = PromiseType::PointerType; + using ValueType = PromiseType::ValueType; + + auto &ref = handle.promise().value; + + if (std::holds_alternative(ref)) { + return *std::get(ref); + } + else { + return std::get(ref); + } + } + + IteratorType begin() { + if (handle) { + handle.resume(); + + if (handle.done()) { + handle.promise().rethrow_if_exception(); + } + } + + return { handle }; + } + + SentinelType end() const noexcept { + return {}; + } + + void swap(Generator &other) noexcept { + std::swap(handle, other.handle); + } + + private: + HandleType handle; + }; + + template + requires (not std::is_void_v) + Generator GeneratorPromise::get_return_object() { + using GeneratorType = Generator; + using HandleType = typename GeneratorType::HandleType; + + return GeneratorType{ HandleType::from_promise(*this) }; + } + +} // namespace arti::lang diff --git a/lib/include/artichoke/Coroutine/Utils.hpp b/lib/include/artichoke/Coroutine/Utils.hpp new file mode 100644 index 0000000..6a6341c --- /dev/null +++ b/lib/include/artichoke/Coroutine/Utils.hpp @@ -0,0 +1,5 @@ +#pragma once + +#define yield co_yield +#define await co_await +#define coret co_return diff --git a/lib/include/artichoke/Tokenizer/Token.hpp b/lib/include/artichoke/Tokenizer/Token.hpp new file mode 100644 index 0000000..e51f8c1 --- /dev/null +++ b/lib/include/artichoke/Tokenizer/Token.hpp @@ -0,0 +1,108 @@ +#pragma once + +#include + +namespace arti::lang { + + enum class TokenV; + + struct Token { + TokenV value; + + std::size_t line; + std::size_t column; + + std::string_view strValue; + }; + + enum class TokenV { + tkEOF, + tkString, + tkDecimal, + tkInteger, + tkCharacter, + tkIdentifier, + + opDot, + opMod, + opPlus, + opHyphen, + opSlash, + opBang, + opStar, + opColon, + opComma, + opAssign, + opAccess, + opSemicolon, + + opCaret, + opTilde, + + opEq, + opNeq, + + opLt, + opGt, + + opLtEq, + opGtEq, + + opLShift, + opRShift, + + opBoolAnd, + opBoolOr, + + opAnd, + opOr, + + opLParen, + opRParen, + + opLBracket, + opRBracket, + + opLSquirly, + opRSquirly, + + opArrow, + + kwOr, + kwNot, + kwAnd, + + kwIf, + kwElse, + + kwFn, + kwEnum, + kwStruct, + kwVariant, + + kwDef, + kwLet, + kwMut, + + kwFor, + kwWhile, + + kwReturn, + kwUnreachable, + + kwDefer, + kwErrDefer, + + kwTrue, + kwFalse, + + kwNull, + + kwImport, + kwExport, + kwModule, + }; + + std::string toString(const Token &value); + +} // namespace arti::lang diff --git a/lib/include/artichoke/Tokenizer/Tokenizer.hpp b/lib/include/artichoke/Tokenizer/Tokenizer.hpp index e69de29..e64690d 100644 --- a/lib/include/artichoke/Tokenizer/Tokenizer.hpp +++ b/lib/include/artichoke/Tokenizer/Tokenizer.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include + +#include +#include + +#include + +namespace arti::lang { + + struct [[nodiscard]] Tokenizer { + Tokenizer() noexcept = delete; + + Tokenizer(std::string source) noexcept; + + ~Tokenizer() noexcept = default; + + Tokenizer(Tokenizer &&rhs) noexcept; + Tokenizer &operator=(Tokenizer &&rhs) noexcept; + + Tokenizer(const Tokenizer &rhs) = delete; + Tokenizer &operator=(const Tokenizer &rhs) = delete; + + Expected consume(std::size_t n = 1) noexcept; + Expected peek(std::size_t n = 0) noexcept; + + bool finished() const noexcept; + + void swap(Tokenizer &other) noexcept; + + private: + Generator> tokenize(); + + void skip_whitespace(); + + Expected readNumber(); + Expected readString(); + Expected readCharacter(); + Expected readIdentifier(); + Expected readOperator(); + + size_t line; + size_t column; + std::string::iterator iter; + + Generator> tokensGenerator; + + std::deque tokensBuffer; + + std::string source; + }; + +} // namespace arti::lang diff --git a/lib/include/artichoke/Util/Demangle.hpp b/lib/include/artichoke/Util/Demangle.hpp new file mode 100644 index 0000000..20db504 --- /dev/null +++ b/lib/include/artichoke/Util/Demangle.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace arti::lang { + + enum struct DemangleExceptCode { + ecMemAlloc, + ecInvalid, + }; + + Expected demangle(std::string_view str); + + std::string demangleErrorString(DemangleExceptCode ec); + +} // namespace arti::lang diff --git a/lib/include/artichoke/Util/Expected.hpp b/lib/include/artichoke/Util/Expected.hpp new file mode 100644 index 0000000..cc6a146 --- /dev/null +++ b/lib/include/artichoke/Util/Expected.hpp @@ -0,0 +1,84 @@ +#pragma once + +#include + +#include +#include +#include + +namespace arti::lang { + + enum struct ExceptCode { + ecEOF, + ecEOB, + ecInvalidToken, + ecStdException, + ecInvalidLiteral, + ecInvalidCharacter, + ecInvalidIndex, + }; + + struct Exception { + size_t line; + size_t column; + std::string message; + }; + + template + using Expected = std::expected; + + template + using Unexpected = std::unexpected; + + template + std::string exceptionMessage(Args &&...args) { + using enum ExceptCode; + + if constexpr (code == ecEOF) { + return "Reached EOF"; + } + else if constexpr (code == ecEOB) { + return "Buffer empty"; + } + else if constexpr (code == ecStdException) { + return std::format( + "Catched instance of {}: '{}'", + std::forward(args)... + ); + } + else if constexpr (code == ecInvalidLiteral) { + return std::format( + "Invalid literal, expected {} got {}", + std::forward(args)... + ); + } + else if constexpr (code == ecInvalidCharacter) { + return std::format( + "Invalid character found '{}'", + std::forward(args)... + ); + } + else if constexpr (code == ecInvalidToken) { + return std::format( + "Invalid token found '{}'", + std::forward(args)... + ); + } + else if constexpr (code == ecInvalidIndex) { + return "Invalid index"; + } + else { + return "Unknown error"; + } + } + + template + static inline Unexpected<> + langException(size_t line, size_t col, Args &&...args) { + return Unexpected{ + Exception{ line, + col, exceptionMessage(std::forward(args)...) } + }; + } + +} // namespace arti::lang diff --git a/lib/include/artichoke/Util/OverloadSet.hpp b/lib/include/artichoke/Util/OverloadSet.hpp index e69de29..3cdcfc2 100644 --- a/lib/include/artichoke/Util/OverloadSet.hpp +++ b/lib/include/artichoke/Util/OverloadSet.hpp @@ -0,0 +1,13 @@ +#pragma once + +namespace arti::lang { + + template + struct OverloadSet { + using Ts::operator()...; + }; + + template + OverloadSet(Ts...) -> OverloadSet; + +} diff --git a/lib/include/artichoke/Util/Strings.hpp b/lib/include/artichoke/Util/Strings.hpp new file mode 100644 index 0000000..748096a --- /dev/null +++ b/lib/include/artichoke/Util/Strings.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include + +namespace arti::lang { + + constexpr size_t CE_TAB_SIZE = 2; + + static inline constexpr bool isLetter(char c) { + return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')); + } + + static inline constexpr char toLower(char c) { + if (isLetter(c)) { + return (c | 0b00100000); + } + + return c; + } + + static inline constexpr bool isDigit(char c) { + return (c >= '0' && c <= '9'); + } + + static inline constexpr bool isFirstIdentChar(char c) { + return isLetter(c) || c == '_'; + } + + static inline constexpr bool isIdentChar(char c) { + return isLetter(c) || c == '_'; + } + + static inline constexpr bool isHexChar(char c) { + return isDigit(c) || ((toLower(c) >= 'a' && toLower(c) <= 'f')); + } + + static inline constexpr bool isBinaryChar(char c) { + return c == '0' || c == '1'; + } + + static inline constexpr bool isOctalChar(char c) { + return (c >= '0' && c <= '7'); + } + + static inline constexpr bool isWhiteSpace(char c) { + return (c == ' ' || c == '\t' || c == '\n'); + } + +} // namespace arti::lang diff --git a/lib/include/artichoke/Util/TrieMap.hpp b/lib/include/artichoke/Util/TrieMap.hpp new file mode 100644 index 0000000..c6cbe3c --- /dev/null +++ b/lib/include/artichoke/Util/TrieMap.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include + +namespace arti::lang { + + template + struct TrieMap { + struct Node { + bool isLeaf; + std::optional value; + std::map childs; + + Node() noexcept + : isLeaf(false) + , value(std::nullopt) + , childs() { } + + ~Node() noexcept = default; + + Node(Node &&) noexcept = default; + Node &operator=(Node &&) noexcept = default; + + Node(const Node &) noexcept = default; + Node &operator=(const Node &) noexcept = default; + }; + + Node root; + + TrieMap() noexcept = default; + ~TrieMap() noexcept = default; + + TrieMap(TrieMap &&) noexcept = default; + TrieMap &operator=(TrieMap &&) noexcept = default; + + TrieMap(const TrieMap &) noexcept = default; + TrieMap &operator=(const TrieMap &) noexcept = default; + + void insert(std::string_view str, T &&value) noexcept { + auto ptrNode = &root; + + for (auto c : str) { + ptrNode = &(ptrNode->childs[c]); + } + + ptrNode->isLeaf = true; + ptrNode->value = std::forward(value); + } + + T *find(std::string_view str) const noexcept { + auto ptrNode = &root; + + for (auto c : str) { + if (not ptrNode->childs.contains(c)) { + return nullptr; + } + + ptrNode = &(ptrNode->childs.at(c)); + } + + return ptrNode; + } + }; + +} // namespace arti::lang diff --git a/lib/src/Tokenizer/Token.cpp b/lib/src/Tokenizer/Token.cpp new file mode 100644 index 0000000..9923869 --- /dev/null +++ b/lib/src/Tokenizer/Token.cpp @@ -0,0 +1,192 @@ +#include + +#include + +namespace arti::lang { + + std::string toString(const Token &value) { + using enum TokenV; + + switch (value.value) { + case tkEOF: + return std::format("Token{{ {} }}", "tkEOF", value.strValue); + case tkString: + return std::format("Token{{ {}, {} }}", "tkString", value.strValue); + case tkDecimal: + return std::format( + "Token{{ {}, {} }}", + "tkDecimal", + value.strValue + ); + case tkInteger: + return std::format( + "Token{{ {}, {} }}", + "tkInteger", + value.strValue + ); + case tkCharacter: + return std::format( + "Token{{ {}, {} }}", + "tkCharacter", + value.strValue + ); + case tkIdentifier: + return std::format( + "Token{{ {}, {} }}", + "tkIdentifier", + value.strValue + ); + case opDot: + return std::format("Token{{ {}, {} }}", "opDot", value.strValue); + case opMod: + return std::format("Token{{ {}, {} }}", "opMod", value.strValue); + case opPlus: + return std::format("Token{{ {}, {} }}", "opPlus", value.strValue); + case opHyphen: + return std::format("Token{{ {}, {} }}", "opHyphen", value.strValue); + case opSlash: + return std::format("Token{{ {}, {} }}", "opSlash", value.strValue); + case opBang: + return std::format("Token{{ {}, {} }}", "opBang", value.strValue); + case opStar: + return std::format("Token{{ {}, {} }}", "opStar", value.strValue); + case opColon: + return std::format("Token{{ {}, {} }}", "opColon", value.strValue); + case opComma: + return std::format("Token{{ {}, {} }}", "opComma", value.strValue); + case opAssign: + return std::format("Token{{ {}, {} }}", "opAssign", value.strValue); + case opAccess: + return std::format("Token{{ {}, {} }}", "opAccess", value.strValue); + case opSemicolon: + return std::format( + "Token{{ {}, {} }}", + "opSemicolon", + value.strValue + ); + case opCaret: + return std::format("Token{{ {}, {} }}", "opCaret", value.strValue); + case opTilde: + return std::format("Token{{ {}, {} }}", "opTilde", value.strValue); + case opEq: + return std::format("Token{{ {}, {} }}", "opEq", value.strValue); + case opNeq: + return std::format("Token{{ {}, {} }}", "opNeq", value.strValue); + case opLt: + return std::format("Token{{ {}, {} }}", "opLt", value.strValue); + case opGt: + return std::format("Token{{ {}, {} }}", "opGt", value.strValue); + case opLtEq: + return std::format("Token{{ {}, {} }}", "opLtEq", value.strValue); + case opGtEq: + return std::format("Token{{ {}, {} }}", "opGtEq", value.strValue); + case opLShift: + return std::format("Token{{ {}, {} }}", "opLShift", value.strValue); + case opRShift: + return std::format("Token{{ {}, {} }}", "opRShift", value.strValue); + case opBoolAnd: + return std::format( + "Token{{ {}, {} }}", + "opBoolAnd", + value.strValue + ); + case opBoolOr: + return std::format("Token{{ {}, {} }}", "opBoolOr", value.strValue); + case opAnd: + return std::format("Token{{ {}, {} }}", "opAnd", value.strValue); + case opOr: + return std::format("Token{{ {}, {} }}", "opOr", value.strValue); + case opLParen: + return std::format("Token{{ {}, {} }}", "opLParen", value.strValue); + case opRParen: + return std::format("Token{{ {}, {} }}", "opRParen", value.strValue); + case opLBracket: + return std::format( + "Token{{ {}, {} }}", + "opLBracket", + value.strValue + ); + case opRBracket: + return std::format( + "Token{{ {}, {} }}", + "opRBracket", + value.strValue + ); + case opLSquirly: + return std::format( + "Token{{ {}, {} }}", + "opLSquirly", + value.strValue + ); + case opRSquirly: + return std::format( + "Token{{ {}, {} }}", + "opRSquirly", + value.strValue + ); + case opArrow: + return std::format("Token{{ {}, {} }}", "opArrow", value.strValue); + case kwOr: + return std::format("Token{{ {}, {} }}", "kwOr", value.strValue); + case kwNot: + return std::format("Token{{ {}, {} }}", "kwNot", value.strValue); + case kwAnd: + return std::format("Token{{ {}, {} }}", "kwAnd", value.strValue); + case kwIf: + return std::format("Token{{ {}, {} }}", "kwIf", value.strValue); + case kwElse: + return std::format("Token{{ {}, {} }}", "kwElse", value.strValue); + case kwFn: + return std::format("Token{{ {}, {} }}", "kwFn", value.strValue); + case kwEnum: + return std::format("Token{{ {}, {} }}", "kwEnum", value.strValue); + case kwStruct: + return std::format("Token{{ {}, {} }}", "kwStruct", value.strValue); + case kwVariant: + return std::format( + "Token{{ {}, {} }}", + "kwVariant", + value.strValue + ); + case kwDef: + return std::format("Token{{ {}, {} }}", "kwDef", value.strValue); + case kwLet: + return std::format("Token{{ {}, {} }}", "kwLet", value.strValue); + case kwMut: + return std::format("Token{{ {}, {} }}", "kwMut", value.strValue); + case kwFor: + return std::format("Token{{ {}, {} }}", "kwFor", value.strValue); + case kwWhile: + return std::format("Token{{ {}, {} }}", "kwWhile", value.strValue); + case kwReturn: + return std::format("Token{{ {}, {} }}", "kwReturn", value.strValue); + case kwUnreachable: + return std::format( + "Token{{ {}, {} }}", + "kwUnreachable", + value.strValue + ); + case kwDefer: + return std::format("Token{{ {}, {} }}", "kwDefer", value.strValue); + case kwErrDefer: + return std::format( + "Token{{ {}, {} }}", + "kwErrDefer", + value.strValue + ); + case kwTrue: + return std::format("Token{{ {}, {} }}", "kwTrue", value.strValue); + case kwFalse: + return std::format("Token{{ {}, {} }}", "kwFalse", value.strValue); + case kwNull: + return std::format("Token{{ {}, {} }}", "kwNull", value.strValue); + case kwImport: + return std::format("Token{{ {}, {} }}", "kwImport", value.strValue); + case kwExport: + return std::format("Token{{ {}, {} }}", "kwExport", value.strValue); + case kwModule: + return std::format("Token{{ {}, {} }}", "kwModule", value.strValue); + } + return ""; + } +} // namespace arti::lang diff --git a/lib/src/Tokenizer/Tokenizer.cpp b/lib/src/Tokenizer/Tokenizer.cpp index e69de29..b20f97d 100644 --- a/lib/src/Tokenizer/Tokenizer.cpp +++ b/lib/src/Tokenizer/Tokenizer.cpp @@ -0,0 +1,661 @@ +#include + +#include +#include + +#include +#include +#include + +namespace arti::lang { + + Tokenizer::Tokenizer(std::string source) noexcept + : line(0) + , column(0) + , tokensGenerator{ tokenize() } + , tokensBuffer{} + , source(std::move(source)) { + iter = this->source.begin(); + } + + Tokenizer::Tokenizer(Tokenizer &&rhs) noexcept + : line(rhs.line) + , column(rhs.column) + , iter(rhs.iter) + , tokensGenerator(std::exchange(rhs.tokensGenerator, {})) + , tokensBuffer(std::exchange(rhs.tokensBuffer, {})) + , source(std::exchange(rhs.source, "")) { } + + Tokenizer &Tokenizer::operator=(Tokenizer &&rhs) noexcept { + line = rhs.line; + column = rhs.column; + iter = std::move(rhs.iter); + source = std::move(rhs.source); + tokensBuffer = std::move(rhs.tokensBuffer); + tokensGenerator = std::move(rhs.tokensGenerator); + return *this; + } + + Expected Tokenizer::consume(std::size_t n) noexcept { + while (not tokensBuffer.empty()) { + tokensBuffer.pop_front(); + n -= 1; + } + + if (n != 0) { + return langException(line, column); + } + + return {}; + } + + Expected Tokenizer::peek(std::size_t n) noexcept { + while (not finished() && tokensBuffer.size() < (n + 1)) { + try { + tokensGenerator.next(); + } + catch (std::exception &e) { + const auto exceptionName = demangle(typeid(e).name()); + + if (not exceptionName) { + return langException( + line, + column, + "abi::__cxa_demangle(...)", + demangleErrorString(exceptionName.error()) + ); + } + + return langException( + line, + column, + exceptionName.value(), + e.what() + ); + } + + auto tok = tokensGenerator.get(); + + if (not tok) { + return tok; + } + + tokensBuffer.push_back(std::move(tok).value()); + } + + if (finished()) { + if (tokensBuffer.size() < (n + 1)) { + return langException(line, column); + } + } + + return tokensBuffer.at(n); + } + + bool Tokenizer::finished() const noexcept { + if (tokensGenerator.finished()) { + return tokensBuffer.empty(); + } + + return false; + } + + void Tokenizer::swap(Tokenizer &other) noexcept { + std::swap(line, other.line); + std::swap(column, other.column); + std::swap(iter, other.iter); + tokensGenerator.swap(other.tokensGenerator); + tokensBuffer.swap(other.tokensBuffer); + source.swap(other.source); + } + + Generator> Tokenizer::tokenize() { + while (iter != source.end()) { + skip_whitespace(); + + if (iter == source.end()) { + break; + } + + if (isDigit(*iter)) { + yield readNumber(); + } + else if (*iter == '"') { + yield readString(); + } + else if (*iter == '\'') { + yield readCharacter(); + } + else if (isFirstIdentChar(*iter)) { + yield readIdentifier(); + } + else { + yield readOperator(); + } + } + + coret Token{ .value = TokenV::tkEOF, .line = line, .column = column }; + } + + void Tokenizer::skip_whitespace() { + while (iter != source.end()) { + switch (*iter) { + case ' ': column += 1; break; + case '\t': column += CE_TAB_SIZE; break; + case '\n': + column = 0; + line += 1; + break; + + default: return; + } + + ++iter; + } + } + + Expected Tokenizer::readNumber() { + auto stIter = iter; + + auto cLine = line; + auto cColumn = column; + + auto forward = [&] { + ++iter; + ++column; + }; + + if (*iter == '0') { + bool continueMatching = false; + + forward(); + + if (iter == source.end()) { + return Token{ + TokenV::tkInteger, + cLine, + cColumn, + { stIter, iter } + }; + } + + if (toLower(*iter) == 'x') { + forward(); + + if (iter == source.end()) { + return langException( + line, + column, + "hex value", + "EOF" + ); + } + + if (not isHexChar(*iter)) { + return langException( + line, + column, + "hex value", + *iter + ); + } + + while (iter != source.end()) { + if (not isHexChar(*iter)) { + break; + } + + forward(); + } + } + else if (toLower(*iter) == 'b') { + forward(); + + if (iter == source.end()) { + return langException( + line, + column, + "binary value", + "EOF" + ); + } + + if (not isBinaryChar(*iter)) { + return langException( + line, + column, + "binary value", + *iter + ); + } + + while (iter != source.end()) { + if (not isBinaryChar(*iter)) { + if (isDigit(*iter)) { + return langException( + line, + column, + "binary value", + *iter + ); + } + + break; + } + + forward(); + } + } + else if (*iter == '.') { + continueMatching = true; + } + else { + while (iter != source.end()) { + if (not isOctalChar(*iter)) { + if (isDigit(*iter)) { + return langException( + line, + column, + "octal value", + *iter + ); + } + + break; + } + + forward(); + } + } + + if (not continueMatching) { + return Token{ + TokenV::tkInteger, + cLine, + cColumn, + { stIter, iter } + }; + } + } + + while (iter != source.end()) { + if (not isDigit(*iter)) { + break; + } + + forward(); + } + + if (iter == source.end()) { + return Token{ + TokenV::tkInteger, + cLine, + cColumn, + { stIter, iter } + }; + } + + if (*iter == '.') { + forward(); + + while (iter != source.end()) { + if (not isDigit(*iter)) { + break; + } + + forward(); + } + + return Token{ + TokenV::tkDecimal, + cLine, + cColumn, + { stIter, iter } + }; + } + + return Token{ + TokenV::tkInteger, + cLine, + cColumn, + { stIter, iter } + }; + } + + Expected Tokenizer::readString() { + auto stIter = iter; + + auto cLine = line; + auto cColumn = column; + + auto forward = [&] { + ++iter; + ++column; + }; + + forward(); + + while (iter != source.end()) { + if (*iter == '\n') { + return langException( + line, + column, + "end of string (\")", + "\\n" + ); + } + + if (*iter == '"') { + forward(); + break; + } + + if (*iter == '\\') { + forward(); + + if (iter == source.end()) { + return langException( + line, + column, + "escaped char", + "EOF" + ); + } + + /* TODO: Validate char escape codes */ + } + + forward(); + } + + return Token{ + TokenV::tkString, + cLine, + cColumn, + { stIter, iter } + }; + } + + Expected Tokenizer::readCharacter() { + auto stIter = iter; + + auto cLine = line; + auto cColumn = column; + + auto forward = [&] { + ++iter; + ++column; + }; + + forward(); + + if (iter == source.end()) { + return langException( + line, + column, + "char literal", + "EOF" + ); + } + + if (*iter == '\n') { + return langException( + line, + column, + "char literal", + "\\n" + ); + } + + if (*iter == '\\') { + forward(); + + if (iter == source.end()) { + return langException( + line, + column, + "char literal", + "EOF" + ); + } + + /* TODO: Validate char escape codes */ + } + + forward(); + + if (iter == source.end()) { + return langException( + line, + column, + "end of string (\")", + "EOF" + ); + } + + if (*iter != '\'') { + return langException( + line, + column, + "end of string (\")", + *iter + ); + } + + return Token{ + TokenV::tkCharacter, + cLine, + cColumn, + { stIter, ++iter } + }; + } + + Expected Tokenizer::readIdentifier() { + auto stIter = iter; + + auto cLine = line; + auto cColumn = column; + + auto forward = [&] { + ++iter; + ++column; + }; + + while (iter != source.end()) { + if (not isIdentChar(*iter)) { + break; + } + + forward(); + } + + auto tok = Token{ + TokenV::tkIdentifier, + cLine, + cColumn, + { stIter, iter } + }; + + std::string_view strToken{ stIter, iter }; + + if (strToken.compare("or") == 0) { + tok.value = TokenV::kwOr; + } + else if (strToken.compare("not") == 0) { + tok.value = TokenV::kwNot; + } + else if (strToken.compare("and") == 0) { + tok.value = TokenV::kwAnd; + } + else if (strToken.compare("if") == 0) { + tok.value = TokenV::kwIf; + } + else if (strToken.compare("else") == 0) { + tok.value = TokenV::kwElse; + } + else if (strToken.compare("fn") == 0) { + tok.value = TokenV::kwFn; + } + else if (strToken.compare("enum") == 0) { + tok.value = TokenV::kwEnum; + } + else if (strToken.compare("struct") == 0) { + tok.value = TokenV::kwStruct; + } + else if (strToken.compare("variant") == 0) { + tok.value = TokenV::kwVariant; + } + else if (strToken.compare("def") == 0) { + tok.value = TokenV::kwDef; + } + else if (strToken.compare("let") == 0) { + tok.value = TokenV::kwLet; + } + else if (strToken.compare("mut") == 0) { + tok.value = TokenV::kwMut; + } + else if (strToken.compare("for") == 0) { + tok.value = TokenV::kwFor; + } + else if (strToken.compare("while") == 0) { + tok.value = TokenV::kwWhile; + } + else if (strToken.compare("return") == 0) { + tok.value = TokenV::kwReturn; + } + else if (strToken.compare("unreachable") == 0) { + tok.value = TokenV::kwUnreachable; + } + else if (strToken.compare("defer") == 0) { + tok.value = TokenV::kwDefer; + } + else if (strToken.compare("errdefer") == 0) { + tok.value = TokenV::kwErrDefer; + } + else if (strToken.compare("true") == 0) { + tok.value = TokenV::kwTrue; + } + else if (strToken.compare("false") == 0) { + tok.value = TokenV::kwFalse; + } + else if (strToken.compare("null") == 0) { + tok.value = TokenV::kwNull; + } + else if (strToken.compare("import") == 0) { + tok.value = TokenV::kwImport; + } + else if (strToken.compare("export") == 0) { + tok.value = TokenV::kwExport; + } + else if (strToken.compare("module") == 0) { + tok.value = TokenV::kwModule; + } + + return tok; + } + + TrieMap buildOperatorsTrieMap(); + + Expected Tokenizer::readOperator() { + auto stIter = iter; + + auto cLine = line; + auto cColumn = column; + + auto forward = [&] { + ++iter; + ++column; + }; + + auto tm = buildOperatorsTrieMap(); + + if (not tm.root.childs.contains(*iter)) { + return langException(line, column, *iter); + } + + auto node = &tm.root.childs.at(*iter); + auto lvNode = node->isLeaf ? node : nullptr; + + long lvNodeDiff = 0; + + forward(); + + while (iter != source.end() and not (isWhiteSpace(*iter))) { + if (not node->childs.contains(*iter)) { + break; + } + + node = &node->childs.at(*iter); + lvNode = node->isLeaf ? node : lvNode; + lvNodeDiff = node->isLeaf ? 0 : (lvNodeDiff + 1); + + forward(); + } + + Token tok{ + TokenV::opStar, + line = cLine, + column = cColumn, + { stIter, iter } + }; + + if (node->isLeaf) { + tok.value = *node->value; + } + else if (lvNode != nullptr) { + iter -= lvNodeDiff; + tok.value = *lvNode->value; + tok.strValue = std::string_view{ stIter, iter }; + } + else { + return langException( + line, + column, + tok.strValue + ); + } + + return tok; + } + + TrieMap buildOperatorsTrieMap() { + TrieMap tm; + + tm.insert(".", TokenV::opDot); + tm.insert("%", TokenV::opMod); + tm.insert("+", TokenV::opPlus); + tm.insert("-", TokenV::opHyphen); + tm.insert("/", TokenV::opSlash); + tm.insert("!", TokenV::opBang); + tm.insert("*", TokenV::opStar); + tm.insert(":", TokenV::opColon); + tm.insert(",", TokenV::opComma); + tm.insert("=", TokenV::opAssign); + tm.insert("::", TokenV::opAccess); + tm.insert(";", TokenV::opSemicolon); + tm.insert("^", TokenV::opCaret); + tm.insert("~", TokenV::opTilde); + tm.insert("==", TokenV::opEq); + tm.insert("!=", TokenV::opNeq); + tm.insert("<", TokenV::opLt); + tm.insert(">", TokenV::opGt); + tm.insert("<=", TokenV::opLtEq); + tm.insert(">=", TokenV::opGtEq); + tm.insert("<<", TokenV::opLShift); + tm.insert(">>", TokenV::opRShift); + tm.insert("&&", TokenV::opBoolAnd); + tm.insert("||", TokenV::opBoolOr); + tm.insert("&", TokenV::opAnd); + tm.insert("|", TokenV::opOr); + tm.insert("(", TokenV::opLParen); + tm.insert(")", TokenV::opRParen); + tm.insert("[", TokenV::opLBracket); + tm.insert("]", TokenV::opRBracket); + tm.insert("{", TokenV::opLSquirly); + tm.insert("}", TokenV::opRSquirly); + tm.insert("->", TokenV::opArrow); + + return tm; + } + +} // namespace arti::lang diff --git a/lib/src/Util/Demangle.cpp b/lib/src/Util/Demangle.cpp new file mode 100644 index 0000000..b52f444 --- /dev/null +++ b/lib/src/Util/Demangle.cpp @@ -0,0 +1,52 @@ +#include + +#include +#include +#include + +namespace arti::lang { + + Expected demangle(std::string_view str) { + using enum DemangleExceptCode; + + int status; + + char *return_value = + abi::__cxa_demangle(str.data(), nullptr, nullptr, &status); + + auto deallocator = [](char *mem) { + if (mem) { + free((void *) mem); + } + }; + + switch (status) { + case -1: return Unexpected(ecMemAlloc); + case -2: return Unexpected(ecInvalid); + case -3: return Unexpected(ecInvalid); + case 0: { + std::unique_ptr ptr( + return_value, + deallocator + ); + + std::string demangled_name{ return_value }; + + return demangled_name; + } break; + default: return Unexpected(ecMemAlloc); + } + } + + std::string demangleErrorString(DemangleExceptCode ec) { + using enum DemangleExceptCode; + + switch(ec) { + case ecMemAlloc: return "Memory allocation failed"; + case ecInvalid: return "Invalid argument"; + } + + std::unreachable(); + } + +} // namespace arti::lang diff --git a/lib/src/Util/Util.cpp b/lib/src/Util/Util.cpp deleted file mode 100644 index e69de29..0000000