feat: Added Token, Tokenizer, Generator, and some utilities

Initial version of Tokenizer and Token
Generator template for coroutines (used in tokenizer)
Utilities like string related functions, TrieMap, and error handling

TODO: Add tests for Tokenizer
TODO: Add tests for Generator

lib/include/artichoke/Coroutine/Generator.hpp

@@ -0,0 +1,272 @@
+#pragma once
+
+#include <ranges>
+#include <utility>
+#include <variant>
+#include <iterator>
+#include <coroutine>
+#include <exception>
+#include <generator>
+#include <type_traits>
+
+#include <artichoke/Coroutine/Utils.hpp>
+#include <artichoke/Util/Expected.hpp>
+
+namespace arti::lang {
+
+  template <typename T>
+  requires (not std::is_void_v<T>)
+  struct Generator;
+
+  template <typename T>
+  requires (not std::is_void_v<T>)
+  struct GeneratorPromise {
+    using value_type = std::remove_cvref_t<T>;
+    using reference_type = value_type &;
+    using pointer_type = value_type *;
+
+    using ValueType = value_type;
+    using ReferenceType = reference_type;
+    using PointerType = pointer_type;
+
+    GeneratorPromise() = default;
+
+    Generator<T> get_return_object();
+
+    std::suspend_always initial_suspend() const noexcept {
+      return {};
+    }
+
+    std::suspend_always final_suspend() const noexcept {
+      return {};
+    }
+
+    void unhandled_exception() {
+      exception = std::current_exception();
+    }
+
+    void rethrow_if_exception() {
+      if (exception) {
+        std::rethrow_exception(exception);
+      }
+    }
+
+    void return_value(const ValueType &val) noexcept {
+      value = val;
+    }
+
+    void return_value(ValueType &&val) noexcept {
+      value = std::forward<ValueType &&>(val);
+    }
+
+    std::suspend_always yield_value(const ValueType &val) noexcept {
+      value = val;
+      return {};
+    }
+
+    std::suspend_always yield_value(ValueType &&val) noexcept {
+      value = std::forward<ValueType &&>(val);
+      return {};
+    }
+
+    decltype(auto) await_transform(auto &&awaitable) = delete;
+
+    std::exception_ptr exception;
+    std::variant<PointerType, ValueType> value;
+  };
+
+  struct GeneratorSentinel { };
+
+  template <typename T>
+  requires (not std::is_void_v<T>)
+  struct GeneratorIterator {
+    using iterator_category = std::input_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+
+    using promise_type = GeneratorPromise<T>;
+
+    using value_type = typename promise_type::value_type;
+    using reference_type = value_type &;
+    using pointer_type = value_type *;
+
+    using handle_type = std::coroutine_handle<promise_type>;
+
+    using IteratorCategory = iterator_category;
+    using DifferenceType = difference_type;
+    using PromiseType = promise_type;
+    using ValueType = value_type;
+    using ReferenceType = reference_type;
+    using PointerType = pointer_type;
+    using HandleType = handle_type;
+
+    GeneratorIterator() noexcept = default;
+    ~GeneratorIterator() noexcept = default;
+
+    GeneratorIterator(HandleType handle) noexcept
+      : handle(handle) { }
+
+    GeneratorIterator(GeneratorIterator &&rhs) noexcept
+      : handle(std::exchange(rhs.handle, nullptr)) { }
+
+    GeneratorIterator &operator=(GeneratorIterator &&rhs) noexcept {
+      handle = std::exchange(rhs.handle, nullptr);
+      return *this;
+    }
+
+    GeneratorIterator(const GeneratorIterator &) noexcept = delete;
+    GeneratorIterator &operator=(const GeneratorIterator &) noexcept = delete;
+
+    GeneratorIterator &operator++() {
+      handle.resume();
+
+      if (handle.done()) {
+        handle.promise().rethrow_if_exception();
+      }
+
+      return *this;
+    }
+
+    void operator++(int) {
+      (void) this->operator++();
+    }
+
+    ReferenceType operator*() const noexcept {
+      auto &ref = handle.promise().value;
+
+      if (std::holds_alternative<PointerType>(ref)) {
+        return *std::get<PointerType>(ref);
+      }
+      else {
+        return std::get<ValueType>(ref);
+      }
+    }
+
+    PointerType operator->() const noexcept {
+      auto &ref = handle.promise().value;
+
+      if (std::holds_alternative<PointerType>(ref)) {
+        return std::get<PointerType>(ref);
+      }
+      else {
+        return &std::get<ValueType>(ref);
+      }
+    }
+
+    friend bool operator==(const GeneratorIterator &it, GeneratorSentinel) {
+      return (! it.handle || it.handle.done());
+    }
+
+    friend bool operator!=(const GeneratorIterator &it, GeneratorSentinel s) {
+      return ! (it == s);
+    }
+
+    friend bool operator==(GeneratorSentinel s, const GeneratorIterator &it) {
+      return it == s;
+    }
+
+    friend bool operator!=(GeneratorSentinel s, const GeneratorIterator &it) {
+      return it != s;
+    }
+
+    HandleType handle;
+  };
+
+  template <typename T>
+  requires (not std::is_void_v<T>)
+  struct [[nodiscard]] Generator {
+    using promise_type = GeneratorPromise<T>;
+    using handle_type = std::coroutine_handle<promise_type>;
+
+    using iterator_type = GeneratorIterator<T>;
+    using sentinel_type = GeneratorSentinel;
+
+    using PromiseType = promise_type;
+    using HandleType = handle_type;
+    using IteratorType = iterator_type;
+    using SentinelType = sentinel_type;
+
+    Generator() noexcept = default;
+
+    ~Generator() noexcept {
+      if (handle) {
+        handle.destroy();
+      }
+    }
+
+    explicit Generator(HandleType handle) noexcept
+      : handle(handle) { }
+
+    Generator(Generator &&rhs) noexcept
+      : handle(std::exchange(rhs.handle, nullptr)) { }
+
+    Generator &operator=(Generator &&rhs) noexcept {
+      this->~Generator();
+      handle = std::exchange(rhs.handle, nullptr);
+      return *this;
+    }
+
+    Generator(const Generator &) noexcept = delete;
+    Generator &operator=(const Generator &) noexcept = delete;
+
+    bool finished() const noexcept {
+      return (not handle || (handle.done() and !handle.promise().exception));
+    }
+
+    void next() {
+      if (not finished()) {
+        handle.resume();
+
+        if (handle.done()) {
+          handle.promise().rethrow_if_exception();
+        }
+      }
+    }
+
+    PromiseType::ReferenceType get() {
+      using PointerType = PromiseType::PointerType;
+      using ValueType = PromiseType::ValueType;
+
+      auto &ref = handle.promise().value;
+
+      if (std::holds_alternative<PointerType>(ref)) {
+        return *std::get<PointerType>(ref);
+      }
+      else {
+        return std::get<ValueType>(ref);
+      }
+    }
+
+    IteratorType begin() {
+      if (handle) {
+        handle.resume();
+
+        if (handle.done()) {
+          handle.promise().rethrow_if_exception();
+        }
+      }
+
+      return { handle };
+    }
+
+    SentinelType end() const noexcept {
+      return {};
+    }
+
+    void swap(Generator &other) noexcept {
+      std::swap(handle, other.handle);
+    }
+
+   private:
+    HandleType handle;
+  };
+
+  template <typename T>
+  requires (not std::is_void_v<T>)
+  Generator<T> GeneratorPromise<T>::get_return_object() {
+    using GeneratorType = Generator<T>;
+    using HandleType = typename GeneratorType::HandleType;
+
+    return GeneratorType{ HandleType::from_promise(*this) };
+  }
+
+} // namespace arti::lang

lib/include/artichoke/Coroutine/Utils.hpp

@@ -0,0 +1,5 @@
+#pragma once
+
+#define yield co_yield
+#define await co_await
+#define coret co_return

lib/include/artichoke/Tokenizer/Token.hpp

@@ -0,0 +1,108 @@
+#pragma once
+
+#include <string>
+
+namespace arti::lang {
+
+  enum class TokenV;
+
+  struct Token {
+    TokenV value;
+
+    std::size_t line;
+    std::size_t column;
+
+    std::string_view strValue;
+  };
+
+  enum class TokenV {
+    tkEOF,
+    tkString,
+    tkDecimal,
+    tkInteger,
+    tkCharacter,
+    tkIdentifier,
+
+    opDot,
+    opMod,
+    opPlus,
+    opHyphen,
+    opSlash,
+    opBang,
+    opStar,
+    opColon,
+    opComma,
+    opAssign,
+    opAccess,
+    opSemicolon,
+
+    opCaret,
+    opTilde,
+
+    opEq,
+    opNeq,
+
+    opLt,
+    opGt,
+
+    opLtEq,
+    opGtEq,
+
+    opLShift,
+    opRShift,
+
+    opBoolAnd,
+    opBoolOr,
+
+    opAnd,
+    opOr,
+
+    opLParen,
+    opRParen,
+
+    opLBracket,
+    opRBracket,
+
+    opLSquirly,
+    opRSquirly,
+
+    opArrow,
+
+    kwOr,
+    kwNot,
+    kwAnd,
+
+    kwIf,
+    kwElse,
+
+    kwFn,
+    kwEnum,
+    kwStruct,
+    kwVariant,
+
+    kwDef,
+    kwLet,
+    kwMut,
+
+    kwFor,
+    kwWhile,
+
+    kwReturn,
+    kwUnreachable,
+
+    kwDefer,
+    kwErrDefer,
+
+    kwTrue,
+    kwFalse,
+
+    kwNull,
+
+    kwImport,
+    kwExport,
+    kwModule,
+  };
+
+  std::string toString(const Token &value);
+
+} // namespace arti::lang

lib/include/artichoke/Tokenizer/Tokenizer.hpp

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <deque>
+#include <vector>
+
+#include <artichoke/Util/Expected.hpp>
+#include <artichoke/Coroutine/Generator.hpp>
+
+#include <artichoke/Tokenizer/Token.hpp>
+
+namespace arti::lang {
+
+  struct [[nodiscard]] Tokenizer {
+    Tokenizer() noexcept = delete;
+
+    Tokenizer(std::string source) noexcept;
+
+    ~Tokenizer() noexcept = default;
+
+    Tokenizer(Tokenizer &&rhs) noexcept;
+    Tokenizer &operator=(Tokenizer &&rhs) noexcept;
+
+    Tokenizer(const Tokenizer &rhs) = delete;
+    Tokenizer &operator=(const Tokenizer &rhs) = delete;
+
+    Expected<void> consume(std::size_t n = 1) noexcept;
+    Expected<Token> peek(std::size_t n = 0) noexcept;
+
+    bool finished() const noexcept;
+
+    void swap(Tokenizer &other) noexcept;
+
+   private:
+    Generator<Expected<Token>> tokenize();
+
+    void skip_whitespace();
+
+    Expected<Token> readNumber();
+    Expected<Token> readString();
+    Expected<Token> readCharacter();
+    Expected<Token> readIdentifier();
+    Expected<Token> readOperator();
+
+    size_t line;
+    size_t column;
+    std::string::iterator iter;
+
+    Generator<Expected<Token>> tokensGenerator;
+
+    std::deque<Token> tokensBuffer;
+
+    std::string source;
+  };
+
+} // namespace arti::lang

lib/include/artichoke/Util/Demangle.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <artichoke/Util/Expected.hpp>
+
+namespace arti::lang {
+
+  enum struct DemangleExceptCode {
+    ecMemAlloc,
+    ecInvalid,
+  };
+
+  Expected<std::string, DemangleExceptCode> demangle(std::string_view str);
+
+  std::string demangleErrorString(DemangleExceptCode ec);
+
+} // namespace arti::lang

lib/include/artichoke/Util/Expected.hpp

@@ -0,0 +1,84 @@
+#pragma once
+
+#include <cstddef>
+
+#include <string>
+#include <expected>
+#include <format>
+
+namespace arti::lang {
+
+  enum struct ExceptCode {
+    ecEOF,
+    ecEOB,
+    ecInvalidToken,
+    ecStdException,
+    ecInvalidLiteral,
+    ecInvalidCharacter,
+    ecInvalidIndex,
+  };
+
+  struct Exception {
+    size_t line;
+    size_t column;
+    std::string message;
+  };
+
+  template <typename T, typename E = Exception>
+  using Expected = std::expected<T, E>;
+
+  template <typename E = Exception>
+  using Unexpected = std::unexpected<E>;
+
+  template <ExceptCode code, typename... Args>
+  std::string exceptionMessage(Args &&...args) {
+    using enum ExceptCode;
+
+    if constexpr (code == ecEOF) {
+      return "Reached EOF";
+    }
+    else if constexpr (code == ecEOB) {
+      return "Buffer empty";
+    }
+    else if constexpr (code == ecStdException) {
+      return std::format(
+        "Catched instance of {}: '{}'",
+        std::forward<Args>(args)...
+      );
+    }
+    else if constexpr (code == ecInvalidLiteral) {
+      return std::format(
+        "Invalid literal, expected {} got {}",
+        std::forward<Args>(args)...
+      );
+    }
+    else if constexpr (code == ecInvalidCharacter) {
+      return std::format(
+        "Invalid character found '{}'",
+        std::forward<Args>(args)...
+      );
+    }
+    else if constexpr (code == ecInvalidToken) {
+      return std::format(
+        "Invalid token found '{}'",
+        std::forward<Args>(args)...
+      );
+    }
+    else if constexpr (code == ecInvalidIndex) {
+      return "Invalid index";
+    }
+    else {
+      return "Unknown error";
+    }
+  }
+
+  template <ExceptCode code, typename... Args>
+  static inline Unexpected<>
+  langException(size_t line, size_t col, Args &&...args) {
+    return Unexpected{
+      Exception{ line,
+                col, exceptionMessage<code>(std::forward<Args>(args)...) }
+    };
+  }
+
+} // namespace arti::lang

lib/include/artichoke/Util/OverloadSet.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+namespace arti::lang {
+
+  template <typename ...Ts>
+  struct OverloadSet {
+    using Ts::operator()...;
+  };
+
+  template <typename ...Ts>
+  OverloadSet(Ts...) -> OverloadSet<Ts...>;
+
+}

lib/include/artichoke/Util/Strings.hpp

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <cstddef>
+
+namespace arti::lang {
+
+  constexpr size_t CE_TAB_SIZE = 2;
+
+  static inline constexpr bool isLetter(char c) {
+    return ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'));
+  }
+
+  static inline constexpr char toLower(char c) {
+    if (isLetter(c)) {
+      return (c | 0b00100000);
+    }
+
+    return c;
+  }
+
+  static inline constexpr bool isDigit(char c) {
+    return (c >= '0' && c <= '9');
+  }
+
+  static inline constexpr bool isFirstIdentChar(char c) {
+    return isLetter(c) || c == '_';
+  }
+
+  static inline constexpr bool isIdentChar(char c) {
+    return isLetter(c) || c == '_';
+  }
+
+  static inline constexpr bool isHexChar(char c) {
+    return isDigit(c) || ((toLower(c) >= 'a' && toLower(c) <= 'f'));
+  }
+
+  static inline constexpr bool isBinaryChar(char c) {
+    return c == '0' || c == '1';
+  }
+
+  static inline constexpr bool isOctalChar(char c) {
+    return (c >= '0' && c <= '7');
+  }
+
+  static inline constexpr bool isWhiteSpace(char c) {
+    return (c == ' ' || c == '\t' || c == '\n');
+  }
+
+} // namespace arti::lang

lib/include/artichoke/Util/TrieMap.hpp

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <map>
+#include <optional>
+#include <string_view>
+
+namespace arti::lang {
+
+  template <typename T>
+  struct TrieMap {
+    struct Node {
+      bool isLeaf;
+      std::optional<T> value;
+      std::map<char, Node> childs;
+
+      Node() noexcept
+        : isLeaf(false)
+        , value(std::nullopt)
+        , childs() { }
+
+      ~Node() noexcept = default;
+
+      Node(Node &&) noexcept = default;
+      Node &operator=(Node &&) noexcept = default;
+
+      Node(const Node &) noexcept = default;
+      Node &operator=(const Node &) noexcept = default;
+    };
+
+    Node root;
+
+    TrieMap() noexcept = default;
+    ~TrieMap() noexcept = default;
+
+    TrieMap(TrieMap &&) noexcept = default;
+    TrieMap &operator=(TrieMap &&) noexcept = default;
+
+    TrieMap(const TrieMap &) noexcept = default;
+    TrieMap &operator=(const TrieMap &) noexcept = default;
+
+    void insert(std::string_view str, T &&value) noexcept {
+      auto ptrNode = &root;
+
+      for (auto c : str) {
+        ptrNode = &(ptrNode->childs[c]);
+      }
+
+      ptrNode->isLeaf = true;
+      ptrNode->value = std::forward<T>(value);
+    }
+
+    T *find(std::string_view str) const noexcept {
+      auto ptrNode = &root;
+
+      for (auto c : str) {
+        if (not ptrNode->childs.contains(c)) {
+          return nullptr;
+        }
+
+        ptrNode = &(ptrNode->childs.at(c));
+      }
+
+      return ptrNode;
+    }
+  };
+
+} // namespace arti::lang

lib/src/Tokenizer/Token.cpp

@@ -0,0 +1,192 @@
+#include <artichoke/Tokenizer/Token.hpp>
+
+#include <format>
+
+namespace arti::lang {
+
+  std::string toString(const Token &value) {
+    using enum TokenV;
+
+    switch (value.value) {
+      case tkEOF:
+        return std::format("Token{{ {} }}", "tkEOF", value.strValue);
+      case tkString:
+        return std::format("Token{{ {}, {} }}", "tkString", value.strValue);
+      case tkDecimal:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "tkDecimal",
+          value.strValue
+        );
+      case tkInteger:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "tkInteger",
+          value.strValue
+        );
+      case tkCharacter:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "tkCharacter",
+          value.strValue
+        );
+      case tkIdentifier:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "tkIdentifier",
+          value.strValue
+        );
+      case opDot:
+        return std::format("Token{{ {}, {} }}", "opDot", value.strValue);
+      case opMod:
+        return std::format("Token{{ {}, {} }}", "opMod", value.strValue);
+      case opPlus:
+        return std::format("Token{{ {}, {} }}", "opPlus", value.strValue);
+      case opHyphen:
+        return std::format("Token{{ {}, {} }}", "opHyphen", value.strValue);
+      case opSlash:
+        return std::format("Token{{ {}, {} }}", "opSlash", value.strValue);
+      case opBang:
+        return std::format("Token{{ {}, {} }}", "opBang", value.strValue);
+      case opStar:
+        return std::format("Token{{ {}, {} }}", "opStar", value.strValue);
+      case opColon:
+        return std::format("Token{{ {}, {} }}", "opColon", value.strValue);
+      case opComma:
+        return std::format("Token{{ {}, {} }}", "opComma", value.strValue);
+      case opAssign:
+        return std::format("Token{{ {}, {} }}", "opAssign", value.strValue);
+      case opAccess:
+        return std::format("Token{{ {}, {} }}", "opAccess", value.strValue);
+      case opSemicolon:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "opSemicolon",
+          value.strValue
+        );
+      case opCaret:
+        return std::format("Token{{ {}, {} }}", "opCaret", value.strValue);
+      case opTilde:
+        return std::format("Token{{ {}, {} }}", "opTilde", value.strValue);
+      case opEq:
+        return std::format("Token{{ {}, {} }}", "opEq", value.strValue);
+      case opNeq:
+        return std::format("Token{{ {}, {} }}", "opNeq", value.strValue);
+      case opLt:
+        return std::format("Token{{ {}, {} }}", "opLt", value.strValue);
+      case opGt:
+        return std::format("Token{{ {}, {} }}", "opGt", value.strValue);
+      case opLtEq:
+        return std::format("Token{{ {}, {} }}", "opLtEq", value.strValue);
+      case opGtEq:
+        return std::format("Token{{ {}, {} }}", "opGtEq", value.strValue);
+      case opLShift:
+        return std::format("Token{{ {}, {} }}", "opLShift", value.strValue);
+      case opRShift:
+        return std::format("Token{{ {}, {} }}", "opRShift", value.strValue);
+      case opBoolAnd:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "opBoolAnd",
+          value.strValue
+        );
+      case opBoolOr:
+        return std::format("Token{{ {}, {} }}", "opBoolOr", value.strValue);
+      case opAnd:
+        return std::format("Token{{ {}, {} }}", "opAnd", value.strValue);
+      case opOr:
+        return std::format("Token{{ {}, {} }}", "opOr", value.strValue);
+      case opLParen:
+        return std::format("Token{{ {}, {} }}", "opLParen", value.strValue);
+      case opRParen:
+        return std::format("Token{{ {}, {} }}", "opRParen", value.strValue);
+      case opLBracket:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "opLBracket",
+          value.strValue
+        );
+      case opRBracket:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "opRBracket",
+          value.strValue
+        );
+      case opLSquirly:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "opLSquirly",
+          value.strValue
+        );
+      case opRSquirly:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "opRSquirly",
+          value.strValue
+        );
+      case opArrow:
+        return std::format("Token{{ {}, {} }}", "opArrow", value.strValue);
+      case kwOr:
+        return std::format("Token{{ {}, {} }}", "kwOr", value.strValue);
+      case kwNot:
+        return std::format("Token{{ {}, {} }}", "kwNot", value.strValue);
+      case kwAnd:
+        return std::format("Token{{ {}, {} }}", "kwAnd", value.strValue);
+      case kwIf:
+        return std::format("Token{{ {}, {} }}", "kwIf", value.strValue);
+      case kwElse:
+        return std::format("Token{{ {}, {} }}", "kwElse", value.strValue);
+      case kwFn:
+        return std::format("Token{{ {}, {} }}", "kwFn", value.strValue);
+      case kwEnum:
+        return std::format("Token{{ {}, {} }}", "kwEnum", value.strValue);
+      case kwStruct:
+        return std::format("Token{{ {}, {} }}", "kwStruct", value.strValue);
+      case kwVariant:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "kwVariant",
+          value.strValue
+        );
+      case kwDef:
+        return std::format("Token{{ {}, {} }}", "kwDef", value.strValue);
+      case kwLet:
+        return std::format("Token{{ {}, {} }}", "kwLet", value.strValue);
+      case kwMut:
+        return std::format("Token{{ {}, {} }}", "kwMut", value.strValue);
+      case kwFor:
+        return std::format("Token{{ {}, {} }}", "kwFor", value.strValue);
+      case kwWhile:
+        return std::format("Token{{ {}, {} }}", "kwWhile", value.strValue);
+      case kwReturn:
+        return std::format("Token{{ {}, {} }}", "kwReturn", value.strValue);
+      case kwUnreachable:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "kwUnreachable",
+          value.strValue
+        );
+      case kwDefer:
+        return std::format("Token{{ {}, {} }}", "kwDefer", value.strValue);
+      case kwErrDefer:
+        return std::format(
+          "Token{{ {}, {} }}",
+          "kwErrDefer",
+          value.strValue
+        );
+      case kwTrue:
+        return std::format("Token{{ {}, {} }}", "kwTrue", value.strValue);
+      case kwFalse:
+        return std::format("Token{{ {}, {} }}", "kwFalse", value.strValue);
+      case kwNull:
+        return std::format("Token{{ {}, {} }}", "kwNull", value.strValue);
+      case kwImport:
+        return std::format("Token{{ {}, {} }}", "kwImport", value.strValue);
+      case kwExport:
+        return std::format("Token{{ {}, {} }}", "kwExport", value.strValue);
+      case kwModule:
+        return std::format("Token{{ {}, {} }}", "kwModule", value.strValue);
+    }
+    return "<Undefined>";
+  }
+} // namespace arti::lang

lib/src/Tokenizer/Tokenizer.cpp

@@ -0,0 +1,661 @@
+#include <artichoke/Tokenizer/Tokenizer.hpp>
+
+#include <print>
+#include <utility>
+
+#include <artichoke/Util/Strings.hpp>
+#include <artichoke/Util/Demangle.hpp>
+#include <artichoke/Util/TrieMap.hpp>
+
+namespace arti::lang {
+
+  Tokenizer::Tokenizer(std::string source) noexcept
+    : line(0)
+    , column(0)
+    , tokensGenerator{ tokenize() }
+    , tokensBuffer{}
+    , source(std::move(source)) {
+    iter = this->source.begin();
+  }
+
+  Tokenizer::Tokenizer(Tokenizer &&rhs) noexcept
+    : line(rhs.line)
+    , column(rhs.column)
+    , iter(rhs.iter)
+    , tokensGenerator(std::exchange(rhs.tokensGenerator, {}))
+    , tokensBuffer(std::exchange(rhs.tokensBuffer, {}))
+    , source(std::exchange(rhs.source, "")) { }
+
+  Tokenizer &Tokenizer::operator=(Tokenizer &&rhs) noexcept {
+    line = rhs.line;
+    column = rhs.column;
+    iter = std::move(rhs.iter);
+    source = std::move(rhs.source);
+    tokensBuffer = std::move(rhs.tokensBuffer);
+    tokensGenerator = std::move(rhs.tokensGenerator);
+    return *this;
+  }
+
+  Expected<void> Tokenizer::consume(std::size_t n) noexcept {
+    while (not tokensBuffer.empty()) {
+      tokensBuffer.pop_front();
+      n -= 1;
+    }
+
+    if (n != 0) {
+      return langException<ExceptCode::ecEOB>(line, column);
+    }
+
+    return {};
+  }
+
+  Expected<Token> Tokenizer::peek(std::size_t n) noexcept {
+    while (not finished() && tokensBuffer.size() < (n + 1)) {
+      try {
+        tokensGenerator.next();
+      }
+      catch (std::exception &e) {
+        const auto exceptionName = demangle(typeid(e).name());
+
+        if (not exceptionName) {
+          return langException<ExceptCode::ecStdException>(
+            line,
+            column,
+            "abi::__cxa_demangle(...)",
+            demangleErrorString(exceptionName.error())
+          );
+        }
+
+        return langException<ExceptCode::ecStdException>(
+          line,
+          column,
+          exceptionName.value(),
+          e.what()
+        );
+      }
+
+      auto tok = tokensGenerator.get();
+
+      if (not tok) {
+        return tok;
+      }
+
+      tokensBuffer.push_back(std::move(tok).value());
+    }
+
+    if (finished()) {
+      if (tokensBuffer.size() < (n + 1)) {
+        return langException<ExceptCode::ecEOF>(line, column);
+      }
+    }
+
+    return tokensBuffer.at(n);
+  }
+
+  bool Tokenizer::finished() const noexcept {
+    if (tokensGenerator.finished()) {
+      return tokensBuffer.empty();
+    }
+
+    return false;
+  }
+
+  void Tokenizer::swap(Tokenizer &other) noexcept {
+    std::swap(line, other.line);
+    std::swap(column, other.column);
+    std::swap(iter, other.iter);
+    tokensGenerator.swap(other.tokensGenerator);
+    tokensBuffer.swap(other.tokensBuffer);
+    source.swap(other.source);
+  }
+
+  Generator<Expected<Token>> Tokenizer::tokenize() {
+    while (iter != source.end()) {
+      skip_whitespace();
+
+      if (iter == source.end()) {
+        break;
+      }
+
+      if (isDigit(*iter)) {
+        yield readNumber();
+      }
+      else if (*iter == '"') {
+        yield readString();
+      }
+      else if (*iter == '\'') {
+        yield readCharacter();
+      }
+      else if (isFirstIdentChar(*iter)) {
+        yield readIdentifier();
+      }
+      else {
+        yield readOperator();
+      }
+    }
+
+    coret Token{ .value = TokenV::tkEOF, .line = line, .column = column };
+  }
+
+  void Tokenizer::skip_whitespace() {
+    while (iter != source.end()) {
+      switch (*iter) {
+        case ' ':  column += 1; break;
+        case '\t': column += CE_TAB_SIZE; break;
+        case '\n':
+          column = 0;
+          line += 1;
+          break;
+
+        default: return;
+      }
+
+      ++iter;
+    }
+  }
+
+  Expected<Token> Tokenizer::readNumber() {
+    auto stIter = iter;
+
+    auto cLine = line;
+    auto cColumn = column;
+
+    auto forward = [&] {
+      ++iter;
+      ++column;
+    };
+
+    if (*iter == '0') {
+      bool continueMatching = false;
+
+      forward();
+
+      if (iter == source.end()) {
+        return Token{
+          TokenV::tkInteger,
+          cLine,
+          cColumn,
+          { stIter, iter }
+        };
+      }
+
+      if (toLower(*iter) == 'x') {
+        forward();
+
+        if (iter == source.end()) {
+          return langException<ExceptCode::ecInvalidLiteral>(
+            line,
+            column,
+            "hex value",
+            "EOF"
+          );
+        }
+
+        if (not isHexChar(*iter)) {
+          return langException<ExceptCode::ecInvalidLiteral>(
+            line,
+            column,
+            "hex value",
+            *iter
+          );
+        }
+
+        while (iter != source.end()) {
+          if (not isHexChar(*iter)) {
+            break;
+          }
+
+          forward();
+        }
+      }
+      else if (toLower(*iter) == 'b') {
+        forward();
+
+        if (iter == source.end()) {
+          return langException<ExceptCode::ecInvalidLiteral>(
+            line,
+            column,
+            "binary value",
+            "EOF"
+          );
+        }
+
+        if (not isBinaryChar(*iter)) {
+          return langException<ExceptCode::ecInvalidLiteral>(
+            line,
+            column,
+            "binary value",
+            *iter
+          );
+        }
+
+        while (iter != source.end()) {
+          if (not isBinaryChar(*iter)) {
+            if (isDigit(*iter)) {
+              return langException<ExceptCode::ecInvalidLiteral>(
+                line,
+                column,
+                "binary value",
+                *iter
+              );
+            }
+
+            break;
+          }
+
+          forward();
+        }
+      }
+      else if (*iter == '.') {
+        continueMatching = true;
+      }
+      else {
+        while (iter != source.end()) {
+          if (not isOctalChar(*iter)) {
+            if (isDigit(*iter)) {
+              return langException<ExceptCode::ecInvalidLiteral>(
+                line,
+                column,
+                "octal value",
+                *iter
+              );
+            }
+
+            break;
+          }
+
+          forward();
+        }
+      }
+
+      if (not continueMatching) {
+        return Token{
+          TokenV::tkInteger,
+          cLine,
+          cColumn,
+          { stIter, iter }
+        };
+      }
+    }
+
+    while (iter != source.end()) {
+      if (not isDigit(*iter)) {
+        break;
+      }
+
+      forward();
+    }
+
+    if (iter == source.end()) {
+      return Token{
+        TokenV::tkInteger,
+        cLine,
+        cColumn,
+        { stIter, iter }
+      };
+    }
+
+    if (*iter == '.') {
+      forward();
+
+      while (iter != source.end()) {
+        if (not isDigit(*iter)) {
+          break;
+        }
+
+        forward();
+      }
+
+      return Token{
+        TokenV::tkDecimal,
+        cLine,
+        cColumn,
+        { stIter, iter }
+      };
+    }
+
+    return Token{
+      TokenV::tkInteger,
+      cLine,
+      cColumn,
+      { stIter, iter }
+    };
+  }
+
+  Expected<Token> Tokenizer::readString() {
+    auto stIter = iter;
+
+    auto cLine = line;
+    auto cColumn = column;
+
+    auto forward = [&] {
+      ++iter;
+      ++column;
+    };
+
+    forward();
+
+    while (iter != source.end()) {
+      if (*iter == '\n') {
+        return langException<ExceptCode::ecInvalidLiteral>(
+          line,
+          column,
+          "end of string (\")",
+          "\\n"
+        );
+      }
+
+      if (*iter == '"') {
+        forward();
+        break;
+      }
+
+      if (*iter == '\\') {
+        forward();
+
+        if (iter == source.end()) {
+          return langException<ExceptCode::ecInvalidLiteral>(
+            line,
+            column,
+            "escaped char",
+            "EOF"
+          );
+        }
+
+        /* TODO: Validate char escape codes */
+      }
+
+      forward();
+    }
+
+    return Token{
+      TokenV::tkString,
+      cLine,
+      cColumn,
+      { stIter, iter }
+    };
+  }
+
+  Expected<Token> Tokenizer::readCharacter() {
+    auto stIter = iter;
+
+    auto cLine = line;
+    auto cColumn = column;
+
+    auto forward = [&] {
+      ++iter;
+      ++column;
+    };
+
+    forward();
+
+    if (iter == source.end()) {
+      return langException<ExceptCode::ecInvalidLiteral>(
+        line,
+        column,
+        "char literal",
+        "EOF"
+      );
+    }
+
+    if (*iter == '\n') {
+      return langException<ExceptCode::ecInvalidLiteral>(
+        line,
+        column,
+        "char literal",
+        "\\n"
+      );
+    }
+
+    if (*iter == '\\') {
+      forward();
+
+      if (iter == source.end()) {
+        return langException<ExceptCode::ecInvalidLiteral>(
+          line,
+          column,
+          "char literal",
+          "EOF"
+        );
+      }
+
+      /* TODO: Validate char escape codes */
+    }
+
+    forward();
+
+    if (iter == source.end()) {
+      return langException<ExceptCode::ecInvalidLiteral>(
+        line,
+        column,
+        "end of string (\")",
+        "EOF"
+      );
+    }
+
+    if (*iter != '\'') {
+      return langException<ExceptCode::ecInvalidLiteral>(
+        line,
+        column,
+        "end of string (\")",
+        *iter
+      );
+    }
+
+    return Token{
+      TokenV::tkCharacter,
+      cLine,
+      cColumn,
+      { stIter, ++iter }
+    };
+  }
+
+  Expected<Token> Tokenizer::readIdentifier() {
+    auto stIter = iter;
+
+    auto cLine = line;
+    auto cColumn = column;
+
+    auto forward = [&] {
+      ++iter;
+      ++column;
+    };
+
+    while (iter != source.end()) {
+      if (not isIdentChar(*iter)) {
+        break;
+      }
+
+      forward();
+    }
+
+    auto tok = Token{
+      TokenV::tkIdentifier,
+      cLine,
+      cColumn,
+      { stIter, iter }
+    };
+
+    std::string_view strToken{ stIter, iter };
+
+    if (strToken.compare("or") == 0) {
+      tok.value = TokenV::kwOr;
+    }
+    else if (strToken.compare("not") == 0) {
+      tok.value = TokenV::kwNot;
+    }
+    else if (strToken.compare("and") == 0) {
+      tok.value = TokenV::kwAnd;
+    }
+    else if (strToken.compare("if") == 0) {
+      tok.value = TokenV::kwIf;
+    }
+    else if (strToken.compare("else") == 0) {
+      tok.value = TokenV::kwElse;
+    }
+    else if (strToken.compare("fn") == 0) {
+      tok.value = TokenV::kwFn;
+    }
+    else if (strToken.compare("enum") == 0) {
+      tok.value = TokenV::kwEnum;
+    }
+    else if (strToken.compare("struct") == 0) {
+      tok.value = TokenV::kwStruct;
+    }
+    else if (strToken.compare("variant") == 0) {
+      tok.value = TokenV::kwVariant;
+    }
+    else if (strToken.compare("def") == 0) {
+      tok.value = TokenV::kwDef;
+    }
+    else if (strToken.compare("let") == 0) {
+      tok.value = TokenV::kwLet;
+    }
+    else if (strToken.compare("mut") == 0) {
+      tok.value = TokenV::kwMut;
+    }
+    else if (strToken.compare("for") == 0) {
+      tok.value = TokenV::kwFor;
+    }
+    else if (strToken.compare("while") == 0) {
+      tok.value = TokenV::kwWhile;
+    }
+    else if (strToken.compare("return") == 0) {
+      tok.value = TokenV::kwReturn;
+    }
+    else if (strToken.compare("unreachable") == 0) {
+      tok.value = TokenV::kwUnreachable;
+    }
+    else if (strToken.compare("defer") == 0) {
+      tok.value = TokenV::kwDefer;
+    }
+    else if (strToken.compare("errdefer") == 0) {
+      tok.value = TokenV::kwErrDefer;
+    }
+    else if (strToken.compare("true") == 0) {
+      tok.value = TokenV::kwTrue;
+    }
+    else if (strToken.compare("false") == 0) {
+      tok.value = TokenV::kwFalse;
+    }
+    else if (strToken.compare("null") == 0) {
+      tok.value = TokenV::kwNull;
+    }
+    else if (strToken.compare("import") == 0) {
+      tok.value = TokenV::kwImport;
+    }
+    else if (strToken.compare("export") == 0) {
+      tok.value = TokenV::kwExport;
+    }
+    else if (strToken.compare("module") == 0) {
+      tok.value = TokenV::kwModule;
+    }
+
+    return tok;
+  }
+
+  TrieMap<TokenV> buildOperatorsTrieMap();
+
+  Expected<Token> Tokenizer::readOperator() {
+    auto stIter = iter;
+
+    auto cLine = line;
+    auto cColumn = column;
+
+    auto forward = [&] {
+      ++iter;
+      ++column;
+    };
+
+    auto tm = buildOperatorsTrieMap();
+
+    if (not tm.root.childs.contains(*iter)) {
+      return langException<ExceptCode::ecInvalidCharacter>(line, column, *iter);
+    }
+
+    auto node = &tm.root.childs.at(*iter);
+    auto lvNode = node->isLeaf ? node : nullptr;
+
+    long lvNodeDiff = 0;
+
+    forward();
+
+    while (iter != source.end() and not (isWhiteSpace(*iter))) {
+      if (not node->childs.contains(*iter)) {
+        break;
+      }
+
+      node = &node->childs.at(*iter);
+      lvNode = node->isLeaf ? node : lvNode;
+      lvNodeDiff = node->isLeaf ? 0 : (lvNodeDiff + 1);
+
+      forward();
+    }
+
+    Token tok{
+      TokenV::opStar,
+      line = cLine,
+      column = cColumn,
+      { stIter, iter }
+    };
+
+    if (node->isLeaf) {
+      tok.value = *node->value;
+    }
+    else if (lvNode != nullptr) {
+      iter -= lvNodeDiff;
+      tok.value = *lvNode->value;
+      tok.strValue = std::string_view{ stIter, iter };
+    }
+    else {
+      return langException<ExceptCode::ecInvalidCharacter>(
+        line,
+        column,
+        tok.strValue
+      );
+    }
+
+    return tok;
+  }
+
+  TrieMap<TokenV> buildOperatorsTrieMap() {
+    TrieMap<TokenV> tm;
+
+    tm.insert(".", TokenV::opDot);
+    tm.insert("%", TokenV::opMod);
+    tm.insert("+", TokenV::opPlus);
+    tm.insert("-", TokenV::opHyphen);
+    tm.insert("/", TokenV::opSlash);
+    tm.insert("!", TokenV::opBang);
+    tm.insert("*", TokenV::opStar);
+    tm.insert(":", TokenV::opColon);
+    tm.insert(",", TokenV::opComma);
+    tm.insert("=", TokenV::opAssign);
+    tm.insert("::", TokenV::opAccess);
+    tm.insert(";", TokenV::opSemicolon);
+    tm.insert("^", TokenV::opCaret);
+    tm.insert("~", TokenV::opTilde);
+    tm.insert("==", TokenV::opEq);
+    tm.insert("!=", TokenV::opNeq);
+    tm.insert("<", TokenV::opLt);
+    tm.insert(">", TokenV::opGt);
+    tm.insert("<=", TokenV::opLtEq);
+    tm.insert(">=", TokenV::opGtEq);
+    tm.insert("<<", TokenV::opLShift);
+    tm.insert(">>", TokenV::opRShift);
+    tm.insert("&&", TokenV::opBoolAnd);
+    tm.insert("||", TokenV::opBoolOr);
+    tm.insert("&", TokenV::opAnd);
+    tm.insert("|", TokenV::opOr);
+    tm.insert("(", TokenV::opLParen);
+    tm.insert(")", TokenV::opRParen);
+    tm.insert("[", TokenV::opLBracket);
+    tm.insert("]", TokenV::opRBracket);
+    tm.insert("{", TokenV::opLSquirly);
+    tm.insert("}", TokenV::opRSquirly);
+    tm.insert("->", TokenV::opArrow);
+
+    return tm;
+  }
+
+} // namespace arti::lang

lib/src/Util/Demangle.cpp

@@ -0,0 +1,52 @@
+#include <artichoke/Util/Demangle.hpp>
+
+#include <utility>
+#include <memory>
+#include <cxxabi.h>
+
+namespace arti::lang {
+
+  Expected<std::string, DemangleExceptCode> demangle(std::string_view str) {
+    using enum DemangleExceptCode;
+
+    int status;
+
+    char *return_value =
+      abi::__cxa_demangle(str.data(), nullptr, nullptr, &status);
+
+    auto deallocator = [](char *mem) {
+      if (mem) {
+        free((void *) mem);
+      }
+    };
+
+    switch (status) {
+      case -1: return Unexpected(ecMemAlloc);
+      case -2: return Unexpected(ecInvalid);
+      case -3: return Unexpected(ecInvalid);
+      case 0:  {
+        std::unique_ptr<char, decltype(deallocator)> ptr(
+          return_value,
+          deallocator
+        );
+
+        std::string demangled_name{ return_value };
+
+        return demangled_name;
+      } break;
+      default: return Unexpected(ecMemAlloc);
+    }
+  }
+
+  std::string demangleErrorString(DemangleExceptCode ec) {
+    using enum DemangleExceptCode;
+
+    switch(ec) {
+      case ecMemAlloc: return "Memory allocation failed";
+      case ecInvalid: return "Invalid argument";
+    }
+
+    std::unreachable();
+  }
+
+} // namespace arti::lang