From 146a48aef9e6b305fa725f173fce3de0306e5167 Mon Sep 17 00:00:00 2001
From: erick-alcachofa <erick@artichoke.dev>
Date: Thu, 1 Jan 2026 07:55:09 +0000
Subject: [PATCH] fix(tokenizer): harden move semantics and operator parsing

Signed-off-by: erick-alcachofa <erick@artichoke.dev>

This commit addresses subtle state-management issues uncovered while
exercising tokenizer move semantics and operator parsing. Moved-from
instances were retaining stale iterators and metadata that could
manifest as incorrect token offsets when the objects were reused.
Additionally, token location bookkeeping during comment skipping and
operator backtracking drifted from the iterator, producing inaccurate
column reports.

- Refines the move constructor by exchanging `line`, `column`, `iter`,
  `tokensGenerator`, `tokensBuffer`, and `source`, ensuring moved-from
  tokenizers reset to neutral defaults without dangling references.
- Aligns the move assignment operator with the constructor by using
  `std::exchange` across all transferred members, preventing stale state
  when a tokenizer is reassigned.
- Adjusts column advancement for single-line comments so the cursor
  increments by one when skipping the comments sequence, matching the
  iterator progression.
- Promotes the operator trie to a function-local static cache, avoiding
  repeated construction each time an operator token is read.
- Restores accurate token metadata by assigning the original line and
  column (`cLine`, `cColumn`) when emitting operator tokens and rolling
  back the column alongside the iterator during longest-valid
  backtracking.
---
 lib/src/Tokenizer/Tokenizer.cpp | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/lib/src/Tokenizer/Tokenizer.cpp b/lib/src/Tokenizer/Tokenizer.cpp
index 11d7862..d73f432 100644
--- a/lib/src/Tokenizer/Tokenizer.cpp
+++ b/lib/src/Tokenizer/Tokenizer.cpp
@@ -22,6 +22,7 @@
 
 #include <artichoke/Tokenizer/Tokenizer.hpp>
 
+#include <print>
 #include <utility>
 
 #include <artichoke/Tokenizer/TokenizerRange.hpp>
@@ -41,20 +42,20 @@ namespace arti::lang {
   }
 
   Tokenizer::Tokenizer(Tokenizer &&rhs) noexcept
-    : line(rhs.line)
-    , column(rhs.column)
-    , iter(rhs.iter)
+    : line(std::exchange(rhs.line, 0))
+    , column(std::exchange(rhs.column, 0))
+    , iter(std::exchange(rhs.iter, {}))
     , tokensGenerator(std::exchange(rhs.tokensGenerator, {}))
     , tokensBuffer(std::exchange(rhs.tokensBuffer, {}))
-    , source(std::exchange(rhs.source, "")) { }
+    , source(std::exchange(rhs.source, {})) { }
 
   Tokenizer &Tokenizer::operator=(Tokenizer &&rhs) noexcept {
-    line = rhs.line;
-    column = rhs.column;
-    iter = std::move(rhs.iter);
-    source = std::move(rhs.source);
-    tokensBuffer = std::move(rhs.tokensBuffer);
-    tokensGenerator = std::move(rhs.tokensGenerator);
+    line = std::exchange(rhs.line, 0);
+    column = std::exchange(rhs.column, 0);
+    iter = std::exchange(rhs.iter, {});
+    source = std::exchange(rhs.source, {});
+    tokensBuffer = std::exchange(rhs.tokensBuffer, {});
+    tokensGenerator = std::exchange(rhs.tokensGenerator, {});
     return *this;
   }
 
@@ -282,7 +283,7 @@ namespace arti::lang {
           }
           else if (*(iter + 1) == '/') {
             iter += 2;
-            column += 2;
+            column += 1;
             return {};
           }
         }
@@ -744,6 +745,8 @@ namespace arti::lang {
   TrieMap<TokenV> buildOperatorsTrieMap();
 
   Expected<Token> Tokenizer::readOperator() {
+    static auto tm = buildOperatorsTrieMap();
+
     auto stIter = iter;
 
     auto cLine = line;
@@ -754,8 +757,6 @@ namespace arti::lang {
       ++column;
     };
 
-    auto tm = buildOperatorsTrieMap();
-
     if (not tm.root.childs.contains(*iter)) {
       return langException<ExceptCode::ecInvalidCharacter>(line, column, *iter);
     }
@@ -781,8 +782,8 @@ namespace arti::lang {
 
     Token tok{
       TokenV::opStar,
-      line = cLine,
-      column = cColumn,
+      cLine,
+      cColumn,
       { stIter, iter }
     };
 
@@ -791,6 +792,7 @@ namespace arti::lang {
     }
     else if (lvNode != nullptr) {
       iter -= lvNodeDiff;
+      column -= static_cast<size_t>(lvNodeDiff);
       tok.value = *lvNode->value;
       tok.strValue = std::string_view{ stIter, iter };
     }