artichoke-lang/tests/Tokenizer/src/Strings.cpp

//============================================================================//
//                                                                            //
//                       artichoke programming language                       //
//                                                                            //
//      Copyright (C) 2025 Erick Saul Guzman Ramos, whoami.artichoke.dev      //
//                                                                            //
//                                                                            //
//  This program is free software: you can redistribute it and/or modify      //
//  it under the terms of the GNU Affero General Public License as published  //
//  by the Free Software Foundation, either version 3 of the License, or      //
//  (at your option) any later version.                                       //
//                                                                            //
//  This program is distributed in the hope that it will be useful,           //
//  but WITHOUT ANY WARRANTY; without even the implied warranty of            //
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             //
//  GNU Affero General Public License for more details.                       //
//                                                                            //
//  You should have received a copy of the GNU Affero General Public License  //
//  along with this program.  If not, see <https://www.gnu.org/licenses/>.    //
//                                                                            //
//============================================================================//

#include <catch2/catch_test_macros.hpp>

#include <array>
#include <string>

#include <artichoke/Tokenizer/Tokenizer.hpp>
#include <artichoke/Util/Expected.hpp>
#include <Utils.hpp>

namespace lang = arti::lang;

template <std::size_t N>
static void CommonLiteralsSuccess(
  lang::TokenV kind,
  const std::array<std::string_view, N> &lexemes
) {
  const std::string source = SourceFromTokens(lexemes);

  std::size_t it = 0;
  lang::Tokenizer tkz{ source };

  for (auto token : tkz.range()) {
    REQUIRE(token.has_value());
    REQUIRE(token->value == kind);
    REQUIRE(token->strValue == lexemes.at(it++));
  }

  REQUIRE(it == lexemes.size());
  REQUIRE(tkz.peek().has_value());
  REQUIRE(tkz.peek()->value == lang::TokenV::tkEOF);
}

TEST_CASE("Strings_Simple", "[strings][valid]") {
  constexpr std::array<std::string_view, 5> lexemes = {
    R"("a")",
    R"("hello")",
    R"("with spaces")",
    R"("12345")",
    R"Q("!@#$%^&*()")Q"
  };

  CommonLiteralsSuccess(lang::TokenV::tkString, lexemes);
}

TEST_CASE("Strings_Escapes", "[strings][valid][escapes]") {
  // Validate common escape sequences remain part of lexeme text.
  constexpr std::array<std::string_view, 5> lexemes = {
    R"("quote: \"")",
    R"("backslash: \\")",
    R"("newline: \n")",
    R"("tab: \t")",
    R"("mix: \"\\\n\t")"
  };

  CommonLiteralsSuccess(lang::TokenV::tkString, lexemes);
}

TEST_CASE("Strings_OperatorsAdjacency", "[strings][operators]") {
  // "foo"+"bar" -> tkString, opPlus, tkString
  const std::string source = R"("foo"+"bar")";
  lang::Tokenizer tkz{ source };

  auto t1 = tkz.peek();
  REQUIRE(t1.has_value());
  REQUIRE(t1->value == lang::TokenV::tkString);
  REQUIRE(t1->strValue == R"("foo")");
  REQUIRE(tkz.consume().has_value());

  auto t2 = tkz.peek();
  REQUIRE(t2.has_value());
  REQUIRE(t2->value == lang::TokenV::opPlus);
  REQUIRE(tkz.consume().has_value());

  auto t3 = tkz.peek();
  REQUIRE(t3.has_value());
  REQUIRE(t3->value == lang::TokenV::tkString);
  REQUIRE(t3->strValue == R"("bar")");
  REQUIRE(tkz.consume().has_value());

  auto eof = tkz.peek();
  REQUIRE(eof.has_value());
  REQUIRE(eof->value == lang::TokenV::tkEOF);
}

TEST_CASE("Strings_Unterminated_Error", "[strings][error]") {
  // Missing closing quote should yield an error.
  const std::string source = "\"unterminated";
  lang::Tokenizer tkz{ source };

  auto errTok = tkz.peek();
  REQUIRE_FALSE(errTok.has_value());
  const auto &err = errTok.error();

  REQUIRE(
    err.message.find("Invalid literal") != std::string::npos
  );
}

TEST_CASE("Chars_Simple", "[chars][valid]") {
  constexpr std::array<std::string_view, 5> lexemes = {
    R"('a')",
    R"('Z')",
    R"('0')",
    R"('_')",
    R"('$')"
  };

  CommonLiteralsSuccess(lang::TokenV::tkCharacter, lexemes);
}

TEST_CASE("Chars_Escapes", "[chars][valid][escapes]") {
  constexpr std::array<std::string_view, 4> lexemes = {
    R"('\n')",
    R"('\t')",
    R"('\\')",
    R"('\'')"
  };

  CommonLiteralsSuccess(lang::TokenV::tkCharacter, lexemes);
}

TEST_CASE("Chars_Invalid_Empty", "[chars][error]") {
  const std::string source = "''";
  lang::Tokenizer tkz{ source };

  auto errTok = tkz.peek();
  REQUIRE_FALSE(errTok.has_value());
  const auto &err = errTok.error();
  REQUIRE(
    err.message.find("Invalid literal") != std::string::npos
  );
}

TEST_CASE("Chars_Invalid_Multiple", "[chars][error]") {
  const std::string source = "'ab'";
  lang::Tokenizer tkz{ source };

  auto errTok = tkz.peek();
  REQUIRE_FALSE(errTok.has_value());
  const auto &err = errTok.error();
  REQUIRE(
    err.message.find("Invalid literal") != std::string::npos
  );
}

TEST_CASE("Chars_Unterminated", "[chars][error]") {
  const std::string source = "'a";
  lang::Tokenizer tkz{ source };

  auto errTok = tkz.peek();
  REQUIRE_FALSE(errTok.has_value());
  const auto &err = errTok.error();
  REQUIRE(
    err.message.find("Invalid literal") != std::string::npos
  );
}

TEST_CASE("Chars_InvalidEscape", "[chars][error][.escapes]") {
  const std::string source = "'\\x'";
  lang::Tokenizer tkz{ source };

  auto errTok = tkz.peek();
  REQUIRE_FALSE(errTok.has_value());
  const auto &err = errTok.error();
  REQUIRE(
    err.message.find("Invalid literal") != std::string::npos
  );
}