diff --git a/lib/include/artichoke/Tokenizer/Token.hpp b/lib/include/artichoke/Tokenizer/Token.hpp index 9a990c4..207f3b8 100644 --- a/lib/include/artichoke/Tokenizer/Token.hpp +++ b/lib/include/artichoke/Tokenizer/Token.hpp @@ -66,7 +66,7 @@ namespace arti::lang { opLShiftAssign, /* <<= */ opRShiftAssign, /* >>= */ opBoolAndAssign, /* &&= */ - opBoolORAssign, /* ||= */ + opBoolOrAssign, /* ||= */ opMut, /* $ */ opOpt, /* ? */ opSliceSize, /* .# */ @@ -91,11 +91,13 @@ namespace arti::lang { kwLoop, /* loop */ kwBreak, /* break */ kwContinue, /* continue */ + kwDo, /* do */ kwWhile, /* while */ kwMatch, /* match */ kwSwitch, /* switch */ kwReturn, /* return */ kwUnreachable, /* unreachable */ + kwTypename, /* typename */ kwDefer, /* defer */ kwErrDefer, /* errdefer */ kwTrue, /* true */ @@ -109,5 +111,6 @@ namespace arti::lang { }; std::string toString(const Token &value); + std::string toString(const TokenV &value); } // namespace arti::lang diff --git a/lib/src/Tokenizer/Token.cpp b/lib/src/Tokenizer/Token.cpp index ad7f47d..957c9c8 100644 --- a/lib/src/Tokenizer/Token.cpp +++ b/lib/src/Tokenizer/Token.cpp @@ -1,107 +1,116 @@ #include #include -#include namespace arti::lang { std::string toString(const Token &value) { using enum TokenV; - std::string_view tokenStr; + std::string tokenStr = toString(value.value); - switch (value.value) { - case tkEOF: return "Token{ tkEOF }"; - case tkString: tokenStr = "tkString"; break; - case tkDecimal: tokenStr = "tkDecimal"; break; - case tkInteger: tokenStr = "tkInteger"; break; - case tkCharacter: tokenStr = "tkCharacter"; break; - case tkIdentifier: tokenStr = "tkIdentifier"; break; - case opDot: tokenStr = "opDot"; break; - case opMod: tokenStr = "opMod"; break; - case opPlus: tokenStr = "opPlus"; break; - case opHyphen: tokenStr = "opHyphen"; break; - case opSlash: tokenStr = "opSlash"; break; - case opBang: tokenStr = "opBang"; break; - case opStar: tokenStr = "opStar"; break; - case opColon: tokenStr = "opColon"; break; - case opComma: tokenStr = "opComma"; break; - case opAssign: tokenStr = "opAssign"; break; - case opAccess: tokenStr = "opAccess"; break; - case opSemicolon: tokenStr = "opSemicolon"; break; - case opCaret: tokenStr = "opCaret"; break; - case opTilde: tokenStr = "opTilde"; break; - case opEq: tokenStr = "opEq"; break; - case opNeq: tokenStr = "opNeq"; break; - case opLt: tokenStr = "opLt"; break; - case opGt: tokenStr = "opGt"; break; - case opLtEq: tokenStr = "opLtEq"; break; - case opGtEq: tokenStr = "opGtEq"; break; - case opLShift: tokenStr = "opLShift"; break; - case opRShift: tokenStr = "opRShift"; break; - case opBoolAnd: tokenStr = "opBoolAnd"; break; - case opBoolOr: tokenStr = "opBoolOr"; break; - case opAnd: tokenStr = "opAnd"; break; - case opOr: tokenStr = "opOr"; break; - case opLParen: tokenStr = "opLParen"; break; - case opRParen: tokenStr = "opRParen"; break; - case opLBracket: tokenStr = "opLBracket"; break; - case opRBracket: tokenStr = "opRBracket"; break; - case opLSquirly: tokenStr = "opLSquirly"; break; - case opRSquirly: tokenStr = "opRSquirly"; break; - case opArrow: tokenStr = "opArrow"; break; - case opPlusAssign: tokenStr = "opPlusAssign"; break; - case opHyphenAssign: tokenStr = "opHyphenAssign"; break; - case opStarAssign: tokenStr = "opStarAssign"; break; - case opSlashAssign: tokenStr = "opSlashAssign"; break; - case opModAssign: tokenStr = "opModAssign"; break; - case opAndAssign: tokenStr = "opAndAssign"; break; - case opOrAssign: tokenStr = "opOrAssign"; break; - case opLShiftAssign: tokenStr = "opLShiftAssign"; break; - case opRShiftAssign: tokenStr = "opRShiftAssign"; break; - case opBoolAndAssign: tokenStr = "opBoolAndAssign"; break; - case opBoolORAssign: tokenStr = "opBoolORAssign"; break; - case opMut: tokenStr = "opMut"; break; - case opOpt: tokenStr = "opOpt"; break; - case opSliceSize: tokenStr = "opSliceSize"; break; - case opPtrSlice: tokenStr = "opPtrSlice"; break; - case opSlicePtr: tokenStr = "opSlicePtr"; break; - case opReflect: tokenStr = "opReflect"; break; - case opLabel: tokenStr = "opLabel"; break; - case kwUnderscore: tokenStr = "kwUnderscore"; break; - case kwOr: tokenStr = "kwOr"; break; - case kwNot: tokenStr = "kwNot"; break; - case kwAnd: tokenStr = "kwAnd"; break; - case kwIf: tokenStr = "kwIf"; break; - case kwElse: tokenStr = "kwElse"; break; - case kwFn: tokenStr = "kwFn"; break; - case kwEnum: tokenStr = "kwEnum"; break; - case kwStruct: tokenStr = "kwStruct"; break; - case kwDef: tokenStr = "kwDef"; break; - case kwLet: tokenStr = "kwLet"; break; - case kwFor: tokenStr = "kwFor"; break; - case kwLoop: tokenStr = "kwLoop"; break; - case kwBreak: tokenStr = "kwBreak"; break; - case kwContinue: tokenStr = "kwContinue"; break; - case kwWhile: tokenStr = "kwWhile"; break; - case kwMatch: tokenStr = "kwMatch"; break; - case kwSwitch: tokenStr = "kwSwitch"; break; - case kwReturn: tokenStr = "kwReturn"; break; - case kwUnreachable: tokenStr = "kwUnreachable"; break; - case kwDefer: tokenStr = "kwDefer"; break; - case kwErrDefer: tokenStr = "kwErrDefer"; break; - case kwTrue: tokenStr = "kwTrue"; break; - case kwFalse: tokenStr = "kwFalse"; break; - case kwNull: tokenStr = "kwNull"; break; - case kwThis: tokenStr = "kwThis"; break; - case kwImport: tokenStr = "kwImport"; break; - case kwExport: tokenStr = "kwExport"; break; - case kwModule: tokenStr = "kwModule"; break; - case kwUsing: tokenStr = "kwUsing"; break; - default: tokenStr = ""; break; + if (value.value == tkEOF) { + return std::format("Token{{ {} }}", tokenStr); } return std::format("Token{{ {}, {} }}", tokenStr, value.strValue); } + std::string toString(const TokenV &value) { + using enum TokenV; + + switch (value) { + case tkEOF: return "tkEOF"; + case tkString: return "tkString"; + case tkDecimal: return "tkDecimal"; + case tkInteger: return "tkInteger"; + case tkCharacter: return "tkCharacter"; + case tkIdentifier: return "tkIdentifier"; + case opDot: return "opDot"; + case opMod: return "opMod"; + case opPlus: return "opPlus"; + case opHyphen: return "opHyphen"; + case opSlash: return "opSlash"; + case opBang: return "opBang"; + case opStar: return "opStar"; + case opColon: return "opColon"; + case opComma: return "opComma"; + case opAssign: return "opAssign"; + case opAccess: return "opAccess"; + case opSemicolon: return "opSemicolon"; + case opCaret: return "opCaret"; + case opTilde: return "opTilde"; + case opEq: return "opEq"; + case opNeq: return "opNeq"; + case opLt: return "opLt"; + case opGt: return "opGt"; + case opLtEq: return "opLtEq"; + case opGtEq: return "opGtEq"; + case opLShift: return "opLShift"; + case opRShift: return "opRShift"; + case opBoolAnd: return "opBoolAnd"; + case opBoolOr: return "opBoolOr"; + case opAnd: return "opAnd"; + case opOr: return "opOr"; + case opLParen: return "opLParen"; + case opRParen: return "opRParen"; + case opLBracket: return "opLBracket"; + case opRBracket: return "opRBracket"; + case opLSquirly: return "opLSquirly"; + case opRSquirly: return "opRSquirly"; + case opArrow: return "opArrow"; + case opPlusAssign: return "opPlusAssign"; + case opHyphenAssign: return "opHyphenAssign"; + case opStarAssign: return "opStarAssign"; + case opSlashAssign: return "opSlashAssign"; + case opModAssign: return "opModAssign"; + case opAndAssign: return "opAndAssign"; + case opOrAssign: return "opOrAssign"; + case opLShiftAssign: return "opLShiftAssign"; + case opRShiftAssign: return "opRShiftAssign"; + case opBoolAndAssign: return "opBoolAndAssign"; + case opBoolOrAssign: return "opBoolORAssign"; + case opMut: return "opMut"; + case opOpt: return "opOpt"; + case opSliceSize: return "opSliceSize"; + case opPtrSlice: return "opPtrSlice"; + case opSlicePtr: return "opSlicePtr"; + case opReflect: return "opReflect"; + case opLabel: return "opLabel"; + case kwUnderscore: return "kwUnderscore"; + case kwOr: return "kwOr"; + case kwNot: return "kwNot"; + case kwAnd: return "kwAnd"; + case kwIf: return "kwIf"; + case kwElse: return "kwElse"; + case kwFn: return "kwFn"; + case kwEnum: return "kwEnum"; + case kwStruct: return "kwStruct"; + case kwDef: return "kwDef"; + case kwLet: return "kwLet"; + case kwFor: return "kwFor"; + case kwLoop: return "kwLoop"; + case kwBreak: return "kwBreak"; + case kwContinue: return "kwContinue"; + case kwDo: return "kwDo"; + case kwWhile: return "kwWhile"; + case kwMatch: return "kwMatch"; + case kwSwitch: return "kwSwitch"; + case kwReturn: return "kwReturn"; + case kwUnreachable: return "kwUnreachable"; + case kwTypename: return "kwTypename"; + case kwDefer: return "kwDefer"; + case kwErrDefer: return "kwErrDefer"; + case kwTrue: return "kwTrue"; + case kwFalse: return "kwFalse"; + case kwNull: return "kwNull"; + case kwThis: return "kwThis"; + case kwImport: return "kwImport"; + case kwExport: return "kwExport"; + case kwModule: return "kwModule"; + case kwUsing: return "kwUsing"; + default: return ""; + } + } + } // namespace arti::lang diff --git a/lib/src/Tokenizer/Tokenizer.cpp b/lib/src/Tokenizer/Tokenizer.cpp index c74574c..f95b457 100644 --- a/lib/src/Tokenizer/Tokenizer.cpp +++ b/lib/src/Tokenizer/Tokenizer.cpp @@ -609,6 +609,9 @@ namespace arti::lang { else if (tok.strValue.compare("continue") == 0) { tok.value = TokenV::kwContinue; } + else if (tok.strValue.compare("do") == 0) { + tok.value = TokenV::kwDo; + } else if (tok.strValue.compare("while") == 0) { tok.value = TokenV::kwWhile; } @@ -624,6 +627,9 @@ namespace arti::lang { else if (tok.strValue.compare("unreachable") == 0) { tok.value = TokenV::kwUnreachable; } + else if (tok.strValue.compare("typename") == 0) { + tok.value = TokenV::kwTypename; + } else if (tok.strValue.compare("defer") == 0) { tok.value = TokenV::kwDefer; } @@ -768,7 +774,7 @@ namespace arti::lang { tm.insert("<<=", TokenV::opLShiftAssign); tm.insert(">>=", TokenV::opRShiftAssign); tm.insert("&&=", TokenV::opBoolAndAssign); - tm.insert("||=", TokenV::opBoolORAssign); + tm.insert("||=", TokenV::opBoolOrAssign); tm.insert("$", TokenV::opMut); tm.insert("?", TokenV::opOpt); tm.insert(".#", TokenV::opSliceSize); diff --git a/tests/Tokenizer/src/Operators.cpp b/tests/Tokenizer/src/Operators.cpp index 9955bc0..cca6eca 100644 --- a/tests/Tokenizer/src/Operators.cpp +++ b/tests/Tokenizer/src/Operators.cpp @@ -61,7 +61,7 @@ TEST_CASE("Operators_MultiChar", "[operators][multi]") { lang::TokenV::opLShift, lang::TokenV::opRShift, lang::TokenV::opBoolAnd, lang::TokenV::opBoolOr, lang::TokenV::opPlusAssign, lang::TokenV::opHyphenAssign, lang::TokenV::opStarAssign, lang::TokenV::opSlashAssign, lang::TokenV::opModAssign, lang::TokenV::opAndAssign, lang::TokenV::opOrAssign, - lang::TokenV::opLShiftAssign, lang::TokenV::opRShiftAssign, lang::TokenV::opBoolAndAssign, lang::TokenV::opBoolORAssign, + lang::TokenV::opLShiftAssign, lang::TokenV::opRShiftAssign, lang::TokenV::opBoolAndAssign, lang::TokenV::opBoolOrAssign, lang::TokenV::opArrow, lang::TokenV::opAccess, lang::TokenV::opLabel }; @@ -86,7 +86,7 @@ TEST_CASE("Operators_GreedyLongestMatch", "[operators][greedy]") { ">=", "<=", "->", "::" }; constexpr std::array kinds = { - lang::TokenV::opRShiftAssign, lang::TokenV::opLShiftAssign, lang::TokenV::opBoolAndAssign, lang::TokenV::opBoolORAssign, + lang::TokenV::opRShiftAssign, lang::TokenV::opLShiftAssign, lang::TokenV::opBoolAndAssign, lang::TokenV::opBoolOrAssign, lang::TokenV::opGtEq, lang::TokenV::opLtEq, lang::TokenV::opArrow, lang::TokenV::opAccess };