diff --git a/lib/include/artichoke/Util/Strings.hpp b/lib/include/artichoke/Util/Strings.hpp index 748096a..e011a69 100644 --- a/lib/include/artichoke/Util/Strings.hpp +++ b/lib/include/artichoke/Util/Strings.hpp @@ -27,7 +27,7 @@ namespace arti::lang { } static inline constexpr bool isIdentChar(char c) { - return isLetter(c) || c == '_'; + return isLetter(c) || c == '_' || isDigit(c); } static inline constexpr bool isHexChar(char c) { diff --git a/lib/src/Tokenizer/Token.cpp b/lib/src/Tokenizer/Token.cpp index 9718b57..78052af 100644 --- a/lib/src/Tokenizer/Token.cpp +++ b/lib/src/Tokenizer/Token.cpp @@ -8,189 +8,76 @@ namespace arti::lang { std::string toString(const Token &value) { using enum TokenV; + std::string_view tokenStr; + switch (value.value) { - case tkEOF: - return std::format("Token{{ {} }}", "tkEOF", value.strValue); - case tkString: - return std::format("Token{{ {}, {} }}", "tkString", value.strValue); - case tkDecimal: - return std::format( - "Token{{ {}, {} }}", - "tkDecimal", - value.strValue - ); - case tkInteger: - return std::format( - "Token{{ {}, {} }}", - "tkInteger", - value.strValue - ); - case tkCharacter: - return std::format( - "Token{{ {}, {} }}", - "tkCharacter", - value.strValue - ); - case tkIdentifier: - return std::format( - "Token{{ {}, {} }}", - "tkIdentifier", - value.strValue - ); - case opDot: - return std::format("Token{{ {}, {} }}", "opDot", value.strValue); - case opMod: - return std::format("Token{{ {}, {} }}", "opMod", value.strValue); - case opPlus: - return std::format("Token{{ {}, {} }}", "opPlus", value.strValue); - case opHyphen: - return std::format("Token{{ {}, {} }}", "opHyphen", value.strValue); - case opSlash: - return std::format("Token{{ {}, {} }}", "opSlash", value.strValue); - case opBang: - return std::format("Token{{ {}, {} }}", "opBang", value.strValue); - case opStar: - return std::format("Token{{ {}, {} }}", "opStar", value.strValue); - case opColon: - return std::format("Token{{ {}, {} }}", "opColon", value.strValue); - case opComma: - return std::format("Token{{ {}, {} }}", "opComma", value.strValue); - case opAssign: - return std::format("Token{{ {}, {} }}", "opAssign", value.strValue); - case opAccess: - return std::format("Token{{ {}, {} }}", "opAccess", value.strValue); - case opSemicolon: - return std::format( - "Token{{ {}, {} }}", - "opSemicolon", - value.strValue - ); - case opCaret: - return std::format("Token{{ {}, {} }}", "opCaret", value.strValue); - case opTilde: - return std::format("Token{{ {}, {} }}", "opTilde", value.strValue); - case opEq: - return std::format("Token{{ {}, {} }}", "opEq", value.strValue); - case opNeq: - return std::format("Token{{ {}, {} }}", "opNeq", value.strValue); - case opLt: - return std::format("Token{{ {}, {} }}", "opLt", value.strValue); - case opGt: - return std::format("Token{{ {}, {} }}", "opGt", value.strValue); - case opLtEq: - return std::format("Token{{ {}, {} }}", "opLtEq", value.strValue); - case opGtEq: - return std::format("Token{{ {}, {} }}", "opGtEq", value.strValue); - case opLShift: - return std::format("Token{{ {}, {} }}", "opLShift", value.strValue); - case opRShift: - return std::format("Token{{ {}, {} }}", "opRShift", value.strValue); - case opBoolAnd: - return std::format( - "Token{{ {}, {} }}", - "opBoolAnd", - value.strValue - ); - case opBoolOr: - return std::format("Token{{ {}, {} }}", "opBoolOr", value.strValue); - case opAnd: - return std::format("Token{{ {}, {} }}", "opAnd", value.strValue); - case opOr: - return std::format("Token{{ {}, {} }}", "opOr", value.strValue); - case opLParen: - return std::format("Token{{ {}, {} }}", "opLParen", value.strValue); - case opRParen: - return std::format("Token{{ {}, {} }}", "opRParen", value.strValue); - case opLBracket: - return std::format( - "Token{{ {}, {} }}", - "opLBracket", - value.strValue - ); - case opRBracket: - return std::format( - "Token{{ {}, {} }}", - "opRBracket", - value.strValue - ); - case opLSquirly: - return std::format( - "Token{{ {}, {} }}", - "opLSquirly", - value.strValue - ); - case opRSquirly: - return std::format( - "Token{{ {}, {} }}", - "opRSquirly", - value.strValue - ); - case opArrow: - return std::format("Token{{ {}, {} }}", "opArrow", value.strValue); - case kwOr: - return std::format("Token{{ {}, {} }}", "kwOr", value.strValue); - case kwNot: - return std::format("Token{{ {}, {} }}", "kwNot", value.strValue); - case kwAnd: - return std::format("Token{{ {}, {} }}", "kwAnd", value.strValue); - case kwIf: - return std::format("Token{{ {}, {} }}", "kwIf", value.strValue); - case kwElse: - return std::format("Token{{ {}, {} }}", "kwElse", value.strValue); - case kwFn: - return std::format("Token{{ {}, {} }}", "kwFn", value.strValue); - case kwEnum: - return std::format("Token{{ {}, {} }}", "kwEnum", value.strValue); - case kwStruct: - return std::format("Token{{ {}, {} }}", "kwStruct", value.strValue); - case kwVariant: - return std::format( - "Token{{ {}, {} }}", - "kwVariant", - value.strValue - ); - case kwDef: - return std::format("Token{{ {}, {} }}", "kwDef", value.strValue); - case kwLet: - return std::format("Token{{ {}, {} }}", "kwLet", value.strValue); - case kwMut: - return std::format("Token{{ {}, {} }}", "kwMut", value.strValue); - case kwFor: - return std::format("Token{{ {}, {} }}", "kwFor", value.strValue); - case kwWhile: - return std::format("Token{{ {}, {} }}", "kwWhile", value.strValue); - case kwReturn: - return std::format("Token{{ {}, {} }}", "kwReturn", value.strValue); - case kwUnreachable: - return std::format( - "Token{{ {}, {} }}", - "kwUnreachable", - value.strValue - ); - case kwDefer: - return std::format("Token{{ {}, {} }}", "kwDefer", value.strValue); - case kwErrDefer: - return std::format( - "Token{{ {}, {} }}", - "kwErrDefer", - value.strValue - ); - case kwTrue: - return std::format("Token{{ {}, {} }}", "kwTrue", value.strValue); - case kwFalse: - return std::format("Token{{ {}, {} }}", "kwFalse", value.strValue); - case kwNull: - return std::format("Token{{ {}, {} }}", "kwNull", value.strValue); - case kwImport: - return std::format("Token{{ {}, {} }}", "kwImport", value.strValue); - case kwExport: - return std::format("Token{{ {}, {} }}", "kwExport", value.strValue); - case kwModule: - return std::format("Token{{ {}, {} }}", "kwModule", value.strValue); - default: - return ""; + case tkEOF: return "Token{{ tkEOF }}"; + case tkString: tokenStr = "tkString"; break; + case tkDecimal: tokenStr = "tkDecimal"; break; + case tkInteger: tokenStr = "tkInteger"; break; + case tkCharacter: tokenStr = "tkCharacter"; break; + case tkIdentifier: tokenStr = "tkIdentifier"; break; + case opDot: tokenStr = "opDot"; break; + case opMod: tokenStr = "opMod"; break; + case opPlus: tokenStr = "opPlus"; break; + case opHyphen: tokenStr = "opHyphen"; break; + case opSlash: tokenStr = "opSlash"; break; + case opBang: tokenStr = "opBang"; break; + case opStar: tokenStr = "opStar"; break; + case opColon: tokenStr = "opColon"; break; + case opComma: tokenStr = "opComma"; break; + case opAssign: tokenStr = "opAssign"; break; + case opAccess: tokenStr = "opAccess"; break; + case opSemicolon: tokenStr = "opSemicolon"; break; + case opCaret: tokenStr = "opCaret"; break; + case opTilde: tokenStr = "opTilde"; break; + case opEq: tokenStr = "opEq"; break; + case opNeq: tokenStr = "opNeq"; break; + case opLt: tokenStr = "opLt"; break; + case opGt: tokenStr = "opGt"; break; + case opLtEq: tokenStr = "opLtEq"; break; + case opGtEq: tokenStr = "opGtEq"; break; + case opLShift: tokenStr = "opLShift"; break; + case opRShift: tokenStr = "opRShift"; break; + case opBoolAnd: tokenStr = "opBoolAnd"; break; + case opBoolOr: tokenStr = "opBoolOr"; break; + case opAnd: tokenStr = "opAnd"; break; + case opOr: tokenStr = "opOr"; break; + case opLParen: tokenStr = "opLParen"; break; + case opRParen: tokenStr = "opRParen"; break; + case opLBracket: tokenStr = "opLBracket"; break; + case opRBracket: tokenStr = "opRBracket"; break; + case opLSquirly: tokenStr = "opLSquirly"; break; + case opRSquirly: tokenStr = "opRSquirly"; break; + case opArrow: tokenStr = "opArrow"; break; + case kwOr: tokenStr = "kwOr"; break; + case kwNot: tokenStr = "kwNot"; break; + case kwAnd: tokenStr = "kwAnd"; break; + case kwIf: tokenStr = "kwIf"; break; + case kwElse: tokenStr = "kwElse"; break; + case kwFn: tokenStr = "kwFn"; break; + case kwEnum: tokenStr = "kwEnum"; break; + case kwStruct: tokenStr = "kwStruct"; break; + case kwVariant: tokenStr = "kwVariant"; break; + case kwDef: tokenStr = "kwDef"; break; + case kwLet: tokenStr = "kwLet"; break; + case kwMut: tokenStr = "kwMut"; break; + case kwFor: tokenStr = "kwFor"; break; + case kwWhile: tokenStr = "kwWhile"; break; + case kwReturn: tokenStr = "kwReturn"; break; + case kwUnreachable: tokenStr = "kwUnreachable"; break; + case kwDefer: tokenStr = "kwDefer"; break; + case kwErrDefer: tokenStr = "kwErrDefer"; break; + case kwTrue: tokenStr = "kwTrue"; break; + case kwFalse: tokenStr = "kwFalse"; break; + case kwNull: tokenStr = "kwNull"; break; + case kwImport: tokenStr = "kwImport"; break; + case kwExport: tokenStr = "kwExport"; break; + case kwModule: tokenStr = "kwModule"; break; + default: tokenStr = ""; break; } - std::unreachable(); + return std::format("Token{{ {}, {} }}", tokenStr, value.strValue); } + } // namespace arti::lang diff --git a/lib/src/Tokenizer/Tokenizer.cpp b/lib/src/Tokenizer/Tokenizer.cpp index 9c1dd47..f20a25f 100644 --- a/lib/src/Tokenizer/Tokenizer.cpp +++ b/lib/src/Tokenizer/Tokenizer.cpp @@ -430,7 +430,7 @@ namespace arti::lang { return langException( line, column, - "end of string (\")", + "end of char literal (')", "EOF" ); } @@ -439,7 +439,7 @@ namespace arti::lang { return langException( line, column, - "end of string (\")", + "end char literal (')", *iter ); } @@ -478,78 +478,76 @@ namespace arti::lang { { stIter, iter } }; - std::string_view strToken{ stIter, iter }; - - if (strToken.compare("or") == 0) { + if (tok.strValue.compare("or") == 0) { tok.value = TokenV::kwOr; } - else if (strToken.compare("not") == 0) { + else if (tok.strValue.compare("not") == 0) { tok.value = TokenV::kwNot; } - else if (strToken.compare("and") == 0) { + else if (tok.strValue.compare("and") == 0) { tok.value = TokenV::kwAnd; } - else if (strToken.compare("if") == 0) { + else if (tok.strValue.compare("if") == 0) { tok.value = TokenV::kwIf; } - else if (strToken.compare("else") == 0) { + else if (tok.strValue.compare("else") == 0) { tok.value = TokenV::kwElse; } - else if (strToken.compare("fn") == 0) { + else if (tok.strValue.compare("fn") == 0) { tok.value = TokenV::kwFn; } - else if (strToken.compare("enum") == 0) { + else if (tok.strValue.compare("enum") == 0) { tok.value = TokenV::kwEnum; } - else if (strToken.compare("struct") == 0) { + else if (tok.strValue.compare("struct") == 0) { tok.value = TokenV::kwStruct; } - else if (strToken.compare("variant") == 0) { + else if (tok.strValue.compare("variant") == 0) { tok.value = TokenV::kwVariant; } - else if (strToken.compare("def") == 0) { + else if (tok.strValue.compare("def") == 0) { tok.value = TokenV::kwDef; } - else if (strToken.compare("let") == 0) { + else if (tok.strValue.compare("let") == 0) { tok.value = TokenV::kwLet; } - else if (strToken.compare("mut") == 0) { + else if (tok.strValue.compare("mut") == 0) { tok.value = TokenV::kwMut; } - else if (strToken.compare("for") == 0) { + else if (tok.strValue.compare("for") == 0) { tok.value = TokenV::kwFor; } - else if (strToken.compare("while") == 0) { + else if (tok.strValue.compare("while") == 0) { tok.value = TokenV::kwWhile; } - else if (strToken.compare("return") == 0) { + else if (tok.strValue.compare("return") == 0) { tok.value = TokenV::kwReturn; } - else if (strToken.compare("unreachable") == 0) { + else if (tok.strValue.compare("unreachable") == 0) { tok.value = TokenV::kwUnreachable; } - else if (strToken.compare("defer") == 0) { + else if (tok.strValue.compare("defer") == 0) { tok.value = TokenV::kwDefer; } - else if (strToken.compare("errdefer") == 0) { + else if (tok.strValue.compare("errdefer") == 0) { tok.value = TokenV::kwErrDefer; } - else if (strToken.compare("true") == 0) { + else if (tok.strValue.compare("true") == 0) { tok.value = TokenV::kwTrue; } - else if (strToken.compare("false") == 0) { + else if (tok.strValue.compare("false") == 0) { tok.value = TokenV::kwFalse; } - else if (strToken.compare("null") == 0) { + else if (tok.strValue.compare("null") == 0) { tok.value = TokenV::kwNull; } - else if (strToken.compare("import") == 0) { + else if (tok.strValue.compare("import") == 0) { tok.value = TokenV::kwImport; } - else if (strToken.compare("export") == 0) { + else if (tok.strValue.compare("export") == 0) { tok.value = TokenV::kwExport; } - else if (strToken.compare("module") == 0) { + else if (tok.strValue.compare("module") == 0) { tok.value = TokenV::kwModule; } @@ -610,7 +608,7 @@ namespace arti::lang { tok.strValue = std::string_view{ stIter, iter }; } else { - return langException( + return langException( line, column, tok.strValue