erick-alcachofa
85a34bdd65
Initial version of Tokenizer and Token Generator template for coroutines (used in tokenizer) Utilities like string related functions, TrieMap, and error handling TODO: Add tests for Tokenizer TODO: Add tests for Generator
109 lines
1.1 KiB
C++
109 lines
1.1 KiB
C++
#pragma once
|
|
|
|
#include <string>
|
|
|
|
namespace arti::lang {
|
|
|
|
enum class TokenV;
|
|
|
|
struct Token {
|
|
TokenV value;
|
|
|
|
std::size_t line;
|
|
std::size_t column;
|
|
|
|
std::string_view strValue;
|
|
};
|
|
|
|
enum class TokenV {
|
|
tkEOF,
|
|
tkString,
|
|
tkDecimal,
|
|
tkInteger,
|
|
tkCharacter,
|
|
tkIdentifier,
|
|
|
|
opDot,
|
|
opMod,
|
|
opPlus,
|
|
opHyphen,
|
|
opSlash,
|
|
opBang,
|
|
opStar,
|
|
opColon,
|
|
opComma,
|
|
opAssign,
|
|
opAccess,
|
|
opSemicolon,
|
|
|
|
opCaret,
|
|
opTilde,
|
|
|
|
opEq,
|
|
opNeq,
|
|
|
|
opLt,
|
|
opGt,
|
|
|
|
opLtEq,
|
|
opGtEq,
|
|
|
|
opLShift,
|
|
opRShift,
|
|
|
|
opBoolAnd,
|
|
opBoolOr,
|
|
|
|
opAnd,
|
|
opOr,
|
|
|
|
opLParen,
|
|
opRParen,
|
|
|
|
opLBracket,
|
|
opRBracket,
|
|
|
|
opLSquirly,
|
|
opRSquirly,
|
|
|
|
opArrow,
|
|
|
|
kwOr,
|
|
kwNot,
|
|
kwAnd,
|
|
|
|
kwIf,
|
|
kwElse,
|
|
|
|
kwFn,
|
|
kwEnum,
|
|
kwStruct,
|
|
kwVariant,
|
|
|
|
kwDef,
|
|
kwLet,
|
|
kwMut,
|
|
|
|
kwFor,
|
|
kwWhile,
|
|
|
|
kwReturn,
|
|
kwUnreachable,
|
|
|
|
kwDefer,
|
|
kwErrDefer,
|
|
|
|
kwTrue,
|
|
kwFalse,
|
|
|
|
kwNull,
|
|
|
|
kwImport,
|
|
kwExport,
|
|
kwModule,
|
|
};
|
|
|
|
std::string toString(const Token &value);
|
|
|
|
} // namespace arti::lang
|