erick-alcachofa
d0599d374f
Signed-off-by: erick-alcachofa <erick@artichoke.dev> This commit lays the foundational groundwork for the artichoke language parser by introducing the formal language grammar specification. The tokenizer was updated to include new operators and keywords, also added the posibility to handle comments. Key Additions: - Implemented support for C-style block comments (`/* ... */`), including error handling for unclosed comments. - Added all necessary tokens for missing keywords (e.g., `module`, `export`, `using`, `match`, `loop`) and operators (e.g., `+=`, `:=`, `.#`, `.*`, `.@`). - The `Token` enum has been expanded to reflect the full language feature set. Documentation: - Added `docs/grammar.ebnf` which contains the official, well-structured EBNF grammar for the language. - Added `docs/readme.md` providing a detailed technical overview of the language's features, syntax, and semantics. BREAKING CHANGE: The `kwVariant` and `kwMut` tokens have been removed to align with the updated language design defined in the new grammar.
114 lines
3.1 KiB
C++
114 lines
3.1 KiB
C++
#pragma once
|
|
|
|
#include <string>
|
|
|
|
namespace arti::lang {
|
|
|
|
enum class TokenV;
|
|
|
|
struct Token {
|
|
TokenV value;
|
|
|
|
std::size_t line;
|
|
std::size_t column;
|
|
|
|
std::string_view strValue;
|
|
};
|
|
|
|
enum class TokenV {
|
|
tkEOF,
|
|
tkString,
|
|
tkDecimal,
|
|
tkInteger,
|
|
tkCharacter,
|
|
tkIdentifier,
|
|
|
|
opDot, /* . */
|
|
opMod, /* % */
|
|
opPlus, /* + */
|
|
opHyphen, /* - */
|
|
opSlash, /* / */
|
|
opBang, /* ! */
|
|
opStar, /* * */
|
|
opColon, /* : */
|
|
opComma, /* , */
|
|
opAssign, /* = */
|
|
opAccess, /* :: */
|
|
opSemicolon, /* ; */
|
|
opCaret, /* ^ */
|
|
opTilde, /* ~ */
|
|
opEq, /* == */
|
|
opNeq, /* != */
|
|
opLt, /* < */
|
|
opGt, /* > */
|
|
opLtEq, /* <= */
|
|
opGtEq, /* >= */
|
|
opLShift, /* << */
|
|
opRShift, /* >> */
|
|
opBoolAnd, /* && */
|
|
opBoolOr, /* || */
|
|
opAnd, /* & */
|
|
opOr, /* | */
|
|
opLParen, /* ( */
|
|
opRParen, /* ) */
|
|
opLBracket, /* [ */
|
|
opRBracket, /* ] */
|
|
opLSquirly, /* { */
|
|
opRSquirly, /* } */
|
|
opArrow, /* -> */
|
|
opPlusAssign, /* += */
|
|
opHyphenAssign, /* -= */
|
|
opStarAssign, /* *= */
|
|
opSlashAssign, /* /= */
|
|
opModAssign, /* %= */
|
|
opAndAssign, /* &= */
|
|
opOrAssign, /* |= */
|
|
opLShiftAssign, /* <<= */
|
|
opRShiftAssign, /* >>= */
|
|
opBoolAndAssign, /* &&= */
|
|
opBoolORAssign, /* ||= */
|
|
opMut, /* $ */
|
|
opOpt, /* ? */
|
|
opSliceSize, /* .# */
|
|
opPtrSlice, /* .[ */
|
|
opSlicePtr, /* .* */
|
|
opReflect, /* .@ */
|
|
opLabel, /* := */
|
|
|
|
/* Keywords */
|
|
kwUnderscore, /* _ */
|
|
kwOr, /* or */
|
|
kwNot, /* not */
|
|
kwAnd, /* and */
|
|
kwIf, /* if */
|
|
kwElse, /* else */
|
|
kwFn, /* fn */
|
|
kwEnum, /* enum */
|
|
kwStruct, /* struct */
|
|
kwDef, /* def */
|
|
kwLet, /* let */
|
|
kwFor, /* for */
|
|
kwLoop, /* loop */
|
|
kwBreak, /* break */
|
|
kwContinue, /* continue */
|
|
kwWhile, /* while */
|
|
kwMatch, /* match */
|
|
kwSwitch, /* switch */
|
|
kwReturn, /* return */
|
|
kwUnreachable, /* unreachable */
|
|
kwDefer, /* defer */
|
|
kwErrDefer, /* errdefer */
|
|
kwTrue, /* true */
|
|
kwFalse, /* false */
|
|
kwNull, /* null */
|
|
kwThis, /* this */
|
|
kwImport, /* import */
|
|
kwExport, /* export */
|
|
kwModule, /* module */
|
|
kwUsing, /* using */
|
|
};
|
|
|
|
std::string toString(const Token &value);
|
|
|
|
} // namespace arti::lang
|