diff --git a/docs/grammar.ebnf b/docs/grammar.ebnf new file mode 100644 index 0000000..134ac36 --- /dev/null +++ b/docs/grammar.ebnf @@ -0,0 +1,348 @@ +/* +================================================================================ +| | +| The Artichoke Programming Language | +| Official EBNF Grammar | +| | +================================================================================ +*/ + + +/* --- Program Structure --- */ +/* A program is a sequence of top-level declarations and statements. */ + + = + ( + | + | + | + | + | )* + + = + "export"? "module" "{" + ( + | + | + | + | )* + "}" + + = + "import" ";" + + = + + | "::" "*" + + = + "using" "=" ";" + + +/* --- Declarations --- */ +/* Rules for defining functions, structs, enums, and their components. */ + + = + "export"? "fn" "(" ")" ( "->" )? + + = + ? + + = + "this" ("," ( "," )* )? + | ( "," )* + + = + ":" + + = + "export"? "struct" "{" "}" + + = + ( "," )* + + = + ":" + + = + "export"? "enum" "{" "}" + + = + ( "," )* + + = + ( "(" ")" )? + + = + ( "<" ">" )? + + = + ( "," )* + + = + "typename" + + +/* --- Statements & Control Flow --- */ +/* Rules for code blocks, variable declarations, and control structures. */ + + = + "{" ? "}" + + = + ( )* + + = + ";" + | + | + | ";" + | ";" + | ";" + | ";" + | ";" + | + | + | + | + + = + ( ":" )? "=" + | ":" ( "=" )? + + = + "let" + | "def" + + = + "if" "(" ")" ? + ? + + = + "else" ? + | "else" + + = + "|" "|" + + = + ( ":=")? ( + + | + | + | + | + ) + + = + "for" "(" ? ";" ";" ")" + + + = + "for" "(" ":=" ")" + + + = + "while" "(" ")" ? + ? + + = + "do" "while" "(" ")" + + = + "loop" + + = + "match" "(" ")" "{" * ? "}" + + = + "switch" "(" ")" "{" * ? "}" + + = + ( | ) ( "(" ")" )? "->" + + = + "->" + + = + "_" "->" + + = + "break" ? + + = + "continue" ? + + = + "defer" ( | ) + + = + "errdefer" ( | ) + + = + "return" ? + + = + ";" + + +/* --- Expressions & Operator Precedence --- */ +/* The full expression hierarchy, from lowest to highest precedence. */ + + = + + + = + ( ( | ) )? + + = + ( ( "||" | "or" ) )* + + = + ( ( "&&" | "and" ) )* + + = + ( )? + + = + ( )* + + = + ( )* + + = + ( )* + + = + ( )* + + = + * + + = + ( | )* + + +/* --- Primary Expressions & Literals --- */ +/* The highest-precedence expressions, including literals and grouped expressions. */ + + = + + | + | + | + | + | + | + | + | + | + + = + "(" ")" + + = + "::" + + = + ( | | ) ".@" ? + + = + "(" ")" + + = + ( ",")* ? + + = + "{" ( | )? ","? "}" + + = + ( "," )* + + = + ":" + + = + ( "," )* + + = + "null" + + = + "true" + | "false" + + = /* Assumed to be defined by the tokenizer */ + = /* Assumed to be defined by the tokenizer */ + = /* Assumed to be defined by the tokenizer */ + + +/* --- Operators --- */ +/* Definitions for all operator token sets. */ + + = "=" + = "+=" | "-=" | "*=" | "/=" | "%=" | "&=" | "|=" | "<<=" | ">>=" | "||=" | "&&=" + = "==" | "!=" | ">" | "<" | ">=" | "<=" + = "&" | "^" | "|" + = "<<" | ">>" + = "+" | "-" + = "*" | "/" | "%" + = "!" | "-" | "~" | "&" | "*" + + = + "[" ? ":" ? "]" + | "[" "]" + | ".[" "]" + | "." + | "->" + | ".#" + | ".*" + +/* --- Type System --- */ +/* Rules for defining types, type names, and type qualifiers. */ + + = + + + = + ( "*" | "[]" ) ? + | "$" ? + | "?" ? + + = + ( "*" | "[]" ) ? + | "$" ? + + = + ( "*" | "[]" ) ? + | "?" ? + + = + ( "<" ">" )? + + = + + | "::" + + = + ( "," )* + + +/* --- Lexical Tokens & Base Definitions --- */ +/* The lowest-level building blocks of the language. */ + + = + + + = + + | + | + + = "_" | [a-z] | [A-Z] + = | + = "0" + = [1-9] + + = E /* Represents an empty terminal string */ + diff --git a/docs/readme.md b/docs/readme.md new file mode 100644 index 0000000..4b3b959 --- /dev/null +++ b/docs/readme.md @@ -0,0 +1,286 @@ +# **The `artichoke` Programming Language: A Technical Overview** + +## **1. Introduction** + +`artichoke` is a statically-typed, general-purpose programming language designed +with an emphasis on performance, safety, and expressive syntax. It combines +low-level control over memory with modern, high-level features like generics, +algebraic data types, and integrated error handling. This document provides an +overview of the language's features as defined by its core grammar. + +Is highly inspired by C, C++, Rust, and mostly Zig. + +## **2. Basic Syntax & Structure** + +### **Modules, Imports, and Aliases** + +`artichoke` code is organized into modules. The `import` statement is used to bring +symbols from other modules into the current scope. + +* **Importing a specific element:** `import my_module::some_function;` +* **Importing all direct elements of a module:** `import std::*;` +* **Importing an entire submodule:** `import std::memory;` + +The `using` keyword creates a local, more convenient alias for a type, function, +or module name. + +``` +using mem = std::memory; +using FileHandle = std::fs::File; +``` + +### **Comments** + +The language uses C-style block comments. + +``` +/* This is a multi-line + comment. */ +``` + +## **3. The Type System** + +`artichoke`'s type system is strong and static, with a rich set of features for +defining complex data structures. + +### **Type Qualifiers** + +Qualifiers modify the type to their immediate right, allowing for precise and +complex type definitions. + +* **`*` (Pointer):** Creates a pointer to a type. Pointers cannot be `null`. +* **`$` (Mutable):** Marks a type as mutable. This is used for function + parameters, local variables, and struct fields to allow modification. +* **`?` (Optional):** Marks a type as nullable. An optional type can hold either a + value of its underlying type or `null`. +* **`[]` (Slice):** A "fat pointer" representing a view into a contiguous + sequence of elements. It contains both a pointer to the data and a length. + +These qualifiers can be combined. For example, `*$?int` defines a **pointer to a +mutable optional integer**. + +### **Generics** + +Generics allow for writing flexible, reusable code that can operate on multiple +types. They are defined using ``. + +``` +/* A generic struct */ +struct Point { + x: T, + y: T +} + +/* A generic function */ +fn scale(lhs: *Point, rhs: T) -> Point { + /* ... */ +} +``` + + +## **4. Declarations** + +### **Variables** + +Variables are declared using the `let` (mutable) and `def` (immutable/constant) +keywords. + +* **Type inference** is supported when the type can be determined from the initializer. +* Variables must be initialized with either a type, a value, or both. + +``` +/* Mutable variable with explicit type */ +let x: i32 = 10; + +/* Immutable variable with type inference */ +def do_you_get_it = meaning_of_life(); +``` + +### **Structs** + +Structs are composite data types that group together variables under one name. +They support generics. + +``` +struct Rectangle { + top: Point, + bot: Point +} +``` + +**Initialization:** Structs can be initialized using positional or named fields, +but not a mix of both. + +``` +/* Positional initialization */ +def top_left = Point{ 0, 10 }; + +/* Named-field initialization */ +def top_right = Point{ x: 10, y: 10 }; +``` + +### **Enums (Tagged Unions)** + +Enums define a type that can be one of several different variants. Variants can +optionally hold data. + +``` +enum AssetType { + Texture, + Model, + Sound, +} + +enum Result { + Ok(T), + Err(E) +} +``` + +**Initialization:** Enum variants are accessed using scope resolution (`::`). + +``` +def my_asset = AssetType::Texture; +def success = Result::Ok(100); +``` + +### **Functions** + +Functions are defined with the fn keyword. The return type is specified after +the parameter list with `->`. + +``` +fn meaning_of_life() -> i32 { + return 42; +} +``` + +#### **Member Functions (`this` parameter)** + +If the first parameter of a function is declared with the `this` keyword, it can +be called using "member function" syntax. + +``` +/* Definition */ +fn add(this *$Point, other: *Point) { + this->x += other->x; + this->y += other->y; +} + +/* Can be called in two ways: */ +/* Member function syntax */ +my_point.add(&other_point); + +/* Normal function syntax */ +add(&my_point, &other_point); +``` + +## **5. Control Flow** + +### **`if`/`else` Statements** + +`artichoke` supports C-style `if`/`else` and `else if` chains. It also integrates a +powerful unwrapping feature for handling `Result` and optional (`?`) types. + +``` + +/* Standard if/else */ +if (argc < 2) { + return Result::Err(-1); +} + +/* Unwrapping a Result */ +if (foo()) |ok| { + /* `ok` holds the success value */ +} +else |err| { + /* `err` holds the error value */ +} +``` + +### **Loops** + +The language provides a comprehensive set of looping constructs. + +* **C-Style `for`:** `for (let i \= 0; i \< 10; i \+= 1\) { ... }` +* **Range-based `for`:** `for (let e := arrSlice) { ... }` +* **`while` Loop:** Can optionally have an `else` block that executes when the loop + condition is no longer met. +* **Iterator `while`:** Supports unwrapping `Result`/optional types, executing as + long as the value is valid. +* **`do-while` Loop:** Guarantees the body executes at least once. +* **Infinite `loop`:** `loop { ... }` + +#### **Loop Labels and Control** + +Loops can be labeled. The `break` and `continue` statements can optionally specify a +label to control nested loops. + +``` +outer_loop := while (condition) { + inner_loop := for (...) { + break outer_loop; + } +} +``` + +## **6. Expressions and Operators** + +### **Pointer and Member Access** + +* **`&` (Address-of):** Gets a pointer to a variable. +* **`*` (Dereference):** Accesses the value a pointer points to. +* **`.` (Member Access):** Accesses a member of a struct value. +* **`->` (Pointer Member Access):** Dereferences a pointer and accesses a member + (`p->x` is shorthand for `(*p).x`). + +### **Slice Operators** + +Slices have a dedicated set of operators for manipulation. + +* **`[start:end]` (Slicing):** Creates a new slice from an existing one. +* **`.*` (Pointer Access):** Gets the underlying raw pointer of the slice. +* **`.#` (Length Access):** Gets the number of elements in the slice. +* **`.[length]` (Slice from Pointer):** Creates a slice from a raw pointer and a length. + +### **Assignment** + +The language supports simple (`=`) and compound assignment (`+=`, `*=`, etc.) +operators. + +## **7. Advanced Features** + +### **Resource Management (`defer` and `errdefer`)** + +`artichoke` uses `defer` for deterministic resource management. + +* **`defer`:** Schedules an expression or code block to be executed when the + current scope is exited. Deferred calls are executed in Last-In, First-Out + (LIFO) order. +* **`errdefer`:** Similar to `defer`, but the code is only executed if the scope is + exited due to a function returning an error (an `Err` variant of a `Result`). + +``` +defer call_cleanup(); + +errdefer { + log("An error occurred!"); +} +``` + +### **Reflection (`.@`)** + +The language provides a compile-time reflection mechanism via the `.@` operator. +It can be applied to values, types, and static members to query metadata. + +* **On values:** `my_variable.@type` +* **On types:** `Point.@size, Point.@alignment` +* **On static members:** `Point::x.@offset` + +``` +/* Gets size in bytes */ +def size_bytes = Point.@size; + +/* Gets string representation of the type */ +def point_name = Point.@typename; +``` diff --git a/lib/include/artichoke/Tokenizer/Token.hpp b/lib/include/artichoke/Tokenizer/Token.hpp index e51f8c1..9a990c4 100644 --- a/lib/include/artichoke/Tokenizer/Token.hpp +++ b/lib/include/artichoke/Tokenizer/Token.hpp @@ -23,84 +23,89 @@ namespace arti::lang { tkCharacter, tkIdentifier, - opDot, - opMod, - opPlus, - opHyphen, - opSlash, - opBang, - opStar, - opColon, - opComma, - opAssign, - opAccess, - opSemicolon, + opDot, /* . */ + opMod, /* % */ + opPlus, /* + */ + opHyphen, /* - */ + opSlash, /* / */ + opBang, /* ! */ + opStar, /* * */ + opColon, /* : */ + opComma, /* , */ + opAssign, /* = */ + opAccess, /* :: */ + opSemicolon, /* ; */ + opCaret, /* ^ */ + opTilde, /* ~ */ + opEq, /* == */ + opNeq, /* != */ + opLt, /* < */ + opGt, /* > */ + opLtEq, /* <= */ + opGtEq, /* >= */ + opLShift, /* << */ + opRShift, /* >> */ + opBoolAnd, /* && */ + opBoolOr, /* || */ + opAnd, /* & */ + opOr, /* | */ + opLParen, /* ( */ + opRParen, /* ) */ + opLBracket, /* [ */ + opRBracket, /* ] */ + opLSquirly, /* { */ + opRSquirly, /* } */ + opArrow, /* -> */ + opPlusAssign, /* += */ + opHyphenAssign, /* -= */ + opStarAssign, /* *= */ + opSlashAssign, /* /= */ + opModAssign, /* %= */ + opAndAssign, /* &= */ + opOrAssign, /* |= */ + opLShiftAssign, /* <<= */ + opRShiftAssign, /* >>= */ + opBoolAndAssign, /* &&= */ + opBoolORAssign, /* ||= */ + opMut, /* $ */ + opOpt, /* ? */ + opSliceSize, /* .# */ + opPtrSlice, /* .[ */ + opSlicePtr, /* .* */ + opReflect, /* .@ */ + opLabel, /* := */ - opCaret, - opTilde, - - opEq, - opNeq, - - opLt, - opGt, - - opLtEq, - opGtEq, - - opLShift, - opRShift, - - opBoolAnd, - opBoolOr, - - opAnd, - opOr, - - opLParen, - opRParen, - - opLBracket, - opRBracket, - - opLSquirly, - opRSquirly, - - opArrow, - - kwOr, - kwNot, - kwAnd, - - kwIf, - kwElse, - - kwFn, - kwEnum, - kwStruct, - kwVariant, - - kwDef, - kwLet, - kwMut, - - kwFor, - kwWhile, - - kwReturn, - kwUnreachable, - - kwDefer, - kwErrDefer, - - kwTrue, - kwFalse, - - kwNull, - - kwImport, - kwExport, - kwModule, + /* Keywords */ + kwUnderscore, /* _ */ + kwOr, /* or */ + kwNot, /* not */ + kwAnd, /* and */ + kwIf, /* if */ + kwElse, /* else */ + kwFn, /* fn */ + kwEnum, /* enum */ + kwStruct, /* struct */ + kwDef, /* def */ + kwLet, /* let */ + kwFor, /* for */ + kwLoop, /* loop */ + kwBreak, /* break */ + kwContinue, /* continue */ + kwWhile, /* while */ + kwMatch, /* match */ + kwSwitch, /* switch */ + kwReturn, /* return */ + kwUnreachable, /* unreachable */ + kwDefer, /* defer */ + kwErrDefer, /* errdefer */ + kwTrue, /* true */ + kwFalse, /* false */ + kwNull, /* null */ + kwThis, /* this */ + kwImport, /* import */ + kwExport, /* export */ + kwModule, /* module */ + kwUsing, /* using */ }; std::string toString(const Token &value); diff --git a/lib/include/artichoke/Tokenizer/Tokenizer.hpp b/lib/include/artichoke/Tokenizer/Tokenizer.hpp index e64690d..3ea09a5 100644 --- a/lib/include/artichoke/Tokenizer/Tokenizer.hpp +++ b/lib/include/artichoke/Tokenizer/Tokenizer.hpp @@ -34,6 +34,7 @@ namespace arti::lang { Generator> tokenize(); void skip_whitespace(); + Expected skip_comment(); Expected readNumber(); Expected readString(); diff --git a/lib/include/artichoke/Util/Expected.hpp b/lib/include/artichoke/Util/Expected.hpp index cc6a146..2c2a20b 100644 --- a/lib/include/artichoke/Util/Expected.hpp +++ b/lib/include/artichoke/Util/Expected.hpp @@ -16,6 +16,7 @@ namespace arti::lang { ecInvalidLiteral, ecInvalidCharacter, ecInvalidIndex, + ecInvalidComment, }; struct Exception { @@ -67,6 +68,9 @@ namespace arti::lang { else if constexpr (code == ecInvalidIndex) { return "Invalid index"; } + else if constexpr (code == ecInvalidComment) { + return "Invalid comment found, missing '*/' end of comment"; + } else { return "Unknown error"; } diff --git a/lib/src/Tokenizer/Token.cpp b/lib/src/Tokenizer/Token.cpp index 78052af..ad7f47d 100644 --- a/lib/src/Tokenizer/Token.cpp +++ b/lib/src/Tokenizer/Token.cpp @@ -11,70 +11,94 @@ namespace arti::lang { std::string_view tokenStr; switch (value.value) { - case tkEOF: return "Token{{ tkEOF }}"; - case tkString: tokenStr = "tkString"; break; - case tkDecimal: tokenStr = "tkDecimal"; break; - case tkInteger: tokenStr = "tkInteger"; break; - case tkCharacter: tokenStr = "tkCharacter"; break; - case tkIdentifier: tokenStr = "tkIdentifier"; break; - case opDot: tokenStr = "opDot"; break; - case opMod: tokenStr = "opMod"; break; - case opPlus: tokenStr = "opPlus"; break; - case opHyphen: tokenStr = "opHyphen"; break; - case opSlash: tokenStr = "opSlash"; break; - case opBang: tokenStr = "opBang"; break; - case opStar: tokenStr = "opStar"; break; - case opColon: tokenStr = "opColon"; break; - case opComma: tokenStr = "opComma"; break; - case opAssign: tokenStr = "opAssign"; break; - case opAccess: tokenStr = "opAccess"; break; - case opSemicolon: tokenStr = "opSemicolon"; break; - case opCaret: tokenStr = "opCaret"; break; - case opTilde: tokenStr = "opTilde"; break; - case opEq: tokenStr = "opEq"; break; - case opNeq: tokenStr = "opNeq"; break; - case opLt: tokenStr = "opLt"; break; - case opGt: tokenStr = "opGt"; break; - case opLtEq: tokenStr = "opLtEq"; break; - case opGtEq: tokenStr = "opGtEq"; break; - case opLShift: tokenStr = "opLShift"; break; - case opRShift: tokenStr = "opRShift"; break; - case opBoolAnd: tokenStr = "opBoolAnd"; break; - case opBoolOr: tokenStr = "opBoolOr"; break; - case opAnd: tokenStr = "opAnd"; break; - case opOr: tokenStr = "opOr"; break; - case opLParen: tokenStr = "opLParen"; break; - case opRParen: tokenStr = "opRParen"; break; - case opLBracket: tokenStr = "opLBracket"; break; - case opRBracket: tokenStr = "opRBracket"; break; - case opLSquirly: tokenStr = "opLSquirly"; break; - case opRSquirly: tokenStr = "opRSquirly"; break; - case opArrow: tokenStr = "opArrow"; break; - case kwOr: tokenStr = "kwOr"; break; - case kwNot: tokenStr = "kwNot"; break; - case kwAnd: tokenStr = "kwAnd"; break; - case kwIf: tokenStr = "kwIf"; break; - case kwElse: tokenStr = "kwElse"; break; - case kwFn: tokenStr = "kwFn"; break; - case kwEnum: tokenStr = "kwEnum"; break; - case kwStruct: tokenStr = "kwStruct"; break; - case kwVariant: tokenStr = "kwVariant"; break; - case kwDef: tokenStr = "kwDef"; break; - case kwLet: tokenStr = "kwLet"; break; - case kwMut: tokenStr = "kwMut"; break; - case kwFor: tokenStr = "kwFor"; break; - case kwWhile: tokenStr = "kwWhile"; break; - case kwReturn: tokenStr = "kwReturn"; break; - case kwUnreachable: tokenStr = "kwUnreachable"; break; - case kwDefer: tokenStr = "kwDefer"; break; - case kwErrDefer: tokenStr = "kwErrDefer"; break; - case kwTrue: tokenStr = "kwTrue"; break; - case kwFalse: tokenStr = "kwFalse"; break; - case kwNull: tokenStr = "kwNull"; break; - case kwImport: tokenStr = "kwImport"; break; - case kwExport: tokenStr = "kwExport"; break; - case kwModule: tokenStr = "kwModule"; break; - default: tokenStr = ""; break; + case tkEOF: return "Token{ tkEOF }"; + case tkString: tokenStr = "tkString"; break; + case tkDecimal: tokenStr = "tkDecimal"; break; + case tkInteger: tokenStr = "tkInteger"; break; + case tkCharacter: tokenStr = "tkCharacter"; break; + case tkIdentifier: tokenStr = "tkIdentifier"; break; + case opDot: tokenStr = "opDot"; break; + case opMod: tokenStr = "opMod"; break; + case opPlus: tokenStr = "opPlus"; break; + case opHyphen: tokenStr = "opHyphen"; break; + case opSlash: tokenStr = "opSlash"; break; + case opBang: tokenStr = "opBang"; break; + case opStar: tokenStr = "opStar"; break; + case opColon: tokenStr = "opColon"; break; + case opComma: tokenStr = "opComma"; break; + case opAssign: tokenStr = "opAssign"; break; + case opAccess: tokenStr = "opAccess"; break; + case opSemicolon: tokenStr = "opSemicolon"; break; + case opCaret: tokenStr = "opCaret"; break; + case opTilde: tokenStr = "opTilde"; break; + case opEq: tokenStr = "opEq"; break; + case opNeq: tokenStr = "opNeq"; break; + case opLt: tokenStr = "opLt"; break; + case opGt: tokenStr = "opGt"; break; + case opLtEq: tokenStr = "opLtEq"; break; + case opGtEq: tokenStr = "opGtEq"; break; + case opLShift: tokenStr = "opLShift"; break; + case opRShift: tokenStr = "opRShift"; break; + case opBoolAnd: tokenStr = "opBoolAnd"; break; + case opBoolOr: tokenStr = "opBoolOr"; break; + case opAnd: tokenStr = "opAnd"; break; + case opOr: tokenStr = "opOr"; break; + case opLParen: tokenStr = "opLParen"; break; + case opRParen: tokenStr = "opRParen"; break; + case opLBracket: tokenStr = "opLBracket"; break; + case opRBracket: tokenStr = "opRBracket"; break; + case opLSquirly: tokenStr = "opLSquirly"; break; + case opRSquirly: tokenStr = "opRSquirly"; break; + case opArrow: tokenStr = "opArrow"; break; + case opPlusAssign: tokenStr = "opPlusAssign"; break; + case opHyphenAssign: tokenStr = "opHyphenAssign"; break; + case opStarAssign: tokenStr = "opStarAssign"; break; + case opSlashAssign: tokenStr = "opSlashAssign"; break; + case opModAssign: tokenStr = "opModAssign"; break; + case opAndAssign: tokenStr = "opAndAssign"; break; + case opOrAssign: tokenStr = "opOrAssign"; break; + case opLShiftAssign: tokenStr = "opLShiftAssign"; break; + case opRShiftAssign: tokenStr = "opRShiftAssign"; break; + case opBoolAndAssign: tokenStr = "opBoolAndAssign"; break; + case opBoolORAssign: tokenStr = "opBoolORAssign"; break; + case opMut: tokenStr = "opMut"; break; + case opOpt: tokenStr = "opOpt"; break; + case opSliceSize: tokenStr = "opSliceSize"; break; + case opPtrSlice: tokenStr = "opPtrSlice"; break; + case opSlicePtr: tokenStr = "opSlicePtr"; break; + case opReflect: tokenStr = "opReflect"; break; + case opLabel: tokenStr = "opLabel"; break; + case kwUnderscore: tokenStr = "kwUnderscore"; break; + case kwOr: tokenStr = "kwOr"; break; + case kwNot: tokenStr = "kwNot"; break; + case kwAnd: tokenStr = "kwAnd"; break; + case kwIf: tokenStr = "kwIf"; break; + case kwElse: tokenStr = "kwElse"; break; + case kwFn: tokenStr = "kwFn"; break; + case kwEnum: tokenStr = "kwEnum"; break; + case kwStruct: tokenStr = "kwStruct"; break; + case kwDef: tokenStr = "kwDef"; break; + case kwLet: tokenStr = "kwLet"; break; + case kwFor: tokenStr = "kwFor"; break; + case kwLoop: tokenStr = "kwLoop"; break; + case kwBreak: tokenStr = "kwBreak"; break; + case kwContinue: tokenStr = "kwContinue"; break; + case kwWhile: tokenStr = "kwWhile"; break; + case kwMatch: tokenStr = "kwMatch"; break; + case kwSwitch: tokenStr = "kwSwitch"; break; + case kwReturn: tokenStr = "kwReturn"; break; + case kwUnreachable: tokenStr = "kwUnreachable"; break; + case kwDefer: tokenStr = "kwDefer"; break; + case kwErrDefer: tokenStr = "kwErrDefer"; break; + case kwTrue: tokenStr = "kwTrue"; break; + case kwFalse: tokenStr = "kwFalse"; break; + case kwNull: tokenStr = "kwNull"; break; + case kwThis: tokenStr = "kwThis"; break; + case kwImport: tokenStr = "kwImport"; break; + case kwExport: tokenStr = "kwExport"; break; + case kwModule: tokenStr = "kwModule"; break; + case kwUsing: tokenStr = "kwUsing"; break; + default: tokenStr = ""; break; } return std::format("Token{{ {}, {} }}", tokenStr, value.strValue); diff --git a/lib/src/Tokenizer/Tokenizer.cpp b/lib/src/Tokenizer/Tokenizer.cpp index f20a25f..09f0fec 100644 --- a/lib/src/Tokenizer/Tokenizer.cpp +++ b/lib/src/Tokenizer/Tokenizer.cpp @@ -129,6 +129,17 @@ namespace arti::lang { else if (isFirstIdentChar(*iter)) { yield readIdentifier(); } + else if (*iter == '/') { + if ((iter + 1) != source.end() && *(iter + 1) == '*') { + if (auto ok = skip_comment(); !ok) { + auto err = ok.error(); + yield Unexpected<>{err}; + } + } + else { + yield readOperator(); + } + } else { yield readOperator(); } @@ -156,6 +167,37 @@ namespace arti::lang { } } + Expected Tokenizer::skip_comment() { + iter += 2; + column += 2; + + bool isEnd = false; + + while (iter != source.end()) { + if (*iter == '\n') { + column = 0; + line += 1; + } + else { + column += 1; + + if (*iter == '*') { + if ((iter + 1) == source.end()) { + return langException(line, column); + } + else if (*(iter + 1) == '/') { + iter += 2; + column += 2; + return {}; + } + } + } + + ++iter; + } + return langException(line, column); + } + Expected Tokenizer::readNumber() { auto stIter = iter; @@ -478,6 +520,9 @@ namespace arti::lang { { stIter, iter } }; + if (tok.strValue.compare("_") == 0) { + tok.value = TokenV::kwUnderscore; + } if (tok.strValue.compare("or") == 0) { tok.value = TokenV::kwOr; } @@ -502,24 +547,33 @@ namespace arti::lang { else if (tok.strValue.compare("struct") == 0) { tok.value = TokenV::kwStruct; } - else if (tok.strValue.compare("variant") == 0) { - tok.value = TokenV::kwVariant; - } else if (tok.strValue.compare("def") == 0) { tok.value = TokenV::kwDef; } else if (tok.strValue.compare("let") == 0) { tok.value = TokenV::kwLet; } - else if (tok.strValue.compare("mut") == 0) { - tok.value = TokenV::kwMut; - } else if (tok.strValue.compare("for") == 0) { tok.value = TokenV::kwFor; } + else if (tok.strValue.compare("loop") == 0) { + tok.value = TokenV::kwLoop; + } + else if (tok.strValue.compare("break") == 0) { + tok.value = TokenV::kwBreak; + } + else if (tok.strValue.compare("continue") == 0) { + tok.value = TokenV::kwContinue; + } else if (tok.strValue.compare("while") == 0) { tok.value = TokenV::kwWhile; } + else if (tok.strValue.compare("match") == 0) { + tok.value = TokenV::kwMatch; + } + else if (tok.strValue.compare("switch") == 0) { + tok.value = TokenV::kwSwitch; + } else if (tok.strValue.compare("return") == 0) { tok.value = TokenV::kwReturn; } @@ -541,6 +595,9 @@ namespace arti::lang { else if (tok.strValue.compare("null") == 0) { tok.value = TokenV::kwNull; } + else if (tok.strValue.compare("this") == 0) { + tok.value = TokenV::kwThis; + } else if (tok.strValue.compare("import") == 0) { tok.value = TokenV::kwImport; } @@ -550,6 +607,9 @@ namespace arti::lang { else if (tok.strValue.compare("module") == 0) { tok.value = TokenV::kwModule; } + else if (tok.strValue.compare("using") == 0) { + tok.value = TokenV::kwUsing; + } return tok; } @@ -654,6 +714,24 @@ namespace arti::lang { tm.insert("{", TokenV::opLSquirly); tm.insert("}", TokenV::opRSquirly); tm.insert("->", TokenV::opArrow); + tm.insert("+=", TokenV::opPlusAssign); + tm.insert("-=", TokenV::opHyphenAssign); + tm.insert("*=", TokenV::opStarAssign); + tm.insert("/=", TokenV::opSlashAssign); + tm.insert("%=", TokenV::opModAssign); + tm.insert("&=", TokenV::opAndAssign); + tm.insert("|=", TokenV::opOrAssign); + tm.insert("<<=", TokenV::opLShiftAssign); + tm.insert(">>=", TokenV::opRShiftAssign); + tm.insert("&&=", TokenV::opBoolAndAssign); + tm.insert("||=", TokenV::opBoolORAssign); + tm.insert("$", TokenV::opMut); + tm.insert("?", TokenV::opOpt); + tm.insert(".#", TokenV::opSliceSize); + tm.insert(".[", TokenV::opPtrSlice); + tm.insert(".*", TokenV::opSlicePtr); + tm.insert(".@", TokenV::opReflect); + tm.insert(":=", TokenV::opLabel); return tm; }