Class Token
Represents a lexical token of the language. This is an abstract class that particular language implementations must extend
to define language-specific token types
Inheritance
System.Object
Token
Inherited Members
System.Object.Equals(System.Object)
System.Object.Equals(System.Object, System.Object)
System.Object.GetHashCode()
System.Object.GetType()
System.Object.MemberwiseClone()
System.Object.ReferenceEquals(System.Object, System.Object)
Assembly: Azos.dll
Syntax
public abstract class Token
Constructors
Token(ILexer, SourcePosition, SourcePosition, String)
Declaration
public Token(ILexer lexer, SourcePosition startPos, SourcePosition endPos, string text)
Parameters
Token(ILexer, SourcePosition, SourcePosition, String, Object)
Declaration
public Token(ILexer lexer, SourcePosition startPos, SourcePosition endPos, string text, object value)
Parameters
Fields
EndPosition
Declaration
public readonly SourcePosition EndPosition
Field Value
Lexer
Declaration
public readonly ILexer Lexer
Field Value
StartPosition
Declaration
public readonly SourcePosition StartPosition
Field Value
Text
Declaration
public readonly string Text
Field Value
| Type |
Description |
| System.String |
|
Value
Declaration
public readonly object Value
Field Value
| Type |
Description |
| System.Object |
|
Properties
IsBOF
Indicates whether this token indicates an BEGINNING-OF-FILE condition
Declaration
public bool IsBOF { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
Indicates whether this token represents a comment
Declaration
public bool IsComment { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsDirective
Indicates whether this token represents a directive
Declaration
public bool IsDirective { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsEOF
Indicates whether this token indicates an END-OF-FILE condition
Declaration
public bool IsEOF { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsIdentifier
Indicates whether this token represents an identifier
Declaration
public bool IsIdentifier { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsKeyword
Indicates whether this token represents a language keyword
Declaration
public bool IsKeyword { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsLiteral
Indicates whether this token represents a literal
Declaration
public bool IsLiteral { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsNonLanguage
This property is needed due to the fact that language lexers may be used to analyze special supersets of regular
language grammars i.e. for pattern matches, template parser etc. Code compilers may elect to throw errors when this property is true.
Returns true for tokens that are not part of the valid language grammar, however they exist for other reasons,
for example - for pattern capture match analysis, or for template processing
Declaration
public abstract bool IsNonLanguage { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsNumericLiteral
Indicates whether this token represents a literal, which is a numeric literal (i.e. int, double)
This flag is useful for pattern searches that look for particular constant values in numbers (i.e. year = 2000)
Declaration
public abstract bool IsNumericLiteral { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsOperator
Indicates whether this token represents an operator
Declaration
public bool IsOperator { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsPrimary
Returns true for tokens that are primary part of the language, not control, metadata, directive, comment etc...
For example BOF,EOF markers are not primary part of the language, compiler directives, comments are not either.
This property is useful for pattern searches, when comments and other non-primary tokens need to be quickly skipped
Declaration
public abstract bool IsPrimary { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsSymbol
Indicates whether this token represents a symbol
Declaration
public bool IsSymbol { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
IsTextualLiteral
Indicates whether this token represents a literal, which is a string or a character sequence.
This flag is useful for pattern searches that examine comments and strings for sub-patterns
Declaration
public abstract bool IsTextualLiteral { get; }
Property Value
| Type |
Description |
| System.Boolean |
|
Kind
Provides language-agnostic classification for token type
Declaration
public abstract TokenKind Kind { get; }
Property Value
Language
Returns language that this token is a part of
Declaration
public abstract Language Language { get; }
Property Value
OrdinalType
Returns token type as a grammar-agnostic ordinal
Declaration
public abstract int OrdinalType { get; }
Property Value
| Type |
Description |
| System.Int32 |
|
Methods
ToString()
Declaration
public override string ToString()
Returns
| Type |
Description |
| System.String |
|
Overrides
System.Object.ToString()
Extension Methods