Hierarchy
- Tokenizer
Constructors
constructor
+ new Tokenizer(input: string, trie: Trie, file: string): Tokenizer
Defined in parser/tokenizer.ts:31
Parameters:
| Name | Type | Default |
|---|---|---|
input |
string | - |
trie |
Trie | - |
file |
string | “” |
Returns: Tokenizer
Properties
N
• N: number
Defined in parser/tokenizer.ts:30
Private file
• file: string
Defined in parser/tokenizer.ts:36
Private input
• input: string
Defined in parser/tokenizer.ts:34
p
• p: number = 0
Defined in parser/tokenizer.ts:29
Private rawBeginAt
• rawBeginAt: number = -1
Defined in parser/tokenizer.ts:31
Private trie
• trie: Trie
Defined in parser/tokenizer.ts:35
Methods
advance
▸ advance(i: number): void
Defined in parser/tokenizer.ts:248
Parameters:
| Name | Type | Default |
|---|---|---|
i |
number | 1 |
Returns: void
end
▸ end(): boolean
Defined in parser/tokenizer.ts:252
Returns: boolean
match
▸ match(word: string): boolean
Defined in parser/tokenizer.ts:343
Parameters:
| Name | Type |
|---|---|
word |
string |
Returns: boolean
mkError
▸ mkError(msg: string, begin: number): TokenizationError
Defined in parser/tokenizer.ts:196
Parameters:
| Name | Type |
|---|---|
msg |
string |
begin |
number |
Returns: TokenizationError
peek
▸ peek(n: number): string
Defined in parser/tokenizer.ts:361
Parameters:
| Name | Type | Default |
|---|---|---|
n |
number | 0 |
Returns: string
peekType
▸ peekType(n: number): number
Defined in parser/tokenizer.ts:357
Parameters:
| Name | Type | Default |
|---|---|---|
n |
number | 0 |
Returns: number
readEndrawOrRawContent
▸ readEndrawOrRawContent(options: NormalizedFullOptions): HTMLToken | TagToken
Defined in parser/tokenizer.ts:169
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
readExpression
▸ readExpression(): Expression
Defined in parser/tokenizer.ts:41
Returns: Expression
readExpressionTokens
▸ readExpressionTokens(): IterableIterator‹Token›
Defined in parser/tokenizer.ts:45
Returns: IterableIterator‹Token›
readFileName
▸ readFileName(): IdentifierToken
Defined in parser/tokenizer.ts:337
Returns: IdentifierToken
readFilter
▸ readFilter(): FilterToken | null
Defined in parser/tokenizer.ts:76
Returns: FilterToken | null
readFilterArg
▸ readFilterArg(): FilterArg | undefined
Defined in parser/tokenizer.ts:97
Returns: FilterArg | undefined
readFilters
▸ readFilters(): FilterToken[]
Defined in parser/tokenizer.ts:68
Returns: FilterToken[]
readHTMLToken
▸ readHTMLToken(options: NormalizedFullOptions): HTMLToken
Defined in parser/tokenizer.ts:125
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: HTMLToken
readHash
▸ readHash(): HashToken | undefined
Defined in parser/tokenizer.ts:228
Returns: HashToken | undefined
readHashes
▸ readHashes(): HashToken[]
Defined in parser/tokenizer.ts:219
Returns: HashToken[]
readIdentifier
▸ readIdentifier(): IdentifierToken
Defined in parser/tokenizer.ts:212
Returns: IdentifierToken
readOperator
▸ readOperator(): OperatorToken | undefined
Defined in parser/tokenizer.ts:62
Returns: OperatorToken | undefined
readOutputToken
▸ readOutputToken(options: NormalizedFullOptions): OutputToken
Defined in parser/tokenizer.ts:159
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: OutputToken
readQuoted
▸ readQuoted(): QuotedToken | undefined
Defined in parser/tokenizer.ts:322
Returns: QuotedToken | undefined
readRange
▸ readRange(): RangeToken | undefined
Defined in parser/tokenizer.ts:304
Returns: RangeToken | undefined
readTagToken
▸ readTagToken(options: NormalizedFullOptions): TagToken
Defined in parser/tokenizer.ts:136
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: TagToken
readTo
▸ readTo(end: string): number
Defined in parser/tokenizer.ts:256
Parameters:
| Name | Type |
|---|---|
end |
string |
Returns: number
readToDelimiter
▸ readToDelimiter(delimiter: string): number
Defined in parser/tokenizer.ts:147
Parameters:
| Name | Type |
|---|---|
delimiter |
string |
Returns: number
readTopLevelToken
▸ readTopLevelToken(options: NormalizedFullOptions): TopLevelToken
Defined in parser/tokenizer.ts:117
Parameters:
| Name | Type |
|---|---|
options |
NormalizedFullOptions |
Returns: TopLevelToken
readTopLevelTokens
▸ readTopLevelTokens(options: NormalizedFullOptions): TopLevelToken[]
Defined in parser/tokenizer.ts:107
Parameters:
| Name | Type | Default |
|---|---|---|
options |
NormalizedFullOptions | defaultOptions |
Returns: TopLevelToken[]
readValue
▸ readValue(): ValueToken | undefined
Defined in parser/tokenizer.ts:264
Returns: ValueToken | undefined
readValueOrThrow
▸ readValueOrThrow(): ValueToken
Defined in parser/tokenizer.ts:316
Returns: ValueToken
readWord
▸ readWord(): IdentifierToken
Defined in parser/tokenizer.ts:207
deprecated
Returns: IdentifierToken
remaining
▸ remaining(): string
Defined in parser/tokenizer.ts:244
Returns: string
rmatch
▸ rmatch(pattern: string): boolean
Defined in parser/tokenizer.ts:350
Parameters:
| Name | Type |
|---|---|
pattern |
string |
Returns: boolean
skipBlank
▸ skipBlank(): void
Defined in parser/tokenizer.ts:365
Returns: void
snapshot
▸ snapshot(begin: number): string
Defined in parser/tokenizer.ts:200
Parameters:
| Name | Type | Default |
|---|---|---|
begin |
number | this.p |
Returns: string