Class Tokenizer

Constructors

constructor

new Tokenizer(input, operators?, file?, range?, groupedExpressions?): Tokenizer
Parameters
- input: string
- operators: Operators = defaultOptions.operators
- Optionalfile: string
- Optionalrange: [number, number]
- groupedExpressions: boolean = false
Returns Tokenizer
- Defined in src/parser/tokenizer.ts:17

Properties

N

N: number

`Optional`file

file?: string

groupedExpressions

groupedExpressions: boolean

input

input: string

p

p: number

Methods

advance

advance(step?): void
Parameters
- step: number = 1
Returns void
- Defined in src/parser/tokenizer.ts:296

assert

assert(pred, msg, pos?): void
Parameters
- pred: unknown
- msg: string | (() => string)
- Optionalpos: number
Returns void
- Defined in src/parser/tokenizer.ts:232

end

end(): boolean
Returns boolean
- Defined in src/parser/tokenizer.ts:300

error

error(msg, pos?): TokenizationError
Parameters
- msg: string
- pos: number = ...
Returns TokenizationError
- Defined in src/parser/tokenizer.ts:228

match

match(word): boolean
Parameters
- word: string
Returns boolean
- Defined in src/parser/tokenizer.ts:456

matchTrie

matchTrie<T>(trie): number
Type Parameters
- T
Parameters
- trie: Trie<T>
Returns number
- Defined in src/parser/tokenizer.ts:56

peek

peek(n?): string
Parameters
- n: number = 0
Returns string
- Defined in src/parser/tokenizer.ts:474

peekType

peekType(n?): number
Parameters
- n: number = 0
Returns number
- Defined in src/parser/tokenizer.ts:470

read

read(): string
Returns string
- Defined in src/parser/tokenizer.ts:303

readEndrawOrRawContent

readEndrawOrRawContent(options): TagToken | HTMLToken
Parameters
- options: NormalizedFullOptions
Returns TagToken | HTMLToken
- Defined in src/parser/tokenizer.ts:182

readExpression

readExpression(): Expression
Returns Expression
- Defined in src/parser/tokenizer.ts:31

readExpressionTokens

readExpressionTokens(): IterableIterator<Token>
Returns IterableIterator<Token>
- Defined in src/parser/tokenizer.ts:35

readFileNameTemplate

readFileNameTemplate(options): IterableIterator<TopLevelToken>
Parameters
- options: NormalizedFullOptions
Returns IterableIterator<TopLevelToken>
- Defined in src/parser/tokenizer.ts:444

readFilter

readFilter(): null | FilterToken
Returns null | FilterToken
- Defined in src/parser/tokenizer.ts:83

readFilterArg

readFilterArg(): undefined | FilterArg
Returns undefined | FilterArg
- Defined in src/parser/tokenizer.ts:111

readFilteredValue

readFilteredValue(): FilteredValueToken
Returns FilteredValueToken
- Defined in src/parser/tokenizer.ts:68

readFilters

readFilters(): FilterToken[]
Returns FilterToken[]
- Defined in src/parser/tokenizer.ts:75

readGroupOrRange

readGroupOrRange(): undefined | FilteredValueToken | RangeToken
Returns undefined | FilteredValueToken | RangeToken
- Defined in src/parser/tokenizer.ts:395

readHTMLToken

readHTMLToken(stopStrings): HTMLToken
Parameters
- stopStrings: string[]
Returns HTMLToken
- Defined in src/parser/tokenizer.ts:139

readHash

readHash(jekyllStyle?): undefined | HashToken
Parameters
- OptionaljekyllStyle: string | boolean
Returns undefined | HashToken
- Defined in src/parser/tokenizer.ts:275

readHashes

readHashes(jekyllStyle?): HashToken[]
Parameters
- OptionaljekyllStyle: string | boolean
Returns HashToken[]
- Defined in src/parser/tokenizer.ts:266

readIdentifier

readIdentifier(): IdentifierToken
Returns IdentifierToken
- Defined in src/parser/tokenizer.ts:247

readLiquidTagToken

readLiquidTagToken(options): undefined | LiquidTagToken
Parameters
- options: NormalizedFullOptions
Returns undefined | LiquidTagToken
- Defined in src/parser/tokenizer.ts:218

readLiquidTagTokens

readLiquidTagTokens(options?): LiquidTagToken[]
Parameters
- options: NormalizedFullOptions = defaultOptions
Returns LiquidTagToken[]
- Defined in src/parser/tokenizer.ts:209

readLiteral

readLiteral(): undefined | LiteralToken
Returns undefined | LiteralToken
- Defined in src/parser/tokenizer.ts:386

readNonEmptyIdentifier

readNonEmptyIdentifier(): undefined | IdentifierToken
Returns undefined | IdentifierToken
- Defined in src/parser/tokenizer.ts:254

readNumber

readNumber(): undefined | NumberToken
Returns undefined | NumberToken
- Defined in src/parser/tokenizer.ts:363

readOperator

readOperator(): undefined | OperatorToken
Returns undefined | OperatorToken
- Defined in src/parser/tokenizer.ts:50

readOutputToken

readOutputToken(options?): OutputToken
Parameters
- options: NormalizedFullOptions = defaultOptions
Returns OutputToken
- Defined in src/parser/tokenizer.ts:172

readQuoted

readQuoted(): undefined | QuotedToken
Returns undefined | QuotedToken
- Defined in src/parser/tokenizer.ts:429

readScopeValue

readScopeValue(): undefined | ValueToken
Returns undefined | ValueToken
- Defined in src/parser/tokenizer.ts:326

readTagName

readTagName(): string
Returns string
- Defined in src/parser/tokenizer.ts:259

readTagToken

readTagToken(options): TagToken
Parameters
- options: NormalizedFullOptions
Returns TagToken
- Defined in src/parser/tokenizer.ts:148

readTo

readTo(end): number
Parameters
- end: string
Returns number
- Defined in src/parser/tokenizer.ts:306

readToDelimiter

readToDelimiter(delimiter, respectQuoted?): number
Parameters
- delimiter: string
- respectQuoted: boolean = false
Returns number
- Defined in src/parser/tokenizer.ts:159

readTopLevelToken

readTopLevelToken(options): TopLevelToken
Parameters
- options: NormalizedFullOptions
Returns TopLevelToken
- Defined in src/parser/tokenizer.ts:131

readTopLevelTokens

readTopLevelTokens(options?): TopLevelToken[]
Parameters
- options: NormalizedFullOptions = defaultOptions
Returns TopLevelToken[]
- Defined in src/parser/tokenizer.ts:121

readValue

readValue(): undefined | ValueToken
Returns undefined | ValueToken
- Defined in src/parser/tokenizer.ts:314

readValueOrThrow

readValueOrThrow(): ValueToken
Returns ValueToken
- Defined in src/parser/tokenizer.ts:423

readWord

readWord(): IdentifierToken
Returns IdentifierToken
Deprecated
use #readIdentifier instead
- Defined in src/parser/tokenizer.ts:243

remaining

remaining(): string
Returns string
- Defined in src/parser/tokenizer.ts:292

rmatch

rmatch(pattern): boolean
Parameters
- pattern: string
Returns boolean
- Defined in src/parser/tokenizer.ts:463

skipBlank

skipBlank(): void
Returns void
- Defined in src/parser/tokenizer.ts:478

snapshot

snapshot(begin?): string
Parameters
- begin: number = ...
Returns string
- Defined in src/parser/tokenizer.ts:236