Class Tokenizer

Constructors

constructor

new Tokenizer(input, operators?, file?, range?): Tokenizer
Parameters
- input: string
- operators: Operators = defaultOptions.operators
- Optionalfile: string
- Optionalrange: [number, number]
Returns Tokenizer
- Defined in src/parser/tokenizer.ts:16

Properties

N

N: number

`Optional`file

file?: string

input

input: string

p

p: number

Methods

advance

advance(step?): void
Parameters
- step: number = 1
Returns void
- Defined in src/parser/tokenizer.ts:292

assert

assert(pred, msg, pos?): void
Parameters
- pred: unknown
- msg: string | (() => string)
- Optionalpos: number
Returns void
- Defined in src/parser/tokenizer.ts:228

end

end(): boolean
Returns boolean
- Defined in src/parser/tokenizer.ts:296

error

error(msg, pos?): TokenizationError
Parameters
- msg: string
- pos: number = ...
Returns TokenizationError
- Defined in src/parser/tokenizer.ts:224

match

match(word): boolean
Parameters
- word: string
Returns boolean
- Defined in src/parser/tokenizer.ts:435

matchTrie

matchTrie<T>(trie): number
Type Parameters
- T
Parameters
- trie: Trie<T>
Returns number
- Defined in src/parser/tokenizer.ts:53

peek

peek(n?): string
Parameters
- n: number = 0
Returns string
- Defined in src/parser/tokenizer.ts:453

peekType

peekType(n?): number
Parameters
- n: number = 0
Returns number
- Defined in src/parser/tokenizer.ts:449

read

read(): string
Returns string
- Defined in src/parser/tokenizer.ts:299

readEndrawOrRawContent

readEndrawOrRawContent(options): TagToken | HTMLToken
Parameters
- options: NormalizedFullOptions
Returns TagToken | HTMLToken
- Defined in src/parser/tokenizer.ts:178

readExpression

readExpression(): Expression
Returns Expression
- Defined in src/parser/tokenizer.ts:28

readExpressionTokens

readExpressionTokens(): IterableIterator<Token>
Returns IterableIterator<Token>
- Defined in src/parser/tokenizer.ts:32

readFileNameTemplate

readFileNameTemplate(options): IterableIterator<TopLevelToken>
Parameters
- options: NormalizedFullOptions
Returns IterableIterator<TopLevelToken>
- Defined in src/parser/tokenizer.ts:423

readFilter

readFilter(): null | FilterToken
Returns null | FilterToken
- Defined in src/parser/tokenizer.ts:80

readFilterArg

readFilterArg(): undefined | FilterArg
Returns undefined | FilterArg
- Defined in src/parser/tokenizer.ts:107

readFilteredValue

readFilteredValue(): FilteredValueToken
Returns FilteredValueToken
- Defined in src/parser/tokenizer.ts:65

readFilters

readFilters(): FilterToken[]
Returns FilterToken[]
- Defined in src/parser/tokenizer.ts:72

readHTMLToken

readHTMLToken(stopStrings): HTMLToken
Parameters
- stopStrings: string[]
Returns HTMLToken
- Defined in src/parser/tokenizer.ts:135

readHash

readHash(jekyllStyle?): undefined | HashToken
Parameters
- OptionaljekyllStyle: string | boolean
Returns undefined | HashToken
- Defined in src/parser/tokenizer.ts:271

readHashes

readHashes(jekyllStyle?): HashToken[]
Parameters
- OptionaljekyllStyle: string | boolean
Returns HashToken[]
- Defined in src/parser/tokenizer.ts:262

readIdentifier

readIdentifier(): IdentifierToken
Returns IdentifierToken
- Defined in src/parser/tokenizer.ts:243

readLiquidTagToken

readLiquidTagToken(options): undefined | LiquidTagToken
Parameters
- options: NormalizedFullOptions
Returns undefined | LiquidTagToken
- Defined in src/parser/tokenizer.ts:214

readLiquidTagTokens

readLiquidTagTokens(options?): LiquidTagToken[]
Parameters
- options: NormalizedFullOptions = defaultOptions
Returns LiquidTagToken[]
- Defined in src/parser/tokenizer.ts:205

readLiteral

readLiteral(): undefined | LiteralToken
Returns undefined | LiteralToken
- Defined in src/parser/tokenizer.ts:379

readNonEmptyIdentifier

readNonEmptyIdentifier(): undefined | IdentifierToken
Returns undefined | IdentifierToken
- Defined in src/parser/tokenizer.ts:250

readNumber

readNumber(): undefined | NumberToken
Returns undefined | NumberToken
- Defined in src/parser/tokenizer.ts:356

readOperator

readOperator(): undefined | OperatorToken
Returns undefined | OperatorToken
- Defined in src/parser/tokenizer.ts:47

readOutputToken

readOutputToken(options?): OutputToken
Parameters
- options: NormalizedFullOptions = defaultOptions
Returns OutputToken
- Defined in src/parser/tokenizer.ts:168

readQuoted

readQuoted(): undefined | QuotedToken
Returns undefined | QuotedToken
- Defined in src/parser/tokenizer.ts:408

readRange

readRange(): undefined | RangeToken
Returns undefined | RangeToken
- Defined in src/parser/tokenizer.ts:388

readScopeValue

readScopeValue(): undefined | ValueToken
Returns undefined | ValueToken
- Defined in src/parser/tokenizer.ts:319

readTagName

readTagName(): string
Returns string
- Defined in src/parser/tokenizer.ts:255

readTagToken

readTagToken(options): TagToken
Parameters
- options: NormalizedFullOptions
Returns TagToken
- Defined in src/parser/tokenizer.ts:144

readTo

readTo(end): number
Parameters
- end: string
Returns number
- Defined in src/parser/tokenizer.ts:302

readToDelimiter

readToDelimiter(delimiter, respectQuoted?): number
Parameters
- delimiter: string
- respectQuoted: boolean = false
Returns number
- Defined in src/parser/tokenizer.ts:155

readTopLevelToken

readTopLevelToken(options): TopLevelToken
Parameters
- options: NormalizedFullOptions
Returns TopLevelToken
- Defined in src/parser/tokenizer.ts:127

readTopLevelTokens

readTopLevelTokens(options?): TopLevelToken[]
Parameters
- options: NormalizedFullOptions = defaultOptions
Returns TopLevelToken[]
- Defined in src/parser/tokenizer.ts:117

readValue

readValue(): undefined | ValueToken
Returns undefined | ValueToken
- Defined in src/parser/tokenizer.ts:310

readValueOrThrow

readValueOrThrow(): ValueToken
Returns ValueToken
- Defined in src/parser/tokenizer.ts:402

readWord

readWord(): IdentifierToken
Returns IdentifierToken
Deprecated
use #readIdentifier instead
- Defined in src/parser/tokenizer.ts:239

remaining

remaining(): string
Returns string
- Defined in src/parser/tokenizer.ts:288

rmatch

rmatch(pattern): boolean
Parameters
- pattern: string
Returns boolean
- Defined in src/parser/tokenizer.ts:442

skipBlank

skipBlank(): void
Returns void
- Defined in src/parser/tokenizer.ts:457

snapshot

snapshot(begin?): string
Parameters
- begin: number = ...
Returns string
- Defined in src/parser/tokenizer.ts:232