Class: Tokenizer

Hierarchy

  • Tokenizer

Constructors

constructor

+ new Tokenizer(input: string, trie: Trie, file: string): Tokenizer

Defined in src/parser/tokenizer.ts:32

Parameters:

Name Type Default
input string -
trie Trie defaultOptions.operatorsTrie
file string “”

Returns: Tokenizer

Properties

N

N: number

Defined in src/parser/tokenizer.ts:31


file

file: string

Defined in src/parser/tokenizer.ts:37


input

input: string

Defined in src/parser/tokenizer.ts:35


p

p: number = 0

Defined in src/parser/tokenizer.ts:30


Private rawBeginAt

rawBeginAt: number = -1

Defined in src/parser/tokenizer.ts:32


Private trie

trie: Trie

Defined in src/parser/tokenizer.ts:36

Methods

advance

advance(i: number): void

Defined in src/parser/tokenizer.ts:273

Parameters:

Name Type Default
i number 1

Returns: void


end

end(): boolean

Defined in src/parser/tokenizer.ts:277

Returns: boolean


match

match(word: string): boolean

Defined in src/parser/tokenizer.ts:374

Parameters:

Name Type
word string

Returns: boolean


mkError

mkError(msg: string, begin: number): TokenizationError‹›

Defined in src/parser/tokenizer.ts:213

Parameters:

Name Type
msg string
begin number

Returns: TokenizationError‹›


peek

peek(n: number): string

Defined in src/parser/tokenizer.ts:392

Parameters:

Name Type Default
n number 0

Returns: string


peekType

peekType(n: number): number

Defined in src/parser/tokenizer.ts:388

Parameters:

Name Type Default
n number 0

Returns: number


readEndrawOrRawContent

readEndrawOrRawContent(options: NormalizedFullOptions): HTMLToken | TagToken

Defined in src/parser/tokenizer.ts:169

Parameters:

Name Type
options NormalizedFullOptions

Returns: HTMLToken | TagToken


readExpression

readExpression(): Expression‹›

Defined in src/parser/tokenizer.ts:42

Returns: Expression‹›


readExpressionTokens

readExpressionTokens(): IterableIterator‹Token

Defined in src/parser/tokenizer.ts:46

Returns: IterableIterator‹Token


readFileNameTemplate

readFileNameTemplate(options: NormalizedFullOptions): IterableIterator‹TopLevelToken

Defined in src/parser/tokenizer.ts:362

Parameters:

Name Type
options NormalizedFullOptions

Returns: IterableIterator‹TopLevelToken


readFilter

readFilter(): FilterToken | null

Defined in src/parser/tokenizer.ts:77

Returns: FilterToken | null


readFilterArg

readFilterArg(): FilterArg | undefined

Defined in src/parser/tokenizer.ts:99

Returns: FilterArg | undefined


readFilters

readFilters(): FilterToken[]

Defined in src/parser/tokenizer.ts:69

Returns: FilterToken[]


readHTMLToken

readHTMLToken(stopStrings: string[]): HTMLToken

Defined in src/parser/tokenizer.ts:127

Parameters:

Name Type
stopStrings string[]

Returns: HTMLToken


readHash

readHash(jekyllStyle?: undefined | false | true): HashToken | undefined

Defined in src/parser/tokenizer.ts:252

Parameters:

Name Type
jekyllStyle? undefined | false | true

Returns: HashToken | undefined


readHashes

readHashes(jekyllStyle?: undefined | false | true): HashToken‹›[]

Defined in src/parser/tokenizer.ts:243

Parameters:

Name Type
jekyllStyle? undefined | false | true

Returns: HashToken‹›[]


readIdentifier

readIdentifier(): IdentifierToken

Defined in src/parser/tokenizer.ts:229

Returns: IdentifierToken


readLiquidTagToken

readLiquidTagToken(options: NormalizedFullOptions): LiquidTagToken

Defined in src/parser/tokenizer.ts:205

Parameters:

Name Type
options NormalizedFullOptions

Returns: LiquidTagToken


readLiquidTagTokens

readLiquidTagTokens(options: NormalizedFullOptions): LiquidTagToken[]

Defined in src/parser/tokenizer.ts:196

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: LiquidTagToken[]


readOperator

readOperator(): OperatorToken | undefined

Defined in src/parser/tokenizer.ts:63

Returns: OperatorToken | undefined


readOutputToken

readOutputToken(options: NormalizedFullOptions): OutputToken

Defined in src/parser/tokenizer.ts:159

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: OutputToken


readQuoted

readQuoted(): QuotedToken | undefined

Defined in src/parser/tokenizer.ts:347

Returns: QuotedToken | undefined


readRange

readRange(): RangeToken | undefined

Defined in src/parser/tokenizer.ts:329

Returns: RangeToken | undefined


readTagName

readTagName(): string

Defined in src/parser/tokenizer.ts:236

Returns: string


readTagToken

readTagToken(options: NormalizedFullOptions): TagToken

Defined in src/parser/tokenizer.ts:136

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: TagToken


readTo

readTo(end: string): number

Defined in src/parser/tokenizer.ts:281

Parameters:

Name Type
end string

Returns: number


readToDelimiter

readToDelimiter(delimiter: string): number

Defined in src/parser/tokenizer.ts:147

Parameters:

Name Type
delimiter string

Returns: number


readTopLevelToken

readTopLevelToken(options: NormalizedFullOptions): TopLevelToken

Defined in src/parser/tokenizer.ts:119

Parameters:

Name Type
options NormalizedFullOptions

Returns: TopLevelToken


readTopLevelTokens

readTopLevelTokens(options: NormalizedFullOptions): TopLevelToken[]

Defined in src/parser/tokenizer.ts:109

Parameters:

Name Type Default
options NormalizedFullOptions defaultOptions

Returns: TopLevelToken[]


readValue

readValue(): ValueToken | undefined

Defined in src/parser/tokenizer.ts:289

Returns: ValueToken | undefined


readValueOrThrow

readValueOrThrow(): ValueToken

Defined in src/parser/tokenizer.ts:341

Returns: ValueToken


readWord

readWord(): IdentifierToken‹›

Defined in src/parser/tokenizer.ts:224

deprecated

Returns: IdentifierToken‹›


remaining

remaining(): string

Defined in src/parser/tokenizer.ts:269

Returns: string


rmatch

rmatch(pattern: string): boolean

Defined in src/parser/tokenizer.ts:381

Parameters:

Name Type
pattern string

Returns: boolean


skipBlank

skipBlank(): void

Defined in src/parser/tokenizer.ts:396

Returns: void


snapshot

snapshot(begin: number): string

Defined in src/parser/tokenizer.ts:217

Parameters:

Name Type Default
begin number this.p

Returns: string