Skip to content

Commit 660d9be

Browse files
harttleharttle
authored
feat: more flexible squared property read expression, fixes #643 (#646)
* fix: more flexible squared property read expression, fixes #643 * fix: unecessary error wrapping in browser bundles * style: update code style and types * perf: use token.value when evalToken
1 parent dc6a301 commit 660d9be

22 files changed

+261
-167
lines changed

‎src/parser/index.ts

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
export * from './tokenizer'
22
export * from './parser'
33
export * from './parse-stream'
4-
export * from './parse-string-literal'
54
export * from './token-kind'

‎src/parser/match-operator.spec.ts

-28
This file was deleted.

‎src/parser/match-operator.ts

-14
This file was deleted.

‎src/parser/parser.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ export class Parser {
4848
}
4949
return new HTML(token)
5050
} catch (e) {
51-
if (e instanceof LiquidError) throw e
51+
if (LiquidError.is(e)) throw e
5252
throw new ParseError(e as Error, token)
5353
}
5454
}

‎src/parser/tokenizer.spec.ts

+67-26
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { LiquidTagToken, HTMLToken, QuotedToken, OutputToken, TagToken, OperatorToken, RangeToken, PropertyAccessToken, NumberToken, IdentifierToken } from '../tokens'
22
import { Tokenizer } from './tokenizer'
3+
import { defaultOperators } from '../render/operator'
4+
import { createTrie } from '../util/operator-trie'
35

46
describe('Tokenizer', function () {
57
it('should read quoted', () => {
@@ -15,12 +17,31 @@ describe('Tokenizer', function () {
1517
// eslint-disable-next-line deprecation/deprecation
1618
expect(new Tokenizer('foo bar').readWord()).toHaveProperty('content', 'foo')
1719
})
18-
it('should read number value', () => {
19-
const token: NumberToken = new Tokenizer('2.33.2').readValueOrThrow() as any
20+
it('should read integer number', () => {
21+
const token: NumberToken = new Tokenizer('123').readValueOrThrow() as any
2022
expect(token).toBeInstanceOf(NumberToken)
21-
expect(token.whole.getText()).toBe('2')
22-
expect(token.decimal!.getText()).toBe('33')
23-
expect(token.getText()).toBe('2.33')
23+
expect(token.getText()).toBe('123')
24+
expect(token.content).toBe(123)
25+
})
26+
it('should read negative number', () => {
27+
const token: NumberToken = new Tokenizer('-123').readValueOrThrow() as any
28+
expect(token).toBeInstanceOf(NumberToken)
29+
expect(token.getText()).toBe('-123')
30+
expect(token.content).toBe(-123)
31+
})
32+
it('should read float number', () => {
33+
const token: NumberToken = new Tokenizer('1.23').readValueOrThrow() as any
34+
expect(token).toBeInstanceOf(NumberToken)
35+
expect(token.getText()).toBe('1.23')
36+
expect(token.content).toBe(1.23)
37+
})
38+
it('should treat 1.2.3 as property read', () => {
39+
const token: PropertyAccessToken = new Tokenizer('1.2.3').readValueOrThrow() as any
40+
expect(token).toBeInstanceOf(PropertyAccessToken)
41+
expect(token.props).toHaveLength(3)
42+
expect(token.props[0].getText()).toBe('1')
43+
expect(token.props[1].getText()).toBe('2')
44+
expect(token.props[2].getText()).toBe('3')
2445
})
2546
it('should read quoted value', () => {
2647
const value = new Tokenizer('"foo"a').readValue()
@@ -33,11 +54,7 @@ describe('Tokenizer', function () {
3354
it('should read quoted property access value', () => {
3455
const value = new Tokenizer('["a prop"]').readValue()
3556
expect(value).toBeInstanceOf(PropertyAccessToken)
36-
expect((value as PropertyAccessToken).variable.getText()).toBe('"a prop"')
37-
})
38-
it('should throw for broken quoted property access', () => {
39-
const tokenizer = new Tokenizer('[5]')
40-
expect(() => tokenizer.readValueOrThrow()).toThrow()
57+
expect((value as QuotedToken).getText()).toBe('["a prop"]')
4158
})
4259
it('should throw for incomplete quoted property access', () => {
4360
const tokenizer = new Tokenizer('["a prop"')
@@ -277,10 +294,10 @@ describe('Tokenizer', function () {
277294

278295
const pa: PropertyAccessToken = token!.args[0] as any
279296
expect(token!.args[0]).toBeInstanceOf(PropertyAccessToken)
280-
expect((pa.variable as any).content).toBe('arr')
281-
expect(pa.props).toHaveLength(1)
282-
expect(pa.props[0]).toBeInstanceOf(NumberToken)
283-
expect(pa.props[0].getText()).toBe('0')
297+
expect(pa.props).toHaveLength(2)
298+
expect((pa.props[0] as any).content).toBe('arr')
299+
expect(pa.props[1]).toBeInstanceOf(NumberToken)
300+
expect(pa.props[1].getText()).toBe('0')
284301
})
285302
it('should read a filter with obj.foo argument', function () {
286303
const tokenizer = new Tokenizer('| plus: obj.foo')
@@ -290,10 +307,10 @@ describe('Tokenizer', function () {
290307

291308
const pa: PropertyAccessToken = token!.args[0] as any
292309
expect(token!.args[0]).toBeInstanceOf(PropertyAccessToken)
293-
expect((pa.variable as any).content).toBe('obj')
294-
expect(pa.props).toHaveLength(1)
295-
expect(pa.props[0]).toBeInstanceOf(IdentifierToken)
296-
expect(pa.props[0].getText()).toBe('foo')
310+
expect(pa.props).toHaveLength(2)
311+
expect((pa.props[0] as any).content).toBe('obj')
312+
expect(pa.props[1]).toBeInstanceOf(IdentifierToken)
313+
expect(pa.props[1].getText()).toBe('foo')
297314
})
298315
it('should read a filter with obj["foo"] argument', function () {
299316
const tokenizer = new Tokenizer('| plus: obj["good luck"]')
@@ -304,8 +321,8 @@ describe('Tokenizer', function () {
304321
const pa: PropertyAccessToken = token!.args[0] as any
305322
expect(token!.args[0]).toBeInstanceOf(PropertyAccessToken)
306323
expect(pa.getText()).toBe('obj["good luck"]')
307-
expect((pa.variable as any).content).toBe('obj')
308-
expect(pa.props[0].getText()).toBe('"good luck"')
324+
expect((pa.props[0] as any).content).toBe('obj')
325+
expect(pa.props[1].getText()).toBe('"good luck"')
309326
})
310327
})
311328
describe('#readFilters()', () => {
@@ -341,7 +358,7 @@ describe('Tokenizer', function () {
341358
expect(tokens[2].args).toHaveLength(1)
342359
expect(tokens[2].args[0]).toBeInstanceOf(PropertyAccessToken)
343360
expect((tokens[2].args[0] as any).getText()).toBe('foo[a.b["c d"]]')
344-
expect((tokens[2].args[0] as any).props[0].getText()).toBe('a.b["c d"]')
361+
expect((tokens[2].args[0] as any).props[1].getText()).toBe('a.b["c d"]')
345362
})
346363
})
347364
describe('#readExpression()', () => {
@@ -358,10 +375,10 @@ describe('Tokenizer', function () {
358375
expect(exp).toHaveLength(1)
359376
const pa = exp[0] as PropertyAccessToken
360377
expect(pa).toBeInstanceOf(PropertyAccessToken)
361-
expect((pa.variable as any).content).toEqual('a')
362-
expect(pa.props).toHaveLength(2)
378+
expect(pa.props).toHaveLength(3)
379+
expect((pa.props[0] as any).content).toEqual('a')
363380

364-
const [p1, p2] = pa.props
381+
const [, p1, p2] = pa.props
365382
expect(p1).toBeInstanceOf(IdentifierToken)
366383
expect(p1.getText()).toBe('')
367384
expect(p2).toBeInstanceOf(PropertyAccessToken)
@@ -373,8 +390,8 @@ describe('Tokenizer', function () {
373390
expect(exp).toHaveLength(1)
374391
const pa = exp[0] as PropertyAccessToken
375392
expect(pa).toBeInstanceOf(PropertyAccessToken)
376-
expect((pa.variable as any).content).toEqual('a')
377-
expect(pa.props).toHaveLength(0)
393+
expect(pa.props).toHaveLength(1)
394+
expect((pa.props[0] as any).content).toEqual('a')
378395
})
379396
it('should read expression `a ==`', () => {
380397
const exp = [...new Tokenizer('a ==').readExpressionTokens()]
@@ -481,6 +498,30 @@ describe('Tokenizer', function () {
481498
expect(rhs.getText()).toEqual('"\\""')
482499
})
483500
})
501+
describe('#matchTrie()', function () {
502+
const opTrie = createTrie(defaultOperators)
503+
it('should match contains', () => {
504+
expect(new Tokenizer('contains').matchTrie(opTrie)).toBe(8)
505+
})
506+
it('should match comparision', () => {
507+
expect(new Tokenizer('>').matchTrie(opTrie)).toBe(1)
508+
expect(new Tokenizer('>=').matchTrie(opTrie)).toBe(2)
509+
expect(new Tokenizer('<').matchTrie(opTrie)).toBe(1)
510+
expect(new Tokenizer('<=').matchTrie(opTrie)).toBe(2)
511+
})
512+
it('should match binary logic', () => {
513+
expect(new Tokenizer('and').matchTrie(opTrie)).toBe(3)
514+
expect(new Tokenizer('or').matchTrie(opTrie)).toBe(2)
515+
})
516+
it('should not match if word not terminate', () => {
517+
expect(new Tokenizer('true1').matchTrie(opTrie)).toBe(-1)
518+
expect(new Tokenizer('containsa').matchTrie(opTrie)).toBe(-1)
519+
})
520+
it('should match if word boundary found', () => {
521+
expect(new Tokenizer('>=1').matchTrie(opTrie)).toBe(2)
522+
expect(new Tokenizer('contains b').matchTrie(opTrie)).toBe(8)
523+
})
524+
})
484525
describe('#readLiquidTagTokens', () => {
485526
it('should read newline terminated tokens', () => {
486527
const tokenizer = new Tokenizer('echo \'hello\'')

‎src/parser/tokenizer.ts

+71-32
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
import { FilteredValueToken, TagToken, HTMLToken, HashToken, QuotedToken, LiquidTagToken, OutputToken, ValueToken, Token, RangeToken, FilterToken, TopLevelToken, PropertyAccessToken, OperatorToken, LiteralToken, IdentifierToken, NumberToken } from '../tokens'
2-
import { Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, IDENTIFIER } from '../util'
2+
import { OperatorHandler } from '../render/operator'
3+
import { TrieNode, LiteralValue, Trie, createTrie, ellipsis, literalValues, TokenizationError, TYPES, QUOTE, BLANK, IDENTIFIER, NUMBER, SIGN } from '../util'
34
import { Operators, Expression } from '../render'
45
import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
56
import { FilterArg } from './filter-arg'
6-
import { matchOperator } from './match-operator'
77
import { whiteSpaceCtrl } from './whitespace-ctrl'
88

99
export class Tokenizer {
1010
p: number
1111
N: number
1212
private rawBeginAt = -1
13-
private opTrie: Trie
13+
private opTrie: Trie<OperatorHandler>
14+
private literalTrie: Trie<LiteralValue>
1415

1516
constructor (
1617
public input: string,
@@ -21,6 +22,7 @@ export class Tokenizer {
2122
this.p = range ? range[0] : 0
2223
this.N = range ? range[1] : input.length
2324
this.opTrie = createTrie(operators)
25+
this.literalTrie = createTrie(literalValues)
2426
}
2527

2628
readExpression () {
@@ -44,10 +46,22 @@ export class Tokenizer {
4446
}
4547
readOperator (): OperatorToken | undefined {
4648
this.skipBlank()
47-
const end = matchOperator(this.input, this.p, this.opTrie)
49+
const end = this.matchTrie(this.opTrie)
4850
if (end === -1) return
4951
return new OperatorToken(this.input, this.p, (this.p = end), this.file)
5052
}
53+
matchTrie<T> (trie: Trie<T>) {
54+
let node: TrieNode<T> = trie
55+
let i = this.p
56+
let info
57+
while (node[this.input[i]] && i < this.N) {
58+
node = node[this.input[i++]]
59+
if (node['end']) info = node
60+
}
61+
if (!info) return -1
62+
if (info['needBoundary'] && (this.peekType(i - this.p) & IDENTIFIER)) return -1
63+
return i
64+
}
5165
readFilteredValue (): FilteredValueToken {
5266
const begin = this.p
5367
const initial = this.readExpression()
@@ -272,8 +286,8 @@ export class Tokenizer {
272286
return this.input.slice(this.p, this.N)
273287
}
274288

275-
advance (i = 1) {
276-
this.p += i
289+
advance (step = 1) {
290+
this.p += step
277291
}
278292

279293
end () {
@@ -289,43 +303,68 @@ export class Tokenizer {
289303
}
290304

291305
readValue (): ValueToken | undefined {
292-
const value = this.readQuoted() || this.readRange()
293-
if (value) return value
294-
295-
if (this.peek() === '[') {
296-
this.p++
297-
const prop = this.readQuoted()
298-
if (!prop) return
299-
if (this.peek() !== ']') return
300-
this.p++
301-
return new PropertyAccessToken(prop, [], this.p)
302-
}
303-
304-
const variable = this.readIdentifier()
305-
if (!variable.size()) return
306-
307-
let isNumber = variable.isNumber(true)
308-
const props: (QuotedToken | IdentifierToken)[] = []
306+
this.skipBlank()
307+
const begin = this.p
308+
const variable = this.readLiteral() || this.readQuoted() || this.readRange() || this.readNumber()
309+
const props: (ValueToken | IdentifierToken)[] = []
309310
while (true) {
310311
if (this.peek() === '[') {
311-
isNumber = false
312312
this.p++
313313
const prop = this.readValue() || new IdentifierToken(this.input, this.p, this.p, this.file)
314-
this.readTo(']')
314+
this.assert(this.readTo(']') !== -1, '[ not closed')
315315
props.push(prop)
316-
} else if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax
316+
continue
317+
}
318+
if (!variable && !props.length) {
319+
const prop = this.readIdentifier()
320+
if (prop.size()) {
321+
props.push(prop)
322+
continue
323+
}
324+
}
325+
if (this.peek() === '.' && this.peek(1) !== '.') { // skip range syntax
317326
this.p++
318327
const prop = this.readIdentifier()
319328
if (!prop.size()) break
320-
if (!prop.isNumber()) isNumber = false
321329
props.push(prop)
330+
continue
331+
}
332+
break
333+
}
334+
if (!props.length) return variable
335+
return new PropertyAccessToken(variable, props, this.input, begin, this.p)
336+
}
337+
338+
readNumber (): NumberToken | undefined {
339+
this.skipBlank()
340+
let decimalFound = false
341+
let digitFound = false
342+
let n = 0
343+
if (this.peekType() & SIGN) n++
344+
while (this.p + n <= this.N) {
345+
if (this.peekType(n) & NUMBER) {
346+
digitFound = true
347+
n++
348+
} else if (this.peek(n) === '.' && this.peek(n + 1) !== '.') {
349+
if (decimalFound || !digitFound) return
350+
decimalFound = true
351+
n++
322352
} else break
323353
}
324-
if (!props.length && literalValues.hasOwnProperty(variable.content)) {
325-
return new LiteralToken(this.input, variable.begin, variable.end, this.file)
354+
if (digitFound && !(this.peekType(n) & IDENTIFIER)) {
355+
const num = new NumberToken(this.input, this.p, this.p + n, this.file)
356+
this.advance(n)
357+
return num
326358
}
327-
if (isNumber) return new NumberToken(variable, props[0] as IdentifierToken)
328-
return new PropertyAccessToken(variable, props, this.p)
359+
}
360+
361+
readLiteral (): LiteralToken | undefined {
362+
this.skipBlank()
363+
const end = this.matchTrie(this.literalTrie)
364+
if (end === -1) return
365+
const literal = new LiteralToken(this.input, this.p, end, this.file)
366+
this.p = end
367+
return literal
329368
}
330369

331370
readRange (): RangeToken | undefined {
@@ -388,7 +427,7 @@ export class Tokenizer {
388427
}
389428

390429
peekType (n = 0) {
391-
return TYPES[this.input.charCodeAt(this.p + n)]
430+
return this.p + n >= this.N ? 0 : TYPES[this.input.charCodeAt(this.p + n)]
392431
}
393432

394433
peek (n = 0): string {

0 commit comments

Comments
 (0)