Skip to content

Commit c33d8f6

Browse files
harttleharttle
committed
feat: throw an Error if delimiter not matched
1 parent c13a16f commit c33d8f6

File tree

5 files changed

+62
-28
lines changed

5 files changed

+62
-28
lines changed

‎src/parser/token.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
export default class Token {
22
type: string
33
line: number
4+
col: number
45
raw: string
56
input: string
67
file: string
78
value: string
8-
constructor (raw, pos, input, file, line) {
9+
constructor (raw, col, input, file, line) {
10+
this.col = col
911
this.line = line
1012
this.raw = raw
1113
this.input = input

‎src/parser/tokenizer.ts

+28-10
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import whiteSpaceCtrl from './whitespace-ctrl'
22
import HTMLToken from './html-token'
33
import TagToken from './tag-token'
4+
import Token from './token'
45
import OutputToken from './output-token'
6+
import { TokenizationError } from 'src/util/error'
57
import { LiquidOptions, defaultOptions } from 'src/liquid-options'
68

79
enum ParseState { HTML, OUTPUT, TAG }
@@ -14,43 +16,59 @@ export default class Tokenizer {
1416
tokenize (input: string, file?: string) {
1517
const tokens = []
1618
let p = 0
17-
let line = 1
19+
let curLine = 1
1820
let state = ParseState.HTML
1921
let buffer = ''
20-
let bufferBegin = 0
22+
let lineBegin = 0
23+
let line = 1
24+
let col = 1
2125

2226
while (p < input.length) {
23-
if (input[p] === '\n') line++
27+
if (input[p] === '\n') {
28+
curLine++
29+
lineBegin = p + 1
30+
}
2431
const bin = input.substr(p, 2)
2532
if (state === ParseState.HTML) {
2633
if (bin === '{{' || bin === '{%') {
27-
if (buffer) tokens.push(new HTMLToken(buffer, bufferBegin, input, file, line))
34+
if (buffer) tokens.push(new HTMLToken(buffer, col, input, file, line))
2835
buffer = bin
29-
bufferBegin = p
36+
line = curLine
37+
col = p - lineBegin + 1
3038
p += 2
3139
state = bin === '{{' ? ParseState.OUTPUT : ParseState.TAG
3240
continue
3341
}
3442
} else if (state === ParseState.OUTPUT && bin === '}}') {
3543
buffer += '}}'
36-
tokens.push(new OutputToken(buffer, bufferBegin, input, file, line))
44+
tokens.push(new OutputToken(buffer, col, input, file, line))
3745
p += 2
3846
buffer = ''
39-
bufferBegin = p
47+
line = curLine
48+
col = p - lineBegin + 1
4049
state = ParseState.HTML
4150
continue
4251
} else if (bin === '%}') {
4352
buffer += '%}'
44-
tokens.push(new TagToken(buffer, bufferBegin, input, file, line))
53+
tokens.push(new TagToken(buffer, col, input, file, line))
4554
p += 2
4655
buffer = ''
47-
bufferBegin = p
56+
line = curLine
57+
col = p - lineBegin + 1
4858
state = ParseState.HTML
4959
continue
5060
}
5161
buffer += input[p++]
5262
}
53-
if (buffer) tokens.push(new HTMLToken(buffer, bufferBegin, input, file, line))
63+
if (state !== ParseState.HTML) {
64+
const t = state === ParseState.OUTPUT ? 'output' : 'tag'
65+
const str = buffer.length > 16 ? buffer.slice(0, 13) + '...' : buffer
66+
throw new TokenizationError(
67+
new Error(`${t} "${str}" not closed`),
68+
new Token(buffer, col, input, file, line)
69+
)
70+
}
71+
if (buffer) tokens.push(new HTMLToken(buffer, col, input, file, line))
5472

5573
whiteSpaceCtrl(tokens, this.options)
5674
return tokens

‎src/util/error.ts

+2-4
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,12 @@ abstract class LiquidError {
1111
name: string
1212
message: string
1313
stack: string
14-
private line: string
1514
private file: string
1615
private input: string
1716
private token: Token
1817
private originalError: Error
1918
constructor (err, token) {
2019
this.input = token.input
21-
this.line = token.line
2220
this.file = token.file
2321
this.originalError = err
2422
this.token = token
@@ -28,7 +26,7 @@ abstract class LiquidError {
2826

2927
captureStack.call(obj)
3028
const err = this.originalError
31-
const context = mkContext(this.input, this.line)
29+
const context = mkContext(this.input, this.token.line)
3230
this.message = mkMessage(err.message, this.token)
3331
this.stack = this.message + '\n' + context +
3432
'\n' + (this.stack || this.message) +
@@ -110,7 +108,7 @@ function mkMessage (msg, token) {
110108
msg += ', file:' + token.file
111109
}
112110
if (token.line) {
113-
msg += ', line:' + token.line
111+
msg += `, line:${token.line}, col:${token.col}`
114112
}
115113
return msg
116114
}

‎test/unit/parser/tokenizer.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import HTMLToken from 'src/parser/html-token'
66

77
describe('tokenizer', function () {
88
const tokenizer = new Tokenizer()
9-
describe('parse', function () {
9+
describe('#tokenize()', function () {
1010
it('should handle plain HTML', function () {
1111
const html = '<html><body><p>Lorem Ipsum</p></body></html>'
1212
const tokens = tokenizer.tokenize(html)
@@ -66,5 +66,15 @@ describe('tokenizer', function () {
6666
expect(tokens[0]).instanceOf(OutputToken)
6767
expect(tokens[0].raw).to.equal('{{foo\n|date:\n"%Y-%m-%d"\n}}')
6868
})
69+
it('should throw if tag not closed', function () {
70+
expect(() => {
71+
tokenizer.tokenize('{% assign foo = bar {{foo}}')
72+
}).to.throw(/tag "{% assign foo..." not closed/)
73+
})
74+
it('should throw if output not closed', function () {
75+
expect(() => {
76+
tokenizer.tokenize('{{name}')
77+
}).to.throw(/output "{{name}" not closed/)
78+
})
6979
})
7080
})

‎test/unit/util/error.ts

+18-12
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ describe('error', function () {
2828
'TokenizationError'
2929
]
3030
const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
31-
expect(err.message).to.equal('illegal tag syntax, line:3')
31+
expect(err.message).to.equal('illegal tag syntax, line:3, col:2')
3232
expect(err.stack).to.contain(message.join('\n'))
3333
expect(err.name).to.equal('TokenizationError')
3434
})
@@ -37,10 +37,10 @@ describe('error', function () {
3737
const err = await expect(engine.parseAndRender(html)).be.rejected
3838
expect(err.input).to.equal(html)
3939
})
40-
it('should contain line number in err.line', async function () {
40+
it('should contain line number in err.token.line', async function () {
4141
const err = await expect(engine.parseAndRender('1\n2\n{% . a %}\n4')).be.rejected
4242
expect(err.name).to.equal('TokenizationError')
43-
expect(err.line).to.equal(3)
43+
expect(err.token.line).to.equal(3)
4444
})
4545
it('should contain stack in err.stack', async function () {
4646
const err = await expect(engine.parseAndRender('{% . a %}')).be.rejected
@@ -68,6 +68,12 @@ describe('error', function () {
6868
expect(err.name).to.equal('TokenizationError')
6969
expect(err.file).to.equal(path.resolve('/foo.html'))
7070
})
71+
it('should throw error with line and pos if tag unmatched', async function () {
72+
const err = await expect(engine.parseAndRender('1\n2\nfoo{% assign a = 4 }\n4')).be.rejected
73+
expect(err.name).to.equal('TokenizationError')
74+
expect(err.token.line).to.equal(3)
75+
expect(err.token.col).to.equal(4)
76+
})
7177
})
7278

7379
describe('RenderError', function () {
@@ -128,7 +134,7 @@ describe('error', function () {
128134
'RenderError'
129135
]
130136
const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
131-
expect(err.message).to.equal('intended render error, line:4')
137+
expect(err.message).to.equal('intended render error, line:4, col:2')
132138
expect(err.stack).to.contain(message.join('\n'))
133139
expect(err.name).to.equal('RenderError')
134140
})
@@ -157,7 +163,7 @@ describe('error', function () {
157163
const err = await expect(engine.parseAndRender(html)).be.rejected
158164
console.log(err.message)
159165
console.log(err.stack)
160-
expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4`)
166+
expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4, col:2`)
161167
expect(err.stack).to.contain(message.join('\n'))
162168
expect(err.name).to.equal('RenderError')
163169
})
@@ -177,7 +183,7 @@ describe('error', function () {
177183
'RenderError'
178184
]
179185
const err = await expect(engine.parseAndRender(html)).be.rejected
180-
expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4`)
186+
expect(err.message).to.equal(`intended render error, file:${path.resolve('/throwing-tag.html')}, line:4, col:2`)
181187
expect(err.stack).to.contain(message.join('\n'))
182188
expect(err.name).to.equal('RenderError')
183189
})
@@ -187,10 +193,10 @@ describe('error', function () {
187193
expect(err.input).to.equal(html)
188194
expect(err.name).to.equal('RenderError')
189195
})
190-
it('should contain line number in err.line', async function () {
196+
it('should contain line number in err.token.line', async function () {
191197
const src = '1\n2\n{{1|throwingFilter}}\n4'
192198
const err = await expect(engine.parseAndRender(src)).be.rejected
193-
expect(err.line).to.equal(3)
199+
expect(err.token.line).to.equal(3)
194200
expect(err.name).to.equal('RenderError')
195201
})
196202
it('should contain stack in err.stack', async function () {
@@ -260,7 +266,7 @@ describe('error', function () {
260266
'ParseError: tag a not found'
261267
]
262268
const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
263-
expect(err.message).to.equal('tag a not found, line:4')
269+
expect(err.message).to.equal('tag a not found, line:4, col:2')
264270
expect(err.stack).to.contain(message.join('\n'))
265271
expect(err.name).to.equal('ParseError')
266272
})
@@ -275,14 +281,14 @@ describe('error', function () {
275281
'ParseError: tag a not found'
276282
]
277283
const err = await expect(engine.parseAndRender(html.join('\n'))).be.rejected
278-
expect(err.message).to.equal('tag a not found, line:2')
284+
expect(err.message).to.equal('tag a not found, line:2, col:2')
279285
expect(err.stack).to.contain(message.join('\n'))
280286
})
281287

282-
it('should contain line number in err.line', async function () {
288+
it('should contain line number in err.token.line', async function () {
283289
const html = '<html>\n<head>\n\n{% raw %}\n\n'
284290
const err = await expect(engine.parseAndRender(html)).be.rejected
285-
expect(err.line).to.equal(4)
291+
expect(err.token.line).to.equal(4)
286292
})
287293

288294
it('should contain stack in err.stack', async function () {

0 commit comments

Comments
 (0)