Skip to content

Commit 9033d42

Browse files
committed
stream: writable buffering
1 parent 1592d0a commit 9033d42

File tree

2 files changed

+70
-152
lines changed

2 files changed

+70
-152
lines changed

lib/_stream_writable.js

Lines changed: 51 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,18 @@ Object.setPrototypeOf(Writable, Stream);
5353

5454
function nop() {}
5555

56+
function bufferedDispatch(err) {
57+
const index = this.index;
58+
for (let i = 0; i < index; i++) {
59+
this[i].callback(err);
60+
this[i] = null;
61+
}
62+
63+
this.splice(0, index);
64+
this.index -= index;
65+
this.allBuffers = this.allBuffers || this.every((request) => request.isBuf);
66+
}
67+
5668
function WritableState(options, stream, isDuplex) {
5769
options = options || {};
5870

@@ -134,8 +146,10 @@ function WritableState(options, stream, isDuplex) {
134146
// The amount that is being written when _write is called.
135147
this.writelen = 0;
136148

137-
this.bufferedRequest = null;
138-
this.lastBufferedRequest = null;
149+
this.buffered = [];
150+
this.buffered.index = 0;
151+
this.buffered.allBuffers = true;
152+
this.buffered.dispatch = bufferedDispatch.bind(this.buffered);
139153

140154
// Number of pending user-supplied write callbacks
141155
// this must be 0 before 'finish' can be emitted
@@ -153,25 +167,10 @@ function WritableState(options, stream, isDuplex) {
153167

154168
// Should .destroy() be called after 'finish' (and potentially 'end')
155169
this.autoDestroy = !!options.autoDestroy;
156-
157-
// Count buffered requests
158-
this.bufferedRequestCount = 0;
159-
160-
// Allocate the first CorkedRequest, there is always
161-
// one allocated and free to use, and we maintain at most two
162-
const corkReq = { next: null, entry: null, finish: undefined };
163-
corkReq.finish = onCorkedFinish.bind(undefined, corkReq, this);
164-
this.corkedRequestsFree = corkReq;
165170
}
166171

167172
WritableState.prototype.getBuffer = function getBuffer() {
168-
var current = this.bufferedRequest;
169-
const out = [];
170-
while (current) {
171-
out.push(current);
172-
current = current.next;
173-
}
174-
return out;
173+
return this.buffered.slice(this.buffered.index);
175174
};
176175

177176
Object.defineProperty(WritableState.prototype, 'buffer', {
@@ -314,12 +313,7 @@ Writable.prototype.uncork = function() {
314313

315314
if (state.corked) {
316315
state.corked--;
317-
318-
if (!state.writing &&
319-
!state.corked &&
320-
!state.bufferProcessing &&
321-
state.bufferedRequest)
322-
clearBuffer(this, state);
316+
clearBuffer(this, state);
323317
}
324318
};
325319

@@ -365,7 +359,7 @@ Object.defineProperty(Writable.prototype, 'writableHighWaterMark', {
365359
// If we're already writing something, then just put this
366360
// in the queue, and wait our turn. Otherwise, call _write
367361
// If we return false, then we need a drain event, so set that flag.
368-
function writeOrBuffer(stream, state, isBuf, chunk, encoding, cb) {
362+
function writeOrBuffer(stream, state, isBuf, chunk, encoding, callback) {
369363
if (!isBuf) {
370364
var newChunk = decodeChunk(state, chunk, encoding);
371365
if (chunk !== newChunk) {
@@ -384,22 +378,16 @@ function writeOrBuffer(stream, state, isBuf, chunk, encoding, cb) {
384378
state.needDrain = true;
385379

386380
if (state.writing || state.corked) {
387-
var last = state.lastBufferedRequest;
388-
state.lastBufferedRequest = {
381+
const buffered = state.buffered;
382+
buffered.push({
389383
chunk,
390384
encoding,
391-
isBuf,
392-
callback: cb,
393-
next: null
394-
};
395-
if (last) {
396-
last.next = state.lastBufferedRequest;
397-
} else {
398-
state.bufferedRequest = state.lastBufferedRequest;
399-
}
400-
state.bufferedRequestCount += 1;
385+
callback,
386+
isBuf
387+
});
388+
buffered.allBuffers = isBuf && buffered.allBuffers;
401389
} else {
402-
doWrite(stream, state, false, len, chunk, encoding, cb);
390+
doWrite(stream, state, false, len, chunk, encoding, callback);
403391
}
404392

405393
return ret;
@@ -426,21 +414,13 @@ function onwriteError(stream, state, sync, er, cb) {
426414
// Defer the callback if we are being called synchronously
427415
// to avoid piling up things on the stack
428416
process.nextTick(cb, er);
429-
// This can emit finish, and it will always happen
430-
// after error
431-
process.nextTick(finishMaybe, stream, state);
432-
stream._writableState.errorEmitted = true;
433-
errorOrDestroy(stream, er);
434417
} else {
435418
// The caller expect this to happen before if
436419
// it is async
437420
cb(er);
438-
stream._writableState.errorEmitted = true;
439-
errorOrDestroy(stream, er);
440-
// This can emit finish, but finish must
441-
// always follow error
442-
finishMaybe(stream, state);
443421
}
422+
stream._writableState.errorEmitted = true;
423+
errorOrDestroy(stream, er);
444424
}
445425

446426
function onwrite(stream, er) {
@@ -462,10 +442,7 @@ function onwrite(stream, er) {
462442
// Check if we're actually ready to finish, but don't emit yet
463443
var finished = needFinish(state) || stream.destroyed;
464444

465-
if (!finished &&
466-
!state.corked &&
467-
!state.bufferProcessing &&
468-
state.bufferedRequest) {
445+
if (!finished) {
469446
clearBuffer(stream, state);
470447
}
471448

@@ -497,67 +474,34 @@ function onwriteDrain(stream, state) {
497474

498475
// If there's something in the buffer waiting, then process it
499476
function clearBuffer(stream, state) {
500-
state.bufferProcessing = true;
501-
var entry = state.bufferedRequest;
502-
503-
if (stream._writev && entry && entry.next) {
504-
// Fast case, write everything using _writev()
505-
var l = state.bufferedRequestCount;
506-
var buffer = new Array(l);
507-
var holder = state.corkedRequestsFree;
508-
holder.entry = entry;
509-
510-
var count = 0;
511-
var allBuffers = true;
512-
while (entry) {
513-
buffer[count] = entry;
514-
if (!entry.isBuf)
515-
allBuffers = false;
516-
entry = entry.next;
517-
count += 1;
518-
}
519-
buffer.allBuffers = allBuffers;
477+
if (state.writing || state.bufferProcessing || state.corked) {
478+
return;
479+
}
520480

521-
doWrite(stream, state, true, state.length, buffer, '', holder.finish);
481+
const buffered = state.buffered;
482+
const bufferedCount = buffered.length - buffered.index;
522483

523-
// doWrite is almost always async, defer these to save a bit of time
524-
// as the hot path ends with doWrite
525-
state.pendingcb++;
526-
state.lastBufferedRequest = null;
527-
if (holder.next) {
528-
state.corkedRequestsFree = holder.next;
529-
holder.next = null;
530-
} else {
531-
var corkReq = { next: null, entry: null, finish: undefined };
532-
corkReq.finish = onCorkedFinish.bind(undefined, corkReq, state);
533-
state.corkedRequestsFree = corkReq;
534-
}
535-
state.bufferedRequestCount = 0;
484+
if (!bufferedCount) {
485+
return;
486+
}
487+
488+
state.bufferProcessing = true;
489+
if (stream._writev) {
490+
buffered.index += bufferedCount;
491+
doWrite(stream, state, true, state.length, buffered, '', buffered.dispatch);
536492
} else {
537493
// Slow case, write chunks one-by-one
538-
while (entry) {
539-
var chunk = entry.chunk;
540-
var encoding = entry.encoding;
541-
var cb = entry.callback;
542-
var len = state.objectMode ? 1 : chunk.length;
543-
544-
doWrite(stream, state, false, len, chunk, encoding, cb);
545-
entry = entry.next;
546-
state.bufferedRequestCount--;
494+
for (const { chunk, encoding, length } of buffered) {
495+
const len = state.objectMode ? 1 : length;
496+
buffered.index += 1;
497+
doWrite(stream, state, false, len, chunk, encoding, buffered.dispatch);
547498
// If we didn't call the onwrite immediately, then
548499
// it means that we need to wait until it does.
549-
// also, that means that the chunk and cb are currently
550-
// being processed, so move the buffer counter past them.
551500
if (state.writing) {
552501
break;
553502
}
554503
}
555-
556-
if (entry === null)
557-
state.lastBufferedRequest = null;
558504
}
559-
560-
state.bufferedRequest = entry;
561505
state.bufferProcessing = false;
562506
}
563507

@@ -606,9 +550,13 @@ Object.defineProperty(Writable.prototype, 'writableLength', {
606550
});
607551

608552
function needFinish(state) {
553+
const buffered = state.buffered;
554+
const bufferedCount = buffered.length - buffered.index;
555+
609556
return (state.ending &&
610557
state.length === 0 &&
611-
state.bufferedRequest === null &&
558+
bufferedCount === 0 &&
559+
!state.errorEmitted &&
612560
!state.finished &&
613561
!state.writing);
614562
}
@@ -670,20 +618,6 @@ function endWritable(stream, state, cb) {
670618
stream.writable = false;
671619
}
672620

673-
function onCorkedFinish(corkReq, state, err) {
674-
var entry = corkReq.entry;
675-
corkReq.entry = null;
676-
while (entry) {
677-
var cb = entry.callback;
678-
state.pendingcb--;
679-
cb(err);
680-
entry = entry.next;
681-
}
682-
683-
// Reuse the free corkReq.
684-
state.corkedRequestsFree.next = corkReq;
685-
}
686-
687621
Object.defineProperty(Writable.prototype, 'destroyed', {
688622
// Making it explicit this property is not enumerable
689623
// because otherwise some prototype manipulation in

test/parallel/test-stream-writable-write-writev-finish.js

Lines changed: 19 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,11 @@ const stream = require('stream');
1414
cb(new Error('write test error'));
1515
};
1616

17-
let firstError = false;
18-
writable.on('finish', common.mustCall(function() {
19-
assert.strictEqual(firstError, true);
20-
}));
21-
22-
writable.on('prefinish', common.mustCall());
17+
writable.on('finish', common.mustNotCall());
18+
writable.on('prefinish', common.mustNotCall());
2319

2420
writable.on('error', common.mustCall((er) => {
2521
assert.strictEqual(er.message, 'write test error');
26-
firstError = true;
2722
}));
2823

2924
writable.end('test');
@@ -36,16 +31,11 @@ const stream = require('stream');
3631
setImmediate(cb, new Error('write test error'));
3732
};
3833

39-
let firstError = false;
40-
writable.on('finish', common.mustCall(function() {
41-
assert.strictEqual(firstError, true);
42-
}));
43-
44-
writable.on('prefinish', common.mustCall());
34+
writable.on('finish', common.mustNotCall());
35+
writable.on('prefinish', common.mustNotCall());
4536

4637
writable.on('error', common.mustCall((er) => {
4738
assert.strictEqual(er.message, 'write test error');
48-
firstError = true;
4939
}));
5040

5141
writable.end('test');
@@ -62,16 +52,11 @@ const stream = require('stream');
6252
cb(new Error('writev test error'));
6353
};
6454

65-
let firstError = false;
66-
writable.on('finish', common.mustCall(function() {
67-
assert.strictEqual(firstError, true);
68-
}));
69-
70-
writable.on('prefinish', common.mustCall());
55+
writable.on('finish', common.mustNotCall());
56+
writable.on('prefinish', common.mustNotCall());
7157

7258
writable.on('error', common.mustCall((er) => {
7359
assert.strictEqual(er.message, 'writev test error');
74-
firstError = true;
7560
}));
7661

7762
writable.cork();
@@ -93,16 +78,11 @@ const stream = require('stream');
9378
setImmediate(cb, new Error('writev test error'));
9479
};
9580

96-
let firstError = false;
97-
writable.on('finish', common.mustCall(function() {
98-
assert.strictEqual(firstError, true);
99-
}));
100-
101-
writable.on('prefinish', common.mustCall());
81+
writable.on('finish', common.mustNotCall());
82+
writable.on('prefinish', common.mustNotCall());
10283

10384
writable.on('error', common.mustCall((er) => {
10485
assert.strictEqual(er.message, 'writev test error');
105-
firstError = true;
10686
}));
10787

10888
writable.cork();
@@ -123,14 +103,9 @@ const stream = require('stream');
123103
rs._read = () => {};
124104

125105
const ws = new stream.Writable();
126-
let firstError = false;
127106

128-
ws.on('finish', common.mustCall(function() {
129-
assert.strictEqual(firstError, true);
130-
}));
131-
ws.on('error', common.mustCall(function() {
132-
firstError = true;
133-
}));
107+
ws.on('finish', common.mustNotCall());
108+
ws.on('error', common.mustCall());
134109

135110
ws._write = (chunk, encoding, done) => {
136111
setImmediate(done, new Error());
@@ -178,3 +153,12 @@ const stream = require('stream');
178153
});
179154
w.end();
180155
}
156+
157+
{
158+
const w = new stream.Writable();
159+
w._write = (chunk, encoding, cb) => {
160+
process.nextTick(cb);
161+
};
162+
w.on('finish', common.mustCall());
163+
w.end();
164+
}

0 commit comments

Comments
 (0)