new changes

This commit is contained in:
Niranjan
2026-04-07 05:05:28 +05:30
parent 7c070224bd
commit a18bba15f2
29975 changed files with 3247495 additions and 2761 deletions

View File

@@ -0,0 +1,50 @@
import { Duplex, type Writable } from 'stream';
export type BufferStreamOptions = {
objectMode: boolean;
};
export type BufferStreamItem<O extends Partial<BufferStreamOptions>, T> = O extends {
objectMode: true;
} ? T : Buffer;
export type BufferStreamPayload<O extends Partial<BufferStreamOptions>, T> = O extends {
objectMode: true;
} ? T[] : Buffer;
export type BufferStreamHandler<O extends Partial<BufferStreamOptions>, T> = (payload: BufferStreamPayload<O, T>) => Promise<BufferStreamPayload<O, T>>;
export type BufferStreamCallback<O extends Partial<BufferStreamOptions>, T> = (err: Error | null, payload: BufferStreamPayload<O, T>, cb: (err: Error | null, payload?: null | BufferStreamPayload<O, T>) => void) => void;
/**
* Buffer the stream content and bring it into the provided callback
*/
declare class BufferStream<T, O extends Partial<BufferStreamOptions>> extends Duplex {
private _options;
private _bufferCallback;
private _finished;
private _buffer;
/**
* @param bufferCallback {Function} A function to handle the buffered content.
* @param options {Object} inherits of Stream.Duplex, the options are passed to the parent constructor so you can use it's options too.
* @param options.objectMode {boolean} Use if piped in streams are in object mode. In this case, an array of the buffered will be transmitted to the callback function.
*/
constructor(bufferCallback: BufferStreamCallback<O, T> | BufferStreamHandler<O, T>, options?: O);
_write(chunk: BufferStreamItem<O, T>, encoding: Parameters<Writable['write']>[1], done: () => void): void;
_read(): void;
_bufferStreamCallbackWrapper(err: Error): void;
_bufferStreamError(err: Error): void;
}
/**
* Utility function if you prefer a functional way of using this lib
* @param bufferCallback
* @param options
* @returns Stream
*/
export declare function bufferStream<T, O extends Partial<BufferStreamOptions>>(bufferCallback: BufferStreamCallback<O, T>, options?: O): BufferStream<T, O>;
/**
* Utility function to buffer objet mode streams
* @param bufferCallback
* @param options
* @returns Stream
*/
export declare function bufferObjects<T>(bufferCallback: BufferStreamCallback<{
objectMode: true;
}, T>, options: Omit<BufferStreamOptions, 'objectMode'>): BufferStream<T, {
objectMode: true;
}>;
export { BufferStream };

View File

@@ -0,0 +1,105 @@
import { Duplex } from 'stream';
import { YError } from 'yerror';
const DEFAULT_BUFFER_STREAM_OPTIONS = {
objectMode: false,
};
/**
* Buffer the stream content and bring it into the provided callback
*/
class BufferStream extends Duplex {
_options = DEFAULT_BUFFER_STREAM_OPTIONS;
_bufferCallback;
_finished = false;
_buffer = [];
/**
* @param bufferCallback {Function} A function to handle the buffered content.
* @param options {Object} inherits of Stream.Duplex, the options are passed to the parent constructor so you can use it's options too.
* @param options.objectMode {boolean} Use if piped in streams are in object mode. In this case, an array of the buffered will be transmitted to the callback function.
*/
constructor(bufferCallback, options) {
super(options);
if (!(bufferCallback instanceof Function)) {
throw new YError('E_BAD_CALLBACK');
}
this._options = {
...DEFAULT_BUFFER_STREAM_OPTIONS,
...options,
};
this._bufferCallback =
bufferCallback.length === 1
? ((err, payload, cb) => {
bufferCallback(payload)
.then((result) => {
cb(err, result);
})
.catch((err) => {
cb(err);
});
})
: bufferCallback;
this.once('finish', this._bufferStreamCallbackWrapper);
this.on('error', this._bufferStreamError);
}
_write(chunk, encoding, done) {
this._buffer.push(chunk);
done();
}
_read() {
if (this._finished) {
while (this._buffer.length) {
if (!this.push(this._buffer.shift())) {
break;
}
}
if (0 === this._buffer.length) {
this.push(null);
}
}
}
_bufferStreamCallbackWrapper(err) {
const buffer = (this._options.objectMode
? this._buffer
: Buffer.concat(this._buffer));
err = err || null;
this._bufferCallback(err, buffer, (err2, buf) => {
setImmediate(() => {
this.removeListener('error', this._bufferStreamError);
if (err2) {
this.emit('error', err2);
}
this._buffer = (buf == null ? [] : buf instanceof Buffer ? [buf] : buf);
this._finished = true;
this._read();
});
});
}
_bufferStreamError(err) {
if (this._finished) {
return;
}
this._bufferStreamCallbackWrapper(err);
}
}
/**
* Utility function if you prefer a functional way of using this lib
* @param bufferCallback
* @param options
* @returns Stream
*/
export function bufferStream(bufferCallback, options = DEFAULT_BUFFER_STREAM_OPTIONS) {
return new BufferStream(bufferCallback, options);
}
/**
* Utility function to buffer objet mode streams
* @param bufferCallback
* @param options
* @returns Stream
*/
export function bufferObjects(bufferCallback, options) {
return new BufferStream(bufferCallback, {
...options,
objectMode: true,
});
}
export { BufferStream };
//# sourceMappingURL=index.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAiB,MAAM,QAAQ,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAsBhC,MAAM,6BAA6B,GAAG;IACpC,UAAU,EAAE,KAAK;CAClB,CAAC;AAEF;;GAEG;AACH,MAAM,YAAwD,SAAQ,MAAM;IAClE,QAAQ,GAAwB,6BAA6B,CAAC;IAC9D,eAAe,CAA6B;IAC5C,SAAS,GAAY,KAAK,CAAC;IAC3B,OAAO,GAA6B,EAAE,CAAC;IAE/C;;;;OAIG;IACH,YACE,cAAsE,EACtE,OAAW;QAEX,KAAK,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,CAAC,cAAc,YAAY,QAAQ,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG;YACd,GAAG,6BAA6B;YAChC,GAAG,OAAO;SACX,CAAC;QACF,IAAI,CAAC,eAAe;YAClB,cAAc,CAAC,MAAM,KAAK,CAAC;gBACzB,CAAC,CAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE;oBACpB,cAA4C,CAAC,OAAO,CAAC;yBACnD,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;wBACf,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;oBAClB,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACb,EAAE,CAAC,GAAG,CAAC,CAAC;oBACV,CAAC,CAAC,CAAC;gBACP,CAAC,CAAgC;gBACnC,CAAC,CAAE,cAA6C,CAAC;QAErD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACvD,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,CACJ,KAA6B,EAC7B,QAA0C,EAC1C,IAAgB;QAEhB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC;IACT,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;oBACrC,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,4BAA4B,CAAC,GAAU;QACrC,MAAM,MAAM,GAAG,CACb,IAAI,CAAC,QAAQ,CAAC,UAAU;YACtB,CAAC,CAAE,IAAI,CAAC,OAAe;YACvB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAmB,CAAC,CAKnC,CAAC;QAEX,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC;QAElB,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;YAC9C,YAAY,CAAC,GAAG,EAAE;gBAChB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBACtD,IAAI,IAAI,EAAE,CAAC;oBACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC3B,CAAC;gBACD,IAAI,CAAC,OAAO,GAAG,CACb,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,YAAY,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAC3B,CAAC;gBAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gBACtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB,CAAC,GAAU;QAC3B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAC1B,cAA0C,EAC1C,UAAa,6BAAkC;IAE/C,OAAO,IAAI,YAAY,CAAO,cAAc,EAAE,OAAO,CAAC,CAAC;AACzD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,cAA6D,EAC7D,OAAgD;IAEhD,OAAO,IAAI,YAAY,CAA0B,cAAc,EAAE;QAC/D,GAAG,OAAO;QACV,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;AACL,CAAC;AAED,OAAO,EAAE,YAAY,EAAE,CAAC"}

View File

@@ -0,0 +1 @@
export {};

View File

@@ -0,0 +1,329 @@
import { describe, expect, test } from '@jest/globals';
import { YError } from 'yerror';
import StreamTest from 'streamtest';
import { BufferStream } from './index.js';
// Helpers
function syncBufferPrefixer(headerText) {
return new BufferStream((err, buf, cb) => {
expect(err).toBeNull();
if (null === buf) {
cb(null, Buffer.from(headerText));
return;
}
cb(null, Buffer.concat([Buffer.from(headerText), buf]));
}, {
objectMode: false,
});
}
function syncObjectsPrefixer(prefixObject) {
return new BufferStream((err, objs, cb) => {
expect(err).toBeNull();
if (null === objs) {
cb(null, [prefixObject]);
return;
}
cb(null, [prefixObject, ...objs]);
}, {
objectMode: true,
});
}
function asyncBufferPrefixer(headerText) {
return new BufferStream((err, buf, cb) => {
expect(err).toBeNull();
if (null === buf) {
setTimeout(() => {
cb(null, Buffer.from(headerText));
}, 0);
}
else {
setTimeout(() => {
cb(null, Buffer.concat([Buffer.from(headerText), buf]));
}, 0);
}
}, {
objectMode: false,
});
}
describe('bufferstreams', () => {
test('should fail when callback is not a function', () => {
try {
new BufferStream(undefined);
throw new YError('E_UNEXPECTED_SUCCESS');
}
catch (err) {
expect(err.code).toEqual('E_BAD_CALLBACK');
}
});
describe('in buffer mode', () => {
describe('synchonously', () => {
test('should work with one pipe', async () => {
const [stream, result] = StreamTest.toText();
StreamTest.fromChunks([Buffer.from('te'), Buffer.from('st')])
.pipe(syncBufferPrefixer('plop'))
.pipe(stream);
expect(await result).toEqual('ploptest');
});
test('should work when returning a null buffer', async () => {
const [stream, result] = StreamTest.toText();
StreamTest.fromChunks([Buffer.from('te'), Buffer.from('st')])
.pipe(new BufferStream((err, buf, cb) => {
if (err) {
cb(err);
return;
}
cb(null, null);
}))
.pipe(stream);
expect(await result).toEqual('');
});
test('should work with an async handler', async () => {
const [stream, result] = StreamTest.toText();
StreamTest.fromChunks([Buffer.from('te'), Buffer.from('st')])
.pipe(new BufferStream(async (buf) => {
return buf;
}))
.pipe(stream);
expect(await result).toEqual('test');
});
test('should work with multiple pipes', async () => {
const [stream, result] = StreamTest.toText();
StreamTest.fromChunks([Buffer.from('te'), Buffer.from('st')])
.pipe(syncBufferPrefixer('plop'))
.pipe(syncBufferPrefixer('plip'))
.pipe(syncBufferPrefixer('plap'))
.pipe(stream);
expect(await result).toEqual('plapplipploptest');
});
});
describe('asynchonously', () => {
test('should work with one pipe', async () => {
const [stream, result] = StreamTest.toText();
StreamTest.fromChunks([Buffer.from('te'), Buffer.from('st')])
.pipe(asyncBufferPrefixer('plop'))
.pipe(stream);
expect(await result).toEqual('ploptest');
});
test('should work when returning a null buffer', async () => {
const [stream, result] = StreamTest.toText();
StreamTest.fromChunks([Buffer.from('te'), Buffer.from('st')])
.pipe(new BufferStream((err, _buf, cb) => {
if (err) {
cb(err);
return;
}
cb(null, null);
}))
.pipe(stream);
expect(await result).toEqual('');
});
test('should work with multiple pipes', async () => {
const [stream, result] = StreamTest.toText();
StreamTest.fromChunks([Buffer.from('te'), Buffer.from('st')])
.pipe(asyncBufferPrefixer('plop'))
.pipe(asyncBufferPrefixer('plip'))
.pipe(asyncBufferPrefixer('plap'))
.pipe(stream);
expect(await result).toEqual('plapplipploptest');
});
test('should report stream errors', async () => {
const [stream, result] = StreamTest.toText();
const bufferStream = new BufferStream((err, _objs, cb) => {
expect(err.code).toEqual('E_ERROR');
cb(null, []);
}, {
objectMode: true,
});
StreamTest.fromErroredChunks(new YError('E_ERROR'), [
Buffer.from('ou'),
Buffer.from('de'),
Buffer.from('la'),
Buffer.from('li'),
])
.on('error', (err) => {
bufferStream.emit('error', err);
})
.pipe(bufferStream)
.pipe(stream);
expect(await result).toEqual('');
});
test('should emit callback errors', async () => {
const [stream, result] = StreamTest.toText();
let caughtError = new YError('E_UNEXPECTED_SUCCESS');
StreamTest.fromChunks([
Buffer.from('ou'),
Buffer.from('de'),
Buffer.from('la'),
Buffer.from('li'),
])
.pipe(new BufferStream((err, _objs, cb) => {
if (err) {
cb(err);
return;
}
cb(new YError('E_ERROR'), Buffer.from(''));
}))
.on('error', (err) => {
caughtError = err;
})
.pipe(stream);
expect(await result).toEqual('');
expect(caughtError.code).toEqual('E_ERROR');
});
});
});
describe('in object mode', () => {
const object1 = { txt: 'te' };
const object2 = { txt: 'st' };
const object3 = { txt: 'e' };
const object4 = { txt: 'd' };
const object5 = { txt: 'u' };
const object6 = { txt: 'ni' };
const object7 = { txt: 't' };
describe('synchonously', () => {
test('should work with one pipe', async () => {
const [stream, result] = StreamTest.toObjects();
StreamTest.fromObjects([object1, object2])
.pipe(syncObjectsPrefixer(object4))
.pipe(stream);
expect(await result).toEqual([object4, object1, object2]);
});
test('should work when returning an empty array', async () => {
const [stream, result] = StreamTest.toObjects();
StreamTest.fromObjects([object1, object2])
.pipe(new BufferStream((err, _objs, cb) => {
if (err) {
cb(err);
return;
}
cb(null, []);
}, {
objectMode: true,
}))
.pipe(stream);
expect((await result).length).toEqual(0);
});
test('should work with multiple pipes', async () => {
const [stream, result] = StreamTest.toObjects();
StreamTest.fromObjects([object1, object2])
.pipe(syncObjectsPrefixer(object4))
.pipe(syncObjectsPrefixer(object5))
.pipe(syncObjectsPrefixer(object6))
.pipe(stream);
expect(await result).toEqual([
object6,
object5,
object4,
object1,
object2,
]);
});
});
describe('asynchonously', () => {
test('should work with one pipe', async () => {
const [stream, result] = StreamTest.toObjects();
StreamTest.fromObjects([object1, object2])
.pipe(syncObjectsPrefixer(object4))
.pipe(stream);
expect(await result).toEqual([object4, object1, object2]);
});
test('should work when returning an empty array', async () => {
const [stream, result] = StreamTest.toObjects();
StreamTest.fromObjects([object1, object2])
.pipe(new BufferStream((err, _objs, cb) => {
if (err) {
cb(err);
return;
}
cb(null, []);
}, {
objectMode: true,
}))
.pipe(stream);
expect((await result).length).toEqual(0);
});
test('should work when returning legacy null', async () => {
const [stream, result] = StreamTest.toObjects();
StreamTest.fromObjects([object1, object2])
.pipe(new BufferStream((err, _objs, cb) => {
if (err) {
cb(err);
return;
}
cb(null, null);
}, {
objectMode: true,
}))
.pipe(stream);
expect((await result).length).toEqual(0);
});
test('should work with multiple pipes', async () => {
const [stream, result] = StreamTest.toObjects();
StreamTest.fromObjects([object1, object2])
.pipe(syncObjectsPrefixer(object4))
.pipe(syncObjectsPrefixer(object5))
.pipe(syncObjectsPrefixer(object6))
.pipe(stream);
expect(await result).toEqual([
object6,
object5,
object4,
object1,
object2,
]);
});
test('should report stream errors', async () => {
const [stream, result] = StreamTest.toObjects();
const bufferStream = new BufferStream((err, _objs, cb) => {
expect(err.code).toEqual('E_ERROR');
cb(null, []);
}, {
objectMode: true,
});
StreamTest.fromErroredObjects(new YError('E_ERROR'), [
object1,
object2,
object3,
object4,
object5,
object6,
object7,
])
.on('error', (err) => {
bufferStream.emit('error', err);
})
.pipe(bufferStream)
.pipe(stream);
expect(await result).toEqual([]);
});
test('should emit callback errors', async () => {
const [stream, result] = StreamTest.toObjects();
let caughtError = new YError('E_UNEXPECTED_SUCCESS');
StreamTest.fromObjects([
object1,
object2,
object3,
object4,
object5,
object6,
object7,
])
.pipe(new BufferStream((err, _objs, cb) => {
if (err) {
cb(err, []);
return;
}
cb(new YError('E_ERROR'), []);
}, {
objectMode: true,
}))
.on('error', (err) => {
caughtError = err;
})
.pipe(stream);
expect(await result).toEqual([]);
expect(caughtError.code).toEqual('E_ERROR');
});
});
});
});
//# sourceMappingURL=index.test.js.map

File diff suppressed because one or more lines are too long