MetaUtil({ 'delay': (process.argv[5] || 100), 'start': Number(process.argv[3]), 'end': Number(process.argv[4]) }).pipe(through( write, //Write function, transform and push buffer at threshold function() { if (bulkBuffer.length > 0) { pushToES(bulkBuffer, 'exit'); } //End function: push contents of buffer and hang up } ))
/* Streaming examples */ var genThroughStreamCounter = function() { var dataCounter = 0; var throughStream = through( function write(data) { dataCounter += data.length; console.log("ThroughStreamCounter: Writing through data ", data.length); this.queue(data); }, function end () { console.log("ThroughStreamCounter: END. Wrote through data ", dataCounter); this.queue(null); }); return throughStream; }
function capture() { const child = exec(cmd, {async: true, silent: true}) child.stdout .pipe(through(function write(data) { data = repairJsonString(data) this.queue(data) })) .pipe(JSONStream.parse('*').on('error', () => { // retry capture() })) .on('error', () => {}) .on('data', processChunk) }
MetaUtil().pipe(through( write, //Write function, transform and push buffer at threshold function() { process.exit(0) } ))
function parseCSVStream(s = new Set()) { let isFirst = true; let ref = { fields: [] }; return through(function (chunk) { let str = chunk.toString(); let parsed = parseCSV(str, {header: isFirst}); let strarr = str.split("\n"); let plen = (isFirst && strarr.length - 1 || strarr.length) - parsed.length; if (!isFirst) { chunk = Buffer.concat([Buffer.from([...s].join("\n")), chunk]).toString(); parsed = parseCSV(str, {header: isFirst}); s = new Set(); } strarr.splice(strarr.length - plen).forEach((value => s.add(value))); chunkBuilder.call(this, isFirst, ref, str, parsed); isFirst = false; }) }
pull.pipe(through(function(msg){ if(++recv > 10000){
through( function(chunk) { template += chunk.toString(); }, function() { options.filename = fileName; options.globalReact = true; try { template = jade.compileClient(template, options); } catch (e) { this.emit('error', e); return; } var moduleBody = 'var React = require("react");\n' + 'module.exports = ' + template; this.queue(moduleBody); this.queue(null); } )
.pipe(through(write, () => this.res.end())) .pipe(this.stringifier);
.snapshot() .cursor({batchSize: 1000}) .pipe(through(function(doc) { const row = doc.toObject({getters: true, virtuals: false});
var passThroughStream = through(
function parseTSVStream(s = new Set()) { let isFirst = true; let ref = { fields: [] }; return through(function (chunk) { let str = chunk.toString(); let parsed = parseTSV(str, {header: isFirst}); let strarr = str.split("\n"); let plen = (isFirst && strarr.length - 1 || strarr.length) - parsed.length; if (!isFirst) { chunk = Buffer.concat([Buffer.from([...s].join("\n")), chunk]).toString(); parsed = parseTSV(str, {header: isFirst}); s = new Set(); } strarr.splice(strarr.length - plen).forEach((value => s.add(value))); chunkBuilder.call(this, isFirst, ref, str, parsed); isFirst = false; }); }