123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110 |
- // Split chunks line-wise for generators passed to the `std*` options
- export const getSplitLinesGenerator = (binary, preserveNewlines, skipped, state) => binary || skipped
- ? undefined
- : initializeSplitLines(preserveNewlines, state);
- // Same but for synchronous methods
- export const splitLinesSync = (chunk, preserveNewlines, objectMode) => objectMode
- ? chunk.flatMap(item => splitLinesItemSync(item, preserveNewlines))
- : splitLinesItemSync(chunk, preserveNewlines);
- const splitLinesItemSync = (chunk, preserveNewlines) => {
- const {transform, final} = initializeSplitLines(preserveNewlines, {});
- return [...transform(chunk), ...final()];
- };
- const initializeSplitLines = (preserveNewlines, state) => {
- state.previousChunks = '';
- return {
- transform: splitGenerator.bind(undefined, state, preserveNewlines),
- final: linesFinal.bind(undefined, state),
- };
- };
- // This imperative logic is much faster than using `String.split()` and uses very low memory.
- const splitGenerator = function * (state, preserveNewlines, chunk) {
- if (typeof chunk !== 'string') {
- yield chunk;
- return;
- }
- let {previousChunks} = state;
- let start = -1;
- for (let end = 0; end < chunk.length; end += 1) {
- if (chunk[end] === '\n') {
- const newlineLength = getNewlineLength(chunk, end, preserveNewlines, state);
- let line = chunk.slice(start + 1, end + 1 - newlineLength);
- if (previousChunks.length > 0) {
- line = concatString(previousChunks, line);
- previousChunks = '';
- }
- yield line;
- start = end;
- }
- }
- if (start !== chunk.length - 1) {
- previousChunks = concatString(previousChunks, chunk.slice(start + 1));
- }
- state.previousChunks = previousChunks;
- };
- const getNewlineLength = (chunk, end, preserveNewlines, state) => {
- if (preserveNewlines) {
- return 0;
- }
- state.isWindowsNewline = end !== 0 && chunk[end - 1] === '\r';
- return state.isWindowsNewline ? 2 : 1;
- };
- const linesFinal = function * ({previousChunks}) {
- if (previousChunks.length > 0) {
- yield previousChunks;
- }
- };
- // Unless `preserveNewlines: true` is used, we strip the newline of each line.
- // This re-adds them after the user `transform` code has run.
- export const getAppendNewlineGenerator = ({binary, preserveNewlines, readableObjectMode, state}) => binary || preserveNewlines || readableObjectMode
- ? undefined
- : {transform: appendNewlineGenerator.bind(undefined, state)};
- const appendNewlineGenerator = function * ({isWindowsNewline = false}, chunk) {
- const {unixNewline, windowsNewline, LF, concatBytes} = typeof chunk === 'string' ? linesStringInfo : linesUint8ArrayInfo;
- if (chunk.at(-1) === LF) {
- yield chunk;
- return;
- }
- const newline = isWindowsNewline ? windowsNewline : unixNewline;
- yield concatBytes(chunk, newline);
- };
- const concatString = (firstChunk, secondChunk) => `${firstChunk}${secondChunk}`;
- const linesStringInfo = {
- windowsNewline: '\r\n',
- unixNewline: '\n',
- LF: '\n',
- concatBytes: concatString,
- };
- const concatUint8Array = (firstChunk, secondChunk) => {
- const chunk = new Uint8Array(firstChunk.length + secondChunk.length);
- chunk.set(firstChunk, 0);
- chunk.set(secondChunk, firstChunk.length);
- return chunk;
- };
- const linesUint8ArrayInfo = {
- windowsNewline: new Uint8Array([0x0D, 0x0A]),
- unixNewline: new Uint8Array([0x0A]),
- LF: 0x0A,
- concatBytes: concatUint8Array,
- };
|