From 918dbde1612158ccbfec8030e56968f4b249bdf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Gu=C3=A9rout?= Date: Sun, 26 Nov 2023 01:46:20 +0100 Subject: [PATCH] Add first types --- .mocharc.json | 5 +- .tools/git-hooks/pre-commit | 6 -- .tools/hooks.sh | 19 ++++ package.json | 4 +- src/accumulateData.ts | 8 +- src/utils/decorateWithPromise.ts | 6 +- src/utils/toAsyncIterator.ts | 11 +- src/writeData.ts | 176 ++++++++++++++++--------------- tsconfig.json | 2 +- tsconfig.test.json | 5 +- 10 files changed, 135 insertions(+), 107 deletions(-) delete mode 100755 .tools/git-hooks/pre-commit create mode 100644 .tools/hooks.sh diff --git a/.mocharc.json b/.mocharc.json index de04a6c..f248fe3 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -1,7 +1,4 @@ { "extension": ["ts"], - "node-option": [ - "experimental-specifier-resolution=node", - "loader=ts-node/esm" - ] + "node-option": ["loader=ts-node/esm"] } diff --git a/.tools/git-hooks/pre-commit b/.tools/git-hooks/pre-commit deleted file mode 100755 index 3ced84b..0000000 --- a/.tools/git-hooks/pre-commit +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -npm test -npm run lint -npm run build diff --git a/.tools/hooks.sh b/.tools/hooks.sh new file mode 100644 index 0000000..0c9ccd3 --- /dev/null +++ b/.tools/hooks.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." +readonly PRE_COMMIT_HOOK="${PROJECT_DIR}/.git/hooks/pre-commit" + +cat <<'EOF' >"${PRE_COMMIT_HOOK}" +#!/usr/bin/env bash +set -euo pipefail +# Do not edit. This file has been generated by oleoduc + +npm test +npm run lint +npm run build + +EOF + +chmod +x "${PRE_COMMIT_HOOK}" +echo "pre-push hooks installed in ${PRE_COMMIT_HOOK}" diff --git a/package.json b/package.json index 31fb300..2b3e377 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,8 @@ }, "scripts": { "build": "bash .tools/build.sh", - "hooks": "git config core.hooksPath .tools/git-hooks && chmod +x .tools/git-hooks/*", - "test": "mocha test/", + "hooks": "bash .tools/hooks.sh", + "test": "TS_NODE_PROJECT=tsconfig.test.json mocha test/", "coverage": "nyc --temp-dir .coverage/.nyc_output --report-dir .coverage --reporter=lcov --reporter=html npm test", "lint": "eslint src/index.ts test/", "release": "bash .tools/release.sh master", diff --git a/src/accumulateData.ts b/src/accumulateData.ts index ceeba99..3784e63 100644 --- a/src/accumulateData.ts +++ b/src/accumulateData.ts @@ -1,6 +1,10 @@ -import { Transform } from "stream"; +import { Transform, TransformOptions } from "stream"; -export function accumulateData(accumulate, options: any = {}) { +type AccumulateDateOptions = { + accumulator?: unknown; +} & TransformOptions; + +export function accumulateData(accumulate, options: AccumulateDateOptions = {}) { const { accumulator, ...rest } = options; let acc = accumulator === undefined ? null : accumulator; let flushed = false; diff --git a/src/utils/decorateWithPromise.ts b/src/utils/decorateWithPromise.ts index b4d3c17..293fcde 100644 --- a/src/utils/decorateWithPromise.ts +++ b/src/utils/decorateWithPromise.ts @@ -1,12 +1,10 @@ export function decorateWithPromise(stream, promise) { const descriptors = ["then", "catch", "finally"].map((property) => { - return [property, Reflect.getOwnPropertyDescriptor(Promise.prototype, property)]; + return { property, descriptor: Reflect.getOwnPropertyDescriptor(Promise.prototype, property) }; }); - for (const [property, descriptor] of descriptors) { - // @ts-ignore + for (const { property, descriptor } of descriptors) { const value = (...args) => Reflect.apply(descriptor.value, promise, args); - // @ts-ignore Reflect.defineProperty(stream, property, { ...descriptor, value }); } return stream; diff --git a/src/utils/toAsyncIterator.ts b/src/utils/toAsyncIterator.ts index 2640a6d..5d17a8f 100644 --- a/src/utils/toAsyncIterator.ts +++ b/src/utils/toAsyncIterator.ts @@ -1,5 +1,9 @@ import { Readable } from "stream"; +type ToAsyncIteratorOptions = { + chunkSize?: number; +}; + /** * Adapted from * - https://iximiuz.com/en/posts/nodejs-readable-streams-distilled/ @@ -8,11 +12,12 @@ import { Readable } from "stream"; * @param options * @returns {AsyncGenerator<*, void, *>} */ -export async function* toAsyncIterator(stream, options: any = {}) { +export async function* toAsyncIterator(stream, options: ToAsyncIteratorOptions = {}) { const chunkSize = options.chunkSize || 1; + if (typeof stream.read !== "function") { - // @ts-ignore - stream = Readable.wrap(stream, { objectMode: true }); + //FIXME seems not used + stream = new Readable().wrap(stream); } let ended = false; diff --git a/src/writeData.ts b/src/writeData.ts index efa6370..9fcccf0 100644 --- a/src/writeData.ts +++ b/src/writeData.ts @@ -1,103 +1,111 @@ import { Writable } from "stream"; import cyclist from "cyclist"; -import { inherits } from "util"; -const ParallelWrite = function (maxParallel, opts, onWrite) { - if (!(this instanceof ParallelWrite)) { - // @ts-ignore - return new ParallelWrite(maxParallel, opts, onWrite); - } +type ParallelWriteOptions = { + parallel?: number; +}; - if (typeof maxParallel === "function") { - onWrite = maxParallel; - opts = null; - maxParallel = 1; - } - if (typeof opts === "function") { - onWrite = opts; - opts = null; - } +class ParallelWrite extends Writable { + private _maxParallel: number; + private _onWrite: (chunk: T, callback: (e: Error, data: T) => void) => void; + private _destroyed: boolean; + private _flushed: boolean; + private _ordered: boolean; + private _buffer: cyclist | []; + private _top: number; + private _bottom: number; + private _ondrain: () => void; + + constructor(maxParallel, opts, onWrite) { + super(); + if (typeof maxParallel === "function") { + onWrite = maxParallel; + opts = null; + maxParallel = 1; + } + if (typeof opts === "function") { + onWrite = opts; + opts = null; + } - if (!opts) opts = {}; - if (!opts.highWaterMark) opts.highWaterMark = Math.max(maxParallel, 16); - if (opts.objectMode !== false) opts.objectMode = true; - - Writable.call(this, opts); - - this._maxParallel = maxParallel; - this._onWrite = onWrite; - this._destroyed = false; - this._flushed = false; - this._ordered = opts.ordered !== false; - this._buffer = this._ordered ? cyclist(maxParallel) : []; - this._top = 0; - this._bottom = 0; - this._ondrain = null; -}; + if (!opts) opts = {}; + if (!opts.highWaterMark) opts.highWaterMark = Math.max(maxParallel, 16); + if (opts.objectMode !== false) opts.objectMode = true; + + Writable.call(this, opts); + + this._maxParallel = maxParallel; + this._onWrite = onWrite; + this._destroyed = false; + this._flushed = false; + this._ordered = opts.ordered !== false; + this._buffer = this._ordered ? cyclist(maxParallel) : []; + this._top = 0; + this._bottom = 0; + this._ondrain = null; + } -inherits(ParallelWrite, Writable); + _destroy() { + if (this._destroyed) return; + this._destroyed = true; + this.emit("close"); + } -ParallelWrite.prototype._destroy = function () { - if (this._destroyed) return; - this._destroyed = true; - this.emit("close"); -}; + _write(chunk, enc, callback) { + const pos = this._top++; + + this._onWrite(chunk, (err, data) => { + if (this._destroyed) return; + if (err) { + this.emit("error", err); + this.destroy(); + return; + } + if (this._ordered) { + this._buffer.put(pos, data === undefined || data === null ? null : data); + } else { + this._buffer.push(data); + } + this._drain(); + }); + + if (this._top - this._bottom < this._maxParallel) return callback(); + this._ondrain = callback; + } -ParallelWrite.prototype._write = function (chunk, enc, callback) { - const self = this; - const pos = this._top++; + _final(callback) { + this._flushed = true; + this._ondrain = callback; + this._drain(); + } - this._onWrite(chunk, function (err, data) { - if (self._destroyed) return; - if (err) { - self.emit("error", err); - self.destroy(); - return; - } - if (self._ordered) { - self._buffer.put(pos, data === undefined || data === null ? null : data); + _drain() { + if (this._ordered) { + while (this._buffer.get(this._bottom) !== undefined) { + this._buffer.del(this._bottom++); + } } else { - self._buffer.push(data); + while (this._buffer.length > 0) { + this._buffer.pop(); + this._bottom++; + } } - self._drain(); - }); - - if (this._top - this._bottom < this._maxParallel) return callback(); - this._ondrain = callback; -}; -ParallelWrite.prototype._final = function (callback) { - this._flushed = true; - this._ondrain = callback; - this._drain(); -}; + if (!this._drained() || !this._ondrain) return; -ParallelWrite.prototype._drain = function () { - if (this._ordered) { - while (this._buffer.get(this._bottom) !== undefined) { - this._buffer.del(this._bottom++); - } - } else { - while (this._buffer.length > 0) { - this._buffer.pop(); - this._bottom++; - } + const ondrain = this._ondrain; + this._ondrain = null; + ondrain(); } - if (!this._drained() || !this._ondrain) return; - - const ondrain = this._ondrain; - this._ondrain = null; - ondrain(); -}; - -ParallelWrite.prototype._drained = function () { - const diff = this._top - this._bottom; - return this._flushed ? !diff : diff < this._maxParallel; -}; + _drained() { + const diff = this._top - this._bottom; + return this._flushed ? !diff : diff < this._maxParallel; + } +} -export function writeData(handleChunk, options: any = {}) { - return ParallelWrite(options.parallel || 1, { objectMode: true }, async (chunk, callback) => { +export function writeData(handleChunk, options: ParallelWriteOptions = {}) { + return new ParallelWrite(options.parallel || 1, { objectMode: true }, async (chunk, callback) => { try { const res = await handleChunk(chunk); callback(null, res); diff --git a/tsconfig.json b/tsconfig.json index ba17f1d..a8d539b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,7 +9,7 @@ "rootDirs": ["src"], "target": "esnext", "lib": ["esnext"], - "types": ["node", "mocha"] + "types": ["node"] }, "exclude": ["node_modules", "dist"], "include": ["src"] diff --git a/tsconfig.test.json b/tsconfig.test.json index 3f6c404..d620e7c 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -4,5 +4,8 @@ "rootDirs": ["./test"], "types": ["node", "mocha"] }, - "include": ["src", "test"] + "include": ["src", "test"], + "ts-node": { + "transpileOnly": true + } }