From 24b415ddaaddac481b361bd6f82a3a3d24c6f97b Mon Sep 17 00:00:00 2001 From: henryhai Date: Thu, 23 Mar 2023 19:40:08 +0700 Subject: [PATCH 1/2] chore: fix issues while receving streaming data --- dist/index.d.ts | 11 +++- dist/index.js | 135 ++++++++++++++++++++++++++++------------------ dist/index.js.map | 2 +- package-lock.json | 65 ++++++++-------------- package.json | 4 -- src/index.ts | 118 ++++++++++++++++++++-------------------- yarn.lock | 28 ++++------ 7 files changed, 184 insertions(+), 179 deletions(-) diff --git a/dist/index.d.ts b/dist/index.d.ts index d06ea4e..c3b6b58 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,4 +1,13 @@ export interface DynamicObject { [key: string]: any; } -export declare const parseGrpcData: (url: string, method: 'POST' | 'GET', headers: DynamicObject, body: DynamicObject, onChunkReceive: (data: any) => void, limiter?: number) => Promise; +export declare const parseGrpcData: (requestObject: { + url: string; + method: 'POST' | 'GET' | 'post' | 'get'; + headers: DynamicObject; + body?: DynamicObject; +}, dataObject: { + limiter?: number; + concatData?: boolean; + objectPrefix?: string; +}, onChunkReceive?: (data: any) => void, onFinish?: (data: any) => void, onError?: (e: any) => void) => Promise; diff --git a/dist/index.js b/dist/index.js index d665fc5..f5f98a3 100644 --- a/dist/index.js +++ b/dist/index.js @@ -35,77 +35,110 @@ var __generator = (this && this.__generator) || function (thisArg, body) { if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; } }; +var __spreadArrays = (this && this.__spreadArrays) || function () { + for (var s = 0, i = 0, il = arguments.length; i < il; i++) s += arguments[i].length; + for (var r = Array(s), k = 0, i = 0; i < il; i++) + for (var a = arguments[i], j = 0, jl = a.length; j < jl; j++, k++) + r[k] = a[j]; + return r; +}; Object.defineProperty(exports, "__esModule", { value: true }); exports.parseGrpcData = void 0; -var lodash_1 = require("lodash"); -exports.parseGrpcData = function (url, method, headers, body, onChunkReceive, limiter) { return __awaiter(void 0, void 0, void 0, function () { - var lastCutData, allData, limiterData, hasLimiter, res, reader, decoder, _a, value, done, chunk, chunkData, lastData, includedParsedData, firstData; - return __generator(this, function (_b) { - switch (_b.label) { +exports.parseGrpcData = function (requestObject, dataObject, onChunkReceive, onFinish, onError) { return __awaiter(void 0, void 0, void 0, function () { + var url, method, headers, _a, limiter, concatData, objectPrefix, allData, limiterData, hasLimiter, fetchProps, res, count, failedCount, reader, decoder, result, startObject, endObjRegex, parsedChunkData, _b, value, done, chunk, startIndex, endIndex, jsonStr, restOfStr, parsedChunk, pushedData, newLimiterData, returnedData, returnedData, error_1; + return __generator(this, function (_c) { + switch (_c.label) { case 0: - lastCutData = ''; + _c.trys.push([0, 5, , 6]); + console.time('parseGrpcData'); + url = requestObject.url, method = requestObject.method, headers = requestObject.headers; + _a = dataObject || {}, limiter = _a.limiter, concatData = _a.concatData, objectPrefix = _a.objectPrefix; allData = []; limiterData = []; hasLimiter = limiter && limiter > 0; - return [4 /*yield*/, fetch(url, { - method: method, - headers: headers, - body: JSON.stringify(body), + fetchProps = { + method: method, + headers: headers, + body: requestObject.body ? JSON.stringify(requestObject.body) : undefined, + }; + return [4 /*yield*/, fetch(url, fetchProps).catch(function (e) { + onError === null || onError === void 0 ? void 0 : onError(e); })]; case 1: - res = _b.sent(); - reader = res.body.getReader(); + res = _c.sent(); + count = 0; + failedCount = 0; + reader = (res === null || res === void 0 ? void 0 : res.body) ? res.body.getReader() : undefined; decoder = new TextDecoder('utf8'); - _b.label = 2; + result = ''; + startObject = '{"result":'; + endObjRegex = /}}\n+/g; + _c.label = 2; case 2: - if (!true) return [3 /*break*/, 4]; + if (!(true && reader)) return [3 /*break*/, 4]; + parsedChunkData = []; return [4 /*yield*/, reader.read()]; case 3: - _a = _b.sent(), value = _a.value, done = _a.done; + _b = _c.sent(), value = _b.value, done = _b.done; if (done) return [3 /*break*/, 4]; chunk = decoder.decode(value); - chunkData = chunk.split(/\r?\n/); - lastData = lodash_1.last(chunkData) || ''; - includedParsedData = void 0; - if (!lodash_1.isEmpty(lastCutData)) { - firstData = lodash_1.first(chunkData) || ''; - includedParsedData = lastCutData + firstData; - chunkData[0] = includedParsedData; - } - if (!lodash_1.isEmpty(lastData)) { - lastCutData = lastData; - delete chunkData[chunkData.length - 1]; - } - else { - lastCutData = ''; - } - lodash_1.forEach(chunkData, function (chunkStr) { - if (!lodash_1.isEmpty(chunkStr)) { - try { - var parsedChunk = JSON.parse(chunkStr); - allData.push(parsedChunk); - if (hasLimiter) { - limiterData.push(parsedChunk); - if (limiterData.length === limiter) { - var newLimiterData = lodash_1.clone(limiterData); - limiterData.splice(0, limiter); - onChunkReceive(newLimiterData); - } + result += chunk; + startIndex = result.indexOf(startObject); + endIndex = result.search(endObjRegex); + if (startIndex !== -1 && endIndex !== -1) { + jsonStr = result.substring(startIndex, endIndex + 2); + restOfStr = result.substring(endIndex + 2); + try { + parsedChunk = JSON.parse(jsonStr); + pushedData = objectPrefix + ? parsedChunk === null || parsedChunk === void 0 ? void 0 : parsedChunk.objectPrefix : parsedChunk; + allData.push(pushedData); + parsedChunkData.push(pushedData); + if (hasLimiter) { + limiterData.push(pushedData); + if (limiterData.length === limiter) { + newLimiterData = __spreadArrays(limiterData); + limiterData.splice(0, limiter); + returnedData = concatData + ? allData + : newLimiterData; + onChunkReceive(returnedData); } } - catch (_err) { - throw new Error('Failed to parse json chunk'); - } } - }); - if (!hasLimiter) - onChunkReceive(chunk); + catch (_err) { + // onError(_err); + console.log('Failed to parse json chunk'); + failedCount++; + } + finally { + result = restOfStr; + count++; + } + } return [3 /*break*/, 2]; case 4: - if (hasLimiter && limiterData.length > 0) - onChunkReceive(limiterData); - return [2 /*return*/]; + if (hasLimiter && limiterData.length > 0) { + returnedData = concatData ? allData : limiterData; + onChunkReceive(returnedData); + } + if (allData.length === 0) { + onChunkReceive([]); + } + if (onFinish) { + console.log("count: ", count); + console.log("failed count: ", failedCount); + console.timeEnd('parseGrpcData'); + onFinish(allData); + } + return [3 /*break*/, 6]; + case 5: + error_1 = _c.sent(); + if (onError) + onError(error_1); + return [3 /*break*/, 6]; + case 6: return [2 /*return*/]; } }); }); }; diff --git a/dist/index.js.map b/dist/index.js.map index 59ca501..41e5278 100644 --- a/dist/index.js.map +++ b/dist/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iCAA8D;AAMjD,QAAA,aAAa,GAAG,UACzB,GAAW,EACX,MAAsB,EACtB,OAAsB,EACtB,IAAmB,EACnB,cAAmC,EACnC,OAAgB;;;;;gBAEZ,WAAW,GAAG,EAAE,CAAC;gBACf,OAAO,GAAoB,EAAE,CAAC;gBAChC,WAAW,GAAoB,EAAE,CAAC;gBAChC,UAAU,GAAG,OAAO,IAAI,OAAO,GAAG,CAAC,CAAC;gBAEzB,qBAAM,KAAK,CAAC,GAAG,EAAE;wBAC9B,MAAM,QAAA;wBACN,OAAO,SAAA;wBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;qBAC7B,CAAC,EAAA;;gBAJI,GAAG,GAAQ,SAIf;gBACI,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChC,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;;;qBAC/B,IAAI;gBACiB,qBAAM,MAAM,CAAC,IAAI,EAAE,EAAA;;gBAArC,KAAkB,SAAmB,EAAnC,KAAK,WAAA,EAAE,IAAI,UAAA;gBACnB,IAAI,IAAI;oBAAE,wBAAM;gBACZ,KAAK,GAAW,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpC,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACjC,QAAQ,GAAG,aAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBACnC,kBAAkB,SAAA,CAAC;gBACvB,IAAI,CAAC,gBAAO,CAAC,WAAW,CAAC,EAAE;oBACjB,SAAS,GAAG,cAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;oBACzC,kBAAkB,GAAG,WAAW,GAAG,SAAS,CAAC;oBAC7C,SAAS,CAAC,CAAC,CAAC,GAAG,kBAAkB,CAAC;iBACrC;gBAED,IAAI,CAAC,gBAAO,CAAC,QAAQ,CAAC,EAAE;oBACpB,WAAW,GAAG,QAAQ,CAAC;oBACvB,OAAO,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;iBAC1C;qBAAM;oBACH,WAAW,GAAG,EAAE,CAAC;iBACpB;gBAED,gBAAO,CAAC,SAAS,EAAE,UAAC,QAAgB;oBAChC,IAAI,CAAC,gBAAO,CAAC,QAAQ,CAAC,EAAE;wBACpB,IAAI;4BACA,IAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;4BACzC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;4BAC1B,IAAI,UAAU,EAAE;gCACZ,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gCAC9B,IAAI,WAAW,CAAC,MAAM,KAAK,OAAO,EAAE;oCAChC,IAAM,cAAc,GAAG,cAAK,CAAC,WAAW,CAAC,CAAC;oCAC1C,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;oCAC/B,cAAc,CAAC,cAAc,CAAC,CAAC;iCAClC;6BACJ;yBACJ;wBAAC,OAAO,IAAI,EAAE;4BACX,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;yBACjD;qBACJ;gBACL,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,UAAU;oBAAE,cAAc,CAAC,KAAK,CAAC,CAAC;;;gBAE3C,IAAI,UAAU,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;oBAAE,cAAc,CAAC,WAAW,CAAC,CAAC;;;;KACzE,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIa,QAAA,aAAa,GAAG,UACzB,aAKC,EACD,UAIC,EACD,cAAoC,EACpC,QAA8B,EAC9B,OAA0B;;;;;;gBAGtB,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;gBACtB,GAAG,GAAsB,aAAa,IAAnC,EAAE,MAAM,GAAc,aAAa,OAA3B,EAAE,OAAO,GAAK,aAAa,QAAlB,CAAmB;gBACzC,KAAwC,UAAU,IAAI,EAAE,EAAtD,OAAO,aAAA,EAAE,UAAU,gBAAA,EAAE,YAAY,kBAAA,CAAsB;gBACzD,OAAO,GAAoB,EAAE,CAAC;gBAC9B,WAAW,GAAoB,EAAE,CAAC;gBAClC,UAAU,GAAG,OAAO,IAAI,OAAO,GAAG,CAAC,CAAC;gBAEpC,UAAU,GAAkB;oBAC9B,MAAM,QAAA;oBACN,OAAO,SAAA;oBACP,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;iBAC5E,CAAC;gBAEe,qBAAM,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,UAAC,CAAM;wBACvD,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAG,CAAC,EAAE;oBACjB,CAAC,CAAC,EAAA;;gBAFI,GAAG,GAAQ,SAEf;gBAEE,KAAK,GAAG,CAAC,CAAC;gBACV,WAAW,GAAG,CAAC,CAAC;gBACd,MAAM,GAAG,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,EAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBACtD,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;gBAEpC,MAAM,GAAG,EAAE,CAAC;gBACV,WAAW,GAAG,YAAY,CAAC;gBAC3B,WAAW,GAAG,QAAQ,CAAC;;;qBAEtB,CAAA,IAAI,IAAI,MAAM,CAAA;gBACX,eAAe,GAAoB,EAAE,CAAC;gBACpB,qBAAM,MAAM,CAAC,IAAI,EAAE,EAAA;;gBAArC,KAAkB,SAAmB,EAAnC,KAAK,WAAA,EAAE,IAAI,UAAA;gBACnB,IAAI,IAAI;oBAAE,wBAAM;gBACV,KAAK,GAAW,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC;gBAEV,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;gBACzC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBAE5C,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE;oBAChC,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;oBACrD,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;oBACjD,IAAI;wBACM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAClC,UAAU,GAAG,YAAY;4BAC3B,CAAC,CAAC,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,YAAY,CAC3B,CAAC,CAAC,WAAW,CAAC;wBAClB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;wBACzB,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;wBACjC,IAAI,UAAU,EAAE;4BACZ,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;4BAC7B,IAAI,WAAW,CAAC,MAAM,KAAK,OAAO,EAAE;gCAC1B,cAAc,kBAAO,WAAW,CAAC,CAAC;gCACxC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gCACzB,YAAY,GAAG,UAAU;oCAC3B,CAAC,CAAC,OAAO;oCACT,CAAC,CAAC,cAAc,CAAC;gCACrB,cAAc,CAAC,YAAY,CAAC,CAAC;6BAChC;yBACJ;qBACJ;oBAAC,OAAO,IAAI,EAAE;wBACX,iBAAiB;wBACjB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;wBAC1C,WAAW,EAAE,CAAC;qBACjB;4BAAS;wBACN,MAAM,GAAG,SAAS,CAAC;wBACnB,KAAK,EAAE,CAAC;qBACX;iBACJ;;;gBAGL,IAAI,UAAU,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC;oBACxD,cAAc,CAAC,YAAY,CAAC,CAAC;iBAChC;gBAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;oBACtB,cAAc,CAAC,EAAE,CAAC,CAAC;iBACtB;gBAED,IAAI,QAAQ,EAAE;oBACV,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;oBAC9B,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;oBACjC,QAAQ,CAAC,OAAO,CAAC,CAAC;iBACrB;;;;gBAED,IAAI,OAAO;oBAAE,OAAO,CAAC,OAAK,CAAC,CAAC;;;;;KAEnC,CAAC"} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0686ede..b08c54b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,10 +8,6 @@ "name": "@alphauslabs/grpc-chunk-parser", "version": "1.1.12", "license": "ISC", - "dependencies": { - "@types/lodash": "^4.14.172", - "lodash": "^4.17.21" - }, "devDependencies": { "@types/node": "^14.11.2", "ts-node": "^9.0.0", @@ -42,11 +38,6 @@ "js-tokens": "^4.0.0" } }, - "node_modules/@types/lodash": { - "version": "4.14.172", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.172.tgz", - "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==" - }, "node_modules/@types/node": { "version": "14.11.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", @@ -262,11 +253,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - }, "node_modules/make-error": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", @@ -274,9 +260,9 @@ "dev": true }, "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "dependencies": { "brace-expansion": "^1.1.7" @@ -286,10 +272,13 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/mkdirp": { "version": "0.5.5", @@ -322,9 +311,9 @@ } }, "node_modules/path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "node_modules/resolve": { @@ -509,11 +498,6 @@ "js-tokens": "^4.0.0" } }, - "@types/lodash": { - "version": "4.14.172", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.172.tgz", - "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==" - }, "@types/node": { "version": "14.11.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", @@ -692,11 +676,6 @@ "esprima": "^4.0.0" } }, - "lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - }, "make-error": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", @@ -704,18 +683,18 @@ "dev": true }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", "dev": true }, "mkdirp": { @@ -743,9 +722,9 @@ "dev": true }, "path-parse": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "resolve": { diff --git a/package.json b/package.json index ea152fd..f32b994 100644 --- a/package.json +++ b/package.json @@ -30,10 +30,6 @@ "tslint": "^6.1.3", "typescript": "~4.0.3" }, - "dependencies": { - "@types/lodash": "^4.14.172", - "lodash": "^4.17.21" - }, "engines": { "node": ">=8" }, diff --git a/src/index.ts b/src/index.ts index 9d62064..c481074 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,3 @@ -import { clone, first, forEach, get, isEmpty, last, isUndefined } from 'lodash'; - export interface DynamicObject { [key: string]: any; } @@ -20,63 +18,57 @@ export const parseGrpcData = async ( onFinish?: (data: any) => void, onError?: (e: any) => void ) => { - const { url, method, headers } = requestObject; - const limiter = get(dataObject, 'limiter'); - const concatData = get(dataObject, 'concatData'); - const objectPrefix = get(dataObject, 'objectPrefix'); - let lastCutData = ''; - const allData: DynamicObject[] = []; - const limiterData: DynamicObject[] = []; - const hasLimiter = limiter && limiter > 0; + try { + console.time('parseGrpcData'); + const { url, method, headers } = requestObject; + const { limiter, concatData, objectPrefix } = dataObject || {}; + const allData: DynamicObject[] = []; + const limiterData: DynamicObject[] = []; + const hasLimiter = limiter && limiter > 0; - const fetchProps: DynamicObject = { - method, - headers, - }; + const fetchProps: DynamicObject = { + method, + headers, + body: requestObject.body ? JSON.stringify(requestObject.body) : undefined, + }; - if (get(requestObject, 'body')) { - fetchProps.body = JSON.stringify(requestObject.body); - } + const res: any = await fetch(url, fetchProps).catch((e: any) => { + onError?.(e); + }); - const res: any = await fetch(url, fetchProps).catch((e: any) => { - if (onError) onError(e); - }); - const reader = get(res, 'body') ? res.body.getReader() : undefined; - const decoder = new TextDecoder('utf8'); - while (true && !isUndefined(reader)) { - const parsedChunkData: DynamicObject[] = []; - const { value, done } = await reader.read(); - if (done) break; - const chunk: string = decoder.decode(value); - const chunkData = chunk.split(/\r?\n/); - const lastData = chunkData.length > 1 ? last(chunkData) || '' : ''; - let includedParsedData; - if (!isEmpty(lastCutData)) { - const firstData = first(chunkData) || ''; - includedParsedData = lastCutData + firstData; - chunkData[0] = includedParsedData; - } + let count = 0; + let failedCount = 0; + const reader = res?.body ? res.body.getReader() : undefined; + const decoder = new TextDecoder('utf8'); - if (!isEmpty(lastData)) { - lastCutData = lastData; - delete chunkData[chunkData.length - 1]; - } else { - lastCutData = ''; - } + let result = ''; + const startObject = '{"result":'; + const endObjRegex = /}}\n+/g; + + while (true && reader) { + const parsedChunkData: DynamicObject[] = []; + const { value, done } = await reader.read(); + if (done) break; + const chunk: string = decoder.decode(value); + result += chunk; + + const startIndex = result.indexOf(startObject); + const endIndex = result.search(endObjRegex); - forEach(chunkData, (chunkStr: string) => { - if (!isEmpty(chunkStr)) { + if (startIndex !== -1 && endIndex !== -1) { + const jsonStr = result.substring(startIndex, endIndex + 2); + const restOfStr = result.substring(endIndex + 2); try { - const parsedChunk = JSON.parse(chunkStr); + const parsedChunk = JSON.parse(jsonStr); const pushedData = objectPrefix - ? get(parsedChunk, objectPrefix) + ? parsedChunk?.objectPrefix : parsedChunk; allData.push(pushedData); parsedChunkData.push(pushedData); if (hasLimiter) { limiterData.push(pushedData); if (limiterData.length === limiter) { - const newLimiterData = clone(limiterData); + const newLimiterData = [...limiterData]; limiterData.splice(0, limiter); const returnedData = concatData ? allData @@ -85,26 +77,32 @@ export const parseGrpcData = async ( } } } catch (_err) { + // onError(_err); console.log('Failed to parse json chunk'); + failedCount++; + } finally { + result = restOfStr; + count++; } } - }); + } - if (!hasLimiter) { - const returnedData = concatData ? allData : parsedChunkData; + if (hasLimiter && limiterData.length > 0) { + const returnedData = concatData ? allData : limiterData; onChunkReceive(returnedData); } - } - if (hasLimiter && limiterData.length > 0) { - const returnedData = concatData ? allData : limiterData; - onChunkReceive(returnedData); - } - if (isEmpty(allData)) { - onChunkReceive([]); - } + if (allData.length === 0) { + onChunkReceive([]); + } - if (onFinish) { - onFinish(allData); + if (onFinish) { + console.log("count: ", count); + console.log("failed count: ", failedCount); + console.timeEnd('parseGrpcData'); + onFinish(allData); + } + } catch (error) { + if (onError) onError(error); } -}; +}; \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index a605463..defa447 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18,11 +18,6 @@ esutils "^2.0.2" js-tokens "^4.0.0" -"@types/lodash@^4.14.172": - version "4.14.172" - resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.172.tgz" - integrity sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw== - "@types/node@^14.11.2": version "14.11.2" resolved "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz" @@ -169,27 +164,22 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" -lodash@^4.17.21: - version "4.17.21" - resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - make-error@^1.1.1: version "1.3.5" resolved "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz" integrity sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g== minimatch@^3.0.4: - version "3.0.4" - resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz" - integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + version "3.1.2" + resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" minimist@^1.2.5: - version "1.2.5" - resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + version "1.2.8" + resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== mkdirp@^0.5.3: version "0.5.5" @@ -211,9 +201,9 @@ path-is-absolute@^1.0.0: integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= path-parse@^1.0.6: - version "1.0.6" - resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz" - integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + version "1.0.7" + resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== resolve@^1.3.2: version "1.15.0" From 6bf4d855dab251f5fb6be449146b1cfe52218549 Mon Sep 17 00:00:00 2001 From: henryhai Date: Fri, 24 Mar 2023 08:23:31 +0700 Subject: [PATCH 2/2] chore: update version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f32b994..ca87601 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alphauslabs/grpc-chunk-parser", - "version": "1.1.12", + "version": "1.2.0", "description": "Typescript package for parsing grpc chunk data", "main": "dist/index.js", "types": "dist/index.d.ts",