From ab640020b733cab5d69bab8623d54b9e76c2577c Mon Sep 17 00:00:00 2001 From: SteVen Batten Date: Fri, 6 Oct 2023 10:01:40 -0700 Subject: [PATCH] remove node-fetch (#21) * remove node-fetch * version bump --- tokenizer_ts/package-lock.json | 125 ++------------------------- tokenizer_ts/package.json | 19 ++-- tokenizer_ts/src/tokenizerBuilder.ts | 1 - 3 files changed, 18 insertions(+), 127 deletions(-) diff --git a/tokenizer_ts/package-lock.json b/tokenizer_ts/package-lock.json index 1a3f246..0209d0e 100644 --- a/tokenizer_ts/package-lock.json +++ b/tokenizer_ts/package-lock.json @@ -9,13 +9,11 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "lru-cache": "^9.1.1", - "node-fetch": "^2.6.1" + "lru-cache": "^9.1.1" }, "devDependencies": { "@types/mocha": "^5.2.7", - "@types/node": "^14.14.35", - "@types/node-fetch": "^2.5.10", + "@types/node": "^18.18.3", "@typescript-eslint/eslint-plugin": "^4.14.1", "@typescript-eslint/parser": "^4.14.1", "eslint": "^7.19.0", @@ -24,6 +22,9 @@ "mocha": "^10.1.0", "prettier": "^1.19.1", "ts-node": "^10.9.1" + }, + "engines": { + "node": ">=18.0.0" } }, "node_modules/@babel/code-frame": { @@ -287,21 +288,11 @@ "dev": true }, "node_modules/@types/node": { - "version": "14.18.46", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.46.tgz", - "integrity": "sha512-n4yVT5FuY5NCcGHCosQSGvvCT74HhowymPN2OEcsHPw6U1NuxV9dvxWbrM2dnBukWjdMYzig1WfIkWdTTQJqng==", + "version": "18.18.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.3.tgz", + "integrity": "sha512-0OVfGupTl3NBFr8+iXpfZ8NR7jfFO+P1Q+IO/q0wbo02wYkP5gy36phojeYWpLQ6WAMjl+VfmqUk2YbUfp0irA==", "dev": true }, - "node_modules/@types/node-fetch": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.3.tgz", - "integrity": "sha512-ETTL1mOEdq/sxUtgtOhKjyB2Irra4cjxksvcMUR5Zr4n+PxVhsCD9WS46oPbHL3et9Zde7CNRr+WUNlcHvsX+w==", - "dev": true, - "dependencies": { - "@types/node": "*", - "form-data": "^3.0.0" - } - }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "4.33.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.33.0.tgz", @@ -584,12 +575,6 @@ "node": ">=8" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true - }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -726,18 +711,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -799,15 +772,6 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -1256,20 +1220,6 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, - "node_modules/form-data": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", - "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", - "dev": true, - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1641,27 +1591,6 @@ "node": ">=8.6" } }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -1807,25 +1736,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "node_modules/node-fetch": { - "version": "2.6.11", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.11.tgz", - "integrity": "sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -2345,11 +2255,6 @@ "node": ">=8.0" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/ts-node": { "version": "10.9.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", @@ -2494,20 +2399,6 @@ "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", "dev": true }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/tokenizer_ts/package.json b/tokenizer_ts/package.json index d557eec..a4c5025 100644 --- a/tokenizer_ts/package.json +++ b/tokenizer_ts/package.json @@ -2,7 +2,7 @@ "name": "@microsoft/tiktokenizer", "displayName": "tiktokenizer", "description": "Tokenizer for OpenAI large language models.", - "version": "1.0.0", + "version": "1.0.1", "author": { "name": "Microsoft Corporation" }, @@ -15,19 +15,21 @@ "bugs": { "url": "https://github.com/Microsoft/Tokenizer/issues" }, + "engines": { + "node": ">=18.0.0" + }, "main": "dist/index.js", "devDependencies": { "@types/mocha": "^5.2.7", - "@types/node": "^14.14.35", - "@types/node-fetch": "^2.5.10", - "mocha": "^10.1.0", - "ts-node": "^10.9.1", + "@types/node": "^18.18.3", "@typescript-eslint/eslint-plugin": "^4.14.1", "@typescript-eslint/parser": "^4.14.1", "eslint": "^7.19.0", "eslint-config-prettier": "^7.1.0", "eslint-plugin-prettier": "^3.3.1", - "prettier": "^1.19.1" + "mocha": "^10.1.0", + "prettier": "^1.19.1", + "ts-node": "^10.9.1" }, "scripts": { "test": "mocha -u tdd --require ts-node/register test/**/*.ts", @@ -36,7 +38,6 @@ "format": "prettier --write \"./**/*.{ts,tsx}\"" }, "dependencies": { - "lru-cache": "^9.1.1", - "node-fetch": "^2.6.1" + "lru-cache": "^9.1.1" } -} \ No newline at end of file +} diff --git a/tokenizer_ts/src/tokenizerBuilder.ts b/tokenizer_ts/src/tokenizerBuilder.ts index f397bf8..41719ae 100644 --- a/tokenizer_ts/src/tokenizerBuilder.ts +++ b/tokenizer_ts/src/tokenizerBuilder.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import fetch from "node-fetch"; import * as fs from "fs"; import * as path from "path"; import { TikTokenizer } from "./tikTokenizer";