perf: cache split regex

Author: johannschopplich

README.md

@@ -11,10 +11,10 @@ The following table shows the accuracy of the token count approximation for diff
 | --- | --- | --- | --- |
 | Short English text | 10 | 11 | 10.00% |
 | German text with umlauts | 56 | 49 | 12.50% |
-| Metamorphosis by Franz Kafka (English) | 31891 | 33928 | 6.39% |
-| Die Verwandlung by Franz Kafka (German) | 40620 | 34908 | 14.06% |
-| 道德經 by Laozi (Chinese) | 14386 | 11919 | 17.15% |
-| TypeScript ES5 Type Declarations (~ 4000 loc) | 47890 | 50464 | 5.37% |
+| Metamorphosis by Franz Kafka (English) | 31892 | 33930 | 6.39% |
+| Die Verwandlung by Franz Kafka (German) | 40621 | 34908 | 14.06% |
+| 道德經 by Laozi (Chinese) | 14387 | 11919 | 17.15% |
+| TypeScript ES5 Type Declarations (~ 4000 loc) | 48408 | 51688 | 6.78% |
 <!-- END GENERATED TOKEN COUNT TABLE -->
 
 ## Features

package.json

@@ -2,7 +2,7 @@
   "name": "tokenx",
   "type": "module",
   "version": "0.4.0",
-  "packageManager": "[email protected].2",
+  "packageManager": "[email protected].4",
   "description": "GPT token estimation and context size utilities without a full tokenizer",
   "author": "Johann Schopplich <[email protected]>",
   "license": "MIT",
@@ -52,14 +52,14 @@
     "release": "bumpp"
   },
   "devDependencies": {
-    "@antfu/eslint-config": "^3.9.2",
-    "@types/node": "^22.9.3",
+    "@antfu/eslint-config": "^3.11.2",
+    "@types/node": "^22.10.1",
     "bumpp": "^9.8.1",
     "eslint": "^9.15.0",
-    "gpt-tokenizer": "^2.6.2",
+    "gpt-tokenizer": "^2.7.0",
     "tsx": "^4.19.2",
-    "typescript": "^5.6.3",
+    "typescript": "^5.7.2",
     "unbuild": "^3.0.0-rc.11",
-    "vitest": "^2.1.5"
+    "vitest": "^2.1.6"
   }
 }