feat(context): tambah context::tokens dengan tiktoken-rs

Ganti tiga heuristik char-count (/3 di shortsend, /4 di loop turn, /4
di status bar) yang saling tidak konsisten dengan satu BPE tokenizer
nyata. tiktoken-rs membundel vocab lewat include_str! saat build, jadi
tidak ada akses jaringan saat runtime.
This commit is contained in:
asepharyana
2026-07-16 07:49:43 +07:00
parent 8bb697a53f
commit c219b6ec58
5 changed files with 112 additions and 0 deletions
Generated
+28
View File
@@ -196,6 +196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79"
dependencies = [
"memchr",
"regex-automata",
"serde_core",
]
@@ -807,6 +808,17 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "fancy-regex"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
dependencies = [
"bit-set 0.8.0",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "fast-srgb8"
version = "1.0.0"
@@ -3521,6 +3533,21 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "tiktoken-rs"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"
dependencies = [
"anyhow",
"base64",
"bstr",
"fancy-regex 0.17.0",
"lazy_static",
"regex",
"rustc-hash",
]
[[package]]
name = "time"
version = "0.3.53"
@@ -4487,6 +4514,7 @@ dependencies = [
"sha2 0.11.0",
"similar",
"syntect",
"tiktoken-rs",
"tokio",
"tracing",
"tracing-subscriber",