feat(context): tambah context::tokens dengan tiktoken-rs
Ganti tiga heuristik char-count (/3 di shortsend, /4 di loop turn, /4 di status bar) yang saling tidak konsisten dengan satu BPE tokenizer nyata. tiktoken-rs membundel vocab lewat include_str! saat build, jadi tidak ada akses jaringan saat runtime.
This commit is contained in:
Generated
+28
@@ -196,6 +196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
@@ -807,6 +808,17 @@ dependencies = [
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
|
||||
dependencies = [
|
||||
"bit-set 0.8.0",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fast-srgb8"
|
||||
version = "1.0.0"
|
||||
@@ -3521,6 +3533,21 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiktoken-rs"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"bstr",
|
||||
"fancy-regex 0.17.0",
|
||||
"lazy_static",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.53"
|
||||
@@ -4487,6 +4514,7 @@ dependencies = [
|
||||
"sha2 0.11.0",
|
||||
"similar",
|
||||
"syntect",
|
||||
"tiktoken-rs",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
|
||||
Reference in New Issue
Block a user