diff --git a/rust/src/parser/benches/utils/adapter.rs b/rust/src/parser/benches/utils/adapter.rs index 103ed18d093..20f8977441c 100644 --- a/rust/src/parser/benches/utils/adapter.rs +++ b/rust/src/parser/benches/utils/adapter.rs @@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer { fn token_to_id(&self, _token: &str) -> Option { Some(u32::MAX) } + + fn id_to_token(&self, _id: u32) -> Option { + Some("\u{FFFD}".to_string()) + } } /// Bench-only adapter that exposes a unified parser through the tool-parser diff --git a/rust/src/tokenizer/src/incremental.rs b/rust/src/tokenizer/src/incremental.rs index 462475fc918..52f87345485 100644 --- a/rust/src/tokenizer/src/incremental.rs +++ b/rust/src/tokenizer/src/incremental.rs @@ -170,6 +170,10 @@ mod tests { fn token_to_id(&self, _token: &str) -> Option { unreachable!() } + + fn id_to_token(&self, _id: u32) -> Option { + unreachable!() + } } #[test] @@ -248,6 +252,10 @@ mod tests { fn token_to_id(&self, _token: &str) -> Option { unreachable!() } + + fn id_to_token(&self, _id: u32) -> Option { + unreachable!() + } } #[test] @@ -320,6 +328,10 @@ mod tests { fn token_to_id(&self, _token: &str) -> Option { unreachable!() } + + fn id_to_token(&self, _id: u32) -> Option { + unreachable!() + } } /// Without the char-boundary fix, this panics slicing mid-emoji. diff --git a/rust/src/tokenizer/src/lib.rs b/rust/src/tokenizer/src/lib.rs index 11c76abdfb6..4f459450c61 100644 --- a/rust/src/tokenizer/src/lib.rs +++ b/rust/src/tokenizer/src/lib.rs @@ -30,11 +30,7 @@ pub trait Tokenizer: Send + Sync { fn token_to_id(&self, token: &str) -> Option; /// Convert one token ID into the tokenizer's raw token string. - fn id_to_token(&self, _id: u32) -> Option { - // TODO: remove default impl and require this to be implemented by all - // tokenizers - None - } + fn id_to_token(&self, id: u32) -> Option; /// Return the vocabulary size. Backends that cannot report it fall back to /// `usize::MAX`, an effectively unbounded value used only by test stubs.