remove default impl on id_to_token

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

rust/src/parser/benches/utils/adapter.rs

@@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer {
     fn token_to_id(&self, _token: &str) -> Option<u32> {
         Some(u32::MAX)
     }
+
+    fn id_to_token(&self, _id: u32) -> Option<String> {
+        Some("\u{FFFD}".to_string())
+    }
 }
 
 /// Bench-only adapter that exposes a unified parser through the tool-parser

rust/src/tokenizer/src/incremental.rs

@@ -170,6 +170,10 @@ mod tests {
         fn token_to_id(&self, _token: &str) -> Option<u32> {
             unreachable!()
         }
+
+        fn id_to_token(&self, _id: u32) -> Option<String> {
+            unreachable!()
+        }
     }
 
     #[test]
@@ -248,6 +252,10 @@ mod tests {
         fn token_to_id(&self, _token: &str) -> Option<u32> {
             unreachable!()
         }
+
+        fn id_to_token(&self, _id: u32) -> Option<String> {
+            unreachable!()
+        }
     }
 
     #[test]
@@ -320,6 +328,10 @@ mod tests {
         fn token_to_id(&self, _token: &str) -> Option<u32> {
             unreachable!()
         }
+
+        fn id_to_token(&self, _id: u32) -> Option<String> {
+            unreachable!()
+        }
     }
 
     /// Without the char-boundary fix, this panics slicing mid-emoji.

rust/src/tokenizer/src/lib.rs

@@ -30,11 +30,7 @@ pub trait Tokenizer: Send + Sync {
     fn token_to_id(&self, token: &str) -> Option<u32>;
 
     /// Convert one token ID into the tokenizer's raw token string.
-    fn id_to_token(&self, _id: u32) -> Option<String> {
-        // TODO: remove default impl and require this to be implemented by all
-        // tokenizers
-        None
-    }
+    fn id_to_token(&self, id: u32) -> Option<String>;
 
     /// Return the vocabulary size. Backends that cannot report it fall back to
     /// `usize::MAX`, an effectively unbounded value used only by test stubs.