From c62832b37a5a4793a15aa2a537e2d0605a9fac0d Mon Sep 17 00:00:00 2001 From: sunyao13 Date: Sat, 30 Aug 2025 08:55:23 +0000 Subject: [PATCH] =?UTF-8?q?update=20ais=5Fbench/benchmark/utils/tokenizer.?= =?UTF-8?q?py.=20=E8=A1=A5=E5=85=85=E7=BC=BA=E5=A4=B1=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8Cmindie=E6=9C=8D=E5=8A=A1=E5=8C=96=E5=90=8E=E7=AB=AF?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E5=8A=9F=E8=83=BD=E6=89=A7=E8=A1=8C=E6=88=90?= =?UTF-8?q?=E5=8A=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: sunyao13 --- ais_bench/benchmark/utils/tokenizer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ais_bench/benchmark/utils/tokenizer.py b/ais_bench/benchmark/utils/tokenizer.py index e9cb76d65b..f039f5d883 100644 --- a/ais_bench/benchmark/utils/tokenizer.py +++ b/ais_bench/benchmark/utils/tokenizer.py @@ -65,6 +65,9 @@ class MindformersTokenizer(Tokenizer): def decode(self, token_ids: list, skip_special_tokens=False) -> str: return self.tokenizer_model.decode(token_ids, skip_special_tokens=skip_special_tokens) + def batch_encode_plus(self, batch_text_or_text_pairs, *args, **kwargs): + return self.tokenizer_model.batch_encode_plus(batch_text_or_text_pairs, *args, **kwargs) + class BenchmarkTokenizer: def __init__(self, model_path: str, tokenizer_type: str = None, trust_remote_code: bool = False, **kwargs): -- Gitee