yuchenlin commited on
Commit
348b4fc
1 Parent(s): f47abfa

add kwargs

Browse files
Files changed (1) hide show
  1. tokenization_rex_qwen2.py +3 -3
tokenization_rex_qwen2.py CHANGED
@@ -41,17 +41,17 @@ class RexQwen2Tokenizer(Qwen2Tokenizer):
41
 
42
 
43
  # redefine the _tokenize method
44
- def tokenize(self, text):
45
  # find the index for first user query
46
  if self.user_prefix not in text or self.rex_size < 1:
47
  # the query is not wrapped with chat template yet
48
  # raise NotImplementedError
49
- return super().tokenize(text)
50
  start_index = text.index(self.user_prefix)
51
  rex_chat_history_tokens = self._rex_query(text[start_index+len(self.user_prefix):])
52
  rex_text = text[:start_index] + rex_chat_history_tokens + text[start_index:]
53
  # print(rex_text)
54
- tokens = super().tokenize(rex_text)
55
  return tokens
56
 
57
 
 
41
 
42
 
43
  # redefine the _tokenize method
44
+ def tokenize(self, text, **kwargs):
45
  # find the index for first user query
46
  if self.user_prefix not in text or self.rex_size < 1:
47
  # the query is not wrapped with chat template yet
48
  # raise NotImplementedError
49
+ return super().tokenize(text, **kwargs)
50
  start_index = text.index(self.user_prefix)
51
  rex_chat_history_tokens = self._rex_query(text[start_index+len(self.user_prefix):])
52
  rex_text = text[:start_index] + rex_chat_history_tokens + text[start_index:]
53
  # print(rex_text)
54
+ tokens = super().tokenize(rex_text, **kwargs)
55
  return tokens
56
 
57