+
    ~j                         R t ^ RIHtHtHtHtHtHt ^ RIH	t	 ^RI
Ht ^RIHt ]P                  ! ]4      tRRRRR	R
/tRR/tRt ! R R]4      tR.tR# )zTokenization classes for Qwen2.)
AddedTokenRegex	Tokenizerdecodersnormalizerspre_tokenizers)BPE)TokenizersBackend)logging
vocab_filez
vocab.jsonmerges_filez
merges.txttokenizer_fileztokenizer.jsonzqwen/qwen-tokenizeri   zn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+c                   R   a a ] tR t^$t o]tRR.t]tRV3R lV 3R lllt	Rt
VtV ;t# )Qwen2Tokenizer	input_idsattention_maskc          
         < V ^8  d   QhRS[ S[S[ S[3,          ,          R,          RS[ S[S[ ,          ,          R,          RS[ RS[ RS[ /# )   vocabNmerges	unk_token	eos_token	pad_token)strdictintlist)format__classdict__s   "ځ/Users/mitch_tango/dev/rabbit-r1-livekit/agent/.venv/lib/python3.14/site-packages/transformers/models/qwen2/tokenization_qwen2.py__annotate__Qwen2Tokenizer.__annotate__)   sc     9` 9`T#s(^#d*9` d3i$&9` 	9` 9` 9`    c                  < Ve   TMRV n         Ve   TMR^ /V n        T;'       g    . V n        \        \	        V P                  V P                  R R RRRRR7      4      V n        \        P                  ! 4       V P
                  n        \        P                  ! 4       V P
                  n        \        P                  ! \        P                  ! \        \         4      RRR7      \        P                  ! V P                   RR7      .4      V P
                  n        \$        S
V `L  ! RRVR	VR
VRVRV/VB  T P)                  V P*                   U	u. uF  p	\-        V	RR7      NK  	  up	4       R # u up	i )NF<|endoftext|> )r   r   dropoutr   continuing_subword_prefixend_of_word_suffixfuse_unkbyte_fallbackisolated)behaviorinvert)add_prefix_space	use_regexr   	bos_tokenr   r   r.   T)special )r.   _vocab_mergesr   r   
_tokenizerr   	ByteLeveldecoderr   NFC
normalizerr   SequenceSplitr   PRETOKENIZE_REGEXpre_tokenizersuper__init__
add_tokensall_special_tokensr   )selfr   r   r   r0   r   r   r.   kwargstoken	__class__s   &&&&&&&&, r   r?   Qwen2Tokenizer.__init__)   so    5E4P 0V[      	 ||#kk||*,#%#	
 #+"4"4"6%0__%6"(6(?(?$$+,' 
 ((%)%:%:#
)
% 	 	
	
	
  	
  		

 .	
 	
 	dF]F]^F]UE48F]^_^s   
E))r4   r5   r3   r.   )NNr$   Nr$   r$   N)__name__
__module____qualname____firstlineno__VOCAB_FILES_NAMESvocab_files_namesmodel_input_namesr   modelr?   __static_attributes____classdictcell____classcell__)rE   r   s   @@r   r   r   $   s.     )$&67E9` 9` 9`r"   r   N)__doc__
tokenizersr   r   r   r   r   r   tokenizers.modelsr   tokenization_utils_tokenizersr	   utilsr
   
get_loggerrG   loggerrK   MAX_MODEL_INPUT_SIZESr<   r   __all__r2   r"   r   <module>r[      su    & Z Z ! >  
		H	% ,<&  /6  J >`& >`B 
r"   