+
    ~j                     |    ^ RI HtHtHtHtHtHt ^ RIHtH	t	H
t
HtHt ^ RIHt ^ RIHt ^RIHt  ! R R]4      tR# )	    )DictIteratorListOptionalTupleUnion)
AddedToken	Tokenizerdecoderspre_tokenizerstrainers)BPE)NFKC)BaseTokenizerc                      a a ] tR t^
t oRtRV3R lV 3R lllt]V3R lR l4       tR	^R.R
. R3V3R lR lltR	^R.R
. RR3V3R lR llt	Rt
VtV ;t# )SentencePieceBPETokenizerzjSentencePiece BPE Tokenizer

Represents the BPE algorithm, with the pretokenization used by SentencePiece
N<unk>Tc                  < V ^8  d   QhRS[ S[S[S[S[S[3,          3,          ,          RS[ S[S[S[S[S[S[3,          ,          3,          ,          RS[S[S[3,          RS[RS[RS[ S[	,          RS[ S[,          /# )   vocabmerges	unk_tokenreplacementadd_prefix_spacedropoutfuse_unk)
r   r   strr   intr   r   r	   boolfloat)format__classdict__s   "ځ/Users/mitch_tango/dev/rabbit-r1-livekit/agent/.venv/lib/python3.14/site-packages/tokenizers/implementations/sentencepiece_bpe.py__annotate__&SentencePieceBPETokenizer.__annotate__   s     0 0c4S>1230 sDsCx$99:;0 j)	0
 0 0 %0 4.0    c           
       < Ve   Ve   \        \        WWcVR7      4      pM\        \        WcVR7      4      pVP                  \        V4      4      e   VP	                  \        V4      .4       \        4       Vn        V'       d   RMRp	\        P                  ! WIR7      Vn	        \        P                  ! WIR7      Vn        RRRVRVR	VR
V/p
\        SV `5  W4       R # )N)r   r   r   alwaysnever)r   prepend_schememodelSentencePieceBPEr   r   r   r   )r
   r   token_to_idr   add_special_tokensr   
normalizerr   	Metaspacepre_tokenizerr   decodersuper__init__)selfr   r   r   r   r   r   r   	tokenizerr*   
parameters	__class__s   &&&&&&&&   r#   r4   "SentencePieceBPETokenizer.__init__   s     !3!#eWdl"mnI!#gU]"^_I  Y0<((#i.)9:#v	%57"0":":{"r	$..;f	 '; 0w

 	/r&   c                &   < V ^8  d   QhRS[ RS[ /# )r   vocab_filenamemerges_filename)r   )r!   r"   s   "r#   r$   r%   2   s     B B# B Br&   c                J    \         P                  ! W4      w  r4\        W43/ VB # )N)r   	read_filer   )r;   r<   kwargsr   r   s   &&,  r#   	from_file#SentencePieceBPETokenizer.from_file1   s"    nF(A&AAr&   i0u  i  c                   < V ^8  d   QhRS[ S[S[S[,          3,          RS[RS[RS[S[ S[S[3,          ,          RS[RS[S[,          RS[/# )r   files
vocab_sizemin_frequencyspecial_tokenslimit_alphabetinitial_alphabetshow_progress)r   r   r   r   r	   r   )r!   r"   s   "r#   r$   r%   6   sp     6 6S$s)^$6 6 	6
 U3
?346 6 s)6 6r&   c           	         \         P                  ! VVVVVVR7      p\        V\        4      '       d   V.pV P                  P                  WR7       R# )z%Train the model using the given filesrD   rE   rF   rG   rH   rI   )trainerN)r   
BpeTrainer
isinstancer   
_tokenizertrain)	r5   rC   rD   rE   rF   rG   rH   rI   rL   s	   &&&&&&&& r#   rP   SentencePieceBPETokenizer.train6   sQ     %%!'))-'
 eS!!GEe5r&   c                   < V ^8  d   QhRS[ S[S[,          S[S[S[,          ,          3,          RS[RS[RS[S[ S[S[3,          ,          RS[RS[S[,          RS[RS[S[,          /# )	r   iteratorrD   rE   rF   rG   rH   rI   length)r   r   r   r   r   r	   r   r   )r!   r"   s   "r#   r$   r%   N   s     
 
x'>>?
 
 	

 U3
?34
 
 s)
 
 
r&   c	           	     z    \         P                  ! VVVVVVR7      p	V P                  P                  VV	VR7       R# )z(Train the model using the given iteratorrK   )rL   rT   N)r   rM   rO   train_from_iterator)
r5   rS   rD   rE   rF   rG   rH   rI   rT   rL   s
   &&&&&&&&& r#   rV   -SentencePieceBPETokenizer.train_from_iteratorN   sH     %%!'))-'
 	++ 	, 	
r&    )NNr   u   ▁TNF)__name__
__module____qualname____firstlineno____doc__r4   staticmethodr@   rP   rV   __static_attributes____classdictcell____classcell__)r8   r"   s   @@r#   r   r   
   ss     
0 0B B B  8?y"&("6 66  8?y"&(" $
 
 
r&   r   N)typingr   r   r   r   r   r   
tokenizersr	   r
   r   r   r   tokenizers.modelsr   tokenizers.normalizersr   base_tokenizerr   r   rX   r&   r#   <module>rg      s(    ? ? P P ! ' )]
 ]
r&   