+
    ~j	                        ^ RI Ht ^ RIt^ RIt^ RIHt ^ RIHt ^RIH	t	H
t
 R.tRR/R	 R
 llt] ! R R4      4       t ! R R]
P                  4      tR# )    )annotationsN)	dataclass)	blingfire)token_stream	tokenizerSentenceTokenizerretain_formatFc               (    V ^8  d   QhRRRRRRRR/# )	   textstrmin_sentence_lenintr	   boolreturnzlist[tuple[str, int, int]] )formats   "v/Users/mitch_tango/dev/rabbit-r1-livekit/agent/.venv/lib/python3.14/site-packages/livekit/agents/tokenize/blingfire.py__annotate__r      s,      
!$8<    c                  \         P                  ! V 4      w  r4. p^ pV Fz  w  r7WV p\        P                  ! RRV4      P	                  4       p	V	'       d   \        V	4      V8  d   KI  V'       d   VP                  WV34       MVP                  WV34       TpK|  	  V\        V 4      8  d_   WR pV'       d   VP                  W\        V 4      34       V# VP	                  4       ;p	'       d   VP                  W\        V 4      34       V# )r   z	\s*\n+\s* N)r   text_to_sentences_with_offsetsresubstriplenappend)
r   r   r	   _offsetsmerged_sentencesstartendraw_sentencesentences
   &&$       r   _split_sentencesr&      s     99$?JAE#66,\:@@B3x=+;;##\#$>?##Xc$:;  s4yF|##\#d)$DE  &++--X-##Xc$i$@Ar   c                  6    ] tR t^.t$ R]R&   R]R&   R]R&   RtR# )_TokenizerOptionsr   r   stream_context_lenr   r	   r   N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r   r   r   r(   r(   .   s    r   r(   c                  Z    ] tR t^5tR^R^
RR/R R lltRR/R	 R
 lltRR/R R lltRtR# )r   r   r)   r	   Fc               (    V ^8  d   QhRRRRRRRR/# )r   r   r   r)   r	   r   r   Noner   )r   s   "r   r   SentenceTokenizer.__annotate__6   s4     
 
 
  	

 
 

r   c               	,    \        VVVR 7      V n        R# ))r   r)   r	   N)r(   _config)selfr   r)   r	   s   &$$$r   __init__SentenceTokenizer.__init__6   s     )-1'
r   languageNc               $    V ^8  d   QhRRRRRR/# )r   r   r   r9   
str | Noner   z	list[str]r   )r   s   "r   r   r3   C   s!     
 
S 
z 
Y 
r   c               	    \        VV P                  P                  V P                  P                  R 7       Uu. uF  pV^ ,          NK  	  up# u upi )r   r	   )r&   r5   r   r	   )r6   r   r9   toks   &&$ r   tokenizeSentenceTokenizer.tokenizeC   sU     (!%!>!>"ll88
 FF
 	
 
s   Ac                    V ^8  d   QhRRRR/# )r   r9   r;   r   ztokenizer.SentenceStreamr   )r   s   "r   r   r3   M   s     	
 	
* 	
8P 	
r   c               	
   \         P                  ! \        P                  ! \        V P
                  P                  V P
                  P                  R 7      V P
                  P                  V P
                  P                  R7      # )r=   )r   min_token_lenmin_ctx_len)	r   BufferedSentenceStream	functoolspartialr&   r5   r   r	   r)   )r6   r9   s   &$r   streamSentenceTokenizer.streamM   s\    22'' !%!>!>"ll88
 ,,7777
 	
r   )r5   )r*   r+   r,   r-   r7   r?   rH   r/   r   r   r   r   r   5   sB    
 !#
 #%	

 $

D 
	
t 	
 	
r   )
__future__r   rF   r   dataclassesr   livekitr    r   r   __all__r&   r(   r   r   r   r   <module>rO      s[    "  	 !  % 
?D<   !
	33 !
r   