+
    ~j                        ^ RI Ht ^ RIt^ RIHt ^RIHtHtHtH	t	H
t
Ht . ROt] ! R R4      4       t ! R	 R]P                  4      t ! R
 R]P                  4      tR R ltRRRR/R R lltR R ltR# )    )annotationsN)	dataclass)_basic_hyphenator_basic_paragraph_basic_sent_basic_wordtoken_stream	tokenizerSentenceTokenizerWordTokenizerc                  @    ] tR t^t$ R]R&   R]R&   R]R&   R]R&   RtR	# )
_TokenizerOptionsstrlanguageintmin_sentence_lenstream_context_lenboolretain_format N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r       r/Users/mitch_tango/dev/rabbit-r1-livekit/agent/.venv/lib/python3.14/site-packages/livekit/agents/tokenize/basic.pyr   r      s    Mr   r   c                  ^    ] tR t^"tRRR^R^
RR/R R lltRR	/R
 R lltRR	/R R lltRtR	# )r   r   englishr   r   r   Fc          
     ,    V ^8  d   QhRRRRRRRRRR	/# )
   r   r   r   r   r   r   r   returnNoner   )formats   "r   __annotate__SentenceTokenizer.__annotate__#   s>     
 
 
 	

  
 
 

r   c               	.    \        VVVVR 7      V n        R# ))r   r   r   r   N)r   _config)selfr   r   r   r   s   &$$$$r   __init__SentenceTokenizer.__init__#   s     )-1'	
r   Nc               $    V ^8  d   QhRRRRRR/# r"   textr   r   
str | Noner#   	list[str]r   )r%   s   "r   r&   r'   2   s!     
 
S 
z 
Y 
r   c               	    \         P                  ! VV P                  P                  V P                  P                  R 7       Uu. uF  pV^ ,          NK  	  up# u upi )r   r   )r   split_sentencesr)   r   r   r*   r/   r   toks   &&$ r   tokenizeSentenceTokenizer.tokenize2   s[     #22!%!>!>"ll88
 FF
 	
 
s   Ac                    V ^8  d   QhRRRR/# )r"   r   r0   r#   ztokenizer.SentenceStreamr   )r%   s   "r   r&   r'   <   s     	
 	
* 	
8P 	
r   c               	   \         P                  ! \        P                  ! \        P
                  V P                  P                  V P                  P                  R 7      V P                  P                  V P                  P                  R7      # )r3   r
   min_token_lenmin_ctx_len)
r	   BufferedSentenceStream	functoolspartialr   r4   r)   r   r   r   r*   r   s   &$r   streamSentenceTokenizer.stream<   sb    22''++!%!>!>"ll88
 ,,7777
 	
r   )r)   r   r   r   r   r+   r7   rB   r   r   r   r   r   r   "   sL    
 "
 !#	

 #%
 $

D 
	
t 	
 	
r   c                  Z    ] tR t^HtRRRRRR/R R lltRR	/R
 R lltRR	/R R lltRtR	# )r   ignore_punctuationTsplit_characterFr   c               (    V ^8  d   QhRRRRRRRR/# )r"   rF   r   rG   r   r#   r$   r   )r%   s   "r   r&   WordTokenizer.__annotate__I   s4     	, 	, !	, 		,
 	, 
	,r   c               	*    Wn         W n        W0n        R # N)_ignore_punctuation_split_character_retain_format)r*   rF   rG   r   s   &$$$r   r+   WordTokenizer.__init__I   s     $6  /+r   r   Nc               $    V ^8  d   QhRRRRRR/# r.   r   )r%   s   "r   r&   rI   T   s!     	
 	
S 	
z 	
Y 	
r   c               	    \         P                  ! VV P                  V P                  V P                  R 7       Uu. uF  pV^ ,          NK  	  up# u upi )rF   rG   r   )r   split_wordsrL   rM   rN   r5   s   &&$ r   r7   WordTokenizer.tokenizeT   s\     #..#'#;#; $ 5 5"11	
 FF
 	
 
s   Ac                    V ^8  d   QhRRRR/# )r"   r   r0   r#   ztokenizer.WordStreamr   )r%   s   "r   r&   rI   _   s     

 

* 

8L 

r   c          	     	    \         P                  ! \        P                  ! \        P
                  V P                  V P                  V P                  R 7      ^^R7      # )rR   r;   )	r	   BufferedWordStreamr?   r@   r   rS   rL   rM   rN   rA   s   &$r   rB   WordTokenizer.stream_   sO    ..''''#'#;#; $ 5 5"11	 	
 		
r   )rL   rN   rM   rD   r   r   r   r   r   H   sB    	, $(	, !&		,
 $	,	
D 	


t 

 

r   c                    V ^8  d   QhRRRR/# )r"   wordr   r#   r1   r   )r%   s   "r   r&   r&   l   s     2 2 2 2r   c                .    \         P                  ! V 4      # rK   )r   hyphenate_word)rZ   s   &r   r\   r\   l   s    ++D11r   rF   TrG   Fc               (    V ^8  d   QhRRRRRRRR/# )r"   r/   r   rF   r   rG   r#   zlist[tuple[str, int, int]]r   )r%   s   "r   r&   r&   p   s,      
&*DHr   c               2    \         P                  ! WVR 7      # ))rF   rG   )r   rS   )r/   rF   rG   s   &$$r   rS   rS   p   s     ""_ r   c                    V ^8  d   QhRRRR/# )r"   r/   r   r#   r1   r   )r%   s   "r   r&   r&   x   s     G Gc Gi Gr   c                d    \         P                  ! V 4       Uu. uF  q^ ,          NK  	  up# u upi )r   )r   split_paragraphs)r/   r6   s   & r   tokenize_paragraphsrb   x   s+    .??EFEsFFEFFFs   -)r   r   r\   rb   )
__future__r   r?   dataclassesr    r   r   r   r   r	   r
   __all__r   r   r   r\   rS   rb   r   r   r   <module>rg      s~    "  !    #
	33 #
L!
I++ !
H2-1KPGr   