+
    ~jD                        ^ RI Ht ^ RIt^ RIHt ^RIHtHt ^RIH	t	H
t
Ht ]].]],          ]]]]]3,          ,          ,          3,          t ! R R4      t ! R R	]]	4      t ! R
 R]]4      tR# )    )annotationsN)Callable)aio	shortuuid)SentenceStream	TokenData
WordStreamc                      ] tR t^tRR/R R llt]P                  R R l4       t]P                  R R l4       tR	 R
 lt	R R lt
R R ltR R ltR R ltRtR# )BufferedTokenStreamretain_formatFc          
     ,    V ^8  d   QhRRRRRRRRRR	/# )
   tokenize_fncTokenizeCallablemin_token_lenintmin_ctx_lenr   boolreturnNone )formats   "y/Users/mitch_tango/dev/rabbit-r1-livekit/agent/.venv/lib/python3.14/site-packages/livekit/agents/tokenize/token_stream.py__annotate__ BufferedTokenStream.__annotate__   s>       ' 	
   
    c               	    \         P                  \        ,          ! 4       V n        Wn        W0n        W n        W@n        \        4       V n	        . V n
        R V n        R V n        R# ) N)r   Chanr   	_event_ch_tokenize_fnc_min_ctx_len_min_token_len_retain_formatr   _current_segment_id_buf_tokens_in_buf_out_buf)selfr   r   r   r   s   &$$$$r   __init__BufferedTokenStream.__init__   sO     ),.)'++#,; &(r   c                    V ^8  d   QhRRRR/# )r   textstrr   r   r   )r   s   "r   r   r   #   s      I  Ic  Id  Ir   c                	   V P                  4        V ;P                  V,          un        \        V P                  4      V P                  8  d   R #  V P	                  V P                  4      p\        V4      ^8:  d   R # V P
                  '       d   V ;P
                  R,          un        VP                  ^ 4      pTp\        V\        4      '       d
   V^ ,          pV ;P
                  V,          un        \        V P
                  4      V P                  8  dB   V P                  P                  \        V P
                  V P                  R7      4       RV n        \        V\        4      '       d   V P                  V^,          R  V n        EK=  \        V P                  P                  V4      ^ 4      pV P                  V\        V4      ,           R  P!                  4       V n        EK  )N token
segment_idr   )_check_not_closedr'   lenr"   r!   r(   pop
isinstancetupler#   r    send_nowaitr   r%   maxfindlstrip)r)   r-   tokenstoktok_texttok_is   &&    r   	push_textBufferedTokenStream.push_text"   sR    t||t000''5F6{a}}}$**Q-CH#u%%q6MMX%M4==!T%8%88**DMMd>V>VW !##u%%#||CFH5DLL--c2A6#||ECH,<,>?FFHr   c                   V ^8  d   QhRR/# r   r   r   r   )r   s   "r   r   r   F   s      t r   c           	     	   V P                  4        V P                  '       g   V P                  '       Ed   V P                  V P                  4      pV'       d   V P                  '       d   V ;P                  R ,          un        \	        V^ ,          \
        4      '       d@   T ;P                  R P                  V Uu. uF  q"^ ,          NK  	  up4      ,          un        M(V ;P                  R P                  V4      ,          un        V P                  '       d;   V P                  P                  \        V P                  V P                  R7      4       \        4       V n
        RV n        RV n        R# u upi )r0   r1   r   N)r4   r'   r(   r!   r7   r8   joinr    r9   r   r%   r   )r)   r=   r>   s   &  r   flushBufferedTokenStream.flushE   s     <<<4===''5F===MMS(MfQi//MMSXX.H#1vv.H%IIMMMSXXf%55M}}}**DMMd>V>VW $-;  /Is   <E3c                   V ^8  d   QhRR/# rD   r   )r   s   "r   r   r   ]   s      4 r   c                	Z    V P                  4        V P                  P                  4        R # N)rG   r    closer)   s   &r   	end_inputBufferedTokenStream.end_input]   s    

r   c                   V ^8  d   QhRR/# rD   r   )r   s   "r   r   r   a   s      d r   c                	B   "   V P                   P                  4        R # 5irK   )r    rL   rM   s   &r   acloseBufferedTokenStream.aclosea   s     s   c                   V ^8  d   QhRR/# rD   r   )r   s   "r   r   r   d   s     M M4 Mr   c                	    V P                   P                  '       d1   \        V 4      p\        VP                   R VP
                   R24      hR# ).z
 is closedN)r    closedtypeRuntimeError
__module____name__)r)   clss   & r   r4   %BufferedTokenStream._check_not_closedd   sA    >>   t*C#..!13<<.
KLL !r   c                   V ^8  d   QhRR/# )r   r   r   r   )r   s   "r   r   r   i   s      . r   c                	    V # rK   r   rM   s   &r   	__aiter__BufferedTokenStream.__aiter__i   s    r   c                   V ^8  d   QhRR/# )r   r   r   r   )r   s   "r   r   r   l   s     0 0 0r   c                	R   "   V P                   P                  4       G R j  xL
 #  L5irK   )r    	__anext__rM   s   &r   rd   BufferedTokenStream.__anext__l   s     ^^--////s   '%')	r&   r%   r    r'   r"   r#   r(   r$   r!   N)r[   rZ   __qualname____firstlineno__r*   typingno_type_checkrA   rG   rN   rR   r4   r`   rd   __static_attributes__r   r   r   r   r      sg     $&  I  ID  .M
0 0r   r   c                  .   a  ] tR t^ptR V 3R lltRtV ;t# )BufferedSentenceStreamc               (    V ^8  d   QhRRRRRRRR/# r   	tokenizerr   r   r   r   r   r   r   )r   s   "r   r   #BufferedSentenceStream.__annotate__q   4     
 
 $
 	

 
 

r   c               	,   < \         SV `  VVVR 7       R# )r   r   r   Nsuperr*   r)   ro   r   r   	__class__s   &$$$r   r*   BufferedSentenceStream.__init__q         	"'# 	 	
r   r   r[   rZ   rf   rg   r*   rj   __classcell__rw   s   @r   rl   rl   p       
 
r   rl   c                  .   a  ] tR t^tR V 3R lltRtV ;t# )BufferedWordStreamc               (    V ^8  d   QhRRRRRRRR/# rn   r   )r   s   "r   r   BufferedWordStream.__annotate__   rq   r   c               	,   < \         SV `  VVVR 7       R# rs   rt   rv   s   &$$$r   r*   BufferedWordStream.__init__   ry   r   r   rz   r|   s   @r   r   r      r}   r   r   )
__future__r   rh   collections.abcr   utilsr   r   ro   r   r   r	   r.   listr8   r   r   r   rl   r   r   r   r   <module>r      so    "  $ " < < SE49tE#sC-4H/I#IIJ _0 _0D
0. 

,j 
r   