o
    ?HhG                     @   s$  d dl mZmZmZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZmZmZ ddlmZmZ dd	lmZ ed
ZedZd dlmZmZmZ d dlm
Z
 d dlmZmZmZ dd Z dVddZ!dd Z"dd Z#dd Z$dd Z%dd Z&ej'(ddd gd!d" Z)d#d$ Z*G d%d& d&e+Z,dWd'd(Z-d)d* Z.d+d, Z/ej'(d-e0e1dd.d/ Z2d0d1 Z3d2d3 Z4d4d5 Z5d6d7 Z6d8d9 Z7d:d; Z8d<d= Z9d>d? Z:d@dA Z;dBdC Z<dDdE Z=ej'(dFdGdHgej'j>ej?dIkoej?dJkdKdLdMdN Z@dOdP ZAej'(dQdRdSgdTdU ZBdS )X    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_config)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributeddask)ClientLocalCluster
get_client)time)clusterinccleanupc                  O      d S N )argskwargsr   r   U/home/air/sanwanet/gpt-api/venv/lib/python3.10/site-packages/joblib/test/test_dask.pynoop   s   r   皙?c                 C   s   t | | r
tdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s   r$   c                    sF   | dd }i }| D ]\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)dask_workerr   r   r   <lambda>$   s    zcount_events.<locals>.<lambda>c                    s   g | ]
}|d   kr|qS    r   ).0event
event_namer   r   
<listcomp>'   s    z count_events.<locals>.<listcomp>)runitemslenlist)r-   clientworker_eventsevent_countsweventsr   r,   r   count_events#   s
    r8   c                 C   sD  t  \}\}}t|d | dp}tdd[ t dd tdD }|dd	 tdD ks0J tt t d
d tdD  W d    n1 sLw   Y  t dd tdD }|dd	 tdD ksjJ W d    n1 stw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Naddressloopr   backendc                 s       | ]	}t t|V  qd S r   r   r   r*   ir   r   r   	<genexpr>0       ztest_simple.<locals>.<genexpr>
   c                 S      g | ]}t |qS r   r   r@   r   r   r   r.   1       ztest_simple.<locals>.<listcomp>c                 s   s     | ]}t t|d kV  qdS )   N)r   r$   r@   r   r   r   rB   4   s    c                 s   r>   r   r?   r@   r   r   r   rB   7   rC   c                 S   rE   r   rF   r@   r   r   r   r.   8   rG   )r   r   r   r
   rangepytestraisesr!   r;   sabr3   seqr   r   r   test_simple,   s$   

"rQ   c                 C   s,  t jtju sJ t \}\}}t|d | d\}tddG t 4}|j}t|t s-J |j	|u s4J |j
dks;J |dd ttdD  |j
d	ksOJ W d    n1 sYw   Y  W d    n1 shw   Y  W d    n1 sww   Y  W d    d S W d    d S 1 sw   Y  d S )
Nr9   r:   r   r<   r)   c                 s   s    | ]
}t d d  V  qdS )c                   S   r   r   r   r   r   r   r   r'   M   s    z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r*   _r   r   r   rB   L   
    
z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @rD   )r   compute_batch_sizer   r   r   r   r
   _backend
isinstanceparallel_effective_batch_sizerI   int)r;   rM   rN   rO   r3   rZ   r=   r   r   r   #test_dask_backend_uses_autobatching;   s.   
"r]   c                   C   s   t  S r   r   r   r   r   r   random2S   s   r^   c              
   C   s   t  V\}\}}t|d | d1}tdd t dd tdD \}}||ks+J W d    n1 s5w   Y  W d    n1 sDw   Y  W d    d S W d    d S 1 s\w   Y  d S )Nr9   r:   r   r<   c                 s       | ]}t t V  qd S r   )r   r^   r@   r   r   r   rB   [   s    z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   rI   )r;   rM   rN   rO   r3   xyr   r   r    test_dont_assume_function_purityW   s   "rc   mixedTFc              
   C   s4  ddl m} |sdd tdD }d}ndd tdD }d}t|||ks)J t d\}\}}t|d	 | d
?}tdd tddd|}	W d    n1 sRw   Y  dd }
|dd}|	|
}t
dd |D sqJ W d    n1 s{w   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r?   r@   r   r   r   r.   c       z&test_dask_funcname.<locals>.<listcomp>   batch_of_inc_4_callsc                 S   s,   g | ]}|d  rt t|nt t|qS )r	   )r   absr   r@   r   r   r   r.   f   s     mixed_batch_of_inc_4_callsr9   r:   r   r<   r	   all)
batch_sizepre_dispatchc                 S   s
   t | jS r   )r2   transition_log)dask_schedulerr   r   r   fr      
ztest_dask_funcname.<locals>.f42c                 s   s    | ]	}d |d v V  qdS )batch_of_incr   Nr   )r*   tupr   r   r   rB   v   rC   z%test_dask_funcname.<locals>.<genexpr>)joblib._daskre   rI   reprr   r   r   r
   replacerun_on_schedulerrk   )r;   rd   re   tasks
batch_reprrM   rN   rO   r3   rT   rp   r%   r   r   r   test_dask_funcname_   s,   
"r|   c               	      s  dd t dD } tdtd d fdd	tdd	d
}t|}ztdd t fdd| D }W d    n1 sDw   Y  | dd t dD ksVJ t	d|}t
| dkseJ tdd |D spJ tdd t  fdd| D }W d    n1 sw   Y  t	d|}t
| dksJ tdd |D sJ W |jdd |jdd d S |jdd |jdd w )Nc                 S      g | ]}g qS r   r   rS   r   r   r   r.          z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d urj |  | t j | S r   )testingassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s   zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr)   r	   )	n_workersthreads_per_workerr   r<   c                 3   s    | ]	}t  |V  qd S r   rR   r*   r   )r   r   r   rB          
z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   r}   r   r   rS   r   r   r   r.      r~   receive-from-scatterr   c                 S      g | ]}t |d kqS r(   r1   r*   rr   r   r   r.      rf   c                 3   s     | ]}t | d V  qdS ))r   NrR   r   )r   r   r   r   rB      s    
c                 S   r   r(   r   r   r   r   r   r.      rf      timeoutr   )rI   rJ   importorskiparanger\   r   r   r   r
   r8   sumvaluesrk   close)listsr   r3   rescountsr   )r   r   r   r   'test_no_undesired_distributed_cache_hity   s8   




r   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )ra   count)selfra   r   r   r   __init__   s   
zCountSerialized.__init__c                 C   s   | j t|d| S )Nra   )ra   getattr)r   otherr   r   r   __add__   s   zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr)   )r   r   ra   )r   r   r   r   
__reduce__   s   zCountSerialized.__reduce__N)__name__
__module____qualname__r   r   __radd__r   r   r   r   r   r      s
    r   c                 C   s   | | | | | S r   r   )rN   rO   cder   r   r   add5   s   r   c                 C   s  t d}t d}t d}t \}\}}t|d | d~}td||gd: tt}||||dd	d
||||d	dd
|||||d	d
||||||d
g}	dd |	D }
t |	}W d    n1 sbw   Y  tt	 td| dd W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  ||
ksJ |j
dksJ |j
dksJ |j
dv sJ d S )Nr)   r	   rH   r9   r:   r   )r=   scatterrg      )r   r   c                 S   s    g | ]\}}}||i |qS r   r   )r*   funcr   r   r   r   r   r.      s    z'test_manual_scatter.<locals>.<listcomp>)r=   r;   r   )rg      )r   r   r   r   r   r   r
   rJ   rK   	TypeErrorr   )r;   ra   rb   zrM   rN   rO   r3   rp   rz   expectedresultsr   r   r   test_manual_scatter   s>   r   c           	   
      s  t d}|jtd|jd |jtd|jd} gd |gd  }t Z\}\}}t|d | d>}tdd t d	d
 t	|D  W d    n1 sPw   Y  t
d|}||d  ||d   dksjJ W d    n1 stw   Y  W d    n1 sw   Y  t i\}\}}t|d | dD}tdd t  fdd
tdD  W d    n1 sw   Y  t
d|}||d  dksJ ||d  dksJ W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   rV   )dtyperH   r9   r:   r   r<   c                 s   s(    | ]\}}t t||||d V  qdS ))optNr   r   )r*   rA   r   r   r   r   rB      s    z$test_auto_scatter.<locals>.<genexpr>r   r	   c                 3   s&    | ]}t t d d |V  qd S )NrH   r   r@   data1r   r   rB      s   $ r   r   )rJ   r   onesr\   uint8r   r   r   r
   	enumerater8   rI   )	loop_in_threadr   data2data_to_processrM   rN   rO   r3   r   r   r   r   test_auto_scatter   s8   



"
"r   retry_noc              
      s   t dd d}fdd fddt V\}\}}t|d | d1}td	d
 dt fddt|D }W d    n1 sKw   Y  W d    n1 sZw   Y  W d    d S W d    d S 1 srw   Y  d S )Nr   rD   c                    s
     | S r   )r   )ra   rA   j)r   r   r   my_sum  rq   z#test_nested_scatter.<locals>.my_sumc                    s^   t  }tdd t  fddtD }W d    t|S 1 s&w   Y  t|S )Nr   r<   c                 3   s(    | ]}t  |d  |V  qd S r   rR   )r*   r   )arrayrA   r   r   r   rB     s    
zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rI   r   )r   rA   r3   r   )NUM_INNER_TASKSr   )r   rA   r   outer_function_joblib  s   

z2test_nested_scatter.<locals>.outer_function_joblibr9   r:   r   r<   i'  c                 3   s&    | ]}t  |d  |V  qd S r   rR   r@   )my_arrayr   r   r   rB      s    
z&test_nested_scatter.<locals>.<genexpr>)rJ   r   r   r   r   r   r
   rI   )r;   r   NUM_OUTER_TASKSrM   rN   rO   rT   r   )r   r   r   r   r   r   test_nested_scatter  s$   
	

"r   c              
      s|  dd  t  \}\}}t|d | d<}tdd' tdd fd	d
tdD }|D ]}tt|dks9J q-W d    n1 sDw   Y  W d    n1 sSw   Y  t|d | d<}tdd' tdd fdd
tdD }|D ]}tt|dksJ qyW d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s       | ]	}t tj V  qd S r   r   osgetpidrS   r   r   r   rB   (  rC   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   r   r   r   rS   r   r   r   rB   )  rC   )setr
   rI   )pidsr   r   r   get_nested_pids'  s    $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr9   r:   r   r<   r	   r   c                 3       | ]}t   V  qd S r   rR   rS   r   r   r   rB   /  
    
z6test_nested_backend_context_manager.<locals>.<genexpr>rD   c                 3   r   r   rR   rS   r   r   r   rB   9  r   )r   r   r   r
   rI   r1   r   )r   rM   rN   rO   r3   
pid_groups	pid_groupr   r   r   #test_nested_backend_context_manager&  s6   
"r   c           	         s6  dd   fddt  \}\}}t|d | d_}tddJ t $} |d	ks-J |jd
ks4J |fddtdD }W d    n1 sKw   Y  |D ]\}}|d	ks\J |d
ksbJ qRW d    n1 smw   Y  W d    n1 s|w   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                 S   s
   | j jjS r   )rX   	__class__r   pr   r   r   _backend_typeE  rq   zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t  }  | | jfW  d    S 1 sw   Y  d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsH  s   $zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr9   r:   r   r<   r   c                 3   r   r   rR   rS   )r   r   r   rB   R  r   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   r   rI   )	r;   rM   rN   rO   r3   r   all_nested_n_jobsbackend_typenested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobsA  s,   
"r   c              	   C   sn   t t}tdd W d    n1 sw   Y  W d    n1 s%w   Y  dt|j v s5J d S )Nr   r<   zcreate a dask client)rJ   rK   r!   r   strvaluelower)r;   infor   r   r   test_errors[  s   r   c              
   C   s4  t  \}\}}t|d | dh}tdd# tdddd td	D }t|d
 d
 d
 ts2J W d    n1 s<w   Y  tdd# tdddd td	D }t|d
 d
 d
 tsbJ W d    n1 slw   Y  W d    n1 s{w   Y  W d    d S W d    d S 1 sw   Y  d S )Nr9   r:   r   r<   r	   r   c                 s   s    | ]
}t td dV  qd S )Nnested_requirer   outerrS   r   r   r   rB   h  s    
z.test_correct_nested_backend.<locals>.<genexpr>r)   r   c                 s   s    | ]
}t td dV  qdS )	sharedmemr   Nr   rS   r   r   r   rB   n  rU   )r   r   r   r
   rI   rY   r   r   )r;   rM   rN   rO   r3   resultr   r   r   test_correct_nested_backendc  s&   "r   c                    s"   t ddd fddtdD S )Nr	   threads)r   preferc                 3   s    | ]	}t t V  qd S r   )r   middlerS   r   r   r   rB   u  r   zouter.<locals>.<genexpr>r)   r
   rI   r   r   r   r   r   t  s   r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   r_   r   )r   innerrS   r   r   r   rB   {  s    

zmiddle.<locals>.<genexpr>r)   r   )r   r   r   r   r   z  s   r   c                   C   s   t  jS r   )r
   rX   r   r   r   r   r     s   r   c              	   C   s   t | ddd4 tdd tdddd	 td
D  W d    n1 s&w   Y  W d    d S W d    d S 1 s>w   Y  d S )NFT)r;   	processesset_as_defaultr   r<   rg   r   c                 s   r>   r   )r   idr@   r   r   r   rB     rC   z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rI   r:   r   r   r   test_secede_with_no_processes  s   "r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r9   )rT   r   r   r   r   _worker_address  s   r   c              
   C   s0  t  \}\}}t|d | df}td|d d t dd tdD }||d gd ks1J W d    n1 s;w   Y  td|d d t dd tdD }||d gd ks`J W d    n1 sjw   Y  W d    n1 syw   Y  W d    d S W d    d S 1 sw   Y  d S )	Nr9   r:   r   )r=   workersc                 s   r>   r   r   r   r@   r   r   r   rB     r   z-test_dask_backend_keywords.<locals>.<genexpr>rD   c                 s   r>   r   r   r@   r   r   r   rB     r   r`   rL   r   r   r   test_dask_backend_keywords  s&   

"r   c              	   C   s   t d| dI}tdd t dd tdD  W d    n1 s#w   Y  t }|jjjrBtd t |d	 k s=J |jjjs0|j	rGJ W d    d S 1 sRw   Y  d S )
NF)r   r;   r   r<   c                 s   r>   r   r?   r@   r   r   r   rB     rC   z/test_scheduler_tasks_cleanup.<locals>.<genexpr>rD   g{Gz?r   )
r   r   r
   rI   r   r   	schedulerrz   r   futures)r;   r3   startr   r   r   test_scheduler_tasks_cleanup  s   

"r  cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc                 C   s   t dddd}t|}| dkr|jddd n	| dkr |d z7tdd	 t d
d tdD  W d    n1 s=w   Y  W |  |  d S W |  |  d S |  |  w )Nr   Fr	   r   r   r   r  )minimummaximumr  r   r<   c                 s   r>   r   r?   r@   r   r   r   rB     rC   z(test_wait_for_workers.<locals>.<genexpr>rD   )r   r   adaptscaler   r
   rI   r   )r  r   r3   r   r   r   test_wait_for_workers  s"   

r  c               	   C   sB  t dddd} t| }ztddd- d}tjt|d	 t d
d tdD  W d    n1 s3w   Y  W d    n1 sBw   Y  tddd- d}tjt|d	 t dd tdD  W d    n1 snw   Y  W d    n1 s}w   Y  W |	  | 	  d S W |	  | 	  d S |	  | 	  w )Nr   Fr	   r	  r   g?)r=   wait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   r>   r   r?   r@   r   r   r   rB     rC   z0test_wait_for_workers_timeout.<locals>.<genexpr>rD   z+DaskDistributedBackend has no active workerc                 s   r>   r   r?   r@   r   r   r   rB     rC   )
r   r   r   rJ   rK   TimeoutErrorr
   rI   RuntimeErrorr   )r   r3   msgr   r   r   test_wait_for_workers_timeout  s0   
r  r=   lokymultiprocessingc              	      s   t dd}t|}z: fdd}||}| }t|dks!J |d j}t|ts-J dt|v s5J W |j	dd	 |j	dd	 d S |j	dd	 |j	dd	 w )
Nr	   )r   c                     sP   t jdd} td ddd tdD  W d    | S 1 s!w   Y  | S )NTrecordr	   )r   r=   c                 s   r>   r   r?   r@   r   r   r   rB     r   zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>rD   )warningscatch_warningsr
   rI   r  r<   r   r   func_using_joblib_parallel  s   

zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr)   r   zdistributed.worker.daemonr   r   )
r   r   submitr   r1   messagerY   UserWarningr   r   )r=   r   r3   r  futr  warningr   r<   r   /test_joblib_warning_inside_dask_daemonic_worker  s   


r!  )r    )r   r   )C
__future__r   r   r   r   r  rJ   r   uuidr   r   r    r
   r   r   rZ   r   r   _daskr   r   r   r   r   r   r   distributed.metricsdistributed.utils_testr   r   r   r   r$   r8   rQ   r]   r^   rc   markparametrizer|   r   objectr   r   r   r   r2   rI   r   r   r   r   r   r   r   r   r   r   r   r  skipif__version__r  r  r!  r   r   r   r   <module>   sf    


	
9
%
