o `^h!@sdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddlm Z ddl mZddlmZddlmZddlZddlZdd lmZdd lmZmZdd lmZdd lmZmZdd lmZddlmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%edurddl&m'Z'ddl(m)Z)zddl*Z*Wn e+ydZ*Ynwzddl,m-Z-Wn e+ydZ-Ynwzddl.Z.Wn e+ydZ.Ynwddl/m0Z0ddl/m1Z1ddl/m2Z2ddl/m3Z3ddl/m4Z4ddl5m6Z6m7Z7ddl5m8Z8ddl5m9Z9ddl5m:Z:ddl5m;Z;mZ>e=?Z@e@AdddgeBe=CZDeDd d!e=D7ZDedur8gZEndd"gZEeEd#gZFeGed$rOeDHeId%e=e>ZJd&d'ZKd(d)ZLd*d+ZMGd,d-d-eNZOdPd/d0ZPd1d2ZQdQd3d4ZRd5d6ZSd7d8ZTd9d:ZUd;d<ZVe!d=e8e9gejWj!d>d?d@e;d@dAfdBggdCdDedEdFZXe!dGeDe!dHgdIe!dJgdKdLdMZYe!dGeDdNdOZZdPdQZ[ee!dRgdSdTdUZ\ee!dGgdVdWdXZ]dYdZZ^e!d[e=e!d\e=d]d^Z_d_d`Z`edadbZadcddZbe!dHgdedfdgZce!dGeFdhdiZdedjdkZeeee!dlgdmdndoZfe!dGeFdpdqZgee!dGeFdrdsZhee!dGeEdtduZiee!dGe=dvdwZjdxdyZke!dGe=e!dzdgd{fd|gd}fgd~dZlee!dGeFddZmddZnee!dGeEddZoddZpddZqddZree!dGeFejWjsddddZtGddde0ZuddZve!dGeDddZwee!dGeFe!dHgdddZxddZyddZze$eduddddZ{ddZ|eFddZ}e}~dd!edDee!dGe}e!d=e9e8gddZGddde0Ze!d=e8e9gddZe!d=e8e9gddZddZddZGdddejj4Ze:deee!dGgde!d=e8e9gddZee!dHgde!dGeFe!d=e8e9gddZddZee!deFe!deFe!d=e8e9gddZee!d=e8e9ge!ddd.gdd„Ze!dgdĢddƄZe!dgdȢddʄZedd̄Zeee!dGeEdd΄ZdZee!dGeEddфZdZdZdZdZee!dGeEedurgnd%ge!deeege!dgdآddڄZdjejejejd܍ZeddބZdjejejejd܍ZeddZddZddZddZeee!dGeEddZddZeeddZddZddZejW!dHgdddZddZddZejW!dHdd|gee!dGee@dhddZejW!dHdd@ge!d=e8e9ge$e.duddddZe!dGeDe!dHgdddZddZddZee!dGe@e!ddd ge!dHgdd d ZeejW!dHdd@ge!ddd ge!d=e8e9ge$e.duddd d Ze!dGe@e!ddd ge!dHgdddZe!dGe@e!ddd ge!dHgdddZe!dGe@e!ddd ge!ddd ge!dHgdddZe!dgde!ddd ge!ddd gddZeee!dGeEddZe!dGedurdd#gngdddZee!dGeEdd Zee!d=e8e9gd!d"Ze!d=e8e9gd#d$Zd%d&ZdRd'd(Zee!dGd"d#ge!d=e8e9gd)d*Zee!d=e8e9ge$e.duddd+d,ZdSd-d.ZejWje!d=e8e9ge!dGedur4d#gnd"d#gd/d0Zd1d2Ze!dGedurRdd"gndge$e-dud3dd4d5Ze!d=e8e9ge$e.duddd6d7Ze!d=e8e9gd8d9ZƐd:d;Zǐd<d=ZȐd>d?Zɐd@dAZeee!dHgdBdCdDZeee!dEgdFe!dHdd@ge!d=e8e9gdGdHZee!dHdd@ge!dIgdJe!d=e8e9gdKdLZee!dHgdMdNdOZdS(Tz Test the parallel module. N)format_exceptionsqrt)sleep) PicklingError) nullcontext TimeoutErrorparallel)dumpload)mp)np with_numpy)with_multiprocessing)IS_PYPY force_gc_pypy) parametrizeraisescheck_subprocess_callskipifwarns)get_reusable_executor)Queue) parallel_sum)SequentialBackend)ThreadingBackend)MultiprocessingBackend)ParallelBackendBase) LokyBackend)Paralleldelayed)parallel_config)parallel_backend)register_parallel_backend)effective_n_jobs cpu_count)rBACKENDSDEFAULT_BACKENDmultiprocessingcCsg|]}t|qS)r)).0 backend_strr,r,W/home/air/shanriGPT/back/venv/lib/python3.10/site-packages/joblib/test/test_parallel.py Msr0loky threading get_contextspawncCst|dt|ddS)N_pool_workers)getattrbackendr,r,r/ get_workers[sr:cCs||SNr,)xyr,r,r/division_r>cC|dSNr,r<r,r,r/squarecr?rDc@seZdZdZddZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cCdSr;r,)selfabcdr,r,r/__init__jz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rLr,r,r,r/rEgs rEFcCs"|dkr|r tddddt|S)NrHrIrJrK)rE ValueError)r<custom_exceptionr,r,r/exception_raiserns rVcCstdt)Ng?)timerKeyboardInterruptrCr,r,r/interrupt_raiserus rYcCs|d||S)zO A module-level function so that it can be spawn with multiprocessing. rBr,)r<r=zr,r,r/fzsr[cCsttdSNr)typer get_active_backendr,r,r,r/_active_backend_typesr_cCst||dddtdDS)Nn_jobsr9cs|] }tt|VqdSr;r#rDr-ir,r,r/   z parallel_func..r"range) inner_n_jobsr9r,r,r/ parallel_funcsrlcCtdksJdSr\)r(r,r,r,r/test_cpu_countrncCrmr\r'r,r,r,r/test_effective_n_jobsrorqcontextzbackend_n_jobs, expected_n_jobs)rhrhraNr)z positive-intz negative-intNone)idscCsT|d|dtdd|ksJWdn1swYtdddks(JdS)Nr2rtrrp)rrbackend_n_jobsexpected_n_jobsr,r,r/test_effective_n_jobs_Nones rzr9ra)rrBrsverbose)rB dcCs:ddtdDt|||dddtdDksJdS)NcSg|]}t|qSr,rDr-r<r,r,r/r0z(test_simple_parallel..)rar9r|csrbr;rcrr,r,r/rfrgz'test_simple_parallel..rjr")r9rar|r,r,r/test_simple_parallels rcCs|jtdddtjdd}td|ddd td D}|gd ks(JWdn1s2wYd d |D}t|dksFJdS)Nname!some_new_name_for_the_main_thread)targetrvalueTrecordrBr`csrbr;rcrr,r,r/rfrgz6test_main_thread_renamed_no_warning..rh)rrcSsg|] }dt|jvr|qS)zworker timeout)strmessager-wr,r,r/r0z7test_main_thread_renamed_no_warning..r)setattrr2current_threadwarningscatch_warningsr"rjlen)r9 monkeypatchwarninforesultsr,r,r/#test_main_thread_renamed_no_warnings rcCstjdd}tdt||dWdn1swYdd|D}|rN|rLtdd|D}ttjd d rBt|d knt|d k}|oK|Sd S|rRJdS) NTralways)r9rkcSsg|]}|jqSr,)rrr,r,r/r0sz*_assert_warning_nested..css|] }d|jdvVqdS)zbacked parallel loops cannotrN)args)r-eachr,r,r/rf  z)_assert_warning_nested..nogilFr) rr simplefilterrlallr7sysflagsr)r9rkexpectedrwarnings_are_correctwarnings_have_the_right_lengthr,r,r/_assert_warning_nesteds"  rz%parent_backend,child_backend,expected))r1r+T)r1r1F)r+r+T)r+r1T)r2r+T)r2r1Tcstd|dfddtdDtd|dfddtdD}|dkr;tr3t|s3tjdd t|s9JdSt|sAJdS) NrBr`c3s"|] }ttdddVqdS)rFr9rkrNr#rr-_ child_backendr,r/rf z0test_nested_parallel_warnings..rc3s"|] }ttddVqdS)rBrNrrrrr,r/rfrr2zThis test often fails in PyPy.reason)r"rjranypytestxfailr)parent_backendrrresr,rr/test_nested_parallel_warningss   r)r1r+r2cCs>dg}dd}tj||fd}|||dsJdS)NFcSshtjdd}tddddtdDWdn1swYtt|t|dk|d<dS) NTrrBrtcs|] }ttdVqdS)皙?Nr#rrr,r,r/rfrgzPtest_background_thread_parallelism..background_thread..rr)rrr"rjprintr)is_run_parallelrr,r,r/background_thread s  z=test_background_thread_parallelism..background_thread)rrr)r2Threadstartjoin)r9rrtr,r,r/"test_background_thread_parallelisms rcCs"td|dddtdDdS)NrBr`csr){Gz?Nrcrr,r,r/rfrgznested_loop..rir8r,r,r/ nested_loops rrrcs&td|dfddtdDdS)NrBr`c3|] }ttVqdSr;)r#rrrr,r/rf!rgz#test_nested_loop..ri)rrr,rr/test_nested_loops rcCstr;)rTr8r,r,r/raise_exception%rMrc Cstt5tddd}|ttdttdgWdn1s$wYWddSWddS1s.N)rr"rjfullr,r,rr/test_mutate_input_with_threads1s r)rrBrhcCs6td}dd|Dt|ddd|DksJdS)z.Check the keyword argument processing of pmap. cSg|]}t|ddqSrr=r[rr,r,r/r0=z(test_parallel_kwargs..rtcs |] }tt|ddVqdSrrNr#r[rr,r,r/rf>z'test_parallel_kwargs..Nr)ralstr,r,r/test_parallel_kwargs9s  rcCstd}dd|D}td|d4}|j}||dd|Dks"J||dd|Dks/Jtdur>t|t|jus>JWdn1sHwYtdurZt|jdusZJ||d d|DksgJtdurtt|jdusvJdSdS) NrcSrrrrr,r,r/r0Drz4test_parallel_as_context_manager..rr`csrrrrr,r,r/rfMrz3test_parallel_as_context_manager..csrrrrr,r,r/rfNrcsrrrrr,r,r/rfZr)rjr"_backendrr:)r9rrpmanaged_backendr,r,r/ test_parallel_as_context_managerAs"rcsdGdddtttddtdddfdd td DWd d S1s+wYd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcStd)N123) RuntimeErrorrGr,r,r/ __reduce__er?z.UnpicklableObject.__reduce__N)rOrPrQrr,r,r,r/UnpicklableObjectds rzthe task to sendmatchrBr1r`c3s|] }ttVqdSr;r#idrrr,r/rfis  z)test_parallel_pickling..rN)objectrrr"rjr,r,rr/test_parallel_pickling_s  "r byteorder)<>=csddtdd|djj}tdddfd d td D}|D]\}}||ks4J||jjks.inspect_byteorder)rBrhi4rBr1r`c3|] }tVqdSr;r#rrr<r,r/rfyrgz5test_parallel_byteorder_corruption..rh) rarangereshapeviewrrr"rjtestingassert_array_equal)rinitial_np_byteorderresult x_returnedbyteorder_in_workerr,rr/"test_parallel_byteorder_corruptionms  rcCs0ttd|ddddtdDdksJdS)NrBrar9timeoutcsr)gMbP?Nrrr,r,r/rfrgz0test_parallel_timeout_success..r)rr"rjr8r,r,r/test_parallel_timeout_successs  rcCsNtttd|ddddtdDWddS1s wYdS)NrBrrcsrrNrrr,r,r/rfrgz-test_parallel_timeout_fail..r)rr r"rjr8r,r,r/test_parallel_timeout_fails  "r c Cstdurtttd|dddtddDWdn1s$wYtttd|ddddDWdn1sEwYtd|d}t|jdusZJt|j}tt|ddtddDWdn1szwYt|jdusJt|j|usJd dtd D|d d td DksJt|j}tt|d ddDWdn1swYt|jdusJt|j|usJddtd D|dd td DksJ|j |j |j |j fWdn 1swYt|jdusJn!tttdddddDWdn 1s2wYtttddddtddDWdn 1sVwYtt tddddd tdDWddS1s{wYdS)NrBr`cSg|] \}}tt||qSr,r#r>r-r<r=r,r,r/r0z&test_error_capture..)rr)rrcSg|]}tt|qSr,r#rYrr,r,r/r0rcSr r,r r r,r,r/r0r cSrrrrr,r,r/r0rrcsrrrrr,r,r/rfrz%test_error_capture..cSrr,rrr,r,r/r0rcSrrrrr,r,r/r0rcsrrrrr,r,r/rfrrtcSrr,rrr,r,r/r0rcSr r,r r r,r,r/r0rr)rar|csr)T)rUNr#rVrdr,r,r/rfr)rrZeroDivisionErrorr"ziprXr:rrj _iteratingn_completed_tasksn_dispatched_tasks _abortingrE)r9r original_workersr,r,r/test_error_capturest           $     $rc Csddd}td|dw}ttdd|dd |dd DWdn1s)wYttdd|d d |d d DWdn1sKwYttdd|d d |dd DWdn1smwYWddSWddS1swYdS)Nrcss*tdD] }||krtd|VqdS)NIterator Raising Error)rjrT)raise_atrer,r,r/ my_generators  z1test_error_in_task_iterator..my_generatorrBr`rrcsrbr;rcrdr,r,r/rfz.test_error_in_task_iterator..)rcsrbr;rcrdr,r,r/rfrrcsrbr;rcrdr,r,r/rfrr)r"rrT)r9rrr,r,r/test_error_in_task_iterators  "r!cCs|d|dS)Nz Consumed %s)append)queueitemr,r,r/consumerror%zbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) r&r(r*r,r'r)r+r-r.r0r/r1csTtfdd}td||dfdd|D|ks Jtdks(JdS) zC Test that with only one job, Parallel does act as a iterator. c3(tdD] }d||VqdSNrz Produced %irjr"rer#r,r/producer  z'test_dispatch_one_job..producerrra batch_sizer9c3s|] }tt|VqdSr;r#r%rr6r,r/rf rz(test_dispatch_one_job.. N)listr"r)r9r:expected_queuer7r,r6r/test_dispatch_one_jobs  r?cst}|fdd}tddd|dfdd|Dt}|d d ks,J|d d d }|dks;J|d}||ksFJtdksNJd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3r2r3r4r5r6r,r/r7r8z/test_dispatch_multiprocessing..producerrBrrh)rar: pre_dispatchr9c3s|] }ttdVqdS)rNr;rr6r,r/rf!rz0test_dispatch_multiprocessing..rr&Nrz Consumed anyrsr,r<)rManagerr=r"indexr)r9managerr7queue_contentsfirst_consumption_indexproduced_3_indexr,r6r/test_dispatch_multiprocessings    rGcCs\tdddd}|ddtdD|jdksJWddS1s'wYdS) NrBautor2r9csrbr;rrdr,r,r/rf8rz/test_batching_auto_threading..rr"rjrcompute_batch_size)rr,r,r/test_batching_auto_threading2s"rLcCs\tdd|d}|ddtdD|jdksJWddS1s'wYdS)NrBrHr9csrbr;rrdr,r,r/rf@rz2test_batching_auto_subprocesses..rIrrJ)r9rr,r,r/test_batching_auto_subprocesses<s"rMcCsNtttddddddtdDWddS1s wYdS) zCMake sure that exception raised during dispatch are indeed capturedrBr)rar@r|csrbr;rrdr,r,r/rfKrgz*test_exception_dispatch..rN)rrTr"rjr,r,r,r/test_exception_dispatchHs  "rOcC tddddtdDdS)NrBrtcsrbr;rr-jr,r,r/rfPrgz(nested_function_inner..rrir5r,r,r/nested_function_innerO rScCrP)NrBrtcsrbr;)r#rSrQr,r,r/rfUrgz(nested_function_outer..rrir5r,r,r/nested_function_outerTrTrUz'https://github.com/joblib/loky/pull/255rcCstt}td|dddtdDWdn1swYt|j|j|j}d|}d|vs7Jd |vs=Jd |vsCJt|jtusLJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. rBr`csrbr;)r#rUrdr,r,r/rfcrgz1test_nested_exception_dispatch..rNrUrSrV) rrTr"rjrr]rtbr)r9excinfo report_linesreportr,r,r/test_nested_exception_dispatchYs      r[c@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr;)r'rar )rGrar backend_argsr,r,r/ configurets zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxrr(rGrar,r,r/r'ysz$FakeParallelBackend.effective_n_jobsr)r)rOrPrQrRr^r'r,r,r,r/r\qs r\c Csttdd tddWdn1swYttddtddWdn1s2wYWdn1sAwYttdd&tddWdn1s]wYWddSWddS1suwYdS)NzInvalid backend:rz unit-testingr8)rrTr"r$r,r,r,r/test_invalid_backends    "racCs,tt}td|dWdn1swYdt|jvs%Jtt}td|dWdn1s.rcss |] \}}|t|kVqdSr;r)r-rerr,r,r/rfr)r"_effective_n_jobsrjr enumerate)r9rarrr,r,r/test_njobs_converted_to_ints rjcCs<ztdtdtvs JtdtksJWtd=dStd=w)N test_backend)r&r\r)r,r,r,r/test_register_parallel_backends   rlcCsTttksJztdtdddttksJWtt_ntt_wttks(JdS)Nr2T) make_default)r_DefaultBackendr&r)rr*r r,r,r,r/test_overwrite_default_backends rozOnly without multiprocessingcCsttddtddddtdDWdn1swYtddtddtdDWddS1sAwYdS) Nz)joblib backend '.*' is not available on.*rr1r8csrbr;rcrdr,r,r/rfrz2test_backend_no_multiprocessing..rhcsrbr;rcrdr,r,r/rfr)r UserWarningr"rjr$r,r,r,r/test_backend_no_multiprocessings "rqcCs^||ddt\}}|dksJtddksJt}|jdks%J|dkr;t|tus1Jt|jtus:JnB|dkrQt|tusGJt|jtusPJn4|dkrgt|t us]Jt|jt usfJn.| drt|t ustJt|jt usJWddSWddSWddSWddSWddS1swYdS)Nrhrtr+r1r2test_) r r^r'r"rar]rrr!r startswithr\)rr backend_nameactive_backend active_n_jobsrr,r,r/check_backend_context_managers8    "rwcCsg|]}d|qS)ztest_backend_%dr,rdr,r,r/r0rrhcsl|tvr |t|tttksJt|ttksJtdddfddtDttks4JdS)NrBr2r`c3s"|] }|stt|VqdSr;)r#rw)r-rIrrr,r/rfs   z/test_backend_context_manager..)r)setitemr\r_rnrwr" all_backends_for_context_manager)rr9rrr,rxr/test_backend_context_managers r{c@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)rTparam)rGr}r,r,r/rLs z%ParameterizedParallelBackend.__init__r;rNr,r,r,r/r|sr|cCs|tdtttksJ|dddd?t\}}t|tus$J|jdks+J|dks1Jt }|j dks;J|j |usBJ|ddt dD}Wdn1sWwY|dd t dDksiJttkspJdS) N param_backend*rh)r}racsrbr;r#rrdr,r,r/rfrz=test_parameterized_backend_context_manager..rcSrr,rrdr,r,r/r0rz>test_parameterized_backend_context_manager..) ryr)r|r_rnr r^r]r}r"rarrj)rrrrurvrrr,r,r/*test_parameterized_backend_context_managers    rcCsttksJ|tdddd?t\}}t|tusJ|jdks&J|dks,Jt}|jdks6J|j |us=J|ddt dD}Wdn1sRwY|ddt dDksdJttkskJdS) N+)r}rrtcsrbr;rrdr,r,r/rf(rzFtest_directly_parameterized_backend_context_manager..cSrr,rrdr,r,r/r0)rzGtest_directly_parameterized_backend_context_manager..) r_rnr|r r^r]r}r"rarrj)rrrurvrrr,r,r/3test_directly_parameterized_backend_context_managers   rcCstdtS)Nr)rosgetpidr,r,r,r/sleep_and_return_pid/srcCs<ttksJtdksJtddddtdDS)NrrBrtcs|]}ttVqdSr;)r#rrr,r,r/rf;sz"get_nested_pids..)r_rr"rhrjr,r,r,r/get_nested_pids4s rcs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSr\)superrget_nested_backendr __class__r,r/rAszMyBackend.get_nested_backend)rOrPrQrRr __classcell__r,r,rr/r?srback_compat_backend)r2r1r+rcCsh||&tddddtdD}|D] }tt|dks!JqWddS1s-wYdS)NrBrtcsrr;)r#rrr,r,r/rfRs  z6test_nested_backend_context_manager..rr)r"rjrset)rrr9 pid_groups pid_groupr,r,r/#test_nested_backend_context_managerIs "r)rBrsNcs~ddtddfddtdD|dtddfddtdDWddS1s8wYdS) NcSs0tt|ks Jt|}t|ksJdSr;)r_r)r'r"rh)expected_backend_typeexpected_n_jobr,r,r/check_nested_backendbsz?test_nested_backend_in_sequential..check_nested_backendrrtc3s|] }ttdVqdSr)r#r*r)rr,r/rflrz4test_nested_backend_in_sequential..rc3s|] }tVqdSr;rrr9rrar,r/rfrrri)r9rarrr,rr/!test_nested_backend_in_sequentialZs  "rcCs\|| }|tur |d}|tur|d}|j|ksJWddS1s'wYdS)Nr9r)r$r% nesting_level)rr inner_backendexpected_levelctxr9r,r,r/check_nesting_levelxs "r outer_backendrcst|dtd|dfddtdDddtfddtdDWddS1s9wYdS) NrrBr`c3 |] }ttdVqdSrr#rrrrrr,r/rfs  z-test_backend_nesting_level..rrtc3rrrrrr,r/rfr)rr"rj)rrrrr,rr/test_backend_nesting_levels  "rwith_retrieve_callbackTcsddlGfdddt}td|dd|d;}tddfd d td D|tur:|d jd ks:J|turG|djd ksOJWddSWddS1sZwYdS)Nrcs"eZdZdZZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrur5rr,r,r/retrieval_contexts z;test_retrieval_context..MyBackend.retrieval_contextN)rOrPrQresupports_retrieve_callbackcontextmanagerrr,) contextlibrr,r/rs r retrievalcSstddddt|DS)NrBrtcsrbr;rrdr,r,r/rfrz>test_retrieval_context..nested_call..ri)nr,r,r/ nested_callsz+test_retrieval_context..nested_callrBrtc3s|] }t|VqdSr;rrd)rr,r/rfs  z)test_retrieval_context..rr9r)rrr&r"rjr$rer%)rrrrrr,)rrrr/test_retrieval_contexts   "rr:)rrsgQ?cCs8ttt|dWddS1swYdS)Nr:)rrTr"rr,r,r/test_invalid_batch_sizes  "rz)n_tasks, n_jobs, pre_dispatch, batch_size) )rBrBrrH)rBrBrarHrrBrarH)irBrarHr)rrrarH)r<rarH)r< 2 * n_jobsr)r<rr)rr<rrS)rr<rrHcCsN|||d}ddt|D}tdi|ddt|D}||ks%JdS)N)rar@r:cSrr,rrdr,r,r/r0rz0test_dispatch_race_condition..csrbr;rcrdr,r,r/rfrz/test_dispatch_race_condition..r,r)n_tasksrar@r:paramsrrr,r,r/test_dispatch_race_conditions  rcCs8t}tddd}|jd}|}||ksJdS)NrBr+r`rr)rget_start_methodr" _backend_argsget)mp_start_methodrrr start_methodr,r,r/test_default_mp_contexts   rcs^|dkr td}tjd}|ddtjtd|dfddt dDdS) Nr+r4rirBr`c3s"|] }ttjjVqdSr;)r#rdotTrdrHr,r/rfs zAtest_no_blas_crash_or_freeze_with_subprocesses..) rr3rrandom RandomStaterandnrrr"rj)r9rngr,rr/.test_no_blas_crash_or_freeze_with_subprocessess    rafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr+forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) rrrskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrr executable)r9coder,r,r/2test_parallel_with_interactively_defined_functionss    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r#rkwargsc Csv|dvr|tks tjdkrtdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r+r4win32zNot picklable with pickle)rr9rjoblib_root_folderzunpicklable_func_script.pyrrr) SQUARE_MAINrplatformrr)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrrpathdirnamejoblib__file__rwriterrstrpath)r9rrtmpdirr code_filer,r,r/0test_parallel_with_unpicklable_functions_in_argsCs     raimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyr)rr)rr5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrrrscriptr,r,r/7test_parallel_with_interactively_defined_functions_lokys    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\] LokyProcess)r stderr_regexr)rr9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrrrr,r,r/:test_parallel_with_interactively_defined_bound_method_lokys    rcCs"tg}tdd|gksJdS)NrBrt)iterr")exhausted_iteratorr,r,r/%test_parallel_with_exhausted_iteratorsrcCsttddS)z+Helper function to force gc in each worker.rN)rrWrr,r,r,r/_cleanup_workersrcCs"t|tjs tdt||S)Nz#Expected np.memmap instance, got %r) isinstancermemmap TypeErrorr]copyrr,r,r/ check_memmaps rcCsdd}tdd|ddd|dD}t||t|D] \}}tj||qtd d|dd d|dD}t||t|D] \}}tj||qBdS) Ncss*t|D] }tjdtjd|VqdS)Nrr)rjronesfloat32)rrer,r,r/generate_arrayss zBtest_auto_memmap_on_arrays_from_generator..generate_arraysrBrra max_nbytesr9csrbr;r#rr-rHr,r,r/rfrgz.r~rcsrbr;rrr,r,r/rfrg)r"rrrrr)r9rrrrr,r,r/)test_auto_memmap_on_arrays_from_generatorsrcCs|Sr;r,)argr,r,r/identityrMrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj s;Jd j |ksDJtj ||dS) Nz test.mmapuint8rrg) mmap_moderBrtc3rr;)r#rrrr,r/rfrz.test_memmap_with_big_offset..r r)rrmmapALLOCATIONGRANULARITYrzerosrr r r"rroffsetrr)rfnamesizeobjrr,rr/test_memmap_with_big_offsets   rcCstjdd}tdddddtdDWdn1s wYt|dks-J|d}t|jts9Jt|jd ksBJdS) NTrr)rarcsrbr;rcrdr,r,r/rfrzFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr"rjrrrrpr)rrr,r,r/3test_warning_about_timeout_not_supported_by_backends  rcCs |||<|Sr;r,) input_listrBrr,r,r/set_list_valuesr )rrBrcsVdgdt|dddfddtdD}t|}tddt|Ds)JdS) Nrr generatorr2ra return_asr9c3s |] }tt||VqdSr;)r#r rdr r,r/rf  zPtest_parallel_return_order_with_return_as_generator_parameter..cs|] \}}||kVqdSr;r,r-vrgr,r,r/rfr)r"rjr=rr)rarr,rr/=test_parallel_return_order_with_return_as_generator_parameters   rcCs|rtdt|S)Nr)rr)edelayr,r,r/_sqrt_with_delaysrcsvt|d|dddtdDtfddtdD}dgttd d}td dt||Ds5JtdS) Ngenerator_unorderedr css&|]}tt|d|dkVqdS)rBrN)r#rrdr,r,r/rf s zK_test_parallel_unordered_generator_returns_fastest_first..rc3s|]}tVqdSr;)nextrrr,r/rf#s rrBcsrr;r,rr,r,r/rf'rg)r"rjsortedr=rrr)r9raquickly_returnedexpected_quickly_returnedr,rr/8_test_parallel_unordered_generator_returns_fastest_firsts  rrB sequentialcCst||dSr;)rr9rar,r,r/7test_parallel_unordered_generator_returns_fastest_first/sr!zThis test requires daskc Csvtjddd*|d td|Wdn1swYWddSWddS1s4wYdSNrB n_workersthreads_per_workerdask) distributedClientr)rarrr,r,r/Atest_parallel_unordered_generator_returns_fastest_first_with_dask9s Pr))rrBr{rscCsvdgdgd}ttt}t||ddd|DWdn1s(wYt|}|dks9JdS)NrHrr~r`css|] }ttj|VqdSr;r#rWrrdr,r,r/rfKrz%test_abort_backend..r)rrrWr")rar9delayst_startdtr,r,r/test_abort_backendEs   r.cCstjtdtd}d|d<|S)NgArFr)rrintbool)rrr,r,r/get_large_objectQsr1cCsbt|||d }|ddtdD}t|t|~tWddS1s*wYdS)N)rar9r csrbr;)r#r1rdr,r,r/rf\rz0_test_deadlock_with_generator..r)r"rjrr)r9r rar rr,r,r/_test_deadlock_with_generatorWs"r2r r rcCst|||dSr;)r2)r9r rar,r,r/test_deadlock_with_generatoresr3c Csxtjddd+|dtd||Wdn1swYWddSWddS1s5wYdSr")r'r(r2)rrr rar,r,r/%test_deadlock_with_generator_and_daskmsPr4cCsttdd)t|||d}|ddtdD}t}|ddtdD}Wdn1s2wYt|d ksCJd ~tdS) N)This Parallel instance is already runningrr9r csrrrrr,r,r/rfrz/test_multiple_generator_call..rcsrbr;rrdr,r,r/rfrr~rBZThe error should be raised immediatly when submitting a new task but it took more than 2s.)rrr"rjrWr)r9r rar gr,gen2r,r,r/test_multiple_generator_callys r:c Cst|||dD}|ddtdD}t}ttdd|ddtdD}Wdn1s3wYt|d ksDJd Wdn1sNwY~tdS) Nr6csrrrrr,r,r/rfrz7test_multiple_generator_call_managed..rr5rcsrbr;rrdr,r,r/rfrr~rBr7)r"rjrWrrr)r9r rar r8r,g2r,r,r/$test_multiple_generator_call_manageds" r< return_as_1 return_as_2cCst|||dddtdD}t|||dddtddD}|dkr)t|}|dkr1t|}tddt|tdDsAJtd dt|tddDsRJdS) Nr6cs |] }tt|dVqdSrBNrrdr,r,r/rfrz9test_multiple_generator_call_separated..rcsr?r@rrdr,r,r/rfrrrcsrr;r,r-rrer,r,r/rfrcsrr;r,rAr,r,r/rfr)r"rjrrr)r9r=r>rar8r;r,r,r/&test_multiple_generator_call_separateds &rBzbackend, error))r1T)r2F)rFc Csb|dkr tdur tdtd||d}|ddtdD}t|dd }|r/ttd d nt }|9t }td||dd dtdd D}t |dkrUt |}t ddt|tdd DsfJWdn1spwYt |dksJd} |jr| dkr| d7} t d|jr| dkst |dksJ|dkr|jsJdSdS)Nr1zRequires multiprocessingrBr6csrrrrdr,r,r/rfrz.rcSstdS)NzGenerator collected)rr,r,r,r/z;test_multiple_generator_call_separated_gc..z The executor underlying Parallelrcsr?r@rrdr,r,r/rfrrrcsrr;r,rAr,r,r/rfrrrrhrrb)rrrr"rjweakreffinalizerrrrWrrrraliverrhr) r9r=r>errorr r8g_wrrr,retryr,r,r/)test_multiple_generator_call_separated_gcs< $   rKcCs |j}tdd||d.}|ddtjdgdDtt|dks'J|ddtdDWdn1s.rrcsrr;r#rrr,r,r/rfr~rz/temporary directory of Parallel was not removedrcsrbr;rrr,r,r/rf#rcsrr;rMrr,r,r/rf$rN) rr"rrrrlistdirrjrAssertionError)r9rrrr,r,r/test_memmapping_leakss,        rQ)Nr1r2cCs<td|dddtdD}|ddtdDksJdS)NrBr`css |] }tdd|VqdS)cSr@rAr,rCr,r,r/rC5rDz2test_lambda_expression...Nrrdr,r,r/rf4rz)test_lambda_expression..rcSsg|]}|dqS)rBr,rdr,r,r/r06rz*test_lambda_expression..ri)r9rr,r,r/test_lambda_expression.srRcsd}d}d|td||d}|fddt|D|jj|jjks&J|jj|jjks0J|fddt|D|jj|jjksGJ|jj|jjksQJd S) zCTest that a parallel backend correctly resets its batch statistics.rBig@r)r|rar9c3|] }ttjVqdSr;r*rd task_timer,r/rfBz6test_backend_batch_statistics_reset..c3rSr;r*rdrTr,r/rfHrVN)r"rjr_effective_batch_size_DEFAULT_EFFECTIVE_BATCH_SIZE_smoothed_batch_duration _DEFAULT_SMOOTHED_BATCH_DURATION)r9ran_inputsrr,rTr/#test_backend_batch_statistics_reset9s$ r\cCsdD];}tt|djtksJt|dd}t|jtusJt|dd}t|jtus.Jt|dd}t|jtus=Jqtdd dd }t|jtusNJ|d ddtdd }t|jtuscJ|jdksjJWdn1stwY|d ddtd dd}t|jtusJ|jd ksJWdn1swY|d ddtdd }t|jtusJ|jdksJWdn1swY|d ddtd dd}t|jtusJ|jd ksJWddS1swYdS)N)rrBrsrtthreadsraprefer processes sharedmemrarequirerBr1)rar9r_r_rhrcr)r]r"rrnrr!ra)rrrarr,r,r/$test_backend_hinting_and_constraintsOs<       "rfcCszGdddt}||&tddd}t|j|usJtddd}t|j|us,JWdn1s6wYGdd d t}||Otddd}t|j|usXJ|\}}|d ksdJ|d ksjJtddd d }t|jtuszJ|\}}d }||ksJ|d ksJWdn1swYttt|ddWddS1swYdS)Nc@$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSrFr;r,rr,r,r/ apply_asyncrMzgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccS|Sr;r,r`r,r,r/r'rMzltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrOrPrQsupports_sharedmem use_threadsrhr'r,r,r,r/MyCustomThreadingBackend  rmrBr`r^rarbc@rg)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSrFr;r,rr,r,r/rhrMzhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSrir;r,r`r,r,r/r'rMzmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNrjr,r,r,r/MyCustomProcessingBackendrnrorVr)rarcr|zUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r9rc) r r"r]r readouterrrstriprrT)capsysrrrmrroouterrrr,r,r/9test_backend_hinting_and_constraints_with_custom_backendss0        "rvcCstt tddWdn1swYtt tddWdn1s-wYtttdddWdn1sGwYtdurtttdddWdn1sewYtttd ddWddS1swYdSdS) Ninvalidrdrer`ra)r_rcr1rpr+)rrTr"rr,r,r,r/,test_invalid_backend_hinting_and_constraintss"       "rxc stdd4}t|jj|jjfg}dkr|WdS|fddtdD}||dWdS1s.r)r"r]rrOrrj)ryrr this_levelrr,r{r/rzs  $rzcCs||dd t}Wdn1swYtdkr#d}d}nd}d}|d}|df|dfd|fd|fg}||ksAJdS)NrBrtrrrBackendr)rzr(title)rrr9backend_types_and_levelssecond_level_backend_type max_leveltop_level_backend_typeexpected_types_and_levelsr,r,r/test_nested_parallelism_limits  rc Cstjdddptjtdtjd}tdD].}|d t|d}Wdn1s,wYt|dks9Jt dd |DsDJq|d t}Wdn1sWwYt|dksdJt d d |DsoJWddS1szwYdS) NrBr#gcArr&)datarcs|] \}}|dkVqdSDaskDistributedBackendNr,r-rrr,r,r/rfz4test_nested_parallelism_with_dask..csrrr,rr,r,r/rfr) r'r(rrr/rrjrzrr)rrrrerr,r,r/!test_nested_parallelism_with_dasks$    "rcCstddtdDS)z6A horrible function that does recursive parallel callscsrr;)r#_recursive_parallelrdr,r,r/rfrNz&_recursive_parallel..rBri) nesting_limitr,r,r/rsrc Cs||ddtt }tWdn1swYWdn1s(wY|j}|dkrFddlm}t||tfrFt dt|t sMJdS)NrBrtr1r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrr&joblib.externals.loky.process_executorrrrrrRecursionError)rrr9rXexcrr,r,r/test_thread_bomb_mitigation s   rcCs*i}dD] }tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSVECLIB_MAXIMUM_THREADSNUMEXPR_NUM_THREADSNUMBA_NUM_THREADS ENABLE_IPCr~)renvironrr)env_varsvarr,r,r/_run_parallel_sum's rzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]-\}}||ks#J|D]\}}|dr9|t|ks8Jq'|dks?J|dksEJq'qdS) NrBr`csrr;)r#rrr,r,r/rf3  z-test_parallel_thread_limit..r_THREADSr1)r"rjr_r(itemsendswithr)r9rexpected_num_threadsworker_env_varsomp_num_threadsrrr,r,r/test_parallel_thread_limit0s    rcCs:ttdd |dWddS1swYdS)NzPlease install daskrr&)rrTrxr,r,r/)test_dask_backend_when_dask_not_installedAs "rc CsGdddt}d}||6tjt|dtddddtdDWdn1s0wYWddSWddS1sHwYdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_dSr\r,)rGrrr,r,r/r^MrMz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSr)NzNo worker availabler)rGfunccallbackr,r,r/rhPr?z?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSrr\r,r`r,r,r/r'SrMzDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsr;)rOrPrQr^rhr'r,r,r,r/ZeroWorkerBackendLs  rz&ZeroWorkerBackend has no active workerrrBrtcsrbr;rrdr,r,r/rfYrz+test_zero_worker_backend..)rrrrr"rj)rrr expected_msgr,r,r/test_zero_worker_backendHs "rcsdadddks JtddfddtdD}t|dhks&Jdadks/Jtddfd dtdD}t|dhksHJdS) Nzoriginal valuecSstSr;)MY_GLOBAL_VARIABLEr,r,r,r/ check_globalsdsz@test_globals_update_at_each_parallel_call..check_globalsrBrtc3|]}tVqdSr;rrdrr,r/rfjrz.z changed valuec3rr;rrdrr,r/rfsr)rr"rjr)workers_global_variabler,rr/)test_globals_update_at_each_parallel_call\srcCs4ddl}|jdd}|||ddlm}|S)Nrr~)threadpool_info)numpyrrr threadpoolctlr)rrHrr,r,r/_check_numpy_threadpool_limits}s   rcCs4|D]}|d|dkr|dSqtd|)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rTr) child_module parent_info parent_moduler,r,r/_parent_max_num_threads_fors rcCs@|D]}|D]}t||}t|||h}|d|vsJqqdS)Nr)rmin) workers_inforrchild_threadpool_inforparent_max_num_threadsrr,r,r/check_child_num_threadssr)rBrr{rscCsdt}t|dkrtjddtd|dddtdD}t|}tt|d }t |||dS) Nr&Need a version of numpy linked to BLASrr1r csrr;r#rrdr,r,r/rfrz;test_threadpool_limitation_in_child_loky..rBr rrrrr"rjr'r_r(r)rarworkers_threadpool_infosexpected_child_num_threadsr,r,r/(test_threadpool_limitation_in_child_lokys  rinner_max_num_threads)rrBrNcCst}t|dkrtjdd|d|dt|dddtd D}Wdn1s.wYt|}|durDtt|d }n|}t |||dS) Nrrrr1rrtcsrr;rrdr,r,r/rfrz>test_threadpool_limitation_in_child_context..rBrr)rrrarrrrr,r,r/+test_threadpool_limitation_in_child_contexts   rvar_name)rrrc stddddtj}z_dtj<t|dfddtd D}|ddgks0J|d d d t|dfd dtd D}Wdn1sRwY|ddgks_JW|durjtj=dS|tj<dS|durztj=w|tj<w)NTreusecSs tj|Sr;)rrr)rr,r,r/_get_envrz>test_threadpool_limitation_in_child_override.._get_env4rtc3rr;rrdrrr,r/rfrgz?test_threadpool_limitation_in_child_override..rBr1rrc3rr;rrdrr,r/rfrgr)rshutdownrrrr"rj)rrraroriginal_var_valuerr,rr/,test_threadpool_limitation_in_child_overrides*     r)rBrrscCsJdd}||tdd}tdD]}||tdd}||ks"JqdS)Ncs,tdt|dfddtdDdS)Nrrtc3rr;)r#sumrdrCr,r/rfrzAtest_loky_reuse_workers..parallel_call..rrtr,rCr/ parallel_calls$z.test_loky_reuse_workers..parallel_callTrr)rrj)rarfirst_executorrexecutorr,r,r/test_loky_reuse_workerss   r)F)rr)rhr;)rRrrrWrrErr2 tracebackrmathrrpicklerrrr+r rrr r r joblib._multiprocessing_helpersrjoblib.test.commonrrrrrjoblib.testingrrrrrjoblib.externals.lokyrr#rposix ImportError _openmp_test_helper.parallel_sumrr'joblib._parallel_backendsrrrr r!joblib.parallelr"r#r$r%r&r'r(r)r*rRETURN_GENERATOR_BACKENDSpoprkeysALL_VALID_BACKENDSPROCESS_BACKENDSPARALLEL_BACKENDShasattrr"r3rnr:r>rD ExceptionrErVrYr[r_rlrnrqmarkrzrrrrrrrrrrrrrrrr rr!r%r?rGrLrMrOrSrUrr[r\rarerjrlrorqrwrzextendrjr{r|rrrr_parallel_backendsrrrrrrrrrrrrrr SQUARE_LOCAL SQUARE_LAMBDArrrrrrrrrrrrrrrrr rrrrr!r)r.r1r2r3r4r:r<rBrKrQrRr\rfrvrxrzrrrno_coverrrrrrrrrrrrrrr,r,r,r/s                                                 D                                   ( ,             2+  . 6          !