
    HWh"                     :   S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SK	r	S SK
r
S SKrS SKrS SKJrJr  S SKJr  S SKJr  S SKJrJrJr  S SKJr  S SKJrJr  S S	KJr  S S
KJr  S SK J!r!  S SK J"r"  S SK J#r#  S SK J$r$  S SK J%r%  S SK J&r&  S SK J'r(  S r)S r*S r+S r,S r-\\S 5       5       r.\\" \R                  S:g  =(       d    SSS9S 5       5       r/\\S 5       5       r0\S 5       r1\\\" S\\Rd                  /SS /S!9S" 5       5       5       r3\\\" S\\Rd                  /SS /S!9S# 5       5       5       r4\\\" S$SS /5      S% 5       5       5       r5\\\" S$SS /5      S& 5       5       5       r6\\\" S$SS /5      S' 5       5       5       r7\\\" S$SS /5      S( 5       5       5       r8\\S) 5       5       r9\\S* 5       5       r:\\\" S$SS /5      S+ 5       5       5       r;\\\" S$SS /5      S, 5       5       5       r<\\\" S$SS /5      S- 5       5       5       r=\\\" S\\Rd                  /SS /S!9S. 5       5       5       r>\\\" S$\R~                  " S\R                  R                  S/S9S09S /5      S1 5       5       5       rB\\\" S\\Rd                  /SS /S!9S2 5       5       5       rC\\\\" S\\Rd                  /SS /S!9S3 5       5       5       5       rD\\\\" S\\Rd                  /SS /S!9S4 5       5       5       5       rE\\\" S\\Rd                  /SS /S!9S5 5       5       5       rFS6 rG\\\" S\\Rd                  /SS /S!9S7 5       5       5       rHS8 rI\\\" S9\J" \R                  " \\Rd                  /\L" S:5      5      5      \R                  " SS /\M" \N\L" S:5      5      5       V Vs/ s H  u  pS;R                  X5      PM     snn S!9S< 5       5       5       rPS= rQS> rR\\" \R                  S:H  S?S9\" S@SASB/5      SC 5       5       5       rS\SD 5       rTSE rU\\SF 5       5       rVgs  snn f )G    N)sleep)
with_numpynp)with_multiprocessing)with_dev_shm)raisesparametrizeskipif)make_memmap)Paralleldelayed)MemmappingPool)_TestingMemmappingExecutorhas_shareable_memory)ArrayMemmapForwardReducer)_strided_from_memmap)_get_temp_dir)_WeakArrayKeyMap_get_backing_memmapc                  .    [         R                  " SSS9  g )Ni,  T)timeoutexit)faulthandlerdump_traceback_later     =/usr/lib/python3/dist-packages/joblib/test/test_memmapping.pysetup_moduler    !   s    %%c=r   c                  .    [         R                  " 5         g N)r   cancel_dump_traceback_laterr   r   r   teardown_moduler$   %   s    ,,.r   c                 "    [        U 5      c   eU $ r"   r   )arrays    r   check_memmap_and_send_backr'   )   s    u%111Lr   c                 R    U u  pn[         R                  R                  X   U5        g)zDummy helper function to be executed in subprocesses

Check that the provided array has the expected values in the provided
range.

Nr   testingassert_array_equalargsdatapositionexpecteds       r   check_arrayr1   .   s$      $DHJJ!!$.(;r   c                     U u  pnX   U:X  d   eX==   S-  ss'   [         R                  R                  X   SU-  5        g)zDummy helper function to be executed in subprocesses


Check that the input array has the right values in the provided range
and perform an inplace modification to double the values in the range by
two.

   Nr)   r,   s       r   inplace_doubler4   9   sD      $DH>X%%%NaNJJ!!$.!h,?r   c           	        ^ [         R                  R                  nU R                  S5      R                  n[         R
                  " U[         R                  SSS9nS[         R                  " UR                  S   UR                  S9-  USS& UR                  5         [         R
                  " U[         R                  S	S
SSS9n[         R                  " S5      R                  UR                  5      USS& USS2SS2SS24   n[         R                  " U5      nUR                  n[        SU R                  SS5      mU4S jnU" U5      n	[        U	5      (       d   e[!        U	[         R
                  5      (       d   eU" X5        U" U5      n
[        U
5      (       d   eU" X5        U" U5      n[!        U[         R
                  5      (       a   e[        U5      (       d   eU" X5        U" U5      n[!        U[         R
                  5      (       a   e[        U5      (       d   eU" X5        US-  n[        U5      (       a   eU" U5      n[        U5      (       a   e[!        U[         R
                  5      (       a   eU" XS-  5        [         R                  " U5      n[        U5      (       a   eU" U5      n[!        U[         R"                  5      (       d   e[        U5      (       a   eU" UU5        g)z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r9   N)         r+Fr?   )r9   r:   r;   orderoffset<      r3   cTc                 "   > T" U 5      u  pU" U6 $ r"   r   xconsr-   reducers      r   reconstruct_array_or_memmapEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmapi       QZ
T{r   r=   )r   r*   r+   joinstrpathmemmapfloat64aranger:   r9   flushreshapeasarrayTr   r   
isinstancendarray)tmpdirr+   filenamebufferabrG   drM   a_reconstructedb_reconstructedc_reconstructedd_reconstructeda3a3_reconstructedb3b3_reconstructedrL   s                    @r    test_memmap_based_array_reducingri   H   s    66{{;'//H YYxrzz4HF 		&,,q/FFF1I
LLN 			("**I3q	2A99R=  )AaD
 	
!B$"ac/A 	

1A	A (fnnc4HG
 2!4O0000oryy1111* 2!4O0000* 2!4O/29955550000*1!4O/29955550000* 
QB#B''''226#$45555*BII6666'Q/ 
BB#B''''226&

3333#$45555',r   win32r   z2PermissionError only easily triggerable on Windows)reasonc                    U R                  S5      R                  nSR                  US9n[        R                  " [
        R                  SU/[        R                  [        R                  S9nUR                  5         UR                  5       u  pEUR                  S:X  d   eUS:X  d   eSR                  U5      nXeR                  5       ;   d   eg )	Nr6   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )r\   -cstderrstdoutr   r   z'tried to unlink {}, got PermissionError)rP   rQ   format
subprocessPopensys
executablePIPEwaitcommunicate
returncodedecode)r[   r\   cmdpouterrmsgs          r   2test_resource_tracker_retries_when_permissionerrorr      s     {{;'//H: 	!; < 	#..$4Z__ *	1AFFH}}HC<<1#::
3
:
:8
DC**,r   c                 >  ^ [         R                  R                  nU R                  S5      R                  n[         R
                  " U[         R                  SSS9n[         R                  " S5      R                  UR                  5      US S & USS nUS S 2SS24   nUS S 2S S 2S S 2S4   nUS	S
S2   n[        S U R                  SS5      mU4S jnU" U5      n	[        U	5      (       d   e[        U	[         R
                  5      (       d   eU" X5        U" U5      n
[        U
5      (       d   eU" X5        U" U5      n[        U5      (       d   eU" X5        U" U5      n[        U5      (       d   eU" X5        U" U5      n[        U5      (       d   eU" X5        g )Nr6   )d      r   r=   r7   r8   i r   
   r>   rE   r=   r?   rG   Tc                 "   > T" U 5      u  pU" U6 $ r"   r   rI   s      r   rM   Ntest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap   rO   r   )r   r*   r+   rP   rQ   rR   rS   rT   rV   r:   r   r   rY   )r[   r+   r\   r^   r_   rG   r`   erM   ra   rb   rc   rd   e_reconstructedrL   s                 @r   )test_high_dimension_memmap_array_reducingr      s    66{{;'//H 			("**4D	A99&'//8AaD 	
!BA	!QrT'
A	!Q1*A	!Aa%A (fnnc4HG 2!4O0000oryy1111*1!4O0000*1!4O0000*1!4O0000*1!4O0000*r   c                    U R                  S5      R                  nS[        R                  -  n[        R                  S-   n[        R
                  " USX#-   S9n[        USSUSUS S S	S
9	n[        U[        R
                  5      (       d   eUR                  U:X  d   e[        USSUSUS-  4SUS	S
9	n[        U5      R                  U:X  d   eg )Nr6   r>   rE   r7   )r;   r:   uint8rCF)r9   r;   rC   rB   r:   stridestotal_buffer_lenunlink_on_gc_collectr3   )r3   )
rP   rQ   mmapALLOCATIONGRANULARITYr   rR   r   rY   rC   r   )r[   fnamesizerC   
memmap_objmemmap_backed_objs         r   test__strided_from_memmapr      s    KK$,,Et)))D''!+F5t4=AJ%e7-33d.2T;@BJ j")),,,,&&&,W3vSqylD4"
 0188FBBBr   factorymultiprocessingloky)idsc                    [         R                  R                  nUR                  S5      R                  nU " SSUS9n UR                  S5      R                  n[         R                  " U[         R                  SSS9nUR                  S	5        UR                  [        [        UR                  S
   5       VVs/ s H)  n[        UR                  S   5        H	  nXgU4S	4PM     M+     snn5        U" US[         R                  " UR                  5      -  5        [         R                  " U[         R                  SSS9n	UR                  [        [        U	R                  S
   5       VVs/ s H)  n[        U	R                  S   5        H	  nXU4S4PM     M+     snn5        [        R                  " U5      / :X  d   eU" US[         R                  " UR                  5      -  5        U" U	S[         R                  " U	R                  5      -  5        [         R                  " U[         R                  SSSS9n
[!        ["        5         UR                  [$        [        U
R                  S
   5       Vs/ s H  ozUS4PM	     sn5        SSS5        [!        [&        [(        45         UR                  [        [        U
R                  S
   5       Vs/ s H  ozUS4PM	     sn5        SSS5        UR+                  5         Ags  snnf s  snnf s  snf ! , (       d  f       N= fs  snf ! , (       d  f       NG= f! UR+                  5         Af = f)z@Check that subprocess can access and update shared memory memmappoolr   r3   
max_nbytestemp_folderr6   r=   r>   r7   r8         ?r   rE   )r>   r=   rG   g       @r   r      )r9   r:   r;   rC   g      @N)r   r*   r+   mkdirrQ   rP   rR   float32fillmapr4   ranger:   onesoslistdirr   AssertionErrorr1   RuntimeError
ValueError	terminate)r   r[   r+   pool_temp_folderr|   r\   r^   ijr_   rG   s              r   test_pool_with_memmapr   
  s    66 ||F+33q.>?A(;;{+33IIhbjjTJ	s	n(-aggaj(9;(91(-aggaj(91 !"q63/(9  0(9; 	< 	1a"''!''"223 IIhbjjSI	n(-aggaj(9;(91(-aggaj(91 !"q63/(9  0(9; 	< zz*+r111 	1a"''!''"2231a"''!''"223 IIhbjjC"$ N#EE+U1771:5FG5FAs5FGH $
 \:./EE.aggaj8I"J8I1q#;8I"JK 0 	
G;;"  H $# #K 0/ 	
s   A<L9 ;0L+BL9 -0LB-L9 
+L5LL
L9 '+L(L# L('L9 L9 L
L L9 #L((
L62L9 9Mc                    [         R                  R                  nUR                  S5      R                  nU " SSUS9n UR                  S5      R                  n[         R                  " U[         R                  SSS9nUR                  S	5        [         R                  " U5      n[        U[         R                  5      (       a   e[        U5      (       d   eUR                  [        [        UR                  S
   5       VV	s/ s H)  n[        UR                  S   5        H	  n	XxU	4S	4PM     M+     sn	n5        U" US[         R                   " UR                  5      -  5        U" US[         R                   " UR                  5      -  5        ["        R$                  " U5      / :X  d   e UR'                  5         Ags  sn	nf ! UR'                  5         Af = f)z?Check that subprocess can access and update shared memory arrayr   r   r3   r   r6   r   r7   r8   r   r   rE   N)r   r*   r+   r   rQ   rP   rR   r   r   rW   rY   r   r   r4   r   r:   r   r   r   r   )
r   r[   r+   r   r|   r\   r^   a_viewr   r   s
             r    test_pool_with_memmap_array_viewr   @  s}    66 ||F+33q.>?A;;{+33IIhbjjTJ	s Afbii0000#F++++	n(-aggaj(9;(91(-aggaj(91 !'A4(9  5(9; 	<
 	1a"''!''"22361rwwqww'7#78 zz*+r111 	
; 	
s   CG 0F?4A8G ?G Gbackendc                 ^   SR                  U S9n[        R                  " [        R                  SU/[        R
                  [        R
                  S9nUR                  5         UR                  5       u  p4UR                  S:X  d)   UR                  5       S-   UR                  5       -   5       eg )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r_   rm   rn   r   

)
rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r   r{   r|   r}   r~   s        r   -test_permission_error_windows_reference_cycler   g  s    $ 	% & 	#..$4Z__ *	1AFFH}}HC<<1BcjjlV3cjjlBBr   c                    SR                  U S9n[        S5       H  n[        R                  R	                  5       n[        R
                  R                  [        5      US'   [        R                  " [        R                  SU/[        R                  [        R                  US9nUR                  5         UR                  5       u  pVUR                  S:X  d   U5       eUS:X  d   e[        R                   S S S	;  d  M  S
U;  a  M   e   g )Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r=   
PYTHONPATHrm   ro   rp   envr   r   )r=      r   )r=   r   rE      resource_tracker)rq   r   r   environcopypathdirname__file__rr   rs   rt   ru   rv   rw   rx   ry   version_info)r   r{   _r   r|   r}   r~   s          r   3test_permission_error_windows_memmap_sent_to_parentr     s    ( 	) , 1XjjooGGOOH5LcnndC8$.OO$.OO> 	
==?||q %#% czzBQ'==
 'c111 r   c                 `  ^ [         R                  " [        S5      5      m[        SU SS9" U4S j[	        S5       5       5      u  n[        SU SS9" U4S j[	        S5       5       5      u  n[
        R                  R                  U5      [
        R                  R                  U5      :w  d   eg )N      Y@r3   r   n_jobsr   r   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr\   Nr   getattr.0r   r&   s     r   	<genexpr>6test_parallel_isolated_temp_folders.<locals>.<genexpr>  $      F5=
++X   #&rE   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   r     r   r   r   rT   intr   r   r   r   r   )r   
filename_1
filename_2r&   s      @r   #test_parallel_isolated_temp_foldersr     s     IIc#hE1g"E F5:1XF LZ 1g"E F5:1XF LZ 77??:&"''//**EEEEr   c                   ^ [         R                  " [        S5      5      m[        SU SS9 nU" U4S j[	        S5       5       5      u  nU" U4S j[	        S5       5       5      u  nS S S 5        [
        R                  R                  W5      [
        R                  R                  W5      :X  d   eg ! , (       d  f       NQ= f)Nr   r3   r   r   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  #      
9AAGGUJ//r   rE   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   r     r   r   r   )r   r|   r   r   r&   s       @r   &test_managed_backend_reuse_temp_folderr     s    
 IIc#hE	G	;q 
9>q
 
  
9>q
 
	 
< 77??:&"''//**EEEE 
<	;s   ;B22
C c                     [         R                  " [        S5      5      n [        5       n[        5       nS n[        R
                  " X0U4S9n[        R
                  " X0U4S9nUR                  5         UR                  5         UR                  5         UR                  5         [        U5      S:X  d   e[        U5      S:X  d   eX:w  d   eg )Nr   c                 
  ^  [        SSSS9 n[        S5       HN  nU" U 4S j[        S5       5       5      u  nUR                  [        R                  R                  U5      5        MP     S S S 5        g ! , (       d  f       g = f)Nr   r3   r   )r   r   r   c              3   P   >#    U  H  n[        [        5      " TS 5      v   M     g7fr   r   r   s     r   r   ]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>  s#      AIAGG$UJ77r   rE   )r   r   addr   r   r   )r&   	temp_dirsr|   r   r\   s   `    r   concurrent_get_filenameJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename  sc    fQ2>!2Y AFq 
 bggooh78	  ?>>s   AA44
B)targetr-   rE   )	r   rT   r   set	threadingThreadstartrP   len)r&   temp_dirs_thread_1temp_dirs_thread_2r   t1t2s         r   )test_memmapping_temp_folder_thread_safetyr     s     IIc#hE9 
		&5G-H
B 
		&5G-H
B HHJHHJGGIGGI!"a'''!"a'''333r   c            
         Sn / SQnU H  u  p#n[         R                  " [        R                  SU R	                  X#S9/[         R
                  [         R
                  S9nUR                  5         UR                  5       u  pgUR                  U:X  d   UR                  5       5       eSU;  a  M   UR                  5       5       e   g )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    ))parallel_get_filenamer   r   )r   parallel_raiserE   )r   r   rE   rm   )f1f2rn   r   )
rr   rs   rt   ru   rq   rv   rw   rx   ry   rz   )r{   functions_and_returncodesr   r   ry   r|   r}   r~   s           r   ?test_multithreaded_parallel_termination_resource_tracker_silentr    s    1Cd! 8
cnndCJJ"J4LM$.OOJOOM	==?||z)73::<7)"#-;szz|;- 8r   c                    SR                  U S9n[        R                  R                  5       n[        R                  R                  [        5      US'   [        R                  " [        R                  SU/[        R                  [        R                  US9nUR                  5         UR                  5       u  pEUR                  S:X  d   U5       eUS:X  d   e[        R                  S S S	;  a	  S
U;  d   eg g )Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   rm   r   r   r   r=   r   r   )rq   r   r   r   r   r   r   rr   rs   rt   ru   rv   rw   rx   ry   r   )r   r{   r   r|   r}   r~   s         r   'test_many_parallel_calls_on_same_objectr  L  s    $ 	% & **//
C1C	s#		A FFH}}HC<<1!c!#::
#99
 ##--- :r   c                    ^ [         R                  " [        S5      5      m[        SU SS9" U4S j[	        S5       5       5      u  n[        U5      b   eg )Ng     @@r3   r   r   c              3   N   >#    U  H  n[        [        5      " T5      v   M     g 7fr"   )r   r'   )r   r   r.   s     r   r   8test_memmap_returned_as_regular_array.<locals>.<genexpr>  s$      CE;Ca*+D118s   "%rE   )r   r   r   r   r   r   )r   resultr.   s     @r   %test_memmap_returned_as_regular_arrayr  |  sS     773s8Dq'cB CE;@8CE EHVv&...r   c                     U S:X  a:  [         R                  R                  S5      (       a  [        R                  " S5        SR                  U S9n[        R                  " [         R                  SU/[        R                  [        R                  S9nUR                  5         UR                  5       u  p4UR                  5       nUR                  5       nUR                  S:X  d   US	-   U-   5       eS
U;  d   U5       eg )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   rm   rn   r   r   resource_tracker)rt   platform
startswithpytestxfailrq   rr   rs   ru   rv   rw   rx   rz   ry   r   s        r   2test_resource_tracker_silent_when_reference_cyclesr    s    " &S\\44U;; 	,	

$ 	% & 	#..$4Z__ *	1AFFH}}HC
**,C
**,C<<10cFlS00S(-#-(r   c           	         [         R                  " UR                  5      / :X  d   eU " SSUR                  SS9n [         R                  " UR                  5      / :X  d   e[         R                  R	                  UR
                  5      (       a   e[        R                  " S[        R                  S9nUR                  S:X  d   eUR                  [        [        UR                  S   5       Vs/ s H  oCUS	4PM	     sn5        [         R                  " UR                  5      / :X  d   e[        R                  " S
[        R                  S9nUR                  S:X  d   eUR                  [        [        UR                  S   5       Vs/ s H  oEUS	4PM	     sn5        [         R                  R                  UR
                  5      (       d   e[         R                  " UR
                  5      n[!        U5      S:X  d   e[        R"                  " S/S
-  SS9nUR                  [$        U/5      nUS   (       a   e UR'                  5         [        S5       H?  n[)        S5        [         R                  R	                  UR
                  5      (       a  M>    Ag   [+        SR-                  UR
                  5      5      es  snf s  snf ! UR'                  5         [        S5       H?  n[)        S5        [         R                  R	                  UR
                  5      (       a  M>    Af    [+        SR-                  UR
                  5      5      e= f)z0Check that large arrays are not copied in memoryr=   (   r3   )r   r   verboser>   r<   r   r   r   r      rE   abcobjectr   皙?z#temporary folder {} was not deletedN)r   r   rQ   r   exists_temp_folderr   r   r   nbytesr   r1   r   r:   rS   isdirr   r&   r   r   r   r   rq   )	r   r[   r|   smallr   largedumped_filenamesobjectsresultss	            r   %test_memmapping_pool_for_large_arraysr!    s    ::fnn%+++ 	bfnnaHA(zz&..)R///77>>!..1111,||r!!!	kU5;;q>5JK5JAsO5JKL zz&..)R/// 2::.||s"""	kU5;;q>5JK5JAsO5JKL ww}}Q^^,,,,::ann5#$))) ((E7S=9%%,wi81:~: 	
rA"I77>>!..11
  
 !5<<Q^^L = L L  	
rA"I77>>!..11
  
 !5<<Q^^L s3   B5K	 0J?
>BK	 
K
B$K	 ?
K	 	AM#)Mz,https://github.com/joblib/joblib/issues/1086)marksc                 x   SR                  U S9n[        R                  R                  5       n[        R                  R                  [        5      US'   [        R                  " [        R                  SU/[        R                  [        R                  US9nUR                  5         UR                  5       u  pEUR                  5       UR                  5       pTUR                  S5      S   nUR                   S:X  d   U=(       d    U5       eUS:X  d   e[        R                  R#                  U5      (       a   eg )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(parallel_obj._backend._workers._temp_folder)
            else:
                return Path(parallel_obj._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   rm   r   
r    )rq   r   r   r   r   r   r   rr   rs   rt   ru   rv   rw   rx   rz   splitry   r  )r   r{   r   r|   r}   r~   r\   s          r   &test_child_raises_parent_exits_cleanlyr'    s    :-Z 	[ \ **//
C1C#..$4Z__ *S	:AFFH}}HCzz|SZZ\yyq!H<<1(cjS("99ww~~h'''''r   c           	         U " SSUR                   S9n [        R                  " UR                   5      / :X  d   e[        R                  " S[        R
                  S9nUR                  S:X  d   eUR                  [        [        UR                  S   5       Vs/ s H  oCUS4PM	     sn5        [        R                  " UR                   5      / :X  d   e UR                  5         Ags  snf ! UR                  5         Af = f)	z2Check that large arrays memmapping can be disabledr=   Nr   r   r<   r  r   r   )rQ   r   r   r   r   rS   r  r   r1   r   r:   r   )r   r[   r|   r  r   s        r   .test_memmapping_pool_for_large_arrays_disabledr)  N  s     	d?A zz&..)R/// 2::.||s"""	kU5;;q>5JK5JAsO5JKL zz&..)R/// 	
 L 	
s   BC, C'
'-C, 'C, ,C?c                    [         R                  n [        S5      [         l        U " SSS9n UR                  nSnUR	                  U5      (       d   e[
        R                  R                  U5      (       d   e[        R                  " S[        R                  S9nUR                  S:X  d   eUR                  [        U/S-  5        [        [
        R                  " U5      5      S	:X  d   e[        R                  " S[        R                  S9S
-  nUR                  S:X  d   eUR                  [        U/S-  5        [        [
        R                  " U5      5      S
:X  d   e UR!                  5         A[#        S5       H4  n[
        R                  R                  U5      (       d    O[%        S5        M6     ['        S5      e U[         l        g! UR!                  5         Af = f! U[         l        f = f)z1Check that memmapping uses /dev/shm when possibleg    ~Ar=   r   r   z"/dev/shm/joblib_memmapping_folder_r   r<   r  rE   r3   r  z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r  r  r   r   r  r   r   rS   r  r   idr   r   r   r   r   r   )r   	orig_sizer|   r   folder_prefixr^   r_   r   s           r   'test_memmapping_on_large_enough_dev_shmr1  i  s    11I+6 -0I)A"%	  !~~@M#..}====77>>"23333 2::.A88s?"?EE"qcBh rzz"234999
 2::.2A88s?"?EE"qcBhrzz"234999 KKMsA77>>"233"I	  !!KLL 
 -6) KKM -6)s$   G( D7G &AG( G%%G( (G5c                 z   [         R                  n [        S5      [         l        U " SSS9n UR                  nUR	                  S5      (       a   e UR                  5         A[        R                  R                  U5      (       a   e U[         l        g ! UR                  5         Af = f! U[         l        f = f)Ng@6Dr=   r   r+  z/dev/shm)	r,  r-  r   r  r  r   r   r   r  )r   r/  r|   r   s       r   $test_memmapping_on_too_small_dev_shmr3    s     11I6 -0J)A"%	  !~~'22:>>>>> KKM 77>>"233333,5) KKM
 -6)s"   B- $B 7B- B**B- -B:c                 f   [         R                  R                  nU " SSUR                  S9n UR	                  [         R
                  SS9nUR                  5       n[        U5      (       a   eU" U[         R
                  " S5      5        UR                  5         Ag! UR                  5         Af = f)z:Check that large arrays are not copied in memory in returnr=   r   r   )  r-   r5  N)	r   r*   r+   rQ   apply_asyncr   getr   r   )r   r[   r+   r|   resr  s         r   /test_memmapping_pool_for_large_arrays_in_returnr:    s     66 	bfnn=AmmBGG'm2	'....5"''$-0	 	
s   AB B0c                 .    [        U 5      (       d   eX-  $ )z4Multiplication function to be executed by subprocessr   )r^   n_timess     r   _worker_multiplyr=    s    """";r   c                    [         R                  R                  nU " SSUR                  S9n [         R                  " [         R
                  " S5      R                  S5      SS9SS2SS	2SS24   nUR                  [        US4S
9R                  5       n[        U5      (       a   eU" USU-  5        UR                  5         Ag! UR                  5         Af = f)zCheck that memmaps with a bad buffer are returned as regular arrays

Unary operations and ufuncs on memmap instances return a new memmap
instance with an in-memory buffer (probably a numpy bug).
r=   r   r   ip  )r5  r3   r=   rA   )rB   NrE   r6  )r   r*   r+   rQ   rW   rT   rV   r7  r=  r8  r   r   )r   r[   r+   r|   r^   r_   s         r   6test_workaround_against_bad_memmap_with_copied_buffersr?    s     66bfnn=A JJryy..|< ""#RaR(,
 MM*!QM8<<>'****1a!e$	 	
s   BC Cc                     U $ r"   r   )args    r   identityrB    s    Jr   zfactory,retry_nor=   z{}, {}c                    UR                  S5      R                  nS[        R                  -  n[        R                  S-   n[	        USUSUS9nU " SUR                  S9nUR                  [        U4S	9R                  5       n[        U[        R                  5      (       d   eUR                  U:X  d   e[        R                  R                  Xh5        UR                  5         g )
Nr6   r>   rE   r7   r   )r;   r:   r9   rC   r3   r   r6  )rP   rQ   r   r   r   r7  rB  r8  rY   r   rR   rC   r*   r+   r   )	r   retry_nor[   r   r   rC   objr|   r  s	            r    test_pool_memmap_with_big_offsetrG    s     KK$,,Et)))D''!+F
e$d'#%C 	v~~.A]]83&]1557Ffbii((((==F"""JJ!!#.KKMr   c                 "   Sn[        XR                  5      u  p#USL d   eX R                  S5      R                  :X  d   e[        US S9u  p#[        R                  R                  S5      (       a  USL d   eUR                  U5      (       d   eg )Ntest.tmpdirFrD  r
  )r   rQ   rP   rt   r  r  endswith)r[   pool_folder_namepool_folder
shared_mems       r   test_pool_get_temp_dirrN    s    $+,<nnMK++m4<<<<<+,<$OK
||u%%U""" 01111r   c                 D   SnSSK n[        UR                  R                  S5      (       a&  UR	                  UR                  R                  S5        [        USS9u  pE[        R                  R                  S5      (       a  USL d   eUR                  U5      (       d   eg)zXCheck that _get_temp_dir works when os.statvfs is not defined

Regression test for #902
rI  r   NstatvfsrD  r
  F)
joblib._memmapping_reducerhasattr_memmapping_reducerr   delattrr   rt   r  r  rJ  )r[   monkeypatchrK  joblibrL  rM  s         r   !test_pool_get_temp_dir_no_statvfsrW  $  s    
 %%v)),,i88F66999E+,<$OK
||u%%U""" 01111r   z1This test fails with a PermissionError on Windows	mmap_moder@   r7   c                    ^ S m[        S5       Vs/ s H  n[        R                  " SSS9PM     nn[        U SSS9" U4S	 j[	        U5       5       5      n[	        U5       H$  u  p[        R
                  R                  XA5        M&     g s  snf )
Nc                     XS S & U $ r"   r   )arrvalues     r   func4test_numpy_arrays_use_different_memory.<locals>.func:  s    A
r   r   )r   r   rS   r<   r   r3   )rX  r   r   c              3   J   >#    U  H  u  p[        T5      " X!5      v   M     g 7fr"   r   )r   r   r[  r]  s      r   r   9test_numpy_arrays_use_different_memory.<locals>.<genexpr>@  s$      D?,=&!c,=s    #)r   r   zerosr   	enumerater*   r+   )rX  r   arraysr   r[  r]  s        @r   &test_numpy_arrays_use_different_memoryre  5  s    
 <A9E9abhhxy19FEqC D?,5f,=D? ?G G$


%%c- % Fs   Bc            	      R   SS jn [         R                  " S5      n[        5       nUR                  US5        UR	                  U5      S:X  d   eUnUR	                  U5      S:X  d   eUR                  US5        UR	                  U5      S:X  d   eA[
        R                  " 5         [        UR                  5      S:X  d   eUR	                  U5      S:X  d   eAU " UR                  5        [         R                  " S5      nUR                  US5        [        UR                  5      S:X  d   eUR	                  U5      S:X  d   e[        [        5         UR	                  [         R                  " S5      5        S S S 5        AU " UR                  5        S n[        [        S	5       Vs/ s H
  oe" X&5      PM     sn5      n[        R                  " 5       S
:X  a6  [        [        R                   SS5      (       a  SOSn[        U5      U:  d   eg g ! , (       d  f       N= fs  snf )Nr   c                     [        U5       H4  n[        U 5      S:X  a    g [        R                  " 5         [	        S5        M6     [        U 5      S:X  d   eg )Nr   r  )r   r   gccollectr   )	containerretriesr   s      r   assert_empty_after_gc_collect>test_weak_array_key_map.<locals>.assert_empty_after_gc_collectJ  sC    wA9~"JJL"I	  
 9~"""r   *   r^   r_   rE   rG   c                    [         R                  " S5      n[        [        5         U R	                  U5        S S S 5        U R                  X!5        U R	                  U5      U:X  d   e[        U5      $ ! , (       d  f       NA= f)Nrn  )r   r   r   KeyErrorr8  r   r.  )mr   r^   s      r   get_set_get_collect4test_weak_array_key_map.<locals>.get_set_get_collectq  sX    GGBKHEE!H 	auuQx1}}!u	 s   A33
Br5  CPythonnogilFi  )r   )r   r   r   r   r8  rh  ri  r   _datar   rp  r   r  python_implementationr   rt   flags)	rl  r^   rq  r_   rG   rr  r   
unique_idsmax_len_unique_idss	            r   test_weak_array_key_mapr{  G  s   # 	AAEE!SM558s??	A558s??EE!SM558s??	JJLqww<1558s??	!!''*
AEE!SMqww<1558s??			bggbk 
 	
!!''* tEA)!/EFJ%%'94 %,CIIw$F$FSC:!3333 5# 
	  Fs   &H+H$
H!c                      [        5       n [        [        R                  5         [        R                  " U 5        S S S 5        g ! , (       d  f       g = fr"   )r   r   picklePicklingErrordumps)rq  s    r   #test_weak_array_key_map_no_picklingr    s/    A	$$	%Q 
&	%	%s   A
Ac                   ^^^^ [        U R                  S5      5      m[        R                  " SSS9nUR	                  T5        U4S jmS mT" 5       n[        SS9" U4S	 jU4 5       5      n[        SS9" U4S
 jU4 5       5      n[        R                  R                  XC5        U4S jm[        SS9" U4S j[        S5       5       5      n[        R                  R                  US   U5        g )Nzarr.datr   r   r<   c                     > [        T5       n [        R                  " U R                  5       S[        R                  SS9nS S S 5        [        R
                  " S[        R                  WSS9$ ! , (       d  f       N3= f)Nr   )accessrC   r   )r9   r]   rC   )openr   filenoACCESS_READr   rZ   r   )fdmmtestfiles     r   _read_array%test_direct_mmap.<locals>._read_array  sR    (^r299;$2B2B1MB zz%rxx1EE ^s   4A..
A<c                     U S-  $ )Nr3   r   )rJ   s    r   r]  test_direct_mmap.<locals>.func  s    !tr   r3   )r   c              3   F   >#    U  H  n[        T5      " U5      v   M     g 7fr"   r`  r   rJ   r]  s     r   r   #test_direct_mmap.<locals>.<genexpr>  s     ;s!WT]1--s   !c              3   F   >#    U  H  n[        T5      " U5      v   M     g 7fr"   r`  r  s     r   r   r    s      A5aq!1!15r  c                     > T " 5       $ r"   r   )r  s   r   worker test_direct_mmap.<locals>.worker  s    }r   c              3   D   >#    U  H  n[        T5      " 5       v   M     g 7fr"   r`  )r   r   r  s     r   r   r    s      EHq!2!2Hs    rE   r   )	strrP   r   rT   tofiler   r*   r+   r   )	r[   r^   r[  refr   r  r]  r  r  s	        @@@@r   test_direct_mmapr    s     6;;y)*H
		"G$AHHXF
 -C !
;s;
;C a  AC5 AAGJJ!!'/ a  EE!H EEGJJ!!'!*c2r   )Wr   r   rt   r  rh  r}  	itertoolstimer   rr   r   r   r  joblib.test.commonr   r   r   r   joblib.testingr   r	   r
   joblib.backportsr   joblib.parallelr   r   joblib.poolr   joblib.executorr   TestExecutorrQ  r   r   r   r   r   r   rS  r,  r    r$   r'   r1   r4   ri   r   r   r   get_memmapping_executorr   r   r   r   r   r   r   r  r  r  r  r!  parammarkr  r'  r)  r1  r3  r:  r=  r?  rB  listproductr   r   r  rq   rG  rN  rW  re  r{  r  r  )rJ   ys   00r   <module>r     sA   	  
  	        - 3 + 6 6 ( - & F ; @ ; 4 7 : (>/
<@ I-  I-X 	 'RCE)E )X *+  *+Z C C* Y)M)MN#V,./.  /d Y)M)MN#V,. .   F Y*F34C 5  C@ Y*F34,2 5  ,2^ Y*F34
F 5  
F Y*F34
F 5  
F #4  #4L C<  C<L Y*F34*. 5  *.Z Y*F34/ 5  / Y*F341. 5  1.h Y)M)MN#V,.1.  1h ++##E $ 	
 	G(  G(T Y)M)MN#V,..  . Y)M)MN#V,..6.   
.6b Y)M)MN#V,.6.   
60 Y)M)MN#V,..  , Y)M)MN#V,..  2 			==>a
J K+4+<+<	F#SeAh%7,9 	: ,941	 ,9 	:	;;  &	22"  )% &[4,'. (& . 94 94x 3  3S	:s   P