U
    ÎZj¤¡  ã                   @   s   d Z ddlZddlmZ ddlZddlZddlmZmZ ddl	Z
ddlZddlmZ ddlmZ ddlmZmZmZmZ ddlm  mZ ddlZddlmZ ddlmZ dd	l m!Z!m"Z"m#Z#m$Z$m%Z% zddl&Z&d
Z'W n e(k
rî   dZ'Y nX z>eƒ �* edde)d� edde*d� ddl+Z+W 5 Q R X d
Z,W n e(k
�rH   dZ,Y nX ej- d¡Z.ej/ej0dej-j1e, �pxedƒdkdd�d�ej0dej-j1e' dd�d�gd�dd„ ƒZ2ej/dd„ ƒZ3ej/dd„ ƒZ4ej/d d!„ ƒZ5ej/d"d#„ ƒZ6ej/d$d%„ ƒZ7ej/ej 8ej9j:¡ej 8ej9j;¡ej 8ej9j<¡ej =d&d'¡ej =d(d'¡ej =d)d'¡ej =d*d'¡gd�d+d,„ ƒZ>dKd.d/„Z?d0d1„ Z@d2d3„ ZAd4d5„ ZBd6d7„ ZCd8d9„ ZDd:d;„ ZEd<d=„ ZFd>d?„ ZGd@dA„ ZHG dBdC„ dCƒZIG dDdE„ dEeIƒZJej- dF¡G dGdH„ dHeIƒƒZKG dIdJ„ dJeIƒZLdS )Lz test parquet compat é    N)ÚBytesIO)Úcatch_warningsÚfilterwarnings)Ú
get_option)Úis_platform_windows)Úpa_version_under2p0Úpa_version_under5p0Úpa_version_under6p0Úpa_version_under8p0)ÚVersion)ÚFastParquetImplÚPyArrowImplÚ
get_engineÚread_parquetÚ
to_parquetTFÚignorez	`np.bool`)Úcategoryz.*Int64Index.*z4ignore:RangeIndex.* is deprecated:DeprecationWarningÚfastparquetúmode.data_managerÚarrayz4fastparquet is not installed or ArrayManager is used©Úreason)ZmarksÚpyarrowúpyarrow is not installed)Úparamsc                 C   s   | j S ©N©Úparam©Úrequest© r    ú_/var/www/html/TRUCKING_PROJECT/venv/lib/python3.8/site-packages/pandas/tests/io/test_parquet.pyÚengineB   s    r"   c                   C   s   t st d¡ dS )Nr   r   )Ú_HAVE_PYARROWÚpytestÚskipr    r    r    r!   ÚpaW   s    
r&   c                   C   s*   t st d¡ ntdƒdkr&t d¡ dS )Nzfastparquet is not installedr   r   z.ArrayManager is not supported with fastparquetr   )Ú_HAVE_FASTPARQUETr$   r%   r   r    r    r    r!   Úfp^   s
    
r(   c                   C   s   t  dddgddœ¡S )Né   é   é   Úfoo©ÚAÚB)ÚpdÚ	DataFramer    r    r    r!   Ú	df_compatg   s    r2   c               
   C   sD   t  tdƒttddƒƒtjdddd�dd	dgt jd
dd�dœ¡} | S )NÚabcr)   é   ç      @ç      @Úfloat64©ÚdtypeTFÚ20130101r+   ©Úperiods)ÚaÚbÚdÚeÚf)r0   r1   ÚlistÚrangeÚnpÚarangeÚ
date_range)Údfr    r    r!   Údf_cross_compatl   s    úÿrH   c                   C   s’   t  tdƒdtjdgdd dgdddgddd	gttd
dƒƒt dd¡ d¡tjdddd�dtjdgdddgt jddd�t  	d¡t j
t  	d¡gdœ¡S )Nr3   r=   Úcó   fooó   bars   bazr,   ÚbarÚbazr)   r4   r+   é   Úu1r5   r6   r7   r8   ç       @ç      @TFr:   r;   Z20130103)ÚstringZstring_with_nanZstring_with_noneÚbytesÚunicodeÚintZuintÚfloatZfloat_with_nanÚboolÚdatetimeZdatetime_with_nat)r0   r1   rB   rD   ÚnanrC   rE   ÚastyperF   Ú	TimestampZNaTr    r    r    r!   Údf_full~   s$    

ýôÿr\   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                 C   s   | j S r   r   r   r    r    r!   Útimezone_aware_date_list–   s    r]   r*   c
              	      s€   ˆp
ddi‰ˆpi ‰ˆdkr ˆ‰|r4|ˆd< |ˆd< ‡ ‡‡‡‡‡‡‡fdd„}
ˆdkrtt  ¡ �‰|
|	ƒ W 5 Q R X n|
|	ƒ dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    ÚcompressionNr"   c              
      sV   t | ƒD ]H}ˆjˆfˆŽ tdd�� tˆfˆŽ}W 5 Q R X tjˆ|ˆˆˆ d� qd S )NT©Úrecord)Úcheck_namesÚ
check_likeÚcheck_dtype)rC   r   r   r   ÚtmÚassert_frame_equal)ÚrepeatÚ_Úactual©rc   rb   ra   rG   ÚexpectedÚpathÚread_kwargsÚwrite_kwargsr    r!   ÚcompareÑ   s    ûz!check_round_trip.<locals>.compare)rd   Úensure_clean)rG   r"   rk   rm   rl   rj   ra   rb   rc   rf   rn   r    ri   r!   Úcheck_round_trip¥   s    "
rp   c                 C   s|   t rLddlm} |j| dd�}t|jjƒt|ƒks6t‚|jjt|ƒksxt‚n,ddl	m
} |j
| dd�}|jjj|ksxt‚dS )zÂCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   NF)Zvalidate_schemaZhive)Úpartitioning)r   Úpyarrow.parquetÚparquetZParquetDatasetÚlenZ
partitionsZpartition_namesÚAssertionErrorÚsetZpyarrow.datasetÚdatasetrq   ÚschemaÚnames)rk   rj   Úpqrw   Zdsr    r    r!   Úcheck_partition_namesæ   s    
r{   c              	   C   s.   d}t jt|d�� t| ddƒ W 5 Q R X d S )Nz.engine must be one of 'pyarrow', 'fastparquet'©Úmatchr,   rL   )r$   ÚraisesÚ
ValueErrorrp   )r2   Úmsgr    r    r!   Útest_invalid_engineý   s    r�   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Núio.parquet.enginer   ©r0   Úoption_contextrp   )r2   r&   r    r    r!   Útest_options_py  s    r…   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Nr‚   r   rƒ   )r2   r(   r    r    r!   Útest_options_fp
  s    r†   c              	   C   s$   t  dd¡� t| ƒ W 5 Q R X d S )Nr‚   Úautorƒ   )r2   r(   r&   r    r    r!   Útest_options_auto  s    rˆ   c              	   C   s  t tdƒtƒst‚t tdƒtƒs$t‚t dd¡�< t tdƒtƒsDt‚t tdƒtƒsVt‚t tdƒtƒsht‚W 5 Q R X t dd¡�< t tdƒtƒs’t‚t tdƒtƒs¤t‚t tdƒtƒs¶t‚W 5 Q R X t dd¡�> t tdƒtƒsàt‚t tdƒtƒsòt‚t tdƒtƒ�st‚W 5 Q R X d S )Nr   r   r‚   r‡   )Ú
isinstancer   r   ru   r   r0   r„   )r(   r&   r    r    r!   Útest_options_get_engine  s    rŠ   c               	   C   s.  ddl m}  |  d¡}|  d¡}ts(dnttjƒt|ƒk }tsBdnttjƒt|ƒk }to\| }tof| }|�s*|�s*|r¨d|› d�}t	j
t|d�� td	ƒ W 5 Q R X n&d
}t	j
t|d�� td	ƒ W 5 Q R X |�rd|› d�}t	j
t|d�� td	ƒ W 5 Q R X n&d}t	j
t|d�� td	ƒ W 5 Q R X d S )Nr   )ÚVERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.r|   r‡   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)Zpandas.compat._optionalr‹   Úgetr#   r   r   Ú__version__r'   r   r$   r~   ÚImportErrorr   )r‹   Z
pa_min_verZ
fp_min_verZhave_pa_bad_versionZhave_fp_bad_versionZhave_usable_paZhave_usable_fpr}   r    r    r!   Ú"test_get_engine_auto_error_message,  s8    

ÿýÿý

r�   c              	   C   sj   | }t  ¡ �T}|j||d d� t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W 5 Q R X d S )N©r"   r^   ©r"   r=   r?   ©r"   Úcolumns)rd   ro   r   r   re   ©rH   r&   r(   rG   rk   Úresultr    r    r!   Útest_cross_engine_pa_fpW  s    
r–   c              
   C   s€   | }t  ¡ �j}|j||d d� tdd��D t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W 5 Q R X W 5 Q R X d S )Nr�   Tr_   r‘   r=   r?   r’   )rd   ro   r   r   r   re   r”   r    r    r!   Útest_cross_engine_fp_pae  s    
r—   c                   @   s:   e Zd Zdd„ Zdd„ Zejjejddd�dd	„ ƒƒZ	d
S )ÚBasec              
   C   sB   t  ¡ �0}tj||d�� t|||d d� W 5 Q R X W 5 Q R X d S )Nr|   ©r^   )rd   ro   r$   r~   r   )ÚselfrG   r"   ÚexcÚerr_msgrk   r    r    r!   Úcheck_error_on_writet  s    
zBase.check_error_on_writec              
   C   s>   t  ¡ �,}t  |¡� t|||d d� W 5 Q R X W 5 Q R X d S )Nr™   )rd   ro   Úexternal_error_raisedr   )rš   rG   r"   r›   rk   r    r    r!   Úcheck_external_error_on_writez  s    
z"Base.check_external_error_on_writeúdhttps://raw.githubusercontent.com/pandas-dev/pandas/main/pandas/tests/io/data/parquet/simple.parquetT)ÚurlZcheck_before_testc                 C   s.   |dkrt  |¡ d}t|ƒ}t ||¡ d S )Nr‡   r    )r$   Úimportorskipr   rd   re   )rš   r2   r"   r¡   rG   r    r    r!   Útest_parquet_read_from_url€  s    	
ÿzBase.test_parquet_read_from_urlN)
Ú__name__Ú
__module__Ú__qualname__r�   rŸ   r$   ÚmarkÚnetworkrd   r£   r    r    r    r!   r˜   s  s   ûr˜   c                   @   sÂ   e Zd Zdd„ Zdd„ Zdd„ Zej ddd	d
dg¡dd„ ƒZ	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zej d$d%d&d'd(d)d*d+d,d-g	¡d.d/„ ƒZdS )0Ú	TestBasicc              	   C   sJ   t  dddg¡ddt  d¡t dddg¡fD ]}d}|  ||t|¡ q,d S )Nr)   r*   r+   r,   r:   z+to_parquet only supports IO with DataFrames)r0   ÚSeriesr[   rD   r   r�   r   )rš   r"   Úobjr€   r    r    r!   Ú
test_error”  s    ûzTestBasic.test_errorc                 C   s6   t  tdƒttddƒƒdœ¡}ddg|_t||ƒ d S )Nr3   r)   r4   ©rR   rU   r,   rL   )r0   r1   rB   rC   r“   rp   )rš   r"   rG   r    r    r!   Útest_columns_dtypesŸ  s    
zTestBasic.test_columns_dtypesc                 C   s�   t  tdƒttddƒƒdœ¡}d}ddg|_|  ||t|¡ ddg|_|  ||t|¡ t d	dddd¡t d	dddd¡g|_|  ||t|¡ d S )
Nr3   r)   r4   r­   ú%parquet must have string column namesr   rJ   rK   iÛ  )r0   r1   rB   rC   r“   r�   r   rX   )rš   r"   rG   r€   r    r    r!   Útest_columns_dtypes_invalid¦  s    

þz%TestBasic.test_columns_dtypes_invalidr^   NÚgzipÚsnappyÚbrotlic                 C   sP   |dkrt  d¡ n|dkr&t  d¡ t ddddgi¡}t||d|id� d S )	Nr²   r³   r.   r)   r*   r+   r^   ©rm   )r$   r¢   r0   r1   rp   )rš   r"   r^   rG   r    r    r!   Útest_compression¹  s    
zTestBasic.test_compressionc                 C   sJ   t  tdƒttddƒƒdœ¡}t  dtdƒi¡}t|||ddgid� d S )Nr3   r)   r4   r­   rR   r“   ©rj   rl   )r0   r1   rB   rC   rp   )rš   r"   rG   rj   r    r    r!   Útest_read_columnsÅ  s       ÿzTestBasic.test_read_columnsc                 C   s¦   |dk}t  ddddgi¡}t||ƒ dddgt jddd�td	ƒdddgg}|D ]2}||_t|t jƒrt|j d ¡|_t|||d
� qPdddg|_d|j_	t||ƒ d S )Nr   r.   r)   r*   r+   r4   r:   r;   r3   )ra   r   r,   )
r0   r1   rp   rF   rB   Úindexr‰   ZDatetimeIndexÚ
_with_freqÚname)rš   r"   ra   rG   Zindexesr¸   r    r    r!   Útest_write_indexÎ  s     
üzTestBasic.test_write_indexc                 C   s>   |}t  ddddgi¡}t j dddg¡}||_t||ƒ d S )Nr.   r)   r*   r+   ©r=   r)   ©r=   r*   ©r>   r)   )r0   r1   Ú
MultiIndexÚfrom_tuplesr¸   rp   )rš   r&   r"   rG   r¸   r    r    r!   Útest_write_multiindexæ  s
    zTestBasic.test_write_multiindexc                 C   s¢   |}t jdddd�}t jtj dt|ƒ d¡tdƒd�}t jj	d	d
g|gddgd�}|j
d d�}||fD ]4}||_t||ƒ t||dddgi|ddg d� qhd S )Nz01-Jan-2018z01-Dec-2018ZMS)Úfreqr*   r+   ÚABC©r“   ÚLevel1ÚLevel2ÚlevelÚdate)ry   r“   r.   r/   ©rl   rj   )r0   rF   r1   rD   ÚrandomÚrandnrt   rB   r¿   Zfrom_productÚcopyr¸   rp   )rš   r&   r"   ÚdatesrG   Zindex1Úindex2r¸   r    r    r!   Útest_multiindex_with_columnsï  s"    $
 ÿ
  
 
ÿz&TestBasic.test_multiindex_with_columnsc              	   C   sê   t  dddgdddgdœ¡}d dd	œ}|jd
d�}t||||d� t jdddgdddgdœdddgd�}t||||d� ddddddddgddddddddgg}t jttdƒƒdd„ tdƒD ƒdœ|d�}|jd
d�}t||||d� d S )Nr)   r*   r+   ÚqÚrÚs)r=   r>   F)r^   r¸   T)Údrop©rm   rj   ZzyxZwvuZtsr)r¸   rL   rM   r,   ÚquxÚoneÚtwoé   c                 S   s   g | ]
}| ‘qS r    r    )Ú.0Úir    r    r!   Ú
<listcomp>  s     z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>)rÖ   r×   )r0   r1   Zreset_indexrp   rB   rC   )rš   r"   rG   rm   rj   Úarraysr    r    r!   Útest_write_ignoring_indexÿ  s$    
 ÿþ ÿz#TestBasic.test_write_ignoring_indexc                 C   sB   t j dddg¡}t jtj dd¡|d�}d}|  ||t|¡ d S )Nr¼   r½   r¾   r4   r+   rÄ   úY\s*parquet must have string column names for all values in\s*each level of the MultiIndex)	r0   r¿   rÀ   r1   rD   rÊ   rË   r�   r   )rš   r"   Z
mi_columnsrG   r€   r    r    r!   Útest_write_column_multiindex  s
    ÿz&TestBasic.test_write_column_multiindexc              	   C   sh   |}ddddddddgddddddddgg}t jtj dd¡|d�}d	d
g|j_d}|  ||t|¡ d S )NrL   rM   r,   rÕ   r)   r*   rØ   rÄ   rÅ   rÆ   rÞ   )	r0   r1   rD   rÊ   rË   r“   ry   r�   r   ©rš   r&   r"   rÜ   rG   r€   r    r    r!   Ú&test_write_column_multiindex_nonstring)  s    þÿz0TestBasic.test_write_column_multiindex_nonstringc              	   C   s^   |}ddddddddgddddddddgg}t jtj dd¡|d�}d	d
g|j_t||ƒ d S )NrL   rM   r,   rÕ   rÖ   r×   rØ   rÄ   Z	ColLevel1Z	ColLevel2)r0   r1   rD   rÊ   rË   r“   ry   rp   ©rš   r&   r"   rÜ   rG   r    r    r!   Ú#test_write_column_multiindex_string;  s    þz-TestBasic.test_write_column_multiindex_stringc                 C   s>   |}ddddg}t jtj dd¡|d�}d|j_t||ƒ d S )	NrL   rM   r,   rÕ   rØ   r4   rÄ   Z	StringCol)r0   r1   rD   rÊ   rË   r“   rº   rp   râ   r    r    r!   Útest_write_column_index_stringJ  s
    z(TestBasic.test_write_column_index_stringc                 C   sH   |}ddddg}t jtj dd¡|d�}d|j_d}|  ||t|¡ d S )	Nr)   r*   r+   r4   rØ   rÄ   ZNonStringColr¯   )	r0   r1   rD   rÊ   rË   r“   rº   r�   r   rà   r    r    r!   Ú!test_write_column_index_nonstringV  s    z+TestBasic.test_write_column_index_nonstringc           
      C   sÖ  dd l m} |dkr.tjjdd�}|j |¡ t t 	dddd gd¡t 	dddd gd	¡t 	d
ddd g¡t 	dddd g¡t 	ddddgd¡t 	dddd gd¡t 	dddd gd¡dœ¡}t
 ¡ �,}| ||¡ t||d�}t||dd�}W 5 Q R X |d
 jt d¡k�st‚t tj	dddd gdd�tj	dddd gdd�tj	d
ddd gdd�tj	dddd gdd�tj	ddddgdd�tj	dddd gdd�tj	dddd gdd�dœ¡}	|dk�rÆ|jddd�}|	jddd�}	t
 ||	¡ d S ) Nr   r   z.Fastparquet nullable dtype support is disabledr   r)   r*   r+   Zint64Zuint8r=   r>   rI   TFr4   ç      ð?rP   rQ   Zfloat32r7   )r=   r>   rI   r?   r@   rA   Úgr‘   ©r"   Úuse_nullable_dtypesÚInt64r8   ÚUInt8rR   ÚbooleanZFloat32ÚFloat64)Zaxis)rr   rs   r$   r§   ÚxfailÚnodeÚ
add_markerr   Útabler   rd   ro   Zwrite_tabler   r9   rD   ru   r0   r1   rÓ   re   )
rš   r"   r   rz   r§   rñ   rk   Zresult1Zresult2rj   r    r    r!   Útest_use_nullable_dtypesb  sF    ÿ÷ÿ
ùÿ
z"TestBasic.test_use_nullable_dtypesr9   rê   rë   rì   Úobjectzdatetime64[ns, UTC]rV   z	period[D]rí   rR   c                 C   sT   t  dt jg |d�i¡}d }|dkr<t  dt jg dd�i¡}t||ddi|d� d S )NÚvaluer8   rV   rí   ré   TrÉ   )r0   r1   r   rp   )rš   r&   r9   rG   rj   r    r    r!   Útest_read_empty_array“  s$     ÿÿ ÿÿ   ÿzTestBasic.test_read_empty_array)r¤   r¥   r¦   r¬   r®   r°   r$   r§   Úparametrizerµ   r·   r»   rÁ   rÏ   rÝ   rß   rá   rã   rä   rå   rò   rõ   r    r    r    r!   r©   “  s<   
		 
1÷þr©   z8ignore:CategoricalBlock is deprecated:DeprecationWarningc                   @   s”  e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	e
jjeƒ dd�e
j deejg¡dd„ ƒƒZdd„ Ze
jjdd„ ƒZe
jjdd„ ƒZe
jje d¡e
j ddgg g¡dd„ ƒƒƒZe d¡d d!„ ƒZe d¡d"d#„ ƒZd$d%„ Zd&d'„ Ze
j deejg¡d(d)„ ƒZd*d+„ Zd,d-„ Z e d¡d.d/„ ƒZ!ejdd0d1�d2d3„ ƒZ"ejdd4d1�d5d6„ ƒZ#d7d8„ Z$d9d:„ Z%ejdd0d1�d;d<„ ƒZ&d=d>„ Z'd?S )@ÚTestParquetPyArrowc                 C   sB   |}t jdddd�}| d ¡}||d< dd dg|d< t||ƒ d S )Nr:   r+   úEurope/Brussels©r<   ÚtzÚdatetime_tzTZbool_with_none)r0   rF   r¹   rp   )rš   r&   r\   rG   Údtir    r    r!   Ú
test_basic·  s    
zTestParquetPyArrow.test_basicc                 C   s<   |}t jdddd�|d< t|||ddg dddgid	� d S )
Nr:   r+   rø   rù   rû   rR   rU   r“   r¶   )r0   rF   rp   )rš   r&   r\   rG   r    r    r!   Útest_basic_subset_columnsÃ  s    

üz,TestParquetPyArrow.test_basic_subset_columnsc                 C   s:   |j |d�}t|tƒst‚t|ƒ}t|ƒ}t ||¡ d S )Nr‘   )r   r‰   rS   ru   r   r   rd   re   )rš   r&   r\   Z	buf_bytesZ
buf_streamÚresr    r    r!   Ú*test_to_bytes_without_path_or_buf_providedÑ  s
    z=TestParquetPyArrow.test_to_bytes_without_path_or_buf_providedc                 C   s8   t jt d¡ dd¡tdƒd� ¡ }|  ||td¡ d S )Né   r4   r+   ÚaaarÄ   zDuplicate column names found©	r0   r1   rD   rE   ZreshaperB   rÌ   r�   r   ©rš   r&   rG   r    r    r!   Útest_duplicate_columnsÜ  s    $z)TestParquetPyArrow.test_duplicate_columnsc                 C   s:   t  dt jddd�i¡}tr,|  ||t¡ n
t||ƒ d S )Nr=   ú1 dayr+   r;   )r0   r1   Útimedelta_ranger
   rŸ   ÚNotImplementedErrorrp   r  r    r    r!   Útest_timedeltaá  s    z!TestParquetPyArrow.test_timedeltac                 C   s(   t  ddddgi¡}|  ||tj¡ d S )Nr=   r)   rP   )r0   r1   rŸ   r   ÚArrowExceptionr  r    r    r!   Útest_unsupportedè  s    z#TestParquetPyArrow.test_unsupportedc                 C   s6   t jddt jd�}tj|dgd�}|  ||tj¡ d S )Nr*   é
   r8   Úfp16©Údatar“   )rD   rE   Úfloat16r0   r1   rŸ   r   r
  )rš   r&   r  rG   r    r    r!   Útest_unsupported_float16ï  s    z+TestParquetPyArrow.test_unsupported_float16zqPyArrow does not cleanup of partial files dumps when unsupported dtypes are passed to_parquet function in windowsr   Ú	path_typec              
   C   sx   t jddt jd�}tj|dgd�}t ¡ �D}||ƒ}t tj	¡� |j
||d� W 5 Q R X tj |¡rjt‚W 5 Q R X d S )Nr*   r  r8   r  r  )rk   r"   )rD   rE   r  r0   r1   rd   ro   rž   r   r
  r   Úosrk   Úisfileru   )rš   r&   r  r  rG   Úpath_strrk   r    r    r!   Ú test_unsupported_float16_cleanupö  s    
z3TestParquetPyArrow.test_unsupported_float16_cleanupc                 C   sx   t  ¡ }t  tdƒ¡|d< t jddddd dgt  dddg¡d�|d< t jddddddgddd	gd
d�|d< t||ƒ d S )NZabcdefr=   rL   r,   rM   r8   r>   rI   r?   T)Ú
categoriesÚordered)r0   r1   ÚCategoricalrB   ZCategoricalDtyperp   r  r    r    r!   Útest_categorical  s    þ
  ÿ
z#TestParquetPyArrow.test_categoricalc                 C   s4   t  d¡}|jf |Ž}d|i}t||d||d� d S )NÚs3fsÚ
filesystemzpandas-test/pyarrow.parquet©rk   rl   rm   )r$   r¢   ZS3FileSystemrp   )rš   r2   Ús3_resourcer&   Ús3sor  Zs3Úkwr    r    r!   Útest_s3_roundtrip_explicit_fs  s    
ûz0TestParquetPyArrow.test_s3_roundtrip_explicit_fsc                 C   s   d|i}t ||d||d� d S )NÚstorage_optionsz s3://pandas-test/pyarrow.parquetr  ©rp   )rš   r2   r  r&   r  r    r    r!   Útest_s3_roundtrip+  s    ûz$TestParquetPyArrow.test_s3_roundtripr  Úpartition_colr.   c              
   C   sR   |  ¡ }|r*trdnd}||  |¡||< t|||dd|i|d |dœddd� d S )	NZint32r   zs3://pandas-test/parquet_dirr"  )Úpartition_colsr^   r"  Tr)   )rj   rk   rl   rm   rb   rf   )rÌ   r   rZ   rp   )rš   r2   r  r&   r%  r  Zexpected_dfZpartition_col_typer    r    r!   Útest_s3_roundtrip_for_dir7  s&    ÿýôz,TestParquetPyArrow.test_s3_roundtrip_for_dirr   c                 C   s(   t ƒ }| |¡ t|ƒ}t ||¡ d S r   )r   r   r   rd   re   )rš   r2   ÚbufferZdf_from_bufr    r    r!   Útest_read_file_like_obj_supportb  s    
z2TestParquetPyArrow.test_read_file_like_obj_supportc              	   C   sb   |  dd¡ |  dd¡ tjtdd�� tdƒ W 5 Q R X tjtdd�� | d¡ W 5 Q R X d S )NÚHOMEZTestingUserÚUSERPROFILEz.*TestingUser.*r|   z~/file.parquet)Zsetenvr$   r~   ÚOSErrorr   r   )rš   r2   Zmonkeypatchr    r    r!   Útest_expand_useri  s    z#TestParquetPyArrow.test_expand_userc              	   C   sR   ddg}|}t  ¡ �4}|j||d d� t||ƒ t|ƒj|jksDt‚W 5 Q R X d S )NrW   rU   ©r&  r^   ©rd   Úensure_clean_dirr   r{   r   Úshaperu   )rš   r&   r\   r&  rG   rk   r    r    r!   Útest_partition_cols_supportedr  s    

z0TestParquetPyArrow.test_partition_cols_supportedc              	   C   sT   d}|g}|}t  ¡ �4}|j||d d� t||ƒ t|ƒj|jksFt‚W 5 Q R X d S )NrW   r.  r/  )rš   r&   r\   r&  Úpartition_cols_listrG   rk   r    r    r!   Útest_partition_cols_string{  s    

z-TestParquetPyArrow.test_partition_cols_stringc           	   	   C   sP   d}|g}|}t  ¡ �0}||ƒ}|j||d� t|ƒj|jksBt‚W 5 Q R X d S )Nr/   )r&  )rd   r0  r   r   r1  ru   )	rš   r&   r2   r  r&  r3  rG   r  rk   r    r    r!   Útest_partition_cols_pathlib…  s    
z.TestParquetPyArrow.test_partition_cols_pathlibc                 C   s   t  ¡ }t||ƒ d S r   )r0   r1   rp   r  r    r    r!   Útest_empty_dataframe’  s    z'TestParquetPyArrow.test_empty_dataframec                 C   sV   dd l }t dddgi¡}| |jd| ¡ d�g¡}| t¡}t||d|i|d� d S )Nr   Úxr)   )Útyperx   rÔ   )	r   r0   r1   rx   ÚfieldZbool_rZ   rW   rp   )rš   r&   r   rG   rx   Zout_dfr    r    r!   Útest_write_with_schema—  s
    
z)TestParquetPyArrow.test_write_with_schemac                 C   sz   t  t jdddgdd�t jdddgdd�t jdd dgd	d�d
œ¡}t||ƒ t  dt jdddd gdd�i¡}t||ƒ d S )Nr)   r*   r+   rê   r8   ZUInt32r=   rI   rR   )r=   r>   rI   )r0   r1   rª   rp   r  r    r    r!   Ú test_additional_extension_arraysŸ  s    ýÿ
 z3TestParquetPyArrow.test_additional_extension_arraysz1.0.0)Úmin_versionc              	   C   sV   t  dt jdd dgdd�i¡}t  d|¡�" t||| d|› d�¡d� W 5 Q R X d S )	Nr=   rI   zstring[pyarrow]r8   Ústring_storagezstring[ú]©rj   )r0   r1   rª   r„   rp   rZ   )rš   r&   r=  rG   r    r    r!   Ú test_pyarrow_backed_string_array¯  s    z3TestParquetPyArrow.test_pyarrow_backed_string_arrayz2.0.0c                 C   sN   t  t j dddg¡t jdddd�t j t jdddd�¡d	œ¡}t||ƒ d S )
N)r   r)   )r)   r*   )r+   r4   z
2012-01-01r+   ÚD)r<   rÂ   r4   )rI   r?   r@   )r0   r1   ZIntervalIndexrÀ   Úperiod_rangeZfrom_breaksrF   rp   r  r    r    r!   Útest_additional_extension_types¶  s    ÿüÿ
z2TestParquetPyArrow.test_additional_extension_typesc                 C   s>   t s
d}nd}t dtjdddd�i¡}t||d|id	� d S )
Nz2.6z2.0r=   z
2017-01-01Z1nr  ©rÂ   r<   Úversionr´   )r	   r0   r1   rF   rp   )rš   r&   ÚverrG   r    r    r!   Útest_timestamp_nanosecondsÆ  s
    z-TestParquetPyArrow.test_timestamp_nanosecondsc                 C   sV   t s(|jtjjkr(|j tjj	dd�¡ d|g }t
j|d|id�}t||dd� d S )Nzitemporary skip this test until it is properly resolved: https://github.com/pandas-dev/pandas/issues/37286r   é   Úindex_as_col©r¸   r  F)rc   )r   ÚtzinforX   ÚtimezoneÚutcrï   rð   r$   r§   rî   r0   r1   rp   )rš   r   r&   r]   ÚidxrG   r    r    r!   Útest_timezone_aware_indexÑ  s    ÿþÿÿ
z,TestParquetPyArrow.test_timezone_aware_indexc              	   C   s^   t  dttddƒƒi¡}t ¡ �$}| ||¡ t||dgdd�}W 5 Q R X t|ƒdksZt	‚d S )Nr=   r   r+   ©r=   z==r   F)ÚfiltersZuse_legacy_datasetr)   )
r0   r1   rB   rC   rd   ro   r   r   rt   ru   )rš   r&   rG   rk   r•   r    r    r!   Útest_filter_row_groupsê  s    
   ÿz)TestParquetPyArrow.test_filter_row_groupsc              	   C   s~   t jtj dd¡dddgd�}t ¡ �}| ||¡ t||ƒ}W 5 Q R X |rdt	|j
t jjjƒszt‚nt	|j
t jjjƒszt‚d S )Nr  r+   r.   r/   ÚCrÄ   )r0   r1   rD   rÊ   rË   rd   ro   r   r   r‰   Z_mgrÚcoreZ	internalsZArrayManagerru   ZBlockManager)rš   r&   Zusing_array_managerrG   rk   r•   r    r    r!   Útest_read_parquet_managerõ  s    
z,TestParquetPyArrow.test_read_parquet_managerN)(r¤   r¥   r¦   rý   rþ   r   r  r	  r  r  r$   r§   rî   r   rö   ÚstrÚpathlibÚPathr  r  Ú
single_cpur!  r$  ÚtdZ
skip_if_nor'  r)  r-  r2  r4  r5  r6  r:  r;  r@  rC  rG  rO  rR  rU  r    r    r    r!   r÷   µ  s`   ý

þþ"

	






r÷   c                   @   s”   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Ze	j
jdd„ ƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!S )"ÚTestParquetFastParquetc                 C   sF   |}t jdddd�}| d ¡}||d< t jddd�|d< t||ƒ d S )	Nr:   r+   z
US/Easternrù   rû   r  r;   Ú	timedelta)r0   rF   r¹   r  rp   )rš   r(   r\   rG   rü   r    r    r!   rý     s    
z!TestParquetFastParquet.test_basicc                 C   s<   t jt d¡ dd¡tdƒd� ¡ }d}|  ||t|¡ d S )Nr  r4   r+   r  rÄ   z9Cannot create parquet dataset with duplicate column namesr  ©rš   r(   rG   r€   r    r    r!   r    s    $z-TestParquetFastParquet.test_duplicate_columnsc                 C   sB   t  ddd dgi¡}t jddtjdgidd�}t|||dd� d S )	Nr=   TFræ   g        r  r8   )rj   rc   )r0   r1   rD   rY   rp   ©rš   r(   rG   rj   r    r    r!   Útest_bool_with_none  s    z*TestParquetFastParquet.test_bool_with_nonec                 C   sV   t  dt jdddd�i¡}|  ||td ¡ t  ddddgi¡}d}|  ||t|¡ d S )	Nr=   Z2013ÚMr+   rD  r)   rP   z"Can't infer object conversion type)r0   r1   rB  r�   r   r]  r    r    r!   r    s
    z'TestParquetFastParquet.test_unsupportedc                 C   s&   t  dt  tdƒ¡i¡}t||ƒ d S )Nr=   r3   )r0   r1   r  rB   rp   )rš   r(   rG   r    r    r!   r  &  s    z'TestParquetFastParquet.test_categoricalc              	   C   sf   dt tddƒƒi}t |¡}t ¡ �(}|j||d dd� t||dgd�}W 5 Q R X t|ƒdksbt	‚d S )Nr=   r   r+   r)   )r^   Zrow_group_offsetsrP  )rQ  )
rB   rC   r0   r1   rd   ro   r   r   rt   ru   )rš   r(   r?   rG   rk   r•   r    r    r!   rR  *  s    

z-TestParquetFastParquet.test_filter_row_groupsc                 C   s    t ||dd|id |dœd� d S )Nz$s3://pandas-test/fastparquet.parquetr"  )r^   r"  r  r#  )rš   r2   r  r(   r  r    r    r!   r$  2  s    ûz(TestParquetFastParquet.test_s3_roundtripc              	   C   sl   ddg}|}t  ¡ �N}|j|d|d d� tj |¡s8t‚dd l}| |d¡j	}t
|ƒdks^t‚W 5 Q R X d S )NrW   rU   r   ©r"   r&  r^   r   Fr*   ©rd   r0  r   r  rk   Úexistsru   r   ZParquetFileZcatsrt   ©rš   r(   r\   r&  rG   rk   r   Zactual_partition_colsr    r    r!   r2  =  s    
üz4TestParquetFastParquet.test_partition_cols_supportedc              	   C   sh   d}|}t  ¡ �N}|j|d|d d� tj |¡s4t‚dd l}| |d¡j	}t
|ƒdksZt‚W 5 Q R X d S )NrW   r   ra  r   Fr)   rb  rd  r    r    r!   r4  N  s    
üz1TestParquetFastParquet.test_partition_cols_stringc              	   C   sl   ddg}|}t  ¡ �N}|j|dd |d� tj |¡s8t‚dd l}| |d¡j	}t
|ƒdks^t‚W 5 Q R X d S )NrW   rU   r   )r"   r^   Úpartition_onr   Fr*   rb  rd  r    r    r!   Útest_partition_on_supported_  s    
üz2TestParquetFastParquet.test_partition_on_supportedc              
   C   sV   ddg}|}d}t jt|d��. t ¡ �}|j|dd ||d� W 5 Q R X W 5 Q R X d S )NrW   rU   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datar|   r   )r"   r^   re  r&  )r$   r~   r   rd   r0  r   )rš   r(   r\   r&  rG   r€   rk   r    r    r!   Ú3test_error_on_using_partition_cols_and_partition_onp  s    ÿ
ûzJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_onc                 C   s*   t  ¡ }| ¡ }d|j_t|||d� d S )Nr¸   r?  ©r0   r1   rÌ   r¸   rº   rp   r^  r    r    r!   r6  ‚  s    z+TestParquetFastParquet.test_empty_dataframec                 C   s>   d|g }t j|d|id�}| ¡ }d|j_t|||d� d S )NrH  rI  rJ  r¸   r?  rh  )rš   r(   r]   rN  rG   rj   r    r    r!   rO  ‰  s
    
z0TestParquetFastParquet.test_timezone_aware_indexc              
   C   s\   t  dddgi¡}t ¡ �8}| |¡ tjtdd�� t|ddd� W 5 Q R X W 5 Q R X d S )	Nr=   r)   r*   z!not supported for the fastparquetr|   r   Trè   )	r0   r1   rd   ro   r   r$   r~   r   r   )rš   r(   rG   rk   r    r    r!   Ú&test_use_nullable_dtypes_not_supported’  s
    

z=TestParquetFastParquet.test_use_nullable_dtypes_not_supportedc              
   C   sb   t  d¡�N}t |¡ d¡ tjtdd�� t|dd� W 5 Q R X t |¡j	dd� W 5 Q R X d S )	Nútest.parquets   breakitÚ r|   r   r‘   F)Ú
missing_ok)
rd   ro   rW  rX  Úwrite_bytesr$   r~   Ú	Exceptionr   Úunlink)rš   rk   r    r    r!   Ú$test_close_file_handle_on_read_errorš  s
    z;TestParquetFastParquet.test_close_file_handle_on_read_errorc              
   C   sp   t jddgddgdœd�}t d¡�6}t| ¡ dƒ�}| |¡ W 5 Q R X t||d�}W 5 Q R X t ||¡ d S )Nr   r)   r-   )r  rj  Úwbr‘   )	r0   r1   rd   ro   ÚopenÚencoder   r   re   )rš   r"   rG   rk   rA   r•   r    r    r!   Útest_bytes_file_name¢  s    z+TestParquetFastParquet.test_bytes_file_nameN)r¤   r¥   r¦   rý   r  r_  r  r  rR  r$   r§   rY  r$  r2  r4  rf  rg  r6  rO  ri  rp  rt  r    r    r    r!   r[    s"   	

	r[  )	NNNNNTFTr*   )MÚ__doc__rX   Úior   r  rW  Úwarningsr   r   ÚnumpyrD   r$   Zpandas._configr   Zpandas.compatr   Zpandas.compat.pyarrowr   r   r	   r
   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsrZ  Zpandasr0   Zpandas._testingZ_testingrd   Zpandas.util.versionr   Zpandas.io.parquetr   r   r   r   r   r   r#   rŽ   ÚDeprecationWarningÚFutureWarningr   r'   r§   Z
pytestmarkZfixturer   Zskipifr"   r&   r(   r2   rH   r\   ÚnowrL  rM  ÚminÚmaxÚstrptimer]   rp   r{   r�   r…   r†   rˆ   rŠ   r�   r–   r—   r˜   r©   r÷   r[  r    r    r    r!   Ú<module>   sÄ   

ÿþþ ÿþøÿ





ùÿ
         ö
A+   $
  N