
    jʟ                        d dl Z d dlZd dlmZ d dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlZd dlZd dlmZ d dlmZ d dlmZmZmZ 	 d dlmZ d dlmZmZ 	 d dlZd dlmZ d d	l m!Z! d d
lm"Z" 	 d dl#Z$ejJ                  j,                  Z&d Z'd Z(ejJ                  jF                  d        Z)ejJ                  jF                  d        Z*ejJ                  j6                  d        Z+ejJ                  j6                  d        Z,d Z-ejJ                  j6                  d        Z.d Z/d Z0d Z1ejJ                  jd                  d        Z3ejJ                  j6                  d        Z4ejJ                  j6                  d        Z5d Z6ejJ                  j6                  d        Z7d Z8ejJ                  js                  dd ejt                         g      ejJ                  js                  dd      d               Z;d  Z<d! Z=d" Z>d# Z?d$ Z@d% ZAd& ZBd' ZCd( ZDd) ZEejJ                  j6                  d*        ZFejJ                  j6                  d+        ZGejJ                  j6                  d,        ZHd- ZId. ZJejJ                  j6                  d/        ZKd0 ZLejJ                  j6                  ejJ                  j                  ejJ                  j                  d1      ejJ                  j                  d2      ejJ                  j                  d3      d4                                    ZOejJ                  js                  d5d6 d7 d8 d9 g      ejJ                  js                  d:d;d<g      d=               ZPd> ZQd? ZRd@ ZSdA ZTdB ZUdC ZVejJ                  j                  dD        ZXejJ                  js                  dEdFdGdHgg      dI        ZYy# e$ r dZY hw xY w# e$ r dxZZY aw xY w# e$ r dZ$Y jw xY w)J    N)OrderedDict)copytree)Decimal)fs)util)_check_roundtrip_roundtrip_table_test_table)_read_table_write_table)dataframe_with_lists)alltypes_samplec                 :   t        j                  dg di      }t        j                  t        d      5  t        || dz  d       d d d        t        j                  t        d      5  t        || dz  d	       d d d        y # 1 sw Y   ?xY w# 1 sw Y   y xY w)
Na         z"Unsupported Parquet format versionmatchztest_version.parquetz2.2versionz%Unsupported Parquet data page version)data_page_version)patablepytestraises
ValueErrorr   )tempdirr   s     G/tmp/pip-target-jj7clzyk/lib/python/pyarrow/tests/parquet/test_basic.pytest_parquet_invalid_versionr!   =   s    HHc9%&E	z)M	N MUG&<<eLM	z *! 
" .UG&<<',	.. .M M. .s   B*BBBc                      t        j                  g ddz        } t         j                  j                  | gdg      }ddg}|D ]  }t	        ||        y )Nr   i f0namesi   i   )data_page_size)r   arrayTablefrom_arraysr   )arrt
page_sizestarget_page_sizes       r    test_set_data_page_sizer.   G   sY    
((9v%
&C
cU4&1A 7#J& =+;<=    c                  8    t        d      } t        | ddd       y )Nd   
   r   2.4)r&   write_batch_sizer   )r
   r   r   s    r    test_set_write_batch_sizer6   Q   s    Eb1er/   c                      t        d      } t        | ddd       t        j                  t              5  t        | ddd       d d d        y # 1 sw Y   y xY w)Nr1   r   r2   r3   )dictionary_pagesize_limitr&   r   r   )r
   r   r   r   	TypeErrorr5   s    r    "test_set_dictionary_pagesize_limitr:   Z   sQ    EUa$&7 
y	! ;#(*E	;; ; ;s   AAc            	         g } t         j                  j                  t        d            }| j	                  t         j
                  j                  |gdz               t               \  }}t         j                  j                  |      }| j	                  t         j
                  j                  |gdz               dD ]  }dD ]  }| D ]  }t        |d||         ! y )Nr2   sizer   )z1.0z2.0)TF2.6)r   r   use_dictionary)	r   RecordBatchfrom_pandasr   appendr(   from_batchesr   r   )tablesbatchdf_r   r?   r   s          r    test_chunked_table_writerH   f   s     FNN&&B'?@E
MM"((''!45 "EBNN&&r*E
MM"((''!45+ 3+ 	3N 3 5&7#133	33r/   c                 H   t        d      }t        j                  j                  |      }t	        |ddid       t        | dz        }t        |d      5 }t        ||d	       d d d        t        j                  |d
      }|j                  |      sJ y # 1 sw Y   4xY w)Nr2   r<   
memory_mapTr>   read_table_kwargsr   tmp_filewbr   )rJ   r   r   r(   rA   r   stropenr   pqread_pandasequalsr   rF   r   filenamef
table_reads         r    test_memory_maprY   y   s    	b	!BHH  $EU|T.B"$ 7Z'(H	h	 .UAu-.T:JU###. .   BB!c                    t        j                  dg di      }t        | dz        }t        t        dd      }t	        j
                  |t        j                  t        j                  z  t        j                  z  |z  d      }t        j                  |d      5 }t        j                  ||       d d d        t	        j
                  |t        j                  |z        }t        j                  |d	      5 }t        j                  |      }d d d        j                  |      sJ y # 1 sw Y   yxY w# 1 sw Y   )xY w)
Nr   r   zraw-fd.parquetO_BINARYr   i  rN   )moderb)r   r   rP   getattrosrQ   O_CREATO_WRONLYO_TRUNCOSFilerR   write_tableO_RDONLY
read_tablerT   )r   r   pathbinary_flagfdsinksourceresults           r    $test_parquet_read_write_table_raw_fdrn      s    HHc9%&Ew))*D"j!,K	rzzBKK/"**<{J
B	2D	! $T
ud#$ 
r{{[0	1B	2D	! 'Vv&' ==$ $' 's   D)7D5)D25D>c                 H   t        d      }t        j                  j                  |      }t	        |ddid       t        | dz        }t        |d      5 }t        ||d	       d d d        t        j                  |d
      }|j                  |      sJ y # 1 sw Y   4xY w)Nr2   r<   buffer_sizei  r>   rK   rM   rN   r   i   )rp   rO   rU   s         r    test_enable_buffered_streamrq      s    	b	!BHH  $EU}d.C"$ 7Z'(H	h	 .UAu-.d;JU###. .rZ   c                 >   t         j                  j                  t        j                  dg      gdg      }d}| |z  }|j	                         rJ t        |t        |             |j	                         sJ t        t        |            }|j                  |      sJ y )N*   intsz	foo # bar)	r   r(   r)   r'   existsr   rP   r   rT   )r   r   rV   rh   rX   s        r    test_special_chars_filenamerv      s    HH  "((B4.!1F8<EHXD{{}D	";;==SY'JU###r/   c                     t        j                  t        d      5  t        j                  d        d d d        t        j                  t        d      5  t        j
                  d        d d d        y # 1 sw Y   CxY w# 1 sw Y   y xY w)NNoner   )r   r   r9   rR   rg   ParquetFile r/   r    test_invalid_sourcer{      si     
y	/ 
d 
y	/ 
t   s   A4B 4A= B	c                 P   ddl m}  G d d      }| dz  }t        j                  dg di      }t	        ||       |j                  d|	      5  t        j                  t        d
      5  t        j                  |dg       d d d        t        j                  t        d      5  t        j                  |ddg       d d d        t        j                  t        d      5  t        j                  ||j                         d d d        t        j                  t        d      5  t        j                  |        d d d        t        j                  |      }||k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   SxY w# 1 sw Y   y xY w)Nr   )mockc                       e Zd Zd Zy);test_read_table_without_dataset.<locals>.MockParquetDatasetc                     t        d      )NMockParquetDataset)ImportError)selfargskwargss      r    __init__zDtest_read_table_without_dataset.<locals>.MockParquetDataset.__init__   s    233r/   N)__name__
__module____qualname__r   rz   r/   r    r   r      s    	4r/   r   test.parquetr   r   z#pyarrow.parquet.core.ParquetDataset)newzthe 'filters' keywordr   )integer=r   )filterszthe 'partitioning' keywordweekcolor)partitioningzthe 'schema' argumentschemathe 'source' argument)unittestr}   r   r   r   patchr   r   r   rR   rg   r   )r   r}   r   rh   r   rm   s         r    test_read_table_without_datasetr      sU   4 4 ^#DHHc9%&E	9?Q	R 
]]:-DE 	?MM$)<(=>	?]]:-IJ 	@MM$fg->?	@]]:-DE 	5MM$u||4	5]]:-DE 	#MM'"	#t$
 
	? 	?	@ 	@	5 	5	# 	#
 
sl   F*E,#F&E8 #F#"F#F(F>$F,E5	1F8F	=FF		FF	FF%c                  t    t        j                  t        t        d            gdg      } t	        | d       y )Ni@  r#   r$   r   )row_group_size)r   r   listranger   )r+   s    r    (test_file_with_over_int16_max_row_groupsr      s,     	$uU|$%dV4AQq)r/   c                  Z   t        d      } t        j                  j                  |       }t        j                  j	                  |j                         D cg c]  }|j                  d      d d  c}|j                  j                        }|j                  j                  d      j                  t        j                         k(  sJ |j                  j                  d      j                  t        j                  t        j                               k(  sJ t        |d       y c c}w )	Nr2   r<   r   r$   null	null_listr>   r   )r   r   r(   rA   r)   itercolumnschunkr   r%   fieldtyper   list_r   )rF   r   cols      r    test_empty_table_roundtripr      s    	b	!B HH  $EHH  %*%6%6%89c1bq	9ll   ! "E <<f%**bggi777<<k*//288BGGI3FFFFu 	:s   D(c                      t        j                         } t        j                  j	                  | d      }t        |       y )NF)preserve_index)pd	DataFramer   r(   rA   r   )rF   emptys     r    test_empty_table_no_columnsr      s.    	BHH  E :EUr/   c            	      H   t        t        j                         t        j                  t        j                                     } g t        dd      gg}|D cg c]:  }t        j
                  |t        j                  |             j                         < }}|D cg c]6  }t        j                  j                  |t        j                  |             8 }}t        j                  j                  |t        j                  |             }t        |       y c c}w c c}w )N)int32list_stringr   )Gr   r   )r   r   r   r   stringr'   structflattenr@   r)   r   r(   rC   r   )colsdatarE   	my_arrays
my_batchestbls         r    1test_write_nested_zero_length_array_chunk_failurer      s    hhjHHRYY[)D 1&9<=D #$ %biio6>>@ $I $  )* ..,,U299T?,K *J *
((


BIIdO
<CS$*s   ?D;Dc                    | dz  }t        j                  dt        j                  dt        j                        i      }t        ||       t        |      }|j                         }t        j                  ||       t        |       dz   }t        j                  dt        j                  dt        j                        i      }t        ||       t        |      }|j                         }t        j                  ||       y )Nzzzz.parquetxr2   dtype)r   r   nparangeint64r   r   	to_pandastmassert_frame_equalrP   )r   rh   rF   rX   df_reads        r    test_multiple_path_typesr     s     ]"D	sBIIb9:	;BTT"J""$G"g& w<-'D	sBIIb9:	;BTT"J""$G"g&r/   c                 X   | dz  }t        j                  dg di      }t        ||       t        j                  |      }t        |      }|j                  |      sJ t        j                  t              5  t        |t        j                                d d d        y # 1 sw Y   y xY w)Nr   r   r   
filesystem)r   r   r   r   FSProtocolClassr   rT   r   r   r9   r   
FileSystem)r   rh   r   fs_protocol_objrm   s        r    test_fspathr   &  s    ^#DHHc9%&E**40O)F== 
y	! AO@A A As   7 B  B)r   name)data.parquetu   例.parquetc                 >   t        j                  dg di      }| |z  }t        j                  |t	        |             t        j                  |       5  t        j                  ||      }d d d        j                  |      sJ |j                          |j                         rJ t        j                  |       5  t        j                  |||       d d d        t        j                  |      }|j                  |      sJ y # 1 sw Y   xY w# 1 sw Y   >xY w)Nr   r   r   )r   r   rR   re   rP   r   
change_cwdrg   rT   unlinkru   )r   r   r   r   rh   rm   s         r    test_relative_pathsr   6  s     HHc9%&ET>D NN5#d)$		! <t
;<==KKM{{} 
	! ;
udz:;]]4 F==< <; ;s   D=DDDc                      t        j                  t              5  t        j                  d       d d d        y # 1 sw Y   y xY w)Nzi-am-not-existing.parquet)r   r   FileNotFoundErrorrR   rg   rz   r/   r    test_read_non_existing_filer   O  s1    	(	) 3
123 3 3s	   9Ac                       G d dt         j                        } t        j                  t        d      5  t        j                   | d             d d d        y # 1 sw Y   y xY w)Nc                       e Zd Zd Zd Zy)3test_file_error_python_exception.<locals>.BogusFilec                     t        d      NzorglubZeroDivisionErrorr   r   s     r    readz8test_file_error_python_exception.<locals>.BogusFile.readW      #I..r/   c                     t        d      r   r   r   s     r    seekz8test_file_error_python_exception.<locals>.BogusFile.seekZ  r   r/   N)r   r   r   r   r   rz   r/   r    	BogusFiler   V  s    	/	/r/   r   r   r   r/   )ioBytesIOr   r   r   rR   rg   )r   s    r     test_file_error_python_exceptionr   U  sH    /BJJ / 
(		: &
in%& & &s   AA#c                    t        j                  dg di      }t        j                  |t	        | dz               t        t	        | dz        d      5 }t        j                  |      }d d d        j                  |      sJ t        t	        | dz        d      5 }t        j                  t        j                  |            }d d d        |j                  |      sJ y # 1 sw Y   yxY w# 1 sw Y   )xY w)Nr   r   r   r^   )	r   r   rR   re   rP   rQ   rg   rT   
PythonFile)r   r   rW   rm   s       r    test_parquet_read_from_bufferr   b  s    HHc9%&ENN5#g678	c'N*+T	2 "aq!"==	c'N*+T	2 1ar}}Q/01==" "1 1s   C")C."C+.C7c            
         t        j                  t        t        t        t        d                        } t        j                  t        t        t        t        d                        }t        j                  ddgdz        }| | g}t         j                  j                  |ddg      }t        ||ddd	       t        ||ddgdg	       t        ||dddgddg	       t         j                  j                  | | ||gg d
      }t        ||ddgddg       t         j                  j                  |gdg      }t        j                  t        d      5  t        ||dd       d d d        y # 1 sw Y   y xY w)Nr1   TF2   r   br$   gzip)expectedcompressionr?   use_byte_stream_splitr   r   cdr   r   )r   r?   r   tmpBYTE_STREAM_SPLIT only supportsr   )r   r   r?   )r   r'   r   mapfloatr   intr(   r)   r   r   r   IOError)	arr_floatarr_intarr_bool
data_floatr   mixed_tables         r    test_byte_stream_splitr   p  sb   c%s456IhhtCU3Z012Gxxu*+HY'JHH  C: >E UU$)G
 UU%(E,/52
 UU%(#J,/:7
 ((&&	9gw'O-A ' CK[;%(#J,/:7 HH  (E7 ;E	w&G	H /d(-	// / /s   E33E<c           
      L   t        j                  t        t        t        t        d                  t        j                  dd            }t        j                  t        t        t        t        d                  t        j                  dd            }t        j                  t        t        t        t        d                  t        j                  dd            }t        j                  dd	gd
z        }|||g}t         j                  j                  |g d      }t        ||dd	d       t        j                  j                  | d      }t        j                  ||dd	d       t        j                  |      }|j                   j#                  d      }	|j                   j#                  d      }
|	j$                  dk(  sJ |
j$                  dk(  sJ t        ||dd	dddd       t         j                  j                  ||||gg d      }t        ||d	d       y )Nr1      r   r      	      TFr   r   r   r   r$   r   )r   r   r?   store_decimal_as_integerr   )r   r?   r  r   r   INT32INT64DELTA_BINARY_PACKEDr   r   )r   r   r?   r  column_encodingr   )r   r?   r  )r   r'   r   r   r   r   
decimal128r(   r)   r   r`   rh   joinrR   re   ry   r   columnphysical_type)r   arr_decimal_1_9arr_decimal_10_18arr_decimal_gt18r   data_decimalr   pqtestfile_path
pqtestfilepqcol_decimal_1_9pqcol_decimal_10_18r   s               r    test_store_decimal_as_integerr    s   hhtCs$<=$&MM!Q$79Oc'5:&>!?&(mmB&:<xxS%*%= >%']]2q%9;xxu*+H#%68HILHH  _ EE U#!'$).2	4 ggll7N;ONN5/%"',02
 0J"))003$++2215**g555,,777 U#!'$).233& ((&&	+-=xH" ' $K [)$).24r/   c                  &   t        j                  t        t        t        t        d                        } t        j                  t        t        t        t        d                        }t        j                  t        d      D cg c]  }t        |       c}t        j                               }t        j                  t        d      D cg c]  }t        |      j                  d       c}t        j                  d            }t        j                  g ddz        }t         j                  j                  | ||||gg d      }t        ||dd	d	d
d	d       t        ||dd
       t        ||dd
dd
d       t        ||dd
ddd       t        ||dd
dddd       t        ||dddi       t        j                  t        d      5  t        ||dd
d
d	d       d d d        t        j                  t         d      5  t        ||ddd
d
d       d d d        t        j                  t"        d      5  t        ||dd       d d d        t        j                  t"        d      5  t        ||dddi       d d d        t        j                  t"              5  t        ||dgdd
i       d d d        t        j                  t"              5  t        ||dd
i       d d d        t        j                  t"              5  t        ||ddgdd	d
d       d d d        t        j                  t"              5  t        ||dddd	d
d       d d d        t        j                  t$              5  t        ||dd       d d d        y c c}w c c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   hxY w# 1 sw Y   @xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w) Nr1   r   r2   )FTFF   )r   r   r   r   er$   FBYTE_STREAM_SPLITPLAINr   )r   r?   r  r
  r  DELTA_LENGTH_BYTE_ARRAYDELTA_BYTE_ARRAYr  RLEr   r   )r   r   r  z)DELTA_BINARY_PACKED encoder only supportsz+'RLE_DICTIONARY' is already used by defaultRLE_DICTIONARYz/Unsupported column encoding: 'MADE_UP_ENCODING'r   MADE_UP_ENCODINGr   )r   r  )r   r?   r   r  T)r   r'   r   r   r   r   r   rP   binaryzfillr(   r)   r   r   r   r   OSErrorr   r9   )r   r   r   arr_binarr_flbar   r   s          r    test_column_encodingr)    s   c%s456IhhtCU3Z012Ghhc
31A3"))+FGxx#(:.aQb	.RYYr]DHxx3b89H((&&	GWh9' ' )K [;u+>+>+2+>&@A [;$)%,.
 [;$)+2+@+2&45 [;$)+2+@+D&FG [;$)+2+@+=+=&?@ [;$)&)5\3 
w>
@ E{(-/6/6/B*D	EE 
wH
J 9{(-/D/6/6*8	99 
zJ
L ;{(-)9	;; 
zN
P D{(-*-/A)B	DD 
z	" 9{),*-w	99 
z	" 9{*-w	99 
z	" 9{(-03u/4/B/6*8	99 
z	" 9{(-/3/4/B/6*8	99 
y	! /{(-)-	// /o 4.`E E9 9; ;D D9 99 99 99 9/ /sx   N!NN"N/N<9O	,O O#O/	O;?P"N,/N9<O	OO #O,/O8;PPc            
      D   t        j                  t        t        t        t        d                        } | | g}t         j                  j                  |ddg      }t        ||dd       t        ||dd       t        ||dd	d
       t        ||dddd
       t        ||dd       t        ||dd       g d}t        j                         }|D ]<  \  }}t        j                  t        t        f      5  t        ||||       d d d        > y # 1 sw Y   IxY w)N  r   r   r$   r   r   )r   r   compression_levelr  snappyr  r   r   r   r   lz4r  ))r-     )r   i)rx   i  )lzo   )r   r,  )r   r'   r   r   r   r   r(   r)   r   r   r   r   r   r   r&  r   )r*   r   r   invalid_combinationsbufcodeclevels          r    test_compression_levelr7  L  s"   
((4Ct-.
/C:DHH  c3Z 8E UU'(*
 UU'(* UU'-H!=? UU-.Q'79
 UU'(* UU'(*8
**,C. 2]]J01 	2+02	2 	22	2 	2s   ;DD	c            
          t        j                  t        t        t        t        d                        } t         j                  j                  | | gddg      }t        ||d       t        ||d       y )Nr+  r   r   r$   lz4_rawr.  LZ4_RAW)	r   r'   r   r   r   r   r(   r)   r   )r*   r   s     r    test_lz4_raw_compression_aliasr;  y  sY    
((4Ct-.
/CHH  #sC: >EUU	BUU	Br/   c            
         t        j                  t        t        t        t        d                        } t        j                  t        d      D cg c]  }t        |       c}t        j                               }| |g}t         j                  j                  |ddg      }t        ||ddddi       t        ||dddd	d
dd       t        ||dddii       t        ||dddii       t        ||di i       t        ||ddi       t        ||ddi       t        j                         }d}t        j                  t        |      5  t!        ||ddddi       d d d        d}t        j                  t        |      5  t!        ||ddddi       d d d        t        j                  t        d      5  t!        ||d       d d d        d}t        j                  t"        |      5  t!        ||dddii       d d d        d}t        j                  t"        t%        j&                  |            5  t!        ||dddii       d d d        d}t        j                  t"        t%        j&                  |            5  t!        ||dddii       d d d        y c c}w # 1 sw Y   IxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nr1   r   r   r   r$   g?)ndvfpp)r   bloom_filter_optionsr2   皙?r  r>  r=  TFz8'bloom_filter_options:ndv' for column 'a' must be an intr   100)r?  z9'bloom_filter_options:fpp' for column 'a' must be a floatz0.05z+'bloom_filter_options' must be a dictionaryzL'bloom_filter_options:ndv' for column 'a' must be greater than zero, got -10izH'bloom_filter_options:fpp' for column 'a' must be in (0.0, 1.0), got 2.0g       @zI'bloom_filter_options:fpp' for column 'a' must be in (0.0, 1.0), got -0.5g      )r   r'   r   r   r   r   rP   r$  r(   r)   r   r   r   r   r   r9   r   r   reescape)r   r   r'  r   r   r4  expects          r    test_bloom_filter_optionsrE    s   hhtCU3Z012Ghhc
31A3"))+FGWDHH  c3Z 8E UU#d3B5 6 UU"%d3"S:QBS T UU5$-B) * UU5#,B( ) UU"B UU$B  ! UU%B! " **,CGF	y	/ 8UC%577 	88
 IF	y	/ 8UC#f577 	88
 
y(U	V <UCd;<
 	W 	z	0 )UC5#,7( 	))
 XF	z6):	; )UC5#,7( 	)) YF	z6):	; *UC5$-7) 	** *y 4B8 88 8
< <) )) )* *sN   J+J0J=K
9KK"K.0J:=K
KK"K+.K7c                      t        j                  g d      } d}t         j                  j                  | g|g      }t	        |ddi      }d}|j
                  d   j                  |k(  sJ y )N)r   r   r   r   r0  zprohib; ,	{}flavorspark)write_table_kwargsprohib______r   )r   r'   r(   r)   r	   r   r   )a0r   r   rm   expected_names        r     test_sanitized_spark_field_namesrM    sb    	/	"BDHH  "v.Ee78KLF"M==  M111r/   c                  >   t        d      } t        j                  j                  |       }t	        j
                         }t        ||dd       |j                  d       t        |d      }|j                  d       t        |d	      }|j                  |      sJ y )
Ni'  r<   SNAPPYr>   )r   r   r   T)use_threadsF)
r   r   r(   rA   r   r   r   r   r   rT   )rF   r   r4  table1table2s        r    test_multithreaded_readrS    sy    	e	$BHH  $E
**,C5AHHQK$/FHHQK%0F==   r/   c                     t        j                  t        j                  d      gg d      } t        j
                  j                  | j                               }t        j                         }t        ||d       |j                  d       t        |      }|j                  |      sJ t        j                  t               5  t        ||d       d d d        y # 1 sw Y   y xY w)Nr0  )ABCD)columns)
chunk_sizer   )r   r   r   r   r   r(   rA   reset_indexr   r   r   r   r   rT   r   r   r   )r   r   r4  rm   s       r    test_min_chunksizer]    s    <<10DEDHH  !1!1!34E
**,C+HHQKF==	z	" /UCA./ / /s   CC&c                 f   t        j                  t        d      t        t        dd            t	        j
                  dd      j                  d      t	        j
                  ddd	
      g dt        j                  t        d            t        j                  dd      t        j                  ddd      t        j                  ddd      d	      }t        j                  j                  |      }| dz  }	 t        ||d       |j                         rJ y # t        j                  $ r Y (w xY w)Nabcr   r0  r      u1      @      @float64r   TFT20130101periodsz
US/Eastern)rh  tzns)rh  freq)	r   r   r   r   r  rW   ghirM   r3   r   )r   r   r   r   r   r   astypeCategorical
date_ranger   r(   rA   r   ArrowExceptionru   )r   rF   pdfrV   s       r    (test_write_error_deletes_incomplete_filert    s     
DK q!-IIaO2248IIc3i@/NN4;7MM*a@MM*a-9;MM*adK	M 	
NB ((

r
"C#H 	S(E2       s   9D D0/D0c                     d}	 t        j                  |       y # t        $ r}||j                  d   v sJ Y d }~y d }~ww xY w)Nznonexistent-file.parquetr   )rR   rg   	Exceptionr   )r   rh   r  s      r    test_read_non_existent_filerw    s>    %D!
d !qvvay   !s    	A ;A c                     t        j                         5  t        j                  d       t        j                  | dz         d d d        y # 1 sw Y   y xY w)Nerror)actionzv0.7.1.parquet)warningscatch_warningssimplefilterrR   rg   )datadirs    r    test_read_table_doesnt_warnr    sC    		 	 	" 2W-
g 0012 2 2s   /AAc                  z   t         j                  j                  t        j                  ddg      gdg      } t	        j
                         }t        j                  | |d       |j                  d       t        j                  |      }t        j                  |j                         | j                                y )Nr_  defsome_colr   r   r   )r   r(   r)   r'   r   r   rR   re   r   rg   r   r   r   )r   rW   	roundtrips      r    test_zlib_compression_bugr    s     HH  "((E5>":!;j\JE


ANN5!0FF1Ia I)--/1BCr/   c                    t        | dz        }t        j                  t        j                  d      5  t        |d      5 }	 d d d        t        j                  |       d d d        t        j                  t        j                  d      5  t        |d      5 }|j                  d       d d d        t        j                  |       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   ?xY w# 1 sw Y   y xY w)Nr   zsize is 0 bytesr   rN   zsize is 4 bytess   ffff)	rP   r   r   r   ArrowInvalidrQ   rR   rg   write)r   rh   rW   s      r    test_parquet_file_too_smallr  '  s    w'(D	r.?	@ $ 		
d
 
r.?	@ $ 	GGG	
d 		 	 	 	 sG   CCCC6C*,C6C	CC'*C3	/C66C?zignore:RangeIndex:FutureWarningz.ignore:tostring:DeprecationWarning:fastparquetz$ignore:unclosed file:ResourceWarningc                 6   t        j                  d      }t        j                  t	        d      t	        t        dd            t        j                  ddd      g d	t        j                  d
d      t        j                  g d      d      }t        j                  |      }t        | dz        }t        j                  ||d        |j                  |      }|j!                         }t#        j$                  ||j'                  dt(        i             t        | dz        }|j+                  ||j'                  dt(        i             t        j,                  |      }t#        j$                  |j!                         |j'                  dt(        i             y )Nfastparquetr_  r   r0  rb  rc  rd  r   re  rf  r   rg  )r   r   r   )r   r   r   r   r  rW   zcross_compat_arrow.parquetr  r   z cross_compat_fastparquet.parquetrW   )r   importorskipr   r   r   r   r   r   rq  rp  r   r   rP   rR   re   ry   r   r   r   ro  objectr  rS   )	r   fprF   r   
file_arrowfp_filedf_fpfile_fastparquettable_fps	            r    $test_fastparquet_cross_compatibilityr  4  s;    
		]	+B	eeAqk"395$z150		

B HHRLE W;;<JNN5*$7nnZ(GE %C=!9: 7%GGHHHryy#v78~~./H (,,.		3-0HIr/   array_factoryc                  6    t        j                  dd gdz        S Nr   r2   r   r'   rz   r/   r    <lambda>r  `  s    BHHaY^$ r/   c                  R    t        j                  dd gdz        j                         S r  r   r'   dictionary_encoderz   r/   r    r  r  a  s     BHHaY^$668 r/   c                  6    t        j                  dd gdz        S N r2   r  rz   r/   r    r  r  b  s    BHHb$Z"_% r/   c                  R    t        j                  dd gdz        j                         S r  r  rz   r/   r    r  r  c  s     BHHb$Z"_%779 r/   read_dictionaryFTc                    t         j                  j                  d |        i      }t        j                         }t        j                  ||d       |j                  d       |rdgnd }t        j                  |d|      }|j                  D ]E  }|j                  \  }|j                         d   }|j                         |j                  dz  k(  rEJ  y )	Nr   T)r?   r   F)rP  r  r       )r   r(   from_pydictr   r   rR   re   r   rg   rY  chunksbuffers
to_pybytesr=   )r  r  
orig_tablebior   r   r   r4  s           r    test_buffer_contentsr  _  s     %%umo&>?J
**,CNN:s48HHQK!0ugdOMM#5*9;E }} 4**mmoa ~~388e#33334r/   c                     t        j                  t        j                  t        d            gdg      }| dz  }t	        j
                  ||d       t	        j                  |      }|j                  |      sJ y )Nr0  rt   r$   zarrow-10480.pyarrow.gzGZIPr  )r   r   r'   r   rR   re   rg   rT   )r   r   rh   rm   s       r    "test_parquet_compression_roundtripr  z  sa    
 HHbhhuQx()&:E--DNN5$F3]]4 F==r/   c                    t         j                  j                  t        j                  g d      gdg      }| dz  }d}t	        j
                  ||j                        5 }t        |      D ]  }|j                  |        	 d d d        t	        j                  |      }|j                  j                  |k(  sJ t        |      D ]$  }|j                  |      j                  |      r$J  y # 1 sw Y   lxY w)Nr   r   r#   zempty_row_groups.parquetr   )r   r(   r)   r'   rR   ParquetWriterr   r   re   ry   metadatanum_row_groupsread_row_grouprT   )r   r   rh   
num_groupswriterrn  readers          r    test_empty_row_groupsr    s    HH  "((2G"<!=vFE//DJ			$	- &z" 	&Au%	&& ^^D!F??))Z777: 6$$Q'..u5556& &s   "C--C6c                     d gdz  }|j                  dg       t        j                  j                  |gdg      }| dz  }t	        j
                  ||       t	        j                  |      }||k(  sJ y )Ni   r   r  zarrow-11607.parquet)rB   r   r(   r)   rR   re   rg   )r   r   r   rh   rR  s        r    test_reads_over_batchr    sk    6WDKK HH  $(4E**DNN5$]]4 FF??r/   c                    | dz  }|j                  d       t        j                  g dg dgddg      }t        j                  ||d	z         t        j                  g d
g dgddg      }t        j                  ||dz         t        j
                  t        |            }t        j                  g dg dgddg      }||k(  sJ y )N dataset_column_order_permutationT)exist_okr   )r@  皙?333333?r   r   r$   zdata1.parquet)皙?      ?333333?)r0  r  r`  zdata2.parquet)r   r   r   r0  r  r`  )r@  r  r  r  r  r  )mkdirr   r   rR   re   rg   rP   )r   casedata1data2r   rR  s         r     test_permutation_of_column_orderr    s    77DJJJHHi.sCjAENN5$01HHlI.sCjAENN5$01MM#d)$EXX)57 #J(F F??r/   c                    | dz  }t        j                  t        t        d                  }d}t        j                  |g|z  t        |      D cg c]  }d| 	 c}      }t        j                  ||       t        j                  t        d      5  t        j                  |d|z  	       d d d        t        j                  t        d      5  t        j                  ||
       d d d        t        j                  |d|z  	      }||k(  sJ t        j                  |d|z  
      }||k(  sJ t        j                  |      }||k(  sJ y c c}w # 1 sw Y   xY w# 1 sw Y   yxY w)Nzlargethrift.parquetr2   r+  some_long_column_name_r$   z1Couldn't deserialize thrift:.*Exceeded size limitr   r   )thrift_string_size_limit)thrift_container_size_limitr1   r   )r   r'   r   r   r   rR   re   r   r   r&  rg   )r   rh   r'   num_colsrn  r   gots          r    test_thrift_size_limitsr    sK   **DHHT%)_%EHHH	(5:8_E's+EGE NN5$	E
G D 	dR(]CD 
E
G B 	dAB
 --sX~
FC%<<
--!h,
GC%<<
--
C%<<# FD DB Bs   E
EE#E #E,c                    | dz  }t        j                  dg di      }t        j                  ||d       t        j                  |d      }||k(  sJ t        |j                               }|d   |d   k7  sJ |d   |d   c|d<   |d<   | d	z  }|j                  |       t        j                  |d
      }||k7  sJ |t        j                  dg di      k(  sJ t        j                  t        d      5  t        j                  |d      }ddd       t        j                  |d
      }|j                         }	|	|k7  sJ |	t        j                  dg di      k(  sJ t        j                  |d      }t        j                  t        d      5  |j                         }ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zUCheck that checksum verification works for datasets created with
    pq.write_table()zcorrect.parquetr   r   r   r   r0  Twrite_page_checksumpage_checksum_verification   $   zcorrupted.parquetFr   r   r   r0  CRC checksum verificationr   N)r   r   rR   re   rg   	bytearray
read_byteswrite_bytesr   r   r&  ry   r   )
r   original_path
table_origtable_checkbin_datacorrupted_pathtable_corruptrG   corrupted_pq_filetable_corrupt2s
             r    +test_page_checksum_verification_write_tabler    s   
 //M3-.JNN:}$G --$OK$$$
 1134H B<8B<'''!)"x|HRL(2, 22Nx( MM.=BDM J&&&BHHc<%89999 
w&A	B KMM.TJK
 ~BGI&++-NZ'''RXXsL&9:::: ~BFH 
w&A	B %""$% %!K K % %s   9F1F=1F:=Gc                    t        j                  dg di      }| dz  }t        j                  ||d       t	        |j                               }t        |      dk(  sJ |d   }t        j                  |d      }||k(  sJ t        |j                               }|d	   |d
   k7  sJ |d
   |d	   c|d	<   |d
<   | dz  }t        ||       ||j                  z  }|j                  |       t        j                  |d      }	|	|k7  sJ |	t        j                  dg di      k(  sJ t        j                  t        d      5  t        j                  |d      }
ddd       y# 1 sw Y   yxY w)zXCheck that checksum verification works for datasets created with
    pq.write_to_datasetr   r  correct_dirTr  r   r   r  r  r  corrupted_dirFr  r  r   N)r   r   rR   write_to_datasetr   iterdirlenrg   r  r  r   r   r  r   r   r&  )r   r  original_dir_pathoriginal_file_path_listr  r  r  corrupted_dir_pathcorrupted_file_pathr  rG   s              r    test_checksum_write_to_datasetr    s   
 3-.J  -/
),02
 ##4#<#<#>?&'1,,,+A.M--$OK$$$
 1134H B<8B<'''!)"x|HRL(2, !?2 23,}/A/AA##H- MM"5=BDM J&&&BHHc<%89999 
w&A	B PMM-$OP P Ps   E##E,rl   z/tmp/z/tmp/file1.parquetz/tmp/file2.parquetc                     | j                  t        j                  dd        t        j                  t
        d      5  t        j                  |       d d d        y # 1 sw Y   y xY w)Nzpyarrow.datasetr   r   )rl   )setitemsysmodulesr   r   r   rR   rg   )monkeypatchrl   s     r    9test_read_table_raises_value_error_when_ds_is_unavailabler  C  sK     %6=	z)@	A %
V$% % %s   AA&)Zr`   r  collectionsr   r   rB  r{  shutilr   decimalr   r   pyarrowr   r   pyarrow.testsr   pyarrow.tests.parquet.commonr   r	   r
   pyarrow.parquetparquetrR   r   r   r   pandasr   pandas.testingtestingr   pyarrow.tests.pandas_examplesr   r   numpyr   mark
pytestmarkr!   r.   r6   r:   rH   rY   rn   rq   rv   r{   r   slowr   r   r   r   r   r   parametrizeLocalFileSystemr   r   r   r   r   r  r)  r7  r;  rE  rM  rS  r]  rt  rw  r  r  r  r  filterwarningsr  r  r  r  r  r  r  r  datasetr  r  rz   r/   r    <module>r     sJ  $ 
 
 # 	 	       7 7 F
B< [[  
.=   ; ; 3 3$ $ $ " $ $$0 * *    8 ' '$A  
"


(  !@A  B *3
& #/L34l}/@*2ZC@*F2 ! !" / /  ! !4!2 D D
 =>LMBC#J D N ?  
#JL $8%9	+  *UDM:4 ;4(	 6$&67%t 1P 1Ph w-/CDEG%G%   	B  NB
  	Bs6   N2 O  ,O 2N=<N= 	OOOO