U
    +Be                     @   s   d dl mZmZ d dlZd dlZd dlZejeje	Z
eje
dddZefeeeef dddZeee d	d
dZdeee edddZdS )    )IterableDictNz..datazHumanEval.jsonl.gz)evalset_filereturnc                 C   s   dd t | D S )Nc                 S   s   i | ]}|d  |qS )task_id ).0taskr   r   g/weka-jd/prod/containers/zhuqihao/dev-cpu/upload_code/deepseek-coder/Evaluation/MBPP/human_eval/data.py
<dictcomp>   s      z!read_problems.<locals>.<dictcomp>)stream_jsonl)r   r   r   r   read_problems   s    r   )filenamer   c              
   c   s   |  drbt| dF}t|d.}|D ]"}tdd |D r(t|V  q(W 5 Q R X W 5 Q R X nBt| ddd.}|D ]"}td	d |D rvt|V  qvW 5 Q R X d
S )z>
    Parses each jsonl line and yields it as a dictionary
    .gzrbrtc                 s   s   | ]}|   V  qd S Nisspacer	   xr   r   r   	<genexpr>   s     zstream_jsonl.<locals>.<genexpr>rutf-8)encodingc                 s   s   | ]}|   V  qd S r   r   r   r   r   r   r      s     N)endswithopengzipanyjsonloads)r   gzfpfpliner   r   r   r      s    
$r   F)r   r   appendc              
   C   s   |r
d}nd}t j| } | drzt| |D}tj|dd*}|D ]}|t	|d 
d qDW 5 Q R X W 5 Q R X n:t| |*}|D ]}|t	|d 
d qW 5 Q R X dS )z5
    Writes an iterable of dictionaries to jsonl
    abwbr   )fileobjmode
r   N)ospath
expanduserr   r   r   GzipFilewriter    dumpsencode)r   r   r%   r)   r#   r"   r   r   r   r   write_jsonl    s    
2r2   )F)typingr   r   r   r    r+   r,   dirnameabspath__file__ROOTjoinZ
HUMAN_EVALstrr   r   boolr2   r   r   r   r   <module>   s   