B
    Xd                 @   s   d dl mZmZ d dlZd dlZd dlZejeje	Z
eje
dddZefeeeef dddZeee d	d
dZdeee edddZdS )    )IterableDictNz..datazHumanEval.jsonl.gz)evalset_filereturnc             C   s   dd t | D S )Nc             S   s   i | ]}||d  qS )task_id ).0taskr   r   Z/weka-jd/prod/containers/zhuqihao/dev-cpu/hugginfaceeval/eval_benchmark/human_eval/data.py
<dictcomp>   s    z!read_problems.<locals>.<dictcomp>)stream_jsonl)r   r   r   r   read_problems   s    r   )filenamer   c          
   c   s   |  drft| dJ}t|d2}x*|D ]"}tdd |D r*t|V  q*W W dQ R X W dQ R X nBt| d2}x*|D ]"}tdd |D rxt|V  qxW W dQ R X dS )	z>
    Parses each jsonl line and yields it as a dictionary
    z.gzrbrtc             s   s   | ]}|   V  qd S )N)isspace)r	   xr   r   r   	<genexpr>   s    zstream_jsonl.<locals>.<genexpr>Nrc             s   s   | ]}|   V  qd S )N)r   )r	   r   r   r   r   r      s    )endswithopengzipanyjsonloads)r   gzfpfpliner   r   r   r      s    

&
r   F)r   r   appendc          
   C   s   |r
d}nd}t j| } | dr~t| |H}tj|dd.}x&|D ]}|t	|d 
d qFW W dQ R X W dQ R X n>t| |.}x&|D ]}|t	|d 
d qW W dQ R X dS )z5
    Writes an iterable of dictionaries to jsonl
    abwbz.gz)fileobjmode
zutf-8N)ospath
expanduserr   r   r   GzipFilewriter   dumpsencode)r   r   r   r#   r   r   r   r   r   r   write_jsonl    s    

4
r,   )F)typingr   r   r   r   r%   r&   dirnameabspath__file__ROOTjoinZ
HUMAN_EVALstrr   r   boolr,   r   r   r   r   <module>   s   