将大模型指令微调数据从parquet转为json格式

import os
import json
import random
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pqdef read_json_file(file_path):try:with open(file_path, 'r', encoding='utf-8') as file:data = json.load(file)return dataexcept FileNotFoundError:print(f"File {file_path} not found.")except json.JSONDecodeError:print(f"File {file_path} is not a valid JSON file.")except Exception as e:print(f"An error occurred: {e}")def read_jsonl_file(file_path):data = []with open(file_path, 'r', encoding='utf-8') as file:for line in file:try:data.append(json.loads(line))except:print(line)1/0return datadef read_praquet_file(file_path):table = pq.read_table(file_path)df = table.to_pandas()result=[row.tolist() for _, row in df.iterrows()]return resultdef save_json(file_path,data):with open(file_path, 'w', encoding='utf-8') as file:json.dump(data, file, indent=4, ensure_ascii=False)print(f'Save {file_path} is ok!')def save_jsonl(file_path,data):try:with open(file_path, 'w', encoding='utf-8') as file:for item in data:file.write(json.dumps(item, ensure_ascii=False) + '\n')print(f"Data saved to {file_path}")except Exception as e:print(f"An error occurred while saving the data: {e}")def save_parquet(file_path, data):if isinstance(data, list):data = pd.DataFrame(data)if not isinstance(data, pd.DataFrame):raise ValueError("data must be a pandas DataFrame or a list of lists")pq.write_table(pa.Table.from_pandas(data), file_path)print(f'Save {file_path} is ok!')def convert_lists_to_json(df):"""Convert lists in DataFrame to JSON strings."""for column in df.columns:if df[column].apply(lambda x: isinstance(x, list)).any():df[column] = df[column].apply(lambda x: json.dumps(x) if isinstance(x, list) else x)return dfroot='/path/to/parquet/dir'
save_path='/path/to/savedir/save_name.json'
new_data=[]
dirs=os.listdir(root)
for one in dirs:if one.endswith('.parquet'):print(one)file_path=root+'/'+onedata=read_praquet_file(file_path)for x in data:inp=x[3]res=x[4]new_entry={"conversations": [{"role": "user","content": inp},{"role": "assistant","content": res}]}if len(inp)>0 and len(res)>0:new_data+=[new_entry]save_json(save_path,new_data)