import json def check_jsonl_file(file_path): with open(file_path, "r", encoding="utf-8") as file: total = 0 custom_id_set = set() for line in file: if line.strip() == "": continue try: line_dict = json.loads(line) except json.decoder.JSONDecodeError: raise Exception(f"Format error in the batch inference input file. The data on line {total + 1} is not in JSON format.") if not line_dict.get("custom_id"): raise Exception(f"Format error in the batch inference input file. The custom_id on line {total + 1} does not exist.") if line_dict.get("custom_id") in custom_id_set: raise Exception( f"Format error in the batch inference input file. The custom_id={line_dict.get('custom_id', '')} is duplicated." ) else: custom_id_set.add(line_dict.get("custom_id")) if not isinstance(line_dict.get("body", ""), dict): raise Exception( f"Format error in the batch inference input file. The body of custom_id={line_dict.get('custom_id', '')} is not a JSON string." ) total += 1 return total # Replace with the path of your JSONL file. file_path = "" total_lines = check_jsonl_file(file_path) print(f"The number of lines with valid JSON data in the file is: {total_lines}")