Update serializer_deserializer.py

This commit is contained in:
eptq002345 2023-09-24 11:57:59 +08:00
parent a7425a55a6
commit 2eda54a755
1 changed files with 16 additions and 3 deletions

View File

@ -25,6 +25,7 @@ from . import datasets as de
def serialize(dataset, json_filepath=""):
"""
Serialize dataset pipeline into a JSON file.
用于将数据集管道序列化为一个JSON文件
Note:
Currently some Python objects are not supported to be serialized.
@ -48,12 +49,14 @@ def serialize(dataset, json_filepath=""):
>>> # serialize it to JSON file
>>> serialized_data = ds.serialize(dataset, json_filepath="/path/to/mnist_dataset_pipeline.json")
"""
# 返回一个字典,其中包含序列化后的数据集图。
return dataset.to_json(json_filepath)
def deserialize(input_dict=None, json_filepath=None):
"""
Construct dataset pipeline from a JSON file produced by de.serialize().
用于从JSON文件中构造数据集管道
Note:
Currently Python function deserialization of map operator are not supported.
@ -82,17 +85,24 @@ def deserialize(input_dict=None, json_filepath=None):
"""
data = None
# 判断input_dict是否为空
if input_dict:
# 将input_dict转换为de.Dataset对象
data = de.DeserializedDataset(input_dict)
# 如果json_filepath不为空
if json_filepath:
# 将json_filepath转换为de.Dataset对象
data = de.DeserializedDataset(json_filepath)
return data
# 定义一个函数expand_path用于将相对路径转换为绝对路径
def expand_path(node_repr, key, val):
"""Convert relative to absolute path."""
"""Convert relative to absolute path.用于将相对路径转换为绝对路径。"""
# 如果val是一个列表
if isinstance(val, list):
# 那么将其中的每个文件路径转换为绝对路径并添加到node_repr字典中
node_repr[key] = [os.path.abspath(file) for file in val]
else:
node_repr[key] = os.path.abspath(val)
@ -101,6 +111,7 @@ def expand_path(node_repr, key, val):
def show(dataset, indentation=2):
"""
Write the dataset pipeline graph to logger.info file.
用于将数据集管道图形写入logger.info文件
Args:
dataset (Dataset): The starting node.
@ -114,14 +125,16 @@ def show(dataset, indentation=2):
>>> dataset = dataset.batch(batch_size=10, drop_remainder=True)
>>> ds.show(dataset)
"""
# 使用json.dumps函数将其转换为格式化的字符串
pipeline = dataset.to_json()
# 写入logger.info文件
logger.info(json.dumps(pipeline, indent=indentation))
def compare(pipeline1, pipeline2):
"""
Compare if two dataset pipelines are the same.
用于比较两个数据集管道的是否相同
Args:
pipeline1 (Dataset): a dataset pipeline.
@ -135,5 +148,5 @@ def compare(pipeline1, pipeline2):
>>> pipeline2 = ds.Cifar10Dataset(cifar10_dataset_dir, num_samples=100)
>>> res = ds.compare(pipeline1, pipeline2)
"""
# 返回一个bool值
return pipeline1.to_json() == pipeline2.to_json()