From 4115f78ced3f964bf7c3573a9517455a6d0b1bbd Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 4 May 2023 17:16:29 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BD=BF=E7=94=A8=E6=B3=A8?= =?UTF-8?q?=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pretrain.py | 9 ++++++--- pretrain_for_c2net.py | 9 ++++++--- train.py | 7 +++++-- train_for_c2net.py | 13 ++++++++----- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pretrain.py b/pretrain.py index 355eeef..3b4b488 100644 --- a/pretrain.py +++ b/pretrain.py @@ -4,7 +4,7 @@ 使用注意事项： 1、本示例只支持单数据集训练，如果是多数据集，请参考多数据集训练示例train_for_multidataset.py 2、本示例支持选择预训练模型单文件或多文件 -3、本示例需要用户定义的参数有--multi_data_url,--pretrain_url,--train_url,--device_target，这4个参数在单数据集任务中必须定义 +3、本示例需要用户定义的参数有--multi_data_url,--pretrain_url,--train_url，这3个参数在单数据集任务中必须定义 具体的含义如下： --multi_data_url是启智平台上选择的数据集的obs路径 --pretrain_url是启智平台上选择的预训练模型文件的obs路径 @@ -78,10 +78,13 @@ if __name__ == "__main__": PretrainToEnv(args.pretrain_url, pretrain_dir) device_num = int(os.getenv('RANK_SIZE')) + #使用单卡时 if device_num == 1: DatasetToEnv(args.multi_data_url,data_dir) context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) - ds_train = create_dataset(os.path.join(data_dir, "train"), cfg.batch_size) + #使用数据集的方式 + ds_train = create_dataset(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) + #使用多卡时 if device_num > 1: # set device_id and init for multi-card training context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) @@ -96,7 +99,7 @@ if __name__ == "__main__": #and Wait for 0th card to finish copying data while not os.path.exists("/cache/download_input.txt"): time.sleep(1) - ds_train = create_dataset_parallel(os.path.join(data_dir, "train"), cfg.batch_size) + ds_train = create_dataset_parallel(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) network = LeNet5(cfg.num_classes) net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") diff --git a/pretrain_for_c2net.py b/pretrain_for_c2net.py index 9680b2d..b97955b 100644 --- a/pretrain_for_c2net.py +++ b/pretrain_for_c2net.py @@ -1,13 +1,13 @@ """ ######################## Attention! ######################## 使用注意事项： -1、本示例需要用户定义的参数有--multi_data_url,--pretrain_url,--train_url,--device_target，这3个参数任务中必须定义 +1、本示例需要用户定义的参数有--multi_data_url,--pretrain_url,--train_url,这3个参数在任务中必须定义 具体的含义如下： --multi_data_url是启智平台上选择的数据集的obs路径 --pretrain_url是启智平台上选择的预训练模型文件的obs路径 --train_url是训练结果回传到启智平台的obs路径 2、用户需要调用OpenI.C2NETMultiDatasetToEnv等函数，来实现数据集、预训练模型文件的拷贝 -3、智算网络中区别于启智的有： +3、智算网络区别于启智： (1)智算的数据集拷贝到训练镜像后需要解压，请使用C2NETMultiDatasetToEnv函数 (2)智算任务结果不需要用户调用函数回传，会在训练结束后自动回传结果 """ @@ -72,10 +72,13 @@ if __name__ == "__main__": PretrainToEnv(args.pretrain_url, pretrain_dir) device_num = int(os.getenv('RANK_SIZE')) + #使用单卡时 if device_num == 1: DatasetToEnv(args.multi_data_url,data_dir) context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) - ds_train = create_dataset(os.path.join(data_dir, "train"), cfg.batch_size) + #使用数据集的方式 + ds_train = create_dataset(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) + #使用多卡时 if device_num > 1: # set device_id and init for multi-card training context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) diff --git a/train.py b/train.py index 7af3f57..661dcf5 100644 --- a/train.py +++ b/train.py @@ -64,10 +64,13 @@ if __name__ == "__main__": DatasetToEnv(args.multi_data_url, data_dir) device_num = int(os.getenv('RANK_SIZE')) + #使用单卡时 if device_num == 1: DatasetToEnv(args.multi_data_url,data_dir) context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) - ds_train = create_dataset(os.path.join(data_dir, "train"), cfg.batch_size) + #使用数据集的方式 + ds_train = create_dataset(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) + #使用多卡时 if device_num > 1: # set device_id and init for multi-card training context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) @@ -81,7 +84,7 @@ if __name__ == "__main__": #If the cache file does not exist, it means that the copy data has not been completed, #and Wait for 0th card to finish copying data while not os.path.exists("/cache/download_input.txt"): - time.sleep(1) + time.sleep(1) ds_train = create_dataset_parallel(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) network = LeNet5(cfg.num_classes) diff --git a/train_for_c2net.py b/train_for_c2net.py index 58f050f..e29205f 100644 --- a/train_for_c2net.py +++ b/train_for_c2net.py @@ -1,12 +1,12 @@ """ ######################## Attention! ######################## 使用注意事项： -1、本示例需要用户定义的参数有--multi_data_url,--train_url,--device_target，这3个参数任务中必须定义 +1、本示例需要用户定义的参数有--multi_data_url,--train_url,这2个参数任务中必须定义 具体的含义如下： --multi_data_url是启智平台上选择的数据集的obs路径 --train_url是训练结果回传到启智平台的obs路径 2、用户需要调用OpenI.C2NETMultiDatasetToEnv等函数，来实现数据集、预训练模型文件的拷贝 -3、智算网络中区别于启智的有： +3、智算网络区别于启智： (1)智算的数据集拷贝到训练镜像后需要解压，请使用C2NETMultiDatasetToEnv函数 (2)智算任务结果不需要用户调用函数回传，会在训练结束后自动回传 """ @@ -30,7 +30,7 @@ from OpenI import C2NETMultiDatasetToEnv as DatasetToEnv parser = argparse.ArgumentParser(description='MindSpore Lenet Example') parser.add_argument('--multi_data_url', - help='使用单数据集训练时，需要定义的参数', + help='使用数据集，需要定义的参数', default= '[{}]') parser.add_argument('--train_url', @@ -61,10 +61,13 @@ if __name__ == "__main__": ###Initialize and copy data to training image DatasetToEnv(args.multi_data_url, data_dir) device_num = int(os.getenv('RANK_SIZE')) + #使用单卡时 if device_num == 1: DatasetToEnv(args.multi_data_url,data_dir) context.set_context(mode=context.GRAPH_MODE,device_target=args.device_target) - ds_train = create_dataset(os.path.join(data_dir, "train"), cfg.batch_size) + #使用数据集的方式 + ds_train = create_dataset(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) + #使用多卡时 if device_num > 1: # set device_id and init for multi-card training context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target, device_id=int(os.getenv('ASCEND_DEVICE_ID'))) @@ -78,7 +81,7 @@ if __name__ == "__main__": #If the cache file does not exist, it means that the copy data has not been completed, #and Wait for 0th card to finish copying data while not os.path.exists("/cache/download_input.txt"): - time.sleep(1) + time.sleep(1) ds_train = create_dataset_parallel(os.path.join(data_dir + "/MNISTData", "train"), cfg.batch_size) network = LeNet5(cfg.num_classes)