From c91ce05db6f7a41fdb3d50009851a7bb90a522ee Mon Sep 17 00:00:00 2001 From: zhengjun10 Date: Thu, 30 Dec 2021 15:14:36 +0800 Subject: [PATCH] fix new java api and example issue --- .../export_models/models/densenet_train_export.py | 2 +- .../export_models/models/effnet_train_export.py | 2 +- .../models/effnet_tune_train_export.py | 2 +- .../export_models/models/googlenet_train_export.py | 2 +- .../export_models/models/lenet_train_export.py | 2 +- .../models/mini_alexnet_train_export.py | 2 +- .../models/mobilenetv1_train_export.py | 2 +- .../models/mobilenetv2_train_export.py | 2 +- .../models/mobilenetv3_train_export.py | 2 +- .../export_models/models/nin_train_export.py | 2 +- .../export_models/models/resnet_train_export.py | 2 +- .../models/shufflenetv2_train_export.py | 2 +- .../export_models/models/tinybert_train_export.py | 2 +- .../export_models/models/vgg_train_export.py | 2 +- .../export_models/models/xception_train_export.py | 2 +- .../examples/train_lenet/model/lenet_export.py | 2 +- .../examples/train_lenet/model/prepare_model.sh | 4 ++-- .../train_lenet_java/model/lenet_export.py | 2 +- .../transfer_learning/model/prepare_model.sh | 5 +++-- .../model/transfer_learning_export.py | 2 +- .../examples/unified_api/model/lenet_export.py | 2 +- .../examples/unified_api/model/prepare_model.sh | 4 ++-- .../com/mindspore/flclient/model/FileUtil.java | 4 ++++ .../java/src/main/java/com/mindspore/Model.java | 5 +++-- .../main/java/com/mindspore/config/MSContext.java | 10 ++++++++++ mindspore/lite/java/src/main/native/model.cpp | 6 +++++- .../src/test/java/com/mindspore/ModelTest.java | 14 ++++++++++++++ 27 files changed, 62 insertions(+), 28 deletions(-) diff --git a/mindspore/lite/examples/export_models/models/densenet_train_export.py b/mindspore/lite/examples/export_models/models/densenet_train_export.py index ea801e5403..8ff6df363c 100644 --- a/mindspore/lite/examples/export_models/models/densenet_train_export.py +++ b/mindspore/lite/examples/export_models/models/densenet_train_export.py @@ -25,7 +25,7 @@ from src.network.densenet import DenseNet121 #pylint: disable=wrong-import-position sys.path.append(os.environ['CLOUD_MODEL_ZOO'] + 'official/cv/densenet121/') -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = DenseNet121(num_classes=10) loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False) diff --git a/mindspore/lite/examples/export_models/models/effnet_train_export.py b/mindspore/lite/examples/export_models/models/effnet_train_export.py index 3384cc28d9..8898cc6e24 100644 --- a/mindspore/lite/examples/export_models/models/effnet_train_export.py +++ b/mindspore/lite/examples/export_models/models/effnet_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = effnet(num_classes=10) loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False) diff --git a/mindspore/lite/examples/export_models/models/effnet_tune_train_export.py b/mindspore/lite/examples/export_models/models/effnet_tune_train_export.py index 3e61b441a6..bb00e61d32 100644 --- a/mindspore/lite/examples/export_models/models/effnet_tune_train_export.py +++ b/mindspore/lite/examples/export_models/models/effnet_tune_train_export.py @@ -24,7 +24,7 @@ from mindspore import context, Tensor, nn from mindspore.train.serialization import export, load_checkpoint from mindspore.common.parameter import ParameterTuple -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) class TransferNet(nn.Cell): diff --git a/mindspore/lite/examples/export_models/models/googlenet_train_export.py b/mindspore/lite/examples/export_models/models/googlenet_train_export.py index 91a0062053..3523635ff3 100644 --- a/mindspore/lite/examples/export_models/models/googlenet_train_export.py +++ b/mindspore/lite/examples/export_models/models/googlenet_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = GoogleNet(num_classes=10) loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False) diff --git a/mindspore/lite/examples/export_models/models/lenet_train_export.py b/mindspore/lite/examples/export_models/models/lenet_train_export.py index 4e03aabd54..de2ba471bc 100644 --- a/mindspore/lite/examples/export_models/models/lenet_train_export.py +++ b/mindspore/lite/examples/export_models/models/lenet_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = LeNet5() loss_fn = nn.MSELoss() diff --git a/mindspore/lite/examples/export_models/models/mini_alexnet_train_export.py b/mindspore/lite/examples/export_models/models/mini_alexnet_train_export.py index 1b9a82d8af..e5301f32b9 100644 --- a/mindspore/lite/examples/export_models/models/mini_alexnet_train_export.py +++ b/mindspore/lite/examples/export_models/models/mini_alexnet_train_export.py @@ -21,7 +21,7 @@ from mini_alexnet import AlexNet from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) # Mini alexnet is designed for MNIST data batch = 2 diff --git a/mindspore/lite/examples/export_models/models/mobilenetv1_train_export.py b/mindspore/lite/examples/export_models/models/mobilenetv1_train_export.py index b3b26de954..29e9a8bdbd 100644 --- a/mindspore/lite/examples/export_models/models/mobilenetv1_train_export.py +++ b/mindspore/lite/examples/export_models/models/mobilenetv1_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = MobileNetV1(10) loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False) diff --git a/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py b/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py index a476b0b89a..83a2257fe0 100644 --- a/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py +++ b/mindspore/lite/examples/export_models/models/mobilenetv2_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) batch = 8 backbone_net = MobileNetV2Backbone() diff --git a/mindspore/lite/examples/export_models/models/mobilenetv3_train_export.py b/mindspore/lite/examples/export_models/models/mobilenetv3_train_export.py index 1f752528ad..bbb696c671 100644 --- a/mindspore/lite/examples/export_models/models/mobilenetv3_train_export.py +++ b/mindspore/lite/examples/export_models/models/mobilenetv3_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = mobilenet_v3_small(num_classes=10) loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False, reduction='mean') diff --git a/mindspore/lite/examples/export_models/models/nin_train_export.py b/mindspore/lite/examples/export_models/models/nin_train_export.py index 786f73919b..ab02562d96 100644 --- a/mindspore/lite/examples/export_models/models/nin_train_export.py +++ b/mindspore/lite/examples/export_models/models/nin_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = NiN(num_classes=10) loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") diff --git a/mindspore/lite/examples/export_models/models/resnet_train_export.py b/mindspore/lite/examples/export_models/models/resnet_train_export.py index c18bcf36c0..638a81f664 100644 --- a/mindspore/lite/examples/export_models/models/resnet_train_export.py +++ b/mindspore/lite/examples/export_models/models/resnet_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) batch = 4 n = resnet50(class_num=10) diff --git a/mindspore/lite/examples/export_models/models/shufflenetv2_train_export.py b/mindspore/lite/examples/export_models/models/shufflenetv2_train_export.py index bf76d48268..2d368a6f1e 100644 --- a/mindspore/lite/examples/export_models/models/shufflenetv2_train_export.py +++ b/mindspore/lite/examples/export_models/models/shufflenetv2_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = ShuffleNetV2(n_class=10) loss_fn = nn.SoftmaxCrossEntropyWithLogits(sparse=False) diff --git a/mindspore/lite/examples/export_models/models/tinybert_train_export.py b/mindspore/lite/examples/export_models/models/tinybert_train_export.py index 7237d67103..c5b9d76fca 100644 --- a/mindspore/lite/examples/export_models/models/tinybert_train_export.py +++ b/mindspore/lite/examples/export_models/models/tinybert_train_export.py @@ -147,7 +147,7 @@ class BertTrainCell(M.nn.Cell): return F.depend(loss, succ) -M.context.set_context(mode=M.context.PYNATIVE_MODE, +M.context.set_context(mode=M.context.GRAPH_MODE, device_target="CPU", save_graphs=False) # get epoch number diff --git a/mindspore/lite/examples/export_models/models/vgg_train_export.py b/mindspore/lite/examples/export_models/models/vgg_train_export.py index c18b33c63e..08ea06bc42 100644 --- a/mindspore/lite/examples/export_models/models/vgg_train_export.py +++ b/mindspore/lite/examples/export_models/models/vgg_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) batch = 2 diff --git a/mindspore/lite/examples/export_models/models/xception_train_export.py b/mindspore/lite/examples/export_models/models/xception_train_export.py index e544d7ee25..09b9490c5b 100644 --- a/mindspore/lite/examples/export_models/models/xception_train_export.py +++ b/mindspore/lite/examples/export_models/models/xception_train_export.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype from mindspore import context, Tensor, nn from mindspore.train.serialization import export -context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) n = Xception(num_classes=1000) diff --git a/mindspore/lite/examples/train_lenet/model/lenet_export.py b/mindspore/lite/examples/train_lenet/model/lenet_export.py index e91bf5dc17..8c75a0c936 100644 --- a/mindspore/lite/examples/train_lenet/model/lenet_export.py +++ b/mindspore/lite/examples/train_lenet/model/lenet_export.py @@ -25,7 +25,7 @@ from train_utils import train_wrap n = LeNet5() n.set_train() -context.set_context(mode=context.PYNATIVE_MODE, device_target="CPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="CPU", save_graphs=False) BATCH_SIZE = int(sys.argv[1]) x = Tensor(np.ones((BATCH_SIZE, 1, 32, 32)), mstype.float32) diff --git a/mindspore/lite/examples/train_lenet/model/prepare_model.sh b/mindspore/lite/examples/train_lenet/model/prepare_model.sh index 7c1c88b3cb..86088dd172 100755 --- a/mindspore/lite/examples/train_lenet/model/prepare_model.sh +++ b/mindspore/lite/examples/train_lenet/model/prepare_model.sh @@ -12,9 +12,9 @@ if [[ -z ${EXPORT} ]]; then fi fi - CONVERTER="../../../build/tools/converter/converter_lite" -if [ ! -f "$CONVERTER" ]; then +$CONVERTER &> /dev/null +if [ "$?" -ne 0 ]; then if ! command -v converter_lite &> /dev/null then tar -xzf ../../../../../output/mindspore-lite-*-linux-x64.tar.gz --strip-components 4 --wildcards --no-anchored converter_lite *so.* *.so diff --git a/mindspore/lite/examples/train_lenet_java/model/lenet_export.py b/mindspore/lite/examples/train_lenet_java/model/lenet_export.py index 6fc5e0ca9a..a729881dc7 100644 --- a/mindspore/lite/examples/train_lenet_java/model/lenet_export.py +++ b/mindspore/lite/examples/train_lenet_java/model/lenet_export.py @@ -23,7 +23,7 @@ from train_utils import train_wrap n = LeNet5() n.set_train() -context.set_context(mode=context.PYNATIVE_MODE, device_target="CPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="CPU", save_graphs=False) BATCH_SIZE = 4 x = Tensor(np.ones((BATCH_SIZE, 1, 32, 32)), mstype.float32) diff --git a/mindspore/lite/examples/transfer_learning/model/prepare_model.sh b/mindspore/lite/examples/transfer_learning/model/prepare_model.sh index ce7d54af05..d76c6643d0 100755 --- a/mindspore/lite/examples/transfer_learning/model/prepare_model.sh +++ b/mindspore/lite/examples/transfer_learning/model/prepare_model.sh @@ -16,10 +16,11 @@ else fi CONVERTER="../../../build/tools/converter/converter_lite" -if [ ! -f "$CONVERTER" ]; then +$CONVERTER &> /dev/null +if [ "$?" -ne 0 ]; then if ! command -v converter_lite &> /dev/null then - tar -xzf ../../../../../output/mindspore-lite-*-linux-x64.tar.gz --strip-components 4 --wildcards --no-anchored converter_lite libglog.so.0 libmslite_converter_plugin.so + tar -xzf ../../../../../output/mindspore-lite-*-linux-x64.tar.gz --strip-components 4 --wildcards --no-anchored converter_lite *so.* *.so if [ -f ./converter_lite ]; then CONVERTER=./converter_lite else diff --git a/mindspore/lite/examples/transfer_learning/model/transfer_learning_export.py b/mindspore/lite/examples/transfer_learning/model/transfer_learning_export.py index 9d10cc9580..4eed62b043 100755 --- a/mindspore/lite/examples/transfer_learning/model/transfer_learning_export.py +++ b/mindspore/lite/examples/transfer_learning/model/transfer_learning_export.py @@ -37,7 +37,7 @@ class TransferNet(Cell): BACKBONE = effnet(num_classes=1000) load_checkpoint("efficient_net_b0.ckpt", BACKBONE) -M.context.set_context(mode=M.context.PYNATIVE_MODE, +M.context.set_context(mode=M.context.GRAPH_MODE, device_target="GPU", save_graphs=False) BATCH_SIZE = 16 X = M.Tensor(np.ones((BATCH_SIZE, 3, 224, 224)), M.float32) diff --git a/mindspore/lite/examples/unified_api/model/lenet_export.py b/mindspore/lite/examples/unified_api/model/lenet_export.py index 43c0310627..07d945f9d8 100644 --- a/mindspore/lite/examples/unified_api/model/lenet_export.py +++ b/mindspore/lite/examples/unified_api/model/lenet_export.py @@ -25,7 +25,7 @@ from train_utils import train_wrap n = LeNet5() n.set_train() -context.set_context(mode=context.PYNATIVE_MODE, device_target="CPU", save_graphs=False) +context.set_context(mode=context.GRAPH_MODE, device_target="CPU", save_graphs=False) BATCH_SIZE = int(sys.argv[1]) x = Tensor(np.ones((BATCH_SIZE, 1, 32, 32)), mstype.float32) diff --git a/mindspore/lite/examples/unified_api/model/prepare_model.sh b/mindspore/lite/examples/unified_api/model/prepare_model.sh index c51b0707c6..5d84abb24e 100755 --- a/mindspore/lite/examples/unified_api/model/prepare_model.sh +++ b/mindspore/lite/examples/unified_api/model/prepare_model.sh @@ -12,9 +12,9 @@ if [[ -z ${EXPORT} ]]; then fi fi - CONVERTER="../../../build/tools/converter/converter_lite" -if [ ! -f "$CONVERTER" ]; then +$CONVERTER &> /dev/null +if [ "$?" -ne 0 ]; then if ! command -v converter_lite &> /dev/null then tar -xzf ../../../../../output/mindspore-lite-*-linux-x64.tar.gz --strip-components 4 --wildcards --no-anchored converter_lite *so.* *.so diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/FileUtil.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/FileUtil.java index 4202b5886e..7d8cbe5ce9 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/FileUtil.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/FileUtil.java @@ -112,6 +112,10 @@ public class FileUtil { if (line.isEmpty()) { continue; } + String[] info = line.split(">>>"); + if (info.length > 1) { + line = info[1]; + } List tokens = customTokenizer.tokenize(line, isTrainMode); Optional feature = customTokenizer.getFeatures(tokens, "other"); if (!feature.isPresent()) { diff --git a/mindspore/lite/java/src/main/java/com/mindspore/Model.java b/mindspore/lite/java/src/main/java/com/mindspore/Model.java index b219c51f11..2d11feb420 100644 --- a/mindspore/lite/java/src/main/java/com/mindspore/Model.java +++ b/mindspore/lite/java/src/main/java/com/mindspore/Model.java @@ -47,10 +47,11 @@ public class Model { * @return build status. */ public boolean build(Graph graph, MSContext context, TrainCfg cfg) { - if (graph == null || context == null || cfg == null) { + if (graph == null || context == null) { return false; } - modelPtr = this.buildByGraph(graph.getGraphPtr(), context.getMSContextPtr(), cfg.getTrainCfgPtr()); + long cfgPtr = cfg != null ? cfg.getTrainCfgPtr() : 0; + modelPtr = this.buildByGraph(graph.getGraphPtr(), context.getMSContextPtr(), cfgPtr); return modelPtr != 0; } diff --git a/mindspore/lite/java/src/main/java/com/mindspore/config/MSContext.java b/mindspore/lite/java/src/main/java/com/mindspore/config/MSContext.java index d4a28b1bc5..2a406ce5e4 100644 --- a/mindspore/lite/java/src/main/java/com/mindspore/config/MSContext.java +++ b/mindspore/lite/java/src/main/java/com/mindspore/config/MSContext.java @@ -53,6 +53,16 @@ public class MSContext { return addDeviceInfo(msContextPtr, deviceType, isEnableFloat16, 3); } + /** + * Init Context,default use 2 thread,no bind mode. + * + * @return init status. + */ + public boolean init() { + this.msContextPtr = createMSContext(2, 0, false); + return this.msContextPtr != 0; + } + /** * Init Context. * diff --git a/mindspore/lite/java/src/main/native/model.cpp b/mindspore/lite/java/src/main/native/model.cpp index ef1cdc9e63..926bb9b565 100644 --- a/mindspore/lite/java/src/main/native/model.cpp +++ b/mindspore/lite/java/src/main/native/model.cpp @@ -45,7 +45,11 @@ extern "C" JNIEXPORT jlong JNICALL Java_com_mindspore_Model_buildByGraph(JNIEnv MS_LOGE("Make train config failed"); return jlong(nullptr); } - cfg.reset(c_cfg_ptr); + if (c_cfg_ptr != nullptr) { + cfg.reset(c_cfg_ptr); + } else { + cfg.reset(); + } auto model = new (std::nothrow) mindspore::Model(); if (model == nullptr) { MS_LOGE("Model new failed"); diff --git a/mindspore/lite/java/src/test/java/com/mindspore/ModelTest.java b/mindspore/lite/java/src/test/java/com/mindspore/ModelTest.java index aaa9b7df16..9d936e4687 100644 --- a/mindspore/lite/java/src/test/java/com/mindspore/ModelTest.java +++ b/mindspore/lite/java/src/test/java/com/mindspore/ModelTest.java @@ -53,6 +53,20 @@ public class ModelTest { liteModel.free(); } + @Test + public void testBuildByInferGraphSuccess() { + String modelFile = "../test/ut/src/runtime/kernel/arm/test_data/nets/lenet_tod_infer.ms"; + Graph g = new Graph(); + assertTrue(g.load(modelFile)); + MSContext context = new MSContext(); + context.init(); + context.addDeviceInfo(DeviceType.DT_CPU, false, 0); + Model liteModel = new Model(); + boolean isSuccess = liteModel.build(g, context, null); + assertTrue(isSuccess); + liteModel.free(); + } + @Test public void testBuildByFileSuccess() { String modelFile = "../test/ut/src/runtime/kernel/arm/test_data/nets/lenet_tod_infer.ms";