yankf21
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!19802 add path check for fl_client in r1.3 

Merge pull request !19802 from zhoushan33/flclient0709_om_r1.3
This commit is contained in:
i-robot 2021-07-10 06:56:57 +00:00 committed by Gitee
parent 2d11b5ea43 3ed4045736
commit 9c868ca486
5 changed files with 65 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java
View File

@ -16,11 +16,14 @@
package com.mindspore.flclient;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -125,11 +128,29 @@ public class Common {
return (new String(message)).contains(safeModTag);
}
public static String getRealPath (String path) {
LOGGER.info(addTag("[original path] " + path));
String[] paths = path.split(",");
for (int i = 0; i < paths.length; i++) {
LOGGER.info(addTag("[original path " + i + "] " + paths[i]));
File file = new File(paths[i]);
try {
paths[i] = file.getCanonicalPath();
} catch (IOException e) {
LOGGER.severe(addTag("[checkPath] catch IOException in file.getCanonicalPath(): " + e.getMessage()));
throw new RuntimeException();
}
}
path = String.join(",", Arrays.asList(paths));
LOGGER.info(addTag("[real path] " + path));
return path;
}
public static boolean checkPath(String path) {
boolean tag = true;
String[] paths = path.split(",");
for (int i = 0; i < paths.length; i++) {
LOGGER.info(addTag("[check path]:" + paths[i]));
LOGGER.info(addTag("[check path " + i + "] " + paths[i]));
File file = new File(paths[i]);
if (!file.exists()) {
tag = false;

14
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java
View File

@ -83,7 +83,13 @@ public class FLParameter {
}
public void setCertPath(String certPath) {
this.certPath = certPath;
certPath = Common.getRealPath(certPath);
if (Common.checkPath(certPath)) {
this.certPath = certPath;
} else {
LOGGER.severe(Common.addTag("[flParameter] the parameter of <certPath> is not exist, please check it before set"));
throw new RuntimeException();
}
}
public boolean isUseHttps() {
@ -103,6 +109,7 @@ public class FLParameter {
}
public void setTrainDataset(String trainDataset) {
trainDataset = Common.getRealPath(trainDataset);
if (Common.checkPath(trainDataset)) {
this.trainDataset = trainDataset;
} else {
@ -120,6 +127,7 @@ public class FLParameter {
}
public void setVocabFile(String vocabFile) {
vocabFile = Common.getRealPath(vocabFile);
if (Common.checkPath(vocabFile)) {
this.vocabFile = vocabFile;
} else {
@ -137,6 +145,7 @@ public class FLParameter {
}
public void setIdsFile(String idsFile) {
idsFile = Common.getRealPath(idsFile);
if (Common.checkPath(idsFile)) {
this.idsFile = idsFile;
} else {
@ -150,6 +159,7 @@ public class FLParameter {
}
public void setTestDataset(String testDataset) {
testDataset = Common.getRealPath(testDataset);
if (Common.checkPath(testDataset)) {
this.testDataset = testDataset;
} else {
@ -184,6 +194,7 @@ public class FLParameter {
}
public void setTrainModelPath(String trainModelPath) {
trainModelPath = Common.getRealPath(trainModelPath);
if (Common.checkPath(trainModelPath)) {
this.trainModelPath = trainModelPath;
} else {
@ -201,6 +212,7 @@ public class FLParameter {
}
public void setInferModelPath(String inferModelPath) {
inferModelPath = Common.getRealPath(inferModelPath);
if (Common.checkPath(inferModelPath)) {
this.inferModelPath = inferModelPath;
} else {

4
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/LocalFLParameter.java
View File

@ -29,8 +29,8 @@ public class LocalFLParameter {
private List<String> albertWeightName = new ArrayList<>();
private String flID;
private String encryptLevel = "NotEncrypt";
private String earlyStopMod = "NotEarlyStop";
private String encryptLevel = EncryptLevel.NOT_ENCRYPT.toString();
private String earlyStopMod = EarlyStopMod.NOT_EARLY_STOP.toString();
private String serverMod = ServerMod.HYBRID_TRAINING.toString();
private String safeMod = "The cluster is in safemode.";

43
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java
View File

@ -41,17 +41,20 @@ public class SyncFLJob {
public SyncFLJob() {
}
public void flJobRun() {
public FLClientStatus flJobRun() {
localFLParameter.setFlID(flParameter.getClientID());
FLLiteClient client = new FLLiteClient();
client.initSession();
FLClientStatus curStatus;
curStatus = client.initSession();
if (curStatus == FLClientStatus.FAILED) {
LOGGER.severe(Common.addTag("init session failed"));
flJobResultCallback.onFlJobFinished(flParameter.getFlName(), client.getIterations(), client.getRetCode());
return curStatus;
}
do {
LOGGER.info(Common.addTag("flName: " + flParameter.getFlName()));
int trainDataSize = client.setInput(flParameter.getTrainDataset());
LOGGER.info(Common.addTag("train path: " + flParameter.getTrainDataset()));
LOGGER.info(Common.addTag("train data size: " + trainDataSize));
if (trainDataSize <= 0) {
LOGGER.severe(Common.addTag("unsolved error code in <client.setInput>: the return trainDataSize<=0"));
curStatus = FLClientStatus.FAILED;
@ -109,7 +112,6 @@ public class SyncFLJob {
failed("[updateModel] updateModel", client.getIteration(), client.getRetCode(), curStatus);
break;
}
LOGGER.info(Common.addTag("[updateModel] updateModel succeed"));
// unmasking
curStatus = client.unMasking();
@ -134,7 +136,6 @@ public class SyncFLJob {
failed("[getModel] getModel", client.getIteration(), client.getRetCode(), curStatus);
break;
}
LOGGER.info(Common.addTag("[getModel] getModel succeed"));
// get the feature map after averaging and update dp_norm_clip
updateDpNormClip(client);
@ -156,6 +157,7 @@ public class SyncFLJob {
client.finalize();
LOGGER.info(Common.addTag("flJobRun finish"));
flJobResultCallback.onFlJobFinished(flParameter.getFlName(), client.getIterations(), client.getRetCode());
return curStatus;
}
private void updateDpNormClip(FLLiteClient client) {
@ -213,19 +215,19 @@ public class SyncFLJob {
return featureMap;
}
public int[] modelInference(String flName, String dataPath, String vocabFile, String idsFile, String modelPath) {
public int[] modelInference() {
int[] labels = new int[0];
if (flName.equals(ALBERT)) {
if (flParameter.getFlName().equals(ALBERT)) {
AlInferBert alInferBert = AlInferBert.getInstance();
LOGGER.info(Common.addTag("===========model inference============="));
labels = alInferBert.inferModel(modelPath, dataPath, vocabFile, idsFile);
labels = alInferBert.inferModel(flParameter.getInferModelPath(), flParameter.getTestDataset(), flParameter.getVocabFile(), flParameter.getIdsFile());
LOGGER.info(Common.addTag("[model inference] the predicted labels: " + Arrays.toString(labels)));
SessionUtil.free(alInferBert.getTrainSession());
LOGGER.info(Common.addTag("[model inference] inference finish"));
} else if (flName.equals(LENET)) {
} else if (flParameter.getFlName().equals(LENET)) {
TrainLenet trainLenet = TrainLenet.getInstance();
LOGGER.info(Common.addTag("===========model inference============="));
labels = trainLenet.inferModel(modelPath, dataPath.split(",")[0]);
labels = trainLenet.inferModel(flParameter.getInferModelPath(), flParameter.getTestDataset().split(",")[0]);
LOGGER.info(Common.addTag("[model inference] the predicted labels: " + Arrays.toString(labels)));
SessionUtil.free(trainLenet.getTrainSession());
LOGGER.info(Common.addTag("[model inference] inference finish"));
@ -363,7 +365,9 @@ public class SyncFLJob {
SyncFLJob syncFLJob = new SyncFLJob();
if (task.equals("train")) {
flParameter.setUseHttps(useHttps);
flParameter.setCertPath(certPath);
if (useHttps) {
flParameter.setCertPath(certPath);
}
flParameter.setHostName(ip);
flParameter.setTrainDataset(trainDataset);
flParameter.setFlName(flName);
@ -382,10 +386,19 @@ public class SyncFLJob {
}
syncFLJob.flJobRun();
} else if (task.equals("inference")) {
syncFLJob.modelInference(flName, testDataset, vocabFile, idsFile, inferModelPath);
flParameter.setFlName(flName);
flParameter.setTestDataset(testDataset);
flParameter.setInferModelPath(inferModelPath);
if (ALBERT.equals(flName)) {
flParameter.setVocabFile(vocabFile);
flParameter.setIdsFile(idsFile);
}
syncFLJob.modelInference();
} else if (task.equals("getModel")) {
flParameter.setUseHttps(useHttps);
flParameter.setCertPath(certPath);
if (useHttps) {
flParameter.setCertPath(certPath);
}
flParameter.setHostName(ip);
flParameter.setFlName(flName);
flParameter.setTrainModelPath(trainModelPath);

3
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/CustomTokenizer.java
View File

@ -70,9 +70,6 @@ public class CustomTokenizer {
for (int i = 0; i < idsLines.size(); ++i) {
vocabs.put(vocabLines.get(i), Integer.parseInt(idsLines.get(i)));
}
if (!trainMod) {
maxSeqLen = 256;
}
}
// is chinses or punctuation