diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java index 25a0a7652fb..aa94b16a80d 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java @@ -16,11 +16,14 @@ package com.mindspore.flclient; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Random; +import java.util.Set; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -125,11 +128,29 @@ public class Common { return (new String(message)).contains(safeModTag); } + public static String getRealPath (String path) { + LOGGER.info(addTag("[original path] " + path)); + String[] paths = path.split(","); + for (int i = 0; i < paths.length; i++) { + LOGGER.info(addTag("[original path " + i + "] " + paths[i])); + File file = new File(paths[i]); + try { + paths[i] = file.getCanonicalPath(); + } catch (IOException e) { + LOGGER.severe(addTag("[checkPath] catch IOException in file.getCanonicalPath(): " + e.getMessage())); + throw new RuntimeException(); + } + } + path = String.join(",", Arrays.asList(paths)); + LOGGER.info(addTag("[real path] " + path)); + return path; + } + public static boolean checkPath(String path) { boolean tag = true; String[] paths = path.split(","); for (int i = 0; i < paths.length; i++) { - LOGGER.info(addTag("[check path]:" + paths[i])); + LOGGER.info(addTag("[check path " + i + "] " + paths[i])); File file = new File(paths[i]); if (!file.exists()) { tag = false; diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java index 6a2bad590c0..7acc9355e5e 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java @@ -83,7 +83,13 @@ public class FLParameter { } public void setCertPath(String certPath) { - this.certPath = certPath; + certPath = Common.getRealPath(certPath); + if (Common.checkPath(certPath)) { + this.certPath = certPath; + } else { + LOGGER.severe(Common.addTag("[flParameter] the parameter of is not exist, please check it before set")); + throw new RuntimeException(); + } } public boolean isUseHttps() { @@ -103,6 +109,7 @@ public class FLParameter { } public void setTrainDataset(String trainDataset) { + trainDataset = Common.getRealPath(trainDataset); if (Common.checkPath(trainDataset)) { this.trainDataset = trainDataset; } else { @@ -120,6 +127,7 @@ public class FLParameter { } public void setVocabFile(String vocabFile) { + vocabFile = Common.getRealPath(vocabFile); if (Common.checkPath(vocabFile)) { this.vocabFile = vocabFile; } else { @@ -137,6 +145,7 @@ public class FLParameter { } public void setIdsFile(String idsFile) { + idsFile = Common.getRealPath(idsFile); if (Common.checkPath(idsFile)) { this.idsFile = idsFile; } else { @@ -150,6 +159,7 @@ public class FLParameter { } public void setTestDataset(String testDataset) { + testDataset = Common.getRealPath(testDataset); if (Common.checkPath(testDataset)) { this.testDataset = testDataset; } else { @@ -184,6 +194,7 @@ public class FLParameter { } public void setTrainModelPath(String trainModelPath) { + trainModelPath = Common.getRealPath(trainModelPath); if (Common.checkPath(trainModelPath)) { this.trainModelPath = trainModelPath; } else { @@ -201,6 +212,7 @@ public class FLParameter { } public void setInferModelPath(String inferModelPath) { + inferModelPath = Common.getRealPath(inferModelPath); if (Common.checkPath(inferModelPath)) { this.inferModelPath = inferModelPath; } else { diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/LocalFLParameter.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/LocalFLParameter.java index 9ceb6736412..a0f5df353e8 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/LocalFLParameter.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/LocalFLParameter.java @@ -29,8 +29,8 @@ public class LocalFLParameter { private List albertWeightName = new ArrayList<>(); private String flID; - private String encryptLevel = "NotEncrypt"; - private String earlyStopMod = "NotEarlyStop"; + private String encryptLevel = EncryptLevel.NOT_ENCRYPT.toString(); + private String earlyStopMod = EarlyStopMod.NOT_EARLY_STOP.toString(); private String serverMod = ServerMod.HYBRID_TRAINING.toString(); private String safeMod = "The cluster is in safemode."; diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java index 433a8b8150e..122b7c43b1e 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java @@ -41,17 +41,20 @@ public class SyncFLJob { public SyncFLJob() { } - public void flJobRun() { + public FLClientStatus flJobRun() { localFLParameter.setFlID(flParameter.getClientID()); FLLiteClient client = new FLLiteClient(); - client.initSession(); - FLClientStatus curStatus; + curStatus = client.initSession(); + if (curStatus == FLClientStatus.FAILED) { + LOGGER.severe(Common.addTag("init session failed")); + flJobResultCallback.onFlJobFinished(flParameter.getFlName(), client.getIterations(), client.getRetCode()); + return curStatus; + } + do { LOGGER.info(Common.addTag("flName: " + flParameter.getFlName())); int trainDataSize = client.setInput(flParameter.getTrainDataset()); - LOGGER.info(Common.addTag("train path: " + flParameter.getTrainDataset())); - LOGGER.info(Common.addTag("train data size: " + trainDataSize)); if (trainDataSize <= 0) { LOGGER.severe(Common.addTag("unsolved error code in : the return trainDataSize<=0")); curStatus = FLClientStatus.FAILED; @@ -109,7 +112,6 @@ public class SyncFLJob { failed("[updateModel] updateModel", client.getIteration(), client.getRetCode(), curStatus); break; } - LOGGER.info(Common.addTag("[updateModel] updateModel succeed")); // unmasking curStatus = client.unMasking(); @@ -134,7 +136,6 @@ public class SyncFLJob { failed("[getModel] getModel", client.getIteration(), client.getRetCode(), curStatus); break; } - LOGGER.info(Common.addTag("[getModel] getModel succeed")); // get the feature map after averaging and update dp_norm_clip updateDpNormClip(client); @@ -156,6 +157,7 @@ public class SyncFLJob { client.finalize(); LOGGER.info(Common.addTag("flJobRun finish")); flJobResultCallback.onFlJobFinished(flParameter.getFlName(), client.getIterations(), client.getRetCode()); + return curStatus; } private void updateDpNormClip(FLLiteClient client) { @@ -213,19 +215,19 @@ public class SyncFLJob { return featureMap; } - public int[] modelInference(String flName, String dataPath, String vocabFile, String idsFile, String modelPath) { + public int[] modelInference() { int[] labels = new int[0]; - if (flName.equals(ALBERT)) { + if (flParameter.getFlName().equals(ALBERT)) { AlInferBert alInferBert = AlInferBert.getInstance(); LOGGER.info(Common.addTag("===========model inference=============")); - labels = alInferBert.inferModel(modelPath, dataPath, vocabFile, idsFile); + labels = alInferBert.inferModel(flParameter.getInferModelPath(), flParameter.getTestDataset(), flParameter.getVocabFile(), flParameter.getIdsFile()); LOGGER.info(Common.addTag("[model inference] the predicted labels: " + Arrays.toString(labels))); SessionUtil.free(alInferBert.getTrainSession()); LOGGER.info(Common.addTag("[model inference] inference finish")); - } else if (flName.equals(LENET)) { + } else if (flParameter.getFlName().equals(LENET)) { TrainLenet trainLenet = TrainLenet.getInstance(); LOGGER.info(Common.addTag("===========model inference=============")); - labels = trainLenet.inferModel(modelPath, dataPath.split(",")[0]); + labels = trainLenet.inferModel(flParameter.getInferModelPath(), flParameter.getTestDataset().split(",")[0]); LOGGER.info(Common.addTag("[model inference] the predicted labels: " + Arrays.toString(labels))); SessionUtil.free(trainLenet.getTrainSession()); LOGGER.info(Common.addTag("[model inference] inference finish")); @@ -363,7 +365,9 @@ public class SyncFLJob { SyncFLJob syncFLJob = new SyncFLJob(); if (task.equals("train")) { flParameter.setUseHttps(useHttps); - flParameter.setCertPath(certPath); + if (useHttps) { + flParameter.setCertPath(certPath); + } flParameter.setHostName(ip); flParameter.setTrainDataset(trainDataset); flParameter.setFlName(flName); @@ -382,10 +386,19 @@ public class SyncFLJob { } syncFLJob.flJobRun(); } else if (task.equals("inference")) { - syncFLJob.modelInference(flName, testDataset, vocabFile, idsFile, inferModelPath); + flParameter.setFlName(flName); + flParameter.setTestDataset(testDataset); + flParameter.setInferModelPath(inferModelPath); + if (ALBERT.equals(flName)) { + flParameter.setVocabFile(vocabFile); + flParameter.setIdsFile(idsFile); + } + syncFLJob.modelInference(); } else if (task.equals("getModel")) { flParameter.setUseHttps(useHttps); - flParameter.setCertPath(certPath); + if (useHttps) { + flParameter.setCertPath(certPath); + } flParameter.setHostName(ip); flParameter.setFlName(flName); flParameter.setTrainModelPath(trainModelPath); diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/CustomTokenizer.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/CustomTokenizer.java index 0266bede0e5..b9df260c993 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/CustomTokenizer.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/model/CustomTokenizer.java @@ -70,9 +70,6 @@ public class CustomTokenizer { for (int i = 0; i < idsLines.size(); ++i) { vocabs.put(vocabLines.get(i), Integer.parseInt(idsLines.get(i))); } - if (!trainMod) { - maxSeqLen = 256; - } } // is chinses or punctuation