diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py index 5982cda733e..0774af58fe8 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_manager.py @@ -9,14 +9,27 @@ class CfgManager(): def __init__(self, cfg) -> None: self.cfg = cfg + @staticmethod + def multistepStrFormat(input: str, placeholder: str, substitution: str): + return input.replace( + '{}{}{}'.format('{', placeholder, '}'), + substitution + ) + def applyTemplate(self): if not "template" in self.cfg: return self.cfg logPath = self.cfg["logPath"] tmplName = self.cfg["template"]["name"] fullCfg = {} + + # todo: generalize tmplcfg generator if tmplName == "bm_simple": fullCfg = self.generatebmSimpleTemplate() + elif tmplName == "e2e": + fullCfg = self.generateE2ETemplate() + elif tmplName == "bm_functional": + fullCfg = self.generatebmFunctionalTemplate() else: raise Exception( "Unknown template '{}'".format(tmplName) @@ -48,3 +61,42 @@ class CfgManager(): else: tmpJSON["runConfig"]["traversal"] = "firstFailedVersion" return tmpJSON + + def generateE2ETemplate(self): + tmpl = self.cfg["template"] + tmpJSON = self.readJsonTmpl("e2e_for_CI.json") + + if "errorPattern" in tmpl and\ + "precommitPath" in tmpl and\ + "testCmd" in tmpl: + tmpJSON["runConfig"]["stopPattern"] = tmpl["errorPattern"] + tmpJSON["dlbConfig"]["commonPath"] = tmpl["precommitPath"] + tmpJSON["cachedPathConfig"]["commonPath"] = tmpl["precommitPath"] + tmpJSON["appCmd"] = CfgManager.multistepStrFormat( + tmpJSON["appCmd"], + tmpl["appCmd"], + "testCmd" + ) + else: + raise("Template is incomplete.") + subPath = "private_linux_manylinux2014_release/" + if "subPath" in tmpl: + subPath = tmpl["subPath"] + tmpJSON["dlbConfig"]["subPath"] = subPath + tmpJSON["cachedPathConfig"]["subPath"] = subPath + + return tmpJSON + + def generatebmFunctionalTemplate(self): + tmpl = self.cfg["template"] + tmpJSON = self.readJsonTmpl("bm_output.json") + stopPattern = "stopPattern" + if "appCmd" in tmpl: + tmpJSON["appCmd"] = tmpl["appCmd"] + else: + raise("No 'appcmd' in template") + if stopPattern in tmpl: + tmpJSON["runConfig"][stopPattern] = tmpl[stopPattern] + else: + raise("No 'stopPattern' in template") + return tmpJSON diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json index 3ec3d96d596..99fce43d90d 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/bm_output.json @@ -1,5 +1,5 @@ { - "appCmd" : "./benchmark_app -m -d CPU -t 10", + "appCmd" : "{appCmd}", "makeCmd" : "cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=OFF -DTHREADING=TBB -DENABLE_INTEL_GPU=OFF -DENABLE_SAMPLES=ON -DENABLE_TESTS=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_CPU_DEBUG_CAPS=OFF -DENABLE_DEBUG_CAPS=OFF -DENABLE_OPENVINO_DEBUG=OFF -DCMAKE_CXX_FLAGS=-Wno-deprecated -DCMAKE_C_FLAGS=-Wno-deprecated -DCMAKE_CXX_FLAGS=-Wno-deprecated-declarations -DCMAKE_C_FLAGS=-Wno-deprecated-declarations ..", "runConfig" : { "commitList" : { @@ -7,6 +7,7 @@ }, "mode" : "checkOutput", "traversal" : "firstFailedVersion", - "stopPattern" : "(.)*(.)*" - } + "stopPattern" : "(.)*{bm_error_message}(.)*" + }, + "extendBuildCommand":true } \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json new file mode 100644 index 00000000000..021ae511a9f --- /dev/null +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/cfg_samples/e2e_for_CI.json @@ -0,0 +1,67 @@ +{ + "appCmd":"source {venvName}/bin/activate && cd {appPath} && {py} -m pip install --upgrade pip && {py} -m pip install openvino-dev[caffe,kaldi,mxnet,onnx,pytorch,tensorflow2]=={wheelVersion} --find-links={precommitPath}wheels/ && {py} -m pip install -r requirements_dev.txt && {testCmd}", + "venvCfg":{ + "venvEnabled":true, + "venvDir":"{workPath}/venv/", + "venvName":"tempVenv" + }, + "commandList":[ + + ], + "runConfig":{ + "mode":"checkOutput", + "traversal":"firstFailedVersion", + "stopPattern":"{ErrorPattern}" + }, + "dlbConfig":{ + "launchedAsJob":true, + "toolName":"e2e", + "wheelVersionsMap":{ + + }, + "commonPath":"{precommitPath}", + "subPath":"{subPath}", + "appPath":"", + "appCmd":"" + }, + "cachedPathConfig":{ + "enabled":true, + "scheme":"mandatory", + "passCmdList":false, + "changeAppPath":false, + "commonPath":"{precommitPath}", + "subPath":"{subPath}", + "cashMap":{ + + } + }, + "substitutionRules":[ + { + "name":"precommitPath", + "enabled":true, + "type":"map", + "placeholder":"precommitPath", + "from":"$.cachedPathConfig.cashMap", + "to":"$.appCmd" + }, + { + "name":"wheelVersion", + "enabled":true, + "type":"map", + "placeholder":"wheelVersion", + "from":"$.dlbConfig.wheelVersionsMap", + "to":"$.appCmd" + } + ], + "subscriptions":[ + { + "name":"wheelPathsMap", + "enabled":true + }, + { + "name":"wheelVersionsMap", + "enabled":true + } + ], + "verboseOutput":false +} \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py b/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py index c705e8b362c..19529f47f74 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/helpers.py @@ -295,7 +295,7 @@ def fetchAppOutput(cfg, commit): {"src": cfg["dlbConfig"]["appCmd"], "dst": "appCmd"}, {"src": cfg["dlbConfig"]["toolPath"], "dst": "toolPath"} ]: - appCmd = multistepStrFormat( + appCmd = CfgManager.multistepStrFormat( appCmd, item["dst"], item["src"] @@ -685,7 +685,7 @@ def applySubstitutionRules(cfg: map, rules: list, commit: str=None): dstPos = formatJSON( dstPos, lambda content: - multistepStrFormat( + CfgManager.multistepStrFormat( content, rule["placeholder"], getMapValueByShortHash(srcPos, commit)\ @@ -704,12 +704,6 @@ def getMapValueByShortHash(map: dict, commit: str): commit, map.keys() )) -def multistepStrFormat(input: str, placeholder: str, substitution: str): - return input.replace( - '{}{}{}'.format('{', placeholder, '}'), - substitution - ) - def deepMapUpdate(content: map, path: list, substitution): if not path: return substitution